├── .gitattributes ├── runtest.sh ├── tests ├── cluster │ ├── tmp │ │ └── .gitignore │ ├── tests │ │ ├── helpers │ │ │ └── onlydots.tcl │ │ ├── 01-faildet.tcl │ │ ├── 09-pubsub.tcl │ │ ├── 11-manual-takeover.tcl │ │ ├── 02-failover.tcl │ │ ├── 00-base.tcl │ │ ├── 12-replica-migration-2.tcl │ │ ├── includes │ │ │ └── init-tests.tcl │ │ ├── 06-slave-stop-cond.tcl │ │ ├── 08-update-msg.tcl │ │ ├── 05-slave-selection.tcl │ │ ├── 07-replica-migration.tcl │ │ └── 03-failover-loop.tcl │ └── run.tcl ├── sentinel │ ├── tmp │ │ └── .gitignore │ ├── tests │ │ ├── 03-runtime-reconf.tcl │ │ ├── 04-slave-selection.tcl │ │ ├── 06-ckquorum.tcl │ │ ├── 01-conf-update.tcl │ │ ├── 05-manual.tcl │ │ ├── 07-down-conditions.tcl │ │ ├── includes │ │ │ └── init-tests.tcl │ │ └── 02-slaves-reconf.tcl │ └── run.tcl ├── unit │ ├── printver.tcl │ ├── type │ │ ├── list-common.tcl │ │ └── list-2.tcl │ ├── limits.tcl │ ├── introspection-2.tcl │ ├── auth.tcl │ ├── quit.tcl │ ├── memefficiency.tcl │ ├── latency-monitor.tcl │ ├── introspection.tcl │ ├── slowlog.tcl │ ├── obuf-limits.tcl │ └── protocol.tcl ├── helpers │ ├── bg_complex_data.tcl │ └── gen_write_load.tcl ├── support │ ├── tmpfile.tcl │ └── test.tcl └── integration │ ├── logging.tcl │ ├── aof-race.tcl │ ├── convert-zipmap-hash-on-load.tcl │ ├── replication-2.tcl │ ├── rdb.tcl │ └── replication-3.tcl ├── doc ├── images │ ├── set.png │ ├── ttl.png │ ├── hash.png │ ├── list.png │ ├── zset.png │ ├── string.png │ ├── kedis_arch.png │ ├── thread_model.png │ ├── kedis_arch.graffle │ ├── replication-state.png │ ├── thread_model.graffle │ ├── replication-state.graffle │ └── kedis-storage-format.graffle ├── misc_test_case.md ├── Readme.md ├── info_keyspace_design.md ├── migrate_design.md ├── zset_double_score_design.md └── support_command.md ├── src ├── 3rd_party │ └── rocksdb-5.6.2.tar.gz ├── server │ ├── kedis_version.h │ ├── config.h │ ├── expire_thread.h │ ├── hyperloglog.h │ ├── migrate_conn.h │ ├── slowlog.h │ ├── cmd_set.h │ ├── cmd_db.h │ ├── migrate.h │ ├── replication.h │ ├── redis_byte_stream.h │ ├── cmd_list.h │ ├── cmd_keys.h │ ├── key_lock.h │ ├── Makefile │ ├── cmd_hash.h │ ├── redis_byte_stream.cpp │ ├── binlog.h │ ├── cmd_zset.h │ ├── expire_thread.cpp │ ├── cmd_string.h │ ├── migrate_conn.cpp │ ├── redis_parser.h │ ├── db_util.h │ ├── client_conn.h │ └── slowlog.cpp └── base │ ├── base_listener.h │ ├── Makefile │ ├── ostype.h │ ├── simple_buffer.h │ ├── simple_log.h │ ├── io_thread_resource.h │ ├── simple_buffer.cpp │ ├── base_conn.h │ ├── event_loop.h │ ├── util.h │ ├── base_socket.h │ ├── thread_pool.h │ ├── simple_log.cpp │ ├── byte_stream.h │ └── thread_pool.cpp ├── tools ├── kedis_to_redis │ ├── crc64.h │ ├── redis_serializer.h │ ├── sync_task.h │ ├── src_kedis_conn.h │ ├── kedis_to_redis.h │ ├── Makefile │ ├── sync_task.cpp │ └── kedis_to_redis.cpp ├── hiredis │ ├── fmacros.h │ ├── .travis.yml │ ├── win32.h │ ├── appveyor.yml │ ├── COPYING │ ├── sdsalloc.h │ └── net.h ├── kedis_port │ ├── sync_task.h │ ├── src_kedis_conn.h │ ├── kedis_port.h │ ├── Makefile │ ├── sync_task.cpp │ └── kedis_port.cpp ├── redis_to_kedis │ ├── kedis_serializer.h │ ├── redis_to_kedis.cpp │ ├── sync_task.h │ ├── Makefile │ ├── src_redis_conn.h │ ├── cmd_line_parser.h │ └── cmd_line_parser.cpp └── hiredis_wrapper │ ├── redis_conn.h │ └── redis_conn.cpp ├── .gitignore ├── LICENSE └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | *.tcl linguist-language=C++ 2 | -------------------------------------------------------------------------------- /runtest.sh: -------------------------------------------------------------------------------- 1 | tclsh tests/test_helper.tcl --clients 2 2 | -------------------------------------------------------------------------------- /tests/cluster/tmp/.gitignore: -------------------------------------------------------------------------------- 1 | redis_* 2 | sentinel_* 3 | -------------------------------------------------------------------------------- /tests/sentinel/tmp/.gitignore: -------------------------------------------------------------------------------- 1 | redis_* 2 | sentinel_* 3 | -------------------------------------------------------------------------------- /doc/images/set.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/set.png -------------------------------------------------------------------------------- /doc/images/ttl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/ttl.png -------------------------------------------------------------------------------- /doc/images/hash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/hash.png -------------------------------------------------------------------------------- /doc/images/list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/list.png -------------------------------------------------------------------------------- /doc/images/zset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/zset.png -------------------------------------------------------------------------------- /doc/images/string.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/string.png -------------------------------------------------------------------------------- /doc/images/kedis_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/kedis_arch.png -------------------------------------------------------------------------------- /tests/sentinel/tests/03-runtime-reconf.tcl: -------------------------------------------------------------------------------- 1 | # Test runtime reconfiguration command SENTINEL SET. 2 | -------------------------------------------------------------------------------- /doc/images/thread_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/thread_model.png -------------------------------------------------------------------------------- /doc/images/kedis_arch.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/kedis_arch.graffle -------------------------------------------------------------------------------- /doc/images/replication-state.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/replication-state.png -------------------------------------------------------------------------------- /doc/images/thread_model.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/thread_model.graffle -------------------------------------------------------------------------------- /src/3rd_party/rocksdb-5.6.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/src/3rd_party/rocksdb-5.6.2.tar.gz -------------------------------------------------------------------------------- /doc/images/replication-state.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/replication-state.graffle -------------------------------------------------------------------------------- /doc/images/kedis-storage-format.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianqingdu/kedis/HEAD/doc/images/kedis-storage-format.graffle -------------------------------------------------------------------------------- /tools/kedis_to_redis/crc64.h: -------------------------------------------------------------------------------- 1 | #ifndef CRC64_H 2 | #define CRC64_H 3 | 4 | #include 5 | 6 | uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /tests/unit/printver.tcl: -------------------------------------------------------------------------------- 1 | start_server {} { 2 | set i [r info] 3 | regexp {redis_version:(.*?)\r\n} $i - version 4 | regexp {redis_git_sha1:(.*?)\r\n} $i - sha1 5 | puts "Testing Redis version $version ($sha1)" 6 | } 7 | -------------------------------------------------------------------------------- /tests/sentinel/tests/04-slave-selection.tcl: -------------------------------------------------------------------------------- 1 | # Test slave selection algorithm. 2 | # 3 | # This unit should test: 4 | # 1) That when there are no suitable slaves no failover is performed. 5 | # 2) That among the available slaves, the one with better offset is picked. 6 | -------------------------------------------------------------------------------- /tests/unit/type/list-common.tcl: -------------------------------------------------------------------------------- 1 | # We need a value larger than list-max-ziplist-value to make sure 2 | # the list has the right encoding when it is swapped in again. 3 | array set largevalue {} 4 | set largevalue(ziplist) "hello" 5 | set largevalue(linkedlist) [string repeat "hello" 4] 6 | -------------------------------------------------------------------------------- /doc/misc_test_case.md: -------------------------------------------------------------------------------- 1 | 这里包括了那些不会被tests测试脚本测到的测试用例 2 | 3 | * master运行时，用redis-benchmark -p 6381 -n 1000000 -r 200000000 -d 128 -c 60 -t mset不停的插入数据，然后挂载一个slave上去，等slave连上master后，关闭插入benchmark，等数据全部同步后，查看master和slave上的数据量是否一致 4 | * 关闭master，测试slave是否会一直重连 5 | * 修改slave的配置文件，把master的ip地址改成一个其他网段不存在的地址，启动slave，测试是否到master的连接在2秒后重连 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.o 3 | *.pid 4 | *.so 5 | *.dylib 6 | 7 | kedis*.tar.gz 8 | 9 | src/3rd_party/rocksdb-5.6.2 10 | src/server/kedis-server 11 | tests/tmp/* 12 | 13 | tools/hiredis/hiredis-test 14 | tools/hiredis/hiredis.pc 15 | tools/kedis_port/kedis_port 16 | tools/redis_to_kedis/redis_to_kedis 17 | tools/kedis_to_redis/kedis_to_redis 18 | -------------------------------------------------------------------------------- /tests/helpers/bg_complex_data.tcl: -------------------------------------------------------------------------------- 1 | source tests/support/redis.tcl 2 | source tests/support/util.tcl 3 | 4 | proc bg_complex_data {host port db ops} { 5 | set r [redis $host $port] 6 | $r select $db 7 | createComplexDataset $r $ops 8 | } 9 | 10 | bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] 11 | -------------------------------------------------------------------------------- /src/server/kedis_version.h: -------------------------------------------------------------------------------- 1 | // 2 | // kedis_version.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/29. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __KEDIS_VERSION_H__ 10 | #define __KEDIS_VERSION_H__ 11 | 12 | #define KEDIS_VERSION "0.1.0" // major.minor.patch 13 | 14 | #endif /* __KEDIS_VERSION_H__ */ 15 | -------------------------------------------------------------------------------- /doc/Readme.md: -------------------------------------------------------------------------------- 1 | ## Document List 2 | * [Design Documentation in English](kedis_server_design_en.md) 3 | * [Design Documentation in Chinese](kedis_server_design.md) 4 | * [info keyspace design doc](info_keyspace_design.md) 5 | * [migrate command design](migrate_design.md) 6 | * [zset double score design](zset_double_score_design.md) 7 | * [support commands](support_command.md) -------------------------------------------------------------------------------- /tools/kedis_to_redis/redis_serializer.h: -------------------------------------------------------------------------------- 1 | // 2 | // redis_serializer.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/28. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __REDIS_SERIALIZER_H__ 10 | #define __REDIS_SERIALIZER_H__ 11 | 12 | #include "util.h" 13 | 14 | int convert_kedis_serialized_value(const string& kedis_value, string& redis_value); 15 | 16 | #endif /* __REDIS_SERIALIZER_H__ */ 17 | -------------------------------------------------------------------------------- /tests/helpers/gen_write_load.tcl: -------------------------------------------------------------------------------- 1 | source tests/support/redis.tcl 2 | 3 | proc gen_write_load {host port seconds} { 4 | set start_time [clock seconds] 5 | set r [redis $host $port 1] 6 | $r select 9 7 | while 1 { 8 | $r set [expr rand()] [expr rand()] 9 | if {[clock seconds]-$start_time > $seconds} { 10 | exit 0 11 | } 12 | } 13 | } 14 | 15 | gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] 16 | -------------------------------------------------------------------------------- /tests/support/tmpfile.tcl: -------------------------------------------------------------------------------- 1 | set ::tmpcounter 0 2 | set ::tmproot "./tests/tmp" 3 | file mkdir $::tmproot 4 | 5 | # returns a dirname unique to this process to write to 6 | proc tmpdir {basename} { 7 | set dir [file join $::tmproot $basename.[pid].[incr ::tmpcounter]] 8 | file mkdir $dir 9 | set _ $dir 10 | } 11 | 12 | # return a filename unique to this process to write to 13 | proc tmpfile {basename} { 14 | file join $::tmproot $basename.[pid].[incr ::tmpcounter] 15 | } 16 | -------------------------------------------------------------------------------- /src/server/config.h: -------------------------------------------------------------------------------- 1 | // 2 | // config.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/8/22. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CONFIG_H__ 10 | #define __CONFIG_H__ 11 | 12 | #include "util.h" 13 | #include "client_conn.h" 14 | 15 | void load_config(const string& filename); 16 | 17 | int rewrite_config(const string& filename); 18 | 19 | void config_command(ClientConn* conn, const vector& cmd_vec); 20 | 21 | #endif /* __CONFIG_H__ */ 22 | -------------------------------------------------------------------------------- /src/server/expire_thread.h: -------------------------------------------------------------------------------- 1 | // 2 | // expire_thread.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/27. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __EXPIRE_THREAD_H__ 10 | #define __EXPIRE_THREAD_H__ 11 | 12 | #include "thread_pool.h" 13 | 14 | class ExpireThread : public Thread { 15 | public: 16 | ExpireThread() {} 17 | virtual ~ExpireThread() {} 18 | 19 | virtual void OnThreadRun(void); 20 | }; 21 | 22 | #endif /* __EXPIRE_THREAD_H__ */ 23 | -------------------------------------------------------------------------------- /tests/unit/limits.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"limits"} overrides {maxclients 10}} { 2 | test {Check if maxclients works refusing connections} { 3 | set c 0 4 | catch { 5 | while {$c < 50} { 6 | incr c 7 | set rd [redis_deferring_client] 8 | $rd ping 9 | $rd read 10 | after 100 11 | } 12 | } e 13 | assert {$c > 8 && $c <= 10} 14 | set e 15 | } {*ERR max*reached*} 16 | } 17 | -------------------------------------------------------------------------------- /src/server/hyperloglog.h: -------------------------------------------------------------------------------- 1 | // 2 | // hyperloglog.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/8/15. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __HYPERLOGLOG_H__ 10 | #define __HYPERLOGLOG_H__ 11 | 12 | #include "server.h" 13 | 14 | void pfadd_command(ClientConn* conn, const vector& cmd_vec); 15 | void pfcount_command(ClientConn* conn, const vector& cmd_vec); 16 | void pfmerge_command(ClientConn* conn, const vector& cmd_vec); 17 | 18 | #endif /* __HYPERLOGLOG_H__ */ 19 | -------------------------------------------------------------------------------- /doc/info_keyspace_design.md: -------------------------------------------------------------------------------- 1 | ## info keyspace的设计实现 2 | 由于以下3个原因，需要特别的设计来实时查询DB里面的key的个数 3 | 4 | * 当key个数很大时，实时遍历每个key来统计太耗费时间 5 | * RocksDB的GetIntProperty(cf_handle, “rocksdb.estimate-num-keys")接口只能估算key的大概个数 6 | * 复杂数据结构的一个key对应到RocksDB的多个key 7 | 8 | 大概的思路如下 9 | 10 | * 用一个内存计数器来存储当前key的个数，每当有key删除和添加时，更新该计数器 11 | * 每隔1秒把该计数器写入到RocksDB进行持久化 12 | * 服务启动时，读取RocksDB里面的计数器，如果计数器个数小于10万，则遍历DB的每个key，来获取精确的key个数，否则就以读取的计数器做为当前key的个数，这样的目的是，当key个数比较少时，让统计尽量精确，当key个数比较大时，有点误差也是可以接受的 13 | * 服务结束时，把计数器写入RocksDB，这样只有当服务进程异常crash，或者机器掉电，而且key个数比较大时，才会出现key计数器不是特别准的情况 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/cluster/tests/helpers/onlydots.tcl: -------------------------------------------------------------------------------- 1 | # Read the standard input and only shows dots in the output, filtering out 2 | # all the other characters. Designed to avoid bufferization so that when 3 | # we get the output of redis-trib and want to show just the dots, we'll see 4 | # the dots as soon as redis-trib will output them. 5 | 6 | fconfigure stdin -buffering none 7 | 8 | while 1 { 9 | set c [read stdin 1] 10 | if {$c eq {}} { 11 | exit 0; # EOF 12 | } elseif {$c eq {.}} { 13 | puts -nonewline . 14 | flush stdout 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /tools/hiredis/fmacros.h: -------------------------------------------------------------------------------- 1 | #ifndef __HIREDIS_FMACRO_H 2 | #define __HIREDIS_FMACRO_H 3 | 4 | #if defined(__linux__) 5 | #define _BSD_SOURCE 6 | #define _DEFAULT_SOURCE 7 | #endif 8 | 9 | #if defined(__CYGWIN__) 10 | #include 11 | #endif 12 | 13 | #if defined(__sun__) 14 | #define _POSIX_C_SOURCE 200112L 15 | #elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) 16 | #define _XOPEN_SOURCE 600 17 | #else 18 | #define _XOPEN_SOURCE 19 | #endif 20 | 21 | #if defined(__APPLE__) && defined(__MACH__) 22 | #define _OSX 23 | #endif 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /tests/sentinel/run.tcl: -------------------------------------------------------------------------------- 1 | # Sentinel test suite. Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com 2 | # This software is released under the BSD License. See the COPYING file for 3 | # more information. 4 | 5 | cd tests/sentinel 6 | source ../instances.tcl 7 | 8 | set ::instances_count 5 ; # How many instances we use at max. 9 | 10 | proc main {} { 11 | parse_options 12 | spawn_instance sentinel $::sentinel_base_port $::instances_count 13 | spawn_instance redis $::redis_base_port $::instances_count 14 | run_tests 15 | cleanup 16 | end_tests 17 | } 18 | 19 | if {[catch main e]} { 20 | puts $::errorInfo 21 | cleanup 22 | exit 1 23 | } 24 | -------------------------------------------------------------------------------- /tools/hiredis/.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | sudo: false 3 | compiler: 4 | - gcc 5 | - clang 6 | 7 | addons: 8 | apt: 9 | packages: 10 | - libc6-dbg 11 | - libc6-dev 12 | - libc6:i386 13 | - libc6-dev-i386 14 | - libc6-dbg:i386 15 | - gcc-multilib 16 | - valgrind 17 | 18 | env: 19 | - CFLAGS="-Werror" 20 | - PRE="valgrind --track-origins=yes --leak-check=full" 21 | - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" 22 | - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" 23 | 24 | script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example 25 | -------------------------------------------------------------------------------- /tests/unit/introspection-2.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"introspection"}} { 2 | test {TTL and TYPYE do not alter the last access time of a key} { 3 | r set foo bar 4 | after 3000 5 | r ttl foo 6 | r type foo 7 | assert {[r object idletime foo] >= 2} 8 | } 9 | 10 | test {TOUCH alters the last access time of a key} { 11 | r set foo bar 12 | after 3000 13 | r touch foo 14 | assert {[r object idletime foo] < 2} 15 | } 16 | 17 | test {TOUCH returns the number of existing keys specified} { 18 | r flushdb 19 | r set key1 1 20 | r set key2 2 21 | r touch key0 key1 key2 key3 22 | } 2 23 | } 24 | -------------------------------------------------------------------------------- /src/server/migrate_conn.h: -------------------------------------------------------------------------------- 1 | // 2 | // migrate_conn.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/19. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __MIGRATE_CONN_H__ 10 | #define __MIGRATE_CONN_H__ 11 | 12 | #include "base_conn.h" 13 | #include "redis_parser.h" 14 | 15 | class MigrateConn : public BaseConn { 16 | public: 17 | MigrateConn(); 18 | virtual ~MigrateConn(); 19 | 20 | virtual void Close(); 21 | virtual void OnConfirm(); 22 | virtual void OnRead(); 23 | virtual void OnTimer(uint64_t curr_tick); 24 | 25 | void SetAddr(const string& addr) { addr_ = addr; } 26 | private: 27 | string addr_; 28 | }; 29 | 30 | 31 | #endif /* __MIGRATE_CONN_H__ */ 32 | -------------------------------------------------------------------------------- /src/server/slowlog.h: -------------------------------------------------------------------------------- 1 | // 2 | // slowlog.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/2. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SLOWLOG_H__ 10 | #define __SLOWLOG_H__ 11 | 12 | #include "server.h" 13 | 14 | #define SLOWLOG_ENTRY_MAX_ARGC 32 15 | #define SLOWLOG_ENTRY_MAX_STRING 128 16 | 17 | typedef struct { 18 | vector cmd_vec; 19 | uint64_t slowlog_id; // Unique entry identifier 20 | uint64_t duration; // Time spent by the query, in milliseconds 21 | time_t time; // Unix time at which the query was executed 22 | } SlowlogEntry; 23 | 24 | void add_slowlog(const vector& cmd_vec, uint64_t duration); 25 | 26 | void slowlog_command(ClientConn* conn, const vector& cmd_vec); 27 | 28 | #endif /* __SLOWLOG_H__ */ 29 | -------------------------------------------------------------------------------- /tests/cluster/run.tcl: -------------------------------------------------------------------------------- 1 | # Cluster test suite. Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com 2 | # This software is released under the BSD License. See the COPYING file for 3 | # more information. 4 | 5 | cd tests/cluster 6 | source cluster.tcl 7 | source ../instances.tcl 8 | source ../../support/cluster.tcl ; # Redis Cluster client. 9 | 10 | set ::instances_count 20 ; # How many instances we use at max. 11 | 12 | proc main {} { 13 | parse_options 14 | spawn_instance redis $::redis_base_port $::instances_count { 15 | "cluster-enabled yes" 16 | "appendonly yes" 17 | } 18 | run_tests 19 | cleanup 20 | end_tests 21 | } 22 | 23 | if {[catch main e]} { 24 | puts $::errorInfo 25 | if {$::pause_on_error} pause_on_error 26 | cleanup 27 | exit 1 28 | } 29 | -------------------------------------------------------------------------------- /tools/kedis_port/sync_task.h: -------------------------------------------------------------------------------- 1 | // 2 | // sync_task.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SYNC_TASK_H__ 10 | #define __SYNC_TASK_H__ 11 | 12 | #include "util.h" 13 | #include "thread_pool.h" 14 | 15 | class SyncCmdTask : public Task { 16 | public: 17 | SyncCmdTask(const string& raw_cmd) : raw_cmd_(raw_cmd) {} 18 | virtual ~SyncCmdTask() {} 19 | 20 | virtual void run(); 21 | private: 22 | string raw_cmd_; 23 | }; 24 | 25 | class KeepalivePingTask : public Task { 26 | public: 27 | KeepalivePingTask() {} 28 | virtual ~KeepalivePingTask() {} 29 | 30 | virtual void run(); 31 | }; 32 | 33 | extern ThreadPool g_thread_pool; 34 | 35 | void init_sync_task(); 36 | 37 | #endif /* __SYNC_TASK_H__ */ 38 | -------------------------------------------------------------------------------- /tools/kedis_to_redis/sync_task.h: -------------------------------------------------------------------------------- 1 | // 2 | // sync_task.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SYNC_TASK_H__ 10 | #define __SYNC_TASK_H__ 11 | 12 | #include "util.h" 13 | #include "thread_pool.h" 14 | 15 | class SyncCmdTask : public Task { 16 | public: 17 | SyncCmdTask(const string& raw_cmd) : raw_cmd_(raw_cmd) {} 18 | virtual ~SyncCmdTask() {} 19 | 20 | virtual void run(); 21 | private: 22 | string raw_cmd_; 23 | }; 24 | 25 | class KeepalivePingTask : public Task { 26 | public: 27 | KeepalivePingTask() {} 28 | virtual ~KeepalivePingTask() {} 29 | 30 | virtual void run(); 31 | }; 32 | 33 | extern ThreadPool g_thread_pool; 34 | 35 | void init_sync_task(); 36 | 37 | #endif /* __SYNC_TASK_H__ */ 38 | -------------------------------------------------------------------------------- /tests/unit/auth.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"auth"}} { 2 | test {AUTH fails if there is no password configured server side} { 3 | catch {r auth foo} err 4 | set _ $err 5 | } {ERR*no password*} 6 | } 7 | 8 | start_server {tags {"auth"} overrides {requirepass foobar}} { 9 | test {AUTH fails when a wrong password is given} { 10 | catch {r auth wrong!} err 11 | set _ $err 12 | } {ERR*invalid password} 13 | 14 | test {Arbitrary command gives an error when AUTH is required} { 15 | catch {r set foo bar} err 16 | set _ $err 17 | } {*NOAUTH*} 18 | 19 | test {AUTH succeeds when the right password is given} { 20 | r auth foobar 21 | } {OK} 22 | 23 | test {Once AUTH succeeded we can actually send commands to the server} { 24 | r set foo 100 25 | r incr foo 26 | } {101} 27 | } 28 | -------------------------------------------------------------------------------- /src/server/cmd_set.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_set.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/20. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_SET_H__ 10 | #define __CMD_SET_H__ 11 | 12 | #include "server.h" 13 | 14 | void sadd_command(ClientConn* conn, const vector& cmd_vec); 15 | void scard_command(ClientConn* conn, const vector& cmd_vec); 16 | void sismember_command(ClientConn* conn, const vector& cmd_vec); 17 | void smembers_command(ClientConn* conn, const vector& cmd_vec); 18 | void spop_command(ClientConn* conn, const vector& cmd_vec); 19 | void srandmember_command(ClientConn* conn, const vector& cmd_vec); 20 | void srem_command(ClientConn* conn, const vector& cmd_vec); 21 | void sscan_command(ClientConn* conn, const vector& cmd_vec); 22 | 23 | #endif /* __CMD_SET_H__ */ 24 | -------------------------------------------------------------------------------- /tools/hiredis/win32.h: -------------------------------------------------------------------------------- 1 | #ifndef _WIN32_HELPER_INCLUDE 2 | #define _WIN32_HELPER_INCLUDE 3 | #ifdef _MSC_VER 4 | 5 | #ifndef inline 6 | #define inline __inline 7 | #endif 8 | 9 | #ifndef va_copy 10 | #define va_copy(d,s) ((d) = (s)) 11 | #endif 12 | 13 | #ifndef snprintf 14 | #define snprintf c99_snprintf 15 | 16 | __inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) 17 | { 18 | int count = -1; 19 | 20 | if (size != 0) 21 | count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); 22 | if (count == -1) 23 | count = _vscprintf(format, ap); 24 | 25 | return count; 26 | } 27 | 28 | __inline int c99_snprintf(char* str, size_t size, const char* format, ...) 29 | { 30 | int count; 31 | va_list ap; 32 | 33 | va_start(ap, format); 34 | count = c99_vsnprintf(str, size, format, ap); 35 | va_end(ap); 36 | 37 | return count; 38 | } 39 | #endif 40 | 41 | #endif 42 | #endif -------------------------------------------------------------------------------- /tests/integration/logging.tcl: -------------------------------------------------------------------------------- 1 | set server_path [tmpdir server.log] 2 | set system_name [string tolower [exec uname -s]] 3 | 4 | if {$system_name eq {linux} || $system_name eq {darwin}} { 5 | start_server [list overrides [list dir $server_path]] { 6 | test "Server is able to generate a stack trace on selected systems" { 7 | r config set watchdog-period 200 8 | r debug sleep 1 9 | set pattern "*debugCommand*" 10 | set retry 10 11 | while {$retry} { 12 | set result [exec tail -100 < [srv 0 stdout]] 13 | if {[string match $pattern $result]} { 14 | break 15 | } 16 | incr retry -1 17 | after 1000 18 | } 19 | if {$retry == 0} { 20 | error "assertion:expected stack trace not found into log file" 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tests/cluster/tests/01-faildet.tcl: -------------------------------------------------------------------------------- 1 | # Check the basic monitoring and failover capabilities. 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "Create a 5 nodes cluster" { 6 | create_cluster 5 5 7 | } 8 | 9 | test "Cluster should start ok" { 10 | assert_cluster_state ok 11 | } 12 | 13 | test "Killing two slave nodes" { 14 | kill_instance redis 5 15 | kill_instance redis 6 16 | } 17 | 18 | test "Cluster should be still up" { 19 | assert_cluster_state ok 20 | } 21 | 22 | test "Killing one master node" { 23 | kill_instance redis 0 24 | } 25 | 26 | # Note: the only slave of instance 0 is already down so no 27 | # failover is possible, that would change the state back to ok. 28 | test "Cluster should be down now" { 29 | assert_cluster_state fail 30 | } 31 | 32 | test "Restarting master node" { 33 | restart_instance redis 0 34 | } 35 | 36 | test "Cluster should be up again" { 37 | assert_cluster_state ok 38 | } 39 | -------------------------------------------------------------------------------- /src/base/base_listener.h: -------------------------------------------------------------------------------- 1 | /* 2 | * base_listener.h 3 | * 4 | * Created on: 2016-3-16 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_BASE_LISTENER_H__ 9 | #define __BASE_BASE_LISTENER_H__ 10 | 11 | #include "util.h" 12 | #include "base_socket.h" 13 | 14 | template 15 | void connect_callback(void* callback_data, uint8_t msg, uint32_t handle, void* pParam) 16 | { 17 | if (msg == NETLIB_MSG_CONNECT) { 18 | T* pConn = new T(); 19 | pConn->OnConnect((BaseSocket*)pParam); 20 | } else { 21 | printf("!!!error msg: %d\n", msg); 22 | } 23 | } 24 | 25 | template 26 | int start_listen(const string& server_ip, uint16_t port) 27 | { 28 | BaseSocket* base_socket = new BaseSocket(); 29 | assert(base_socket); 30 | net_handle_t handle = base_socket->Listen(server_ip.c_str(), port, connect_callback, NULL); 31 | if (handle == NETLIB_INVALID_HANDLE) { 32 | return 1; 33 | } else { 34 | return 0; 35 | } 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/base/Makefile: -------------------------------------------------------------------------------- 1 | CC = g++ 2 | 3 | ver=release 4 | CFLAGS=-Wall -g -std=c++11 5 | ifeq ($(ver), release) 6 | CFLAGS := $(CFLAGS) -O2 7 | endif 8 | 9 | OS_NAME=$(shell uname) 10 | ifeq ($(OS_NAME), Linux) 11 | CFLAGS := $(CFLAGS) -fPIC 12 | endif 13 | 14 | LDFLAGS= -lpthread 15 | AR=ar 16 | CP=/bin/cp 17 | RM=/bin/rm -rf 18 | 19 | # target binary object 20 | BIN=libbase.a 21 | 22 | SrcDir= . 23 | IncDir= . 24 | LibDir= ./ 25 | 26 | SRCS=$(foreach dir,$(SrcDir),$(wildcard $(dir)/*.cpp)) 27 | INCS=$(foreach dir,$(IncDir),$(addprefix -I,$(dir))) 28 | LINKS=$(foreach dir,$(LibDir),$(addprefix -L,$(dir))) 29 | CFLAGS := $(CFLAGS) $(INCS) 30 | LDFLAGS:= $(LINKS) $(LDFLAGS) 31 | 32 | OBJS = $(SRCS:%.cpp=%.o) 33 | .PHONY:all clean 34 | 35 | all:$(BIN) 36 | $(BIN):$(OBJS) 37 | $(AR) -r $(BIN) $(OBJS) 38 | @echo " OK!\tCompile $@ " 39 | @echo 40 | 41 | %.o:%.cpp 42 | $(CC) $(CFLAGS) -c $< -o $@ 43 | 44 | .PHONY: clean 45 | clean: 46 | @echo "[base] Cleaning files..." 47 | @$(RM) $(OBJS) $(BIN) 48 | -------------------------------------------------------------------------------- /src/server/cmd_db.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_db.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/27. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_DB_H__ 10 | #define __CMD_DB_H__ 11 | 12 | #include "server.h" 13 | 14 | void auth_command(ClientConn* conn, const vector& cmd_vec); 15 | void ping_command(ClientConn* conn, const vector& cmd_vec); 16 | void echo_command(ClientConn* conn, const vector& cmd_vec); 17 | void select_command(ClientConn* conn, const vector& cmd_vec); 18 | void dbsize_command(ClientConn* conn, const vector& cmd_vec); 19 | void info_command(ClientConn* conn, const vector& cmd_vec); 20 | 21 | void generic_flushdb(int db_idx); 22 | void flushdb_command(ClientConn* conn, const vector& cmd_vec); 23 | void flushall_command(ClientConn* conn, const vector& cmd_vec); 24 | void debug_command(ClientConn* conn, const vector& cmd_vec); 25 | void command_command(ClientConn* conn, const vector& cmd_vec); 26 | 27 | #endif /* __CMD_DB_H__ */ 28 | -------------------------------------------------------------------------------- /tools/kedis_port/src_kedis_conn.h: -------------------------------------------------------------------------------- 1 | // 2 | // src_kedis_conn.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SRC_KEDIS_CONN_H__ 10 | #define __SRC_KEDIS_CONN_H__ 11 | 12 | #include "base_conn.h" 13 | 14 | enum { 15 | CONN_STATE_IDLE = 0, 16 | CONN_STATE_RECV_AUTH, 17 | CONN_STATE_RECV_PORT, 18 | CONN_STATE_RECV_PSYNC, 19 | CONN_STATE_RECV_SNAPSHOT, 20 | CONN_STATE_CONNECTED, 21 | }; 22 | 23 | class SrcKedisConn : public BaseConn { 24 | public: 25 | SrcKedisConn(); 26 | virtual ~SrcKedisConn(); 27 | 28 | virtual void Close(); 29 | virtual void OnConfirm(); 30 | virtual void OnRead(); 31 | virtual void OnTimer(uint64_t curr_tick); 32 | private: 33 | void _RemoveKeyPrefix(string& key); 34 | void _HandleRedisCommand(vector& cmd_vec); 35 | private: 36 | int state_; 37 | int cur_db_num_; 38 | char* req_buf; 39 | int req_len; 40 | }; 41 | 42 | #endif /* __SRC_KEDIS_CONN_H__ */ 43 | -------------------------------------------------------------------------------- /tools/kedis_to_redis/src_kedis_conn.h: -------------------------------------------------------------------------------- 1 | // 2 | // src_kedis_conn.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SRC_KEDIS_CONN_H__ 10 | #define __SRC_KEDIS_CONN_H__ 11 | 12 | #include "base_conn.h" 13 | 14 | enum { 15 | CONN_STATE_IDLE = 0, 16 | CONN_STATE_RECV_AUTH, 17 | CONN_STATE_RECV_PORT, 18 | CONN_STATE_RECV_PSYNC, 19 | CONN_STATE_RECV_SNAPSHOT, 20 | CONN_STATE_CONNECTED, 21 | }; 22 | 23 | class SrcKedisConn : public BaseConn { 24 | public: 25 | SrcKedisConn(); 26 | virtual ~SrcKedisConn(); 27 | 28 | virtual void Close(); 29 | virtual void OnConfirm(); 30 | virtual void OnRead(); 31 | virtual void OnTimer(uint64_t curr_tick); 32 | private: 33 | void _RemoveKeyPrefix(string& key); 34 | void _HandleRedisCommand(vector& cmd_vec); 35 | private: 36 | int state_; 37 | int cur_db_num_; 38 | char* req_buf; 39 | int req_len; 40 | }; 41 | 42 | #endif /* __SRC_KEDIS_CONN_H__ */ 43 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/kedis_serializer.h: -------------------------------------------------------------------------------- 1 | // 2 | // kedis_serializer.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/21. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __KEDIS_SERIALIZER_H__ 10 | #define __KEDIS_SERIALIZER_H__ 11 | 12 | #include "util.h" 13 | 14 | string serialize_string(uint64_t ttl, const string& key, const string& value); 15 | string serialize_list(uint64_t ttl, const string& key, const list& value_list); 16 | string serialize_hash(uint64_t ttl, const string& key, const map& fv_map); 17 | string serialize_set(uint64_t ttl, const string& key, const set& member_set); 18 | string serialize_zset(uint64_t ttl, const string& key, const map& score_map); 19 | 20 | string serialize_list_vec(uint64_t ttl, const string& key, const vector& vec); 21 | string serialize_hash_vec(uint64_t ttl, const string& key, const vector& vec); 22 | string serialize_zset_vec(uint64_t ttl, const string& key, const vector& vec); 23 | 24 | #endif /* __KEDIS_SERIALIZER_H__ */ 25 | -------------------------------------------------------------------------------- /src/server/migrate.h: -------------------------------------------------------------------------------- 1 | // 2 | // migrate.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/18. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __MIGRATE_H__ 10 | #define __MIGRATE_H__ 11 | 12 | #include "server.h" 13 | 14 | int get_migrate_conn_number(); 15 | 16 | int dump_string(const string& key, uint64_t ttl, const string& encode_value, string& serialized_value); 17 | int dump_complex_struct(int db_idx, uint8_t type, const string& key, uint64_t ttl, const string& encode_value, 18 | uint64_t count, string& serialized_value, const rocksdb::Snapshot* snapshot = NULL); 19 | 20 | void migrate_conn_established(const string& addr); 21 | void migrate_conn_closed(const string& addr); 22 | void continue_migrate_command(const string& addr, const RedisReply& reply); 23 | 24 | void migrate_command(ClientConn* conn, const vector& cmd_vec); 25 | void restore_command(ClientConn* conn, const vector& cmd_vec); 26 | void dump_command(ClientConn* conn, const vector& cmd_vec); 27 | 28 | #endif /* __MIGRATE_H__ */ 29 | -------------------------------------------------------------------------------- /src/server/replication.h: -------------------------------------------------------------------------------- 1 | // 2 | // replication.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/8/30. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __REPLICATION_H__ 10 | #define __REPLICATION_H__ 11 | 12 | #include "server.h" 13 | #include "client_conn.h" 14 | 15 | void replconf_command(ClientConn* conn, const vector& cmd_vec); 16 | void psync_command(ClientConn* conn, const vector& cmd_vec); 17 | void slaveof_command(ClientConn* conn, const vector& cmd_vec); 18 | void repl_snapshot_complete_command(ClientConn* conn, const vector& cmd_vec); 19 | 20 | void replication_cron(); 21 | 22 | void prepare_snapshot_replication(); 23 | 24 | class ReplicationSnapshot { 25 | public: 26 | ReplicationSnapshot(const rocksdb::Snapshot* snapshot); 27 | ~ReplicationSnapshot(); 28 | 29 | void SendTo(ClientConn* conn); 30 | bool IsCompelte(); 31 | private: 32 | const rocksdb::Snapshot* snapshot_; 33 | rocksdb::Iterator* iterator_; 34 | int cur_db_idx_; 35 | }; 36 | 37 | #endif /* __REPLICATION_H__ */ 38 | -------------------------------------------------------------------------------- /tools/kedis_port/kedis_port.h: -------------------------------------------------------------------------------- 1 | // 2 | // kedis_port.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __KEDIS_PORT_H__ 10 | #define __KEDIS_PORT_H__ 11 | 12 | #include "util.h" 13 | 14 | struct Config { 15 | string src_kedis_host; 16 | int src_kedis_port; 17 | int src_kedis_db; // select from which db, -1 means any db 18 | string src_kedis_password; 19 | string dst_kedis_host; 20 | int dst_kedis_port; 21 | int dst_kedis_db; // write to which db, -1 means do not need to select db 22 | string dst_kedis_password; 23 | string prefix; 24 | 25 | Config() { 26 | src_kedis_host = "127.0.0.1"; 27 | src_kedis_port = 6375; 28 | src_kedis_db = -1; 29 | src_kedis_password = ""; 30 | dst_kedis_host = "127.0.0.1"; 31 | dst_kedis_port = 7400; 32 | dst_kedis_db = -1; 33 | dst_kedis_password = ""; 34 | prefix = ""; 35 | } 36 | }; 37 | 38 | extern Config g_config; 39 | 40 | #endif /* __KEDIS_PORT_H__ */ 41 | -------------------------------------------------------------------------------- /tools/kedis_to_redis/kedis_to_redis.h: -------------------------------------------------------------------------------- 1 | // 2 | // kedis_port.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __KEDIS_PORT_H__ 10 | #define __KEDIS_PORT_H__ 11 | 12 | #include "util.h" 13 | 14 | struct Config { 15 | string src_kedis_host; 16 | int src_kedis_port; 17 | int src_kedis_db; // select from which db, -1 means any db 18 | string src_kedis_password; 19 | string dst_redis_host; 20 | int dst_redis_port; 21 | int dst_redis_db; // write to which db, -1 means do not need to select db 22 | string dst_redis_password; 23 | string prefix; 24 | 25 | Config() { 26 | src_kedis_host = "127.0.0.1"; 27 | src_kedis_port = 6375; 28 | src_kedis_db = -1; 29 | src_kedis_password = ""; 30 | dst_redis_host = "127.0.0.1"; 31 | dst_redis_port = 7400; 32 | dst_redis_db = -1; 33 | dst_redis_password = ""; 34 | prefix = ""; 35 | } 36 | }; 37 | 38 | extern Config g_config; 39 | 40 | #endif /* __KEDIS_PORT_H__ */ 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 jianqingdu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/base/ostype.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ostype.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_OS_TYPE_H__ 9 | #define __BASE_OS_TYPE_H__ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | const int SOCKET_ERROR = -1; 29 | const int INVALID_SOCKET = -1; 30 | const int NETLIB_INVALID_HANDLE = -1; 31 | 32 | typedef unsigned char uchar_t; 33 | typedef int net_handle_t; 34 | 35 | enum 36 | { 37 | NETLIB_MSG_CONNECT = 1, 38 | NETLIB_MSG_CONFIRM, 39 | NETLIB_MSG_READ, 40 | NETLIB_MSG_WRITE, 41 | NETLIB_MSG_CLOSE, 42 | NETLIB_MSG_TIMER, 43 | NETLIB_MSG_LOOP, 44 | }; 45 | 46 | typedef void (*callback_t)(void* callback_data, uint8_t msg, uint32_t handle, void* pParam); 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /tests/unit/quit.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"quit"}} { 2 | proc format_command {args} { 3 | set cmd "*[llength $args]\r\n" 4 | foreach a $args { 5 | append cmd "$[string length $a]\r\n$a\r\n" 6 | } 7 | set _ $cmd 8 | } 9 | 10 | test "QUIT returns OK" { 11 | reconnect 12 | assert_equal OK [r quit] 13 | assert_error * {r ping} 14 | } 15 | 16 | test "Pipelined commands after QUIT must not be executed" { 17 | reconnect 18 | r write [format_command quit] 19 | r write [format_command set foo bar] 20 | r flush 21 | assert_equal OK [r read] 22 | assert_error * {r read} 23 | 24 | reconnect 25 | assert_equal {} [r get foo] 26 | } 27 | 28 | test "Pipelined commands after QUIT that exceed read buffer size" { 29 | reconnect 30 | r write [format_command quit] 31 | r write [format_command set foo [string repeat "x" 1024]] 32 | r flush 33 | assert_equal OK [r read] 34 | assert_error * {r read} 35 | 36 | reconnect 37 | assert_equal {} [r get foo] 38 | 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/redis_to_kedis.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // redis_to_kedis.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17-11-20. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "util.h" 10 | #include "cmd_line_parser.h" 11 | #include "event_loop.h" 12 | #include "src_redis_conn.h" 13 | #include "sync_task.h" 14 | #include "simple_log.h" 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | parse_cmd_line(argc, argv); 19 | 20 | int io_thread_num = 0; // all nonblock network io in main thread 21 | init_thread_event_loops(io_thread_num); 22 | init_thread_base_conn(io_thread_num); 23 | 24 | init_simple_log(kLogLevelDebug, "log"); 25 | 26 | g_thread_pool.Init(1); // only one thread in the pool，so tasks will be executed in order 27 | 28 | if (g_config.src_from_rdb) { 29 | SyncRdbTask* task = new SyncRdbTask(); 30 | g_thread_pool.AddTask(task); 31 | } else { 32 | SrcRedisConn* src_conn = new SrcRedisConn(); 33 | src_conn->Connect(g_config.src_redis_host, g_config.src_redis_port); 34 | } 35 | 36 | get_main_event_loop()->Start(); 37 | 38 | return 0; 39 | } 40 | -------------------------------------------------------------------------------- /doc/migrate_design.md: -------------------------------------------------------------------------------- 1 | MIGRATE命令格式 2 | 3 | MIGRATE host port key destination_db timeout [use_prefix] 4 | 5 | 参数说明 6 | 7 | * host 迁移的目的IP 8 | * port 迁移的目的端口 9 | * key 需要迁移的一个key或者key的前缀，具体看use_prefix参数 10 | * destination_db 迁移的目的db号 11 | * timeout 超时时间，如果有错误或者迁移超时，会返回IOERROR错误 12 | * use_prefix 可选，如果存在，表示migrate迁移一个以key为前缀的任何一个key，这个主要是为了支持扩缩容迁移工具 13 | 14 | 返回值 15 | 16 | * 当key迁移成功是，返回OK状态 17 | * 当key不存在时，返回NOKEY错误 18 | * 当发生错误时，比如超时，返回IOERROR错误 19 | 20 | Redis的migrate命令是同步式的，但由于Redis是操作内存数据，所有很快会返回，但是由于kedis-server是基于RocksDB实现的，对于那些复杂数据结构的迁移，可能需要比较长的时间，所以需要实现异步的方式，来减少对其他命令的影响 21 | 22 | kedis-server需要维持到host:port的长连接，用于减少每次migrate命令的连接建立消耗，同时也用于保存实现异步migrate命令的上下文信息： 23 | 24 | struct MigrateContext { 25 | int from_handle; 26 | int timeout; 27 | string key; 28 | }; 29 | 30 | struct MigrateServer { 31 | int handle; 32 | int cur_db_index; 33 | list request_list; 34 | }; 35 | 36 | kedis-server每次收到一个migrate请求，检查是否有对应的MigrateServer，如果没有，则建立连接，然后把MigrateContext加入request_list里面，获取key的所有数据序列化，发送RESTORE请求，异步收到回复后，取出第一个请求的MigrateContext，然后发送回复给其中的from_handle 37 | 38 | key数据的序列化格式，就用kv在RocksdB的encoding方式来序列化 39 | 40 | 需要定时器检查所有MigrateContext的超时情况 41 | 42 | 43 | -------------------------------------------------------------------------------- /tools/hiredis_wrapper/redis_conn.h: -------------------------------------------------------------------------------- 1 | // 2 | // redis_conn.h 3 | // kv-store 4 | // 5 | // Created by ziteng on 17-11-16. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __REDIS_CONN_H__ 10 | #define __REDIS_CONN_H__ 11 | 12 | #include "util.h" 13 | #include "hiredis.h" 14 | 15 | class RedisConn 16 | { 17 | public: 18 | RedisConn(const string& ip, int port); 19 | RedisConn(); 20 | virtual ~RedisConn(); 21 | 22 | void SetAddr(const string& ip, int port) { ip_ = ip; port_ = port; } 23 | void SetPassword(const string& password) { password_ = password; } 24 | int Init(); 25 | 26 | // Caution: do not freeReplyObject(), reply will be freed by the RedisConn object 27 | redisReply* DoRawCmd(const string& cmd); 28 | redisReply* DoCmd(const string& cmd); 29 | 30 | void PipelineRawCmd(const string& cmd); 31 | void PipelineCmd(const string& cmd); 32 | redisReply* GetReply(); 33 | private: 34 | redisContext* context_; 35 | redisReply* reply_; //reply_ will be freed in the next request 36 | string ip_; 37 | int port_; 38 | string password_; 39 | }; 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kedis-Server -- A Redis Compatible persistance Nosql 2 | 3 | Kedis-Server is a Redis-Protocol compatible persistance NoSQL with RocksDB as its storage engine, support most Redis comamnds and those Reids complex data structure: string, list, hash, set, sorted set and hyperloglog 4 | 5 | ## Build 6 | Platform 7 | 8 | Linux 9 | Mac OS X (intel) 10 | 11 | Dependency: 12 | 13 | C++11 compiler, nothing more 14 | recommend install jemalloc, snappy for performance improvement 15 | 16 | How to build 17 | 18 | ./build.sh 19 | 20 | If everything is OK, the executable file will be in src/server/kedis-server. 21 | The server config file is in src/server/kedis.conf 22 | 23 | ## Run 24 | 25 | ./kedis-server -c ./kedis.conf 26 | 27 | After that, you can connect to server with any redis client, suck as redis-cli 28 | 29 | redis-cli -p 6379 set k v 30 | redis-cli -p 6379 get k 31 | 32 | ## Run The Test 33 | 34 | After build the server, you can run the test case with 35 | 36 | ./runtest.sh 37 | 38 | ## Supported Commands 39 | 40 | [Currrent supported Commands](doc/support_command.md) 41 | 42 | ## Architecture 43 | 44 | [Kedis-Server Deisgn Documentation](doc/kedis_server_design_en.md) 45 | -------------------------------------------------------------------------------- /tests/cluster/tests/09-pubsub.tcl: -------------------------------------------------------------------------------- 1 | # Test PUBLISH propagation across the cluster. 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "Create a 5 nodes cluster" { 6 | create_cluster 5 5 7 | } 8 | 9 | proc test_cluster_publish {instance instances} { 10 | # Subscribe all the instances but the one we use to send. 11 | for {set j 0} {$j < $instances} {incr j} { 12 | if {$j != $instance} { 13 | R $j deferred 1 14 | R $j subscribe testchannel 15 | R $j read; # Read the subscribe reply 16 | } 17 | } 18 | 19 | set data [randomValue] 20 | R $instance PUBLISH testchannel $data 21 | 22 | # Read the message back from all the nodes. 23 | for {set j 0} {$j < $instances} {incr j} { 24 | if {$j != $instance} { 25 | set msg [R $j read] 26 | assert {$data eq [lindex $msg 2]} 27 | R $j unsubscribe testchannel 28 | R $j read; # Read the unsubscribe reply 29 | R $j deferred 0 30 | } 31 | } 32 | } 33 | 34 | test "Test publishing to master" { 35 | test_cluster_publish 0 10 36 | } 37 | 38 | test "Test publishing to slave" { 39 | test_cluster_publish 5 10 40 | } 41 | -------------------------------------------------------------------------------- /src/server/redis_byte_stream.h: -------------------------------------------------------------------------------- 1 | // 2 | // redis_byte_stream.h 3 | // kedis 4 | // 5 | // Created by ziteng on 16-6-14. 6 | // Copyright (c) 2016年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __REDIS_BYTE_STREAM_H__ 10 | #define __REDIS_BYTE_STREAM_H__ 11 | 12 | #include "util.h" 13 | 14 | enum { 15 | PARSE_REDIS_ERROR_NO_MORE_DATA = 0, 16 | PARSE_REDIS_ERROR_INVALID_FORMAT = 1, 17 | }; 18 | 19 | class ParseRedisException { 20 | public: 21 | ParseRedisException(uint32_t code, const string& msg) : error_code_(code), error_msg_(msg) { } 22 | virtual ~ParseRedisException() {} 23 | 24 | uint32_t GetErrorCode() { return error_code_; } 25 | char* GetErrorMsg() { return (char*)error_msg_.c_str(); } 26 | private: 27 | uint32_t error_code_; 28 | string error_msg_; 29 | }; 30 | 31 | class RedisByteStream { 32 | public: 33 | RedisByteStream(char* data, int len); 34 | virtual ~RedisByteStream(); 35 | 36 | int GetOffset() { return offset_; } 37 | int ReadByte(); 38 | void ReadLine(string& line); 39 | void ReadBytes(int size, string& value); 40 | private: 41 | char* data_; 42 | int len_; 43 | int offset_; // current read position 44 | }; 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /src/server/cmd_list.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_list.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/20. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_LIST_H__ 10 | #define __CMD_LIST_H__ 11 | 12 | #include "server.h" 13 | 14 | void lindex_command(ClientConn* conn, const vector& cmd_vec); 15 | void linsert_command(ClientConn* conn, const vector& cmd_vec); 16 | void llen_command(ClientConn* conn, const vector& cmd_vec); 17 | void lpush_command(ClientConn* conn, const vector& cmd_vec); 18 | void lpop_command(ClientConn* conn, const vector& cmd_vec); 19 | void lrange_command(ClientConn* conn, const vector& cmd_vec); 20 | void rpush_command(ClientConn* conn, const vector& cmd_vec); 21 | void rpop_command(ClientConn* conn, const vector& cmd_vec); 22 | 23 | void lpushx_command(ClientConn* conn, const vector& cmd_vec); 24 | void rpushx_command(ClientConn* conn, const vector& cmd_vec); 25 | void lrem_command(ClientConn* conn, const vector& cmd_vec); 26 | void lset_command(ClientConn* conn, const vector& cmd_vec); 27 | void ltrim_command(ClientConn* conn, const vector& cmd_vec); 28 | 29 | #endif /* __CMD_LIST_H__ */ 30 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/sync_task.h: -------------------------------------------------------------------------------- 1 | // 2 | // sync_task.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17-11-20. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SYNC_TASK_H__ 10 | #define __SYNC_TASK_H__ 11 | 12 | #include "util.h" 13 | #include "redis_conn.h" 14 | #include "thread_pool.h" 15 | 16 | class SyncRdbTask : public Task { 17 | public: 18 | SyncRdbTask() {} 19 | virtual ~SyncRdbTask() {} 20 | 21 | virtual void run(); 22 | }; 23 | 24 | class SyncAofTask : public Task { 25 | public: 26 | SyncAofTask() {} 27 | virtual ~SyncAofTask() {} 28 | 29 | virtual void run(); 30 | }; 31 | 32 | class SyncCmdTask : public Task { 33 | public: 34 | SyncCmdTask(const string& raw_cmd) : raw_cmd_(raw_cmd) {} 35 | virtual ~SyncCmdTask() {} 36 | 37 | virtual void run(); 38 | private: 39 | string raw_cmd_; 40 | }; 41 | 42 | class KeepalivePingTask : public Task { 43 | public: 44 | KeepalivePingTask() {} 45 | virtual ~KeepalivePingTask() {} 46 | 47 | virtual void run(); 48 | }; 49 | 50 | extern ThreadPool g_thread_pool; 51 | extern bool g_sync_rdb_finished; 52 | 53 | void execute_pipeline(int pipeline_cnt, RedisConn& redis_conn); 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /src/server/cmd_keys.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_keys.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/25. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_KEYS_H__ 10 | #define __CMD_KEYS_H__ 11 | 12 | #include "server.h" 13 | 14 | void del_command(ClientConn* conn, const vector& cmd_vec); 15 | void exists_command(ClientConn* conn, const vector& cmd_vec); 16 | void ttl_command(ClientConn* conn, const vector& cmd_vec); 17 | void pttl_command(ClientConn* conn, const vector& cmd_vec); 18 | void type_command(ClientConn* conn, const vector& cmd_vec); 19 | 20 | void expire_command(ClientConn* conn, const vector& cmd_vec); 21 | void expireat_command(ClientConn* conn, const vector& cmd_vec); 22 | void pexpire_command(ClientConn* conn, const vector& cmd_vec); 23 | void pexpireat_command(ClientConn* conn, const vector& cmd_vec); 24 | void persist_command(ClientConn* conn, const vector& cmd_vec); 25 | void randomkey_command(ClientConn* conn, const vector& cmd_vec); 26 | void keys_command(ClientConn* conn, const vector& cmd_vec); 27 | void scan_command(ClientConn* conn, const vector& cmd_vec); 28 | 29 | #endif /* __CMD_KEYS_H__ */ 30 | -------------------------------------------------------------------------------- /tools/kedis_port/Makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | 3 | ver=release 4 | CFLAGS=-Wall -g -std=c++11 5 | ifeq ($(ver), release) 6 | CFLAGS += -O2 7 | else 8 | CFLAGS += -DDEBUG 9 | endif 10 | 11 | LDFLAGS= -lbase -lpthread ../hiredis/libhiredis.a ../../src/server/redis_parser.o ../../src/server/redis_byte_stream.o 12 | 13 | RM=/bin/rm -rf 14 | ARCH=PC 15 | 16 | # target binary object 17 | BIN=kedis_port 18 | 19 | SrcDir= . ../hiredis_wrapper 20 | IncDir= ../../src/base ../../src/server ../hiredis ../hiredis_wrapper 21 | LibDir= ../../src/base/ 22 | 23 | SRCS=$(foreach dir,$(SrcDir),$(wildcard $(dir)/*.cpp)) 24 | INCS=$(foreach dir,$(IncDir),$(addprefix -I,$(dir))) 25 | LINKS=$(foreach dir,$(LibDir),$(addprefix -L,$(dir))) 26 | CFLAGS := $(CFLAGS) $(INCS) 27 | LDFLAGS:= $(LINKS) $(LDFLAGS) 28 | 29 | OBJS = $(SRCS:%.cpp=%.o) 30 | .PHONY:all clean 31 | 32 | all:$(BIN) 33 | $(BIN):$(OBJS) ../../src/base/libbase.a ../hiredis/libhiredis.a 34 | $(CC) -o $(BIN) $(OBJS) $(LDFLAGS) 35 | @echo " OK!\tCompile $@ " 36 | @echo 37 | 38 | ../hiredis/libhiredis.a: 39 | make -C ../hiredis 40 | 41 | %.o:%.cpp 42 | @echo "$(CC) $(CFLAGS) -c $< -o $@" 43 | @$(CC) $(CFLAGS) -c $< -o $@ 44 | 45 | .PHONY: clean 46 | clean: 47 | @echo "[$(ARCH)] \tCleaning files..." 48 | @$(RM) $(OBJS) $(BIN) 49 | -------------------------------------------------------------------------------- /tools/kedis_to_redis/Makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | 3 | ver=release 4 | CFLAGS=-Wall -g -std=c++11 5 | ifeq ($(ver), release) 6 | CFLAGS += -O2 7 | else 8 | CFLAGS += -DDEBUG 9 | endif 10 | 11 | LDFLAGS= -lbase -lpthread ../hiredis/libhiredis.a ../../src/server/redis_parser.o ../../src/server/redis_byte_stream.o 12 | 13 | RM=/bin/rm -rf 14 | ARCH=PC 15 | 16 | # target binary object 17 | BIN=kedis_to_redis 18 | 19 | SrcDir= . ../hiredis_wrapper 20 | IncDir= ../../src/base ../../src/server ../hiredis ../hiredis_wrapper 21 | LibDir= ../../src/base/ 22 | 23 | SRCS=$(foreach dir,$(SrcDir),$(wildcard $(dir)/*.cpp)) 24 | INCS=$(foreach dir,$(IncDir),$(addprefix -I,$(dir))) 25 | LINKS=$(foreach dir,$(LibDir),$(addprefix -L,$(dir))) 26 | CFLAGS := $(CFLAGS) $(INCS) 27 | LDFLAGS:= $(LINKS) $(LDFLAGS) 28 | 29 | OBJS = $(SRCS:%.cpp=%.o) 30 | .PHONY:all clean 31 | 32 | all:$(BIN) 33 | $(BIN):$(OBJS) ../../src/base/libbase.a ../hiredis/libhiredis.a 34 | $(CC) -o $(BIN) $(OBJS) $(LDFLAGS) 35 | @echo " OK!\tCompile $@ " 36 | @echo 37 | 38 | ../hiredis/libhiredis.a: 39 | make -C ../hiredis 40 | 41 | %.o:%.cpp 42 | @echo "$(CC) $(CFLAGS) -c $< -o $@" 43 | @$(CC) $(CFLAGS) -c $< -o $@ 44 | 45 | .PHONY: clean 46 | clean: 47 | @echo "[$(ARCH)] \tCleaning files..." 48 | @$(RM) $(OBJS) $(BIN) 49 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/Makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | 3 | ver=release 4 | CFLAGS=-Wall -g -std=c++11 5 | ifeq ($(ver), release) 6 | CFLAGS += -O2 7 | else 8 | CFLAGS += -DDEBUG 9 | endif 10 | 11 | LDFLAGS= -lbase -lpthread ../hiredis/libhiredis.a ../../src/server/redis_parser.o ../../src/server/redis_byte_stream.o 12 | 13 | RM=/bin/rm -rf 14 | ARCH=PC 15 | 16 | # target binary object 17 | BIN=redis_to_kedis 18 | 19 | SrcDir= . ../hiredis_wrapper 20 | IncDir= ../../src/base ../../src/server ../hiredis ../hiredis_wrapper 21 | LibDir= ../../src/base/ 22 | 23 | SRCS=$(foreach dir,$(SrcDir),$(wildcard $(dir)/*.cpp)) 24 | INCS=$(foreach dir,$(IncDir),$(addprefix -I,$(dir))) 25 | LINKS=$(foreach dir,$(LibDir),$(addprefix -L,$(dir))) 26 | CFLAGS := $(CFLAGS) $(INCS) 27 | LDFLAGS:= $(LINKS) $(LDFLAGS) 28 | 29 | OBJS = $(SRCS:%.cpp=%.o) 30 | .PHONY:all clean 31 | 32 | all:$(BIN) 33 | $(BIN):$(OBJS) ../../src/base/libbase.a ../hiredis/libhiredis.a 34 | $(CC) -o $(BIN) $(OBJS) $(LDFLAGS) 35 | @echo " OK!\tCompile $@ " 36 | @echo 37 | 38 | ../hiredis/libhiredis.a: 39 | make -C ../hiredis 40 | 41 | %.o:%.cpp 42 | @echo "$(CC) $(CFLAGS) -c $< -o $@" 43 | @$(CC) $(CFLAGS) -c $< -o $@ 44 | 45 | .PHONY: clean 46 | clean: 47 | @echo "[$(ARCH)] \tCleaning files..." 48 | @$(RM) $(OBJS) $(BIN) 49 | -------------------------------------------------------------------------------- /tests/sentinel/tests/06-ckquorum.tcl: -------------------------------------------------------------------------------- 1 | # Test for the SENTINEL CKQUORUM command 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | set num_sentinels [llength $::sentinel_instances] 5 | 6 | test "CKQUORUM reports OK and the right amount of Sentinels" { 7 | foreach_sentinel_id id { 8 | assert_match "*OK $num_sentinels usable*" [S $id SENTINEL CKQUORUM mymaster] 9 | } 10 | } 11 | 12 | test "CKQUORUM detects quorum cannot be reached" { 13 | set orig_quorum [expr {$num_sentinels/2+1}] 14 | S 0 SENTINEL SET mymaster quorum [expr {$num_sentinels+1}] 15 | catch {[S 0 SENTINEL CKQUORUM mymaster]} err 16 | assert_match "*NOQUORUM*" $err 17 | S 0 SENTINEL SET mymaster quorum $orig_quorum 18 | } 19 | 20 | test "CKQUORUM detects failover authorization cannot be reached" { 21 | set orig_quorum [expr {$num_sentinels/2+1}] 22 | S 0 SENTINEL SET mymaster quorum 1 23 | kill_instance sentinel 1 24 | kill_instance sentinel 2 25 | kill_instance sentinel 3 26 | after 5000 27 | catch {[S 0 SENTINEL CKQUORUM mymaster]} err 28 | assert_match "*NOQUORUM*" $err 29 | S 0 SENTINEL SET mymaster quorum $orig_quorum 30 | restart_instance sentinel 1 31 | restart_instance sentinel 2 32 | restart_instance sentinel 3 33 | } 34 | 35 | -------------------------------------------------------------------------------- /src/base/simple_buffer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * simple_buffer.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_SIMPLE_BUFFER_H__ 9 | #define __BASE_SIMPLE_BUFFER_H__ 10 | 11 | #include "ostype.h" 12 | 13 | class SimpleBuffer 14 | { 15 | public: 16 | SimpleBuffer(); 17 | ~SimpleBuffer(); 18 | uchar_t* GetBuffer() const { return m_buffer; } 19 | uchar_t* GetWriteBuffer() const { return m_buffer + m_write_offset; } 20 | uchar_t* GetReadBuffer() const { return m_buffer + m_read_offset; } 21 | uint32_t GetAllocSize() const { return m_alloc_size; } 22 | uint32_t GetWriteOffset() const { return m_write_offset; } 23 | uint32_t GetReadOffset() const { return m_read_offset; } 24 | uint32_t GetWritableLen() const { return m_alloc_size - m_write_offset; } 25 | uint32_t GetReadableLen() const { return m_write_offset - m_read_offset; } 26 | void IncWriteOffset(uint32_t len) { m_write_offset += len; } 27 | 28 | void Extend(uint32_t len); 29 | uint32_t Write(void* buf, uint32_t len); 30 | uint32_t Read(void* buf, uint32_t len); 31 | void ResetOffset(); 32 | void Clear(); 33 | private: 34 | uchar_t* m_buffer; 35 | uint32_t m_alloc_size; 36 | uint32_t m_write_offset; 37 | uint32_t m_read_offset; 38 | }; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /tests/unit/memefficiency.tcl: -------------------------------------------------------------------------------- 1 | proc test_memory_efficiency {range} { 2 | r flushall 3 | set rd [redis_deferring_client] 4 | set base_mem [s used_memory] 5 | set written 0 6 | for {set j 0} {$j < 10000} {incr j} { 7 | set key key:$j 8 | set val [string repeat A [expr {int(rand()*$range)}]] 9 | $rd set $key $val 10 | incr written [string length $key] 11 | incr written [string length $val] 12 | incr written 2 ;# A separator is the minimum to store key-value data. 13 | } 14 | for {set j 0} {$j < 10000} {incr j} { 15 | $rd read ; # Discard replies 16 | } 17 | 18 | set current_mem [s used_memory] 19 | set used [expr {$current_mem-$base_mem}] 20 | set efficiency [expr {double($written)/$used}] 21 | return $efficiency 22 | } 23 | 24 | start_server {tags {"memefficiency"}} { 25 | foreach {size_range expected_min_efficiency} { 26 | 32 0.15 27 | 64 0.25 28 | 128 0.35 29 | 1024 0.75 30 | 16384 0.82 31 | } { 32 | test "Memory efficiency with values in range $size_range" { 33 | set efficiency [test_memory_efficiency $size_range] 34 | assert {$efficiency >= $expected_min_efficiency} 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /doc/zset_double_score_design.md: -------------------------------------------------------------------------------- 1 | ## zset数据结构和score的关系 2 | 在Redis里面zset是一种有序集合，一个zset的key下面，所有(score，member)的元素，都按score排序，这样可以获取一个区间的所有元素，比如通过ZRANGEBYSCORE命令来获取[score1, score2]这个区间内的所有元素 3 | 4 | RocksDB只支持kv结构，而且key是按字节序排序的，所有为了实现zset数据类型，在RocksDB上的key必须用下面的方式来编码： 5 | 6 | key:score:member 7 | 8 | 要是score的编码符合字节序和score的double数值大小顺序一致，就可以实现zset的数据结构 9 | 10 | ## double类型的score的编码方式 11 | 按照IEEE-754标准，double类型占用8个字节，按大端字节序分成3个部分： 12 | 13 | * 符号: 1 bit （0为正，1为负） 14 | * 指数: 11 bits 15 | * 分数: 52 bits 16 | 17 | 可以参考这个[wiki](https://en.wikipedia.org/wiki/Double-precision_floating-point_format) 18 | 19 | 对于正数部分，按照IEEE-754的编码，如果用大端字节序保存，则字节序和double的大小顺序一致。对于负数部分，如果用大端字节序保存，则字节序和double的顺序刚好相反。所以采用如下的编码方式，则所有double类型的字节序和大小一致： 20 | 21 | uint64_t double_to_uint64(double d) 22 | { 23 | uint64_t u = *(uint64_t*)&d; 24 | if (d >= 0) { 25 | u |= 0x8000000000000000; 26 | } else { 27 | u = ~u; 28 | } 29 | 30 | return u; 31 | } 32 | 33 | double uint64_to_double(uint64_t u) 34 | { 35 | if ((u & 0x8000000000000000) > 0) { 36 | u &= ~0x8000000000000000; 37 | } else { 38 | u = ~u; 39 | } 40 | 41 | return *(double*)&u; 42 | } 43 | 44 | 就是对于所有正数，最高位置1，对于所以负数，则取反。 45 | 这样处理后正数的最高位是1，负数的最高位是0，所以按字节序，所有正数肯定大于所有负数 46 | ，所有正数按字节序排序，所有负数也按字节序排序 47 | -------------------------------------------------------------------------------- /src/server/key_lock.h: -------------------------------------------------------------------------------- 1 | // 2 | // key_lock.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/25. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __KEY_LOCK_H__ 10 | #define __KEY_LOCK_H__ 11 | 12 | #include "util.h" 13 | 14 | // key lock is used when multiple thread are operating on the same key in complex structure 15 | void init_key_lock(); 16 | 17 | void lock_key(int db_idx, const string& key); 18 | void unlock_key(int db_idx, const string& key); 19 | 20 | // keys are in set, so they are sorted and unique, which prevent deadlock 21 | void lock_keys(int db_idx, set& keys); 22 | void unlock_keys(int db_idx, set& keys); 23 | 24 | void spinlock_db(int db_idx); 25 | void lock_db(int db_idx); 26 | void unlock_db(int db_idx); 27 | 28 | // when get snapshot/flushdb we need block/freeze all threads use these functions 29 | void spinlock_all(); 30 | void lock_all(); 31 | void unlock_all(); 32 | 33 | class KeyLockGuard { 34 | public: 35 | KeyLockGuard(int db_idx, const string& key) : db_idx_(db_idx), key_(key) { 36 | lock_key(db_idx_, key_); 37 | } 38 | ~KeyLockGuard() { 39 | unlock_key(db_idx_, key_); 40 | } 41 | private: 42 | int db_idx_; 43 | string key_; 44 | //bool incr_; 45 | }; 46 | 47 | #endif /* __KEY_LOCK_H__ */ 48 | -------------------------------------------------------------------------------- /src/server/Makefile: -------------------------------------------------------------------------------- 1 | ROCKSDB_PATH=../3rd_party/rocksdb-5.6.2 2 | ifdef rocksdb_ver 3 | ROCKSDB_PATH=../3rd_party/rocksdb-$(rocksdb_ver) 4 | endif 5 | 6 | include $(ROCKSDB_PATH)/make_config.mk 7 | 8 | CC=g++ 9 | 10 | ver=release 11 | CFLAGS=-Wall -g -std=c++11 12 | ifeq ($(ver), release) 13 | CFLAGS += -O2 14 | else 15 | CFLAGS += -DDEBUG 16 | endif 17 | 18 | LDFLAGS= -lbase $(PLATFORM_LDFLAGS) $(ROCKSDB_PATH)/librocksdb.a 19 | 20 | LN=/bin/ln -s 21 | AR=ar 22 | CP=/bin/cp 23 | RM=/bin/rm -rf 24 | 25 | # target binary object 26 | BIN=kedis-server 27 | 28 | SrcDir= . 29 | IncDir= ../base $(ROCKSDB_PATH)/include 30 | LibDir= ../base/ 31 | 32 | SRCS=$(foreach dir,$(SrcDir),$(wildcard $(dir)/*.cpp)) 33 | INCS=$(foreach dir,$(IncDir),$(addprefix -I,$(dir))) 34 | LINKS=$(foreach dir,$(LibDir),$(addprefix -L,$(dir))) 35 | CFLAGS := $(CFLAGS) $(INCS) -Wno-deprecated-declarations 36 | LDFLAGS:= $(LINKS) $(LDFLAGS) 37 | 38 | OBJS = $(SRCS:%.cpp=%.o) 39 | .PHONY:all clean 40 | 41 | all:$(BIN) 42 | $(BIN):$(OBJS) ../base/libbase.a $(ROCKSDB_PATH)/librocksdb.a 43 | $(CC) -o $(BIN) $(OBJS) $(LDFLAGS) 44 | @echo " OK!\tCompile $@ " 45 | @echo 46 | 47 | %.o:%.cpp 48 | @echo "$(CC) $(CFLAGS) -c $< -o $@" 49 | @$(CC) $(CFLAGS) -c $< -o $@ 50 | 51 | .PHONY: clean 52 | clean: 53 | @echo "[server] Cleaning files..." 54 | @$(RM) $(OBJS) $(BIN) 55 | -------------------------------------------------------------------------------- /src/server/cmd_hash.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_hash.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/20. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_HASH_H__ 10 | #define __CMD_HASH_H__ 11 | 12 | #include "server.h" 13 | 14 | void hdel_command(ClientConn* conn, const vector& cmd_vec); 15 | void hexists_command(ClientConn* conn, const vector& cmd_vec); 16 | void hget_command(ClientConn* conn, const vector& cmd_vec); 17 | void hgetall_command(ClientConn* conn, const vector& cmd_vec); 18 | void hkeys_command(ClientConn* conn, const vector& cmd_vec); 19 | void hvals_command(ClientConn* conn, const vector& cmd_vec); 20 | 21 | void hincrby_command(ClientConn* conn, const vector& cmd_vec); 22 | void hincrbyfloat_command(ClientConn* conn, const vector& cmd_vec); 23 | 24 | void hlen_command(ClientConn* conn, const vector& cmd_vec); 25 | void hmget_command(ClientConn* conn, const vector& cmd_vec); 26 | void hmset_command(ClientConn* conn, const vector& cmd_vec); 27 | void hscan_command(ClientConn* conn, const vector& cmd_vec); 28 | void hset_command(ClientConn* conn, const vector& cmd_vec); 29 | void hsetnx_command(ClientConn* conn, const vector& cmd_vec); 30 | void hstrlen_command(ClientConn* conn, const vector& cmd_vec); 31 | 32 | #endif /* __CMD_HASH_H__ */ 33 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/src_redis_conn.h: -------------------------------------------------------------------------------- 1 | // 2 | // src_redis_conn.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17-11-20. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __SRC_REDIS_CONN_H__ 10 | #define __SRC_REDIS_CONN_H__ 11 | 12 | #include "base_conn.h" 13 | 14 | enum { 15 | CONN_STATE_AUTH, 16 | CONN_STATE_READ_RDB_LEN, 17 | CONN_STATE_READ_RDB_DATA, 18 | CONN_STATE_SYNC_CMD, 19 | }; 20 | 21 | class SrcRedisConn : public BaseConn { 22 | public: 23 | SrcRedisConn() : rdb_file_(NULL), aof_file_(NULL), cur_db_num_(-1) {} 24 | virtual ~SrcRedisConn() {} 25 | 26 | virtual void OnConfirm(); 27 | virtual void OnRead(); 28 | virtual void OnClose(); 29 | virtual void OnTimer(uint64_t curr_tick); 30 | 31 | private: 32 | void _LimitRDBReceiveSpeed(int recv_bytes); 33 | void _HandleAuth(); 34 | void _ReadRdbLength(); 35 | void _ReadRdbData(); 36 | void _SyncWriteCommand(); 37 | void _RemoveKeyPrefix(string& key); 38 | void _CommitAofTask(); 39 | void _HandleRedisCommand(vector& cmd_vec); 40 | private: 41 | int conn_state_; 42 | FILE* rdb_file_; 43 | long rdb_remain_len_; 44 | FILE* aof_file_; 45 | int cur_db_num_; 46 | char* req_buf; 47 | int req_len; 48 | uint64_t start_tick_; // 接收数据每秒开始的时刻 49 | int recv_bytes_; // 当前秒接收的数据量 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/server/redis_byte_stream.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // redis_byte_stream.cpp 3 | // kv-store 4 | // 5 | // Created by ziteng on 16-6-14. 6 | // Copyright (c) 2016年 mgj. All rights reserved. 7 | // 8 | 9 | #include "redis_byte_stream.h" 10 | 11 | RedisByteStream::RedisByteStream(char* data, int len) 12 | { 13 | data_ = data; 14 | len_ = len; 15 | offset_ = 0; 16 | } 17 | 18 | RedisByteStream::~RedisByteStream() 19 | { 20 | 21 | } 22 | 23 | int RedisByteStream::ReadByte() 24 | { 25 | if (offset_ >= len_) { 26 | throw ParseRedisException(PARSE_REDIS_ERROR_NO_MORE_DATA, "no more data"); 27 | } 28 | 29 | char ch = data_[offset_]; 30 | offset_++; 31 | return ch; 32 | } 33 | 34 | void RedisByteStream::ReadLine(string& line) 35 | { 36 | char* start_pos = data_ + offset_; 37 | char* pos = strstr(start_pos, "\r\n"); 38 | if (!pos) { 39 | throw ParseRedisException(PARSE_REDIS_ERROR_NO_MORE_DATA, "no more data"); 40 | } 41 | 42 | int size = (int)(pos - start_pos); 43 | line.append(start_pos, size); 44 | offset_ += size + 2; 45 | } 46 | 47 | void RedisByteStream::ReadBytes(int size, string& value) 48 | { 49 | if (offset_ + size + 2 > len_) { 50 | throw ParseRedisException(PARSE_REDIS_ERROR_NO_MORE_DATA, "no more data"); 51 | } 52 | 53 | char* start_pos = data_ + offset_; 54 | value.append(start_pos, size); 55 | offset_ += size + 2; 56 | } 57 | -------------------------------------------------------------------------------- /tests/integration/aof-race.tcl: -------------------------------------------------------------------------------- 1 | set defaults { appendonly {yes} appendfilename {appendonly.aof} } 2 | set server_path [tmpdir server.aof] 3 | set aof_path "$server_path/appendonly.aof" 4 | 5 | proc start_server_aof {overrides code} { 6 | upvar defaults defaults srv srv server_path server_path 7 | set config [concat $defaults $overrides] 8 | start_server [list overrides $config] $code 9 | } 10 | 11 | tags {"aof"} { 12 | # Specific test for a regression where internal buffers were not properly 13 | # cleaned after a child responsible for an AOF rewrite exited. This buffer 14 | # was subsequently appended to the new AOF, resulting in duplicate commands. 15 | start_server_aof [list dir $server_path] { 16 | set client [redis [srv host] [srv port]] 17 | set bench [open "|src/redis-benchmark -q -p [srv port] -c 20 -n 20000 incr foo" "r+"] 18 | after 100 19 | 20 | # Benchmark should be running by now: start background rewrite 21 | $client bgrewriteaof 22 | 23 | # Read until benchmark pipe reaches EOF 24 | while {[string length [read $bench]] > 0} {} 25 | 26 | # Check contents of foo 27 | assert_equal 20000 [$client get foo] 28 | } 29 | 30 | # Restart server to replay AOF 31 | start_server_aof [list dir $server_path] { 32 | set client [redis [srv host] [srv port]] 33 | assert_equal 20000 [$client get foo] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tests/sentinel/tests/01-conf-update.tcl: -------------------------------------------------------------------------------- 1 | # Test Sentinel configuration consistency after partitions heal. 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "We can failover with Sentinel 1 crashed" { 6 | set old_port [RI $master_id tcp_port] 7 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 8 | assert {[lindex $addr 1] == $old_port} 9 | 10 | # Crash Sentinel 1 11 | kill_instance sentinel 1 12 | 13 | kill_instance redis $master_id 14 | foreach_sentinel_id id { 15 | if {$id != 1} { 16 | wait_for_condition 1000 50 { 17 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 18 | } else { 19 | fail "Sentinel $id did not received failover info" 20 | } 21 | } 22 | } 23 | restart_instance redis $master_id 24 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 25 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] 26 | } 27 | 28 | test "After Sentinel 1 is restarted, its config gets updated" { 29 | restart_instance sentinel 1 30 | wait_for_condition 1000 50 { 31 | [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 32 | } else { 33 | fail "Restarted Sentinel did not received failover info" 34 | } 35 | } 36 | 37 | test "New master [join $addr {:}] role matches" { 38 | assert {[RI $master_id role] eq {master}} 39 | } 40 | -------------------------------------------------------------------------------- /tests/integration/convert-zipmap-hash-on-load.tcl: -------------------------------------------------------------------------------- 1 | # Copy RDB with zipmap encoded hash to server path 2 | set server_path [tmpdir "server.convert-zipmap-hash-on-load"] 3 | 4 | exec cp -f tests/assets/hash-zipmap.rdb $server_path 5 | start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] { 6 | test "RDB load zipmap hash: converts to ziplist" { 7 | r select 0 8 | 9 | assert_match "*ziplist*" [r debug object hash] 10 | assert_equal 2 [r hlen hash] 11 | assert_match {v1 v2} [r hmget hash f1 f2] 12 | } 13 | } 14 | 15 | exec cp -f tests/assets/hash-zipmap.rdb $server_path 16 | start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-entries" 1]] { 17 | test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-entries is exceeded" { 18 | r select 0 19 | 20 | assert_match "*hashtable*" [r debug object hash] 21 | assert_equal 2 [r hlen hash] 22 | assert_match {v1 v2} [r hmget hash f1 f2] 23 | } 24 | } 25 | 26 | exec cp -f tests/assets/hash-zipmap.rdb $server_path 27 | start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-value" 1]] { 28 | test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-value is exceeded" { 29 | r select 0 30 | 31 | assert_match "*hashtable*" [r debug object hash] 32 | assert_equal 2 [r hlen hash] 33 | assert_match {v1 v2} [r hmget hash f1 f2] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/server/binlog.h: -------------------------------------------------------------------------------- 1 | // 2 | // binlog.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/12. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __BINLOG_H__ 10 | #define __BINLOG_H__ 11 | 12 | #include "util.h" 13 | #include "thread_pool.h" 14 | #include "rocksdb/db.h" 15 | 16 | class Binlog { 17 | public: 18 | Binlog(); 19 | virtual ~Binlog(); 20 | 21 | int Init(); 22 | void InitWithMaster(const string& binlog_id, int cur_db_idx, uint64_t seq); 23 | void Drop(); // clear all data 24 | int Store(int db_idx, const string& command); 25 | int Extract(uint64_t seq, string& command); 26 | void Purge(); 27 | 28 | string GetBinlogId() { return binlog_id_; } 29 | int GetCurDbIdx() { return cur_db_idx_; } 30 | uint64_t GetMinSeq() { return min_seq_; } 31 | uint64_t GetMaxSeq() { return max_seq_; } 32 | private: 33 | void GenerateBinlogId(); 34 | string EncodeKey(uint64_t seq); 35 | uint64_t DecodeKey(const rocksdb::Slice& slice); 36 | private: 37 | rocksdb::DB* db_; 38 | mutex mtx_; 39 | string binlog_id_; 40 | int cur_db_idx_; 41 | uint64_t min_seq_; 42 | uint64_t max_seq_; 43 | bool empty_; 44 | }; 45 | 46 | class PurgeBinlogThread : public Thread { 47 | public: 48 | PurgeBinlogThread() {} 49 | virtual ~PurgeBinlogThread() {} 50 | 51 | virtual void OnThreadRun(void); 52 | }; 53 | 54 | #endif /* __BINLOG_H__ */ 55 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/cmd_line_parser.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_line_parser.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17-11-20. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_LINE_PARSER_H__ 10 | #define __CMD_LINE_PARSER_H__ 11 | 12 | #include "util.h" 13 | 14 | struct Config { 15 | string src_redis_host; 16 | int src_redis_port; 17 | int src_redis_db; // select from which db, -1 means any db 18 | string src_redis_password; 19 | string dst_kedis_host; 20 | int dst_kedis_port; 21 | int dst_kedis_db; // write to which db, -1 means do not need to select db 22 | string dst_kedis_password; 23 | int pipeline_cnt; 24 | string rdb_file; 25 | string aof_file; 26 | string prefix; 27 | int network_limit; // limit network speed when receive RDB file 28 | bool src_from_rdb; 29 | 30 | Config() { 31 | src_redis_host = "127.0.0.1"; 32 | src_redis_port = 6375; 33 | src_redis_db = -1; 34 | src_redis_password = ""; 35 | dst_kedis_host = "127.0.0.1"; 36 | dst_kedis_port = 7400; 37 | dst_kedis_db = -1; 38 | dst_kedis_password = ""; 39 | pipeline_cnt = 32; 40 | rdb_file = "dump.rdb"; 41 | aof_file = "dump.aof"; 42 | prefix = ""; 43 | network_limit = 50 * 1024 * 1024; 44 | src_from_rdb = false; 45 | } 46 | }; 47 | 48 | extern Config g_config; 49 | 50 | void parse_cmd_line(int argc, char* argv[]); 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /tests/cluster/tests/11-manual-takeover.tcl: -------------------------------------------------------------------------------- 1 | # Manual takeover test 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "Create a 5 nodes cluster" { 6 | create_cluster 5 5 7 | } 8 | 9 | test "Cluster is up" { 10 | assert_cluster_state ok 11 | } 12 | 13 | test "Cluster is writable" { 14 | cluster_write_test 0 15 | } 16 | 17 | test "Killing majority of master nodes" { 18 | kill_instance redis 0 19 | kill_instance redis 1 20 | kill_instance redis 2 21 | } 22 | 23 | test "Cluster should eventually be down" { 24 | assert_cluster_state fail 25 | } 26 | 27 | test "Use takeover to bring slaves back" { 28 | R 5 cluster failover takeover 29 | R 6 cluster failover takeover 30 | R 7 cluster failover takeover 31 | } 32 | 33 | test "Cluster should eventually be up again" { 34 | assert_cluster_state ok 35 | } 36 | 37 | test "Cluster is writable" { 38 | cluster_write_test 4 39 | } 40 | 41 | test "Instance #5, #6, #7 are now masters" { 42 | assert {[RI 5 role] eq {master}} 43 | assert {[RI 6 role] eq {master}} 44 | assert {[RI 7 role] eq {master}} 45 | } 46 | 47 | test "Restarting the previously killed master nodes" { 48 | restart_instance redis 0 49 | restart_instance redis 1 50 | restart_instance redis 2 51 | } 52 | 53 | test "Instance #0, #1, #2 gets converted into a slaves" { 54 | wait_for_condition 1000 50 { 55 | [RI 0 role] eq {slave} && [RI 1 role] eq {slave} && [RI 2 role] eq {slave} 56 | } else { 57 | fail "Old masters not converted into slaves" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /tests/sentinel/tests/05-manual.tcl: -------------------------------------------------------------------------------- 1 | # Test manual failover 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "Manual failover works" { 6 | set old_port [RI $master_id tcp_port] 7 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 8 | assert {[lindex $addr 1] == $old_port} 9 | catch {S 0 SENTINEL FAILOVER mymaster} reply 10 | assert {$reply eq "OK"} 11 | foreach_sentinel_id id { 12 | wait_for_condition 1000 50 { 13 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 14 | } else { 15 | fail "At least one Sentinel did not received failover info" 16 | } 17 | } 18 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 19 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] 20 | } 21 | 22 | test "New master [join $addr {:}] role matches" { 23 | assert {[RI $master_id role] eq {master}} 24 | } 25 | 26 | test "All the other slaves now point to the new master" { 27 | foreach_redis_id id { 28 | if {$id != $master_id && $id != 0} { 29 | wait_for_condition 1000 50 { 30 | [RI $id master_port] == [lindex $addr 1] 31 | } else { 32 | fail "Redis ID $id not configured to replicate with new master" 33 | } 34 | } 35 | } 36 | } 37 | 38 | test "The old master eventually gets reconfigured as a slave" { 39 | wait_for_condition 1000 50 { 40 | [RI 0 master_port] == [lindex $addr 1] 41 | } else { 42 | fail "Old master not reconfigured as slave of new master" 43 | } 44 | } 45 | 46 | -------------------------------------------------------------------------------- /src/base/simple_log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * simple_log.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_SIMPLE_LOG_H__ 9 | #define __BASE_SIMPLE_LOG_H__ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | using std::string; 16 | using std::mutex; 17 | 18 | enum LogLevel { 19 | kLogLevelError = 0, 20 | kLogLevelWarning = 1, 21 | kLogLevelInfo = 2, 22 | kLogLevelDebug = 3, 23 | }; 24 | 25 | const string kLogLevelName[] = {"[ERROR]", "[WARNING]", "[INFO]", "[DEBUG]"}; 26 | 27 | const int kMaxLogLineSize = 4096; 28 | 29 | class SimpleLog { 30 | public: 31 | SimpleLog(); 32 | virtual ~SimpleLog(); 33 | 34 | void Init(LogLevel level, const string& path); 35 | void LogMessage(LogLevel level, const char* fmt, ...); 36 | void SetLogLevel(LogLevel level) { log_level_ = level; } 37 | private: 38 | void OpenFile(); 39 | bool IsNewDay(time_t current_time); 40 | private: 41 | LogLevel log_level_; 42 | string log_path_; 43 | FILE* log_file_; 44 | time_t last_log_time_; 45 | char log_buf_[kMaxLogLineSize]; 46 | mutex mutex_; 47 | }; 48 | 49 | // if application only has one log path, use these two api to simplify the log process 50 | // see http://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html for explanation of ##__VA_ARGS__ 51 | extern SimpleLog g_simple_log; 52 | void init_simple_log(LogLevel level, const string& path); 53 | #define log_message(level, fmt, ...) g_simple_log.LogMessage(level, fmt, ##__VA_ARGS__) 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /tests/cluster/tests/02-failover.tcl: -------------------------------------------------------------------------------- 1 | # Check the basic monitoring and failover capabilities. 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "Create a 5 nodes cluster" { 6 | create_cluster 5 5 7 | } 8 | 9 | test "Cluster is up" { 10 | assert_cluster_state ok 11 | } 12 | 13 | test "Cluster is writable" { 14 | cluster_write_test 0 15 | } 16 | 17 | test "Instance #5 is a slave" { 18 | assert {[RI 5 role] eq {slave}} 19 | } 20 | 21 | test "Instance #5 synced with the master" { 22 | wait_for_condition 1000 50 { 23 | [RI 5 master_link_status] eq {up} 24 | } else { 25 | fail "Instance #5 master link status is not up" 26 | } 27 | } 28 | 29 | set current_epoch [CI 1 cluster_current_epoch] 30 | 31 | test "Killing one master node" { 32 | kill_instance redis 0 33 | } 34 | 35 | test "Wait for failover" { 36 | wait_for_condition 1000 50 { 37 | [CI 1 cluster_current_epoch] > $current_epoch 38 | } else { 39 | fail "No failover detected" 40 | } 41 | } 42 | 43 | test "Cluster should eventually be up again" { 44 | assert_cluster_state ok 45 | } 46 | 47 | test "Cluster is writable" { 48 | cluster_write_test 1 49 | } 50 | 51 | test "Instance #5 is now a master" { 52 | assert {[RI 5 role] eq {master}} 53 | } 54 | 55 | test "Restarting the previously killed master node" { 56 | restart_instance redis 0 57 | } 58 | 59 | test "Instance #0 gets converted into a slave" { 60 | wait_for_condition 1000 50 { 61 | [RI 0 role] eq {slave} 62 | } else { 63 | fail "Old master was not converted into slave" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tools/hiredis/appveyor.yml: -------------------------------------------------------------------------------- 1 | # Appveyor configuration file for CI build of hiredis on Windows (under Cygwin) 2 | environment: 3 | matrix: 4 | - CYG_ROOT: C:\cygwin64 5 | CYG_SETUP: setup-x86_64.exe 6 | CYG_MIRROR: http://cygwin.mirror.constant.com 7 | CYG_CACHE: C:\cygwin64\var\cache\setup 8 | CYG_BASH: C:\cygwin64\bin\bash 9 | CC: gcc 10 | - CYG_ROOT: C:\cygwin 11 | CYG_SETUP: setup-x86.exe 12 | CYG_MIRROR: http://cygwin.mirror.constant.com 13 | CYG_CACHE: C:\cygwin\var\cache\setup 14 | CYG_BASH: C:\cygwin\bin\bash 15 | CC: gcc 16 | TARGET: 32bit 17 | TARGET_VARS: 32bit-vars 18 | 19 | # Cache Cygwin files to speed up build 20 | cache: 21 | - '%CYG_CACHE%' 22 | clone_depth: 1 23 | 24 | # Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail 25 | init: 26 | - git config --global core.autocrlf input 27 | 28 | # Install needed build dependencies 29 | install: 30 | - ps: 'Start-FileDownload "http://cygwin.com/$env:CYG_SETUP" -FileName "$env:CYG_SETUP"' 31 | - '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm > NUL 2>&1' 32 | - '%CYG_BASH% -lc "cygcheck -dc cygwin"' 33 | 34 | build_script: 35 | - 'echo building...' 36 | - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 2 | Copyright (c) 2010-2011, Pieter Noordhuis 3 | 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, 10 | this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of Redis nor the names of its contributors may be used 17 | to endorse or promote products derived from this software without specific 18 | prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 24 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /tests/unit/latency-monitor.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"latency-monitor"}} { 2 | # Set a threshold high enough to avoid spurious latency events. 3 | r config set latency-monitor-threshold 200 4 | r latency reset 5 | 6 | test {Test latency events logging} { 7 | r debug sleep 0.3 8 | after 1100 9 | r debug sleep 0.4 10 | after 1100 11 | r debug sleep 0.5 12 | assert {[r latency history command] >= 3} 13 | } 14 | 15 | test {LATENCY HISTORY output is ok} { 16 | set min 250 17 | set max 450 18 | foreach event [r latency history command] { 19 | lassign $event time latency 20 | assert {$latency >= $min && $latency <= $max} 21 | incr min 100 22 | incr max 100 23 | set last_time $time ; # Used in the next test 24 | } 25 | } 26 | 27 | test {LATENCY LATEST output is ok} { 28 | foreach event [r latency latest] { 29 | lassign $event eventname time latency max 30 | assert {$eventname eq "command"} 31 | assert {$max >= 450 & $max <= 650} 32 | assert {$time == $last_time} 33 | break 34 | } 35 | } 36 | 37 | test {LATENCY HISTORY / RESET with wrong event name is fine} { 38 | assert {[llength [r latency history blabla]] == 0} 39 | assert {[r latency reset blabla] == 0} 40 | } 41 | 42 | test {LATENCY DOCTOR produces some output} { 43 | assert {[string length [r latency doctor]] > 0} 44 | } 45 | 46 | test {LATENCY RESET is able to reset events} { 47 | assert {[r latency reset] > 0} 48 | assert {[r latency latest] eq {}} 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/server/cmd_zset.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_zset.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/20. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_ZSET_H__ 10 | #define __CMD_ZSET_H__ 11 | 12 | #include "server.h" 13 | 14 | void zadd_command(ClientConn* conn, const vector& cmd_vec); 15 | void zincrby_command(ClientConn* conn, const vector& cmd_vec); 16 | void zcard_command(ClientConn* conn, const vector& cmd_vec); 17 | void zcount_command(ClientConn* conn, const vector& cmd_vec); 18 | 19 | void zrange_command(ClientConn* conn, const vector& cmd_vec); 20 | void zrevrange_command(ClientConn* conn, const vector& cmd_vec); 21 | void zrangebyscore_command(ClientConn* conn, const vector& cmd_vec); 22 | void zrevrangebyscore_command(ClientConn* conn, const vector& cmd_vec); 23 | void zlexcount_command(ClientConn* conn, const vector& cmd_vec); 24 | void zrangebylex_command(ClientConn* conn, const vector& cmd_vec); 25 | void zrevrangebylex_command(ClientConn* conn, const vector& cmd_vec); 26 | 27 | void zrank_command(ClientConn* conn, const vector& cmd_vec); 28 | void zrevrank_command(ClientConn* conn, const vector& cmd_vec); 29 | 30 | void zrem_command(ClientConn* conn, const vector& cmd_vec); 31 | void zremrangebylex_command(ClientConn* conn, const vector& cmd_vec); 32 | void zremrangebyrank_command(ClientConn* conn, const vector& cmd_vec); 33 | void zremrangebyscore_command(ClientConn* conn, const vector& cmd_vec); 34 | void zscore_command(ClientConn* conn, const vector& cmd_vec); 35 | void zscan_command(ClientConn* conn, const vector& cmd_vec); 36 | 37 | #endif /* __CMD_ZSET_H__ */ 38 | -------------------------------------------------------------------------------- /tests/unit/type/list-2.tcl: -------------------------------------------------------------------------------- 1 | start_server { 2 | tags {"list"} 3 | overrides { 4 | } 5 | } { 6 | source "tests/unit/type/list-common.tcl" 7 | 8 | foreach {type large} [array get largevalue] { 9 | tags {"slow"} { 10 | test "LTRIM stress testing - $type" { 11 | set mylist {} 12 | set startlen 32 13 | r del mylist 14 | 15 | # Start with the large value to ensure the 16 | # right encoding is used. 17 | r rpush mylist $large 18 | lappend mylist $large 19 | 20 | for {set i 0} {$i < $startlen} {incr i} { 21 | set str [randomInt 9223372036854775807] 22 | r rpush mylist $str 23 | lappend mylist $str 24 | } 25 | 26 | for {set i 0} {$i < 1000} {incr i} { 27 | set min [expr {int(rand()*$startlen)}] 28 | set max [expr {$min+int(rand()*$startlen)}] 29 | set before_len [llength $mylist] 30 | set before_len_r [r llen mylist] 31 | set mylist [lrange $mylist $min $max] 32 | r ltrim mylist $min $max 33 | assert_equal $mylist [r lrange mylist 0 -1] "failed trim" 34 | 35 | set starting [r llen mylist] 36 | for {set j [r llen mylist]} {$j < $startlen} {incr j} { 37 | set str [randomInt 9223372036854775807] 38 | r rpush mylist $str 39 | lappend mylist $str 40 | assert_equal $mylist [r lrange mylist 0 -1] "failed append match" 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/server/expire_thread.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // expire_thread.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/27. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "expire_thread.h" 10 | #include "server.h" 11 | #include "encoding.h" 12 | #include "db_util.h" 13 | #include "simple_log.h" 14 | 15 | void ExpireThread::OnThreadRun() 16 | { 17 | log_message(kLogLevelInfo, "expire thread start\n"); 18 | m_running = true; 19 | while (m_running) { 20 | usleep(100000); 21 | 22 | uint64_t current_tick = get_tick_count(); 23 | for (int i = 0; i < g_server.db_num; i++) { 24 | ScanKeyGuard scan_key_guard(i); 25 | 26 | rocksdb::ColumnFamilyHandle* cf_handle = g_server.cf_handles_map[i]; 27 | rocksdb::Iterator* it = g_server.db->NewIterator(g_server.read_option, cf_handle); 28 | rocksdb::Slice prefix_key((char*)&KEY_TYPE_ZSET_SORT, 1); 29 | 30 | for (it->Seek(prefix_key); it->Valid(); it->Next()) { 31 | if (it->key()[0] != KEY_TYPE_TTL_SORT) { 32 | break; 33 | } 34 | 35 | uint64_t ttl; 36 | string key; 37 | if (DecodeKey::Decode(it->key().ToString(), KEY_TYPE_TTL_SORT, ttl, key) != kDecodeOK) { 38 | log_message(kLogLevelError, "db must be collapsed\n"); 39 | continue; 40 | } 41 | 42 | if (ttl <= current_tick) { 43 | uint8_t key_type = it->value()[0]; 44 | delete_key(i, key, ttl, key_type); 45 | } else { 46 | break; 47 | } 48 | } 49 | 50 | delete it; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tests/cluster/tests/00-base.tcl: -------------------------------------------------------------------------------- 1 | # Check the basic monitoring and failover capabilities. 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | if {$::simulate_error} { 6 | test "This test will fail" { 7 | fail "Simulated error" 8 | } 9 | } 10 | 11 | test "Different nodes have different IDs" { 12 | set ids {} 13 | set numnodes 0 14 | foreach_redis_id id { 15 | incr numnodes 16 | # Every node should just know itself. 17 | set nodeid [dict get [get_myself $id] id] 18 | assert {$nodeid ne {}} 19 | lappend ids $nodeid 20 | } 21 | set numids [llength [lsort -unique $ids]] 22 | assert {$numids == $numnodes} 23 | } 24 | 25 | test "It is possible to perform slot allocation" { 26 | cluster_allocate_slots 5 27 | } 28 | 29 | test "After the join, every node gets a different config epoch" { 30 | set trynum 60 31 | while {[incr trynum -1] != 0} { 32 | # We check that this condition is true for *all* the nodes. 33 | set ok 1 ; # Will be set to 0 every time a node is not ok. 34 | foreach_redis_id id { 35 | set epochs {} 36 | foreach n [get_cluster_nodes $id] { 37 | lappend epochs [dict get $n config_epoch] 38 | } 39 | if {[lsort $epochs] != [lsort -unique $epochs]} { 40 | set ok 0 ; # At least one collision! 41 | } 42 | } 43 | if {$ok} break 44 | after 1000 45 | puts -nonewline . 46 | flush stdout 47 | } 48 | if {$trynum == 0} { 49 | fail "Config epoch conflict resolution is not working." 50 | } 51 | } 52 | 53 | test "Nodes should report cluster_state is ok now" { 54 | assert_cluster_state ok 55 | } 56 | 57 | test "It is possible to write and read from the cluster" { 58 | cluster_write_test 0 59 | } 60 | -------------------------------------------------------------------------------- /src/server/cmd_string.h: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_string.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/20. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CMD_STRING_H__ 10 | #define __CMD_STRING_H__ 11 | 12 | #include "server.h" 13 | 14 | void append_command(ClientConn* conn, const vector& cmd_vec); 15 | void getrange_command(ClientConn* conn, const vector& cmd_vec); 16 | void setrange_command(ClientConn* conn, const vector& cmd_vec); 17 | void strlen_command(ClientConn* conn, const vector& cmd_vec); 18 | 19 | void get_command(ClientConn* conn, const vector& cmd_vec); 20 | void getset_command(ClientConn* conn, const vector& cmd_vec); 21 | void set_command(ClientConn* conn, const vector& cmd_vec); 22 | void setex_command(ClientConn* conn, const vector& cmd_vec); 23 | void setnx_command(ClientConn* conn, const vector& cmd_vec); 24 | void psetex_command(ClientConn* conn, const vector& cmd_vec); 25 | 26 | void mset_command(ClientConn* conn, const vector& cmd_vec); 27 | void msetnx_command(ClientConn* conn, const vector& cmd_vec); 28 | void mget_command(ClientConn* conn, const vector& cmd_vec); 29 | 30 | void incr_command(ClientConn* conn, const vector& cmd_vec); 31 | void decr_command(ClientConn* conn, const vector& cmd_vec); 32 | void incrby_command(ClientConn* conn, const vector& cmd_vec); 33 | void decrby_command(ClientConn* conn, const vector& cmd_vec); 34 | void incrbyfloat_command(ClientConn* conn, const vector& cmd_vec); 35 | 36 | void setbit_command(ClientConn* conn, const vector& cmd_vec); 37 | void getbit_command(ClientConn* conn, const vector& cmd_vec); 38 | void bitcount_command(ClientConn* conn, const vector& cmd_vec); 39 | void bitpos_command(ClientConn* conn, const vector& cmd_vec); 40 | 41 | #endif /* __CMD_STRING_H__ */ 42 | -------------------------------------------------------------------------------- /src/base/io_thread_resource.h: -------------------------------------------------------------------------------- 1 | /* 2 | * io_thread_resource.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_IO_THREAD_RESOURCE_H__ 9 | #define __BASE_IO_THREAD_RESOURCE_H__ 10 | 11 | #include 12 | 13 | // io thread resource management, EventLoop, PendingPktMgr, all will use this 14 | template 15 | class IoThreadResource { 16 | public: 17 | IoThreadResource(); 18 | ~IoThreadResource(); 19 | 20 | void Init(int io_thread_num); 21 | int GetThreadNum() { return io_thread_num_; } 22 | T* GetMainResource() { return main_thread_resource_; } 23 | T* GetIOResource(int handle); 24 | bool IsInited() { return inited_; } 25 | private: 26 | T* main_thread_resource_; 27 | T* io_thread_resources_; 28 | int io_thread_num_; 29 | bool inited_; 30 | }; 31 | 32 | 33 | template 34 | IoThreadResource::IoThreadResource() 35 | { 36 | inited_ = false; 37 | io_thread_num_ = 0; 38 | main_thread_resource_ = NULL; 39 | io_thread_resources_ = NULL; 40 | } 41 | 42 | template 43 | IoThreadResource::~IoThreadResource() 44 | { 45 | if (main_thread_resource_) { 46 | delete main_thread_resource_; 47 | } 48 | 49 | if (io_thread_resources_) { 50 | delete [] io_thread_resources_; 51 | } 52 | } 53 | 54 | template 55 | void IoThreadResource::Init(int io_thread_num) 56 | { 57 | io_thread_num_ = io_thread_num; 58 | main_thread_resource_ = new T; 59 | assert(main_thread_resource_); 60 | if (io_thread_num_ > 0) { 61 | io_thread_resources_ = new T[io_thread_num_]; 62 | } 63 | 64 | inited_ = true; 65 | } 66 | 67 | template 68 | T* IoThreadResource::GetIOResource(int handle) 69 | { 70 | if (io_thread_num_ > 0) { 71 | return &io_thread_resources_[handle % io_thread_num_]; 72 | } else { 73 | return main_thread_resource_; 74 | } 75 | } 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /doc/support_command.md: -------------------------------------------------------------------------------- 1 | ## DB 2 | * AUTH 3 | * PING 4 | * ECHO 5 | * SELECT 6 | * DBSIZE 7 | * INFO 8 | * FLUSHDB 9 | * FLUSHALL 10 | * DEBUG 11 | * COMMAND 12 | * CONFIG 13 | * SLOWLOG 14 | 15 | ## Replication 16 | * REPLCONF 17 | * PSYNC 18 | * SLAVEOF 19 | * REPL_SNAPSHOT_COMPLETE (inner command) 20 | 21 | ## Migrate 22 | * MIGRATE 23 | * RESTORE 24 | * DUMP 25 | 26 | ## Keys 27 | * DEL 28 | * EXISTS 29 | * TTL 30 | * PTTL 31 | * TYPE 32 | * EXPIRE 33 | * EXPIREAT 34 | * PEXPIRE 35 | * PEXPIREAT 36 | * PERSIST 37 | * RANDOMKEY 38 | * KEYS 39 | * SCAN 40 | 41 | ## String 42 | * GET 43 | * GETSET 44 | * SET 45 | * SETEX 46 | * SETNX 47 | * PSETEX 48 | * MSET 49 | * MSETNX 50 | * MGET 51 | * APPEND 52 | * GETRANGE 53 | * SETRANGE 54 | * STRLEN 55 | * INCR 56 | * DECR 57 | * INCRBY 58 | * DECRBY 59 | * INCRBYFLOAT 60 | * SETBIT 61 | * GETBIT 62 | * BITCOUNT 63 | * BITPOS 64 | 65 | ## Hash 66 | * HDEL 67 | * HEXISTS 68 | * HGET 69 | * HGETALL 70 | * HKEYS 71 | * HVALS 72 | * HINCRBY 73 | * HINCRBYFLOAT 74 | * HLEN 75 | * HMGET 76 | * HMSET 77 | * HSCAN 78 | * HSET 79 | * HSETNX 80 | * HSTRLEN 81 | 82 | ## List 83 | * LINDEX 84 | * LINSERT 85 | * LLEN 86 | * LPUSH 87 | * LPOP 88 | * LRANGE 89 | * RPUSH 90 | * RPOP 91 | * LPUSHX 92 | * RPUSHX 93 | * LREM 94 | * LSET 95 | * LTRIM 96 | 97 | ## Set 98 | * SADD 99 | * SCARD 100 | * SISMEMBER 101 | * SMEMBERS 102 | * SPOP 103 | * SRANDMEMBER 104 | * SREM 105 | * SSCAN 106 | 107 | ## Sorted Set 108 | * ZADD 109 | * ZINCRBY 110 | * ZCARD 111 | * ZCOUNT 112 | * ZRANGE 113 | * ZREVRANGE 114 | * ZRANGEBYSCORE 115 | * ZREVRANGEBYSCORE 116 | * ZRANGEBYLEX 117 | * ZREVRANGEBYLEX 118 | * ZLEXCOUNT 119 | * ZRANK 120 | * ZREVRANK 121 | * ZREM 122 | * ZREMRANGEBYLEX 123 | * ZREMRANGEBYRANK 124 | * ZREMRANGEBYSCORE 125 | * ZSCORE 126 | * ZSCAN 127 | 128 | ## Hyperloglog 129 | * PFADD 130 | * PFCOUNT 131 | * PFMERGE 132 | -------------------------------------------------------------------------------- /src/base/simple_buffer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * simple_buffer.cpp 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #include "simple_buffer.h" 9 | #include 10 | #include 11 | 12 | SimpleBuffer::SimpleBuffer() 13 | { 14 | m_buffer = NULL; 15 | 16 | m_alloc_size = 0; 17 | m_write_offset = 0; 18 | m_read_offset = 0; 19 | } 20 | 21 | SimpleBuffer::~SimpleBuffer() 22 | { 23 | if (m_buffer) { 24 | free(m_buffer); 25 | m_buffer = NULL; 26 | } 27 | 28 | m_alloc_size = 0; 29 | m_write_offset = 0; 30 | m_read_offset = 0; 31 | } 32 | 33 | void SimpleBuffer::Extend(uint32_t len) 34 | { 35 | m_alloc_size = m_write_offset + len; 36 | m_alloc_size += m_alloc_size >> 2; // increase by 1/4 allocate size 37 | uchar_t* new_buf = (uchar_t*)realloc(m_buffer, m_alloc_size); 38 | m_buffer = new_buf; 39 | } 40 | 41 | uint32_t SimpleBuffer::Write(void* buf, uint32_t len) 42 | { 43 | if (m_write_offset + len > m_alloc_size) { 44 | Extend(len); 45 | } 46 | 47 | if (buf) { 48 | memcpy(m_buffer + m_write_offset, buf, len); 49 | } 50 | 51 | m_write_offset += len; 52 | 53 | return len; 54 | } 55 | 56 | uint32_t SimpleBuffer::Read(void* buf, uint32_t len) 57 | { 58 | if (len > GetReadableLen()) 59 | len = GetReadableLen(); 60 | 61 | if (buf) 62 | memcpy(buf, m_buffer + m_read_offset, len); 63 | 64 | m_read_offset += len; 65 | return len; 66 | } 67 | 68 | // move the rest of data to the start position of the buffer, reset read_offset and write_offset 69 | void SimpleBuffer::ResetOffset() 70 | { 71 | if (m_read_offset == 0) 72 | return; 73 | 74 | if (m_read_offset < m_write_offset) 75 | memmove(m_buffer, m_buffer + m_read_offset, GetReadableLen()); 76 | 77 | m_write_offset = GetReadableLen(); 78 | m_read_offset = 0; 79 | } 80 | 81 | void SimpleBuffer::Clear() 82 | { 83 | m_write_offset = m_read_offset = 0; 84 | m_alloc_size = 0; 85 | if (m_buffer) { 86 | free(m_buffer); 87 | m_buffer = NULL; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /tests/cluster/tests/12-replica-migration-2.tcl: -------------------------------------------------------------------------------- 1 | # Replica migration test #2. 2 | # 3 | # Check that the status of master that can be targeted by replica migration 4 | # is acquired again, after being getting slots again, in a cluster where the 5 | # other masters have slaves. 6 | 7 | source "../tests/includes/init-tests.tcl" 8 | 9 | # Create a cluster with 5 master and 15 slaves, to make sure there are no 10 | # empty masters and make rebalancing simpler to handle during the test. 11 | test "Create a 5 nodes cluster" { 12 | create_cluster 5 15 13 | } 14 | 15 | test "Cluster is up" { 16 | assert_cluster_state ok 17 | } 18 | 19 | test "Each master should have at least two replicas attached" { 20 | foreach_redis_id id { 21 | if {$id < 5} { 22 | wait_for_condition 1000 50 { 23 | [llength [lindex [R 0 role] 2]] >= 2 24 | } else { 25 | fail "Master #$id does not have 2 slaves as expected" 26 | } 27 | } 28 | } 29 | } 30 | 31 | set master0_id [dict get [get_myself 0] id] 32 | test "Resharding all the master #0 slots away from it" { 33 | set output [exec \ 34 | ../../../src/redis-trib.rb rebalance \ 35 | --weight ${master0_id}=0 \ 36 | 127.0.0.1:[get_instance_attrib redis 0 port] >@ stdout] 37 | } 38 | 39 | test "Master #0 should lose its replicas" { 40 | wait_for_condition 1000 50 { 41 | [llength [lindex [R 0 role] 2]] == 0 42 | } else { 43 | fail "Master #0 still has replicas" 44 | } 45 | } 46 | 47 | test "Resharding back some slot to master #0" { 48 | # Wait for the cluster config to propagate before attempting a 49 | # new resharding. 50 | after 10000 51 | set output [exec \ 52 | ../../../src/redis-trib.rb rebalance \ 53 | --weight ${master0_id}=.01 \ 54 | --use-empty-masters \ 55 | 127.0.0.1:[get_instance_attrib redis 0 port] >@ stdout] 56 | } 57 | 58 | test "Master #0 should re-acquire one or more replicas" { 59 | wait_for_condition 1000 50 { 60 | [llength [lindex [R 0 role] 2]] >= 1 61 | } else { 62 | fail "Master #0 has no has replicas" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/server/migrate_conn.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // migrate_conn.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/19. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "migrate_conn.h" 10 | #include "migrate.h" 11 | #include "simple_log.h" 12 | 13 | MigrateConn::MigrateConn() 14 | { 15 | 16 | } 17 | 18 | MigrateConn::~MigrateConn() 19 | { 20 | 21 | } 22 | 23 | void MigrateConn::Close() 24 | { 25 | BaseConn::Close(); 26 | migrate_conn_closed(addr_); 27 | } 28 | 29 | void MigrateConn::OnConfirm() 30 | { 31 | BaseConn::OnConfirm(); 32 | migrate_conn_established(addr_); 33 | } 34 | 35 | void MigrateConn::OnRead() 36 | { 37 | _RecvData(); 38 | 39 | // when parse redis protocol, this makes sure searching string pattern will not pass the boundary 40 | m_in_buf.GetWriteBuffer()[0] = 0; 41 | 42 | while (true) { 43 | RedisReply reply; 44 | int ret = parse_redis_response((const char*)m_in_buf.GetReadBuffer(), m_in_buf.GetReadableLen(), reply); 45 | if (ret > 0) { 46 | if (reply.GetType() == REDIS_TYPE_ERROR) { 47 | log_message(kLogLevelError, "migrate error: %s\n", reply.GetStrValue().c_str()); 48 | return; 49 | } 50 | 51 | continue_migrate_command(addr_, reply); 52 | m_in_buf.Read(NULL, ret); 53 | } else if (ret < 0) { 54 | log_message(kLogLevelError, "parse redis protocol error in migrate connection\n"); 55 | Close(); 56 | return; 57 | } else { 58 | // not yet receive a whole command 59 | m_in_buf.ResetOffset(); 60 | break; 61 | } 62 | } 63 | } 64 | 65 | void MigrateConn::OnTimer(uint64_t curr_tick) 66 | { 67 | if (!IsOpen()) { 68 | if (curr_tick >= m_last_send_tick + 1000) { 69 | // if connection has not establish after 1 second, close the connection 70 | Close(); 71 | } 72 | } else { 73 | if (curr_tick >= m_last_send_tick + 15000) { 74 | // if connection idled 15 seconds, close the connection 75 | Close(); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /tests/unit/introspection.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"introspection"}} { 2 | test {CLIENT LIST} { 3 | r client list 4 | } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} 5 | 6 | test {MONITOR can log executed commands} { 7 | set rd [redis_deferring_client] 8 | $rd monitor 9 | r set foo bar 10 | r get foo 11 | list [$rd read] [$rd read] [$rd read] 12 | } {*OK*"set" "foo"*"get" "foo"*} 13 | 14 | test {MONITOR can log commands issued by the scripting engine} { 15 | set rd [redis_deferring_client] 16 | $rd monitor 17 | r eval {redis.call('set',KEYS[1],ARGV[1])} 1 foo bar 18 | $rd read ;# Discard the OK 19 | assert_match {*eval*} [$rd read] 20 | assert_match {*lua*"set"*"foo"*"bar"*} [$rd read] 21 | } 22 | 23 | test {CLIENT GETNAME should return NIL if name is not assigned} { 24 | r client getname 25 | } {} 26 | 27 | test {CLIENT LIST shows empty fields for unassigned names} { 28 | r client list 29 | } {*name= *} 30 | 31 | test {CLIENT SETNAME does not accept spaces} { 32 | catch {r client setname "foo bar"} e 33 | set e 34 | } {ERR*} 35 | 36 | test {CLIENT SETNAME can assign a name to this connection} { 37 | assert_equal [r client setname myname] {OK} 38 | r client list 39 | } {*name=myname*} 40 | 41 | test {CLIENT SETNAME can change the name of an existing connection} { 42 | assert_equal [r client setname someothername] {OK} 43 | r client list 44 | } {*name=someothername*} 45 | 46 | test {After CLIENT SETNAME, connection can still be closed} { 47 | set rd [redis_deferring_client] 48 | $rd client setname foobar 49 | assert_equal [$rd read] "OK" 50 | assert_match {*foobar*} [r client list] 51 | $rd close 52 | # Now the client should no longer be listed 53 | wait_for_condition 50 100 { 54 | [string match {*foobar*} [r client list]] == 0 55 | } else { 56 | fail "Client still listed in CLIENT LIST after SETNAME." 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /tools/hiredis/sdsalloc.h: -------------------------------------------------------------------------------- 1 | /* SDSLib 2.0 -- A C dynamic strings library 2 | * 3 | * Copyright (c) 2006-2015, Salvatore Sanfilippo 4 | * Copyright (c) 2015, Oran Agra 5 | * Copyright (c) 2015, Redis Labs, Inc 6 | * All rights reserved. 7 | * 8 | * Redistribution and use in source and binary forms, with or without 9 | * modification, are permitted provided that the following conditions are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright notice, 12 | * this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * * Neither the name of Redis nor the names of its contributors may be used 17 | * to endorse or promote products derived from this software without 18 | * specific prior written permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | * POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | /* SDS allocator selection. 34 | * 35 | * This file is used in order to change the SDS allocator at compile time. 36 | * Just define the following defines to what you want to use. Also add 37 | * the include of your alternate allocator if needed (not needed in order 38 | * to use the default libc allocator). */ 39 | 40 | #define s_malloc malloc 41 | #define s_realloc realloc 42 | #define s_free free 43 | -------------------------------------------------------------------------------- /tests/sentinel/tests/07-down-conditions.tcl: -------------------------------------------------------------------------------- 1 | # Test conditions where an instance is considered to be down 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | proc ensure_master_up {} { 6 | wait_for_condition 1000 50 { 7 | [dict get [S 4 sentinel master mymaster] flags] eq "master" 8 | } else { 9 | fail "Master flags are not just 'master'" 10 | } 11 | } 12 | 13 | proc ensure_master_down {} { 14 | wait_for_condition 1000 50 { 15 | [string match *down* \ 16 | [dict get [S 4 sentinel master mymaster] flags]] 17 | } else { 18 | fail "Master is not flagged SDOWN" 19 | } 20 | } 21 | 22 | test "Crash the majority of Sentinels to prevent failovers for this unit" { 23 | for {set id 0} {$id < $quorum} {incr id} { 24 | kill_instance sentinel $id 25 | } 26 | } 27 | 28 | test "SDOWN is triggered by non-responding but not crashed instance" { 29 | lassign [S 4 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port 30 | ensure_master_up 31 | exec ../../../src/redis-cli -h $host -p $port debug sleep 10 > /dev/null & 32 | ensure_master_down 33 | ensure_master_up 34 | } 35 | 36 | test "SDOWN is triggered by crashed instance" { 37 | lassign [S 4 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port 38 | ensure_master_up 39 | kill_instance redis 0 40 | ensure_master_down 41 | restart_instance redis 0 42 | ensure_master_up 43 | } 44 | 45 | test "SDOWN is triggered by masters advertising as slaves" { 46 | ensure_master_up 47 | R 0 slaveof 127.0.0.1 34567 48 | ensure_master_down 49 | R 0 slaveof no one 50 | ensure_master_up 51 | } 52 | 53 | test "SDOWN is triggered by misconfigured instance repling with errors" { 54 | ensure_master_up 55 | set orig_dir [lindex [R 0 config get dir] 1] 56 | set orig_save [lindex [R 0 config get save] 1] 57 | # Set dir to / and filename to "tmp" to make sure it will fail. 58 | R 0 config set dir / 59 | R 0 config set dbfilename tmp 60 | R 0 config set save "1000000 1000000" 61 | R 0 bgsave 62 | ensure_master_down 63 | R 0 config set save $orig_save 64 | R 0 config set dir $orig_dir 65 | R 0 config set dbfilename dump.rdb 66 | R 0 bgsave 67 | ensure_master_up 68 | } 69 | -------------------------------------------------------------------------------- /tests/cluster/tests/includes/init-tests.tcl: -------------------------------------------------------------------------------- 1 | # Initialization tests -- most units will start including this. 2 | 3 | test "(init) Restart killed instances" { 4 | foreach type {redis} { 5 | foreach_${type}_id id { 6 | if {[get_instance_attrib $type $id pid] == -1} { 7 | puts -nonewline "$type/$id " 8 | flush stdout 9 | restart_instance $type $id 10 | } 11 | } 12 | } 13 | } 14 | 15 | test "Cluster nodes are reachable" { 16 | foreach_redis_id id { 17 | # Every node should be reachable. 18 | wait_for_condition 1000 50 { 19 | ([catch {R $id ping} ping_reply] == 0) && 20 | ($ping_reply eq {PONG}) 21 | } else { 22 | catch {R $id ping} err 23 | fail "Node #$id keeps replying '$err' to PING." 24 | } 25 | } 26 | } 27 | 28 | test "Cluster nodes hard reset" { 29 | foreach_redis_id id { 30 | if {$::valgrind} { 31 | set node_timeout 10000 32 | } else { 33 | set node_timeout 3000 34 | } 35 | catch {R $id flushall} ; # May fail for readonly slaves. 36 | R $id MULTI 37 | R $id cluster reset hard 38 | R $id cluster set-config-epoch [expr {$id+1}] 39 | R $id EXEC 40 | R $id config set cluster-node-timeout $node_timeout 41 | R $id config set cluster-slave-validity-factor 10 42 | R $id config rewrite 43 | } 44 | } 45 | 46 | test "Cluster Join and auto-discovery test" { 47 | # Join node 0 with 1, 1 with 2, ... and so forth. 48 | # If auto-discovery works all nodes will know every other node 49 | # eventually. 50 | set ids {} 51 | foreach_redis_id id {lappend ids $id} 52 | for {set j 0} {$j < [expr [llength $ids]-1]} {incr j} { 53 | set a [lindex $ids $j] 54 | set b [lindex $ids [expr $j+1]] 55 | set b_port [get_instance_attrib redis $b port] 56 | R $a cluster meet 127.0.0.1 $b_port 57 | } 58 | 59 | foreach_redis_id id { 60 | wait_for_condition 1000 50 { 61 | [llength [get_cluster_nodes $id]] == [llength $ids] 62 | } else { 63 | fail "Cluster failed to join into a full mesh." 64 | } 65 | } 66 | } 67 | 68 | test "Before slots allocation, all nodes report cluster failure" { 69 | assert_cluster_state fail 70 | } 71 | -------------------------------------------------------------------------------- /tests/cluster/tests/06-slave-stop-cond.tcl: -------------------------------------------------------------------------------- 1 | # Slave stop condition test 2 | # Check that if there is a disconnection time limit, the slave will not try 3 | # to failover its master. 4 | 5 | source "../tests/includes/init-tests.tcl" 6 | 7 | # Create a cluster with 5 master and 5 slaves. 8 | test "Create a 5 nodes cluster" { 9 | create_cluster 5 5 10 | } 11 | 12 | test "Cluster is up" { 13 | assert_cluster_state ok 14 | } 15 | 16 | test "The first master has actually one slave" { 17 | assert {[llength [lindex [R 0 role] 2]] == 1} 18 | } 19 | 20 | test {Slaves of #0 is instance #5 as expected} { 21 | set port0 [get_instance_attrib redis 0 port] 22 | assert {[lindex [R 5 role] 2] == $port0} 23 | } 24 | 25 | test "Instance #5 synced with the master" { 26 | wait_for_condition 1000 50 { 27 | [RI 5 master_link_status] eq {up} 28 | } else { 29 | fail "Instance #5 master link status is not up" 30 | } 31 | } 32 | 33 | test "Lower the slave validity factor of #5 to the value of 2" { 34 | assert {[R 5 config set cluster-slave-validity-factor 2] eq {OK}} 35 | } 36 | 37 | test "Break master-slave link and prevent further reconnections" { 38 | # Stop the slave with a multi/exec transaction so that the master will 39 | # be killed as soon as it can accept writes again. 40 | R 5 multi 41 | R 5 client kill 127.0.0.1:$port0 42 | # here we should sleep 6 or more seconds (node_timeout * slave_validity) 43 | # but the actual validity time is actually incremented by the 44 | # repl-ping-slave-period value which is 10 seconds by default. So we 45 | # need to wait more than 16 seconds. 46 | R 5 debug sleep 20 47 | R 5 deferred 1 48 | R 5 exec 49 | 50 | # Prevent the master from accepting new slaves. 51 | # Use a large pause value since we'll kill it anyway. 52 | R 0 CLIENT PAUSE 60000 53 | 54 | # Wait for the slave to return available again 55 | R 5 deferred 0 56 | assert {[R 5 read] eq {OK OK}} 57 | 58 | # Kill the master so that a reconnection will not be possible. 59 | kill_instance redis 0 60 | } 61 | 62 | test "Slave #5 is reachable and alive" { 63 | assert {[R 5 ping] eq {PONG}} 64 | } 65 | 66 | test "Slave #5 should not be able to failover" { 67 | after 10000 68 | assert {[RI 5 role] eq {slave}} 69 | } 70 | 71 | test "Cluster should be down" { 72 | assert_cluster_state fail 73 | } 74 | -------------------------------------------------------------------------------- /tests/cluster/tests/08-update-msg.tcl: -------------------------------------------------------------------------------- 1 | # Test UPDATE messages sent by other nodes when the currently authorirative 2 | # master is unavaialble. The test is performed in the following steps: 3 | # 4 | # 1) Master goes down. 5 | # 2) Slave failover and becomes new master. 6 | # 3) New master is partitoned away. 7 | # 4) Old master returns. 8 | # 5) At this point we expect the old master to turn into a slave ASAP because 9 | # of the UPDATE messages it will receive from the other nodes when its 10 | # configuration will be found to be outdated. 11 | 12 | source "../tests/includes/init-tests.tcl" 13 | 14 | test "Create a 5 nodes cluster" { 15 | create_cluster 5 5 16 | } 17 | 18 | test "Cluster is up" { 19 | assert_cluster_state ok 20 | } 21 | 22 | test "Cluster is writable" { 23 | cluster_write_test 0 24 | } 25 | 26 | test "Instance #5 is a slave" { 27 | assert {[RI 5 role] eq {slave}} 28 | } 29 | 30 | test "Instance #5 synced with the master" { 31 | wait_for_condition 1000 50 { 32 | [RI 5 master_link_status] eq {up} 33 | } else { 34 | fail "Instance #5 master link status is not up" 35 | } 36 | } 37 | 38 | set current_epoch [CI 1 cluster_current_epoch] 39 | 40 | test "Killing one master node" { 41 | kill_instance redis 0 42 | } 43 | 44 | test "Wait for failover" { 45 | wait_for_condition 1000 50 { 46 | [CI 1 cluster_current_epoch] > $current_epoch 47 | } else { 48 | fail "No failover detected" 49 | } 50 | } 51 | 52 | test "Cluster should eventually be up again" { 53 | assert_cluster_state ok 54 | } 55 | 56 | test "Cluster is writable" { 57 | cluster_write_test 1 58 | } 59 | 60 | test "Instance #5 is now a master" { 61 | assert {[RI 5 role] eq {master}} 62 | } 63 | 64 | test "Killing the new master #5" { 65 | kill_instance redis 5 66 | } 67 | 68 | test "Cluster should be down now" { 69 | assert_cluster_state fail 70 | } 71 | 72 | test "Restarting the old master node" { 73 | restart_instance redis 0 74 | } 75 | 76 | test "Instance #0 gets converted into a slave" { 77 | wait_for_condition 1000 50 { 78 | [RI 0 role] eq {slave} 79 | } else { 80 | fail "Old master was not converted into slave" 81 | } 82 | } 83 | 84 | test "Restarting the new master node" { 85 | restart_instance redis 5 86 | } 87 | 88 | test "Cluster is up again" { 89 | assert_cluster_state ok 90 | } 91 | -------------------------------------------------------------------------------- /src/server/redis_parser.h: -------------------------------------------------------------------------------- 1 | // 2 | // redis_parser.h 3 | // kv-store 4 | // 5 | // Created by ziteng on 16-5-18. 6 | // Copyright (c) 2016年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __REDIS_PARSER_H__ 10 | #define __REDIS_PARSER_H__ 11 | 12 | #include "util.h" 13 | 14 | const string kNullBulkString = "$-1\r\n"; 15 | const string kEmptyBulkString = "$0\r\n\r\n"; 16 | const string kOKString = "+OK\r\n"; 17 | const string kWrongTypeError = "-WRONGTYPE Operation against a key holding the wrong kind of value\r\n"; 18 | const string kIoErrorString = "-IOERROR\r\n"; 19 | const string kNoKeyString = "-NOKEY\r\n"; 20 | 21 | enum { 22 | REDIS_TYPE_STRING = 1, 23 | REDIS_TYPE_ARRAY = 2, 24 | REDIS_TYPE_INTEGER = 3, 25 | REDIS_TYPE_NIL = 4, 26 | REDIS_TYPE_STATUS = 5, 27 | REDIS_TYPE_ERROR = 6, 28 | }; 29 | 30 | class RedisReply { 31 | public: 32 | RedisReply(): type_(REDIS_TYPE_NIL), int_value_(0) {} 33 | RedisReply(int type, long value): type_(type), int_value_(value) {} 34 | RedisReply(int type, string& value): type_(type), int_value_(0), str_value_(value) {} 35 | RedisReply(int type): type_(type), int_value_(0) {} 36 | virtual ~RedisReply() {} 37 | 38 | void AddRedisReply(const RedisReply& element) { elements_.push_back(element); } 39 | 40 | int GetType() const { return type_; } 41 | long GetIntValue() const { return int_value_; } 42 | const string& GetStrValue() const { return str_value_; } 43 | const vector& GetElements() const { return elements_; } 44 | private: 45 | int type_; 46 | long int_value_; // The integer when type is INTEGER 47 | string str_value_; // value for STATUS, ERROR, STRING 48 | vector elements_; // elements for ARRAY 49 | }; 50 | 51 | /* 52 | * 返回值: 53 | * -1 -- 解析失败，redis格式出错 54 | * 0 -- 没有接收到完整的数据包 55 | * >0 -- 解析成功，返回redis请求的长度 56 | */ 57 | int parse_redis_request(const char* redis_cmd, int redis_len, vector& cmd_vec, string& err_msg); 58 | 59 | /* 60 | * 返回值: 61 | * -1 -- 解析失败，redis格式出错 62 | * 0 -- 没有接收到完整的数据包 63 | * >0 -- 解析成功，返回redis请求的长度 64 | */ 65 | int parse_redis_response(const char* redis_resp, int redis_len, RedisReply& reply); 66 | 67 | int build_prefix(char* buf, int len, char start_char, int size); 68 | 69 | void build_request(const vector& cmd_vec, string& request); 70 | 71 | void build_response(const vector& cmd_vec, string& response); 72 | 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /tools/kedis_to_redis/sync_task.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // sync_task.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "sync_task.h" 10 | #include "redis_conn.h" 11 | #include "simple_log.h" 12 | #include "kedis_to_redis.h" 13 | #include "event_loop.h" 14 | #include "redis_parser.h" 15 | 16 | ThreadPool g_thread_pool; 17 | RedisConn g_redis_conn; 18 | time_t g_last_cmd_time; 19 | 20 | static void ping_timer_callback(void* callback_data, uint8_t msg, uint32_t handle, void* pParam) 21 | { 22 | time_t current_time = time(NULL); 23 | if (current_time > g_last_cmd_time + 60) { 24 | // start a ping command if idled for 1 minutes, 25 | // so the redis connection will not be close by the server for idled too long 26 | KeepalivePingTask* task = new KeepalivePingTask(); 27 | g_thread_pool.AddTask(task); 28 | } 29 | } 30 | 31 | void init_sync_task() 32 | { 33 | g_thread_pool.Init(1); // only one thread in the pool，so tasks will be executed in order 34 | 35 | g_redis_conn.SetAddr(g_config.dst_redis_host, g_config.dst_redis_port); 36 | g_redis_conn.SetPassword(g_config.dst_redis_password); 37 | g_redis_conn.Init(); 38 | if (g_config.dst_redis_db != -1) { 39 | string request; 40 | vector cmd_vec = {"SELECT", to_string(g_config.dst_redis_db)}; 41 | build_request(cmd_vec, request); 42 | redisReply* reply = g_redis_conn.DoRawCmd(request); 43 | if ((reply == NULL) || (reply->type == REDIS_REPLY_ERROR)) { 44 | log_message(kLogLevelError, "RedisConn DoRawCmd failed: %s\n", request.c_str()); 45 | exit(1); 46 | } 47 | } 48 | 49 | g_last_cmd_time = time(NULL); 50 | get_main_event_loop()->AddTimer(ping_timer_callback, NULL, 5000); 51 | } 52 | 53 | void SyncCmdTask::run() 54 | { 55 | redisReply* reply = g_redis_conn.DoRawCmd(raw_cmd_); 56 | if ((reply == NULL) || (reply->type == REDIS_REPLY_ERROR)) { 57 | log_message(kLogLevelInfo, "RedisConn DoRawCmd failed: %s, error:%s\n", raw_cmd_.c_str(), reply ? reply->str : ""); 58 | } 59 | 60 | g_last_cmd_time = time(NULL); 61 | int remain_task_cnt = g_thread_pool.GetTotalTaskCnt() - 1; 62 | if (remain_task_cnt > 0) { 63 | log_message(kLogLevelInfo, "SyncCmdTask, remain_cnt=%d\n", remain_task_cnt); 64 | } 65 | } 66 | 67 | void KeepalivePingTask::run() 68 | { 69 | g_redis_conn.DoCmd("PING"); 70 | g_last_cmd_time = time(NULL); 71 | } 72 | -------------------------------------------------------------------------------- /tools/kedis_port/sync_task.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // sync_task.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "sync_task.h" 10 | #include "redis_conn.h" 11 | #include "simple_log.h" 12 | #include "kedis_port.h" 13 | #include "event_loop.h" 14 | #include "redis_parser.h" 15 | 16 | ThreadPool g_thread_pool; 17 | RedisConn g_kedis_conn; 18 | time_t g_last_cmd_time; 19 | 20 | static void ping_timer_callback(void* callback_data, uint8_t msg, uint32_t handle, void* pParam) 21 | { 22 | time_t current_time = time(NULL); 23 | if (current_time > g_last_cmd_time + 60) { 24 | // start a ping command if idled for 1 minutes, 25 | // so the redis connection will not be close by the server for idled too long 26 | KeepalivePingTask* task = new KeepalivePingTask(); 27 | g_thread_pool.AddTask(task); 28 | } 29 | } 30 | 31 | void init_sync_task() 32 | { 33 | g_thread_pool.Init(1); // only one thread in the pool，so tasks will be executed in order 34 | 35 | g_kedis_conn.SetAddr(g_config.dst_kedis_host, g_config.dst_kedis_port); 36 | g_kedis_conn.SetPassword(g_config.dst_kedis_password); 37 | g_kedis_conn.Init(); 38 | 39 | if (g_config.dst_kedis_db != -1) { 40 | string request; 41 | vector cmd_vec = {"SELECT", to_string(g_config.dst_kedis_db)}; 42 | build_request(cmd_vec, request); 43 | redisReply* reply = g_kedis_conn.DoRawCmd(request); 44 | if ((reply == NULL) || (reply->type == REDIS_REPLY_ERROR)) { 45 | log_message(kLogLevelError, "RedisConn DoRawCmd failed: %s\n", request.c_str()); 46 | exit(1); 47 | } 48 | } 49 | 50 | g_last_cmd_time = time(NULL); 51 | get_main_event_loop()->AddTimer(ping_timer_callback, NULL, 5000); 52 | } 53 | 54 | void SyncCmdTask::run() 55 | { 56 | redisReply* reply = g_kedis_conn.DoRawCmd(raw_cmd_); 57 | if ((reply == NULL) || (reply->type == REDIS_REPLY_ERROR)) { 58 | log_message(kLogLevelInfo, "RedisConn DoRawCmd failed: %s, error:%s\n", raw_cmd_.c_str(), reply ? reply->str : ""); 59 | } 60 | 61 | g_last_cmd_time = time(NULL); 62 | int remain_task_cnt = g_thread_pool.GetTotalTaskCnt() - 1; 63 | if (remain_task_cnt > 0) { 64 | log_message(kLogLevelInfo, "SyncCmdTask, remain_cnt=%d\n", remain_task_cnt); 65 | } 66 | } 67 | 68 | void KeepalivePingTask::run() 69 | { 70 | g_kedis_conn.DoCmd("PING"); 71 | g_last_cmd_time = time(NULL); 72 | } 73 | -------------------------------------------------------------------------------- /src/server/db_util.h: -------------------------------------------------------------------------------- 1 | // 2 | // db_util.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/9/21. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __DB_UTIL_H__ 10 | #define __DB_UTIL_H__ 11 | 12 | #include "util.h" 13 | #include "key_lock.h" 14 | #include "simple_log.h" 15 | #include "client_conn.h" 16 | #include "rocksdb/db.h" 17 | #include "redis_parser.h" 18 | #include "server.h" 19 | 20 | #define FIELD_NOT_EXIST 0 21 | #define FIELD_EXIST 1 22 | #define DB_ERROR 2 23 | 24 | const int kExpireKeyNotExist = 0; 25 | const int kExpireKeyExist = 1; // also return key_type, ttl, count or value, this can save another Get method 26 | const int kExpireDBError = 2; 27 | 28 | struct MetaData { 29 | uint8_t type; 30 | uint64_t ttl; 31 | string value; 32 | uint64_t count; 33 | uint64_t head_seq; 34 | uint64_t tail_seq; 35 | uint64_t current_seq; 36 | }; 37 | 38 | void delete_key(int db_idx, const string& key, uint64_t ttl, uint8_t key_type); 39 | 40 | int expire_key_if_needed(int db_idx, const string& key, MetaData& mdata, string* raw_value = nullptr); 41 | 42 | rocksdb::Status put_meta_data(int db_idx, uint8_t key_type, const string& key, uint64_t ttl, uint64_t count, 43 | rocksdb::WriteBatch* batch = NULL); 44 | rocksdb::Status put_meta_data(int db_idx, uint8_t key_type, const string& key, uint64_t ttl, uint64_t count, 45 | uint64_t head_seq, uint64_t tail_seq, uint64_t current_seq, 46 | rocksdb::WriteBatch* batch = NULL); 47 | rocksdb::Status put_ttl_data(int db_idx, uint64_t ttl, const string& key, uint8_t key_type, rocksdb::WriteBatch* batch = NULL); 48 | rocksdb::Status del_ttl_data(int db_idx, uint64_t ttl, const string& key, rocksdb::WriteBatch* batch = NULL); 49 | void put_kv_data(int db_idx, const string& key, const string& value, uint64_t ttl, rocksdb::WriteBatch* batch = NULL); 50 | 51 | #define DB_BATCH_UPDATE(batch) \ 52 | rocksdb::Status s = g_server.db->Write(g_server.write_option, &batch); \ 53 | if (!s.ok()) { \ 54 | log_message(kLogLevelError, "write batch failed: %s\n", batch.Data().c_str()); \ 55 | } 56 | 57 | string double_to_string(double value); 58 | 59 | int parse_scan_param(ClientConn* conn, const vector& cmd_vec, int start_index, string& pattern, long& count); 60 | 61 | // Glob-style pattern matching, 1-match, 0-unmatch 62 | int stringmatchlen(const char *pattern, int patternLen, const char *str, int strLen, int nocase); 63 | 64 | #endif /* __DB_UTIL_H__ */ 65 | -------------------------------------------------------------------------------- /src/base/base_conn.h: -------------------------------------------------------------------------------- 1 | /* 2 | * base_conn.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_BASE_CONN_H_ 9 | #define __BASE_BASE_CONN_H_ 10 | 11 | #include "util.h" 12 | #include "simple_buffer.h" 13 | #include "base_socket.h" 14 | 15 | const int kHeartBeartInterval = 3000; 16 | const int kConnTimeout = 16000; 17 | 18 | const int kMaxSendSize = 128 * 1024; 19 | const int kReadBufSize = 2048; 20 | 21 | 22 | class BaseConn : public RefCount 23 | { 24 | public: 25 | BaseConn(); 26 | virtual ~BaseConn(); 27 | 28 | bool IsBusy() { return m_busy; } 29 | bool IsOpen() { return m_open; } 30 | void SetHeartbeatInterval(int interval) { m_heartbeat_interval = interval; } 31 | void SetConnTimerout(int timeout) { m_conn_timeout = timeout; } 32 | net_handle_t GetHandle() { return m_handle; } 33 | char* GetPeerIP() { return (char*)m_peer_ip.c_str(); } 34 | uint16_t GetPeerPort() { return m_peer_port; } 35 | 36 | virtual net_handle_t Connect(const string& server_ip, uint16_t server_port, int thread_index = -1); 37 | int Send(void* data, int len); 38 | virtual void Close(); 39 | 40 | virtual void OnConnect(BaseSocket* base_socket); 41 | virtual void OnConfirm(); 42 | virtual void OnRead(); 43 | virtual void OnWrite(); 44 | virtual void OnClose(); 45 | virtual void OnTimer(uint64_t curr_tick); 46 | virtual void OnLoop() {} // be called everytime before waiting for event 47 | 48 | static int Send(net_handle_t handle, void* data, int len); 49 | static int CloseHandle(net_handle_t handle); // used for other thread to close the connection 50 | 51 | static long GetTotalNetInputBytes() { return m_total_net_input_bytes; } 52 | static long GetTotalNetOutputBytes() { return m_total_net_output_bytes; } 53 | protected: 54 | void _RecvData(); 55 | protected: 56 | int m_thread_index; 57 | BaseSocket* m_base_socket; 58 | net_handle_t m_handle; 59 | bool m_busy; 60 | bool m_open; 61 | int m_heartbeat_interval; 62 | int m_conn_timeout; 63 | 64 | string m_peer_ip; 65 | uint16_t m_peer_port; 66 | SimpleBuffer m_in_buf; 67 | SimpleBuffer m_out_buf; 68 | 69 | uint64_t m_last_send_tick; 70 | uint64_t m_last_recv_tick; 71 | 72 | static atomic m_total_net_input_bytes; 73 | static atomic m_total_net_output_bytes; 74 | }; 75 | 76 | void init_thread_base_conn(int io_thread_num); 77 | void destroy_thread_base_conn(int io_thread_num); 78 | 79 | #endif /* __BASE_CONN_H_ */ 80 | -------------------------------------------------------------------------------- /src/base/event_loop.h: -------------------------------------------------------------------------------- 1 | /* 2 | * event_loop.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_EVENT_LOOP_H__ 9 | #define __BASE_EVENT_LOOP_H__ 10 | 11 | #include "ostype.h" 12 | #include "util.h" 13 | 14 | enum { 15 | SOCKET_READ = 0x1, 16 | SOCKET_WRITE = 0x2, 17 | SOCKET_EXCEP = 0x4, 18 | SOCKET_ALL = 0x7, 19 | SOCKET_CONNECT_CB = 0x8, 20 | SOCKET_ADD_CONN = 0x10, 21 | SOCKET_DEL_CONN = 0x20, 22 | }; 23 | 24 | class BaseSocket; 25 | typedef unordered_map SocketMap; 26 | 27 | class EventLoop 28 | { 29 | public: 30 | EventLoop(); 31 | virtual ~EventLoop(); 32 | 33 | void AddEvent(int fd, uint8_t socket_event, BaseSocket* pSocket); 34 | void RemoveEvent(int fd, uint8_t socket_event, BaseSocket* pSocket); 35 | 36 | void AddTimer(callback_t callback, void* user_data, uint64_t interval); 37 | void RemoveTimer(callback_t callback, void* user_data); 38 | 39 | void AddLoop(callback_t callback, void* user_data); 40 | 41 | void Start(uint32_t wait_timeout = 100); 42 | 43 | void Stop() { stop_ = true; Wakeup(); } // call by another thread 44 | void Wakeup(); 45 | 46 | void SetThreadId(pthread_t tid) { thread_id_ = tid; } 47 | bool IsInLoopThread() { return pthread_self() == thread_id_; } 48 | pthread_t GetThreadId() { return thread_id_; } 49 | 50 | void OnTimer(); 51 | 52 | BaseSocket* FindBaseSocket(net_handle_t handle); 53 | private: 54 | void _CheckTimer(); 55 | void _CheckLoop(); 56 | void _ReadWakeupData(); 57 | void _RegisterEventList(); 58 | 59 | typedef struct { 60 | callback_t callback; 61 | void* user_data; 62 | uint64_t interval; 63 | uint64_t next_tick; 64 | } TimerItem; 65 | 66 | typedef struct { 67 | int fd; 68 | uint8_t socket_event; 69 | BaseSocket* base_socket; 70 | } RegisterEvent; 71 | private: 72 | pthread_t thread_id_; 73 | int event_fd_; 74 | bool stop_; 75 | int wakeup_fds_[2]; 76 | list timer_list_; 77 | list loop_list_; 78 | SocketMap socket_map_; 79 | 80 | mutex mutex_; 81 | list register_event_list_; 82 | }; 83 | 84 | void init_thread_event_loops(int io_thread_num); 85 | void destroy_thread_event_loops(int io_thread_num); 86 | 87 | EventLoop* get_main_event_loop(); 88 | EventLoop* get_io_event_loop(net_handle_t handle); 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /tests/sentinel/tests/includes/init-tests.tcl: -------------------------------------------------------------------------------- 1 | # Initialization tests -- most units will start including this. 2 | 3 | test "(init) Restart killed instances" { 4 | foreach type {redis sentinel} { 5 | foreach_${type}_id id { 6 | if {[get_instance_attrib $type $id pid] == -1} { 7 | puts -nonewline "$type/$id " 8 | flush stdout 9 | restart_instance $type $id 10 | } 11 | } 12 | } 13 | } 14 | 15 | test "(init) Remove old master entry from sentinels" { 16 | foreach_sentinel_id id { 17 | catch {S $id SENTINEL REMOVE mymaster} 18 | } 19 | } 20 | 21 | set redis_slaves 4 22 | test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" { 23 | create_redis_master_slave_cluster [expr {$redis_slaves+1}] 24 | } 25 | set master_id 0 26 | 27 | test "(init) Sentinels can start monitoring a master" { 28 | set sentinels [llength $::sentinel_instances] 29 | set quorum [expr {$sentinels/2+1}] 30 | foreach_sentinel_id id { 31 | S $id SENTINEL MONITOR mymaster \ 32 | [get_instance_attrib redis $master_id host] \ 33 | [get_instance_attrib redis $master_id port] $quorum 34 | } 35 | foreach_sentinel_id id { 36 | assert {[S $id sentinel master mymaster] ne {}} 37 | S $id SENTINEL SET mymaster down-after-milliseconds 2000 38 | S $id SENTINEL SET mymaster failover-timeout 20000 39 | S $id SENTINEL SET mymaster parallel-syncs 10 40 | } 41 | } 42 | 43 | test "(init) Sentinels can talk with the master" { 44 | foreach_sentinel_id id { 45 | wait_for_condition 1000 50 { 46 | [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0 47 | } else { 48 | fail "Sentinel $id can't talk with the master." 49 | } 50 | } 51 | } 52 | 53 | test "(init) Sentinels are able to auto-discover other sentinels" { 54 | set sentinels [llength $::sentinel_instances] 55 | foreach_sentinel_id id { 56 | wait_for_condition 1000 50 { 57 | [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1) 58 | } else { 59 | fail "At least some sentinel can't detect some other sentinel" 60 | } 61 | } 62 | } 63 | 64 | test "(init) Sentinels are able to auto-discover slaves" { 65 | foreach_sentinel_id id { 66 | wait_for_condition 1000 50 { 67 | [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves 68 | } else { 69 | fail "At least some sentinel can't detect some slave" 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/base/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * util.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_UTIL_H__ 9 | #define __BASE_UTIL_H__ 10 | 11 | #include "ostype.h" 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | using namespace std; 26 | 27 | #define NOTUSED_ARG(v) ((void)v) // used this to remove warning C4100, unreferenced parameter 28 | 29 | #define CODE_OK 0 30 | #define CODE_ERROR 1 31 | 32 | class RefCount 33 | { 34 | public: 35 | RefCount() : ref_count_(1), mutex_(NULL) {} 36 | virtual ~RefCount() {} 37 | 38 | void SetMutex(mutex* mtx) { mutex_ = mtx; } 39 | 40 | void AddRef() { 41 | if (mutex_) { 42 | mutex_->lock(); 43 | ++ref_count_; 44 | mutex_->unlock(); 45 | } else { 46 | ++ref_count_; 47 | } 48 | } 49 | 50 | void ReleaseRef() { 51 | if (mutex_) { 52 | lock_guard mg(*mutex_); 53 | --ref_count_; 54 | if (ref_count_ == 0) { 55 | delete this; 56 | } 57 | } else { 58 | --ref_count_; 59 | if (ref_count_ == 0) 60 | delete this; 61 | } 62 | } 63 | private: 64 | int ref_count_; 65 | mutex* mutex_; 66 | }; 67 | 68 | uint64_t get_tick_count(); 69 | 70 | bool is_valid_ip(const char *ip); 71 | 72 | void write_pid(const string& name); 73 | 74 | // read small config file content 75 | int get_file_content(const char* filename, string& file_content); 76 | 77 | // system daemon() function will report deprecated warning under MacOS X, so just rewrite the function 78 | void run_as_daemon(); 79 | 80 | vector split(const string& str, const string& sep); 81 | 82 | // create a path if not exist 83 | // side effect: if path is not end with '/'， it will add '/' at the end 84 | void create_path(string& path); 85 | 86 | void parse_command_args(int argc, char* argv[], const char* version, string& config_file); 87 | 88 | int get_long_from_string(const string& str, long& l); 89 | int get_ulong_from_string(const string& str, unsigned long& ul); 90 | int get_double_from_string(const string& str, double& d); 91 | 92 | uint64_t double_to_uint64(double d); 93 | double uint64_to_double(uint64_t u); 94 | 95 | bool get_ip_port(const string& addr, string& ip, int& port); 96 | 97 | #endif 98 | -------------------------------------------------------------------------------- /tests/unit/slowlog.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000}} { 2 | test {SLOWLOG - check that it starts with an empty log} { 3 | r slowlog len 4 | } {0} 5 | 6 | test {SLOWLOG - only logs commands taking more time than specified} { 7 | r config set slowlog-log-slower-than 100 8 | r ping 9 | assert_equal [r slowlog len] 0 10 | r debug sleep 0.2 11 | assert_equal [r slowlog len] 1 12 | } 13 | 14 | test {SLOWLOG - max entries is correctly handled} { 15 | r config set slowlog-log-slower-than 0 16 | r config set slowlog-max-len 10 17 | for {set i 0} {$i < 100} {incr i} { 18 | r ping 19 | } 20 | r slowlog len 21 | } {10} 22 | 23 | test {SLOWLOG - GET optional argument to limit output len works} { 24 | llength [r slowlog get 5] 25 | } {5} 26 | 27 | test {SLOWLOG - RESET subcommand works} { 28 | r config set slowlog-log-slower-than 100 29 | r slowlog reset 30 | r slowlog len 31 | } {0} 32 | 33 | test {SLOWLOG - logged entry sanity check} { 34 | r debug sleep 0.2 35 | set e [lindex [r slowlog get] 0] 36 | assert_equal [llength $e] 4 37 | assert_equal [lindex $e 0] 105 38 | assert_equal [expr {[lindex $e 2] > 100}] 1 39 | assert_equal [lindex $e 3] {DEBUG sleep 0.2} 40 | } 41 | 42 | test {SLOWLOG - commands with too many arguments are trimmed} { 43 | r config set slowlog-log-slower-than 0 44 | r slowlog reset 45 | r sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 46 | set e [lindex [r slowlog get] 0] 47 | lindex $e 3 48 | } {SADD set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 {... (2 more arguments)}} 49 | 50 | test {SLOWLOG - too long arguments are trimmed} { 51 | r config set slowlog-log-slower-than 0 52 | r slowlog reset 53 | set arg [string repeat A 129] 54 | r sadd set foo $arg 55 | set e [lindex [r slowlog get] 0] 56 | lindex $e 3 57 | } {SADD set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} 58 | 59 | #test {SLOWLOG - EXEC is not logged, just executed commands} { 60 | # r config set slowlog-log-slower-than 100 61 | # r slowlog reset 62 | # assert_equal [r slowlog len] 0 63 | # r multi 64 | # r debug sleep 0.2 65 | # r exec 66 | # assert_equal [r slowlog len] 1 67 | # set e [lindex [r slowlog get] 0] 68 | # assert_equal [lindex $e 3] {debug sleep 0.2} 69 | #} 70 | } 71 | -------------------------------------------------------------------------------- /tools/hiredis/net.h: -------------------------------------------------------------------------------- 1 | /* Extracted from anet.c to work properly with Hiredis error reporting. 2 | * 3 | * Copyright (c) 2009-2011, Salvatore Sanfilippo 4 | * Copyright (c) 2010-2014, Pieter Noordhuis 5 | * Copyright (c) 2015, Matt Stancliff , 6 | * Jan-Erik Rediger 7 | * 8 | * All rights reserved. 9 | * 10 | * Redistribution and use in source and binary forms, with or without 11 | * modification, are permitted provided that the following conditions are met: 12 | * 13 | * * Redistributions of source code must retain the above copyright notice, 14 | * this list of conditions and the following disclaimer. 15 | * * Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * * Neither the name of Redis nor the names of its contributors may be used 19 | * to endorse or promote products derived from this software without 20 | * specific prior written permission. 21 | * 22 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | * POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | #ifndef __NET_H 36 | #define __NET_H 37 | 38 | #include "hiredis.h" 39 | 40 | #if defined(__sun) 41 | #define AF_LOCAL AF_UNIX 42 | #endif 43 | 44 | int redisCheckSocketError(redisContext *c); 45 | int redisContextSetTimeout(redisContext *c, const struct timeval tv); 46 | int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); 47 | int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, 48 | const struct timeval *timeout, 49 | const char *source_addr); 50 | int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); 51 | int redisKeepAlive(redisContext *c, int interval); 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /tests/unit/obuf-limits.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"obuf-limits"}} { 2 | test {Client output buffer hard limit is enforced} { 3 | r config set client-output-buffer-limit {pubsub 100000 0 0} 4 | set rd1 [redis_deferring_client] 5 | 6 | $rd1 subscribe foo 7 | set reply [$rd1 read] 8 | assert {$reply eq "subscribe foo 1"} 9 | 10 | set omem 0 11 | while 1 { 12 | r publish foo bar 13 | set clients [split [r client list] "\r\n"] 14 | set c [split [lindex $clients 1] " "] 15 | if {![regexp {omem=([0-9]+)} $c - omem]} break 16 | if {$omem > 200000} break 17 | } 18 | assert {$omem >= 90000 && $omem < 200000} 19 | $rd1 close 20 | } 21 | 22 | test {Client output buffer soft limit is not enforced if time is not overreached} { 23 | r config set client-output-buffer-limit {pubsub 0 100000 10} 24 | set rd1 [redis_deferring_client] 25 | 26 | $rd1 subscribe foo 27 | set reply [$rd1 read] 28 | assert {$reply eq "subscribe foo 1"} 29 | 30 | set omem 0 31 | set start_time 0 32 | set time_elapsed 0 33 | while 1 { 34 | r publish foo bar 35 | set clients [split [r client list] "\r\n"] 36 | set c [split [lindex $clients 1] " "] 37 | if {![regexp {omem=([0-9]+)} $c - omem]} break 38 | if {$omem > 100000} { 39 | if {$start_time == 0} {set start_time [clock seconds]} 40 | set time_elapsed [expr {[clock seconds]-$start_time}] 41 | if {$time_elapsed >= 5} break 42 | } 43 | } 44 | assert {$omem >= 100000 && $time_elapsed >= 5 && $time_elapsed <= 10} 45 | $rd1 close 46 | } 47 | 48 | test {Client output buffer soft limit is enforced if time is overreached} { 49 | r config set client-output-buffer-limit {pubsub 0 100000 3} 50 | set rd1 [redis_deferring_client] 51 | 52 | $rd1 subscribe foo 53 | set reply [$rd1 read] 54 | assert {$reply eq "subscribe foo 1"} 55 | 56 | set omem 0 57 | set start_time 0 58 | set time_elapsed 0 59 | while 1 { 60 | r publish foo bar 61 | set clients [split [r client list] "\r\n"] 62 | set c [split [lindex $clients 1] " "] 63 | if {![regexp {omem=([0-9]+)} $c - omem]} break 64 | if {$omem > 100000} { 65 | if {$start_time == 0} {set start_time [clock seconds]} 66 | set time_elapsed [expr {[clock seconds]-$start_time}] 67 | if {$time_elapsed >= 10} break 68 | } 69 | } 70 | assert {$omem >= 100000 && $time_elapsed < 6} 71 | $rd1 close 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/server/client_conn.h: -------------------------------------------------------------------------------- 1 | // 2 | // client_conn.h 3 | // kedis 4 | // 5 | // Created by ziteng on 17/7/19. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #ifndef __CLIENT_CONN_H__ 10 | #define __CLIENT_CONN_H__ 11 | 12 | #include "base_conn.h" 13 | #include "redis_parser.h" 14 | 15 | enum { 16 | CONN_STATE_IDLE = 0, 17 | CONN_STATE_RECV_AUTH, 18 | CONN_STATE_RECV_PORT, 19 | CONN_STATE_RECV_PSYNC, 20 | CONN_STATE_RECV_SNAPSHOT, 21 | CONN_STATE_CONNECTED, 22 | }; 23 | 24 | enum { 25 | CLIENT_NORMAL = 0, // normal client 26 | CLIENT_SLAVE, // the client is a slave 27 | CLIENT_MASTER, // the client is a master 28 | }; 29 | 30 | class ReplicationSnapshot; 31 | 32 | class ClientConn : public BaseConn { 33 | public: 34 | ClientConn(); 35 | virtual ~ClientConn(); 36 | 37 | virtual void Close(); 38 | virtual void OnConnect(BaseSocket* base_socket); 39 | virtual void OnConfirm(); 40 | virtual void OnRead(); 41 | virtual void OnTimer(uint64_t curr_tick); 42 | virtual void OnLoop(); 43 | 44 | void SendRawResponse(const string& resp); 45 | void SendError(const string& error_msg); 46 | void SendInteger(long i); 47 | void SendSimpleString(const string& str); 48 | void SendBulkString(const string& str); 49 | void SendArray(const vector& str_vec); 50 | void SendMultiBuldLen(long len); 51 | 52 | int GetDBIndex() { return db_index_; } 53 | void SetAuth(bool auth) { authenticated_ = auth; } 54 | string GetCurReqCommand() { return string(cur_req_buf_, cur_req_len_); } 55 | void SetState(int state) { state_ = state; } 56 | int GetState() { return state_; } 57 | void SetFlag(int flag) { flag_ = flag; } 58 | void SetSlavePort(int port) { slave_port_ = port; } 59 | void SetSyncSeq(uint64_t seq) { sync_seq_ = seq; } 60 | void SetDbIndex(int db_idx) { db_index_ = db_idx; } 61 | void SetReplSnapshot(ReplicationSnapshot* snapshot) { repl_snapshot_ = snapshot; } 62 | bool IsSendingSnapshot() { return repl_snapshot_ != nullptr; } 63 | uint64_t GetSyncSeq() { return sync_seq_; }; 64 | int GetSlavePort() { return slave_port_; } 65 | string GetSlaveName() { return m_peer_ip + ":" + to_string(slave_port_); } 66 | private: 67 | void _HandleRedisCommand(vector& cmd_vec); 68 | private: 69 | int db_index_; 70 | string pipeline_response_; 71 | bool authenticated_; 72 | char* cur_req_buf_; // the buffer of the current processing request 73 | int cur_req_len_; // the length of current processing request 74 | int state_; 75 | int flag_; // client type 76 | int slave_port_; 77 | uint64_t sync_seq_; // used for slave conn, the binlog sequence that need to send to slave 78 | ReplicationSnapshot* repl_snapshot_; 79 | }; 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /src/base/base_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * base_socket.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_BASE_SOCKET_H__ 9 | #define __BASE_BASE_SOCKET_H__ 10 | 11 | #include "ostype.h" 12 | #include "util.h" 13 | 14 | enum 15 | { 16 | SOCKET_STATE_IDLE, 17 | SOCKET_STATE_LISTENING, 18 | SOCKET_STATE_CONNECTING, 19 | SOCKET_STATE_CONNECTED, 20 | SOCKET_STATE_PEER_CLOSING, 21 | SOCKET_STATE_CLOSING 22 | }; 23 | 24 | class EventLoop; 25 | 26 | class BaseSocket : public RefCount 27 | { 28 | public: 29 | BaseSocket(); 30 | virtual ~BaseSocket(); 31 | 32 | net_handle_t GetHandle() { return m_handle; } 33 | int GetSocket() { return m_socket; } 34 | void SetEventLoop(EventLoop* el) { m_event_loop = el; } 35 | void SetSocket(int fd) { m_socket = fd; } 36 | void SetState(uint8_t state) { m_state = state; } 37 | void SetCallback(callback_t callback) { m_callback = callback; } 38 | void SetCallbackData(void* data) { m_callback_data = data; } 39 | void SetRemoteIP(char* ip) { m_remote_ip = ip; } 40 | void SetRemotePort(uint16_t port) { m_remote_port = port; } 41 | 42 | EventLoop* GetEventLoop() { return m_event_loop; } 43 | const char* GetRemoteIP() { return m_remote_ip.c_str(); } 44 | uint16_t GetRemotePort() { return m_remote_port; } 45 | const char* GetLocalIP() { return m_local_ip.c_str(); } 46 | uint16_t GetLocalPort() { return m_local_port; } 47 | public: 48 | net_handle_t Listen( 49 | const char* server_ip, 50 | uint16_t port, 51 | callback_t callback, 52 | void* callback_data); 53 | 54 | net_handle_t Connect( 55 | const char* server_ip, 56 | uint16_t port, 57 | callback_t callback, 58 | void* callback_data, 59 | EventLoop* event_loop = NULL); // put the connection object to eventloop if presented 60 | 61 | int Send(void* buf, int len); 62 | 63 | int Recv(void* buf, int len); 64 | 65 | int Close(); 66 | 67 | public: 68 | void OnConnect(); 69 | void OnRead(); 70 | void OnWrite(); 71 | void OnClose(); 72 | void OnTimer(uint64_t curr_tick); 73 | 74 | void SetFastAck(); 75 | static void SetNonblock(int fd, bool nonblock); 76 | static void SetReuseAddr(int fd); 77 | static void SetNoDelay(int fd); 78 | static void SetAddr(const char* ip, const uint16_t port, sockaddr_in* pAddr); 79 | static void GetBindAddr(int fd, string& bind_ip, uint16_t& bind_port); 80 | 81 | private: 82 | bool _IsBlock(int error_code); 83 | void _AcceptNewSocket(); 84 | 85 | private: 86 | EventLoop* m_event_loop; 87 | string m_remote_ip; 88 | uint16_t m_remote_port; 89 | string m_local_ip; 90 | uint16_t m_local_port; 91 | 92 | callback_t m_callback; 93 | void* m_callback_data; 94 | 95 | uint8_t m_state; 96 | int m_socket; 97 | net_handle_t m_handle; 98 | }; 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /tests/cluster/tests/05-slave-selection.tcl: -------------------------------------------------------------------------------- 1 | # Slave selection test 2 | # Check the algorithm trying to pick the slave with the most complete history. 3 | 4 | source "../tests/includes/init-tests.tcl" 5 | 6 | # Create a cluster with 5 master and 10 slaves, so that we have 2 7 | # slaves for each master. 8 | test "Create a 5 nodes cluster" { 9 | create_cluster 5 10 10 | } 11 | 12 | test "Cluster is up" { 13 | assert_cluster_state ok 14 | } 15 | 16 | test "The first master has actually two slaves" { 17 | assert {[llength [lindex [R 0 role] 2]] == 2} 18 | } 19 | 20 | test {Slaves of #0 are instance #5 and #10 as expected} { 21 | set port0 [get_instance_attrib redis 0 port] 22 | assert {[lindex [R 5 role] 2] == $port0} 23 | assert {[lindex [R 10 role] 2] == $port0} 24 | } 25 | 26 | test "Instance #5 and #10 synced with the master" { 27 | wait_for_condition 1000 50 { 28 | [RI 5 master_link_status] eq {up} && 29 | [RI 10 master_link_status] eq {up} 30 | } else { 31 | fail "Instance #5 or #10 master link status is not up" 32 | } 33 | } 34 | 35 | set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]] 36 | 37 | test "Slaves are both able to receive and acknowledge writes" { 38 | for {set j 0} {$j < 100} {incr j} { 39 | $cluster set $j $j 40 | } 41 | assert {[R 0 wait 2 60000] == 2} 42 | } 43 | 44 | test "Write data while slave #10 is paused and can't receive it" { 45 | # Stop the slave with a multi/exec transaction so that the master will 46 | # be killed as soon as it can accept writes again. 47 | R 10 multi 48 | R 10 debug sleep 10 49 | R 10 client kill 127.0.0.1:$port0 50 | R 10 deferred 1 51 | R 10 exec 52 | 53 | # Write some data the slave can't receive. 54 | for {set j 0} {$j < 100} {incr j} { 55 | $cluster set $j $j 56 | } 57 | 58 | # Prevent the master from accepting new slaves. 59 | # Use a large pause value since we'll kill it anyway. 60 | R 0 CLIENT PAUSE 60000 61 | 62 | # Wait for the slave to return available again 63 | R 10 deferred 0 64 | assert {[R 10 read] eq {OK OK}} 65 | 66 | # Kill the master so that a reconnection will not be possible. 67 | kill_instance redis 0 68 | } 69 | 70 | test "Wait for instance #5 (and not #10) to turn into a master" { 71 | wait_for_condition 1000 50 { 72 | [RI 5 role] eq {master} 73 | } else { 74 | fail "No failover detected" 75 | } 76 | } 77 | 78 | test "Wait for the node #10 to return alive before ending the test" { 79 | R 10 ping 80 | } 81 | 82 | test "Cluster should eventually be up again" { 83 | assert_cluster_state ok 84 | } 85 | 86 | test "Node #10 should eventually replicate node #5" { 87 | set port5 [get_instance_attrib redis 5 port] 88 | wait_for_condition 1000 50 { 89 | ([lindex [R 10 role] 2] == $port5) && 90 | ([lindex [R 10 role] 3] eq {connected}) 91 | } else { 92 | fail "#10 didn't became slave of #5" 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /tests/cluster/tests/07-replica-migration.tcl: -------------------------------------------------------------------------------- 1 | # Replica migration test. 2 | # Check that orphaned masters are joined by replicas of masters having 3 | # multiple replicas attached, according to the migration barrier settings. 4 | 5 | source "../tests/includes/init-tests.tcl" 6 | 7 | # Create a cluster with 5 master and 10 slaves, so that we have 2 8 | # slaves for each master. 9 | test "Create a 5 nodes cluster" { 10 | create_cluster 5 10 11 | } 12 | 13 | test "Cluster is up" { 14 | assert_cluster_state ok 15 | } 16 | 17 | test "Each master should have two replicas attached" { 18 | foreach_redis_id id { 19 | if {$id < 5} { 20 | wait_for_condition 1000 50 { 21 | [llength [lindex [R 0 role] 2]] == 2 22 | } else { 23 | fail "Master #$id does not have 2 slaves as expected" 24 | } 25 | } 26 | } 27 | } 28 | 29 | test "Killing all the slaves of master #0 and #1" { 30 | kill_instance redis 5 31 | kill_instance redis 10 32 | kill_instance redis 6 33 | kill_instance redis 11 34 | after 4000 35 | } 36 | 37 | foreach_redis_id id { 38 | if {$id < 5} { 39 | test "Master #$id should have at least one replica" { 40 | wait_for_condition 1000 50 { 41 | [llength [lindex [R $id role] 2]] >= 1 42 | } else { 43 | fail "Master #$id has no replicas" 44 | } 45 | } 46 | } 47 | } 48 | 49 | # Now test the migration to a master which used to be a slave, after 50 | # a failver. 51 | 52 | source "../tests/includes/init-tests.tcl" 53 | 54 | # Create a cluster with 5 master and 10 slaves, so that we have 2 55 | # slaves for each master. 56 | test "Create a 5 nodes cluster" { 57 | create_cluster 5 10 58 | } 59 | 60 | test "Cluster is up" { 61 | assert_cluster_state ok 62 | } 63 | 64 | test "Kill slave #7 of master #2. Only slave left is #12 now" { 65 | kill_instance redis 7 66 | } 67 | 68 | set current_epoch [CI 1 cluster_current_epoch] 69 | 70 | test "Killing master node #2, #12 should failover" { 71 | kill_instance redis 2 72 | } 73 | 74 | test "Wait for failover" { 75 | wait_for_condition 1000 50 { 76 | [CI 1 cluster_current_epoch] > $current_epoch 77 | } else { 78 | fail "No failover detected" 79 | } 80 | } 81 | 82 | test "Cluster should eventually be up again" { 83 | assert_cluster_state ok 84 | } 85 | 86 | test "Cluster is writable" { 87 | cluster_write_test 1 88 | } 89 | 90 | test "Instance 12 is now a master without slaves" { 91 | assert {[RI 12 role] eq {master}} 92 | } 93 | 94 | # The remaining instance is now without slaves. Some other slave 95 | # should migrate to it. 96 | 97 | test "Master #12 should get at least one migrated replica" { 98 | wait_for_condition 1000 50 { 99 | [llength [lindex [R 12 role] 2]] >= 1 100 | } else { 101 | fail "Master #12 has no replicas" 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /tests/sentinel/tests/02-slaves-reconf.tcl: -------------------------------------------------------------------------------- 1 | # Check that slaves are reconfigured at a latter time if they are partitioned. 2 | # 3 | # Here we should test: 4 | # 1) That slaves point to the new master after failover. 5 | # 2) That partitioned slaves point to new master when they are partitioned 6 | # away during failover and return at a latter time. 7 | 8 | source "../tests/includes/init-tests.tcl" 9 | 10 | proc 02_test_slaves_replication {} { 11 | uplevel 1 { 12 | test "Check that slaves replicate from current master" { 13 | set master_port [RI $master_id tcp_port] 14 | foreach_redis_id id { 15 | if {$id == $master_id} continue 16 | if {[instance_is_killed redis $id]} continue 17 | wait_for_condition 1000 50 { 18 | ([RI $id master_port] == $master_port) && 19 | ([RI $id master_link_status] eq {up}) 20 | } else { 21 | fail "Redis slave $id is replicating from wrong master" 22 | } 23 | } 24 | } 25 | } 26 | } 27 | 28 | proc 02_crash_and_failover {} { 29 | uplevel 1 { 30 | test "Crash the master and force a failover" { 31 | set old_port [RI $master_id tcp_port] 32 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 33 | assert {[lindex $addr 1] == $old_port} 34 | kill_instance redis $master_id 35 | foreach_sentinel_id id { 36 | wait_for_condition 1000 50 { 37 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 38 | } else { 39 | fail "At least one Sentinel did not received failover info" 40 | } 41 | } 42 | restart_instance redis $master_id 43 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 44 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] 45 | } 46 | } 47 | } 48 | 49 | 02_test_slaves_replication 50 | 02_crash_and_failover 51 | 02_test_slaves_replication 52 | 53 | test "Kill a slave instance" { 54 | foreach_redis_id id { 55 | if {$id == $master_id} continue 56 | set killed_slave_id $id 57 | kill_instance redis $id 58 | break 59 | } 60 | } 61 | 62 | 02_crash_and_failover 63 | 02_test_slaves_replication 64 | 65 | test "Wait for failover to end" { 66 | set inprogress 1 67 | while {$inprogress} { 68 | set inprogress 0 69 | foreach_sentinel_id id { 70 | if {[dict exists [S $id SENTINEL MASTER mymaster] failover-state]} { 71 | incr inprogress 72 | } 73 | } 74 | if {$inprogress} {after 100} 75 | } 76 | } 77 | 78 | test "Restart killed slave and test replication of slaves again..." { 79 | restart_instance redis $killed_slave_id 80 | } 81 | 82 | # Now we check if the slave rejoining the partition is reconfigured even 83 | # if the failover finished. 84 | 02_test_slaves_replication 85 | -------------------------------------------------------------------------------- /src/base/thread_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * thread_pool.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_THREAD_POOL_H__ 9 | #define __BASE_THREAD_POOL_H__ 10 | 11 | #include "ostype.h" 12 | #include "util.h" 13 | 14 | class Thread 15 | { 16 | public: 17 | Thread() : m_thread_id(0), m_running(false) {} 18 | virtual ~Thread() {} 19 | 20 | static void* StartRoutine(void* arg) { 21 | Thread* pThread = (Thread*)arg; 22 | pThread->OnThreadRun(); 23 | return NULL; 24 | } 25 | 26 | void StartThread(void) { 27 | pthread_create(&m_thread_id, NULL, StartRoutine, this); 28 | } 29 | 30 | void StopThread() { 31 | if (m_running) { 32 | m_running = false; 33 | pthread_join(m_thread_id, NULL); 34 | } 35 | } 36 | 37 | bool IsRunning() { return m_running; } 38 | 39 | virtual void OnThreadRun(void) = 0; 40 | /* 41 | the format of OnThreadRun() in subclass must be: 42 | void OnThreadRun(void) 43 | { 44 | m_running = true; 45 | while (m_running) { 46 | // do usefull things 47 | } 48 | } 49 | */ 50 | protected: 51 | pthread_t m_thread_id; 52 | bool m_running; 53 | }; 54 | 55 | class ThreadNotify 56 | { 57 | public: 58 | ThreadNotify(); 59 | ~ThreadNotify(); 60 | void Lock() { pthread_mutex_lock(&m_mutex); } 61 | void Unlock() { pthread_mutex_unlock(&m_mutex); } 62 | void Wait() { pthread_cond_wait(&m_cond, &m_mutex); } 63 | void Signal() { pthread_cond_signal(&m_cond); } 64 | private: 65 | pthread_mutex_t m_mutex; 66 | pthread_cond_t m_cond; 67 | }; 68 | 69 | class Task { 70 | public: 71 | Task() {} 72 | virtual ~Task() {} 73 | 74 | virtual void run() = 0; 75 | 76 | void setThreadIdx(uint32_t idx) { m_thread_idx = idx; } 77 | uint32_t getThreadIdx() { return m_thread_idx; } 78 | private: 79 | uint32_t m_thread_idx; 80 | }; 81 | 82 | class WorkerThread { 83 | public: 84 | WorkerThread(); 85 | ~WorkerThread(); 86 | 87 | static void* StartRoutine(void* arg); 88 | 89 | void Start(); 90 | void Stop(); 91 | void Execute(); 92 | void PushTask(Task* pTask); 93 | 94 | pthread_t GetThreadId() { return m_thread_id; } 95 | void SetThreadIdx(uint32_t idx) { m_thread_idx = idx; } 96 | int GetTaskCnt() { return m_task_cnt; } 97 | private: 98 | bool m_stop; 99 | uint32_t m_thread_idx; 100 | pthread_t m_thread_id; 101 | ThreadNotify m_thread_notify; 102 | list m_task_list; 103 | atomic m_task_cnt; 104 | }; 105 | 106 | class ThreadPool { 107 | public: 108 | ThreadPool(); 109 | virtual ~ThreadPool(); 110 | 111 | int Init(uint32_t thread_num); 112 | void AddTask(Task* pTask); 113 | void AddTask(Task* pTask, uint32_t thread_idx); 114 | void Destory(); 115 | int GetTotalTaskCnt(); 116 | private: 117 | uint32_t m_thread_num; 118 | WorkerThread* m_worker_threads; 119 | }; 120 | 121 | 122 | #endif /* __BASE_THREAD_POOL_H__ */ 123 | -------------------------------------------------------------------------------- /src/base/simple_log.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * simple_log.cpp 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #include "simple_log.h" 9 | #include "util.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | SimpleLog::SimpleLog() 16 | { 17 | log_file_ = NULL; 18 | last_log_time_ = 0; 19 | log_level_ = kLogLevelDebug; 20 | } 21 | 22 | SimpleLog::~SimpleLog() 23 | { 24 | if (log_file_) { 25 | fclose(log_file_); 26 | log_file_ = NULL; 27 | } 28 | } 29 | 30 | 31 | void SimpleLog::Init(LogLevel level, const string& path) 32 | { 33 | // use last_log_time_ to determine if initialized 34 | if (last_log_time_) { 35 | return; 36 | } 37 | 38 | log_level_ = level; 39 | log_path_ = path; 40 | create_path(log_path_); 41 | 42 | last_log_time_ = time(NULL); 43 | OpenFile(); 44 | } 45 | 46 | void SimpleLog::LogMessage(LogLevel level, const char* fmt, ...) 47 | { 48 | if (level > log_level_) { 49 | return; 50 | } 51 | 52 | struct timeval tval; 53 | struct tm tm_now; 54 | time_t curr_time; 55 | time(&curr_time); 56 | localtime_r(&curr_time, &tm_now); 57 | gettimeofday(&tval, NULL); 58 | 59 | lock_guard mg(mutex_); 60 | int prefix_len = snprintf(log_buf_, sizeof(log_buf_), "%02d:%02d:%02d.%03d(tid=%d) %s ", 61 | tm_now.tm_hour, tm_now.tm_min, tm_now.tm_sec, (int)tval.tv_usec / 1000, 62 | (uint16_t)(long)pthread_self(), kLogLevelName[level].c_str()); 63 | 64 | va_list ap; 65 | va_start(ap, fmt); 66 | vsnprintf(log_buf_ + prefix_len, kMaxLogLineSize - prefix_len, fmt, ap); 67 | va_end(ap); 68 | 69 | if (IsNewDay(curr_time)) { 70 | if (log_file_) { 71 | fclose(log_file_); 72 | log_file_ = NULL; 73 | } 74 | 75 | OpenFile(); 76 | } 77 | 78 | if (log_file_) { 79 | fwrite(log_buf_, strlen(log_buf_), 1, log_file_); 80 | fflush(log_file_); 81 | } 82 | 83 | last_log_time_ = curr_time; 84 | } 85 | 86 | void SimpleLog::OpenFile() 87 | { 88 | time_t now = time(NULL); 89 | struct tm tm_now; 90 | localtime_r(&now, &tm_now); 91 | char date_buf[64]; 92 | snprintf(date_buf, sizeof(date_buf), "%04d-%02d-%02d.log", 93 | tm_now.tm_year + 1900, tm_now.tm_mon + 1, tm_now.tm_mday); 94 | string file = log_path_ + date_buf; 95 | log_file_ = fopen(file.c_str(), "a+"); 96 | assert(log_file_ != NULL); 97 | } 98 | 99 | bool SimpleLog::IsNewDay(time_t current_time) 100 | { 101 | struct tm tm_last, tm_now; 102 | localtime_r(&last_log_time_, &tm_last); 103 | localtime_r(¤t_time, &tm_now); 104 | 105 | if ((tm_last.tm_year == tm_now.tm_year) && 106 | (tm_last.tm_mon == tm_now.tm_mon) && 107 | (tm_last.tm_mday == tm_now.tm_mday)) { 108 | return false; 109 | } else { 110 | return true; 111 | } 112 | } 113 | 114 | 115 | SimpleLog g_simple_log; 116 | 117 | void init_simple_log(LogLevel level, const string& path) 118 | { 119 | g_simple_log.Init(level, path); 120 | } 121 | 122 | -------------------------------------------------------------------------------- /tests/integration/replication-2.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"repl"}} { 2 | start_server {} { 3 | test {First server should have role slave after SLAVEOF} { 4 | r -1 slaveof [srv 0 host] [srv 0 port] 5 | after 1000 6 | s -1 role 7 | } {slave} 8 | 9 | test {If min-slaves-to-write is honored, write is accepted} { 10 | r config set min-slaves-to-write 1 11 | r config set min-slaves-max-lag 10 12 | r set foo 12345 13 | wait_for_condition 50 100 { 14 | [r -1 get foo] eq {12345} 15 | } else { 16 | fail "Write did not reached slave" 17 | } 18 | } 19 | 20 | test {No write if min-slaves-to-write is < attached slaves} { 21 | r config set min-slaves-to-write 2 22 | r config set min-slaves-max-lag 10 23 | catch {r set foo 12345} err 24 | set err 25 | } {NOREPLICAS*} 26 | 27 | test {If min-slaves-to-write is honored, write is accepted (again)} { 28 | r config set min-slaves-to-write 1 29 | r config set min-slaves-max-lag 10 30 | r set foo 12345 31 | wait_for_condition 50 100 { 32 | [r -1 get foo] eq {12345} 33 | } else { 34 | fail "Write did not reached slave" 35 | } 36 | } 37 | 38 | test {No write if min-slaves-max-lag is > of the slave lag} { 39 | r -1 deferred 1 40 | r config set min-slaves-to-write 1 41 | r config set min-slaves-max-lag 2 42 | r -1 debug sleep 6 43 | assert {[r set foo 12345] eq {OK}} 44 | after 4000 45 | catch {r set foo 12345} err 46 | assert {[r -1 read] eq {OK}} 47 | r -1 deferred 0 48 | set err 49 | } {NOREPLICAS*} 50 | 51 | test {min-slaves-to-write is ignored by slaves} { 52 | r config set min-slaves-to-write 1 53 | r config set min-slaves-max-lag 10 54 | r -1 config set min-slaves-to-write 1 55 | r -1 config set min-slaves-max-lag 10 56 | r set foo aaabbb 57 | wait_for_condition 50 100 { 58 | [r -1 get foo] eq {aaabbb} 59 | } else { 60 | fail "Write did not reached slave" 61 | } 62 | } 63 | 64 | # Fix parameters for the next test to work 65 | r config set min-slaves-to-write 0 66 | r -1 config set min-slaves-to-write 0 67 | r flushall 68 | 69 | test {MASTER and SLAVE dataset should be identical after complex ops} { 70 | createComplexDataset r 10000 71 | after 500 72 | if {[r debug digest] ne [r -1 debug digest]} { 73 | set csv1 [csvdump r] 74 | set csv2 [csvdump {r -1}] 75 | set fd [open /tmp/repldump1.txt w] 76 | puts -nonewline $fd $csv1 77 | close $fd 78 | set fd [open /tmp/repldump2.txt w] 79 | puts -nonewline $fd $csv2 80 | close $fd 81 | puts "Master - Slave inconsistency" 82 | puts "Run diff -u against /tmp/repldump*.txt for more info" 83 | } 84 | assert_equal [r debug digest] [r -1 debug digest] 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/base/byte_stream.h: -------------------------------------------------------------------------------- 1 | /* 2 | * byte_stream.h 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #ifndef __BASE_BYTE_STREAM_H__ 9 | #define __BASE_BYTE_STREAM_H__ 10 | 11 | #include "ostype.h" 12 | #include "util.h" 13 | #include "simple_buffer.h" 14 | 15 | class ParseException { 16 | public: 17 | ParseException(const char* error_msg) { 18 | error_msg_ = error_msg; 19 | } 20 | virtual ~ParseException() {} 21 | 22 | char* GetErrorMsg() { return (char*)error_msg_.c_str(); } 23 | private: 24 | string error_msg_; 25 | }; 26 | 27 | class ByteStream 28 | { 29 | public: 30 | ByteStream(); 31 | ByteStream(uchar_t* buf, uint32_t len); 32 | ByteStream(SimpleBuffer* pSimpBuf, uint32_t pos = 0); 33 | ByteStream(string* pStr); 34 | ~ByteStream() {} 35 | 36 | void SetBuf(uchar_t* buf, uint32_t len) { m_pBuf = buf; m_len = len; } 37 | void SetSimpleBuffer(SimpleBuffer* buf) { m_pSimpBuf = buf; } 38 | void SetString(string* buf) { m_pStr = buf; } 39 | unsigned char* GetBuf() { 40 | if (m_pSimpBuf) { 41 | return m_pSimpBuf->GetBuffer(); 42 | } else if (m_pStr) { 43 | return (uchar_t*)m_pStr->data(); 44 | } else { 45 | return m_pBuf; 46 | } 47 | } 48 | uint32_t GetPos() { return m_pos; } 49 | uint32_t GetLen() { return m_len; } 50 | void Skip(uint32_t len) { 51 | m_pos += len; 52 | if (m_pos > m_len) { 53 | throw ParseException("skip passed boundary"); 54 | } 55 | } 56 | 57 | static int16_t ReadInt16(uchar_t* buf); 58 | static uint16_t ReadUint16(uchar_t* buf); 59 | static int32_t ReadInt32(uchar_t* buf); 60 | static uint32_t ReadUint32(uchar_t* buf); 61 | static int64_t ReadInt64(uchar_t* buf); 62 | static uint64_t ReadUint64(uchar_t* buf); 63 | 64 | static void WriteInt16(uchar_t* buf, int16_t data); 65 | static void WriteUint16(uchar_t* buf, uint16_t data); 66 | static void WriteInt32(uchar_t* buf, int32_t data); 67 | static void WriteUint32(uchar_t* buf, uint32_t data); 68 | static void WriteInt64(uchar_t* buf, int64_t data); 69 | static void WriteUint64(uchar_t* buf, uint64_t data); 70 | 71 | void operator << (int8_t data); 72 | void operator << (uint8_t data); 73 | void operator << (int16_t data); 74 | void operator << (uint16_t data); 75 | void operator << (int32_t data); 76 | void operator << (uint32_t data); 77 | void operator << (int64_t data); 78 | void operator << (uint64_t data); 79 | void operator << (const string& str); 80 | 81 | void operator >> (int8_t& data); 82 | void operator >> (uint8_t& data); 83 | void operator >> (int16_t& data); 84 | void operator >> (uint16_t& data); 85 | void operator >> (int32_t& data); 86 | void operator >> (uint32_t& data); 87 | void operator >> (int64_t& data); 88 | void operator >> (uint64_t& data); 89 | void operator >> (string& str); 90 | 91 | void WriteVarUInt(uint32_t len); 92 | uint32_t ReadVarUInt(); 93 | 94 | void WriteData(uchar_t* data, uint32_t len); 95 | uchar_t* ReadData(uint32_t& len); 96 | 97 | void WriteStringWithoutLen(const string& str); 98 | void ReadStringWithoutLen(string& str); 99 | private: 100 | void _WriteByte(void* buf, uint32_t len); 101 | void _ReadByte(void* buf, uint32_t len); 102 | private: 103 | SimpleBuffer* m_pSimpBuf; 104 | string* m_pStr; 105 | uchar_t* m_pBuf; 106 | uint32_t m_len; 107 | uint32_t m_pos; 108 | }; 109 | 110 | #endif 111 | -------------------------------------------------------------------------------- /tools/kedis_port/kedis_port.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // kedis_port.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "kedis_port.h" 10 | #include "util.h" 11 | #include "base_conn.h" 12 | #include "event_loop.h" 13 | #include "simple_log.h" 14 | #include "sync_task.h" 15 | #include "src_kedis_conn.h" 16 | #include "kedis_version.h" 17 | 18 | Config g_config; 19 | 20 | static void print_usage(const char* program) 21 | { 22 | fprintf(stderr, "%s [OPTIONS]\n" 23 | " --src_addr source kedis server ip:port (default: 127.0.0.1:6375)\n" 24 | " --dst_addr destination kedis server ip:port (default: 127.0.0.1:7400)\n" 25 | " --src_db source db number (default: -1)\n" 26 | " --dst_db destination db number (default: -1)\n" 27 | " --src_password source kedis password (default: no password)\n" 28 | " --dst_password destination kedis password (default: no password)\n" 29 | " --prefix remove key prefix\n" 30 | " --version show version\n" 31 | " --help\n", program); 32 | } 33 | 34 | void parse_cmd_line(int argc, char* argv[]) 35 | { 36 | for (int i = 1; i < argc; ++i) { 37 | bool last_arg = (i == argc - 1); 38 | 39 | if (!strcmp(argv[i], "--help")) { 40 | print_usage(argv[0]); 41 | exit(0); 42 | } else if (!strcmp(argv[i], "--src_addr") && !last_arg) { 43 | string addr = argv[++i]; 44 | if (!get_ip_port(addr, g_config.src_kedis_host, g_config.src_kedis_port)) { 45 | fprintf(stderr, "invalid src addr: %s\n", addr.c_str()); 46 | exit(1); 47 | } 48 | } else if (!strcmp(argv[i], "--dst_addr") && !last_arg) { 49 | string addr = argv[++i]; 50 | if (!get_ip_port(addr, g_config.dst_kedis_host, g_config.dst_kedis_port)) { 51 | fprintf(stderr, "invalid dst addr: %s\n", addr.c_str()); 52 | exit(1); 53 | } 54 | } else if (!strcmp(argv[i], "--src_db") && !last_arg) { 55 | g_config.src_kedis_db = atoi(argv[++i]); 56 | } else if (!strcmp(argv[i], "--dst_db") && !last_arg) { 57 | g_config.dst_kedis_db = atoi(argv[++i]); 58 | } else if (!strcmp(argv[i], "--src_password") && !last_arg) { 59 | g_config.src_kedis_password = argv[++i]; 60 | } else if (!strcmp(argv[i], "--dst_password") && !last_arg) { 61 | g_config.dst_kedis_password = argv[++i]; 62 | } else if (!strcmp(argv[i], "--prefix") && !last_arg) { 63 | g_config.prefix = argv[++i]; 64 | } else if (!strcmp(argv[i], "--version")) { 65 | printf("redis_port Version: %s\n", KEDIS_VERSION); 66 | printf("redis_port Build: %s %s\n", __DATE__, __TIME__); 67 | exit(0); 68 | } else { 69 | print_usage(argv[0]); 70 | exit(1); 71 | } 72 | } 73 | } 74 | 75 | int main(int argc, char* argv[]) 76 | { 77 | parse_cmd_line(argc, argv); 78 | 79 | int io_thread_num = 0; // all nonblock network io in main thread 80 | init_thread_event_loops(io_thread_num); 81 | init_thread_base_conn(io_thread_num); 82 | 83 | init_simple_log(kLogLevelDebug, "log"); 84 | 85 | init_sync_task(); 86 | 87 | SrcKedisConn* src_conn = new SrcKedisConn(); 88 | src_conn->Connect(g_config.src_kedis_host, g_config.src_kedis_port); 89 | 90 | get_main_event_loop()->Start(); 91 | 92 | return 0; 93 | } 94 | -------------------------------------------------------------------------------- /tools/kedis_to_redis/kedis_to_redis.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // kedis_port.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/16. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "kedis_to_redis.h" 10 | #include "util.h" 11 | #include "base_conn.h" 12 | #include "event_loop.h" 13 | #include "simple_log.h" 14 | #include "sync_task.h" 15 | #include "src_kedis_conn.h" 16 | #include "kedis_version.h" 17 | 18 | Config g_config; 19 | 20 | static void print_usage(const char* program) 21 | { 22 | fprintf(stderr, "%s [OPTIONS]\n" 23 | " --src_addr source kedis server ip:port (default: 127.0.0.1:6375)\n" 24 | " --dst_addr destination redis server ip:port (default: 127.0.0.1:7400)\n" 25 | " --src_db source db number (default: -1)\n" 26 | " --dst_db destination db number (default: -1)\n" 27 | " --src_password source kedis password (default: no password)\n" 28 | " --dst_password destination redis password (default: no password)\n" 29 | " --prefix remove key prefix\n" 30 | " --version show version\n" 31 | " --help\n", program); 32 | } 33 | 34 | void parse_cmd_line(int argc, char* argv[]) 35 | { 36 | for (int i = 1; i < argc; ++i) { 37 | bool last_arg = (i == argc - 1); 38 | 39 | if (!strcmp(argv[i], "--help")) { 40 | print_usage(argv[0]); 41 | exit(0); 42 | } else if (!strcmp(argv[i], "--src_addr") && !last_arg) { 43 | string addr = argv[++i]; 44 | if (!get_ip_port(addr, g_config.src_kedis_host, g_config.src_kedis_port)) { 45 | fprintf(stderr, "invalid src addr: %s\n", addr.c_str()); 46 | exit(1); 47 | } 48 | } else if (!strcmp(argv[i], "--dst_addr") && !last_arg) { 49 | string addr = argv[++i]; 50 | if (!get_ip_port(addr, g_config.dst_redis_host, g_config.dst_redis_port)) { 51 | fprintf(stderr, "invalid dst addr: %s\n", addr.c_str()); 52 | exit(1); 53 | } 54 | } else if (!strcmp(argv[i], "--src_db") && !last_arg) { 55 | g_config.src_kedis_db = atoi(argv[++i]); 56 | } else if (!strcmp(argv[i], "--dst_db") && !last_arg) { 57 | g_config.dst_redis_db = atoi(argv[++i]); 58 | } else if (!strcmp(argv[i], "--src_password") && !last_arg) { 59 | g_config.src_kedis_password = argv[++i]; 60 | } else if (!strcmp(argv[i], "--dst_password") && !last_arg) { 61 | g_config.dst_redis_password = argv[++i]; 62 | } else if (!strcmp(argv[i], "--prefix") && !last_arg) { 63 | g_config.prefix = argv[++i]; 64 | } else if (!strcmp(argv[i], "--version")) { 65 | printf("redis_port Version: %s\n", KEDIS_VERSION); 66 | printf("redis_port Build: %s %s\n", __DATE__, __TIME__); 67 | exit(0); 68 | } else { 69 | print_usage(argv[0]); 70 | exit(1); 71 | } 72 | } 73 | } 74 | 75 | int main(int argc, char* argv[]) 76 | { 77 | parse_cmd_line(argc, argv); 78 | 79 | int io_thread_num = 0; // all nonblock network io in main thread 80 | init_thread_event_loops(io_thread_num); 81 | init_thread_base_conn(io_thread_num); 82 | 83 | init_simple_log(kLogLevelDebug, "log"); 84 | 85 | init_sync_task(); 86 | 87 | SrcKedisConn* src_conn = new SrcKedisConn(); 88 | src_conn->Connect(g_config.src_kedis_host, g_config.src_kedis_port); 89 | 90 | get_main_event_loop()->Start(); 91 | 92 | return 0; 93 | } 94 | -------------------------------------------------------------------------------- /tests/cluster/tests/03-failover-loop.tcl: -------------------------------------------------------------------------------- 1 | # Failover stress test. 2 | # In this test a different node is killed in a loop for N 3 | # iterations. The test checks that certain properties 4 | # are preseved across iterations. 5 | 6 | source "../tests/includes/init-tests.tcl" 7 | 8 | test "Create a 5 nodes cluster" { 9 | create_cluster 5 5 10 | } 11 | 12 | test "Cluster is up" { 13 | assert_cluster_state ok 14 | } 15 | 16 | set iterations 20 17 | set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]] 18 | 19 | while {[incr iterations -1]} { 20 | set tokill [randomInt 10] 21 | set other [expr {($tokill+1)%10}] ; # Some other instance. 22 | set key [randstring 20 20 alpha] 23 | set val [randstring 20 20 alpha] 24 | set role [RI $tokill role] 25 | if {$role eq {master}} { 26 | set slave {} 27 | set myid [dict get [get_myself $tokill] id] 28 | foreach_redis_id id { 29 | if {$id == $tokill} continue 30 | if {[dict get [get_myself $id] slaveof] eq $myid} { 31 | set slave $id 32 | } 33 | } 34 | if {$slave eq {}} { 35 | fail "Unable to retrieve slave's ID for master #$tokill" 36 | } 37 | } 38 | 39 | puts "--- Iteration $iterations ---" 40 | 41 | if {$role eq {master}} { 42 | test "Wait for slave of #$tokill to sync" { 43 | wait_for_condition 1000 50 { 44 | [string match {*state=online*} [RI $tokill slave0]] 45 | } else { 46 | fail "Slave of node #$tokill is not ok" 47 | } 48 | } 49 | set slave_config_epoch [CI $slave cluster_my_epoch] 50 | } 51 | 52 | test "Cluster is writable before failover" { 53 | for {set i 0} {$i < 100} {incr i} { 54 | catch {$cluster set $key:$i $val:$i} err 55 | assert {$err eq {OK}} 56 | } 57 | # Wait for the write to propagate to the slave if we 58 | # are going to kill a master. 59 | if {$role eq {master}} { 60 | R $tokill wait 1 20000 61 | } 62 | } 63 | 64 | test "Killing node #$tokill" { 65 | kill_instance redis $tokill 66 | } 67 | 68 | if {$role eq {master}} { 69 | test "Wait failover by #$slave with old epoch $slave_config_epoch" { 70 | wait_for_condition 1000 50 { 71 | [CI $slave cluster_my_epoch] > $slave_config_epoch 72 | } else { 73 | fail "No failover detected, epoch is still [CI $slave cluster_my_epoch]" 74 | } 75 | } 76 | } 77 | 78 | test "Cluster should eventually be up again" { 79 | assert_cluster_state ok 80 | } 81 | 82 | test "Cluster is writable again" { 83 | for {set i 0} {$i < 100} {incr i} { 84 | catch {$cluster set $key:$i:2 $val:$i:2} err 85 | assert {$err eq {OK}} 86 | } 87 | } 88 | 89 | test "Restarting node #$tokill" { 90 | restart_instance redis $tokill 91 | } 92 | 93 | test "Instance #$tokill is now a slave" { 94 | wait_for_condition 1000 50 { 95 | [RI $tokill role] eq {slave} 96 | } else { 97 | fail "Restarted instance is not a slave" 98 | } 99 | } 100 | 101 | test "We can read back the value we set before" { 102 | for {set i 0} {$i < 100} {incr i} { 103 | catch {$cluster get $key:$i} err 104 | assert {$err eq "$val:$i"} 105 | catch {$cluster get $key:$i:2} err 106 | assert {$err eq "$val:$i:2"} 107 | } 108 | } 109 | } 110 | 111 | test "Post condition: current_epoch >= my_epoch everywhere" { 112 | foreach_redis_id id { 113 | assert {[CI $id cluster_current_epoch] >= [CI $id cluster_my_epoch]} 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /tests/unit/protocol.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"protocol"}} { 2 | test "Handle an empty query" { 3 | reconnect 4 | r write "\r\n" 5 | r flush 6 | assert_equal "PONG" [r ping] 7 | } 8 | 9 | test "Negative multibulk length" { 10 | reconnect 11 | r write "*-10\r\n" 12 | r flush 13 | assert_equal PONG [r ping] 14 | } 15 | 16 | test "Out of range multibulk length" { 17 | reconnect 18 | r write "*20000000\r\n" 19 | r flush 20 | assert_error "*invalid multibulk length*" {r read} 21 | } 22 | 23 | test "Wrong multibulk payload header" { 24 | reconnect 25 | r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\nfooz\r\n" 26 | r flush 27 | assert_error "*expected '$', got 'f'*" {r read} 28 | } 29 | 30 | test "Negative multibulk payload length" { 31 | reconnect 32 | r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\n\$-10\r\n" 33 | r flush 34 | assert_error "*invalid bulk length*" {r read} 35 | } 36 | 37 | test "Out of range multibulk payload length" { 38 | reconnect 39 | r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\n\$2000000000\r\n" 40 | r flush 41 | assert_error "*invalid bulk length*" {r read} 42 | } 43 | 44 | test "Non-number multibulk payload length" { 45 | reconnect 46 | r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\n\$blabla\r\n" 47 | r flush 48 | assert_error "*invalid bulk length*" {r read} 49 | } 50 | 51 | test "Multi bulk request not followed by bulk arguments" { 52 | reconnect 53 | r write "*1\r\nfoo\r\n" 54 | r flush 55 | assert_error "*expected '$', got 'f'*" {r read} 56 | } 57 | 58 | test "Generic wrong number of args" { 59 | reconnect 60 | assert_error "*wrong*arguments*PING*" {r ping x y z} 61 | } 62 | 63 | test "Unbalanced number of quotes" { 64 | reconnect 65 | r write "set \"\"\"test-key\"\"\" test-value\r\n" 66 | r write "ping\r\n" 67 | r flush 68 | assert_error "*unbalanced*" {r read} 69 | } 70 | 71 | set c 0 72 | foreach seq [list "\x00" "*\x00" "$\x00"] { 73 | incr c 74 | test "Protocol desync regression test #$c" { 75 | set s [socket [srv 0 host] [srv 0 port]] 76 | puts -nonewline $s $seq 77 | set payload [string repeat A 1024]"\n" 78 | set test_start [clock seconds] 79 | set test_time_limit 30 80 | while 1 { 81 | if {[catch { 82 | puts -nonewline $s payload 83 | flush $s 84 | incr payload_size [string length $payload] 85 | }]} { 86 | set retval [gets $s] 87 | close $s 88 | break 89 | } else { 90 | set elapsed [expr {[clock seconds]-$test_start}] 91 | if {$elapsed > $test_time_limit} { 92 | close $s 93 | error "assertion:Redis did not closed connection after protocol desync" 94 | } 95 | } 96 | } 97 | set retval 98 | } {*Protocol error*} 99 | } 100 | unset c 101 | } 102 | 103 | start_server {tags {"regression"}} { 104 | test "Regression for a crash with blocking ops and pipelining" { 105 | set rd [redis_deferring_client] 106 | set fd [r channel] 107 | set proto "*3\r\n\$5\r\nBLPOP\r\n\$6\r\nnolist\r\n\$1\r\n0\r\n" 108 | puts -nonewline $fd $proto$proto 109 | flush $fd 110 | set res {} 111 | 112 | $rd rpush nolist a 113 | $rd read 114 | $rd rpush nolist a 115 | $rd read 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/base/thread_pool.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * thread_pool.cpp 3 | * 4 | * Created on: 2016-3-14 5 | * Author: ziteng 6 | */ 7 | 8 | #include 9 | #include 10 | #include "util.h" 11 | #include "thread_pool.h" 12 | 13 | 14 | ThreadNotify::ThreadNotify() 15 | { 16 | pthread_mutex_init(&m_mutex, NULL); 17 | pthread_cond_init(&m_cond, NULL); 18 | } 19 | 20 | ThreadNotify::~ThreadNotify() 21 | { 22 | pthread_mutex_destroy(&m_mutex); 23 | pthread_cond_destroy(&m_cond); 24 | } 25 | 26 | /////////// 27 | WorkerThread::WorkerThread() 28 | { 29 | m_stop = false; 30 | m_task_cnt = 0; 31 | } 32 | 33 | WorkerThread::~WorkerThread() 34 | { 35 | 36 | } 37 | 38 | void* WorkerThread::StartRoutine(void* arg) 39 | { 40 | WorkerThread* pThread = (WorkerThread*)arg; 41 | 42 | pThread->Execute(); 43 | 44 | return NULL; 45 | } 46 | 47 | void WorkerThread::Start() 48 | { 49 | (void)pthread_create(&m_thread_id, NULL, StartRoutine, this); 50 | } 51 | 52 | void WorkerThread::Stop() 53 | { 54 | m_stop = true; 55 | m_thread_notify.Signal(); 56 | } 57 | 58 | void WorkerThread::Execute() 59 | { 60 | while (!m_stop) { 61 | m_thread_notify.Lock(); 62 | 63 | // put wait in while cause there can be spurious wake up (due to signal/ENITR) 64 | while (m_task_list.empty() && !m_stop) { 65 | m_thread_notify.Wait(); 66 | } 67 | 68 | list tmp_task_list; 69 | tmp_task_list.swap(m_task_list); 70 | 71 | m_thread_notify.Unlock(); 72 | 73 | for (list::iterator it = tmp_task_list.begin(); it != tmp_task_list.end(); ++it) { 74 | Task* task = *it; 75 | task->setThreadIdx(m_thread_idx); 76 | task->run(); 77 | delete task; 78 | m_task_cnt--; 79 | } 80 | } 81 | } 82 | 83 | void WorkerThread::PushTask(Task* pTask) 84 | { 85 | m_thread_notify.Lock(); 86 | m_task_list.push_back(pTask); 87 | m_thread_notify.Signal(); 88 | m_thread_notify.Unlock(); 89 | 90 | m_task_cnt++; 91 | } 92 | 93 | 94 | ////////////// 95 | ThreadPool::ThreadPool() 96 | { 97 | m_thread_num = 0; 98 | m_worker_threads = NULL; 99 | } 100 | 101 | ThreadPool::~ThreadPool() 102 | { 103 | 104 | } 105 | 106 | int ThreadPool::Init(uint32_t thread_num) 107 | { 108 | m_thread_num = thread_num; 109 | m_worker_threads = new WorkerThread [m_thread_num]; 110 | assert(m_worker_threads); 111 | 112 | for (uint32_t i = 0; i < m_thread_num; ++i) { 113 | m_worker_threads[i].SetThreadIdx(i); 114 | m_worker_threads[i].Start(); 115 | } 116 | 117 | return 0; 118 | } 119 | 120 | void ThreadPool::Destory() 121 | { 122 | for (uint32_t i = 0; i < m_thread_num; ++i) { 123 | m_worker_threads[i].Stop(); 124 | } 125 | 126 | for (uint32_t i = 0; i < m_thread_num; ++i) { 127 | pthread_join(m_worker_threads[i].GetThreadId(), NULL); 128 | } 129 | 130 | delete [] m_worker_threads; 131 | } 132 | 133 | void ThreadPool::AddTask(Task* pTask) 134 | { 135 | /* 136 | * select a random thread to push task 137 | * we can also select a thread that has less task to do 138 | * but that will scan the whole threads vector and use thread lock to get each task size 139 | */ 140 | uint32_t thread_idx = random() % m_thread_num; 141 | m_worker_threads[thread_idx].PushTask(pTask); 142 | } 143 | 144 | void ThreadPool::AddTask(Task* pTask, uint32_t thread_idx) 145 | { 146 | if (thread_idx >= m_thread_num) { 147 | AddTask(pTask); 148 | return; 149 | } 150 | 151 | m_worker_threads[thread_idx].PushTask(pTask); 152 | } 153 | 154 | // not very precise because of without a mutex to synchronization 155 | int ThreadPool::GetTotalTaskCnt() 156 | { 157 | int total_task_cnt = 0; 158 | for (uint32_t i = 0; i < m_thread_num; ++i) { 159 | total_task_cnt += m_worker_threads[i].GetTaskCnt(); 160 | } 161 | return total_task_cnt; 162 | } 163 | -------------------------------------------------------------------------------- /src/server/slowlog.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // slowlog.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17/11/2. 6 | // Copyright © 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "slowlog.h" 10 | #include "db_util.h" 11 | 12 | mutex g_slowlog_mtx; 13 | list g_slowlog_list; 14 | uint64_t g_slowlog_id = 0; 15 | 16 | void add_slowlog(const vector& cmd_vec, uint64_t duration) 17 | { 18 | int cmd_size = (int)cmd_vec.size(); 19 | int sl_argc = cmd_size; 20 | if (sl_argc > SLOWLOG_ENTRY_MAX_ARGC) { 21 | sl_argc = SLOWLOG_ENTRY_MAX_ARGC; 22 | } 23 | 24 | SlowlogEntry* sl_entry = new SlowlogEntry; 25 | 26 | lock_guard guard(g_slowlog_mtx); 27 | if ((int)g_slowlog_list.size() >= g_server.slowlog_max_len) { 28 | SlowlogEntry* old_entry = g_slowlog_list.front(); 29 | delete old_entry; 30 | g_slowlog_list.pop_front(); 31 | } 32 | 33 | for (int i = 0; i < sl_argc; i++) { 34 | if ((sl_argc != cmd_size) && (i == sl_argc - 1)) { 35 | char buf[256]; 36 | snprintf(buf, sizeof(buf), "... (%d more arguments)", cmd_size - sl_argc + 1); 37 | sl_entry->cmd_vec.push_back(buf); 38 | } else { 39 | // Trim too long strings 40 | int item_size = (int)cmd_vec[i].size(); 41 | if (item_size > SLOWLOG_ENTRY_MAX_STRING) { 42 | char buf[256]; 43 | string prefix = cmd_vec[i].substr(0, SLOWLOG_ENTRY_MAX_STRING); 44 | snprintf(buf, sizeof(buf), "%s... (%d more bytes)", 45 | prefix.c_str(), item_size - SLOWLOG_ENTRY_MAX_STRING); 46 | sl_entry->cmd_vec.push_back(buf); 47 | } else { 48 | sl_entry->cmd_vec.push_back(cmd_vec[i]); 49 | } 50 | } 51 | } 52 | 53 | sl_entry->time = time(NULL); 54 | sl_entry->duration = duration; 55 | sl_entry->slowlog_id = g_slowlog_id++; 56 | g_slowlog_list.push_back(sl_entry); 57 | } 58 | 59 | void slowlog_command(ClientConn* conn, const vector& cmd_vec) 60 | { 61 | int cmd_size = (int)cmd_vec.size(); 62 | if (!strcasecmp(cmd_vec[1].c_str(), "reset") && (cmd_size == 2)) { 63 | g_slowlog_mtx.lock(); 64 | for (SlowlogEntry* entry : g_slowlog_list) { 65 | delete entry; 66 | } 67 | g_slowlog_list.clear(); 68 | g_slowlog_mtx.unlock(); 69 | 70 | conn->SendRawResponse(kOKString); 71 | } else if (!strcasecmp(cmd_vec[1].c_str(), "len") && (cmd_size == 2)) { 72 | g_slowlog_mtx.lock(); 73 | long len = (long)g_slowlog_list.size(); 74 | g_slowlog_mtx.unlock(); 75 | 76 | conn->SendInteger(len); 77 | } else if (!strcasecmp(cmd_vec[1].c_str(), "get") && (cmd_size == 2 || cmd_size == 3)) { 78 | long count = 10; 79 | if (cmd_size == 3) { 80 | if (get_long_from_string(cmd_vec[2], count) == CODE_ERROR) { 81 | conn->SendError("value is not integer or out of range"); 82 | return; 83 | } 84 | } 85 | 86 | lock_guard guard(g_slowlog_mtx); 87 | long sl_size = (long)g_slowlog_list.size(); 88 | if (count > sl_size) { 89 | count = sl_size; 90 | } 91 | 92 | conn->SendMultiBuldLen(count); 93 | for (auto it = g_slowlog_list.rbegin(); it != g_slowlog_list.rend() && count; it++) { 94 | SlowlogEntry* entry = *it; 95 | conn->SendMultiBuldLen(4); 96 | conn->SendInteger(entry->slowlog_id); 97 | conn->SendInteger(entry->time); 98 | conn->SendInteger(entry->duration); 99 | conn->SendArray(entry->cmd_vec); 100 | count--; 101 | } 102 | } else { 103 | conn->SendError("Unknown SLOWLOG subcommand or wrong # of args. Try GET, RESET, LEN"); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /tools/redis_to_kedis/cmd_line_parser.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // cmd_line_parser.cpp 3 | // kedis 4 | // 5 | // Created by ziteng on 17-11-20. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "cmd_line_parser.h" 10 | #include "kedis_version.h" 11 | 12 | Config g_config; 13 | 14 | static void print_usage(const char* program) 15 | { 16 | fprintf(stderr, "%s [OPTIONS]\n" 17 | " --src_addr source redis server ip:port (default: 127.0.0.1:6375)\n" 18 | " --dst_addr destination kedis server ip:port (default: 127.0.0.1:7400)\n" 19 | " --src_db source db number (default: -1)\n" 20 | " --dst_db destination kedis db number (default: -1)\n" 21 | " --src_password source redis password (default: no password)\n" 22 | " --dst_password destination kedis password (default: no password)\n" 23 | " --pipeline how many pipeline command (default: 32)\n" 24 | " --rdb_file rdb dump file (default: dump.rdb)\n" 25 | " --aof_file aof dump file (default: dump.aof)\n" 26 | " --prefix remove key prefix\n" 27 | " --network_limit limit network speed when receive RDB file, unit MB (default: 50MB)\n" 28 | " --src_from_rdb use RDB file as a data source\n" 29 | " --version show version\n" 30 | " --help\n", program); 31 | } 32 | 33 | void parse_cmd_line(int argc, char* argv[]) 34 | { 35 | for (int i = 1; i < argc; ++i) { 36 | bool last_arg = (i == argc - 1); 37 | 38 | if (!strcmp(argv[i], "--help")) { 39 | print_usage(argv[0]); 40 | exit(0); 41 | } else if (!strcmp(argv[i], "--src_addr") && !last_arg) { 42 | string addr = argv[++i]; 43 | if (!get_ip_port(addr, g_config.src_redis_host, g_config.src_redis_port)) { 44 | fprintf(stderr, "invalid src addr: %s\n", addr.c_str()); 45 | exit(1); 46 | } 47 | } else if (!strcmp(argv[i], "--dst_addr") && !last_arg) { 48 | string addr = argv[++i]; 49 | if (!get_ip_port(addr, g_config.dst_kedis_host, g_config.dst_kedis_port)) { 50 | fprintf(stderr, "invalid dst addr: %s\n", addr.c_str()); 51 | exit(1); 52 | } 53 | } else if (!strcmp(argv[i], "--src_db") && !last_arg) { 54 | g_config.src_redis_db = atoi(argv[++i]); 55 | } else if (!strcmp(argv[i], "--dst_db") && !last_arg) { 56 | g_config.dst_kedis_db = atoi(argv[++i]); 57 | } else if (!strcmp(argv[i], "--src_password") && !last_arg) { 58 | g_config.src_redis_password = argv[++i]; 59 | } else if (!strcmp(argv[i], "--dst_password") && !last_arg) { 60 | g_config.dst_kedis_password = argv[++i]; 61 | } else if (!strcmp(argv[i], "--pipeline") && !last_arg) { 62 | g_config.pipeline_cnt = atoi(argv[++i]); 63 | } else if (!strcmp(argv[i], "--rdb_file") && !last_arg) { 64 | g_config.rdb_file = argv[++i]; 65 | } else if (!strcmp(argv[i], "--aof_file") && !last_arg) { 66 | g_config.aof_file = argv[++i]; 67 | } else if (!strcmp(argv[i], "--prefix") && !last_arg) { 68 | g_config.prefix = argv[++i]; 69 | } else if (!strcmp(argv[i], "--network_limit") && !last_arg) { 70 | g_config.network_limit = atoi(argv[++i]) * 1024 * 1024; 71 | } else if (!strcmp(argv[i], "--src_from_rdb")) { 72 | g_config.src_from_rdb = true; 73 | } else if (!strcmp(argv[i], "--version")) { 74 | printf("redis_to_kedis Version: %s\n", KEDIS_VERSION); 75 | printf("redis_to_kedis Build: %s %s\n", __DATE__, __TIME__); 76 | exit(0); 77 | } else { 78 | print_usage(argv[0]); 79 | exit(1); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /tools/hiredis_wrapper/redis_conn.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // redis_conn.cpp 3 | // kv-store 4 | // 5 | // Created by ziteng on 17-11-16. 6 | // Copyright (c) 2017年 mgj. All rights reserved. 7 | // 8 | 9 | #include "redis_conn.h" 10 | #include "simple_log.h" 11 | 12 | RedisConn::RedisConn(const string& ip, int port) : ip_(ip), port_(port) 13 | { 14 | context_ = NULL; 15 | reply_ = NULL; 16 | } 17 | 18 | RedisConn::RedisConn() 19 | { 20 | context_ = NULL; 21 | reply_ = NULL; 22 | } 23 | 24 | RedisConn::~RedisConn() 25 | { 26 | if (context_) { 27 | redisFree(context_); 28 | context_ = NULL; 29 | } 30 | 31 | if (reply_) { 32 | freeReplyObject(reply_); 33 | reply_ = NULL; 34 | } 35 | } 36 | 37 | int RedisConn::Init() 38 | { 39 | if (context_) { 40 | return 0; 41 | } 42 | 43 | // 200ms timeout 44 | struct timeval timeout = {0, 200000}; 45 | context_ = redisConnectWithTimeout(ip_.c_str(), port_, timeout); 46 | if (!context_ || context_->err) { 47 | if (context_) { 48 | log_message(kLogLevelError, "redisConnect failed: %s\n", context_->errstr); 49 | redisFree(context_); 50 | context_ = NULL; 51 | } else { 52 | log_message(kLogLevelError, "redisConnect failed\n"); 53 | } 54 | 55 | return 1; 56 | } 57 | 58 | if (!password_.empty()) { 59 | string auth_cmd = "AUTH " + password_; 60 | reply_ = (redisReply *)redisCommand(context_, auth_cmd.c_str()); 61 | if (reply_ && (reply_->type == REDIS_REPLY_STATUS) && !strncmp(reply_->str, "OK", 2)) { 62 | return 0; 63 | } else if (reply_ && (reply_->type == REDIS_REPLY_ERROR) && !strncmp(reply_->str, "ERR Client sent AUTH", 20)) { 64 | return 0; 65 | } else { 66 | log_message(kLogLevelError, "redis auth failed\n"); 67 | return 2; 68 | } 69 | } 70 | 71 | return 0; 72 | } 73 | 74 | redisReply* RedisConn::DoRawCmd(const string& cmd) 75 | { 76 | if (reply_) { 77 | freeReplyObject(reply_); 78 | reply_ = NULL; 79 | } 80 | 81 | if (Init()) { 82 | return NULL; 83 | } 84 | 85 | reply_ = (redisReply *)redisRawCommand(context_, cmd.c_str() , (int)cmd.size()); 86 | if (!reply_) { 87 | log_message(kLogLevelError, "redisRawCommand failed: %s\n", context_->errstr); 88 | redisFree(context_); 89 | context_ = NULL; 90 | return NULL; 91 | } 92 | 93 | return reply_; 94 | } 95 | 96 | redisReply* RedisConn::DoCmd(const string& cmd) 97 | { 98 | if (reply_) { 99 | freeReplyObject(reply_); 100 | reply_ = NULL; 101 | } 102 | 103 | if (Init()) { 104 | return NULL; 105 | } 106 | 107 | reply_ = (redisReply *)redisCommand(context_, cmd.c_str()); 108 | if (!reply_) { 109 | log_message(kLogLevelError, "redisCommand failed: %s\n", context_->errstr); 110 | redisFree(context_); 111 | context_ = NULL; 112 | return NULL; 113 | } 114 | 115 | return reply_; 116 | } 117 | 118 | void RedisConn::PipelineRawCmd(const string& cmd) 119 | { 120 | if (reply_) { 121 | freeReplyObject(reply_); 122 | reply_ = NULL; 123 | } 124 | 125 | if (Init()) { 126 | return; 127 | } 128 | 129 | redisAppendFormattedCommand(context_, cmd.c_str(), cmd.size()); 130 | } 131 | 132 | void RedisConn::PipelineCmd(const string& cmd) 133 | { 134 | if (reply_) { 135 | freeReplyObject(reply_); 136 | reply_ = NULL; 137 | } 138 | 139 | if (Init()) { 140 | return; 141 | } 142 | 143 | redisAppendCommand(context_, cmd.c_str()); 144 | } 145 | 146 | redisReply* RedisConn::GetReply() 147 | { 148 | if (reply_) { 149 | freeReplyObject(reply_); 150 | reply_ = NULL; 151 | } 152 | 153 | redisGetReply(context_, (void **)&reply_); 154 | 155 | return reply_; 156 | } 157 | -------------------------------------------------------------------------------- /tests/integration/rdb.tcl: -------------------------------------------------------------------------------- 1 | set server_path [tmpdir "server.rdb-encoding-test"] 2 | 3 | # Copy RDB with different encodings in server path 4 | exec cp tests/assets/encodings.rdb $server_path 5 | 6 | start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] { 7 | test "RDB encoding loading test" { 8 | r select 0 9 | csvdump r 10 | } {"0","compressible","string","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" 11 | "0","hash","hash","a","1","aa","10","aaa","100","b","2","bb","20","bbb","200","c","3","cc","30","ccc","300","ddd","400","eee","5000000000", 12 | "0","hash_zipped","hash","a","1","b","2","c","3", 13 | "0","list","list","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000", 14 | "0","list_zipped","list","1","2","3","a","b","c","100000","6000000000", 15 | "0","number","string","10" 16 | "0","set","set","1","100000","2","3","6000000000","a","b","c", 17 | "0","set_zipped_1","set","1","2","3","4", 18 | "0","set_zipped_2","set","100000","200000","300000","400000", 19 | "0","set_zipped_3","set","1000000000","2000000000","3000000000","4000000000","5000000000","6000000000", 20 | "0","string","string","Hello World" 21 | "0","zset","zset","a","1","b","2","c","3","aa","10","bb","20","cc","30","aaa","100","bbb","200","ccc","300","aaaa","1000","cccc","123456789","bbbb","5000000000", 22 | "0","zset_zipped","zset","a","1","b","2","c","3", 23 | } 24 | } 25 | 26 | set server_path [tmpdir "server.rdb-startup-test"] 27 | 28 | start_server [list overrides [list "dir" $server_path]] { 29 | test {Server started empty with non-existing RDB file} { 30 | r debug digest 31 | } {0000000000000000000000000000000000000000} 32 | # Save an RDB file, needed for the next test. 33 | r save 34 | } 35 | 36 | start_server [list overrides [list "dir" $server_path]] { 37 | test {Server started empty with empty RDB file} { 38 | r debug digest 39 | } {0000000000000000000000000000000000000000} 40 | } 41 | 42 | # Helper function to start a server and kill it, just to check the error 43 | # logged. 44 | set defaults {} 45 | proc start_server_and_kill_it {overrides code} { 46 | upvar defaults defaults srv srv server_path server_path 47 | set config [concat $defaults $overrides] 48 | set srv [start_server [list overrides $config]] 49 | uplevel 1 $code 50 | kill_server $srv 51 | } 52 | 53 | # Make the RDB file unreadable 54 | file attributes [file join $server_path dump.rdb] -permissions 0222 55 | 56 | # Detect root account (it is able to read the file even with 002 perm) 57 | set isroot 0 58 | catch { 59 | open [file join $server_path dump.rdb] 60 | set isroot 1 61 | } 62 | 63 | # Now make sure the server aborted with an error 64 | if {!$isroot} { 65 | start_server_and_kill_it [list "dir" $server_path] { 66 | test {Server should not start if RDB file can't be open} { 67 | wait_for_condition 50 100 { 68 | [string match {*Fatal error loading*} \ 69 | [exec tail -n1 < [dict get $srv stdout]]] 70 | } else { 71 | fail "Server started even if RDB was unreadable!" 72 | } 73 | } 74 | } 75 | } 76 | 77 | # Fix permissions of the RDB file. 78 | file attributes [file join $server_path dump.rdb] -permissions 0666 79 | 80 | # Corrupt its CRC64 checksum. 81 | set filesize [file size [file join $server_path dump.rdb]] 82 | set fd [open [file join $server_path dump.rdb] r+] 83 | fconfigure $fd -translation binary 84 | seek $fd -8 end 85 | puts -nonewline $fd "foobar00"; # Corrupt the checksum 86 | close $fd 87 | 88 | # Now make sure the server aborted with an error 89 | start_server_and_kill_it [list "dir" $server_path] { 90 | test {Server should not start if RDB is corrupted} { 91 | wait_for_condition 50 100 { 92 | [string match {*CRC error*} \ 93 | [exec tail -n10 < [dict get $srv stdout]]] 94 | } else { 95 | fail "Server started even if RDB was corrupted!" 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /tests/integration/replication-3.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"repl"}} { 2 | start_server {} { 3 | test {First server should have role slave after SLAVEOF} { 4 | r -1 slaveof [srv 0 host] [srv 0 port] 5 | wait_for_condition 50 100 { 6 | [s -1 master_link_status] eq {up} 7 | } else { 8 | fail "Replication not started." 9 | } 10 | } 11 | 12 | if {$::accurate} {set numops 50000} else {set numops 5000} 13 | 14 | test {MASTER and SLAVE consistency with expire} { 15 | createComplexDataset r $numops useexpire 16 | after 4000 ;# Make sure everything expired before taking the digest 17 | r keys * ;# Force DEL syntesizing to slave 18 | after 1000 ;# Wait another second. Now everything should be fine. 19 | #if {[r debug digest] ne [r -1 debug digest]} { 20 | # set csv1 [csvdump r] 21 | # set csv2 [csvdump {r -1}] 22 | # set fd [open /tmp/repldump1.txt w] 23 | # puts -nonewline $fd $csv1 24 | # close $fd 25 | # set fd [open /tmp/repldump2.txt w] 26 | # puts -nonewline $fd $csv2 27 | # close $fd 28 | # puts "Master - Slave inconsistency" 29 | # puts "Run diff -u against /tmp/repldump*.txt for more info" 30 | #} 31 | assert_equal [r dbsize] [r -1 dbsize] 32 | } 33 | } 34 | } 35 | 36 | start_server {tags {"repl"}} { 37 | start_server {} { 38 | test {First server should have role slave after SLAVEOF} { 39 | r -1 slaveof [srv 0 host] [srv 0 port] 40 | wait_for_condition 50 100 { 41 | [s -1 master_link_status] eq {up} 42 | } else { 43 | fail "Replication not started." 44 | } 45 | } 46 | 47 | set numops 20000 ;# Enough to trigger the Script Cache LRU eviction. 48 | 49 | # While we are at it, enable AOF to test it will be consistent as well 50 | # after the test. 51 | #r config set appendonly yes 52 | 53 | test {MASTER and SLAVE consistency with EVALSHA replication} { 54 | array set oldsha {} 55 | for {set j 0} {$j < $numops} {incr j} { 56 | set key "key:$j" 57 | # Make sure to create scripts that have different SHA1s 58 | set script "return redis.call('incr','$key')" 59 | set sha1 [r eval "return redis.sha1hex(\"$script\")" 0] 60 | set oldsha($j) $sha1 61 | r eval $script 0 62 | set res [r evalsha $sha1 0] 63 | assert {$res == 2} 64 | # Additionally call one of the old scripts as well, at random. 65 | set res [r evalsha $oldsha([randomInt $j]) 0] 66 | assert {$res > 2} 67 | 68 | # Trigger an AOF rewrite while we are half-way, this also 69 | # forces the flush of the script cache, and we will cover 70 | # more code as a result. 71 | if {$j == $numops / 2} { 72 | catch {r bgrewriteaof} 73 | } 74 | } 75 | 76 | wait_for_condition 50 100 { 77 | [r dbsize] == $numops && 78 | [r -1 dbsize] == $numops 79 | } else { 80 | set csv1 [csvdump r] 81 | set csv2 [csvdump {r -1}] 82 | set fd [open /tmp/repldump1.txt w] 83 | puts -nonewline $fd $csv1 84 | close $fd 85 | set fd [open /tmp/repldump2.txt w] 86 | puts -nonewline $fd $csv2 87 | close $fd 88 | puts "Master - Slave inconsistency" 89 | puts "Run diff -u against /tmp/repldump*.txt for more info" 90 | 91 | } 92 | 93 | #set old_digest [r debug digest] 94 | #r config set appendonly no 95 | #r debug loadaof 96 | #set new_digest [r debug digest] 97 | #assert {$old_digest eq $new_digest} 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /tests/support/test.tcl: -------------------------------------------------------------------------------- 1 | set ::num_tests 0 2 | set ::num_passed 0 3 | set ::num_failed 0 4 | set ::tests_failed {} 5 | 6 | proc fail {msg} { 7 | error "assertion:$msg" 8 | } 9 | 10 | proc assert {condition} { 11 | if {![uplevel 1 [list expr $condition]]} { 12 | error "assertion:Expected condition '$condition' to be true ([uplevel 1 [list subst -nocommands $condition]])" 13 | } 14 | } 15 | 16 | proc assert_match {pattern value} { 17 | if {![string match $pattern $value]} { 18 | error "assertion:Expected '$value' to match '$pattern'" 19 | } 20 | } 21 | 22 | proc assert_equal {expected value {detail ""}} { 23 | if {$expected ne $value} { 24 | if {$detail ne ""} { 25 | set detail " (detail: $detail)" 26 | } 27 | error "assertion:Expected '$value' to be equal to '$expected'$detail" 28 | } 29 | } 30 | 31 | proc assert_error {pattern code} { 32 | if {[catch {uplevel 1 $code} error]} { 33 | assert_match $pattern $error 34 | } else { 35 | error "assertion:Expected an error but nothing was caught" 36 | } 37 | } 38 | 39 | proc assert_encoding {enc key} { 40 | #set dbg [r debug object $key] 41 | #assert_match "* encoding:$enc *" $dbg 42 | } 43 | 44 | proc assert_type {type key} { 45 | assert_equal $type [r type $key] 46 | } 47 | 48 | # Wait for the specified condition to be true, with the specified number of 49 | # max retries and delay between retries. Otherwise the 'elsescript' is 50 | # executed. 51 | proc wait_for_condition {maxtries delay e _else_ elsescript} { 52 | while {[incr maxtries -1] >= 0} { 53 | set errcode [catch {uplevel 1 [list expr $e]} result] 54 | if {$errcode == 0} { 55 | if {$result} break 56 | } else { 57 | return -code $errcode $result 58 | } 59 | after $delay 60 | } 61 | if {$maxtries == -1} { 62 | set errcode [catch [uplevel 1 $elsescript] result] 63 | return -code $errcode $result 64 | } 65 | } 66 | 67 | proc test {name code {okpattern undefined}} { 68 | # abort if tagged with a tag to deny 69 | foreach tag $::denytags { 70 | if {[lsearch $::tags $tag] >= 0} { 71 | return 72 | } 73 | } 74 | 75 | # check if tagged with at least 1 tag to allow when there *is* a list 76 | # of tags to allow, because default policy is to run everything 77 | if {[llength $::allowtags] > 0} { 78 | set matched 0 79 | foreach tag $::allowtags { 80 | if {[lsearch $::tags $tag] >= 0} { 81 | incr matched 82 | } 83 | } 84 | if {$matched < 1} { 85 | return 86 | } 87 | } 88 | 89 | incr ::num_tests 90 | set details {} 91 | lappend details "$name in $::curfile" 92 | 93 | send_data_packet $::test_server_fd testing $name 94 | 95 | if {[catch {set retval [uplevel 1 $code]} error]} { 96 | if {[string match "assertion:*" $error]} { 97 | set msg [string range $error 10 end] 98 | lappend details $msg 99 | lappend ::tests_failed $details 100 | 101 | incr ::num_failed 102 | send_data_packet $::test_server_fd err [join $details "\n"] 103 | } else { 104 | # Re-raise, let handler up the stack take care of this. 105 | error $error $::errorInfo 106 | } 107 | } else { 108 | if {$okpattern eq "undefined" || $okpattern eq $retval || [string match $okpattern $retval]} { 109 | incr ::num_passed 110 | send_data_packet $::test_server_fd ok $name 111 | } else { 112 | set msg "Expected '$okpattern' to equal or match '$retval'" 113 | lappend details $msg 114 | lappend ::tests_failed $details 115 | 116 | incr ::num_failed 117 | send_data_packet $::test_server_fd err [join $details "\n"] 118 | } 119 | } 120 | 121 | if {$::traceleaks} { 122 | set output [exec leaks redis-server] 123 | if {![string match {*0 leaks*} $output]} { 124 | send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output" 125 | } 126 | } 127 | } 128 | --------------------------------------------------------------------------------