├── .gitignore ├── cluster ├── .gitignore ├── ananas │ ├── CMakeLists.txt │ ├── net │ │ ├── AnanasLogo.h │ │ ├── AnanasDebug.h │ │ ├── AnanasDebug.cc │ │ ├── CMakeLists.txt │ │ ├── log │ │ │ ├── README.md │ │ │ └── MmapFile.h │ │ ├── Kqueue.h │ │ ├── Epoller.h │ │ ├── Typedefs.h │ │ ├── Acceptor.h │ │ ├── TimeUtil.h │ │ ├── Connector.h │ │ ├── DatagramSocket.h │ │ ├── Poller.h │ │ ├── TimeUtil.cc │ │ ├── Connection.h │ │ ├── Socket.h │ │ ├── Kqueue.cc │ │ ├── ThreadPool.cc │ │ └── Timer.cc │ └── util │ │ ├── Scheduler.h │ │ ├── Util.h │ │ └── Delegate.h ├── cluster_conn │ ├── CMakeLists.txt │ ├── ClusterConn.h │ └── zookeeper │ │ ├── CMakeLists.txt │ │ ├── ZkResponse.h │ │ ├── recordio.h │ │ └── ZookeeperContext.h ├── Makefile ├── qedis_proxy │ ├── ProxyLog.h │ ├── CMakeLists.txt │ ├── ClientConn.h │ ├── ProxyConfig.h │ ├── ClientManager.h │ ├── ProxyConfig.cc │ ├── ClusterManager.h │ ├── main.cc │ ├── Command.h │ ├── QedisManager.h │ ├── ZookeeperConn.h │ ├── QedisConn.h │ ├── QedisConn.cc │ ├── ConfigParser.h │ ├── ClientManager.cc │ ├── Protocol.h │ └── ConfigParser.cc ├── CMakeLists.txt ├── CMakeCommon └── README.md ├── performance.png ├── QBase ├── README.md ├── CMakeLists.txt ├── Kqueue.h ├── EPoller.h ├── ListenSocket.h ├── ClientSocket.h ├── AsyncBuffer.h ├── Poller.h ├── TaskManager.h ├── UnboundedBuffer.h ├── ConfigParser.h ├── StreamSocket.h ├── Server.h ├── Log │ └── MemoryFile.h ├── Threads │ ├── ThreadPool.h │ └── ThreadPool.cc ├── NetThreadPool.h ├── Delegate.h ├── TaskManager.cc ├── AsyncBuffer.cc ├── ConfigParser.cc ├── Kqueue.cc ├── ListenSocket.cc ├── ClientSocket.cc └── EPoller.cc ├── Modules ├── QModuleInit.h ├── QListModule.h ├── QSetModule.h ├── QHashModule.h ├── CMakeLists.txt ├── QHashModule.cc ├── QSetModule.cc ├── QListModule.cc └── QModuleInit.cc ├── .travis.yml ├── QedisCore ├── QList.h ├── QSlaveClient.h ├── QSet.h ├── QHash.h ├── QString.h ├── CMakeLists.txt ├── QDumpInterface.h ├── QMulti.h ├── QSlaveClient.cc ├── QProtoParser.h ├── QSlowLog.h ├── QLeveldb.h ├── QMigration.h ├── QSortedSet.h ├── QPubsub.h ├── QSlowLog.cc ├── QModule.h ├── QGlobRegex.h ├── QDB.h ├── QConfig.h ├── redisIntset.h ├── redisZipList.h ├── QReplication.h ├── QAOF.h ├── QHelper.h └── QProtoParser.cc ├── CMakeLists.txt ├── UnitTest ├── CMakeLists.txt ├── UnitTest.cc ├── UnitTest.h └── QGlobRegex_unittest.cc ├── Makefile ├── QSentinel ├── CMakeLists.txt ├── QClusterClient.h ├── README.md ├── zookeeper │ ├── zookeeper_version.h │ ├── ZookeeperConn.h │ └── recordio.h ├── QClusterInterface.h └── QClusterClient.cc ├── QedisSvr ├── QedisLogo.h ├── CMakeLists.txt └── Qedis.h ├── CMakeCommon ├── LICENSE ├── README.zh.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # MAC OS 2 | .DS_Store 3 | 4 | # 5 | build/ 6 | bin/ 7 | -------------------------------------------------------------------------------- /cluster/.gitignore: -------------------------------------------------------------------------------- 1 | # MAC OS 2 | .DS_Store 3 | 4 | # 5 | build/ 6 | bin/ 7 | -------------------------------------------------------------------------------- /performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loveyacper/Qedis/HEAD/performance.png -------------------------------------------------------------------------------- /cluster/ananas/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | SUBDIRS(net) 4 | -------------------------------------------------------------------------------- /cluster/cluster_conn/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | SUBDIRS(zookeeper) 4 | -------------------------------------------------------------------------------- /cluster/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | mkdir -p build && cd build && cmake .. && make 3 | clean: 4 | cd build && make clean 5 | -------------------------------------------------------------------------------- /QBase/README.md: -------------------------------------------------------------------------------- 1 | # QBase 2 | 3 | QBase最初是C++03代码，由于历史原因，再加上它也极其稳定，我不打算替换了。 4 | 5 | 感兴趣的可以看我的另外一个最新后台开发基础库[ananas](https://github.com/loveyacper/ananas) 6 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ProxyLog.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_PROXYLOG_H 2 | #define BERT_PROXYLOG_H 3 | 4 | #include "net/log/Logger.h" 5 | 6 | extern std::shared_ptr g_logger; 7 | 8 | #endif 9 | 10 | -------------------------------------------------------------------------------- /Modules/QModuleInit.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QMODULEINIT_H 2 | #define BERT_QMODULEINIT_H 3 | 4 | extern "C" 5 | bool QedisModule_OnLoad(); 6 | 7 | extern "C" 8 | void QedisModule_OnUnLoad(); 9 | #endif 10 | 11 | -------------------------------------------------------------------------------- /cluster/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | PROJECT(QEDIS_CLUSTER) 4 | 5 | SUBDIRS(ananas) 6 | 7 | SET(USE_ZOOKEEPER 1) 8 | 9 | ADD_DEFINITIONS(-DUSE_ZOOKEEPER=${USE_ZOOKEEPER}) 10 | 11 | SUBDIRS(cluster_conn) 12 | SUBDIRS(qedis_proxy) 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: cpp 3 | compiler: 4 | - clang 5 | script: make 6 | branches: 7 | only: 8 | - master 9 | notifications: 10 | recipients: 11 | - bertyoung666@gmail.com 12 | email: 13 | on_success: change 14 | on_failure: always 15 | os: 16 | - osx 17 | -------------------------------------------------------------------------------- /QedisCore/QList.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QLIST_H 2 | #define BERT_QLIST_H 3 | 4 | #include "QString.h" 5 | #include 6 | 7 | namespace qedis 8 | { 9 | 10 | enum class ListPosition 11 | { 12 | head, 13 | tail, 14 | }; 15 | 16 | using QList = std::list; 17 | } 18 | 19 | #endif 20 | 21 | -------------------------------------------------------------------------------- /QedisCore/QSlaveClient.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QSLAVECLIENT_H 2 | #define BERT_QSLAVECLIENT_H 3 | 4 | #include "StreamSocket.h" 5 | 6 | namespace qedis 7 | { 8 | 9 | class QSlaveClient : public StreamSocket 10 | { 11 | public: 12 | void OnConnect() override; 13 | private: 14 | PacketLength _HandlePacket(const char* msg, std::size_t len) override; 15 | }; 16 | 17 | } 18 | 19 | #endif 20 | 21 | -------------------------------------------------------------------------------- /QedisCore/QSet.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QSET_H 2 | #define BERT_QSET_H 3 | 4 | #include "QHelper.h" 5 | #include 6 | 7 | namespace qedis 8 | { 9 | 10 | using QSet = std::unordered_set >; 13 | 14 | size_t SScanKey(const QSet& qset, size_t cursor, size_t count, std::vector& res); 15 | 16 | } 17 | 18 | #endif 19 | 20 | -------------------------------------------------------------------------------- /cluster/ananas/net/AnanasLogo.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ANANASLOGO_H 2 | #define BERT_ANANASLOGO_H 3 | 4 | namespace ananas 5 | { 6 | namespace internal 7 | { 8 | 9 | const char* logo = "" 10 | " __ _ _ __ __ _ _ __ __ _ ___ \n" 11 | " / _` | '_ \\ / _` | '_ \\ / _` / __| \n" 12 | "| (_| | | | | (_| | | | | (_| \\__ \\ \n" 13 | " \\__,_|_| |_|\\__,_|_| |_|\\__,_|___/ \n"; 14 | 15 | } 16 | } 17 | #endif 18 | 19 | -------------------------------------------------------------------------------- /QedisCore/QHash.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QHASH_H 2 | #define BERT_QHASH_H 3 | 4 | #include "QString.h" 5 | #include "QHelper.h" 6 | 7 | #include 8 | 9 | namespace qedis 10 | { 11 | 12 | using QHash = std::unordered_map >; 13 | 14 | 15 | size_t HScanKey(const QHash& hash, size_t cursor, size_t count, std::vector& res); 16 | 17 | } 18 | 19 | #endif 20 | 21 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | PROJECT(QEDIS) 4 | 5 | SUBDIRS(QBase) 6 | SUBDIRS(QedisCore) 7 | SUBDIRS(QedisSvr) 8 | SUBDIRS(Modules) 9 | SUBDIRS(UnitTest) 10 | 11 | 12 | SET(QEDIS_CLUSTER 0) 13 | SET(USE_ZOOKEEPER 0) 14 | 15 | ADD_DEFINITIONS(-DQEDIS_CLUSTER=${QEDIS_CLUSTER}) 16 | ADD_DEFINITIONS(-DUSE_ZOOKEEPER=${USE_ZOOKEEPER}) 17 | if(${QEDIS_CLUSTER} EQUAL 1) 18 | SUBDIRS(QSentinel) 19 | endif() 20 | -------------------------------------------------------------------------------- /cluster/ananas/net/AnanasDebug.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ANANASDEBUG_H 2 | #define BERT_ANANASDEBUG_H 3 | 4 | #include 5 | #include "log/Logger.h" 6 | 7 | namespace ananas 8 | { 9 | 10 | namespace internal 11 | { 12 | 13 | extern std::shared_ptr g_debug; 14 | extern std::once_flag g_logInit; 15 | void InitDebugLog(unsigned int level); 16 | 17 | } // end namespace internal 18 | 19 | } // end namespace ananas 20 | 21 | #endif 22 | 23 | -------------------------------------------------------------------------------- /Modules/QListModule.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QLISTMODULE_H 2 | #define BERT_QLISTMODULE_H 3 | 4 | #include 5 | #include "QString.h" 6 | #include "QCommon.h" 7 | 8 | namespace qedis 9 | { 10 | class UnboundedBuffer; 11 | } 12 | 13 | 14 | // ldel list_key item_index 15 | // For delete list item by index 16 | // 17 | extern "C" 18 | qedis::QError ldel(const std::vector& params, qedis::UnboundedBuffer* reply); 19 | 20 | #endif 21 | 22 | -------------------------------------------------------------------------------- /QedisCore/QString.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QSTRING_H 2 | #define BERT_QSTRING_H 3 | 4 | #include 5 | #include 6 | 7 | namespace qedis 8 | { 9 | 10 | using QString = std::string; 11 | 12 | //typedef std::basic_string, Bert::Allocator > QString; 13 | 14 | struct QObject; 15 | 16 | std::unique_ptr 17 | GetDecodedString(const QObject* value); 18 | 19 | } 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /cluster/cluster_conn/ClusterConn.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CLUSTERCONN_H 2 | #define BERT_CLUSTERCONN_H 3 | 4 | namespace qedis 5 | { 6 | 7 | class ClusterConn 8 | { 9 | public: 10 | virtual ~ClusterConn() 11 | { 12 | } 13 | 14 | public: 15 | virtual bool OnData(const char*& data, size_t len) = 0; 16 | virtual void OnConnect() = 0; 17 | virtual void OnDisconnect() = 0; 18 | }; 19 | 20 | } // end namespace qedis 21 | 22 | #endif // endif BERT_CLUSTERCONN_H 23 | 24 | -------------------------------------------------------------------------------- /Modules/QSetModule.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QSETMODULE_H 2 | #define BERT_QSETMODULE_H 3 | 4 | #include 5 | #include "QString.h" 6 | #include "QCommon.h" 7 | 8 | namespace qedis 9 | { 10 | class UnboundedBuffer; 11 | } 12 | 13 | 14 | // skeys set_key filter_pattern 15 | // for example, `skeys cities sh*` may return ["shanghai", "shenzhen"] 16 | // 17 | extern "C" 18 | qedis::QError skeys(const std::vector& params, qedis::UnboundedBuffer* reply); 19 | 20 | #endif 21 | 22 | -------------------------------------------------------------------------------- /cluster/ananas/net/AnanasDebug.cc: -------------------------------------------------------------------------------- 1 | #include "AnanasDebug.h" 2 | 3 | namespace ananas 4 | { 5 | 6 | namespace internal 7 | { 8 | 9 | std::shared_ptr g_debug; 10 | std::once_flag g_logInit; 11 | 12 | void InitDebugLog(unsigned int level) 13 | { 14 | std::call_once(g_logInit, [level]() { 15 | g_debug = LogManager::Instance().CreateLog(level, logFile, "ananas_debug_log"); 16 | }); 17 | } 18 | 19 | } // end namespace internal 20 | 21 | } // end namespace ananas 22 | 23 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 2 | 3 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}) 4 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/ananas) 5 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/cluster_conn) 6 | 7 | AUX_SOURCE_DIRECTORY(. PROXY_SRC) 8 | ADD_EXECUTABLE(qedisproxy ${PROXY_SRC}) 9 | 10 | SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin/) 11 | 12 | TARGET_LINK_LIBRARIES(qedisproxy ananas;zk) 13 | 14 | ADD_DEPENDENCIES(qedisproxy ananas; zk) 15 | 16 | -------------------------------------------------------------------------------- /UnitTest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 2 | 3 | AUX_SOURCE_DIRECTORY(. UNITTEST_SRC) 4 | 5 | LINK_DIRECTORIES(../../leveldb) 6 | LINK_DIRECTORIES(${PROJECT_SOURCE_DIR}/QedisCore) 7 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QedisCore) 8 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QBase) 9 | 10 | ADD_EXECUTABLE(qunittest ${UNITTEST_SRC}) 11 | SET(EXECUTABLE_OUTPUT_PATH ../../bin) 12 | TARGET_LINK_LIBRARIES(qunittest qediscore; leveldb) 13 | ADD_DEPENDENCIES(qunittest qediscore) 14 | -------------------------------------------------------------------------------- /cluster/cluster_conn/zookeeper/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 4 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/ananas) 5 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/zk) 6 | 7 | AUX_SOURCE_DIRECTORY(. ZK_SRC) 8 | 9 | SET(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) 10 | ADD_LIBRARY(zk ${ZK_SRC}) 11 | TARGET_LINK_LIBRARIES(zk; ananas; pthread) 12 | ADD_DEPENDENCIES(zk; ananas) 13 | 14 | SET_TARGET_PROPERTIES(zk PROPERTIES LINKER_LANGUAGE CXX) 15 | -------------------------------------------------------------------------------- /QBase/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 4 | 5 | AUX_SOURCE_DIRECTORY(. SDK_SRC) 6 | AUX_SOURCE_DIRECTORY(./Log SDK_SRC) 7 | AUX_SOURCE_DIRECTORY(./Threads SDK_SRC) 8 | AUX_SOURCE_DIRECTORY(./SmartPtr SDK_SRC) 9 | AUX_SOURCE_DIRECTORY(./lzf SDK_SRC) 10 | SET(LIBRARY_OUTPUT_PATH ../../bin) 11 | ADD_LIBRARY(qbaselib SHARED ${SDK_SRC}) 12 | 13 | TARGET_LINK_LIBRARIES(qbaselib; pthread) 14 | 15 | SET_TARGET_PROPERTIES(qbaselib PROPERTIES LINKER_LANGUAGE CXX) 16 | -------------------------------------------------------------------------------- /Modules/QHashModule.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QHASHMODULE_H 2 | #define BERT_QHASHMODULE_H 3 | 4 | #include 5 | #include "QString.h" 6 | #include "QCommon.h" 7 | 8 | namespace qedis 9 | { 10 | class UnboundedBuffer; 11 | } 12 | 13 | 14 | // hgets hash_key filter_pattern 15 | // for example, `hgets profile c*y` may return ["city":"shanghai", "country":"china"] 16 | // because c*y matches "city" and "country" 17 | // 18 | extern "C" 19 | qedis::QError hgets(const std::vector& params, qedis::UnboundedBuffer* reply); 20 | 21 | #endif 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | @if [ ! -d "leveldb" ]; then echo "leveldb not present. Fetching leveldb-1.18 from internet..."; curl -s -L -O https://github.com/google/leveldb/archive/v1.18.tar.gz; tar xzvf v1.18.tar.gz; rm -f v1.18.tar.gz; mv leveldb-1.18 leveldb; fi 3 | cd leveldb && make && cd - 4 | mkdir -p bin && ln -s -f ../leveldb/libleveldb.so.1.18 bin/libleveldb.so.1 5 | mkdir -p build && cd build && cmake .. && make 6 | 7 | clean: 8 | @if [ -d "build" ]; then cd build && make clean && cd -; fi 9 | cleanall:clean 10 | @if [ -d "leveldb" ]; then cd leveldb && make clean && cd -; rm -rf leveldb; fi 11 | -------------------------------------------------------------------------------- /QedisCore/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | AUX_SOURCE_DIRECTORY(. QEDIS_SRC) 2 | 3 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 4 | 5 | LINK_DIRECTORIES(../../leveldb) 6 | 7 | ADD_LIBRARY(qediscore SHARED ${QEDIS_SRC}) 8 | SET(LIBRARY_OUTPUT_PATH ../../bin) 9 | 10 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QBase) 11 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/) 12 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/leveldb/include) 13 | 14 | ADD_DEPENDENCIES(qediscore qbaselib leveldb) 15 | TARGET_LINK_LIBRARIES(qediscore; qbaselib; dl; leveldb) 16 | 17 | SET_TARGET_PROPERTIES(qediscore PROPERTIES LINKER_LANGUAGE CXX) 18 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ClientConn.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CLIENTCONN_H 2 | #define BERT_CLIENTCONN_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "net/Connection.h" 10 | #include "Protocol.h" 11 | 12 | class ClientConn final 13 | { 14 | public: 15 | explicit 16 | ClientConn(ananas::Connection* conn); 17 | 18 | ananas::PacketLen_t OnRecv(ananas::Connection* conn, const char* data, ananas::PacketLen_t len); 19 | 20 | private: 21 | ServerProtocol proto_; 22 | ananas::Connection* hostConn_; 23 | }; 24 | 25 | #endif 26 | 27 | -------------------------------------------------------------------------------- /QBase/Kqueue.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_KQUEUE_H 2 | #define BERT_KQUEUE_H 3 | 4 | #if defined(__APPLE__) 5 | 6 | #include "Poller.h" 7 | #include 8 | 9 | class Kqueue : public Poller 10 | { 11 | public: 12 | Kqueue(); 13 | ~Kqueue(); 14 | 15 | bool AddSocket(int sock, int events, void* userPtr); 16 | bool ModSocket(int sock, int events, void* userPtr); 17 | bool DelSocket(int sock, int events); 18 | 19 | int Poll(std::vector& events, std::size_t maxEvent, int timeoutMs); 20 | 21 | private: 22 | std::vector events_; 23 | }; 24 | 25 | #endif 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ProxyConfig.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_PROXYCONFIG_H 2 | #define BERT_PROXYCONFIG_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace qedis 9 | { 10 | 11 | struct ProxyConfig 12 | { 13 | std::string bindAddr; 14 | std::string logDir; 15 | std::vector clusters; 16 | 17 | static const std::string kProxyPrefixPath; 18 | static const std::string kQedisSetsPath; 19 | 20 | ProxyConfig(); 21 | }; 22 | 23 | extern ProxyConfig g_config; 24 | 25 | extern bool LoadProxyConfig(const char* cfgFile, ProxyConfig& cfg); 26 | 27 | } 28 | 29 | #endif 30 | 31 | -------------------------------------------------------------------------------- /QBase/EPoller.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_EPOLLER_H 2 | #define BERT_EPOLLER_H 3 | 4 | #if defined(__gnu_linux__) 5 | 6 | #include 7 | #include 8 | #include "Poller.h" 9 | 10 | class Epoller : public Poller 11 | { 12 | public: 13 | Epoller(); 14 | ~Epoller(); 15 | 16 | bool AddSocket(int sock, int events, void* userPtr); 17 | bool ModSocket(int sock, int events, void* userPtr); 18 | bool DelSocket(int sock, int events); 19 | 20 | int Poll(std::vector& events, std::size_t maxEvent, int timeoutMs); 21 | 22 | private: 23 | std::vector events_; 24 | }; 25 | 26 | #endif 27 | 28 | #endif 29 | 30 | -------------------------------------------------------------------------------- /QSentinel/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 4 | 5 | LINK_DIRECTORIES(../../leveldb) 6 | 7 | AUX_SOURCE_DIRECTORY(. CLUSTER_SRC) 8 | AUX_SOURCE_DIRECTORY(./zookeeper CLUSTER_SRC) 9 | 10 | SET(LIBRARY_OUTPUT_PATH ../../bin) 11 | ADD_LIBRARY(qcluster SHARED ${CLUSTER_SRC}) 12 | 13 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QBase) 14 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QedisCore) 15 | 16 | TARGET_LINK_LIBRARIES(qcluster; qbaselib; qediscore; leveldb; pthread) 17 | ADD_DEPENDENCIES(qcluster qediscore qbaselib leveldb) 18 | 19 | SET_TARGET_PROPERTIES(qcluster PROPERTIES LINKER_LANGUAGE CXX) 20 | -------------------------------------------------------------------------------- /Modules/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 2 | 3 | AUX_SOURCE_DIRECTORY(. QEDIS_MODULE_SRC) 4 | 5 | LINK_DIRECTORIES(../../leveldb) 6 | 7 | SET(LIBRARY_OUTPUT_PATH ../../bin) 8 | SET(QEDIS_MODULE qedismodule) 9 | 10 | ADD_LIBRARY(${QEDIS_MODULE} SHARED ${QEDIS_MODULE_SRC}) 11 | 12 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QedisCore) 13 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QBase) 14 | 15 | ADD_DEPENDENCIES(${QEDIS_MODULE} qediscore leveldb) 16 | # clang needs below, not necessary for gcc 17 | TARGET_LINK_LIBRARIES(${QEDIS_MODULE} qbaselib; qediscore; leveldb) 18 | 19 | SET_TARGET_PROPERTIES(${QEDIS_MODULE} PROPERTIES LINKER_LANGUAGE CXX) 20 | -------------------------------------------------------------------------------- /QedisSvr/QedisLogo.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QEDISLOGO_H 2 | #define BERT_QEDISLOGO_H 3 | 4 | const char* qedisLogo = "\n _____ _____ _____ _ _____ \n" 5 | " / _ \\ | ____| | _ \\ | | / ___/\n" 6 | " | | | | | |__ | | | | | | | |___ Qedis(%s) %d bits, another redis written in C++11\n" // version and server bits 7 | " | | | | | __| | | | | | | \\___ \\ Port: %d\n" 8 | " | |_| |_ | |___ | |_| | | | ___| | Author: Bert Young\n" 9 | " \\_______| |_____| |_____/ |_| /_____/ https://github.com/loveyacper/Qedis\n\n\n"; 10 | 11 | #endif 12 | 13 | -------------------------------------------------------------------------------- /QBase/ListenSocket.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_LISTENSOCKET_H 3 | #define BERT_LISTENSOCKET_H 4 | 5 | #include "Socket.h" 6 | 7 | namespace Internal 8 | { 9 | 10 | class ListenSocket : public Socket 11 | { 12 | static const int LISTENQ; 13 | public: 14 | explicit 15 | ListenSocket(int tag); 16 | ~ListenSocket(); 17 | 18 | SocketType GetSocketType() const { return SocketType_Listen; } 19 | 20 | bool Bind(const SocketAddr& addr); 21 | bool OnReadable(); 22 | bool OnWritable(); 23 | bool OnError(); 24 | 25 | private: 26 | int _Accept(); 27 | sockaddr_in addrClient_; 28 | unsigned short localPort_; 29 | const int tag_; 30 | }; 31 | 32 | } 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /QedisSvr/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #PROJECT(QEDISSERVER) 2 | 3 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 4 | 5 | AUX_SOURCE_DIRECTORY(. QEDISSERVER_SRC) 6 | 7 | LINK_DIRECTORIES(../../leveldb) 8 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QBase; ${PROJECT_SOURCE_DIR}/QedisCore) 9 | 10 | ADD_EXECUTABLE(qedis_server ${QEDISSERVER_SRC}) 11 | SET(EXECUTABLE_OUTPUT_PATH ../../bin) 12 | 13 | TARGET_LINK_LIBRARIES(qedis_server qediscore; qbaselib; leveldb) 14 | ADD_DEPENDENCIES(qedis_server qbaselib; qediscore; leveldb) 15 | IF(${QEDIS_CLUSTER} EQUAL 1) 16 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/QSentinel) 17 | TARGET_LINK_LIBRARIES(qedis_server qcluster) 18 | ADD_DEPENDENCIES(qedis_server qcluster) 19 | ENDIF() 20 | 21 | -------------------------------------------------------------------------------- /cluster/ananas/net/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8) 2 | 3 | INCLUDE(${PROJECT_SOURCE_DIR}/CMakeCommon) 4 | 5 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}) 6 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/ananas) 7 | 8 | AUX_SOURCE_DIRECTORY(. ANANAS_SRC) 9 | AUX_SOURCE_DIRECTORY(./log ANANAS_SRC) 10 | SET(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) 11 | 12 | IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") 13 | REMOVE(${ANANAS_SRC} Kqueue.cc Kqueue.h) 14 | ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") 15 | REMOVE(${ANANAS_SRC} Epoller.cc Epoller.h) 16 | ENDIF() 17 | 18 | 19 | ADD_LIBRARY(ananas ${ANANAS_SRC}) 20 | TARGET_LINK_LIBRARIES(ananas; pthread) 21 | SET_TARGET_PROPERTIES(ananas PROPERTIES LINKER_LANGUAGE CXX) 22 | -------------------------------------------------------------------------------- /CMakeCommon: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") 4 | SET(CMAKE_CXX_FLAGS_DEBUG "-g -Wall -std=c++0x") 5 | SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -Wall -std=c++0x") 6 | ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") 7 | SET(CMAKE_CXX_FLAGS_DEBUG "-g -Wall -std=c++1y -stdlib=libc++ -Wc++11-extensions") 8 | SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -Wall -std=c++1y -stdlib=libc++ -Wc++11-extensions") 9 | ELSE() 10 | message(FATAL_ERROR "Only support linux or OS X, support for windows is in plan") 11 | ENDIF() 12 | 13 | OPTION(DEBUG "Debug or release" OFF) 14 | 15 | IF(DEBUG) 16 | SET(CMAKE_BUILD_TYPE "Debug") 17 | ELSE() 18 | SET(CMAKE_BUILD_TYPE "Release") 19 | ENDIF() 20 | 21 | -------------------------------------------------------------------------------- /QBase/ClientSocket.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_CLIENTSOCKET_H 3 | #define BERT_CLIENTSOCKET_H 4 | 5 | #include 6 | #include "Socket.h" 7 | 8 | // Abstraction for a TCP client socket 9 | class ClientSocket : public Socket 10 | { 11 | public: 12 | explicit 13 | ClientSocket(int tag); 14 | ~ClientSocket(); 15 | bool Connect(const SocketAddr& addr); 16 | bool OnWritable(); 17 | bool OnError(); 18 | SocketType GetSockType() const { return SocketType_Client; } 19 | 20 | void SetFailCallback(const std::function& cb) { onConnectFail_ = cb; } 21 | 22 | private: 23 | const int tag_; 24 | SocketAddr peerAddr_; 25 | std::function onConnectFail_; 26 | }; 27 | 28 | #endif 29 | 30 | -------------------------------------------------------------------------------- /cluster/ananas/net/log/README.md: -------------------------------------------------------------------------------- 1 | # Multi-thread log 2 | # 多线程日志库 3 | 4 | ## Requirements 5 | * C++11 6 | * Linux or MAC OS 7 | 8 | ## High Performance 9 | O2优化, 10 | 11 | 单线程写100万条日志600ms左右,多线程速度可提高到每100万日志300-400ms左右 12 | 13 | 即每秒300w+ 14 | 15 | 但线程过多(`超过CPU个数`)可能会退化到单线程的水平. 16 | 17 | ## High Reliability 18 | 19 | 在进程关闭时候，IO线程退出前，同步等待生产者线程完成. 20 | 21 | 并且此时将不能再创建新的logger对象，确保迅速收敛. 22 | 23 | 生产者线程在日志关闭后写日志将得到提示并输出到屏幕. 24 | 25 | 这一现象并不应该发生，只有在全局对象的析构函数调用日志可能会触发. 26 | 27 | 由于C++不保证不同编译单元全局对象的构造和析构顺序，在析构函数中做过多的 28 | 29 | 工作似乎也不是合理的。 30 | 31 | ## Rationale 32 | 33 | 每个生产者线程在格式化日志数据的时候是不需要锁的，因为访问的是thread-local数据. 34 | 35 | 每个生产者线程有独立的输出buffer，通过thread id来索引. 36 | 37 | 当输出buffer超过阈值，主动唤醒IO线程 38 | 39 | 唯一的IO线程，将buffer输出到mmap的日志文件中. 40 | 41 | -------------------------------------------------------------------------------- /QSentinel/QClusterClient.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QCLUSTERCLIENT_H 2 | #define BERT_QCLUSTERCLIENT_H 3 | 4 | #if QEDIS_CLUSTER 5 | 6 | #include "StreamSocket.h" 7 | #include "QClusterInterface.h" 8 | 9 | namespace ConnectionTag 10 | { 11 | const int kSentinelClient = 2; 12 | } 13 | 14 | 15 | namespace qedis 16 | { 17 | 18 | class QClusterClient: public StreamSocket 19 | { 20 | public: 21 | void OnConnect() override; 22 | void OnDisconnect() override; 23 | bool Init(int fd, const SocketAddr& peer); 24 | 25 | private: 26 | PacketLength _HandlePacket(const char*, std::size_t) override; 27 | 28 | std::unique_ptr conn_; 29 | }; 30 | 31 | } 32 | 33 | #endif // endif QEDIS_CLUSTER 34 | 35 | #endif // endif BERT_QCLUSTERCLIENT_H 36 | 37 | -------------------------------------------------------------------------------- /cluster/CMakeCommon: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") 4 | SET(CMAKE_CXX_FLAGS_DEBUG "-g -Wall -std=c++0x -lpthread") 5 | SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -Wall -std=c++0x -lpthread") 6 | ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") 7 | SET(CMAKE_CXX_FLAGS_DEBUG "-g -Wall -std=c++1y -stdlib=libc++ -Wc++11-extensions") 8 | SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -Wall -std=c++1y -stdlib=libc++ -Wc++11-extensions") 9 | ADD_DEFINITIONS(-Dthread_local=__thread) 10 | ELSE() 11 | message(FATAL_ERROR "Only support linux or OS X") 12 | ENDIF() 13 | 14 | OPTION(DEBUG "Debug or release" ON) 15 | 16 | IF(DEBUG) 17 | SET(CMAKE_BUILD_TYPE "Debug") 18 | ELSE() 19 | SET(CMAKE_BUILD_TYPE "Release") 20 | ENDIF() 21 | 22 | -------------------------------------------------------------------------------- /QedisCore/QDumpInterface.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QDUMPINTERFACE_H 2 | #define BERT_QDUMPINTERFACE_H 3 | 4 | #include 5 | #include "QString.h" 6 | 7 | namespace qedis 8 | { 9 | 10 | struct QObject; 11 | 12 | class QDumpInterface 13 | { 14 | public: 15 | virtual ~QDumpInterface() {} 16 | 17 | virtual QObject Get(const QString& key) = 0; 18 | virtual bool Put(const QString& key, const QObject& obj, int64_t ttl = 0) = 0; 19 | virtual bool Put(const QString& key) = 0; 20 | virtual bool Delete(const QString& key) = 0; 21 | 22 | //std::vector MultiGet(const QString& key); 23 | //bool MultiPut(const QString& key, const QObject& obj, int64_t ttl = 0); 24 | //SaveAllRedisTolevelDb(); 25 | //LoadAllLeveldbToRedis(); 26 | }; 27 | 28 | } 29 | 30 | #endif 31 | 32 | -------------------------------------------------------------------------------- /cluster/ananas/net/Kqueue.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_KQUEUE_H 2 | #define BERT_KQUEUE_H 3 | 4 | #if defined(__APPLE__) 5 | 6 | #include 7 | #include 8 | #include "Poller.h" 9 | 10 | namespace ananas 11 | { 12 | namespace internal 13 | { 14 | 15 | class Kqueue : public Poller 16 | { 17 | public: 18 | Kqueue(); 19 | ~Kqueue(); 20 | 21 | bool Register(int fd, int events, void* userPtr) override; 22 | bool Modify(int fd, int events, void* userPtr) override; 23 | bool Unregister(int fd, int events) override; 24 | 25 | int Poll(std::size_t maxEvent, int timeoutMs) override; 26 | 27 | private: 28 | std::vector events_; 29 | }; 30 | 31 | } // end namespace internal 32 | } // end namespace ananas 33 | 34 | #endif // end #if defined(__APPLE__) 35 | 36 | #endif 37 | 38 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ClientManager.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CLIENTMANAGER_H 2 | #define BERT_CLIENTMANAGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace ananas 9 | { 10 | class EventLoop; 11 | class Connection; 12 | } 13 | 14 | class ClientManager 15 | { 16 | public: 17 | static ClientManager& Instance(); 18 | 19 | ClientManager(); 20 | 21 | void SetEventLoop(ananas::EventLoop* loop); 22 | bool Listen(const std::string& addr); 23 | 24 | void OnNewConnection(ananas::Connection* conn); 25 | void OnConnect(ananas::Connection* conn); 26 | void OnDisconnect(ananas::Connection* conn); 27 | 28 | private: 29 | //std::unordered_map connMap_; 30 | 31 | ananas::EventLoop* loop_; 32 | bool listening_; 33 | }; 34 | 35 | #endif 36 | 37 | -------------------------------------------------------------------------------- /QBase/AsyncBuffer.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ASYNCBUFFER_H 2 | #define BERT_ASYNCBUFFER_H 3 | 4 | #include 5 | #include 6 | 7 | #include "Buffer.h" 8 | #include "UnboundedBuffer.h" 9 | 10 | class AsyncBuffer 11 | { 12 | public: 13 | explicit 14 | AsyncBuffer(std::size_t size = 128 * 1024); 15 | ~AsyncBuffer(); 16 | 17 | void Write(const void* data, std::size_t len); 18 | void Write(const BufferSequence& data); 19 | 20 | void ProcessBuffer(BufferSequence& data); 21 | void Skip(std::size_t size); 22 | 23 | private: 24 | // for async write 25 | Buffer buffer_; 26 | 27 | // double buffer 28 | qedis::UnboundedBuffer tmpBuf_; 29 | 30 | std::mutex backBufLock_; 31 | std::atomic backBytes_; 32 | qedis::UnboundedBuffer backBuf_; 33 | }; 34 | 35 | #endif 36 | 37 | -------------------------------------------------------------------------------- /cluster/ananas/net/Epoller.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_EPOLLER_H 2 | #define BERT_EPOLLER_H 3 | 4 | #ifdef __gnu_linux__ 5 | 6 | #include 7 | #include 8 | #include "Poller.h" 9 | 10 | namespace ananas 11 | { 12 | namespace internal 13 | { 14 | 15 | class Epoller : public Poller 16 | { 17 | public: 18 | Epoller(); 19 | ~Epoller(); 20 | 21 | Epoller(const Epoller& ) = delete; 22 | void operator= (const Epoller& ) = delete; 23 | 24 | bool Register(int fd, int events, void* userPtr) override; 25 | bool Modify(int fd, int events, void* userPtr) override; 26 | bool Unregister(int fd, int events) override; 27 | 28 | int Poll(std::size_t maxEvent, int timeoutMs) override; 29 | 30 | private: 31 | std::vector events_; 32 | }; 33 | 34 | } // end namespace internal 35 | } // end namespace ananas 36 | 37 | #endif // end #ifdef __gnu_linux__ 38 | 39 | #endif 40 | 41 | -------------------------------------------------------------------------------- /QBase/Poller.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_POLLER_H 2 | #define BERT_POLLER_H 3 | 4 | #include 5 | 6 | enum EventType 7 | { 8 | EventTypeRead = 0x1, 9 | EventTypeWrite = 0x1 << 1, 10 | EventTypeError = 0x1 << 2, 11 | }; 12 | 13 | struct FiredEvent 14 | { 15 | int events; 16 | void* userdata; 17 | 18 | FiredEvent() : events(0), userdata(0) 19 | { 20 | } 21 | }; 22 | 23 | class Poller 24 | { 25 | public: 26 | Poller() : multiplexer_(-1) 27 | { 28 | } 29 | 30 | virtual ~Poller() 31 | { 32 | } 33 | 34 | virtual bool AddSocket(int sock, int events, void* userPtr) = 0; 35 | virtual bool ModSocket(int sock, int events, void* userPtr) = 0; 36 | virtual bool DelSocket(int sock, int events) = 0; 37 | 38 | virtual int Poll(std::vector& events, std::size_t maxEv, int timeoutMs) = 0; 39 | 40 | protected: 41 | int multiplexer_; 42 | }; 43 | 44 | #endif 45 | 46 | -------------------------------------------------------------------------------- /QedisCore/QMulti.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QMULTI_H 2 | #define BERT_QMULTI_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "QString.h" 8 | 9 | namespace qedis 10 | { 11 | 12 | class QClient; 13 | class QMulti 14 | { 15 | public: 16 | static QMulti& Instance(); 17 | 18 | QMulti(const QMulti& ) = delete; 19 | void operator= (const QMulti& ) = delete; 20 | 21 | void Watch(QClient* client, int dbno, const QString& key); 22 | bool Multi(QClient* client); 23 | bool Exec(QClient* client); 24 | void Discard(QClient* client); 25 | 26 | void NotifyDirty(int dbno, const QString& key); 27 | void NotifyDirtyAll(int dbno); 28 | private: 29 | QMulti() {} 30 | 31 | using Clients = std::vector >; 32 | using WatchedClients = std::map >; 33 | 34 | WatchedClients clients_; 35 | }; 36 | 37 | } 38 | 39 | #endif 40 | 41 | -------------------------------------------------------------------------------- /Modules/QHashModule.cc: -------------------------------------------------------------------------------- 1 | #include "QHashModule.h" 2 | #include "QHash.h" 3 | #include "QStore.h" 4 | #include "QGlobRegex.h" 5 | 6 | using namespace qedis; 7 | 8 | QError hgets(const std::vector& params, UnboundedBuffer* reply) 9 | { 10 | QObject* value; 11 | QError err = QSTORE.GetValueByType(params[1], value, QType_hash); 12 | if (err != QError_ok) 13 | { 14 | ReplyError(err, reply); 15 | return err; 16 | } 17 | 18 | const PHASH& hash= value->CastHash(); 19 | std::vector res; 20 | for (const auto& kv : *hash) 21 | { 22 | if (glob_match(params[2], kv.first)) 23 | { 24 | res.push_back(&kv.first); 25 | res.push_back(&kv.second); 26 | } 27 | } 28 | 29 | PreFormatMultiBulk(res.size(), reply); 30 | for (auto v : res) 31 | { 32 | FormatBulk(*v, reply); 33 | } 34 | 35 | return QError_ok; 36 | } 37 | 38 | -------------------------------------------------------------------------------- /Modules/QSetModule.cc: -------------------------------------------------------------------------------- 1 | #include "QSetModule.h" 2 | #include "QSet.h" 3 | #include "QStore.h" 4 | #include "QGlobRegex.h" 5 | 6 | using namespace qedis; 7 | 8 | QError skeys(const std::vector& params, UnboundedBuffer* reply) 9 | { 10 | QObject* value; 11 | QError err = QSTORE.GetValueByType(params[1], value, QType_set); 12 | if (err != QError_ok) 13 | { 14 | if (err == QError_notExist) 15 | FormatNull(reply); 16 | else 17 | ReplyError(err, reply); 18 | 19 | return err; 20 | } 21 | 22 | std::vector res; 23 | const PSET& set = value->CastSet(); 24 | for (const auto& k : *set) 25 | { 26 | if (glob_match(params[2], k)) 27 | { 28 | res.push_back(&k); 29 | } 30 | } 31 | 32 | PreFormatMultiBulk(res.size(), reply); 33 | for (auto v : res) 34 | { 35 | FormatBulk(*v, reply); 36 | } 37 | 38 | return QError_ok; 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /QedisSvr/Qedis.h: -------------------------------------------------------------------------------- 1 | // 2 | // qedis.h 3 | // 4 | // Created by Bert Young on 16-1-22. 5 | // Copyright (c) 2016年 Bert Young. All rights reserved. 6 | // 7 | 8 | #include "QString.h" 9 | #include "Server.h" 10 | 11 | #define QEDIS_VERSION "1.0.0" 12 | 13 | class Qedis : public Server 14 | { 15 | public: 16 | Qedis(); 17 | ~Qedis(); 18 | 19 | bool ParseArgs(int ac, char* av[]); 20 | const qedis::QString& GetConfigName() const { return cfgFile_; } 21 | 22 | private: 23 | std::shared_ptr _OnNewConnection(int fd, int tag) override; 24 | bool _Init() override; 25 | bool _RunLogic() override; 26 | void _Recycle() override; 27 | 28 | qedis::QString cfgFile_; 29 | unsigned short port_; 30 | qedis::QString logLevel_; 31 | 32 | qedis::QString master_; 33 | unsigned short masterPort_; 34 | 35 | #if QEDIS_CLUSTER 36 | // cluster 37 | size_t clusterIndex_ = 0; 38 | #endif 39 | 40 | static const unsigned kRunidSize; 41 | }; 42 | -------------------------------------------------------------------------------- /cluster/ananas/net/Typedefs.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_TYPEDEFS_H 2 | #define BERT_TYPEDEFS_H 3 | 4 | #include 5 | 6 | namespace ananas 7 | { 8 | typedef size_t PacketLen_t; 9 | 10 | struct SocketAddr; 11 | class Connection; 12 | class DatagramSocket; 13 | class EventLoop; 14 | 15 | using NewTcpConnCallback = std::function; 16 | using TcpConnFailCallback = std::function; 17 | using TcpMessageCallback = std::function; 18 | using TcpWriteCompleteCallback = std::function; 19 | using TcpWriteHighWaterCallback = std::function; 20 | 21 | using UDPMessageCallback = std::function; 22 | using UDPCreateCallback = std::function; 23 | 24 | using SocketPairCreateCallback = std::function; 25 | } 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /QedisCore/QSlaveClient.cc: -------------------------------------------------------------------------------- 1 | #include "QSlaveClient.h" 2 | #include "QConfig.h" 3 | #include "QCommon.h" 4 | #include "Log/Logger.h" 5 | 6 | namespace qedis 7 | { 8 | 9 | void QSlaveClient::OnConnect() 10 | { 11 | std::string cmd = BuildInlineRequest("slaveof ", g_config.ip, std::to_string(g_config.port)); 12 | INF << "Send to slave cmd " << cmd; 13 | 14 | SendPacket(cmd.data(), cmd.size()); 15 | 16 | auto wk = std::weak_ptr(std::static_pointer_cast(this->shared_from_this())); 17 | Timer* timer = TimerManager::Instance().CreateTimer(); 18 | timer->Init(3 * 1000, 1); 19 | timer->SetCallback([wk]() { 20 | auto me = wk.lock(); 21 | if (me) { 22 | USR << "OnTimer close " << me->GetPeerAddr().ToString(); 23 | me->OnError(); 24 | } 25 | }); 26 | TimerManager::Instance().AsyncAddTimer(timer); 27 | } 28 | 29 | PacketLength QSlaveClient::_HandlePacket(const char* msg, std::size_t len) 30 | { 31 | return static_cast(len); 32 | } 33 | 34 | } 35 | 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Bert Young 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cluster/ananas/util/Scheduler.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_SCHEDULER_H 2 | #define BERT_SCHEDULER_H 3 | 4 | namespace ananas 5 | { 6 | 7 | class Scheduler 8 | { 9 | public: 10 | virtual ~Scheduler() {} 11 | 12 | /** 13 | * The following functions need not to be thread-safe. 14 | * It is nonsense that schedule callback and submit request are in different threads. 15 | * Do it like this: 16 | * e.g. 17 | * 18 | * // In this_loop thread. 19 | * 20 | * Future ft(ReadFileInSeparateThread(very_big_file)); 21 | * 22 | * ft.Then(&this_loop, [](const Buffer& file_contents) { 23 | * // SUCCESS : process file_content; 24 | * }) 25 | * .OnTimeout(std::chrono::seconds(10), [=very_big_file]() { 26 | * // FAILED OR TIMEOUT: 27 | * printf("Read file %s failed\n", very_big_file); 28 | * }, 29 | * &this_loop); 30 | */ 31 | virtual void ScheduleOnceAfter(std::chrono::milliseconds duration, std::function f) = 0; 32 | virtual void ScheduleOnce(std::function f) = 0; 33 | }; 34 | 35 | } // end namespace ananas 36 | 37 | #endif 38 | 39 | -------------------------------------------------------------------------------- /QSentinel/README.md: -------------------------------------------------------------------------------- 1 | # Qedis高可用集群 2 | 3 | 利用Zookeeper实现的Qedis高可用集群，功能类似redis-sentinel。在此基础上可实现sacle-out集群。 4 | 5 | ## 环境需求 6 | * C++11、CMake 7 | * zookeeper 8 | * Linux 或 MAC OS 9 | 10 | ## 集群特性 11 | 搭建Zookeeper，监视一组互为主备的Qedis进程以实现高可用; 12 | 13 | 当然也可以使用官方redis-sentinel。 14 | 15 | scale-out集群正在开发中... 16 | 17 | ## 简单原理 18 | 19 | /servers 20 | /set-1 21 | /qedis-(127.0.0.1:6379)-0001 22 | /qedis-(127.0.0.1:6381)-0003 23 | /qedis-(127.0.0.1:6382)-0004 24 | /qedis-(127.0.0.1:6385)-0007 25 | /set-2 26 | /qedis-(127.0.0.1:16379)-0004 27 | /qedis-(127.0.0.1:16389)-0007 28 | /qedis-(127.0.0.1:16399)-0008 29 | 30 | 一组Qedis进程形成一个set，set内最多只有一个master，其它都是slave，且没有级联复制结构。 31 | 32 | 通过配置文件中setid来配置set，相同setid的Qedis进程将形成一个set。 33 | 34 | 通过设置配置文件中cluster开关，Qedis在启动时，将尝试向Zookeeper的/servers/set-{id}/下创建自己的临时顺序节点。 35 | 36 | 创建成功后，获取孩子列表，看自己的节点序号是不是最小。 37 | 38 | 如果是最小，则是master，向所有孩子发送slaveof my_addr的命令； 39 | 40 | 如果不是，则监视比自己序号大的孩子中序号最小的节点。比如我的序号是7，孩子列表序号是1，3，4，7，则我监视4节点。 41 | 42 | 当我收到监视的节点被删除的通知，则判断自己是否是master（因为启动时已经获得孩子列表了）。 43 | 44 | 是或不是，都继续重复上述过的逻辑。 45 | -------------------------------------------------------------------------------- /QedisCore/QProtoParser.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QPROTOPARSER_H 2 | #define BERT_QPROTOPARSER_H 3 | 4 | #include 5 | #include "QString.h" 6 | 7 | namespace qedis 8 | { 9 | 10 | class QProtoParser 11 | { 12 | public: 13 | void Reset(); 14 | QParseResult ParseRequest(const char*& ptr, const char* end); 15 | 16 | const std::vector& GetParams() const { return params_; } 17 | void SetParams(std::vector p) { params_ = std::move(p); } 18 | 19 | bool IsInitialState() const { return multi_ == -1; } 20 | 21 | private: 22 | QParseResult _ParseMulti(const char*& ptr, const char* end, int& result); 23 | QParseResult _ParseStrlist(const char*& ptr, const char* end, std::vector& results); 24 | QParseResult _ParseStr(const char*& ptr, const char* end, QString& result); 25 | QParseResult _ParseStrval(const char*& ptr, const char* end, QString& result); 26 | QParseResult _ParseStrlen(const char*& ptr, const char* end, int& result); 27 | 28 | int multi_ = -1; 29 | int paramLen_ = -1; 30 | 31 | size_t numOfParam_ = 0; // for optimize 32 | std::vector params_; 33 | }; 34 | 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /cluster/ananas/net/Acceptor.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_ACCEPTOR_H 3 | #define BERT_ACCEPTOR_H 4 | 5 | #include "Socket.h" 6 | #include "Typedefs.h" 7 | 8 | namespace ananas 9 | { 10 | namespace internal 11 | { 12 | 13 | class Acceptor : public EventSource 14 | { 15 | public: 16 | explicit 17 | Acceptor(EventLoop* loop); 18 | ~Acceptor(); 19 | 20 | Acceptor(const Acceptor& ) = delete; 21 | void operator= (const Acceptor& ) = delete; 22 | 23 | void SetNewConnCallback(NewTcpConnCallback cb); 24 | bool Bind(const SocketAddr& addr); 25 | 26 | int Identifier() const override; 27 | bool HandleReadEvent() override; 28 | bool HandleWriteEvent() override; 29 | void HandleErrorEvent() override; 30 | 31 | private: 32 | int _Accept(); 33 | 34 | SocketAddr peer_; 35 | int localSock_; 36 | uint16_t localPort_; 37 | 38 | EventLoop* const loop_; // which loop belong to 39 | 40 | //register msg callback and on connect callback for conn 41 | NewTcpConnCallback newConnCallback_; 42 | 43 | static const int kListenQueue; 44 | }; 45 | 46 | } // end namespace internal 47 | } // end namespace ananas 48 | 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /QBase/TaskManager.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_TASKMANAGER_H 2 | #define BERT_TASKMANAGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | class StreamSocket; 11 | 12 | namespace Internal 13 | { 14 | 15 | class TaskManager 16 | { 17 | typedef std::shared_ptr PTCPSOCKET; 18 | typedef std::vector NEWTASKS_T; 19 | 20 | public: 21 | TaskManager() : newCnt_(0) { } 22 | ~TaskManager(); 23 | 24 | bool AddTask(PTCPSOCKET ); 25 | 26 | bool Empty() const { return tcpSockets_.empty(); } 27 | void Clear() { tcpSockets_.clear(); } 28 | PTCPSOCKET FindTCP(unsigned int id) const; 29 | 30 | size_t TCPSize() const { return tcpSockets_.size(); } 31 | 32 | bool DoMsgParse(); 33 | 34 | private: 35 | bool _AddTask(PTCPSOCKET task); 36 | void _RemoveTask(std::map::iterator& ); 37 | std::map tcpSockets_; 38 | 39 | // Lock for new tasks 40 | std::mutex lock_; 41 | NEWTASKS_T newTasks_; 42 | std::atomic newCnt_; // vector::empty() is not thread-safe !!! 43 | }; 44 | 45 | } 46 | 47 | #endif 48 | 49 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ProxyConfig.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "ProxyConfig.h" 5 | #include "ConfigParser.h" 6 | 7 | namespace qedis 8 | { 9 | 10 | static void EraseQuotes(std::string& str) 11 | { 12 | // convert "hello" to hello 13 | if (str.size() < 2) 14 | return; 15 | 16 | if (str[0] == '"' && str[str.size() - 1] == '"') 17 | { 18 | str.pop_back(); 19 | str.erase(str.begin()); 20 | } 21 | } 22 | 23 | const std::string ProxyConfig::kProxyPrefixPath = "/proxy/qedis_proxy_"; 24 | const std::string ProxyConfig::kQedisSetsPath = "/servers"; 25 | 26 | ProxyConfig g_config; 27 | 28 | ProxyConfig::ProxyConfig() : 29 | bindAddr("127.0.0.1:6379"), 30 | logDir("log_proxy"), 31 | clusters({"127.0.0.1:2181"}) 32 | { 33 | } 34 | 35 | bool LoadProxyConfig(const char* cfgFile, ProxyConfig& cfg) 36 | { 37 | ConfigParser parser; 38 | if (!parser.Load(cfgFile)) 39 | return false; 40 | 41 | cfg.bindAddr = parser.GetData("bind", cfg.bindAddr); 42 | cfg.logDir = parser.GetData("logdir", cfg.logDir); 43 | 44 | return true; 45 | } 46 | 47 | } // end namespace qedis 48 | 49 | -------------------------------------------------------------------------------- /cluster/ananas/net/log/MmapFile.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_MMAPFILE_H 2 | #define BERT_MMAPFILE_H 3 | 4 | #include 5 | 6 | namespace ananas 7 | { 8 | namespace internal 9 | { 10 | 11 | class OMmapFile 12 | { 13 | public: 14 | OMmapFile(); 15 | ~OMmapFile(); 16 | 17 | bool Open(const std::string& file, bool bAppend = true); 18 | bool Open(const char* file, bool bAppend = true); 19 | void Close(); 20 | bool Sync(); 21 | 22 | void Truncate(std::size_t size); 23 | 24 | void Write(const void* data, std::size_t len); 25 | template 26 | void Write(const T& t); 27 | 28 | std::size_t Offset() const { return offset_; } 29 | bool IsOpen() const; 30 | 31 | private: 32 | bool _MapWriteOnly(); 33 | void _ExtendFileSize(std::size_t size); 34 | void _AssureSpace(std::size_t size); 35 | 36 | int file_; 37 | char* memory_; 38 | std::size_t offset_; 39 | std::size_t size_; 40 | std::size_t syncPos_; 41 | }; 42 | 43 | 44 | template 45 | inline void OMmapFile::Write(const T& t) 46 | { 47 | this->Write(&t, sizeof t); 48 | } 49 | 50 | } // end namespace internal 51 | 52 | } // end namespace ananas 53 | 54 | #endif 55 | 56 | -------------------------------------------------------------------------------- /QSentinel/zookeeper/zookeeper_version.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | #ifndef ZOOKEEPER_VERSION_H_ 19 | #define ZOOKEEPER_VERSION_H_ 20 | 21 | #ifdef __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | #define ZOO_MAJOR_VERSION 3 26 | #define ZOO_MINOR_VERSION 4 27 | #define ZOO_PATCH_VERSION 6 28 | 29 | #ifdef __cplusplus 30 | } 31 | #endif 32 | 33 | #endif /* ZOOKEEPER_VERSION_H_ */ 34 | -------------------------------------------------------------------------------- /QSentinel/QClusterInterface.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CLUSTERINTERFACE_H 2 | #define BERT_CLUSTERINTERFACE_H 3 | 4 | #if QEDIS_CLUSTER 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "Socket.h" 11 | 12 | namespace ConnectionTag 13 | { 14 | const int kSlaveClient = 3; 15 | } 16 | 17 | namespace qedis 18 | { 19 | 20 | class QClusterConn 21 | { 22 | public: 23 | virtual ~QClusterConn() 24 | { 25 | } 26 | 27 | void SetOnBecomeMaster(std::function& )> cb) 28 | { 29 | onBecomeMaster_ = std::move(cb); 30 | } 31 | 32 | void SetOnBecomeSlave(std::function cb) 33 | { 34 | onBecomeSlave_ = std::move(cb); 35 | } 36 | 37 | public: 38 | virtual bool ParseMessage(const char*& data, size_t len) = 0; 39 | virtual void OnConnect() = 0; 40 | virtual void OnDisconnect() = 0; 41 | 42 | protected: 43 | std::function& )> onBecomeMaster_; 44 | std::function onBecomeSlave_; 45 | 46 | }; 47 | 48 | } // end namespace qedis 49 | 50 | #endif // endif QEDIS_CLUSTER 51 | 52 | #endif // endif BERT_CLUSTERINTERFACE_H 53 | 54 | -------------------------------------------------------------------------------- /QedisCore/QSlowLog.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QSLOWLOG_H 2 | #define BERT_QSLOWLOG_H 3 | 4 | #include 5 | #include 6 | 7 | #include "QString.h" 8 | 9 | class Logger; 10 | 11 | namespace qedis 12 | { 13 | 14 | struct SlowLogItem 15 | { 16 | unsigned used; 17 | std::vector cmds; 18 | 19 | SlowLogItem() : used(0) 20 | { 21 | } 22 | 23 | SlowLogItem(SlowLogItem&& item) : used(item.used), cmds(std::move(item.cmds)) 24 | { 25 | } 26 | }; 27 | 28 | class QSlowLog 29 | { 30 | public: 31 | static QSlowLog& Instance(); 32 | 33 | QSlowLog(const QSlowLog& ) = delete; 34 | void operator= (const QSlowLog& ) = delete; 35 | 36 | void Begin(); 37 | void EndAndStat(const std::vector& cmds); 38 | 39 | void SetThreshold(unsigned int ); 40 | void SetLogLimit(std::size_t maxCount); 41 | 42 | void ClearLogs() { logs_.clear(); } 43 | std::size_t GetLogsCount() const { return logs_.size(); } 44 | const std::deque& GetLogs() const { return logs_; } 45 | 46 | private: 47 | QSlowLog(); 48 | ~QSlowLog(); 49 | 50 | unsigned int threshold_; 51 | long long beginUs_; 52 | Logger* logger_; 53 | 54 | std::size_t logMaxCount_; 55 | std::deque logs_; 56 | 57 | }; 58 | 59 | } 60 | 61 | #endif 62 | 63 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ClusterManager.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CLUSTERMANAGER_H 2 | #define BERT_CLUSTERMANAGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace ananas 9 | { 10 | class EventLoop; 11 | class Connection; 12 | struct SocketAddr; 13 | } 14 | 15 | class ClusterManager 16 | { 17 | public: 18 | static ClusterManager& Instance(); 19 | 20 | ClusterManager(); 21 | 22 | void SetEventLoop(ananas::EventLoop* loop); 23 | void AddClusterAddr(const std::string& host); 24 | bool Connect(); 25 | 26 | void OnNewConnection(ananas::Connection* conn); 27 | void OnConnect(ananas::Connection* conn); 28 | void OnDisconnect(ananas::Connection* conn); 29 | void OnConnFail(ananas::EventLoop* loop, const ananas::SocketAddr& peer); 30 | 31 | void AddShardingInfo(int setid, const std::vector& shardings); 32 | void AddServerInfo(int setid, const std::string& host); 33 | 34 | const std::string& GetServer(const std::string& key) const; 35 | 36 | private: 37 | int index_ = -1; 38 | std::vector cluster_; 39 | 40 | // sharding-id -> set-id 41 | std::unordered_map shardingInfo_; 42 | 43 | // set-id -> server-list 44 | std::unordered_map> hostInfo_; 45 | 46 | ananas::EventLoop* loop_; 47 | }; 48 | 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /cluster/ananas/net/TimeUtil.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_TIMEUTIL_H 3 | #define BERT_TIMEUTIL_H 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace ananas 11 | { 12 | 13 | class Time 14 | { 15 | public: 16 | Time(); 17 | //Time(const Time& other) = delete; 18 | //Time& operator= (const Time& ) = delete; 19 | 20 | void Now(); 21 | int64_t MilliSeconds() const; 22 | int64_t MicroSeconds() const; 23 | std::size_t FormatTime(char* buf) const; 24 | 25 | int GetYear() const { _UpdateTm(); return tm_.tm_year + 1900; } 26 | int GetMonth() const { _UpdateTm(); return tm_.tm_mon + 1; } 27 | int GetDay() const { _UpdateTm(); return tm_.tm_mday; } 28 | int GetHour() const { _UpdateTm(); return tm_.tm_hour; } 29 | int GetMinute() const { _UpdateTm(); return tm_.tm_min; } 30 | int GetSecond() const { _UpdateTm(); return tm_.tm_sec; } 31 | 32 | operator int64_t() const { return MilliSeconds(); } 33 | 34 | private: 35 | std::chrono::system_clock::time_point now_; 36 | mutable tm tm_; 37 | mutable bool valid_; 38 | 39 | void _UpdateTm() const; 40 | 41 | // for FormatTime 42 | static std::once_flag init_; 43 | static void Init(); 44 | static const char* YEAR[]; 45 | static char NUMBER[60][2]; 46 | }; 47 | 48 | } // end namespace ananas 49 | 50 | #endif 51 | 52 | -------------------------------------------------------------------------------- /Modules/QListModule.cc: -------------------------------------------------------------------------------- 1 | #include "QListModule.h" 2 | #include "QList.h" 3 | #include "QStore.h" 4 | #include 5 | 6 | using namespace qedis; 7 | 8 | 9 | QError ldel(const std::vector& params, UnboundedBuffer* reply) 10 | { 11 | QObject* value; 12 | 13 | QError err = QSTORE.GetValueByType(params[1], value, QType_list); 14 | if (err != QError_ok) 15 | { 16 | Format0(reply); 17 | return err; 18 | } 19 | 20 | long idx; 21 | if (!Strtol(params[2].c_str(), params[2].size(), &idx)) 22 | { 23 | ReplyError(QError_nan, reply); 24 | return QError_nan; 25 | } 26 | 27 | const PLIST& list = value->CastList(); 28 | const int size = static_cast(list->size()); 29 | if (idx < 0) 30 | idx += size; 31 | 32 | if (idx < 0 || idx >= size) 33 | { 34 | Format0(reply); 35 | return QError_nop; 36 | } 37 | 38 | if (2 * idx < size) 39 | { 40 | auto it = list->begin(); 41 | std::advance(it, idx); 42 | list->erase(it); 43 | } 44 | else 45 | { 46 | auto it = list->rbegin(); 47 | idx = size - 1 - idx; 48 | std::advance(it, idx); 49 | list->erase((++it).base()); 50 | } 51 | 52 | if (list->empty()) 53 | QSTORE.DeleteKey(params[1]); 54 | 55 | Format1(reply); 56 | return QError_ok; 57 | } 58 | 59 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "ProxyLog.h" 4 | #include "ProxyConfig.h" 5 | #include "ananas/net/EventLoop.h" 6 | #include "ClusterManager.h" 7 | #include "ClientManager.h" 8 | #include "QedisManager.h" 9 | #include "Command.h" 10 | 11 | std::shared_ptr g_logger; 12 | 13 | int main(int ac, char* av[]) 14 | { 15 | using namespace qedis; 16 | 17 | if (ac > 1) 18 | { 19 | if (!LoadProxyConfig(av[1], g_config)) 20 | { 21 | std::cout << "LoadProxyConfig " << av[1] << " failed!\n"; 22 | return -1; 23 | } 24 | } 25 | 26 | ananas::LogManager::Instance().Start(); 27 | g_logger = ananas::LogManager::Instance().CreateLog(logERROR, logALL, g_config.logDir.data()); 28 | 29 | CommandTable::Init(); 30 | 31 | for (const auto& addr : g_config.clusters) 32 | ClusterManager::Instance().AddClusterAddr(addr); 33 | 34 | ananas::EventLoop loop; 35 | ClusterManager::Instance().SetEventLoop(&loop); 36 | if (!ClusterManager::Instance().Connect()) 37 | ananas::EventLoop::ExitApplication(); 38 | 39 | ClientManager::Instance().SetEventLoop(&loop); 40 | if (!ClientManager::Instance().Listen(g_config.bindAddr)) 41 | ananas::EventLoop::ExitApplication(); 42 | 43 | QedisManager::Instance().SetEventLoop(&loop); 44 | 45 | loop.Run(); 46 | 47 | return 0; 48 | } 49 | 50 | -------------------------------------------------------------------------------- /cluster/ananas/net/Connector.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_CONNECTOR_H 3 | #define BERT_CONNECTOR_H 4 | 5 | #include "Socket.h" 6 | #include "Timer.h" 7 | #include "Typedefs.h" 8 | 9 | namespace ananas 10 | { 11 | namespace internal 12 | { 13 | 14 | enum class ConnectState 15 | { 16 | none, 17 | connecting, 18 | connected, 19 | failed, 20 | }; 21 | 22 | class Connector : public EventSource 23 | { 24 | public: 25 | explicit 26 | Connector(EventLoop* loop); 27 | ~Connector(); 28 | 29 | Connector(const Connector& ) = delete; 30 | void operator= (const Connector& ) = delete; 31 | 32 | void SetNewConnCallback(NewTcpConnCallback cb); 33 | void SetFailCallback(TcpConnFailCallback cb); 34 | bool Connect(const SocketAddr& addr, DurationMs timeout); 35 | 36 | int Identifier() const override; 37 | bool HandleReadEvent() override; 38 | bool HandleWriteEvent() override; 39 | void HandleErrorEvent() override; 40 | 41 | ConnectState State() const { return state_; } 42 | 43 | private: 44 | void _OnSuccess(); 45 | void _OnFailed(); 46 | 47 | int localSock_ = kInvalid; 48 | SocketAddr peer_; 49 | EventLoop* const loop_; 50 | 51 | ConnectState state_ = ConnectState::none; 52 | 53 | TimerId timeoutId_; 54 | 55 | TcpConnFailCallback onConnectFail_; 56 | NewTcpConnCallback newConnCallback_; 57 | }; 58 | 59 | } // end namespace internal 60 | } // end namespace ananas 61 | 62 | #endif 63 | 64 | -------------------------------------------------------------------------------- /QBase/UnboundedBuffer.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_UNBOUNDEDBUFFER_H 3 | #define BERT_UNBOUNDEDBUFFER_H 4 | 5 | #include 6 | #include 7 | 8 | namespace qedis 9 | { 10 | 11 | class UnboundedBuffer 12 | { 13 | public: 14 | UnboundedBuffer() : 15 | readPos_(0), 16 | writePos_(0) 17 | { 18 | } 19 | 20 | std::size_t PushDataAt(const void* pData, std::size_t nSize, std::size_t offset = 0); 21 | std::size_t PushData(const void* pData, std::size_t nSize); 22 | std::size_t Write(const void* pData, std::size_t nSize); 23 | void AdjustWritePtr(std::size_t nBytes) { writePos_ += nBytes; } 24 | 25 | std::size_t PeekDataAt(void* pBuf, std::size_t nSize, std::size_t offset = 0); 26 | std::size_t PeekData(void* pBuf, std::size_t nSize); 27 | void AdjustReadPtr(std::size_t nBytes) { readPos_ += nBytes; } 28 | 29 | char* ReadAddr() { return &buffer_[readPos_]; } 30 | char* WriteAddr() { return &buffer_[writePos_]; } 31 | 32 | bool IsEmpty() const { return ReadableSize() == 0; } 33 | std::size_t ReadableSize() const { return writePos_ - readPos_; } 34 | std::size_t WritableSize() const { return buffer_.size() - writePos_; } 35 | 36 | void Shrink(bool tight = false); 37 | void Clear(); 38 | void Swap(UnboundedBuffer& buf); 39 | 40 | static const std::size_t MAX_BUFFER_SIZE; 41 | private: 42 | void _AssureSpace(std::size_t size); 43 | std::size_t readPos_; 44 | std::size_t writePos_; 45 | std::vector buffer_; 46 | }; 47 | 48 | } 49 | 50 | #endif 51 | 52 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/Command.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_COMMAND_H 2 | #define BERT_COMMAND_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | enum CommandAttr 9 | { 10 | Attr_read = 0x1, 11 | Attr_write = 0x1 << 1, 12 | Attr_multikey = 0x1 << 2, 13 | }; 14 | 15 | 16 | using CommandHandler = std::string (const std::vector& params); 17 | 18 | // proxy commands 19 | CommandHandler ping; 20 | CommandHandler info; 21 | 22 | struct CommandInfo 23 | { 24 | std::string cmd; 25 | int attr; 26 | int params; 27 | CommandHandler* handler; 28 | 29 | CommandInfo(const std::string& c, int a, int p, CommandHandler* ch = nullptr) : 30 | cmd(c), 31 | attr(a), 32 | params(p), 33 | handler(ch) 34 | { 35 | } 36 | 37 | bool CheckParamsCount(int nParams) const; 38 | }; 39 | 40 | class CommandTable 41 | { 42 | public: 43 | static void Init(); 44 | static const CommandInfo* GetCommandInfo(const std::string& cmd); 45 | 46 | private: 47 | static const CommandInfo s_info[]; 48 | static std::map s_handlers; 49 | }; 50 | 51 | enum QedisError 52 | { 53 | QError_nop = -1, 54 | QError_ok = 0, 55 | QError_param = 1, 56 | QError_unknowCmd = 2, 57 | QError_syntax = 3, 58 | QError_notready = 4, 59 | QError_timeout = 5, 60 | QError_dead = 6, 61 | QError_max, 62 | }; 63 | 64 | extern struct QedisErrorInfo 65 | { 66 | size_t len; 67 | const char* errorStr; 68 | } g_errorInfo[]; 69 | 70 | 71 | #endif 72 | 73 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/QedisManager.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QEDISMANAGER_H 2 | #define BERT_QEDISMANAGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "future/Future.h" 9 | 10 | 11 | class QedisConn; 12 | 13 | namespace ananas 14 | { 15 | class EventLoop; 16 | class Connection; 17 | struct SocketAddr; 18 | } 19 | 20 | class QedisManager 21 | { 22 | public: 23 | static QedisManager& Instance(); 24 | 25 | QedisManager(); 26 | 27 | void SetEventLoop(ananas::EventLoop* loop); 28 | 29 | ananas::Future Connect(const std::string& addr); 30 | 31 | void OnNewConnection(ananas::Connection* conn); 32 | void OnConnect(ananas::Connection* conn); 33 | void OnDisconnect(ananas::Connection* conn); 34 | void OnConnFail(ananas::EventLoop* loop, const ananas::SocketAddr& peer); 35 | 36 | ananas::Future GetConnection(const std::string& peer); 37 | 38 | private: 39 | ananas::EventLoop* loop_; 40 | std::unordered_map connMap_; 41 | 42 | struct Request 43 | { 44 | std::string peer; 45 | ananas::Promise promise; 46 | 47 | Request() 48 | { 49 | } 50 | 51 | Request(Request const& ) = delete; 52 | void operator= (Request const& ) = delete; 53 | 54 | Request(Request&& ) = default; 55 | Request& operator= (Request&& ) = default; 56 | }; 57 | 58 | using ConnectPromise = ananas::Promise; 59 | std::unordered_map > pending_; 60 | }; 61 | 62 | #endif 63 | 64 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ZookeeperConn.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ZOOKEEPERCONN_H 2 | #define BERT_ZOOKEEPERCONN_H 3 | 4 | #include 5 | #include "ClusterConn.h" 6 | #include "zookeeper/ZookeeperContext.h" 7 | 8 | namespace ananas 9 | { 10 | class Connection; 11 | } 12 | 13 | // 当连接zk成功时，先发起握手，再获取QEDIS节点列表，然后再监听集群客户端 14 | // 15 | // 对于集群客户端，先连接zk，成功后，同时注册自己和获取PROXY节点列表，然后就可以发起请求 16 | // PROXY节点无状态，可以随意负载均衡的请求 17 | class ZookeeperConn final : public qedis::ClusterConn 18 | { 19 | public: 20 | explicit 21 | ZookeeperConn(ananas::Connection* c); 22 | 23 | virtual ~ZookeeperConn(); 24 | 25 | public: 26 | bool OnData(const char*& data, size_t len) override final; 27 | void OnConnect() override final; 28 | void OnDisconnect() override final; 29 | 30 | private: 31 | ananas::Try 32 | _ProcessHandshake(const ZkResponse& ); 33 | 34 | ananas::Future>> 35 | _RegisterAndGetServers(ananas::Try&& ); 36 | 37 | ananas::Future>> 38 | _GetShardingInfo(const std::vector>& ); 39 | 40 | bool _ProcessShardingInfo(const std::vector>& ); 41 | 42 | ananas::Future>> 43 | _GetServers(const std::vector>& ); 44 | 45 | bool _ProcessServerInfo(const std::vector>& ); 46 | void _InitPingTimer(); 47 | 48 | ananas::Connection* conn_; 49 | std::unique_ptr ctx_; 50 | 51 | ananas::TimerId pingId_; 52 | }; 53 | 54 | #endif 55 | 56 | -------------------------------------------------------------------------------- /QedisCore/QLeveldb.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QLEVELDB_H 2 | #define BERT_QLEVELDB_H 3 | 4 | #include "QDumpInterface.h" 5 | #include "QStore.h" 6 | 7 | namespace leveldb 8 | { 9 | class DB; 10 | } 11 | 12 | namespace qedis 13 | { 14 | 15 | class UnboundedBuffer; 16 | 17 | class QLeveldb : public QDumpInterface 18 | { 19 | public: 20 | QLeveldb(); 21 | ~QLeveldb(); 22 | 23 | bool Open(const char* path); 24 | bool IsOpen() const ; 25 | 26 | QObject Get(const QString& key) override; 27 | bool Put(const QString& key) override; 28 | 29 | bool Put(const QString& key, const QObject& obj, int64_t ttl = 0) override; 30 | bool Delete(const QString& key) override; 31 | 32 | private: 33 | leveldb::DB* db_; 34 | 35 | // encoding stuff 36 | 37 | // value format: type + ttl(if has) + qobject 38 | void _EncodeObject(const QObject& obj, int64_t absttl, UnboundedBuffer& v); 39 | 40 | void _EncodeString(const QString& str, UnboundedBuffer& v); 41 | void _EncodeHash(const PHASH& , UnboundedBuffer& v); 42 | void _EncodeList(const PLIST& , UnboundedBuffer& v); 43 | void _EncodeSet(const PSET& , UnboundedBuffer& v); 44 | void _EncodeSSet(const PSSET& , UnboundedBuffer& v); 45 | 46 | // decoding stuff 47 | QObject _DecodeObject(const char* data, size_t len, int64_t& remainTtlSeconds); 48 | 49 | QString _DecodeString(const char* data, size_t len); 50 | QObject _DecodeHash(const char* data, size_t len); 51 | QObject _DecodeList(const char* data, size_t len); 52 | QObject _DecodeSet(const char* data, size_t len); 53 | QObject _DecodeSSet(const char* data, size_t len); 54 | }; 55 | 56 | } 57 | 58 | #endif 59 | 60 | -------------------------------------------------------------------------------- /cluster/ananas/net/DatagramSocket.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_DATAGRAMSOCKET_H 3 | #define BERT_DATAGRAMSOCKET_H 4 | 5 | #include 6 | #include "Socket.h" 7 | #include "Typedefs.h" 8 | #include "Poller.h" 9 | 10 | namespace ananas 11 | { 12 | 13 | class EventLoop; 14 | 15 | class DatagramSocket : public internal::EventSource 16 | { 17 | public: 18 | explicit 19 | DatagramSocket(EventLoop* loop); 20 | ~DatagramSocket(); 21 | 22 | DatagramSocket(const DatagramSocket& ) = delete; 23 | void operator= (const DatagramSocket& ) = delete; 24 | 25 | void SetMaxPacketSize(std::size_t s); 26 | bool Bind(const SocketAddr* addr); 27 | 28 | int Identifier() const override; 29 | bool HandleReadEvent() override; 30 | bool HandleWriteEvent() override; 31 | void HandleErrorEvent() override; 32 | 33 | bool SendPacket(const void* , size_t , const SocketAddr* = nullptr); 34 | 35 | const SocketAddr& PeerAddr() const { return srcAddr_; } 36 | 37 | void SetMessageCallback(UDPMessageCallback mcb) { onMessage_ = std::move(mcb); } 38 | void SetCreateCallback(UDPCreateCallback ccb) { onCreate_ = std::move(ccb); } 39 | 40 | private: 41 | void _PutSendBuf(const void* data, size_t size, const SocketAddr* dst); 42 | int _Send(const void* data, size_t size, const SocketAddr& dst); 43 | 44 | EventLoop* const loop_; 45 | int localSock_; 46 | std::size_t maxPacketSize_; 47 | SocketAddr srcAddr_; 48 | 49 | struct Package 50 | { 51 | SocketAddr dst; 52 | std::string data; 53 | }; 54 | std::list sendList_; 55 | 56 | UDPMessageCallback onMessage_; 57 | UDPCreateCallback onCreate_; 58 | }; 59 | 60 | } // namespace ananas 61 | 62 | #endif 63 | 64 | -------------------------------------------------------------------------------- /QedisCore/QMigration.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QMIGRATION_H 2 | #define BERT_QMIGRATION_H 3 | 4 | #include 5 | #include "QClient.h" 6 | 7 | namespace ConnectionTag 8 | { 9 | const int kMigrateClient = 4; 10 | } 11 | 12 | namespace qedis 13 | { 14 | 15 | enum class MigrateState 16 | { 17 | None, 18 | Processing, 19 | Timeout, 20 | Done, 21 | }; 22 | 23 | struct MigrationItem 24 | { 25 | SocketAddr dst; 26 | int dstDb = 0; 27 | time_t timeout = 0; 28 | bool copy = false; 29 | bool replace = false; 30 | std::vector keys; 31 | std::weak_ptr client; 32 | 33 | MigrateState state = MigrateState::None; 34 | }; 35 | 36 | class QMigrateClient; 37 | 38 | class QMigrationManager 39 | { 40 | friend class QMigrateClient; 41 | public: 42 | static QMigrationManager& Instance(); 43 | 44 | void Add(const MigrationItem& item); 45 | void Add(MigrationItem&& item); 46 | 47 | void InitMigrationTimer(); 48 | void LoopCheck(); 49 | 50 | std::shared_ptr GetConnection(const SocketAddr& dst); 51 | void OnConnectMigrateFail(const SocketAddr& dst); 52 | void OnConnect(QMigrateClient* ); 53 | 54 | private: 55 | QMigrationManager() { } 56 | 57 | std::unordered_map > items_; 58 | std::unordered_map > conns_; 59 | 60 | std::unordered_set pendingConnect_; 61 | }; 62 | 63 | class QMigrateClient : public StreamSocket 64 | { 65 | public: 66 | void OnConnect() override; 67 | 68 | private: 69 | PacketLength _HandlePacket(const char* msg, std::size_t len) override; 70 | size_t readyRsp_ = 0; 71 | }; 72 | 73 | } // end namespace qedis 74 | 75 | #endif 76 | 77 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/QedisConn.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QEDISCONN_H 2 | #define BERT_QEDISCONN_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "net/Connection.h" 8 | #include "future/Future.h" 9 | #include "Protocol.h" 10 | 11 | // Build redis request from multiple strings, use inline protocol 12 | template 13 | std::string BuildQedisRequest(Args&& ...); 14 | 15 | template 16 | std::string BuildQedisRequest(S&& s) 17 | { 18 | return std::string(std::forward(s)) + "\r\n"; 19 | } 20 | 21 | template 22 | std::string BuildQedisRequest(H&& head, T&&... tails) 23 | { 24 | std::string h(std::forward(head)); 25 | return h + " " + BuildQedisRequest(std::forward(tails)...); 26 | } 27 | 28 | class QedisConn final 29 | { 30 | public: 31 | explicit 32 | QedisConn(ananas::Connection* conn); 33 | 34 | ananas::Future 35 | ForwardRequest(const std::vector& params); 36 | 37 | ananas::Future 38 | ForwardRequest(const std::string& rawRequest); 39 | 40 | ananas::PacketLen_t OnRecv(ananas::Connection* conn, const char* data, ananas::PacketLen_t len); 41 | private: 42 | ananas::Connection* hostConn_; 43 | 44 | struct Request 45 | { 46 | //std::vector request; 47 | ananas::Promise promise; 48 | 49 | Request() = default; 50 | 51 | Request(Request const& ) = delete; 52 | void operator= (Request const& ) = delete; 53 | 54 | Request(Request&& ) = default; 55 | Request& operator= (Request&& ) = default; 56 | }; 57 | 58 | std::queue pending_; 59 | 60 | ClientProtocol proto_; 61 | }; 62 | 63 | #endif 64 | 65 | -------------------------------------------------------------------------------- /QedisCore/QSortedSet.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QSORTEDSET_H 2 | #define BERT_QSORTEDSET_H 3 | 4 | #include "QString.h" 5 | #include "QHelper.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace qedis 12 | { 13 | 14 | class QSortedSet 15 | { 16 | public: 17 | using Members = std::set; 18 | using Score2Members = std::map; 19 | 20 | using Member2Score = std::unordered_map >; 23 | 24 | Member2Score::iterator FindMember(const QString& member); 25 | Member2Score::const_iterator begin() const { return members_.begin(); }; 26 | Member2Score::iterator begin() { return members_.begin(); }; 27 | Member2Score::const_iterator end() const { return members_.end(); }; 28 | Member2Score::iterator end() { return members_.end(); }; 29 | void AddMember (const QString& member, double score); 30 | double UpdateMember(const Member2Score::iterator& itMem, double delta); 31 | 32 | int Rank (const QString& member) const;// 0-based 33 | int RevRank (const QString& member) const;// 0-based 34 | bool DelMember (const QString& member); 35 | Member2Score::value_type 36 | GetMemberByRank(std::size_t rank) const; 37 | 38 | std::vector 39 | RangeByRank(long start, long end) const; 40 | 41 | std::vector 42 | RangeByScore(double minScore, double maxScore); 43 | 44 | std::size_t Size() const; 45 | 46 | private: 47 | Score2Members scores_; 48 | Member2Score members_; 49 | }; 50 | 51 | } 52 | 53 | #endif 54 | 55 | -------------------------------------------------------------------------------- /cluster/ananas/net/Poller.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_POLLER_H 2 | #define BERT_POLLER_H 3 | 4 | #include 5 | #include 6 | 7 | namespace ananas 8 | { 9 | namespace internal 10 | { 11 | 12 | enum EventType 13 | { 14 | eET_None = 0, 15 | eET_Read = 0x1 << 0, 16 | eET_Write = 0x1 << 1, 17 | eET_Error = 0x1 << 2, 18 | }; 19 | 20 | class EventSource : public std::enable_shared_from_this 21 | { 22 | public: 23 | EventSource() { } 24 | virtual ~EventSource() { } 25 | 26 | virtual int Identifier() const = 0; // the socket 27 | 28 | unsigned int GetUniqueId() const { return unique_id_; } 29 | void SetUniqueId(unsigned int id) { unique_id_ = id; } 30 | 31 | virtual bool HandleReadEvent() = 0; 32 | virtual bool HandleWriteEvent() = 0; 33 | virtual void HandleErrorEvent() = 0; 34 | 35 | private: 36 | unsigned int unique_id_ = 0; // dispatch by ioloop 37 | }; 38 | 39 | 40 | struct FiredEvent 41 | { 42 | int events; 43 | void* userdata; 44 | 45 | FiredEvent() : events(0), userdata(nullptr) 46 | { 47 | } 48 | }; 49 | 50 | class Poller 51 | { 52 | public: 53 | Poller() : multiplexer_(-1) 54 | { 55 | } 56 | 57 | virtual ~Poller() 58 | { 59 | } 60 | 61 | virtual bool Register(int fd, int events, void* userPtr) = 0; 62 | virtual bool Modify(int fd, int events, void* userPtr) = 0; 63 | virtual bool Unregister(int fd, int events) = 0; 64 | 65 | virtual int Poll(std::size_t maxEv, int timeoutMs) = 0; 66 | const std::vector& GetFiredEvents() const { return firedEvents_; } 67 | 68 | protected: 69 | int multiplexer_; 70 | std::vector firedEvents_; 71 | }; 72 | 73 | } // end namespace internal 74 | } // end namespace ananas 75 | 76 | #endif 77 | 78 | -------------------------------------------------------------------------------- /QedisCore/QPubsub.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QPUBSUB_H 2 | #define BERT_QPUBSUB_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "QString.h" 8 | #include 9 | 10 | namespace qedis 11 | { 12 | 13 | class QClient; 14 | class QPubsub 15 | { 16 | public: 17 | static QPubsub& Instance(); 18 | 19 | QPubsub(const QPubsub& ) = delete; 20 | void operator= (const QPubsub& ) = delete; 21 | 22 | std::size_t Subscribe(QClient* client, const QString& channel); 23 | std::size_t UnSubscribe(QClient* client, const QString& channel); 24 | std::size_t UnSubscribeAll(QClient* client); 25 | std::size_t PublishMsg(const QString& channel, const QString& msg); 26 | 27 | std::size_t PSubscribe(QClient* client, const QString& pchannel); 28 | std::size_t PUnSubscribeAll(QClient* client); 29 | std::size_t PUnSubscribe(QClient* client, const QString& pchannel); 30 | 31 | // introspect 32 | void PubsubChannels(std::vector& res, const char* pattern = 0) const; 33 | std::size_t PubsubNumsub(const QString& channel) const; 34 | std::size_t PubsubNumpat() const; 35 | 36 | void InitPubsubTimer(); 37 | void RecycleClients(QString& startChannel, QString& startPattern); 38 | 39 | private: 40 | QPubsub() {} 41 | 42 | using Clients = std::set, std::owner_less > >; 43 | using ChannelClients = std::map; 44 | 45 | ChannelClients channels_; 46 | ChannelClients patternChannels_; 47 | 48 | QString startChannel_; 49 | QString startPattern_; 50 | static void _RecycleClients(ChannelClients& channels, QString& start); 51 | 52 | size_t _Publish(QPubsub::Clients& clients, const std::vector& args); 53 | }; 54 | 55 | } 56 | 57 | #endif 58 | 59 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/QedisConn.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "QedisConn.h" 4 | 5 | QedisConn::QedisConn(ananas::Connection* conn) : 6 | hostConn_(conn) 7 | { 8 | } 9 | 10 | static 11 | std::string BuildRequest(const std::vector& params) 12 | { 13 | assert (!params.empty()); 14 | 15 | std::string req; 16 | for (const auto& e : params) 17 | { 18 | req += e + " "; 19 | } 20 | 21 | req.pop_back(); 22 | req += "\r\n"; 23 | 24 | return req; 25 | } 26 | 27 | ananas::Future 28 | QedisConn::ForwardRequest(const std::vector& params) 29 | { 30 | std::string buf = BuildRequest(params); 31 | return ForwardRequest(buf); 32 | } 33 | 34 | ananas::Future 35 | QedisConn::ForwardRequest(const std::string& rawReq) 36 | { 37 | hostConn_->SendPacket(rawReq.data(), rawReq.size()); 38 | 39 | QedisConn::Request req; 40 | 41 | auto fut = req.promise.GetFuture(); 42 | pending_.push(std::move(req)); 43 | 44 | return fut; 45 | } 46 | 47 | ananas::PacketLen_t QedisConn::OnRecv(ananas::Connection* conn, const char* data, ananas::PacketLen_t len) 48 | { 49 | const char* ptr = data; 50 | auto parseRet = proto_.Parse(ptr, ptr + len); 51 | if (parseRet == ParseResult::error) 52 | { 53 | conn->ActiveClose(); 54 | return 0; 55 | } 56 | else if (parseRet != ParseResult::ok) 57 | { 58 | // wait 59 | return static_cast(ptr - data); 60 | } 61 | 62 | assert (parseRet == ParseResult::ok); 63 | 64 | auto& req = pending_.front(); 65 | req.promise.SetValue(proto_.GetContent()); 66 | pending_.pop(); 67 | 68 | proto_.Reset(); 69 | 70 | return static_cast(ptr - data); 71 | } 72 | 73 | -------------------------------------------------------------------------------- /QedisCore/QSlowLog.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "Log/Logger.h" 6 | #include "QSlowLog.h" 7 | 8 | namespace qedis 9 | { 10 | 11 | QSlowLog& QSlowLog::Instance() 12 | { 13 | static QSlowLog slog; 14 | 15 | return slog; 16 | } 17 | 18 | QSlowLog::QSlowLog() : threshold_(0), logger_(nullptr) 19 | { 20 | } 21 | 22 | QSlowLog::~QSlowLog() 23 | { 24 | } 25 | 26 | void QSlowLog::SetThreshold(unsigned int v) 27 | { 28 | threshold_ = v; 29 | } 30 | 31 | void QSlowLog::SetLogLimit(std::size_t maxCount) 32 | { 33 | logMaxCount_ = maxCount; 34 | } 35 | 36 | void QSlowLog::Begin() 37 | { 38 | if (!threshold_) 39 | return; 40 | 41 | timeval begin; 42 | gettimeofday(&begin, 0); 43 | beginUs_ = begin.tv_sec * 1000000 + begin.tv_usec; 44 | } 45 | 46 | 47 | 48 | void QSlowLog::EndAndStat(const std::vector& cmds) 49 | { 50 | if (!threshold_ || beginUs_ == 0) 51 | return; 52 | 53 | timeval end; 54 | gettimeofday(&end, 0); 55 | auto used = end.tv_sec * 1000000 + end.tv_usec - beginUs_; 56 | 57 | if (used >= threshold_) 58 | { 59 | if (logger_ == nullptr) 60 | logger_ = LogManager::Instance().CreateLog(logALL, logFILE, "slowlog.qedis"); 61 | 62 | LOG_INF(logger_) << "+ Used:(us) " << used; 63 | 64 | for (const auto& param : cmds) 65 | { 66 | LOG_INF(logger_) << param; 67 | } 68 | 69 | if (cmds[0] == "slowlog") 70 | return; 71 | 72 | SlowLogItem item; 73 | item.used = static_cast(used); 74 | item.cmds = cmds; 75 | 76 | logs_.emplace_front(std::move(item)); 77 | if (logs_.size() > logMaxCount_) 78 | logs_.pop_back(); 79 | } 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ConfigParser.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CONFIGPARSER_H 2 | #define BERT_CONFIGPARSER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef CONFIG_DEBUG 9 | #include 10 | #endif 11 | 12 | class ConfigParser 13 | { 14 | public: 15 | bool Load(const char* FileName); 16 | 17 | template 18 | T GetData(const char* key, const T& default_ = T()) const; 19 | 20 | const std::vector& GetDataVector(const char* key) const; 21 | 22 | 23 | #ifdef CONFIG_DEBUG 24 | void Print() 25 | { 26 | std::cout << "//////////////////"<< std::endl; 27 | Data::const_iterator it = data_.begin(); 28 | while (it != data_.end()) 29 | { 30 | std::cout << it->first << ":" << it->second[0] << "\n"; 31 | ++ it; 32 | } 33 | } 34 | #endif 35 | 36 | private: 37 | typedef std::map > Data; 38 | 39 | Data data_; 40 | 41 | template 42 | T _ToType(const std::string& data) const; 43 | }; 44 | 45 | 46 | template 47 | inline T ConfigParser::_ToType(const std::string& data) const 48 | { 49 | T t; 50 | std::istringstream os(data); 51 | os >> t; 52 | return t; 53 | } 54 | 55 | template <> 56 | inline const char* ConfigParser::_ToType(const std::string& data) const 57 | { 58 | return data.c_str(); 59 | } 60 | 61 | template <> 62 | inline std::string ConfigParser::_ToType(const std::string& data) const 63 | { 64 | return data; 65 | } 66 | 67 | 68 | template 69 | inline T ConfigParser::GetData(const char* key, const T& default_) const 70 | { 71 | auto it = data_.find(key); 72 | if (it == data_.end()) 73 | return default_; 74 | 75 | return _ToType(it->second[0]); // only return first value 76 | } 77 | 78 | #endif 79 | 80 | -------------------------------------------------------------------------------- /QBase/ConfigParser.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_CONFIGPARSER_H 2 | #define BERT_CONFIGPARSER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef CONFIG_DEBUG 9 | #include 10 | #endif 11 | 12 | class ConfigParser 13 | { 14 | public: 15 | bool Load(const char* FileName); 16 | 17 | template 18 | T GetData(const char* key, const T& default_ = T()) const; 19 | 20 | const std::vector& GetDataVector(const char* key) const; 21 | 22 | 23 | #ifdef CONFIG_DEBUG 24 | void Print() 25 | { 26 | std::cout << "//////////////////"<< std::endl; 27 | std::map::const_iterator it = data_.begin(); 28 | while (it != data_.end()) 29 | { 30 | std::cout << it->first << ":" << it->second << "\n"; 31 | ++ it; 32 | } 33 | } 34 | #endif 35 | 36 | private: 37 | typedef std::map > Data; 38 | 39 | Data data_; 40 | 41 | template 42 | T _ToType(const std::string& data) const; 43 | }; 44 | 45 | 46 | template 47 | inline T ConfigParser::_ToType(const std::string& data) const 48 | { 49 | T t; 50 | std::istringstream os(data); 51 | os >> t; 52 | return t; 53 | } 54 | 55 | template <> 56 | inline const char* ConfigParser::_ToType(const std::string& data) const 57 | { 58 | return data.c_str(); 59 | } 60 | 61 | template <> 62 | inline std::string ConfigParser::_ToType(const std::string& data) const 63 | { 64 | return data; 65 | } 66 | 67 | 68 | template 69 | inline T ConfigParser::GetData(const char* key, const T& default_) const 70 | { 71 | auto it = data_.find(key); 72 | if (it == data_.end()) 73 | return default_; 74 | 75 | return _ToType(it->second[0]); // only return first value 76 | } 77 | 78 | #endif 79 | 80 | -------------------------------------------------------------------------------- /README.zh.md: -------------------------------------------------------------------------------- 1 | 2 | _____ _____ _____ _ ______ 3 | / _ \ | ____| | _ \ | | / ___/ 4 | | | | | | |__ | | | | | | | |____ 5 | | | | | | __| | | | | | | \__ \ 6 | | |_| |_ | |___ | |_| | | | ___| | 7 | \_______| |_____| |_____/ |_| /_____/ 8 | 9 | 10 | [![Build Status](https://travis-ci.org/loveyacper/Qedis.svg?branch=master)](https://travis-ci.org/loveyacper/Qedis) 11 | 12 | [Click me switch to English](README.md) 13 | 14 | C++11实现的增强版分布式Redis服务器,使用Leveldb作为持久化存储引擎。 15 | 16 | ## 环境需求 17 | * C++11、CMake 18 | * Linux 或 MAC OS 19 | 20 | ## 集群特性 21 | 可以搭建Zookeeper，监视一组互为主备的Qedis进程以实现高可用; 22 | 23 | 当然也可以使用官方redis-sentinel。 24 | 25 | 详见[cluster Readme](QCluster/README.md) 26 | 27 | scale-out集群正在开发中... 28 | 29 | ## 与Redis完全兼容 30 | 你可以用redis的各种工具来测试Qedis，比如官方的redis-cli, redis-benchmark。 31 | 32 | Qedis可以和redis之间进行复制，可以读取redis的rdb文件或aof文件。当然，Qedis生成的aof或rdb文件也可以被redis读取。 33 | 34 | 你还可以用redis-sentinel来实现Qedis的高可用！ 35 | 36 | 总之，Qedis与Redis完全兼容。 37 | 38 | ## 高性能 39 | - Qedis性能大约比Redis3.2高出20%(使用redis-benchmark测试pipeline请求，比如设置-P=50或更高) 40 | - Qedis的高性能有一部分得益于独立的网络线程处理IO，因此和redis比占了便宜。但Qedis逻辑仍然是单线程的。 41 | - 另一部分得益于C++ STL的高效率（CLANG的表现比GCC更好）。 42 | - 在测试前，你要确保std::list的size()是O(1)复杂度，这才遵循C++11的标准。否则list相关命令不可测。 43 | 44 | 运行下面这个命令，试试和redis比一比~ 45 | ```bash 46 | ./redis-benchmark -q -n 1000000 -P 50 -c 50 47 | ``` 48 | 49 | 我在rMBP late2013笔记本上测试结果如图： 50 | 51 | ![image](https://github.com/loveyacper/Qedis/blob/master/performance.png) 52 | 53 | 54 | ## 编写扩展模块 55 | Qedis支持动态库模块，可以在运行时添加新命令。 56 | 我添加了三个命令(ldel, skeys, hgets)作为演示。 57 | 58 | ## 支持冷数据淘汰 59 | 是的，在内存受限的情况下，你可以让Qedis根据简单的LRU算法淘汰一些key以释放内存。 60 | 61 | ## 主从复制，事务，RDB/AOF持久化，慢日志，发布订阅 62 | 这些特性Qedis都有:-) 63 | 64 | ## 持久化：内存不再是上限 65 | Leveldb可以配置为Qedis的持久化存储引擎，可以存储更多的数据。 66 | 67 | 68 | ## 命令列表 69 | #### 展示Qedis支持的所有命令，目前支持140个命令 70 | - cmdlist 71 | 72 | ## TODO 73 | * 支持lua 74 | * Qedis Cluster多语言客户端 75 | 76 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ClientManager.cc: -------------------------------------------------------------------------------- 1 | #include "ClientManager.h" 2 | #include "ClientConn.h" 3 | #include "ProxyLog.h" 4 | 5 | #include "net/EventLoop.h" 6 | #include "net/Connection.h" 7 | #include "net/Socket.h" 8 | 9 | ClientManager& ClientManager::Instance() 10 | { 11 | static ClientManager mgr; 12 | return mgr; 13 | } 14 | 15 | ClientManager::ClientManager() : 16 | loop_(nullptr), 17 | listening_(false) 18 | { 19 | } 20 | 21 | void ClientManager::SetEventLoop(ananas::EventLoop* loop) 22 | { 23 | assert (!loop_); 24 | loop_ = loop; 25 | } 26 | 27 | bool ClientManager::Listen(const std::string& addr) 28 | { 29 | assert (!listening_); 30 | listening_ = loop_->Listen(ananas::SocketAddr(addr), 31 | std::bind(&ClientManager::OnNewConnection, this, std::placeholders::_1)); 32 | return listening_; 33 | } 34 | 35 | void ClientManager::OnNewConnection(ananas::Connection* conn) 36 | { 37 | INF(g_logger) << "ClientManager::OnNewConnection " << conn->Identifier(); 38 | 39 | auto ctx = std::make_shared(conn); 40 | 41 | conn->SetUserData(ctx); 42 | conn->SetMinPacketSize(4); 43 | conn->SetOnMessage(std::bind(&ClientConn::OnRecv, 44 | ctx.get(), 45 | std::placeholders::_1, 46 | std::placeholders::_2, 47 | std::placeholders::_3)); 48 | conn->SetOnConnect(std::bind(&ClientManager::OnConnect, this, std::placeholders::_1)); 49 | conn->SetOnDisconnect(std::bind(&ClientManager::OnDisconnect, this, std::placeholders::_1)); 50 | } 51 | 52 | void ClientManager::OnConnect(ananas::Connection* conn) 53 | { 54 | INF(g_logger) << "ClientManager::OnConnect " << conn->Identifier(); 55 | } 56 | 57 | void ClientManager::OnDisconnect(ananas::Connection* conn) 58 | { 59 | INF(g_logger) << "ClientManager::OnDisconnect " << conn->Identifier(); 60 | } 61 | 62 | -------------------------------------------------------------------------------- /cluster/ananas/util/Util.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_UTIL_H 2 | #define BERT_UTIL_H 3 | 4 | // Ananas tool box 5 | 6 | #include 7 | #include 8 | 9 | namespace ananas 10 | { 11 | 12 | namespace 13 | { 14 | 15 | // The defer class for C++11 16 | class ExecuteOnScopeExit 17 | { 18 | public: 19 | ExecuteOnScopeExit() { } 20 | 21 | ExecuteOnScopeExit(ExecuteOnScopeExit&& e) 22 | { 23 | func_ = std::move(e.func_); 24 | } 25 | 26 | ExecuteOnScopeExit(const ExecuteOnScopeExit& e) = delete; 27 | void operator=(const ExecuteOnScopeExit& f) = delete; 28 | 29 | template 30 | ExecuteOnScopeExit(F&& f, Args&&... args) 31 | { 32 | auto temp = std::bind(std::forward(f), std::forward(args)...); 33 | func_ = [temp]() { (void)temp(); }; 34 | } 35 | 36 | ~ExecuteOnScopeExit() noexcept 37 | { 38 | if (func_) func_(); 39 | } 40 | 41 | private: 42 | std::function func_; 43 | }; 44 | 45 | } // end namespace 46 | 47 | #define _CONCAT(a, b) a##b 48 | #define _MAKE_DEFER_HELPER_(line) ananas::ExecuteOnScopeExit _CONCAT(defer, line) = [&]() 49 | 50 | #undef ANANAS_DEFER 51 | #define ANANAS_DEFER _MAKE_DEFER_HELPER_(__LINE__) 52 | 53 | inline 54 | std::vector SplitString(const std::string& str, char seperator) 55 | { 56 | std::vector results; 57 | 58 | std::string::size_type start = 0; 59 | std::string::size_type sep = str.find(seperator); 60 | while (sep != std::string::npos) 61 | { 62 | if (start < sep) 63 | results.emplace_back(str.substr(start, sep - start)); 64 | 65 | start = sep + 1; 66 | sep = str.find(seperator, start); 67 | } 68 | 69 | if (start != str.size()) 70 | results.emplace_back(str.substr(start)); 71 | 72 | return results; 73 | } 74 | 75 | } // end namespace ananas 76 | 77 | #endif 78 | 79 | -------------------------------------------------------------------------------- /QBase/StreamSocket.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_STREAMSOCKET_H 3 | #define BERT_STREAMSOCKET_H 4 | 5 | #include "AsyncBuffer.h" 6 | #include "Socket.h" 7 | #include 8 | #include 9 | 10 | using PacketLength = int32_t; 11 | 12 | // Abstraction for a TCP connection 13 | class StreamSocket : public Socket 14 | { 15 | friend class SendThread; 16 | public: 17 | StreamSocket(); 18 | ~StreamSocket(); 19 | 20 | bool Init(int localfd, const SocketAddr& peer); 21 | SocketType GetSocketType() const { return SocketType_Stream; } 22 | 23 | public: 24 | // Receive data 25 | int Recv(); 26 | public: 27 | // Send data 28 | bool SendPacket(const void* , std::size_t ); 29 | bool SendPacket(Buffer& bf); 30 | bool SendPacket(AttachedBuffer& abf); 31 | bool SendPacket(qedis::UnboundedBuffer& ubf); 32 | template 33 | bool SendPacket(StackBuffer& sb); 34 | 35 | bool OnReadable(); 36 | bool OnWritable(); 37 | bool OnError(); 38 | 39 | bool DoMsgParse(); // false if no msg 40 | 41 | void SetOnDisconnect(const std::function& cb = std::function()) { onDisconnect_ = cb; } 42 | 43 | // send thread 44 | bool Send(); 45 | 46 | const SocketAddr& GetPeerAddr() const { return peerAddr_; } 47 | 48 | protected: 49 | SocketAddr peerAddr_; 50 | 51 | private: 52 | std::function onDisconnect_; 53 | 54 | int _Send(const BufferSequence& bf); 55 | virtual PacketLength _HandlePacket(const char* msg, std::size_t len) = 0; 56 | 57 | // For human readability 58 | enum 59 | { 60 | TIMEOUTSOCKET = 0, 61 | ERRORSOCKET = -1, 62 | EOFSOCKET = -2, 63 | }; 64 | 65 | Buffer recvBuf_; 66 | AsyncBuffer sendBuf_; 67 | }; 68 | 69 | template 70 | inline bool StreamSocket::SendPacket(StackBuffer& sf) 71 | { 72 | return SendPacket(sf.ReadAddr(), sf.ReadableSize()); 73 | } 74 | 75 | #endif 76 | 77 | -------------------------------------------------------------------------------- /QedisCore/QModule.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QMODULE_H 2 | #define BERT_QMODULE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "QString.h" 9 | 10 | namespace qedis 11 | { 12 | 13 | class ModuleExist : public std::logic_error 14 | { 15 | public: 16 | explicit ModuleExist(const QString& what) : std::logic_error(what) 17 | { 18 | } 19 | }; 20 | 21 | class ModuleNotExist : public std::logic_error 22 | { 23 | public: 24 | explicit ModuleNotExist(const QString& what) : std::logic_error(what) 25 | { 26 | } 27 | }; 28 | 29 | class ModuleNoLoad : public std::logic_error 30 | { 31 | public: 32 | explicit ModuleNoLoad(const QString& what) : std::logic_error(what) 33 | { 34 | } 35 | }; 36 | 37 | class ModuleNoUnLoad : public std::logic_error 38 | { 39 | public: 40 | explicit ModuleNoUnLoad(const QString& what) : std::logic_error(what) 41 | { 42 | } 43 | }; 44 | 45 | 46 | class QModule 47 | { 48 | public: 49 | QModule(); 50 | ~QModule(); 51 | 52 | const QString& Name() const; 53 | 54 | void Load(const char* so, bool lazy = false) throw(std::runtime_error); 55 | void UnLoad(); 56 | void* Symbol(const char* symbol); 57 | 58 | private: 59 | QString soname_; 60 | void* handler_; 61 | }; 62 | 63 | class QModuleManager 64 | { 65 | public: 66 | static QModuleManager& Instance(); 67 | 68 | QModuleManager(const QModuleManager& ) = delete; 69 | void operator= (const QModuleManager& ) = delete; 70 | 71 | QModule* Load(const char* so, bool lazy = false) throw(std::logic_error, std::runtime_error); 72 | void UnLoad(const char* so); 73 | 74 | QModule* GetModule(const char* so); 75 | 76 | std::vector NameList() const; 77 | 78 | private: 79 | QModuleManager() {} 80 | 81 | std::unordered_map > modules_; 82 | }; 83 | 84 | #define MODULES ::qedis::QModuleManager::Instance() 85 | 86 | } 87 | 88 | #endif 89 | 90 | -------------------------------------------------------------------------------- /QedisCore/QGlobRegex.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QGLOBREGEX_H 2 | #define BERT_QGLOBREGEX_H 3 | 4 | #include 5 | #include "QString.h" 6 | 7 | namespace qedis 8 | { 9 | 10 | bool NotGlobRegex(const char* pattern, std::size_t plen); 11 | 12 | class QGlobRegex 13 | { 14 | public: 15 | explicit QGlobRegex(const char* pattern = 0, std::size_t plen = 0, 16 | const char* text = 0, std::size_t tlen = 0); 17 | 18 | void SetPattern(const char* pattern, std::size_t plen); 19 | void SetText(const char* text, std::size_t tlen); 20 | bool TryMatch(); 21 | 22 | 23 | private: 24 | bool _ProcessStar(); 25 | bool _ProcessQuestion(); 26 | bool _ProcessBracket(); 27 | bool _IsMatch() const; 28 | 29 | const char* pattern_; 30 | std::size_t pLen_; 31 | std::size_t pOff_; 32 | 33 | const char* text_; 34 | std::size_t tLen_; 35 | std::size_t tOff_; 36 | }; 37 | 38 | inline bool glob_match(const char* pattern, std::size_t plen, 39 | const char* text, std::size_t tlen) 40 | { 41 | QGlobRegex rgx; 42 | rgx.SetPattern(pattern, plen); 43 | rgx.SetText(text, tlen); 44 | 45 | return rgx.TryMatch(); 46 | } 47 | 48 | inline bool glob_match(const char* pattern, 49 | const char* text) 50 | { 51 | return glob_match(pattern, strlen(pattern), text, strlen(text)); 52 | } 53 | 54 | inline bool glob_match(const QString& pattern, 55 | const QString& text) 56 | { 57 | return glob_match(pattern.c_str(), pattern.size(), 58 | text.c_str(), text.size()); 59 | } 60 | 61 | inline bool glob_search(const char* pattern, 62 | const char* str) 63 | { 64 | QString sPattern("*"); 65 | sPattern += pattern; 66 | sPattern += "*"; 67 | 68 | QGlobRegex rgx; 69 | rgx.SetPattern(sPattern.c_str(), sPattern.size()); 70 | rgx.SetText(str, strlen(str)); 71 | 72 | return rgx.TryMatch(); 73 | } 74 | 75 | } 76 | 77 | #endif 78 | 79 | -------------------------------------------------------------------------------- /Modules/QModuleInit.cc: -------------------------------------------------------------------------------- 1 | #include "QModuleInit.h" 2 | #include "QCommand.h" 3 | #include 4 | 5 | extern "C" 6 | qedis::QError ldel(const std::vector& params, qedis::UnboundedBuffer* reply); 7 | 8 | extern "C" 9 | qedis::QError hgets(const std::vector& params, qedis::UnboundedBuffer* reply); 10 | 11 | extern "C" 12 | qedis::QError skeys(const std::vector& params, qedis::UnboundedBuffer* reply); 13 | 14 | bool QedisModule_OnLoad() 15 | { 16 | printf("enter %s\n", __FUNCTION__); 17 | using namespace qedis; 18 | 19 | // register list ldel command 20 | static QCommandInfo ldelinfo; 21 | ldelinfo.cmd = "ldel"; 22 | ldelinfo.attr = QAttr_write; 23 | ldelinfo.params = 3; 24 | ldelinfo.handler = &ldel; 25 | 26 | if (!QCommandTable::AddCommand("ldel", &ldelinfo)) 27 | return false; 28 | 29 | // register hash hgets command 30 | static QCommandInfo hgetsinfo; 31 | hgetsinfo.cmd = "hgets"; 32 | hgetsinfo.attr = QAttr_read; 33 | hgetsinfo.params = 3; 34 | hgetsinfo.handler = &hgets; 35 | 36 | if (!QCommandTable::AddCommand("hgets", &hgetsinfo)) 37 | { 38 | QCommandTable::DelCommand("ldel"); 39 | return false; 40 | } 41 | 42 | // register set skeys command 43 | static QCommandInfo skeysinfo; 44 | skeysinfo.cmd = "skeys"; 45 | skeysinfo.attr = QAttr_read; 46 | skeysinfo.params = 3; 47 | skeysinfo.handler = &skeys; 48 | 49 | if (!QCommandTable::AddCommand("skeys", &skeysinfo)) 50 | { 51 | QCommandTable::DelCommand("hgets"); 52 | QCommandTable::DelCommand("ldel"); 53 | return false; 54 | } 55 | 56 | printf("exit %s\n", __FUNCTION__); 57 | return true; 58 | } 59 | 60 | 61 | void QedisModule_OnUnLoad() 62 | { 63 | printf("enter %s\n", __FUNCTION__); 64 | qedis::QCommandTable::DelCommand("skeys"); 65 | qedis::QCommandTable::DelCommand("hgets"); 66 | qedis::QCommandTable::DelCommand("ldel"); 67 | printf("exit %s\n", __FUNCTION__); 68 | } 69 | -------------------------------------------------------------------------------- /cluster/cluster_conn/zookeeper/ZkResponse.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ZKRESPONSE_H 2 | #define BERT_ZKRESPONSE_H 3 | 4 | #include "zookeeper.jute.h" 5 | #include "proto.h" 6 | 7 | using ZkResponse = std::shared_ptr; 8 | 9 | template 10 | inline 11 | std::shared_ptr NewResponse() 12 | { 13 | return std::make_shared(); 14 | } 15 | 16 | template 17 | inline std::shared_ptr AnyCast(const ZkResponse& rsp) 18 | { 19 | return std::static_pointer_cast(rsp); 20 | } 21 | 22 | struct CreateRsp 23 | { 24 | std::string path; 25 | }; 26 | 27 | inline 28 | CreateRsp Convert(const CreateResponse& rsp) 29 | { 30 | CreateRsp crsp; 31 | crsp.path = rsp.path; 32 | return crsp; 33 | } 34 | 35 | struct ChildrenRsp 36 | { 37 | std::string parent; 38 | std::vector children; 39 | struct Stat stat; 40 | }; 41 | 42 | inline 43 | ChildrenRsp Convert(const std::string& parent, const GetChildren2Response& children) 44 | { 45 | ChildrenRsp rsp; 46 | rsp.parent = parent; 47 | 48 | for (int i = 0; i < children.children.count; ++ i) 49 | { 50 | rsp.children.push_back(children.children.data[i]); 51 | } 52 | 53 | rsp.stat = children.stat; 54 | return rsp; 55 | } 56 | 57 | struct DataRsp 58 | { 59 | std::string path; 60 | std::string data; 61 | struct Stat stat; 62 | }; 63 | 64 | inline 65 | DataRsp Convert(const std::string& path, const GetDataResponse& rsp) 66 | { 67 | DataRsp drsp; 68 | drsp.path = path; 69 | drsp.data.assign(rsp.data.buff, rsp.data.len); 70 | drsp.stat = rsp.stat; 71 | return drsp; 72 | } 73 | 74 | #if 0 75 | union ZkResponse 76 | { 77 | prime_struct handshakeRsp; 78 | long recvPing; 79 | CreateResponse createRsp; 80 | 81 | struct 82 | { 83 | struct buffer parent; 84 | GetChildren2Response children; 85 | } child2Rsp; 86 | 87 | struct { 88 | struct buffer path; 89 | GetDataResponse data; 90 | } dataRsp; 91 | }; 92 | #endif 93 | 94 | #endif //BERT_ZKRESPONSE_H 95 | 96 | -------------------------------------------------------------------------------- /QBase/Server.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_SERVER_H 3 | #define BERT_SERVER_H 4 | 5 | #include 6 | #include 7 | #include "TaskManager.h" 8 | 9 | struct SocketAddr; 10 | 11 | class Server // Should singleton 12 | { 13 | protected: 14 | virtual bool _RunLogic(); 15 | virtual void _Recycle() { } 16 | virtual bool _Init() = 0; 17 | 18 | Server(); 19 | 20 | Server(const Server& ) = delete; 21 | void operator= (const Server& ) = delete; 22 | 23 | public: 24 | virtual ~Server(); 25 | 26 | bool TCPBind(const SocketAddr& listenAddr, int tag); 27 | void TCPReconnect(const SocketAddr& peer, int tag); 28 | 29 | static Server* Instance() { return sinstance_; } 30 | 31 | bool IsTerminate() const { return bTerminate_; } 32 | void Terminate() { bTerminate_ = true; } 33 | 34 | void MainLoop(bool daemon = false); 35 | void NewConnection(int sock, int tag, const std::function& cb = std::function()); 36 | 37 | void TCPConnect(const SocketAddr& peer, int tag); 38 | void TCPConnect(const SocketAddr& peer, const std::function& cb, int tag); 39 | 40 | size_t TCPSize() const { return tasks_.TCPSize(); } 41 | 42 | // SIGHUP handler, in fact, you should load config use this function; 43 | virtual void ReloadConfig() { } 44 | 45 | static void IntHandler(int sig); 46 | static void HupHandler(int sig); 47 | 48 | std::shared_ptr FindTCP(unsigned int id) const { return tasks_.FindTCP(id); } 49 | 50 | static void AtForkHandler(); 51 | static void DelListenSock(int sock); 52 | 53 | private: 54 | virtual std::shared_ptr _OnNewConnection(int tcpsock, int tag); 55 | void _TCPConnect(const SocketAddr& peer, const std::function* cb = nullptr, int tag = -1); 56 | 57 | std::atomic bTerminate_; 58 | Internal::TaskManager tasks_; 59 | bool reloadCfg_; 60 | static Server* sinstance_; 61 | 62 | static std::set slistenSocks_; 63 | }; 64 | 65 | #endif 66 | 67 | -------------------------------------------------------------------------------- /QBase/Log/MemoryFile.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_MEMORYFILE_H 2 | #define BERT_MEMORYFILE_H 3 | 4 | #include 5 | #include 6 | 7 | class InputMemoryFile 8 | { 9 | public: 10 | InputMemoryFile(); 11 | ~InputMemoryFile(); 12 | 13 | bool Open(const char* file); 14 | void Close(); 15 | 16 | void Attach(const char* data, size_t len); 17 | 18 | const char* Read(std::size_t& len); 19 | template 20 | T Read(); 21 | void Skip(std::size_t len); 22 | 23 | bool IsOpen() const; 24 | 25 | private: 26 | bool _MapReadOnly(); 27 | void _CheckAvail(std::size_t len); 28 | 29 | int file_; 30 | char* memory_; 31 | std::size_t offset_; 32 | std::size_t size_; 33 | }; 34 | 35 | template 36 | inline T InputMemoryFile::Read() 37 | { 38 | _CheckAvail(sizeof(T)); 39 | 40 | T res(*reinterpret_cast(memory_ + offset_)); 41 | offset_ += sizeof(T); 42 | 43 | return res; 44 | } 45 | 46 | 47 | class OutputMemoryFile 48 | { 49 | public: 50 | OutputMemoryFile(); 51 | ~OutputMemoryFile(); 52 | 53 | bool Open(const std::string& file, bool bAppend = true); 54 | bool Open(const char* file, bool bAppend = true); 55 | void Close(); 56 | bool Sync(); 57 | 58 | void Truncate(std::size_t size); 59 | //!! if process terminated abnormally, erase the trash data 60 | //requirement: text file. 61 | void TruncateTailZero(); 62 | 63 | void Write(const void* data, std::size_t len); 64 | template 65 | void Write(const T& t); 66 | 67 | std::size_t Offset() const { return offset_; } 68 | bool IsOpen() const; 69 | 70 | private: 71 | bool _MapWriteOnly(); 72 | void _ExtendFileSize(std::size_t size); 73 | void _AssureSpace(std::size_t size); 74 | 75 | int file_; 76 | char* memory_; 77 | std::size_t offset_; 78 | std::size_t size_; 79 | std::size_t syncPos_; 80 | }; 81 | 82 | 83 | template 84 | inline void OutputMemoryFile::Write(const T& t) 85 | { 86 | this->Write(&t, sizeof t); 87 | } 88 | 89 | #endif 90 | 91 | -------------------------------------------------------------------------------- /cluster/README.md: -------------------------------------------------------------------------------- 1 | # Qedis分布式集群 2 | 3 | ## 环境需求 4 | * C++11、CMake 5 | * zookeeper 6 | * Linux 或 MAC OS 7 | 8 | ## 代码目录 9 | 10 | * ananas 11 | 12 | 一个C++11编写的网络库，提供了强大的future异步编程模式. 13 | 14 | * cluster_conn 15 | 16 | 针对zookeeper或etcd的包装。目前只提供了zookeeper. 17 | 18 | * qedis_proxy 19 | 20 | Qedis代理服务器，负责发现Qedis服务，转发客户端请求和Qedis服务器响应. 21 | 22 | ## Future模式 23 | 本目录代码采用了基于Future模式的异步编程,例如与zookeeper集群连接时,需要进行7个步骤: 24 | 25 | 26 | * 握手 27 | * 注册自己，同时获取Qedis set信息,这是两个并行的异步请求 28 | * 根据set信息，获取分片信息 29 | * 存储分片信息 30 | * 发起异步请求:获取Qedis服务列表 31 | * 存储Qedis服务信息 32 | * 初始化ping定时器 33 | * 如若以上操作任一无响应，触发超时逻辑 34 | 这一连串的操作使用future模式编写如下,每一个Then都是前面异步请求的回调: 35 | ```cpp 36 | ctx_->DoHandshake() 37 | .Then([me = this](const ZkResponse& rsp) mutable { 38 | return me->_ProcessHandshake(rsp); 39 | }) 40 | .Then([me = this](ananas::Try&& tctx) mutable { 41 | return me->_RegisterAndGetServers(std::move(tctx)); 42 | }) 43 | .Then([me = this](const std::vector >& rsps) mutable { 44 | return me->_GetShardingInfo(rsps); 45 | }) 46 | .Then([me = this](const std::vector >& vrsp) mutable { 47 | if (!me->_ProcessShardingInfo(vrsp)) { 48 | using InnerType = std::vector>; 49 | auto exp = std::runtime_error("ProcessShardingInfo failed"); 50 | return ananas::MakeExceptionFuture(exp); 51 | } 52 | 53 | // 5. get qedis server's list and watch the qedis server list 54 | return me->_GetServers(vrsp); 55 | }) 56 | .Then([me = this](const std::vector >& vrsp) mutable { 57 | return me->_ProcessServerInfo(vrsp); 58 | }) 59 | .Then([me = this](bool succ) { 60 | if (succ) 61 | me->_InitPingTimer(); 62 | }) 63 | .OnTimeout(std::chrono::seconds(3), []() { 64 | // 3秒钟超时 65 | std::cout << "OnTimeout handshake\n"; 66 | ananas::EventLoop::ExitApplication(); 67 | }, conn_->GetLoop() 68 | ); 69 | ``` 70 | 71 | ## 集群特性 72 | 待写，代码实现中。。。 73 | 74 | -------------------------------------------------------------------------------- /QedisCore/QDB.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QDB_H 2 | #define BERT_QDB_H 3 | 4 | #include "Log/MemoryFile.h" 5 | #include "QStore.h" 6 | 7 | namespace qedis 8 | { 9 | 10 | class QDBSaver 11 | { 12 | public: 13 | explicit 14 | QDBSaver(const char* file = nullptr); 15 | void Save(const char* qdbFile); 16 | void SaveType(const QObject& obj); 17 | void SaveKey(const QString& key); 18 | void SaveObject(const QObject& obj); 19 | void SaveString(const QString& str); 20 | void SaveLength(uint64_t len); // big endian 21 | void SaveString(int64_t intVal); 22 | bool SaveLZFString(const QString& str); 23 | 24 | static void SaveDoneHandler(int exit, int signal); 25 | 26 | private: 27 | void _SaveDoubleValue(double val); 28 | 29 | void _SaveList(const PLIST& l); 30 | void _SaveSet(const PSET& s); 31 | void _SaveHash(const PHASH& h); 32 | void _SaveSSet(const PSSET& ss); 33 | 34 | OutputMemoryFile qdb_; 35 | }; 36 | 37 | extern time_t g_lastQDBSave; 38 | extern pid_t g_qdbPid; 39 | 40 | class QDBLoader 41 | { 42 | public: 43 | explicit 44 | QDBLoader(const char* data = nullptr, size_t len = 0); 45 | int Load(const char* filename); 46 | 47 | int8_t LoadByte(); 48 | size_t LoadLength(bool& special); 49 | QObject LoadSpecialStringObject(size_t specialVal); 50 | QString LoadString(size_t strLen); 51 | QString LoadLZFString(); 52 | 53 | QString LoadKey(); 54 | QObject LoadObject(int8_t type); 55 | 56 | private: 57 | QString _LoadGenericString(); 58 | QObject _LoadList(); 59 | QObject _LoadSet(); 60 | QObject _LoadHash(); 61 | QObject _LoadSSet(); 62 | double _LoadDoubleValue(); 63 | QObject _LoadZipList(int8_t type); 64 | QObject _LoadZipList(const QString& zl, int8_t type); 65 | QObject _LoadIntset(); 66 | QObject _LoadQuickList(); 67 | 68 | void _LoadAux(); 69 | void _LoadResizeDB(); 70 | 71 | InputMemoryFile qdb_; 72 | }; 73 | 74 | std::string DumpObject(const QObject& val); 75 | QObject RestoreObject(const char* data, size_t len); 76 | 77 | } 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /cluster/ananas/net/TimeUtil.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "TimeUtil.h" 3 | 4 | namespace ananas 5 | { 6 | 7 | Time::Time() : valid_(false) 8 | { 9 | this->Now(); 10 | } 11 | 12 | int64_t Time::MilliSeconds() const 13 | { 14 | return std::chrono::duration_cast(now_.time_since_epoch()).count(); 15 | } 16 | 17 | int64_t Time::MicroSeconds() const 18 | { 19 | return std::chrono::duration_cast(now_.time_since_epoch()).count(); 20 | } 21 | 22 | void Time::Now() 23 | { 24 | now_ = std::chrono::system_clock::now(); 25 | valid_ = false; 26 | } 27 | 28 | void Time::_UpdateTm() const 29 | { 30 | // lazy compute 31 | if (valid_) 32 | return; 33 | 34 | valid_ = true; 35 | const time_t now(MilliSeconds() / 1000UL); 36 | ::localtime_r(&now, &tm_); 37 | } 38 | 39 | std::once_flag Time::init_; 40 | 41 | // from 2015 to 2029 42 | const char* Time::YEAR[] = { "2015", "2016", "2017", "2018", "2019", "2020", 43 | "2021", "2022", "2023", "2024", "2025", "2026", "2027", "2028", "2029", 44 | nullptr, 45 | }; 46 | 47 | char Time::NUMBER[60][2] = {""}; 48 | 49 | void Time::Init() 50 | { 51 | for (size_t i = 0; i < sizeof NUMBER / sizeof NUMBER[0]; ++ i) 52 | { 53 | char tmp[3]; 54 | snprintf(tmp, 3, "%02d", static_cast(i)); 55 | memcpy(NUMBER[i], tmp, 2); 56 | } 57 | } 58 | 59 | std::size_t Time::FormatTime(char* buf) const 60 | { 61 | std::call_once(init_, &Time::Init); 62 | 63 | _UpdateTm(); 64 | 65 | memcpy(buf, YEAR[tm_.tm_year + 1900 - 2015], 4); 66 | buf[4] = '-'; 67 | memcpy(buf + 5, NUMBER[tm_.tm_mon + 1], 2); 68 | buf[7] = '-'; 69 | memcpy(buf + 8, NUMBER[tm_.tm_mday], 2); 70 | buf[10] = '['; 71 | memcpy(buf + 11, NUMBER[tm_.tm_hour], 2); 72 | buf[13] = ':'; 73 | memcpy(buf + 14, NUMBER[tm_.tm_min], 2); 74 | buf[16] = ':'; 75 | memcpy(buf + 17, NUMBER[tm_.tm_sec], 2); 76 | buf[19] = '.'; 77 | auto msec = MicroSeconds(); 78 | snprintf(buf + 20, 8, "%06d]", static_cast(msec % 1000)); 79 | 80 | return 27; 81 | } 82 | 83 | } // end namespace ananas 84 | 85 | -------------------------------------------------------------------------------- /QBase/Threads/ThreadPool.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_THREADPOOL_H 2 | #define BERT_THREADPOOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | 14 | class ThreadPool final 15 | { 16 | public: 17 | ~ThreadPool(); 18 | 19 | ThreadPool(const ThreadPool& ) = delete; 20 | void operator=(const ThreadPool& ) = delete; 21 | 22 | static ThreadPool& Instance(); 23 | 24 | template 25 | auto ExecuteTask(F&& f, Args&&... args) -> std::future::type>; 26 | 27 | void JoinAll(); 28 | void SetMaxIdleThread(unsigned int m); 29 | 30 | private: 31 | ThreadPool(); 32 | 33 | void _CreateWorker(); 34 | void _WorkerRoutine(); 35 | void _MonitorRoutine(); 36 | 37 | std::thread monitor_; 38 | std::atomic maxIdleThread_; 39 | std::atomic pendingStopSignal_; 40 | 41 | static __thread bool working_; 42 | std::deque workers_; 43 | 44 | std::mutex mutex_; 45 | std::condition_variable cond_; 46 | unsigned waiters_; 47 | bool shutdown_; 48 | std::deque > tasks_; 49 | 50 | static const int kMaxThreads = 256; 51 | }; 52 | 53 | 54 | template 55 | auto ThreadPool::ExecuteTask(F&& f, Args&&... args) -> std::future::type> 56 | { 57 | using resultType = typename std::result_of::type; 58 | 59 | auto task = std::make_shared >(std::bind(std::forward(f), std::forward(args)...)); 60 | 61 | { 62 | std::unique_lock guard(mutex_); 63 | if (shutdown_) 64 | { 65 | return std::future(); 66 | } 67 | 68 | tasks_.emplace_back( [=]() { (*task)(); } ); 69 | if (waiters_ == 0) 70 | { 71 | _CreateWorker(); 72 | } 73 | 74 | cond_.notify_one(); 75 | } 76 | 77 | return task->get_future(); 78 | } 79 | 80 | #endif 81 | 82 | -------------------------------------------------------------------------------- /QSentinel/QClusterClient.cc: -------------------------------------------------------------------------------- 1 | #if QEDIS_CLUSTER 2 | 3 | #include "Log/Logger.h" 4 | #include "Server.h" 5 | 6 | #include "QConfig.h" 7 | #include "QCommand.h" 8 | #include "QClusterClient.h" 9 | 10 | #if USE_ZOOKEEPER 11 | #include "zookeeper/ZookeeperConn.h" 12 | #endif 13 | 14 | namespace qedis 15 | { 16 | 17 | PacketLength QClusterClient::_HandlePacket(const char* data, std::size_t bytes) 18 | { 19 | const char* ptr = data; 20 | 21 | bool result = conn_->ParseMessage(ptr, bytes); 22 | if (!result) 23 | { 24 | OnError(); 25 | return 0; 26 | } 27 | 28 | return static_cast(ptr - data); 29 | } 30 | 31 | bool QClusterClient::Init(int fd, const SocketAddr& peer) 32 | { 33 | if (!StreamSocket::Init(fd, peer)) 34 | return false; 35 | 36 | auto me = std::static_pointer_cast(shared_from_this()); 37 | SocketAddr myAddr(g_config.ip.c_str(), g_config.port); 38 | 39 | #if USE_ZOOKEEPER 40 | QClusterConn* conn = new ZookeeperConn(me, g_config.setid, myAddr.ToString()); 41 | #else 42 | #error "Only support zookeeper for now, supporting etcd is in progress" 43 | #endif 44 | 45 | conn->SetOnBecomeMaster([](const std::vector& slaves) { 46 | INF << "I become master"; 47 | std::vector cmd {"slaveof", "no", "one"}; 48 | slaveof(cmd, nullptr); 49 | 50 | for (const auto& addr : slaves) 51 | { 52 | INF << "Try connect to slave " << addr.ToString(); 53 | // connect to slaves and send 'slave of me' 54 | Server::Instance()->TCPConnect(addr, ConnectionTag::kSlaveClient); 55 | } 56 | // 读取所有set的数据保存其版本号 57 | // *set的数据格式是 1,3,4,7|2:1,4 58 | // 含义:本set负责slot 1347，但是现在正在将slot 1,4迁移到set2上 59 | }); 60 | 61 | conn->SetOnBecomeSlave([](const std::string& master) { 62 | INF << "I become slave of " << master; 63 | std::vector cmd(SplitString(master, ':')); 64 | slaveof({"slaveof", cmd[0], cmd[1]}, nullptr); 65 | }); 66 | 67 | conn_.reset(conn); 68 | 69 | return true; 70 | } 71 | 72 | void QClusterClient::OnConnect() 73 | { 74 | conn_->OnConnect(); 75 | } 76 | 77 | void QClusterClient::OnDisconnect() 78 | { 79 | conn_->OnDisconnect(); 80 | } 81 | 82 | } // end namespace qedis 83 | 84 | #endif 85 | 86 | -------------------------------------------------------------------------------- /QedisCore/QConfig.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QCONFIG_H 2 | #define BERT_QCONFIG_H 3 | 4 | #include 5 | #include 6 | #include "QString.h" 7 | 8 | namespace qedis 9 | { 10 | 11 | enum BackEndType 12 | { 13 | BackEndNone = 0, 14 | BackEndLeveldb = 1, 15 | BackEndMax = 2, 16 | }; 17 | 18 | struct QConfig 19 | { 20 | bool daemonize; 21 | QString pidfile; 22 | 23 | QString ip; 24 | unsigned short port; 25 | 26 | int timeout; 27 | 28 | QString loglevel; 29 | QString logdir; // the log directory, differ from redis 30 | 31 | int databases; 32 | 33 | // auth 34 | QString password; 35 | 36 | std::map aliases; 37 | 38 | // @ rdb 39 | // save seconds changes 40 | int saveseconds; 41 | int savechanges; 42 | bool rdbcompression; // yes 43 | bool rdbchecksum; // yes 44 | QString rdbfullname; // ./dump.rdb 45 | 46 | int maxclients; // 10000 47 | 48 | bool appendonly; // no 49 | QString appendfilename; // appendonly.aof 50 | int appendfsync; // no, everysec, always 51 | 52 | int slowlogtime; // 1000 microseconds 53 | int slowlogmaxlen; // 128 54 | 55 | int hz; // 10 [1,500] 56 | 57 | QString masterIp; 58 | unsigned short masterPort; // replication 59 | QString masterauth; 60 | 61 | QString runid; 62 | 63 | QString includefile; // the template config 64 | 65 | std::vector modules; // modules 66 | 67 | // use redis as cache, level db as backup 68 | uint64_t maxmemory; // default 2GB 69 | int maxmemorySamples; // default 5 70 | bool noeviction; // default true 71 | 72 | int backend; // enum BackEndType 73 | QString backendPath; 74 | int backendHz; // the frequency of dump to backend 75 | 76 | // cluster 77 | bool enableCluster = false; 78 | std::vector centers; 79 | int setid = -1; // sharding set id 80 | 81 | QConfig(); 82 | 83 | bool CheckArgs() const; 84 | bool CheckPassword(const QString& pwd) const; 85 | }; 86 | 87 | extern QConfig g_config; 88 | 89 | extern bool LoadQedisConfig(const char* cfgFile, QConfig& cfg); 90 | 91 | } 92 | 93 | #endif 94 | 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | _____ _____ _____ _ ______ 3 | / _ \ | ____| | _ \ | | / ___/ 4 | | | | | | |__ | | | | | | | |____ 5 | | | | | | __| | | | | | | \__ \ 6 | | |_| |_ | |___ | |_| | | | ___| | 7 | \_______| |_____| |_____/ |_| /_____/ 8 | 9 | 10 | [![Build Status](https://travis-ci.org/loveyacper/Qedis.svg?branch=master)](https://travis-ci.org/loveyacper/Qedis) 11 | 12 | [看中文说明请点我](README.zh.md) 13 | 14 | A C++11 implementation of distributed redis server, use Leveldb for persist storage.(including cluster) 15 | 16 | ## Requirements 17 | * C++11 & CMake 18 | * Linux or OS X 19 | 20 | ## Cluster Features 21 | Use zookeeper for leader election, to reach high availability. 22 | 23 | Of course you can also use redis-sentinel. 24 | 25 | See details in [cluster Readme](QCluster/README.md), still in development. 26 | 27 | ## Fully compatible with redis 28 | You can test Qedis with redis-cli, redis-benchmark, or use redis as master with Qedis as slave or conversely, it also can work with redis sentinel. 29 | In a word, Qedis is full compatible with Redis. 30 | 31 | ## Support module for write your own extensions 32 | Qedis supports module now, still in progress, much work to do. 33 | I added three commands(ldel, skeys, hgets) for demonstration. 34 | 35 | ## Persistence: Not limited to memory 36 | Leveldb can be configured as backend for Qedis. 37 | 38 | ## High Performance 39 | - Qedis is approximately 20-25% faster than redis if run benchmark with pipeline requests(set -P = 50 or higher). 40 | - Average 80K requests per seconds for write, and 90K requests per seconds for read. 41 | - Before run test, please ensure that std::list::size() is O(1), obey the C++11 standards. 42 | 43 | Run this command, compare with redis use pipeline commands, try it. 44 | ```bash 45 | ./redis-benchmark -q -n 1000000 -P 50 -c 50 46 | ``` 47 | 48 | ![image](https://github.com/loveyacper/Qedis/blob/master/performance.png) 49 | 50 | ## Support LRU cache 51 | When memory is low, you can make Qedis to free memory by evict some key according to LRU. 52 | 53 | ## Master-slave Replication, transaction, RDB/AOF, slow log, publish-subscribe 54 | Qedis supports them all :-) 55 | 56 | ## Command List 57 | #### show all supported commands list, about 140 commands 58 | - cmdlist 59 | 60 | ## TODO 61 | * Support lua 62 | * Golang Cluster client 63 | 64 | -------------------------------------------------------------------------------- /QedisCore/redisIntset.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2012, Pieter Noordhuis 3 | * Copyright (c) 2009-2012, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef __INTSET_H 32 | #define __INTSET_H 33 | #include 34 | 35 | typedef struct intset { 36 | uint32_t encoding; 37 | uint32_t length; 38 | int8_t contents[]; 39 | } intset; 40 | 41 | intset *intsetNew(void); 42 | intset *intsetAdd(intset *is, int64_t value, uint8_t *success); 43 | intset *intsetRemove(intset *is, int64_t value, int *success); 44 | uint8_t intsetFind(intset *is, int64_t value); 45 | int64_t intsetRandom(intset *is); 46 | uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value); 47 | uint32_t intsetLen(intset *is); 48 | size_t intsetBlobLen(intset *is); 49 | 50 | #endif // __INTSET_H 51 | -------------------------------------------------------------------------------- /QBase/NetThreadPool.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_NETTHREADPOOL_H 2 | #define BERT_NETTHREADPOOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "Poller.h" 11 | #include "Threads/ThreadPool.h" 12 | 13 | inline long GetCpuNum() 14 | { 15 | return sysconf(_SC_NPROCESSORS_ONLN); 16 | } 17 | 18 | class Socket; 19 | typedef std::shared_ptr PSOCKET; 20 | 21 | namespace Internal 22 | { 23 | 24 | 25 | class NetThread 26 | { 27 | public: 28 | NetThread(); 29 | virtual ~NetThread(); 30 | 31 | bool IsAlive() const { return running_; } 32 | void Stop() { running_ = false;} 33 | 34 | void AddSocket(PSOCKET , uint32_t event); 35 | void ModSocket(PSOCKET , uint32_t event); 36 | void RemoveSocket(PSOCKET, uint32_t event); 37 | 38 | protected: 39 | std::unique_ptr poller_; 40 | std::vector firedEvents_; 41 | std::deque tasks_; 42 | void _TryAddNewTasks(); 43 | 44 | private: 45 | std::atomic running_; 46 | 47 | std::mutex mutex_; 48 | typedef std::vector, uint32_t> > NewTasks; 49 | NewTasks newTasks_; 50 | std::atomic newCnt_; 51 | void _AddSocket(PSOCKET , uint32_t event); 52 | }; 53 | 54 | class RecvThread : public NetThread 55 | { 56 | public: 57 | void Run(); 58 | }; 59 | 60 | class SendThread : public NetThread 61 | { 62 | public: 63 | void Run(); 64 | }; 65 | 66 | 67 | /////////////////////////////////////////////// 68 | class NetThreadPool 69 | { 70 | std::shared_ptr recvThread_; 71 | std::shared_ptr sendThread_; 72 | 73 | public: 74 | NetThreadPool() = default; 75 | 76 | NetThreadPool(const NetThreadPool& ) = delete; 77 | void operator= (const NetThreadPool& ) = delete; 78 | 79 | bool AddSocket(PSOCKET , uint32_t event); 80 | bool StartAllThreads(); 81 | void StopAllThreads(); 82 | 83 | void EnableRead(const std::shared_ptr& sock); 84 | void EnableWrite(const std::shared_ptr& sock); 85 | void DisableRead(const std::shared_ptr& sock); 86 | void DisableWrite(const std::shared_ptr& sock); 87 | 88 | static NetThreadPool& Instance() 89 | { 90 | static NetThreadPool pool; 91 | return pool; 92 | } 93 | }; 94 | 95 | } 96 | 97 | #endif 98 | 99 | -------------------------------------------------------------------------------- /QSentinel/zookeeper/ZookeeperConn.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ZOOKEEPERCONN_H 2 | #define BERT_ZOOKEEPERCONN_H 3 | 4 | #include 5 | #include 6 | #include "StreamSocket.h" 7 | #include "../QClusterInterface.h" 8 | 9 | struct prime_struct; 10 | struct RequestHeader; 11 | struct ReplyHeader; 12 | struct iarchive; 13 | struct oarchive; 14 | 15 | class Timer; 16 | 17 | namespace qedis 18 | { 19 | 20 | class ZookeeperConn : public QClusterConn 21 | { 22 | public: 23 | ZookeeperConn(const std::shared_ptr& c, int setId, const std::string& addr); 24 | ~ZookeeperConn(); 25 | 26 | bool ParseMessage(const char*& data, size_t len) override; 27 | void OnConnect() override; 28 | void OnDisconnect() override; 29 | 30 | private: 31 | void _RunForMaster(int setid, const std::string& val); 32 | bool _ProcessHandshake(const prime_struct& rsp); 33 | bool _ProcessResponse(const ReplyHeader& header, iarchive* ia); 34 | bool _ProcessWatchEvent(const ReplyHeader& header, iarchive* ia); 35 | bool _GetSiblings(const std::string& parent); 36 | bool _GetData(const std::string& node, bool watch = true); 37 | bool _Exists(const std::string& sibling, bool watch = true); 38 | void _InitPingTimer(); 39 | bool _IsMaster() const; 40 | bool _SendPacket(const RequestHeader& h, struct oarchive* oa, const std::string* = nullptr); 41 | void _OnNodeDelete(const std::string& node); 42 | 43 | int _GetXid() const; 44 | mutable int xid_; 45 | 46 | const int setId_; 47 | const std::string addr_; 48 | 49 | enum class State 50 | { 51 | kNone, 52 | kHandshaking, 53 | kConnected, 54 | } state_; 55 | 56 | struct Request 57 | { 58 | int type; 59 | int xid; 60 | std::string path; 61 | }; 62 | std::list pendingRequests_; 63 | 64 | timeval lastPing_; 65 | 66 | // my node & seq 67 | std::string node_; 68 | int seq_; 69 | // nodes in my set 70 | std::map siblings_; 71 | 72 | #pragma pack(1) 73 | struct SessionInfo 74 | { 75 | int64_t sessionId{0}; 76 | char passwd[16]; 77 | // ? zk_cli doesn't persist this field 78 | int64_t lastSeenZxid {0}; 79 | } sessionInfo_; 80 | #pragma pack() 81 | 82 | std::string sessionFile_; 83 | Timer* pingTimer_; 84 | std::weak_ptr sock_; 85 | }; 86 | 87 | } // end namespace qedis 88 | 89 | #endif //endif BERT_ZOOKEEPERCONN_H 90 | 91 | -------------------------------------------------------------------------------- /QBase/Delegate.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | 5 | template 6 | class Delegate; 7 | 8 | template 9 | class Delegate 10 | { 11 | public: 12 | typedef Delegate Self; 13 | 14 | Delegate() = default; 15 | 16 | Delegate(const Self& ) = delete; 17 | Self& operator= (const Self& ) = delete; 18 | 19 | template 20 | Delegate(F&& f) 21 | { 22 | connect(std::forward(f)); 23 | } 24 | 25 | Delegate(Self&& other) : funcs_(std::move(other.funcs_)) 26 | { 27 | } 28 | 29 | template 30 | Self& operator+=(F&& f) 31 | { 32 | connect(std::forward(f)); 33 | return *this; 34 | } 35 | 36 | template 37 | Self& operator-=(F&& f) 38 | { 39 | disconnect(std::forward(f)); 40 | return *this; 41 | } 42 | 43 | void operator()(Args&&... args) 44 | { 45 | call(std::forward(args)...); 46 | } 47 | 48 | private: 49 | std::list > funcs_; 50 | 51 | template 52 | void connect(F&& f) 53 | { 54 | funcs_.emplace_back(std::forward(f)); 55 | } 56 | 57 | template 58 | void disconnect(F&& f) 59 | { 60 | //std::cerr << "&f = " << typeid(&f).name() << ", and target = " << typeid(decltype(std::addressof(f))).name() << std::endl; 61 | for (auto it(funcs_.begin()); it != funcs_.end(); ++ it) 62 | { 63 | const auto& target = it->template target(); 64 | if (target) 65 | { 66 | if(*target == &f) 67 | { 68 | funcs_.erase(it); 69 | return; 70 | } 71 | } 72 | else 73 | { 74 | const auto& target2 = it->template target::type>(); 75 | 76 | // the function object must implement operator == 77 | if (target2 && *target2 == f) 78 | { 79 | funcs_.erase(it); 80 | return; 81 | } 82 | } 83 | } 84 | } 85 | 86 | void call(Args&&... args) 87 | { 88 | // But what if rvalue args? FIXME 89 | for (const auto& f : funcs_) 90 | f(std::forward(args)...); 91 | } 92 | }; 93 | 94 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/Protocol.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_PROTOCOL_H 2 | #define BERT_PROTOCOL_H 3 | 4 | #include 5 | #include 6 | 7 | enum class ParseResult : int8_t 8 | { 9 | ok, 10 | wait, 11 | error, 12 | }; 13 | 14 | class ServerProtocol 15 | { 16 | public: 17 | void Reset(); 18 | ParseResult ParseRequest(const char*& ptr, const char* end); 19 | 20 | const std::vector& GetParams() const & { return params_; } 21 | std::vector& GetParams()& { return params_; } 22 | std::vector&& GetParams()&& { return std::move(params_); } 23 | 24 | const std::string& GetRawRequest() const & { return content_; } 25 | std::string& GetRawRequest() & { return content_; } 26 | std::string&& GetRawRequest() && { return std::move(content_); } 27 | 28 | bool IsInitialState() const { return multi_ == -1; } 29 | 30 | private: 31 | ParseResult _ParseMulti(const char*& ptr, const char* end, int& result); 32 | ParseResult _ParseStrlist(const char*& ptr, const char* end, std::vector& results); 33 | ParseResult _ParseStr(const char*& ptr, const char* end, std::string& res); 34 | ParseResult _ParseStrval(const char*& ptr, const char* end, std::string& res); 35 | ParseResult _ParseStrlen(const char*& ptr, const char* end, int& result); 36 | 37 | int multi_ = -1; 38 | int paramLen_ = -1; 39 | std::vector params_; 40 | int nParams_ = 0; 41 | std::string content_; 42 | }; 43 | 44 | 45 | enum class ResponseType 46 | { 47 | None, 48 | Fine, // + 49 | Error, // - 50 | String, // $ 51 | Number, // : 52 | Multi, // * 53 | }; 54 | 55 | class ClientProtocol 56 | { 57 | public: 58 | ClientProtocol(); 59 | 60 | void Reset(); 61 | ParseResult Parse(const char*& data, const char* end); 62 | 63 | const std::string& GetContent() const& { return content_; } 64 | std::string& GetContent() & { return content_; } 65 | std::string&& GetContent()&& { return std::move(content_); } 66 | 67 | private: 68 | // $3\r\nabc\r\n 69 | ParseResult _ParseStr(const char*& ptr, const char* end); 70 | ParseResult _ParseStrval(const char*& ptr, const char* end); 71 | ParseResult _ParseStrlen(const char*& ptr, const char* end, int& result); 72 | 73 | // *2\r\n$4\r\nname\r\n$4\r\nbert\r\n 74 | ParseResult _ParseMulti(const char*& ptr, const char* end); 75 | 76 | static const int kInvalid = -2; 77 | 78 | ResponseType type_; 79 | std::string content_; 80 | int multi_ = kInvalid; 81 | int paramLen_ = kInvalid; 82 | int nParams_ = 0; 83 | }; 84 | 85 | 86 | #endif 87 | 88 | -------------------------------------------------------------------------------- /cluster/ananas/util/Delegate.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_DELEGATE_H 2 | #define BERT_DELEGATE_H 3 | 4 | #include 5 | #include 6 | 7 | 8 | namespace ananas 9 | { 10 | 11 | template 12 | class Delegate; 13 | 14 | template 15 | class Delegate 16 | { 17 | public: 18 | typedef Delegate Self; 19 | 20 | Delegate() = default; 21 | 22 | Delegate(const Self& ) = delete; 23 | Self& operator= (const Self& ) = delete; 24 | 25 | template 26 | Delegate(F&& f) 27 | { 28 | connect(std::forward(f)); 29 | } 30 | 31 | Delegate(Self&& other) : 32 | funcs_(std::move(other.funcs_)) 33 | { 34 | } 35 | 36 | template 37 | Self& operator+=(F&& f) 38 | { 39 | connect(std::forward(f)); 40 | return *this; 41 | } 42 | 43 | template 44 | Self& operator-=(F&& f) 45 | { 46 | disconnect(std::forward(f)); 47 | return *this; 48 | } 49 | 50 | template 51 | void operator()(ARGS&&... args) 52 | { 53 | call(std::forward(args)...); 54 | } 55 | 56 | private: 57 | std::list > funcs_; 58 | 59 | template 60 | void connect(F&& f) 61 | { 62 | funcs_.emplace_back(std::forward(f)); 63 | } 64 | 65 | template 66 | void disconnect(F&& f) 67 | { 68 | for (auto it(funcs_.begin()); it != funcs_.end(); ++ it) 69 | { 70 | const auto& target = it->template target(); 71 | if (target) 72 | { 73 | if (*target == &f) 74 | { 75 | funcs_.erase(it); 76 | return; 77 | } 78 | } 79 | else 80 | { 81 | const auto& target2 = it->template target::type>(); 82 | 83 | // the function object must implement operator == 84 | if (target2 && *target2 == f) 85 | { 86 | funcs_.erase(it); 87 | return; 88 | } 89 | } 90 | } 91 | } 92 | 93 | template 94 | void call(ARGS&&... args) 95 | { 96 | // But what if rvalue args? 97 | for (const auto& f : funcs_) 98 | f(std::forward(args)...); 99 | } 100 | }; 101 | 102 | } // end namespace ananas 103 | 104 | #endif 105 | 106 | -------------------------------------------------------------------------------- /QedisCore/redisZipList.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2012, Pieter Noordhuis 3 | * Copyright (c) 2009-2012, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #define ZIPLIST_HEAD 0 32 | #define ZIPLIST_TAIL 1 33 | 34 | unsigned char *ziplistNew(void); 35 | unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where); 36 | unsigned char *ziplistIndex(unsigned char *zl, int index); 37 | unsigned char *ziplistNext(unsigned char *zl, unsigned char *p); 38 | unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p); 39 | unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval); 40 | unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen); 41 | unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p); 42 | unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num); 43 | unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen); 44 | unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip); 45 | unsigned int ziplistLen(unsigned char *zl); 46 | size_t ziplistBlobLen(unsigned char *zl); 47 | -------------------------------------------------------------------------------- /QBase/TaskManager.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "TaskManager.h" 3 | #include "StreamSocket.h" 4 | #include "Log/Logger.h" 5 | 6 | namespace Internal 7 | { 8 | 9 | TaskManager::~TaskManager() 10 | { 11 | assert(Empty() && "Why you do not clear container before exit?"); 12 | } 13 | 14 | 15 | bool TaskManager::AddTask(PTCPSOCKET task) 16 | { 17 | std::lock_guard guard(lock_); 18 | newTasks_.push_back(task); 19 | ++ newCnt_; 20 | 21 | return true; 22 | } 23 | 24 | TaskManager::PTCPSOCKET TaskManager::FindTCP(unsigned int id) const 25 | { 26 | if (id > 0) 27 | { 28 | auto it = tcpSockets_.find(id); 29 | if (it != tcpSockets_.end()) 30 | return it->second; 31 | } 32 | 33 | return PTCPSOCKET(); 34 | } 35 | 36 | bool TaskManager::_AddTask(PTCPSOCKET task) 37 | { 38 | //bool succ = tcpSockets_.insert(std::map::value_type(task->GetID(), task)).second; 39 | bool succ = tcpSockets_.insert({task->GetID(), task}).second; 40 | return succ; 41 | } 42 | 43 | 44 | void TaskManager::_RemoveTask(std::map::iterator& it) 45 | { 46 | tcpSockets_.erase(it ++); 47 | } 48 | 49 | 50 | bool TaskManager::DoMsgParse() 51 | { 52 | if (newCnt_ > 0 && lock_.try_lock()) 53 | { 54 | NEWTASKS_T tmpNewTask; 55 | tmpNewTask.swap(newTasks_); 56 | newCnt_ = 0; 57 | lock_.unlock(); 58 | 59 | for (const auto& task : tmpNewTask) 60 | { 61 | if (!_AddTask(task)) 62 | { 63 | ERR << "Why can not insert tcp socket " 64 | << task->GetSocket() 65 | << ", id = " 66 | << task->GetID(); 67 | } 68 | else 69 | { 70 | INF << "New connection from " 71 | << task->GetPeerAddr().ToString() 72 | << ", id = " 73 | << task->GetID(); 74 | 75 | task->OnConnect(); 76 | } 77 | } 78 | } 79 | 80 | bool busy = false; 81 | 82 | for (auto it(tcpSockets_.begin()); it != tcpSockets_.end(); ) 83 | { 84 | if (!it->second || it->second->Invalid()) 85 | { 86 | if (it->second) 87 | { 88 | INF << "Close connection from " 89 | << it->second->GetPeerAddr().ToString() 90 | << ", id = " 91 | << it->second->GetID(); 92 | 93 | it->second->OnDisconnect(); 94 | } 95 | _RemoveTask(it); 96 | } 97 | else 98 | { 99 | if (it->second->DoMsgParse() && !busy) 100 | busy = true; 101 | 102 | ++ it; 103 | } 104 | } 105 | 106 | return busy; 107 | } 108 | 109 | } 110 | 111 | -------------------------------------------------------------------------------- /QBase/Threads/ThreadPool.cc: -------------------------------------------------------------------------------- 1 | #include "ThreadPool.h" 2 | 3 | __thread bool ThreadPool::working_ = true; 4 | 5 | ThreadPool::ThreadPool() : waiters_(0), shutdown_(false) 6 | { 7 | monitor_ = std::thread([this]() { this->_MonitorRoutine(); } ); 8 | maxIdleThread_ = std::max(1U, std::thread::hardware_concurrency()); 9 | pendingStopSignal_ = 0; 10 | } 11 | 12 | ThreadPool::~ThreadPool() 13 | { 14 | JoinAll(); 15 | } 16 | 17 | ThreadPool& ThreadPool::Instance() 18 | { 19 | static ThreadPool pool; 20 | return pool; 21 | } 22 | 23 | void ThreadPool::SetMaxIdleThread(unsigned int m) 24 | { 25 | if (0 < m && m <= kMaxThreads) 26 | maxIdleThread_ = m; 27 | } 28 | 29 | void ThreadPool::JoinAll() 30 | { 31 | decltype(workers_) tmp; 32 | 33 | { 34 | std::unique_lock guard(mutex_); 35 | if (shutdown_) 36 | return; 37 | 38 | shutdown_ = true; 39 | cond_.notify_all(); 40 | 41 | tmp.swap(workers_); 42 | workers_.clear(); 43 | } 44 | 45 | for (auto& t : tmp) 46 | { 47 | if (t.joinable()) 48 | t.join(); 49 | } 50 | 51 | if (monitor_.joinable()) 52 | monitor_.join(); 53 | } 54 | 55 | void ThreadPool::_CreateWorker() 56 | { 57 | std::thread t([this]() { this->_WorkerRoutine(); } ); 58 | workers_.push_back(std::move(t)); 59 | } 60 | 61 | void ThreadPool::_WorkerRoutine() 62 | { 63 | working_ = true; 64 | 65 | while (working_) 66 | { 67 | std::function task; 68 | 69 | { 70 | std::unique_lock guard(mutex_); 71 | 72 | ++ waiters_; 73 | cond_.wait(guard, [this]()->bool { return this->shutdown_ || !tasks_.empty(); } ); 74 | -- waiters_; 75 | 76 | if (this->shutdown_ && tasks_.empty()) 77 | return; 78 | 79 | task = std::move(tasks_.front()); 80 | tasks_.pop_front(); 81 | } 82 | 83 | task(); 84 | } 85 | 86 | // if reach here, this thread is recycled by monitor thread 87 | -- pendingStopSignal_; 88 | } 89 | 90 | void ThreadPool::_MonitorRoutine() 91 | { 92 | while (!shutdown_) 93 | { 94 | std::this_thread::sleep_for(std::chrono::seconds(1)); 95 | 96 | std::unique_lock guard(mutex_); 97 | if (shutdown_) 98 | return; 99 | 100 | auto nw = waiters_; 101 | 102 | // if there is any pending stop signal to consume waiters 103 | nw -= pendingStopSignal_; 104 | 105 | while (nw -- > maxIdleThread_) 106 | { 107 | tasks_.push_back([this]() { working_ = false; }); 108 | cond_.notify_one(); 109 | ++ pendingStopSignal_; 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /UnitTest/UnitTest.cc: -------------------------------------------------------------------------------- 1 | #include "UnitTest.h" 2 | #include 3 | 4 | enum Color 5 | { 6 | Color_red = 1, 7 | Color_green , 8 | Color_yellow , 9 | Color_normal , 10 | Color_blue , 11 | Color_purple , 12 | Color_white , 13 | Color_max , 14 | }; 15 | 16 | void SetColor(Color c) 17 | { 18 | static const char* colors[Color_max] = { 19 | "", 20 | "\033[1;31;40m", 21 | "\033[1;32;40m", 22 | "\033[1;33;40m", 23 | "\033[0m", 24 | "\033[1;34;40m", 25 | "\033[1;35;40m", 26 | "\033[1;37;40m", 27 | }; 28 | 29 | fprintf(stdout, "%s", colors[c]); 30 | } 31 | 32 | UnitTestBase::UnitTestBase() : pass_(true), abort_(false) 33 | { 34 | UnitTestManager::Instance().AddTest(this); 35 | } 36 | 37 | 38 | UnitTestBase& UnitTestBase::SetInfo(const std::string& exprInfo, bool pass, bool abort) 39 | { 40 | pass_ = pass; 41 | abort_ = abort; 42 | expr_ = (pass_ ? "[passed]: " : "[failed]: ") + exprInfo; 43 | return *this; 44 | } 45 | 46 | 47 | void UnitTestBase::Print() const 48 | { 49 | SetColor(Color_red); 50 | 51 | for (const auto& e : errors_) 52 | std::cout << e << std::endl; 53 | 54 | SetColor(Color_normal); 55 | } 56 | 57 | 58 | void UnitTestBase::FlushError() 59 | { 60 | if (pass_) 61 | { 62 | SetColor(Color_green); 63 | std::cout << expr_ << std::endl; 64 | return; 65 | } 66 | 67 | errors_.push_back(expr_); 68 | if (abort_) 69 | { 70 | Print(); 71 | ::abort(); 72 | } 73 | } 74 | 75 | 76 | // test mgr 77 | UnitTestManager& UnitTestManager::Instance() 78 | { 79 | static UnitTestManager mgr; 80 | return mgr; 81 | } 82 | 83 | void UnitTestManager::AddTest(UnitTestBase* test) 84 | { 85 | tests_.push_back(test); 86 | } 87 | 88 | void UnitTestManager::Clear() 89 | { 90 | tests_.clear(); 91 | } 92 | 93 | void UnitTestManager::Run() 94 | { 95 | std::size_t pass = 0; 96 | std::size_t fail = 0; 97 | 98 | for (const auto& ut : tests_) 99 | { 100 | ut->Run(); 101 | if (ut->IsFine()) 102 | { 103 | ++ pass; 104 | 105 | SetColor(Color_white); 106 | std::cout << "ALL PASSED! " << ut->GetName() << std::endl; 107 | } 108 | else 109 | { 110 | ++ fail; 111 | 112 | ut->Print(); 113 | SetColor(Color_purple); 114 | std::cout << "FAILED! " << ut->GetName() << std::endl; 115 | } 116 | } 117 | 118 | Clear(); 119 | 120 | SetColor(fail == 0 ? Color_blue: Color_yellow); 121 | std::cout << (pass + fail) << " cases: " 122 | << pass << " passed, " 123 | << fail << " failed\n"; 124 | SetColor(Color_normal); 125 | } 126 | 127 | int main() 128 | { 129 | RUN_ALL_TESTS(); 130 | return 0; 131 | } 132 | 133 | -------------------------------------------------------------------------------- /UnitTest/UnitTest.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_UNITTEST_H 2 | #define BERT_UNITTEST_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class UnitTestBase 9 | { 10 | public: 11 | friend class MsgHelper; 12 | 13 | UnitTestBase(); 14 | // stack only, no need virtual destructor, but the warning... 15 | virtual ~UnitTestBase() {} 16 | 17 | const std::string& GetName() const { return name_; } 18 | 19 | virtual void Run() = 0; 20 | 21 | bool IsFine() const { return errors_.empty(); } 22 | void Print() const; 23 | 24 | template 25 | UnitTestBase& operator << (const T & t); 26 | 27 | protected: 28 | UnitTestBase& SetInfo(const std::string& exprInfo, bool pass = true, bool abort = false); 29 | std::string name_; 30 | 31 | private: 32 | void FlushError(); 33 | 34 | bool pass_; 35 | bool abort_; 36 | 37 | std::string expr_; 38 | std::vector errors_; 39 | 40 | private: 41 | UnitTestBase(const UnitTestBase& ) = delete; 42 | UnitTestBase& operator= (const UnitTestBase& ) = delete; 43 | 44 | void* operator new(std::size_t ); // stack only 45 | }; 46 | 47 | template 48 | inline UnitTestBase& UnitTestBase::operator<< (const T & t) 49 | { 50 | if (!pass_) 51 | { 52 | std::ostringstream str; 53 | str << t; 54 | expr_ += str.str(); 55 | } 56 | 57 | return *this; 58 | } 59 | 60 | class MsgHelper 61 | { 62 | public: 63 | void operator=(UnitTestBase& test) 64 | { 65 | test.FlushError(); 66 | } 67 | }; 68 | 69 | 70 | 71 | #define TEST_CASE(name) \ 72 | class UnitTestBase##name: public UnitTestBase \ 73 | { \ 74 | public: \ 75 | UnitTestBase##name() { \ 76 | name_ = #name; \ 77 | } \ 78 | virtual void Run(); \ 79 | } test_##name##_obj; \ 80 | void UnitTestBase##name::Run() 81 | 82 | 83 | #define EXPECT_TRUE(expr) \ 84 | MsgHelper()=((expr) ? SetInfo("'"#expr"'", true) : SetInfo("'"#expr"'", false)) 85 | 86 | #define EXPECT_FALSE(expr) \ 87 | EXPECT_TRUE(!(expr)) 88 | 89 | #define ASSERT_TRUE(expr) \ 90 | MsgHelper()=((expr) ? SetInfo("'"#expr"'", true) : SetInfo("'"#expr"' ", false, true)) 91 | 92 | #define ASSERT_FALSE(expr) \ 93 | ASSERT_TRUE(!(expr)) 94 | 95 | 96 | 97 | class UnitTestManager 98 | { 99 | public: 100 | static UnitTestManager& Instance(); 101 | 102 | void AddTest(UnitTestBase* test); 103 | void Clear(); 104 | void Run(); 105 | private: 106 | UnitTestManager() {} 107 | 108 | std::vector tests_; 109 | }; 110 | 111 | #define RUN_ALL_TESTS UnitTestManager::Instance().Run 112 | 113 | #endif 114 | 115 | -------------------------------------------------------------------------------- /QBase/AsyncBuffer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #if defined(__APPLE__) 4 | #include 5 | #endif 6 | 7 | #include "AsyncBuffer.h" 8 | 9 | using std::size_t; 10 | 11 | AsyncBuffer::AsyncBuffer(size_t size) : buffer_(size), 12 | backBytes_(0) 13 | { 14 | } 15 | 16 | AsyncBuffer::~AsyncBuffer() 17 | { 18 | // assert (buffer_.IsEmpty()); 19 | // assert (backBytes_ == 0); 20 | } 21 | 22 | 23 | void AsyncBuffer::Write(const void* data, size_t len) 24 | { 25 | BufferSequence bf; 26 | bf.buffers[0].iov_base = const_cast(data); 27 | bf.buffers[0].iov_len = len; 28 | bf.count = 1; 29 | 30 | this->Write(bf); 31 | } 32 | 33 | void AsyncBuffer::Write(const BufferSequence& data) 34 | { 35 | auto len = data.TotalBytes(); 36 | 37 | if (backBytes_ > 0 || buffer_.WritableSize() < len) 38 | { 39 | std::lock_guard guard(backBufLock_); 40 | 41 | if (backBytes_ > 0 || buffer_.WritableSize() < len) 42 | { 43 | for (size_t i = 0; i < data.count; ++ i) 44 | { 45 | backBuf_.PushData(data.buffers[i].iov_base, 46 | data.buffers[i].iov_len); 47 | } 48 | 49 | backBytes_ += len; 50 | assert (backBytes_ == backBuf_.ReadableSize()); 51 | 52 | return; 53 | } 54 | } 55 | 56 | assert(backBytes_ == 0 && buffer_.WritableSize() >= len); 57 | 58 | for (size_t i = 0; i < data.count; ++ i) 59 | { 60 | buffer_.PushData(data.buffers[i].iov_base, data.buffers[i].iov_len); 61 | } 62 | } 63 | 64 | void AsyncBuffer::ProcessBuffer(BufferSequence& data) 65 | { 66 | data.count = 0; 67 | 68 | // Here be dragons! see below... 69 | if (!tmpBuf_.IsEmpty()) 70 | { 71 | data.count = 1; 72 | data.buffers[0].iov_base = tmpBuf_.ReadAddr(); 73 | data.buffers[0].iov_len = tmpBuf_.ReadableSize(); 74 | } 75 | else if (!buffer_.IsEmpty()) 76 | { 77 | auto nLen = buffer_.ReadableSize(); 78 | 79 | buffer_.GetDatum(data, nLen); 80 | assert (nLen == data.TotalBytes()); 81 | } 82 | else 83 | { 84 | if (backBytes_ > 0 && backBufLock_.try_lock()) 85 | { 86 | // tmpBuf_ is used for process backBuf_ without held mutex! 87 | backBytes_ = 0; 88 | tmpBuf_.Swap(backBuf_); 89 | backBufLock_.unlock(); 90 | 91 | data.count = 1; 92 | data.buffers[0].iov_base = tmpBuf_.ReadAddr(); 93 | data.buffers[0].iov_len = tmpBuf_.ReadableSize(); 94 | } 95 | } 96 | } 97 | 98 | void AsyncBuffer::Skip(size_t size) 99 | { 100 | if (!tmpBuf_.IsEmpty()) 101 | { 102 | assert(size <= tmpBuf_.ReadableSize()); 103 | tmpBuf_.AdjustReadPtr(size); 104 | } 105 | else 106 | { 107 | assert(buffer_.ReadableSize() >= size); 108 | buffer_.AdjustReadPtr(size); 109 | } 110 | } 111 | 112 | -------------------------------------------------------------------------------- /QedisCore/QReplication.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QREPLICATION_H 2 | #define BERT_QREPLICATION_H 3 | 4 | #include 5 | #include 6 | #include "UnboundedBuffer.h" 7 | #include "Socket.h" 8 | #include "Log/MemoryFile.h" 9 | 10 | namespace qedis 11 | { 12 | 13 | // master side 14 | enum QSlaveState 15 | { 16 | QSlaveState_none, 17 | QSlaveState_wait_bgsave_start, // 有非sync的bgsave进行 要等待 18 | QSlaveState_wait_bgsave_end, // sync bgsave正在进行 19 | //QSlaveState_send_rdb, // 这个slave在接受rdb文件 20 | QSlaveState_online, 21 | }; 22 | 23 | struct QSlaveInfo 24 | { 25 | QSlaveState state; 26 | unsigned short listenPort; // slave listening port 27 | 28 | QSlaveInfo() : state(QSlaveState_none), listenPort(0) 29 | { 30 | } 31 | }; 32 | 33 | // slave side 34 | enum QReplState 35 | { 36 | QReplState_none, 37 | QReplState_connecting, 38 | QReplState_connected, 39 | QReplState_wait_auth, // wait auth to be confirmed 40 | QReplState_wait_replconf, // wait replconf to be confirmed 41 | QReplState_wait_rdb, // wait to recv rdb file 42 | QReplState_online, 43 | }; 44 | 45 | struct QMasterInfo 46 | { 47 | SocketAddr addr; 48 | QReplState state; 49 | time_t downSince; 50 | 51 | // For recv rdb 52 | std::size_t rdbSize; 53 | std::size_t rdbRecved; 54 | 55 | QMasterInfo() 56 | { 57 | state = QReplState_none; 58 | downSince = 0; 59 | rdbSize = std::size_t(-1); 60 | rdbRecved = 0; 61 | } 62 | }; 63 | 64 | //tmp filename 65 | const char* const slaveRdbFile = "slave.rdb"; 66 | 67 | class QClient; 68 | 69 | class QReplication 70 | { 71 | public: 72 | static QReplication& Instance(); 73 | 74 | QReplication(const QReplication& ) = delete; 75 | void operator= (const QReplication& ) = delete; 76 | 77 | void Cron(); 78 | 79 | // master side 80 | bool IsBgsaving() const; 81 | bool HasAnyWaitingBgsave() const; 82 | void AddSlave(QClient* cli); 83 | void TryBgsave(); 84 | bool StartBgsave(); 85 | void OnStartBgsave(); 86 | void OnRdbSaveDone(); 87 | void SendToSlaves(const std::vector& params); 88 | 89 | // slave side 90 | void SaveTmpRdb(const char* data, std::size_t& len); 91 | void SetMaster(const std::shared_ptr& cli); 92 | void SetMasterState(QReplState s); 93 | void SetMasterAddr(const char* ip, unsigned short port); 94 | void SetRdbSize(std::size_t s); 95 | QReplState GetMasterState() const; 96 | SocketAddr GetMasterAddr() const; 97 | std::size_t GetRdbSize() const; 98 | 99 | // info command 100 | void OnInfoCommand(UnboundedBuffer& res); 101 | 102 | private: 103 | QReplication(); 104 | void _OnStartBgsave(bool succ); 105 | 106 | // master side 107 | bool bgsaving_; 108 | UnboundedBuffer buffer_; 109 | std::list > slaves_; 110 | 111 | //slave side 112 | QMasterInfo masterInfo_; 113 | std::weak_ptr master_; 114 | OutputMemoryFile rdb_; 115 | }; 116 | 117 | } 118 | 119 | #define QREPL qedis::QReplication::Instance() 120 | 121 | #endif 122 | -------------------------------------------------------------------------------- /QSentinel/zookeeper/recordio.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | #ifndef __RECORDIO_H__ 19 | #define __RECORDIO_H__ 20 | 21 | #include 22 | #define STRUCT_INITIALIZER(l,r) .l = r 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | struct buffer { 29 | int32_t len; 30 | char *buff; 31 | }; 32 | 33 | void deallocate_String(char **s); 34 | void deallocate_Buffer(struct buffer *b); 35 | void deallocate_vector(void *d); 36 | struct iarchive { 37 | int (*start_record)(struct iarchive *ia, const char *tag); 38 | int (*end_record)(struct iarchive *ia, const char *tag); 39 | int (*start_vector)(struct iarchive *ia, const char *tag, int32_t *count); 40 | int (*end_vector)(struct iarchive *ia, const char *tag); 41 | int (*deserialize_Bool)(struct iarchive *ia, const char *name, int32_t *); 42 | int (*deserialize_Int)(struct iarchive *ia, const char *name, int32_t *); 43 | int (*deserialize_Long)(struct iarchive *ia, const char *name, int64_t *); 44 | int (*deserialize_Buffer)(struct iarchive *ia, const char *name, 45 | struct buffer *); 46 | int (*deserialize_String)(struct iarchive *ia, const char *name, char **); 47 | void *priv; 48 | }; 49 | struct oarchive { 50 | int (*start_record)(struct oarchive *oa, const char *tag); 51 | int (*end_record)(struct oarchive *oa, const char *tag); 52 | int (*start_vector)(struct oarchive *oa, const char *tag, const int32_t *count); 53 | int (*end_vector)(struct oarchive *oa, const char *tag); 54 | int (*serialize_Bool)(struct oarchive *oa, const char *name, const int32_t *); 55 | int (*serialize_Int)(struct oarchive *oa, const char *name, const int32_t *); 56 | int (*serialize_Long)(struct oarchive *oa, const char *name, 57 | const int64_t *); 58 | int (*serialize_Buffer)(struct oarchive *oa, const char *name, 59 | const struct buffer *); 60 | int (*serialize_String)(struct oarchive *oa, const char *name, char **); 61 | void *priv; 62 | }; 63 | 64 | struct oarchive *create_buffer_oarchive(void); 65 | void close_buffer_oarchive(struct oarchive **oa, int free_buffer); 66 | struct iarchive *create_buffer_iarchive(char *buffer, int len); 67 | void close_buffer_iarchive(struct iarchive **ia); 68 | char *get_buffer(struct oarchive *); 69 | int get_buffer_len(struct oarchive *); 70 | 71 | #if defined(__APPLE__) 72 | #else 73 | int64_t htonll(int64_t v); 74 | #endif 75 | 76 | #ifdef __cplusplus 77 | } 78 | #endif 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /cluster/cluster_conn/zookeeper/recordio.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | #ifndef __RECORDIO_H__ 19 | #define __RECORDIO_H__ 20 | 21 | #include 22 | #define STRUCT_INITIALIZER(l,r) .l = r 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | struct buffer { 29 | int32_t len; 30 | char *buff; 31 | }; 32 | 33 | void deallocate_String(char **s); 34 | void deallocate_Buffer(struct buffer *b); 35 | void deallocate_vector(void *d); 36 | struct iarchive { 37 | int (*start_record)(struct iarchive *ia, const char *tag); 38 | int (*end_record)(struct iarchive *ia, const char *tag); 39 | int (*start_vector)(struct iarchive *ia, const char *tag, int32_t *count); 40 | int (*end_vector)(struct iarchive *ia, const char *tag); 41 | int (*deserialize_Bool)(struct iarchive *ia, const char *name, int32_t *); 42 | int (*deserialize_Int)(struct iarchive *ia, const char *name, int32_t *); 43 | int (*deserialize_Long)(struct iarchive *ia, const char *name, int64_t *); 44 | int (*deserialize_Buffer)(struct iarchive *ia, const char *name, 45 | struct buffer *); 46 | int (*deserialize_String)(struct iarchive *ia, const char *name, char **); 47 | void *priv; 48 | }; 49 | struct oarchive { 50 | int (*start_record)(struct oarchive *oa, const char *tag); 51 | int (*end_record)(struct oarchive *oa, const char *tag); 52 | int (*start_vector)(struct oarchive *oa, const char *tag, const int32_t *count); 53 | int (*end_vector)(struct oarchive *oa, const char *tag); 54 | int (*serialize_Bool)(struct oarchive *oa, const char *name, const int32_t *); 55 | int (*serialize_Int)(struct oarchive *oa, const char *name, const int32_t *); 56 | int (*serialize_Long)(struct oarchive *oa, const char *name, 57 | const int64_t *); 58 | int (*serialize_Buffer)(struct oarchive *oa, const char *name, 59 | const struct buffer *); 60 | int (*serialize_String)(struct oarchive *oa, const char *name, char **); 61 | void *priv; 62 | }; 63 | 64 | struct oarchive *create_buffer_oarchive(void); 65 | void close_buffer_oarchive(struct oarchive **oa, int free_buffer); 66 | struct iarchive *create_buffer_iarchive(char *buffer, int len); 67 | void close_buffer_iarchive(struct iarchive **ia); 68 | char *get_buffer(struct oarchive *); 69 | int get_buffer_len(struct oarchive *); 70 | 71 | #if defined(__APPLE__) 72 | #else 73 | int64_t htonll(int64_t v); 74 | #endif 75 | 76 | #ifdef __cplusplus 77 | } 78 | #endif 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /QBase/ConfigParser.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "ConfigParser.h" 3 | #include "Log/MemoryFile.h" 4 | 5 | static const int SPACE = ' '; 6 | static const int TAB = '\t'; 7 | static const int NEWLINE = '\n'; 8 | static const int COMMENT = '#'; 9 | 10 | 11 | static size_t SkipBlank(const char* data, size_t len, size_t off) 12 | { 13 | while (++ off < len) 14 | { 15 | if (SPACE != data[off] && TAB != data[off]) 16 | { 17 | -- off; 18 | break; 19 | } 20 | } 21 | 22 | return off; 23 | } 24 | 25 | bool ConfigParser::Load(const char* FileName) 26 | { 27 | InputMemoryFile file; 28 | if (!file.Open(FileName)) 29 | return false; // no such file 30 | 31 | data_.clear(); 32 | 33 | size_t maxLen = size_t(-1); 34 | const char* data = file.Read(maxLen); 35 | 36 | bool bReadKey = true; 37 | std::string key, value; 38 | key.reserve(64); 39 | value.reserve(64); 40 | 41 | size_t off = 0; 42 | while (off < maxLen) 43 | { 44 | switch (data[off]) 45 | { 46 | case COMMENT: 47 | while (++ off < maxLen) 48 | { 49 | if (NEWLINE == data[off]) 50 | { 51 | -- off; 52 | break; 53 | } 54 | } 55 | break; 56 | 57 | case NEWLINE: 58 | bReadKey = true; 59 | 60 | if (!key.empty()) 61 | { 62 | data_[key].push_back(value); 63 | 64 | key.clear(); 65 | value.clear(); 66 | } 67 | 68 | off = SkipBlank(data, maxLen, off); 69 | break; 70 | 71 | case SPACE: 72 | case TAB: 73 | // 支持value中有空格 74 | if (bReadKey) 75 | { 76 | bReadKey = false; 77 | off = SkipBlank(data, maxLen, off); // 跳过所有分界空格 78 | } 79 | else 80 | { 81 | value += data[off]; 82 | } 83 | break; 84 | 85 | case '\r': 86 | break; 87 | 88 | default: 89 | if (bReadKey) 90 | key += data[off]; 91 | else 92 | value += data[off]; 93 | 94 | break; 95 | } 96 | 97 | ++ off; 98 | } 99 | 100 | file.Close(); 101 | return true; 102 | } 103 | 104 | const std::vector& ConfigParser::GetDataVector(const char* key) const 105 | { 106 | auto it = data_.find(key); 107 | if (it == data_.end()) 108 | { 109 | static const std::vector kEmpty; 110 | return kEmpty; 111 | } 112 | 113 | return it->second; 114 | } 115 | 116 | #ifdef CONFIG_DEBUG 117 | int main() 118 | { 119 | ConfigParser csv; 120 | csv.Load("config"); 121 | csv.Print(); 122 | 123 | std::cout << "=====================" << std::endl; 124 | } 125 | #endif 126 | 127 | -------------------------------------------------------------------------------- /UnitTest/QGlobRegex_unittest.cc: -------------------------------------------------------------------------------- 1 | #include "UnitTest.h" 2 | #include "QGlobRegex.h" 3 | 4 | using namespace qedis; 5 | 6 | TEST_CASE(regex_star) 7 | { 8 | EXPECT_TRUE(glob_match("a*b?*?", "abxy")); 9 | EXPECT_TRUE(glob_match("*a", "aa")); 10 | EXPECT_TRUE(glob_match("*a", "abaa")); 11 | EXPECT_TRUE(glob_match("***a", "aaaa")); 12 | EXPECT_FALSE(glob_match("a*b?*?", "cb")); 13 | 14 | EXPECT_TRUE(glob_match("a*b", "ab")); 15 | EXPECT_TRUE(glob_match("a*b", "acb")); 16 | EXPECT_TRUE(glob_match("a*b", "acccb")); 17 | EXPECT_FALSE(glob_match("a*b", "accc")); 18 | } 19 | 20 | TEST_CASE(regex_question) 21 | { 22 | EXPECT_FALSE(glob_match("a?b", "ab")); 23 | EXPECT_TRUE(glob_match("a?b", "acb")); 24 | EXPECT_FALSE(glob_match("a?b", "acc")); 25 | EXPECT_FALSE(glob_match("a?b", "accb")); 26 | } 27 | 28 | TEST_CASE(regex_bracket) 29 | { 30 | EXPECT_TRUE(glob_match("[-]", "-")); 31 | EXPECT_TRUE(glob_match("[-abc]", "-")); 32 | EXPECT_TRUE(glob_match("[-]??", "--x")); 33 | 34 | EXPECT_TRUE(glob_match("[a-ce-a]", "a")); 35 | 36 | EXPECT_TRUE(glob_match("[a-eg-z]", "u")); 37 | EXPECT_FALSE(glob_match("[^a-eg-z]", "u")); 38 | EXPECT_TRUE(glob_match("[abcx-z]", "c")); 39 | EXPECT_TRUE(glob_match("[abcx-z]", "y")); 40 | EXPECT_FALSE(glob_match("[^abcx-z]", "c")); 41 | EXPECT_FALSE(glob_match("[^abcx-z]", "y")); 42 | 43 | EXPECT_FALSE(glob_match("[a-z]", "U")); 44 | EXPECT_TRUE(glob_match("[^a-z]", "U")); 45 | 46 | EXPECT_TRUE(glob_match("[a--]", "-")); 47 | EXPECT_FALSE(glob_match("[^a--]", "-")); 48 | 49 | EXPECT_TRUE(glob_match("[---]", "-")); 50 | EXPECT_TRUE(glob_match("[-]", "-")); 51 | EXPECT_TRUE(glob_match("[-xyz]", "-")); 52 | EXPECT_TRUE(glob_match("[]]", "]")); 53 | EXPECT_TRUE(glob_match("x[]abc]", "x]")); 54 | EXPECT_FALSE(glob_match("x[ab]c]", "x]")); 55 | 56 | EXPECT_TRUE(glob_match("x\\[ab]y", "x[ab]y")); 57 | EXPECT_TRUE(glob_match("x*\\", "xab\\")); 58 | EXPECT_TRUE(glob_match("\\*\\", "\\fuckyou\\")); 59 | } 60 | 61 | TEST_CASE(regex_star_brackets) 62 | { 63 | EXPECT_FALSE(glob_match("[a-e]", "xa")); 64 | EXPECT_FALSE(glob_match("*[a-e]*", "x")); 65 | EXPECT_FALSE(glob_match("*[a-e]*", "xy")); 66 | EXPECT_TRUE(glob_match("*[a-e]*", "xya")); 67 | EXPECT_TRUE(glob_match("*[a-e]*", "xay")); 68 | EXPECT_TRUE(glob_match("*[a-e]*", "axy")); 69 | } 70 | 71 | TEST_CASE(regex_search) 72 | { 73 | EXPECT_TRUE(glob_search("[a-e]", "xa")); 74 | EXPECT_FALSE(glob_search("[a-e]", "x")); 75 | EXPECT_FALSE(glob_search("[a-e]", "xy")); 76 | EXPECT_TRUE(glob_search("[a-e]", "xya")); 77 | EXPECT_TRUE(glob_search("[a-e]", "xay")); 78 | EXPECT_TRUE(glob_search("[a-e]", "axy")); 79 | } 80 | 81 | TEST_CASE(regex_strange) 82 | { 83 | EXPECT_FALSE(glob_match("[a-e", "a")); 84 | EXPECT_TRUE(glob_match("*a-e]*", "a-e]")); 85 | EXPECT_FALSE(glob_match("[a-e*", "a")); 86 | EXPECT_TRUE(glob_match("a[-]", "a-")); 87 | EXPECT_FALSE(glob_match("a\[", "a[")); 88 | EXPECT_TRUE(glob_match("a\\[", "a[")); 89 | EXPECT_TRUE(glob_match("a[]]", "a]")); 90 | 91 | EXPECT_TRUE(glob_match("a[^^]", "a$")); 92 | EXPECT_FALSE(glob_match("a[^^]", "a^")); 93 | 94 | EXPECT_TRUE(glob_match("a**[^^]", "a^!")); 95 | EXPECT_FALSE(glob_match("**a**", "xyz")); 96 | } 97 | 98 | -------------------------------------------------------------------------------- /QBase/Kqueue.cc: -------------------------------------------------------------------------------- 1 | #if defined(__APPLE__) 2 | 3 | #include "Kqueue.h" 4 | #include "Log/Logger.h" 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | Kqueue::Kqueue() 12 | { 13 | multiplexer_ = ::kqueue(); 14 | INF << "create kqueue: " << multiplexer_; 15 | } 16 | 17 | Kqueue::~Kqueue() 18 | { 19 | INF << "close kqueue: " << multiplexer_; 20 | if (multiplexer_ != -1) 21 | ::close(multiplexer_); 22 | } 23 | 24 | bool Kqueue::AddSocket(int sock, int events, void* userPtr) 25 | { 26 | struct kevent change[2]; 27 | 28 | int cnt = 0; 29 | 30 | if (events & EventTypeRead) 31 | { 32 | EV_SET(change + cnt, sock, EVFILT_READ, EV_ADD, 0, 0, userPtr); 33 | ++ cnt; 34 | } 35 | 36 | if (events & EventTypeWrite) 37 | { 38 | EV_SET(change + cnt, sock, EVFILT_WRITE, EV_ADD, 0, 0, userPtr); 39 | ++ cnt; 40 | } 41 | 42 | return kevent(multiplexer_, change, cnt, NULL, 0, NULL) != -1; 43 | } 44 | 45 | bool Kqueue::DelSocket(int sock, int events) 46 | { 47 | struct kevent change[2]; 48 | int cnt = 0; 49 | 50 | if (events & EventTypeRead) 51 | { 52 | EV_SET(change + cnt, sock, EVFILT_READ, EV_DELETE, 0, 0, NULL); 53 | ++ cnt; 54 | } 55 | 56 | if (events & EventTypeWrite) 57 | { 58 | EV_SET(change + cnt, sock, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); 59 | ++ cnt; 60 | } 61 | 62 | if (cnt == 0) 63 | return false; 64 | 65 | return -1 != kevent(multiplexer_, change, cnt, NULL, 0, NULL); 66 | } 67 | 68 | 69 | bool Kqueue::ModSocket(int sock, int events, void* userPtr) 70 | { 71 | bool ret = DelSocket(sock, EventTypeRead | EventTypeWrite); 72 | if (events == 0) 73 | return ret; 74 | 75 | return AddSocket(sock, events, userPtr); 76 | } 77 | 78 | 79 | int Kqueue::Poll(std::vector& events, std::size_t maxEvent, int timeoutMs) 80 | { 81 | if (maxEvent == 0) 82 | return 0; 83 | 84 | while (events_.size() < maxEvent) 85 | events_.resize(2 * events_.size() + 1); 86 | 87 | struct timespec* pTimeout = NULL; 88 | struct timespec timeout; 89 | if (timeoutMs >= 0) 90 | { 91 | pTimeout = &timeout; 92 | timeout.tv_sec = timeoutMs / 1000; 93 | timeout.tv_nsec = timeoutMs % 1000 * 1000000; 94 | } 95 | 96 | int nFired = ::kevent(multiplexer_, NULL, 0, &events_[0], static_cast(maxEvent), pTimeout); 97 | if (nFired == -1) 98 | return -1; 99 | 100 | if (nFired > 0 && static_cast(nFired) > events.size()) 101 | events.resize(nFired); 102 | 103 | for (int i = 0; i < nFired; ++ i) 104 | { 105 | FiredEvent& fired = events[i]; 106 | fired.events = 0; 107 | fired.userdata = events_[i].udata; 108 | 109 | if (events_[i].filter == EVFILT_READ) 110 | fired.events |= EventTypeRead; 111 | 112 | if (events_[i].filter == EVFILT_WRITE) 113 | fired.events |= EventTypeWrite; 114 | 115 | if (events_[i].flags & EV_ERROR) 116 | fired.events |= EventTypeError; 117 | } 118 | 119 | return nFired; 120 | } 121 | 122 | #endif 123 | 124 | -------------------------------------------------------------------------------- /cluster/ananas/net/Connection.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_CONNECTION_H 3 | #define BERT_CONNECTION_H 4 | 5 | #include 6 | #include "Socket.h" 7 | #include "Poller.h" 8 | #include "Buffer.h" 9 | #include "Typedefs.h" 10 | 11 | namespace ananas 12 | { 13 | 14 | namespace internal 15 | { 16 | class Acceptor; 17 | class Connector; 18 | } 19 | 20 | enum class ShutdownMode 21 | { 22 | eSM_Both, 23 | eSM_Read, 24 | eSM_Write, 25 | }; 26 | 27 | class Connection : public internal::EventSource 28 | { 29 | public: 30 | explicit 31 | Connection(EventLoop* loop); 32 | ~Connection(); 33 | 34 | Connection(const Connection& ) = delete; 35 | void operator= (const Connection& ) = delete; 36 | 37 | bool Init(int sock, const SocketAddr& peer); 38 | void SetMaxPacketSize(std::size_t s); 39 | const SocketAddr& Peer() const { return peer_; } 40 | void ActiveClose(); 41 | EventLoop* GetLoop() const { return loop_; } 42 | 43 | void Shutdown(ShutdownMode mode); 44 | void SetNodelay(bool enable); 45 | 46 | int Identifier() const override; 47 | bool HandleReadEvent() override; 48 | bool HandleWriteEvent() override; 49 | void HandleErrorEvent() override; 50 | 51 | bool SendPacket(const void* data, std::size_t len); 52 | bool SendPacket(const BufferVector& datum); 53 | bool SendPacket(const SliceVector& slice); 54 | 55 | void SetOnConnect(std::function cb); 56 | void SetOnDisconnect(std::function cb); 57 | void SetOnMessage(TcpMessageCallback cb); 58 | void SetFailCallback(TcpConnFailCallback cb); 59 | void SetOnWriteComplete(TcpWriteCompleteCallback wccb); 60 | void SetOnWriteHighWater(TcpWriteHighWaterCallback whwcb); 61 | 62 | void SetMinPacketSize(size_t s); 63 | void SetWriteHighWater(size_t s); 64 | 65 | void SetUserData(std::shared_ptr user); 66 | 67 | template 68 | std::shared_ptr GetUserData() const; 69 | 70 | private: 71 | enum State 72 | { 73 | eS_None, 74 | eS_Connected, 75 | eS_CloseWaitWrite, // peer close or shutdown write, but I have data to send 76 | eS_PassiveClose, // should close 77 | eS_ActiveClose, // should close 78 | eS_Error, 79 | eS_Closed, 80 | }; 81 | 82 | friend class internal::Acceptor; 83 | friend class internal::Connector; 84 | void _OnConnect(); 85 | int _Send(const void* data, size_t len); 86 | 87 | EventLoop* const loop_; 88 | State state_ = State::eS_None; 89 | int localSock_; 90 | size_t minPacketSize_; 91 | size_t sendBufHighWater_; 92 | 93 | Buffer recvBuf_; 94 | BufferVector sendBuf_; 95 | 96 | SocketAddr peer_; 97 | 98 | std::function onConnect_; 99 | std::function onDisconnect_; 100 | 101 | TcpMessageCallback onMessage_; 102 | TcpConnFailCallback onConnFail_; 103 | TcpWriteCompleteCallback onWriteComplete_; 104 | TcpWriteHighWaterCallback onWriteHighWater; 105 | 106 | std::shared_ptr userData_; 107 | }; 108 | 109 | 110 | template 111 | inline std::shared_ptr Connection::GetUserData() const 112 | { 113 | return std::static_pointer_cast(userData_); 114 | } 115 | 116 | } // end namespace ananas 117 | 118 | #endif 119 | 120 | -------------------------------------------------------------------------------- /cluster/ananas/net/Socket.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BERT_SOCKET_H 3 | #define BERT_SOCKET_H 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace ananas 12 | { 13 | 14 | struct SocketAddr 15 | { 16 | static const uint16_t kInvalidPort = -1; 17 | 18 | SocketAddr() 19 | { 20 | memset(&addr_, 0, sizeof addr_); 21 | } 22 | 23 | SocketAddr(const sockaddr_in& addr) 24 | { 25 | Init(addr); 26 | } 27 | 28 | SocketAddr(uint32_t netip, uint16_t netport) 29 | { 30 | Init(netip, netport); 31 | } 32 | 33 | SocketAddr(const char* ip, uint16_t hostport) 34 | { 35 | Init(ip, hostport); 36 | } 37 | 38 | // ip port format: 127.0.0.1:6379 39 | SocketAddr(const std::string& ipport) 40 | { 41 | std::string::size_type p = ipport.find_first_of(':'); 42 | std::string ip = ipport.substr(0, p); 43 | std::string port = ipport.substr(p + 1); 44 | 45 | Init(ip.c_str(), static_cast(std::stoi(port))); 46 | } 47 | 48 | void Init(const sockaddr_in& addr) 49 | { 50 | memcpy(&addr_, &addr, sizeof(addr)); 51 | } 52 | 53 | void Init(uint32_t netip, uint16_t netport) 54 | { 55 | addr_.sin_family = AF_INET; 56 | addr_.sin_addr.s_addr = netip; 57 | addr_.sin_port = netport; 58 | } 59 | 60 | void Init(const char* ip, uint16_t hostport) 61 | { 62 | addr_.sin_family = AF_INET; 63 | addr_.sin_addr.s_addr = ::inet_addr(ip); 64 | addr_.sin_port = htons(hostport); 65 | } 66 | 67 | const sockaddr_in& GetAddr() const 68 | { 69 | return addr_; 70 | } 71 | 72 | std::string GetIP() const 73 | { 74 | char tmp[32]; 75 | const char* res = inet_ntop(AF_INET, &addr_.sin_addr, 76 | tmp, (socklen_t)(sizeof tmp)); 77 | return std::string(res); 78 | } 79 | 80 | uint16_t GetPort() const 81 | { 82 | return ntohs(addr_.sin_port); 83 | } 84 | 85 | std::string ToString() const 86 | { 87 | char tmp[32]; 88 | const char* res = inet_ntop(AF_INET, &addr_.sin_addr, tmp, (socklen_t)(sizeof tmp)); 89 | 90 | return std::string(res) + ":" + std::to_string(ntohs(addr_.sin_port)); 91 | } 92 | 93 | bool IsValid() const { return addr_.sin_family != 0; } 94 | 95 | private: 96 | sockaddr_in addr_; 97 | }; 98 | 99 | extern const int kInvalid; 100 | 101 | extern const int kTimeout; 102 | extern const int kError; 103 | extern const int kEof; 104 | 105 | int CreateTCPSocket(); 106 | int CreateUDPSocket(); 107 | bool CreateSocketPair(int& readSock, int& writeSock); 108 | void CloseSocket(int &sock); 109 | void SetNonBlock(int sock, bool nonBlock = true); 110 | void SetNodelay(int sock, bool enable = true); 111 | void SetSndBuf(int sock, socklen_t size = 64 * 1024); 112 | void SetRcvBuf(int sock, socklen_t size = 64 * 1024); 113 | void SetReuseAddr(int sock); 114 | bool GetLocalAddr(int sock, SocketAddr& ); 115 | bool GetPeerAddr(int sock, SocketAddr& ); 116 | 117 | in_addr_t GetLocalAddrInfo(); 118 | 119 | rlim_t GetMaxOpenFd(); 120 | bool SetMaxOpenFd(rlim_t maxfdPlus1); 121 | 122 | std::string ConvertIp(const char* ip); 123 | 124 | } // end namespace ananas 125 | 126 | #endif 127 | 128 | -------------------------------------------------------------------------------- /cluster/ananas/net/Kqueue.cc: -------------------------------------------------------------------------------- 1 | #if defined(__APPLE__) 2 | 3 | #include "Kqueue.h" 4 | #include "AnanasDebug.h" 5 | 6 | #include 7 | #include 8 | 9 | namespace ananas 10 | { 11 | namespace internal 12 | { 13 | 14 | Kqueue::Kqueue() 15 | { 16 | multiplexer_ = ::kqueue(); 17 | INF(internal::g_debug) << "create kqueue: " << multiplexer_; 18 | } 19 | 20 | Kqueue::~Kqueue() 21 | { 22 | INF(internal::g_debug) << "close kqueue: " << multiplexer_; 23 | if (multiplexer_ != -1) 24 | ::close(multiplexer_); 25 | } 26 | 27 | bool Kqueue::Register(int sock, int events, void* userPtr) 28 | { 29 | struct kevent change[2]; 30 | 31 | int cnt = 0; 32 | 33 | if (events & eET_Read) 34 | { 35 | EV_SET(change + cnt, sock, EVFILT_READ, EV_ADD, 0, 0, userPtr); 36 | ++ cnt; 37 | } 38 | 39 | if (events & eET_Write) 40 | { 41 | EV_SET(change + cnt, sock, EVFILT_WRITE, EV_ADD, 0, 0, userPtr); 42 | ++ cnt; 43 | } 44 | 45 | return kevent(multiplexer_, change, cnt, NULL, 0, NULL) != -1; 46 | } 47 | 48 | bool Kqueue::Unregister(int sock, int events) 49 | { 50 | struct kevent change[2]; 51 | int cnt = 0; 52 | 53 | if (events & eET_Read) 54 | { 55 | EV_SET(change + cnt, sock, EVFILT_READ, EV_DELETE, 0, 0, NULL); 56 | ++ cnt; 57 | } 58 | 59 | if (events & eET_Write) 60 | { 61 | EV_SET(change + cnt, sock, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); 62 | ++ cnt; 63 | } 64 | 65 | if (cnt == 0) 66 | return false; 67 | 68 | return -1 != kevent(multiplexer_, change, cnt, NULL, 0, NULL); 69 | } 70 | 71 | 72 | bool Kqueue::Modify(int sock, int events, void* userPtr) 73 | { 74 | bool ret = Unregister(sock, eET_Read | eET_Write); 75 | if (events == 0) 76 | return ret; 77 | 78 | return Register(sock, events, userPtr); 79 | } 80 | 81 | 82 | int Kqueue::Poll(std::size_t maxEvent, int timeoutMs) 83 | { 84 | if (maxEvent == 0) 85 | return 0; 86 | 87 | while (events_.size() < maxEvent) 88 | events_.resize(2 * events_.size() + 1); 89 | 90 | struct timespec* pTimeout = NULL; 91 | struct timespec timeout; 92 | if (timeoutMs >= 0) 93 | { 94 | pTimeout = &timeout; 95 | timeout.tv_sec = timeoutMs / 1000; 96 | timeout.tv_nsec = timeoutMs % 1000 * 1000000; 97 | } 98 | 99 | int nFired = ::kevent(multiplexer_, NULL, 0, &events_[0], static_cast(maxEvent), pTimeout); 100 | if (nFired == -1) 101 | return -1; 102 | 103 | auto& events = firedEvents_; 104 | if (nFired > 0 && static_cast(nFired) > events.size()) 105 | events.resize(nFired); 106 | 107 | for (int i = 0; i < nFired; ++ i) 108 | { 109 | FiredEvent& fired = events[i]; 110 | fired.events = 0; 111 | fired.userdata = events_[i].udata; 112 | 113 | if (events_[i].filter == EVFILT_READ) 114 | fired.events |= eET_Read; 115 | 116 | if (events_[i].filter == EVFILT_WRITE) 117 | fired.events |= eET_Write; 118 | 119 | if (events_[i].flags & EV_ERROR) 120 | fired.events |= eET_Error; 121 | } 122 | 123 | return nFired; 124 | } 125 | 126 | } // end namespace internal 127 | } // end namespace ananas 128 | 129 | #endif 130 | 131 | -------------------------------------------------------------------------------- /cluster/qedis_proxy/ConfigParser.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "ConfigParser.h" 5 | 6 | static const int SPACE = ' '; 7 | static const int TAB = '\t'; 8 | static const int NEWLINE = '\n'; 9 | static const int COMMENT = '#'; 10 | 11 | 12 | static size_t SkipBlank(const char* data, size_t len, size_t off) 13 | { 14 | while (++ off < len) 15 | { 16 | if (SPACE != data[off] && TAB != data[off]) 17 | { 18 | -- off; 19 | break; 20 | } 21 | } 22 | 23 | return off; 24 | } 25 | 26 | bool ConfigParser::Load(const char* fileName) 27 | { 28 | std::ifstream ifs(fileName, std::ifstream::binary); 29 | if (!ifs) 30 | return false; // no such file 31 | 32 | std::filebuf* fbuf = ifs.rdbuf(); 33 | const std::size_t size = fbuf->pubseekoff (0, ifs.end, ifs.in); 34 | fbuf->pubseekpos (0, ifs.in); 35 | 36 | // allocate memory to hold file data 37 | std::unique_ptr data(new char[size]); 38 | fbuf->sgetn(data.get(), size); 39 | ifs.close(); 40 | 41 | data_.clear(); 42 | 43 | bool bReadKey = true; 44 | std::string key, value; 45 | key.reserve(64); 46 | value.reserve(64); 47 | 48 | size_t off = 0; 49 | while (off < size) 50 | { 51 | switch (data[off]) 52 | { 53 | case COMMENT: 54 | while (++ off < size) 55 | { 56 | if (NEWLINE == data[off]) 57 | { 58 | -- off; 59 | break; 60 | } 61 | } 62 | break; 63 | 64 | case NEWLINE: 65 | bReadKey = true; 66 | 67 | if (!key.empty()) 68 | { 69 | data_[key].push_back(value); 70 | 71 | key.clear(); 72 | value.clear(); 73 | } 74 | 75 | off = SkipBlank(data.get(), size, off); 76 | break; 77 | 78 | case SPACE: 79 | case TAB: 80 | // 支持value中有空格 81 | if (bReadKey) 82 | { 83 | bReadKey = false; 84 | off = SkipBlank(data.get(), size, off); // 跳过所有分界空格 85 | } 86 | else 87 | { 88 | value += data[off]; 89 | } 90 | break; 91 | 92 | case '\r': 93 | break; 94 | 95 | default: 96 | if (bReadKey) 97 | key += data[off]; 98 | else 99 | value += data[off]; 100 | 101 | break; 102 | } 103 | 104 | ++ off; 105 | } 106 | 107 | return true; 108 | } 109 | 110 | const std::vector& ConfigParser::GetDataVector(const char* key) const 111 | { 112 | auto it = data_.find(key); 113 | if (it == data_.end()) 114 | { 115 | static const std::vector kEmpty; 116 | return kEmpty; 117 | } 118 | 119 | return it->second; 120 | } 121 | 122 | #ifdef CONFIG_DEBUG 123 | int main() 124 | { 125 | ConfigParser csv; 126 | csv.Load("config"); 127 | csv.Print(); 128 | 129 | std::cout << "=====================" << std::endl; 130 | } 131 | #endif 132 | 133 | -------------------------------------------------------------------------------- /cluster/ananas/net/ThreadPool.cc: -------------------------------------------------------------------------------- 1 | #include "ThreadPool.h" 2 | 3 | namespace ananas 4 | { 5 | 6 | thread_local bool ThreadPool::working_ = true; 7 | std::thread::id ThreadPool::s_mainThread; 8 | 9 | ThreadPool::ThreadPool() : currentThreads_{0}, waiters_(0), shutdown_(false) 10 | { 11 | monitor_ = std::thread([this]() { _MonitorRoutine(); } ); 12 | maxIdleThreads_ = maxThreads_ = std::max(1U, std::thread::hardware_concurrency()); 13 | pendingStopSignal_ = 0; 14 | 15 | // init main thread id 16 | s_mainThread = std::this_thread::get_id(); 17 | } 18 | 19 | ThreadPool::~ThreadPool() 20 | { 21 | JoinAll(); 22 | } 23 | 24 | ThreadPool& ThreadPool::Instance() 25 | { 26 | static ThreadPool pool; 27 | return pool; 28 | } 29 | 30 | void ThreadPool::SetMaxIdleThreads(unsigned int m) 31 | { 32 | if (0 < m && m <= kMaxThreads) 33 | maxIdleThreads_ = m; 34 | } 35 | 36 | void ThreadPool::SetMaxThreads(unsigned int m) 37 | { 38 | if (0 < m && m <= kMaxThreads) 39 | maxThreads_ = m; 40 | } 41 | 42 | void ThreadPool::JoinAll() 43 | { 44 | if (s_mainThread != std::this_thread::get_id()) 45 | return; 46 | 47 | decltype(workers_) tmp; 48 | 49 | { 50 | std::unique_lock guard(mutex_); 51 | if (shutdown_) 52 | return; 53 | 54 | shutdown_ = true; 55 | cond_.notify_all(); 56 | 57 | tmp.swap(workers_); 58 | } 59 | 60 | for (auto& t : tmp) 61 | { 62 | if (t.joinable()) 63 | t.join(); 64 | } 65 | 66 | if (monitor_.joinable()) 67 | { 68 | monitor_.join(); 69 | } 70 | } 71 | 72 | void ThreadPool::_CreateWorker() 73 | { 74 | // guarded by mutex. 75 | ++ currentThreads_; 76 | std::thread t([this]() { this->_WorkerRoutine(); } ); 77 | workers_.push_back(std::move(t)); 78 | } 79 | 80 | void ThreadPool::_WorkerRoutine() 81 | { 82 | working_ = true; 83 | 84 | while (working_) 85 | { 86 | std::function task; 87 | 88 | { 89 | std::unique_lock guard(mutex_); 90 | 91 | ++ waiters_; 92 | cond_.wait(guard, [this]() { return shutdown_ || !tasks_.empty(); } ); 93 | -- waiters_; 94 | 95 | if (shutdown_ && tasks_.empty()) 96 | { 97 | -- currentThreads_; 98 | return; 99 | } 100 | 101 | task = std::move(tasks_.front()); 102 | tasks_.pop_front(); 103 | } 104 | 105 | task(); 106 | } 107 | 108 | // if reach here, this thread is recycled by monitor thread 109 | -- currentThreads_; 110 | -- pendingStopSignal_; 111 | } 112 | 113 | void ThreadPool::_MonitorRoutine() 114 | { 115 | while (!shutdown_) 116 | { 117 | std::this_thread::sleep_for(std::chrono::seconds(1)); 118 | 119 | std::unique_lock guard(mutex_); 120 | if (shutdown_) 121 | return; 122 | 123 | auto nw = waiters_; 124 | 125 | // if there is any pending stop signal to consume waiters 126 | nw -= pendingStopSignal_; 127 | 128 | while (nw -- > maxIdleThreads_) 129 | { 130 | tasks_.push_back([this]() { working_ = false; }); 131 | cond_.notify_one(); 132 | ++ pendingStopSignal_; 133 | } 134 | } 135 | } 136 | 137 | } // end namespace ananas 138 | 139 | -------------------------------------------------------------------------------- /QedisCore/QAOF.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_QAOF_H 2 | #define BERT_QAOF_H 3 | 4 | #include 5 | #include 6 | #include "Log/MemoryFile.h" 7 | #include "AsyncBuffer.h" 8 | #include "QString.h" 9 | #include "QStore.h" 10 | 11 | namespace qedis 12 | { 13 | 14 | extern pid_t g_rewritePid; 15 | 16 | class QAOFThreadController 17 | { 18 | public: 19 | static QAOFThreadController& Instance(); 20 | 21 | QAOFThreadController(const QAOFThreadController& ) = delete; 22 | void operator= (const QAOFThreadController& ) = delete; 23 | 24 | void Start(); 25 | void Stop(); 26 | void Join(); 27 | 28 | void SaveCommand(const std::vector& params, int db); 29 | bool ProcessTmpBuffer(BufferSequence& bf); 30 | void SkipTmpBuffer(size_t n); 31 | 32 | static void RewriteDoneHandler(int exit, int signal); 33 | 34 | private: 35 | QAOFThreadController() : lastDb_(-1) {} 36 | 37 | class AOFThread 38 | { 39 | friend class QAOFThreadController; 40 | public: 41 | AOFThread() : alive_(false) { } 42 | ~AOFThread(); 43 | 44 | void SetAlive() { alive_ = true; } 45 | bool IsAlive() const { return alive_; } 46 | void Stop() { alive_ = false; } 47 | 48 | //void Close(); 49 | void SaveCommand(const std::vector& params); 50 | 51 | bool Flush(); 52 | 53 | void Run(); 54 | 55 | std::atomic alive_; 56 | 57 | OutputMemoryFile file_; 58 | AsyncBuffer buf_; 59 | 60 | std::promise pro_; // Effective modern C++ : Item 39 61 | }; 62 | 63 | void _WriteSelectDB(int db, AsyncBuffer& dst); 64 | 65 | std::shared_ptr aofThread_; 66 | AsyncBuffer aofBuffer_; 67 | int lastDb_; 68 | }; 69 | 70 | 71 | class QAOFLoader 72 | { 73 | public: 74 | QAOFLoader(); 75 | 76 | bool Load(const char* name); 77 | 78 | const std::vector >& GetCmds() const 79 | { 80 | return cmds_; 81 | } 82 | 83 | private: 84 | std::vector > cmds_; 85 | }; 86 | 87 | template 88 | inline void WriteBulkString(const char* str, size_t strLen, DEST& dst) 89 | { 90 | char tmp[32]; 91 | size_t n = snprintf(tmp, sizeof tmp, "$%lu\r\n", strLen); 92 | 93 | dst.Write(tmp, n); 94 | dst.Write(str, strLen); 95 | dst.Write("\r\n", 2); 96 | } 97 | 98 | 99 | template 100 | inline void WriteBulkString(const QString& str, DEST& dst) 101 | { 102 | WriteBulkString(str.data(), str.size(), dst); 103 | } 104 | 105 | template 106 | inline void WriteMultiBulkLong(long val, DEST& dst) 107 | { 108 | char tmp[32]; 109 | size_t n = snprintf(tmp, sizeof tmp, "*%lu\r\n", val); 110 | dst.Write(tmp, n); 111 | } 112 | 113 | template 114 | inline void WriteBulkLong(long val, DEST& dst) 115 | { 116 | char tmp[32]; 117 | size_t n = snprintf(tmp, sizeof tmp, "%lu", val); 118 | 119 | WriteBulkString(tmp, n, dst); 120 | } 121 | 122 | 123 | template 124 | inline void SaveCommand(const std::vector& params, DEST& dst) 125 | { 126 | WriteMultiBulkLong(params.size(), dst); 127 | 128 | for (const auto& s : params) 129 | { 130 | WriteBulkString(s, dst); 131 | } 132 | } 133 | 134 | } 135 | 136 | #endif 137 | -------------------------------------------------------------------------------- /cluster/cluster_conn/zookeeper/ZookeeperContext.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_ZOOKEEPERCONTEXT_H 2 | #define BERT_ZOOKEEPERCONTEXT_H 3 | 4 | #include 5 | #include 6 | #include "future/Future.h" 7 | #include "net/Typedefs.h" 8 | #include "ZkResponse.h" 9 | 10 | struct oarchive; 11 | struct iarchive; 12 | 13 | namespace ananas 14 | { 15 | class Connection; 16 | } 17 | 18 | namespace qedis 19 | { 20 | 21 | class ZookeeperContext final 22 | { 23 | public: 24 | explicit 25 | ZookeeperContext(ananas::Connection* c); 26 | ~ZookeeperContext(); 27 | 28 | bool ParseMessage(const char*& data, ananas::PacketLen_t len); 29 | 30 | bool IsResumed() const { return resumed_; } 31 | 32 | // requests 33 | ananas::Future DoHandshake(); 34 | bool ProcessHandshake(const ZkResponse& rsp); 35 | 36 | // Ping 37 | ananas::Future Ping(); 38 | void ProcessPing(long lastPing, ZkResponse now); // Unit: millseconds 39 | 40 | // Create node 41 | ananas::Future CreateNode(bool empher, bool seq, 42 | const std::string* data = nullptr, 43 | const std::string* pathPrefix = nullptr); 44 | void ProcessCreateNode(const ZkResponse& rsp); 45 | 46 | // Get children2 47 | ananas::Future GetChildren2(const std::string& parent, bool watch = false); 48 | void ProcessGetChildren2(const ZkResponse& rsp); 49 | 50 | // Get data 51 | ananas::Future GetData(const std::string& node, bool watch = false); 52 | void ProcessGetData(const ZkResponse& rsp); 53 | 54 | private: 55 | 56 | void _SendPacket(struct oarchive* oa); 57 | 58 | ananas::Connection* const conn_; 59 | 60 | int _GetXid() const; 61 | mutable int xid_; 62 | 63 | int lastZxid_; 64 | 65 | struct Request { 66 | ananas::Promise promise; 67 | int type; 68 | int xid; 69 | std::string path; 70 | 71 | Request() : type(-1), xid(-1) { } 72 | 73 | Request(const Request& ) = delete; 74 | void operator= (const Request & ) = delete; 75 | 76 | Request(Request&& r) { 77 | _Move(std::move(r)); 78 | } 79 | 80 | Request& operator= (Request&& r) { 81 | return _Move(std::move(r)); 82 | } 83 | private: 84 | Request& _Move(Request&& r) { 85 | if (&r != this) { 86 | promise = std::move(r.promise); 87 | type = r.type; 88 | xid = r.xid; 89 | path = std::move(r.path); 90 | r.type = r.xid = -1; 91 | } 92 | return *this; 93 | } 94 | }; 95 | 96 | ananas::Future _PendingRequest(int type, int xid, const std::string* path = nullptr); 97 | 98 | std::queue pendingReq_; 99 | 100 | enum class State 101 | { 102 | kNone, 103 | kHandshaking, 104 | kConnected, 105 | } state_; 106 | 107 | bool _ProcessResponse(const ReplyHeader& hdr, iarchive* ia); 108 | 109 | #pragma pack(1) 110 | struct SessionInfo 111 | { 112 | int64_t sessionId{0}; 113 | char passwd[16]; 114 | int64_t lastZxidSeen {0}; // zk_cli doesn't persist this field 115 | } sessionInfo_; 116 | #pragma pack() 117 | std::string sessionFile_; 118 | bool resumed_ {false}; 119 | }; 120 | 121 | } // end namespace qedis 122 | 123 | #endif //endif BERT_ZOOKEEPERCONN_H 124 | 125 | -------------------------------------------------------------------------------- /QBase/ListenSocket.cc: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "Server.h" 8 | #include "NetThreadPool.h" 9 | #include "ListenSocket.h" 10 | #include "Log/Logger.h" 11 | 12 | namespace Internal 13 | { 14 | 15 | const int ListenSocket::LISTENQ = 1024; 16 | 17 | ListenSocket::ListenSocket(int tag) : 18 | localPort_(INVALID_PORT), 19 | tag_(tag) 20 | { 21 | } 22 | 23 | ListenSocket::~ListenSocket() 24 | { 25 | Server::Instance()->DelListenSock(localSock_); 26 | USR << "Close ListenSocket " << localSock_; 27 | } 28 | 29 | bool ListenSocket::Bind(const SocketAddr& addr) 30 | { 31 | if (addr.Empty()) 32 | return false; 33 | 34 | if (localSock_ != INVALID_SOCKET) 35 | return false; 36 | 37 | localPort_ = addr.GetPort(); 38 | 39 | localSock_ = CreateTCPSocket(); 40 | SetNonBlock(localSock_); 41 | SetNodelay(localSock_); 42 | SetReuseAddr(localSock_); 43 | SetRcvBuf(localSock_); 44 | SetSndBuf(localSock_); 45 | 46 | struct sockaddr_in serv = addr.GetAddr(); 47 | 48 | int ret = ::bind(localSock_, (struct sockaddr*)&serv, sizeof serv); 49 | if (SOCKET_ERROR == ret) 50 | { 51 | CloseSocket(localSock_); 52 | ERR << "Cannot bind port " << addr.GetPort(); 53 | return false; 54 | } 55 | ret = ::listen(localSock_, ListenSocket::LISTENQ); 56 | if (SOCKET_ERROR == ret) 57 | { 58 | CloseSocket(localSock_); 59 | ERR << "Cannot listen on port " << addr.GetPort(); 60 | return false; 61 | } 62 | 63 | if (!NetThreadPool::Instance().AddSocket(shared_from_this(), EventTypeRead)) 64 | return false; 65 | 66 | USR << "Success: listen on (" 67 | << addr.GetIP() 68 | << ":" 69 | << addr.GetPort() 70 | << ")"; 71 | 72 | return true; 73 | } 74 | 75 | int ListenSocket::_Accept() 76 | { 77 | socklen_t addrLength = sizeof addrClient_; 78 | return ::accept(localSock_, (struct sockaddr *)&addrClient_, &addrLength); 79 | } 80 | 81 | bool ListenSocket::OnReadable() 82 | { 83 | while (true) 84 | { 85 | int connfd = _Accept(); 86 | if (connfd >= 0) 87 | { 88 | Server::Instance()->NewConnection(connfd, tag_); 89 | } 90 | else 91 | { 92 | bool result = false; 93 | switch (errno) 94 | { 95 | case EWOULDBLOCK: 96 | case ECONNABORTED: 97 | case EINTR: 98 | result = true; 99 | break; 100 | 101 | case EMFILE: 102 | case ENFILE: 103 | ERR << "Not enough file descriptor available!!!"; 104 | result = true; 105 | break; 106 | 107 | case ENOBUFS: 108 | ERR << "Not enough memory"; 109 | result = true; 110 | 111 | default: 112 | ERR << "When accept, unknown error happened : " << errno; 113 | break; 114 | } 115 | 116 | return result; 117 | } 118 | } 119 | 120 | return true; 121 | } 122 | 123 | bool ListenSocket::OnWritable() 124 | { 125 | return false; 126 | } 127 | 128 | bool ListenSocket::OnError() 129 | { 130 | if (Socket::OnError()) 131 | { 132 | Server::Instance()->DelListenSock(localSock_); 133 | return true; 134 | } 135 | 136 | return false; 137 | } 138 | 139 | } 140 | 141 | -------------------------------------------------------------------------------- /cluster/ananas/net/Timer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "Timer.h" 4 | 5 | namespace ananas 6 | { 7 | namespace internal 8 | { 9 | 10 | unsigned int TimerManager::s_timerIdGen_ = 0; 11 | 12 | TimerManager::TimerManager() 13 | { 14 | } 15 | 16 | TimerManager::~TimerManager() 17 | { 18 | } 19 | 20 | void TimerManager::Update() 21 | { 22 | if (timers_.empty()) 23 | return; 24 | 25 | const auto now = std::chrono::steady_clock::now(); 26 | 27 | for (auto it(timers_.begin()); it != timers_.end(); ) 28 | { 29 | if (it->first > now) 30 | return; 31 | 32 | // support cancel self 33 | it->second.OnTimer(); 34 | 35 | // steal and erase it 36 | Timer timer(std::move(it->second)); 37 | it = timers_.erase(it); 38 | 39 | if (timer.count_ != 0) 40 | { 41 | // need reschedule 42 | const auto tp = timer.id_->first; 43 | auto itNew = timers_.insert(std::make_pair(tp, std::move(timer))); 44 | if (it == timers_.end() || itNew->first < it->first) 45 | it = itNew; 46 | } 47 | } 48 | } 49 | 50 | bool TimerManager::Cancel(TimerId id) 51 | { 52 | //time point + uid 53 | auto begin = timers_.lower_bound(id->first); 54 | if (begin == timers_.end()) 55 | return false; 56 | 57 | auto end = timers_.upper_bound(id->first); 58 | for (auto it(begin); it != end; ++ it) 59 | { 60 | if (it->second.UniqueId() == id->second) 61 | { 62 | // lazy delete, in case cancel another timer during on timer call. 63 | it->second.count_ = 0; 64 | return true; 65 | } 66 | } 67 | 68 | return false; 69 | } 70 | 71 | DurationMs TimerManager::NearestTimer() const 72 | { 73 | if (timers_.empty()) 74 | return DurationMs::max(); 75 | 76 | const auto& timer = timers_.begin()->second; 77 | auto now = std::chrono::steady_clock::now(); 78 | if (now > timer.Id()->first) 79 | return DurationMs::min(); 80 | else 81 | return std::chrono::duration_cast(timer.Id()->first - now); 82 | } 83 | 84 | TimerManager::Timer::Timer(const TimePoint& tp) : 85 | id_(new std::pair{tp, ++ TimerManager::s_timerIdGen_}), 86 | count_(kForever) 87 | { 88 | } 89 | 90 | TimerManager::Timer::Timer(Timer&& timer) 91 | { 92 | _Move(std::move(timer)); 93 | } 94 | 95 | TimerManager::Timer& TimerManager::Timer::operator= (Timer&& timer) 96 | { 97 | if (this != &timer) 98 | _Move(std::move(timer)); 99 | 100 | return *this; 101 | } 102 | 103 | void TimerManager::Timer::_Move(Timer&& timer) 104 | { 105 | this->id_ = std::move(timer.id_); 106 | this->func_ = std::move(timer.func_); 107 | this->interval_ = std::move(timer.interval_); 108 | this->count_ = timer.count_; 109 | } 110 | 111 | 112 | void TimerManager::Timer::OnTimer() 113 | { 114 | if (!func_ || count_ == 0) 115 | return; 116 | 117 | if (count_ == kForever || count_-- > 0) 118 | { 119 | func_(); 120 | id_->first += interval_; 121 | } 122 | else 123 | { 124 | count_ = 0; // in case if count_ other than -1 125 | } 126 | } 127 | 128 | TimerId TimerManager::Timer::Id() const 129 | { 130 | return id_; 131 | } 132 | 133 | unsigned int TimerManager::Timer::UniqueId() const 134 | { 135 | return id_->second; 136 | } 137 | 138 | } // end namespace internal 139 | } // end namespace ananas 140 | 141 | -------------------------------------------------------------------------------- /QBase/ClientSocket.cc: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "Server.h" 9 | #include "ClientSocket.h" 10 | #include "Log/Logger.h" 11 | #include "NetThreadPool.h" 12 | 13 | 14 | ClientSocket::ClientSocket(int tag) : tag_(tag) 15 | { 16 | } 17 | 18 | ClientSocket::~ClientSocket() 19 | { 20 | if (localSock_ != INVALID_SOCKET) 21 | WRN << "Destruct ClientSocket " << localSock_; 22 | } 23 | 24 | bool ClientSocket::Connect(const SocketAddr& dst) 25 | { 26 | if (dst.Empty()) 27 | return false; 28 | 29 | if (INVALID_SOCKET != localSock_) 30 | return false; 31 | 32 | peerAddr_ = dst; 33 | 34 | localSock_ = CreateTCPSocket(); 35 | SetNonBlock(localSock_); 36 | SetNodelay(localSock_); 37 | SetRcvBuf(localSock_); 38 | SetSndBuf(localSock_); 39 | 40 | int result = ::connect(localSock_, (sockaddr*)&peerAddr_.GetAddr(), sizeof(sockaddr_in)); 41 | 42 | if (0 == result) 43 | { 44 | INF << "ClientSocket immediately connected to [" 45 | << peerAddr_.GetIP() 46 | << ":" 47 | << peerAddr_.GetPort() 48 | << "]"; 49 | 50 | Server::Instance()->NewConnection(localSock_, tag_, onConnectFail_); 51 | localSock_ = INVALID_SOCKET; 52 | return true; 53 | } 54 | else 55 | { 56 | if (EINPROGRESS == errno) 57 | { 58 | 59 | INF << "EINPROGRESS: ClientSocket connected to (" 60 | << peerAddr_.GetIP() 61 | << ":" 62 | << peerAddr_.GetPort() 63 | << ")"; 64 | 65 | Internal::NetThreadPool::Instance().AddSocket(shared_from_this(), EventTypeWrite); 66 | epollOut_ = true; 67 | return true; 68 | } 69 | 70 | ERR << "Failed: ClientSocket connected to (" 71 | << peerAddr_.GetIP() 72 | << ":" 73 | << peerAddr_.GetPort() 74 | << ")"; 75 | 76 | if (onConnectFail_) 77 | onConnectFail_(); 78 | 79 | return false; 80 | } 81 | 82 | return true; 83 | } 84 | 85 | 86 | bool ClientSocket::OnWritable() 87 | { 88 | epollOut_ = false; 89 | int error = 0; 90 | socklen_t len = sizeof error; 91 | 92 | bool bSucc = (::getsockopt(localSock_, SOL_SOCKET, SO_ERROR, (char*)&error, &len) >= 0 && 0 == error); 93 | if (!bSucc) 94 | { 95 | errno = error; 96 | ERR << "Failed: ClientSocket connected to (" 97 | << peerAddr_.GetIP() 98 | << ":" 99 | << peerAddr_.GetPort() 100 | << "), error:" << error; 101 | 102 | return false; 103 | } 104 | 105 | INF << "Success: ClientSocket connected to (" 106 | << peerAddr_.GetIP() 107 | << ":" 108 | << peerAddr_.GetPort() << ")"; 109 | 110 | Server::Instance()->NewConnection(localSock_, tag_, onConnectFail_); 111 | localSock_ = INVALID_SOCKET; 112 | 113 | return true; 114 | } 115 | 116 | bool ClientSocket::OnError() 117 | { 118 | if (Socket::OnError()) 119 | { 120 | ERR << __FUNCTION__ << " connected to (" 121 | << peerAddr_.GetIP() 122 | << ":" 123 | << peerAddr_.GetPort() << ")"; 124 | 125 | if (onConnectFail_) 126 | onConnectFail_(); 127 | 128 | return true; 129 | } 130 | 131 | return false; 132 | } 133 | 134 | -------------------------------------------------------------------------------- /QedisCore/QHelper.h: -------------------------------------------------------------------------------- 1 | #ifndef BERT_HELPER_H 2 | #define BERT_HELPER_H 3 | 4 | #include "QString.h" 5 | #include 6 | #include 7 | 8 | namespace qedis 9 | { 10 | 11 | // hash func from redis 12 | extern unsigned int dictGenHashFunction(const void* key, int len); 13 | 14 | // hash function 15 | struct my_hash 16 | { 17 | size_t operator() (const QString& str) const; 18 | }; 19 | 20 | std::size_t BitCount(const uint8_t* buf, std::size_t len); 21 | 22 | template 23 | inline typename HASH::const_local_iterator RandomHashMember(const HASH& container) 24 | { 25 | if (container.empty()) 26 | { 27 | return typename HASH::const_local_iterator(); 28 | } 29 | 30 | while (true) 31 | { 32 | size_t bucket = random() % container.bucket_count(); 33 | if (container.bucket_size(bucket) == 0) 34 | continue; 35 | 36 | long lucky = random() % container.bucket_size(bucket); 37 | typename HASH::const_local_iterator it = container.begin(bucket); 38 | while (lucky > 0) 39 | { 40 | ++ it; 41 | -- lucky; 42 | } 43 | 44 | return it; 45 | } 46 | 47 | 48 | return typename HASH::const_local_iterator(); 49 | } 50 | 51 | // scan 52 | template 53 | inline size_t ScanHashMember(const HASH& container, 54 | size_t cursor, 55 | size_t count, 56 | std::vector& res) 57 | { 58 | if (cursor >= container.size()) 59 | { 60 | return 0; 61 | } 62 | 63 | auto idx = cursor; 64 | for (decltype(container.bucket_count()) bucket = 0; 65 | bucket < container.bucket_count(); 66 | ++ bucket) 67 | { 68 | const auto bktSize = container.bucket_size(bucket); 69 | if (idx < bktSize) 70 | { 71 | // find the bucket; 72 | auto it = container.begin(bucket); 73 | while (idx > 0) 74 | { 75 | ++ it; 76 | -- idx; 77 | } 78 | 79 | size_t newCursor = cursor; 80 | auto end = container.end(bucket); 81 | while (res.size() < count && it != end) 82 | { 83 | ++ newCursor; 84 | res.push_back(it ++); 85 | 86 | if (it == end) 87 | { 88 | while (++ bucket < container.bucket_count()) 89 | { 90 | if (container.bucket_size(bucket) > 0) 91 | { 92 | it = container.begin(bucket); 93 | end = container.end(bucket); 94 | break; 95 | } 96 | } 97 | 98 | if (bucket == container.bucket_count()) 99 | return 0; 100 | } 101 | } 102 | 103 | return newCursor; 104 | } 105 | else 106 | { 107 | idx -= bktSize; 108 | } 109 | } 110 | 111 | return 0; // never here 112 | } 113 | 114 | extern void getRandomHexChars(char *p, unsigned int len); 115 | 116 | enum MemoryInfoType { 117 | VmPeak = 0, 118 | VmSize = 1, 119 | VmLck = 2, 120 | VmHWM = 3, 121 | VmRSS = 4, 122 | VmSwap = 5, 123 | 124 | VmMax = VmSwap + 1, 125 | }; 126 | 127 | extern std::vector getMemoryInfo(); 128 | extern size_t getMemoryInfo(MemoryInfoType type); 129 | 130 | } 131 | 132 | #endif 133 | 134 | -------------------------------------------------------------------------------- /QBase/EPoller.cc: -------------------------------------------------------------------------------- 1 | 2 | #if defined(__gnu_linux__) 3 | 4 | #include "EPoller.h" 5 | #include "Log/Logger.h" 6 | 7 | #include 8 | #include 9 | 10 | namespace Epoll 11 | { 12 | bool ModSocket(int epfd, int socket, uint32_t events, void* ptr); 13 | 14 | bool AddSocket(int epfd, int socket, uint32_t events, void* ptr) 15 | { 16 | if (socket < 0) 17 | return false; 18 | 19 | epoll_event ev; 20 | ev.data.ptr= ptr; 21 | 22 | ev.events = 0; 23 | if (events & EventTypeRead) 24 | ev.events |= EPOLLIN; 25 | if (events & EventTypeWrite) 26 | ev.events |= EPOLLOUT; 27 | 28 | return 0 == epoll_ctl(epfd, EPOLL_CTL_ADD, socket, &ev); 29 | } 30 | 31 | bool DelSocket(int epfd, int socket) 32 | { 33 | if (socket < 0) 34 | return false; 35 | 36 | epoll_event dummy; 37 | 38 | return 0 == epoll_ctl(epfd, EPOLL_CTL_DEL, socket, &dummy) ; 39 | } 40 | 41 | bool ModSocket(int epfd, int socket, uint32_t events, void* ptr) 42 | { 43 | if (socket < 0) 44 | return false; 45 | 46 | epoll_event ev; 47 | ev.data.ptr= ptr; 48 | 49 | ev.events = 0; 50 | if (events & EventTypeRead) 51 | ev.events |= EPOLLIN; 52 | if (events & EventTypeWrite) 53 | ev.events |= EPOLLOUT; 54 | 55 | return 0 == epoll_ctl(epfd, EPOLL_CTL_MOD, socket, &ev); 56 | } 57 | } 58 | 59 | 60 | Epoller::Epoller() 61 | { 62 | multiplexer_ = ::epoll_create(512); 63 | INF << "create epoll: " << multiplexer_; 64 | } 65 | 66 | Epoller::~Epoller() 67 | { 68 | INF << "close epoll: " << multiplexer_; 69 | if (multiplexer_ != -1) 70 | ::close(multiplexer_); 71 | } 72 | 73 | bool Epoller::AddSocket(int sock, int events, void* userPtr) 74 | { 75 | if (Epoll::AddSocket(multiplexer_, sock, events, userPtr)) 76 | return true; 77 | 78 | return (errno == EEXIST) && ModSocket(sock, events, userPtr); 79 | } 80 | 81 | bool Epoller::DelSocket(int sock, int events) 82 | { 83 | return Epoll::DelSocket(multiplexer_, sock); 84 | } 85 | 86 | 87 | bool Epoller::ModSocket(int sock, int events, void* userPtr) 88 | { 89 | if (events == 0) 90 | return DelSocket(sock, 0); 91 | 92 | if (Epoll::ModSocket(multiplexer_, sock, events, userPtr)) 93 | return true; 94 | 95 | return errno == ENOENT && AddSocket(sock, events, userPtr); 96 | } 97 | 98 | 99 | int Epoller::Poll(std::vector& events, size_t maxEvent, int timeoutMs) 100 | { 101 | if (maxEvent == 0) 102 | return 0; 103 | 104 | while (events_.size() < maxEvent) 105 | events_.resize(2 * events_.size() + 1); 106 | 107 | int nFired = TEMP_FAILURE_RETRY(::epoll_wait(multiplexer_, &events_[0], maxEvent, timeoutMs)); 108 | if (nFired == -1 && errno != EINTR && errno != EWOULDBLOCK) 109 | return -1; 110 | 111 | if (nFired > 0 && static_cast(nFired) > events.size()) 112 | events.resize(nFired); 113 | 114 | for (int i = 0; i < nFired; ++ i) 115 | { 116 | FiredEvent& fired = events[i]; 117 | fired.events = 0; 118 | fired.userdata = events_[i].data.ptr; 119 | 120 | if (events_[i].events & EPOLLIN) 121 | fired.events |= EventTypeRead; 122 | 123 | if (events_[i].events & EPOLLOUT) 124 | fired.events |= EventTypeWrite; 125 | 126 | if (events_[i].events & (EPOLLERR | EPOLLHUP)) 127 | fired.events |= EventTypeError; 128 | } 129 | 130 | return nFired; 131 | } 132 | 133 | #endif 134 | 135 | -------------------------------------------------------------------------------- /QedisCore/QProtoParser.cc: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | #include "QCommon.h" 5 | #include "QProtoParser.h" 6 | 7 | #include 8 | 9 | // 1 request -> multi strlist 10 | // 2 multi -> * number crlf 11 | // 3 strlist -> str strlist | empty 12 | // 4 str -> strlen strval 13 | // 5 strlen -> $ number crlf 14 | // 6 strval -> string crlf 15 | 16 | namespace qedis 17 | { 18 | 19 | void QProtoParser::Reset() 20 | { 21 | multi_ = -1; 22 | paramLen_ = -1; 23 | numOfParam_ = 0; 24 | 25 | // Optimize: Most redis command has 3 args 26 | while (params_.size() > 3) 27 | params_.pop_back(); 28 | } 29 | 30 | QParseResult QProtoParser::ParseRequest(const char*& ptr, const char* end) 31 | { 32 | if (multi_ == -1) 33 | { 34 | auto parseRet = _ParseMulti(ptr, end, multi_); 35 | if (parseRet == QParseResult::error || 36 | multi_ < -1) 37 | return QParseResult::error; 38 | 39 | if (parseRet != QParseResult::ok) 40 | return QParseResult::wait; 41 | } 42 | 43 | return _ParseStrlist(ptr, end, params_); 44 | } 45 | 46 | QParseResult QProtoParser::_ParseMulti(const char*& ptr, const char* end, int& result) 47 | { 48 | if (end - ptr < 3) 49 | return QParseResult::wait; 50 | 51 | if (*ptr != '*') 52 | return QParseResult::error; 53 | 54 | ++ ptr; 55 | 56 | return GetIntUntilCRLF(ptr, end - ptr, result); 57 | } 58 | 59 | QParseResult QProtoParser::_ParseStrlist(const char*& ptr, const char* end, std::vector& results) 60 | { 61 | while (static_cast(numOfParam_) < multi_) 62 | { 63 | if (results.size() < numOfParam_ + 1) 64 | results.resize(numOfParam_ + 1); 65 | 66 | auto parseRet = _ParseStr(ptr, end, results[numOfParam_]); 67 | 68 | if (parseRet == QParseResult::ok) 69 | { 70 | ++ numOfParam_; 71 | } 72 | else 73 | { 74 | return parseRet; 75 | } 76 | } 77 | 78 | results.resize(numOfParam_); 79 | return QParseResult::ok; 80 | } 81 | 82 | QParseResult QProtoParser::_ParseStr(const char*& ptr, const char* end, QString& result) 83 | { 84 | if (paramLen_ == -1) 85 | { 86 | auto parseRet = _ParseStrlen(ptr, end, paramLen_); 87 | if (parseRet == QParseResult::error || 88 | paramLen_ < -1) 89 | return QParseResult::error; 90 | 91 | if (parseRet != QParseResult::ok) 92 | return QParseResult::wait; 93 | } 94 | 95 | if (paramLen_ == -1) 96 | { 97 | result.clear(); // or should be "(nil)" ? 98 | return QParseResult::ok; 99 | } 100 | else 101 | { 102 | return _ParseStrval(ptr, end, result); 103 | } 104 | } 105 | 106 | QParseResult QProtoParser::_ParseStrval(const char*& ptr, const char* end, QString& result) 107 | { 108 | assert (paramLen_ >= 0); 109 | 110 | if (static_cast(end - ptr) < paramLen_ + 2) 111 | return QParseResult::wait; 112 | 113 | auto tail = ptr + paramLen_; 114 | if (tail[0] != '\r' || tail[1] != '\n') 115 | return QParseResult::error; 116 | 117 | result.assign(ptr, tail - ptr); 118 | ptr = tail + 2; 119 | paramLen_ = -1; 120 | 121 | return QParseResult::ok; 122 | } 123 | 124 | QParseResult QProtoParser::_ParseStrlen(const char*& ptr, const char* end, int& result) 125 | { 126 | if (end - ptr < 3) 127 | return QParseResult::wait; 128 | 129 | if (*ptr != '$') 130 | return QParseResult::error; 131 | 132 | ++ ptr; 133 | 134 | const auto ret = GetIntUntilCRLF(ptr, end - ptr, result); 135 | if (ret != QParseResult::ok) 136 | -- ptr; 137 | 138 | return ret; 139 | } 140 | 141 | } 142 | 143 | --------------------------------------------------------------------------------