├── .gitignore ├── CMakeLists.txt ├── README.md ├── README └── image-20230911212756277.png ├── app ├── CMakeLists.txt ├── client.cc └── server.cc ├── cmake └── modules │ ├── Findrdmacm.cmake │ └── Findverbs.cmake ├── include ├── client.h ├── connection.h ├── const.h ├── context.h ├── handler.h ├── message.h ├── misc.h ├── server.h └── util.h ├── setup_env.sh └── src ├── CMakeLists.txt ├── client.cc ├── connection.cc ├── context.cc ├── handler.cc ├── message.cc └── server.cc /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .vscode/ -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(rdma-example) 3 | 4 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/") 5 | 6 | find_package(verbs) 7 | find_package(rdmacm) 8 | find_package(Threads) 9 | 10 | find_package(PkgConfig REQUIRED) 11 | pkg_check_modules(libevent REQUIRED IMPORTED_TARGET libevent) 12 | 13 | add_subdirectory(src) 14 | add_subdirectory(app) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RDMA-RPC 2 | 3 | This is a multi-send & single-receive RPC based on RDMA. 4 | 5 | ### Env 6 | 7 | - Build tool: `CMake` (version 3.10 +) 8 | - Dependent library: `rdmacm`、`ibverbs`、`event`、`pthread` 9 | - Dependent Environment: 10 | - RDMA-enabled NIC; 11 | - or, Soft-RoCE instead. 12 | 13 | ``` bash 14 | # load rxe 15 | sudo modprobe rdma_rxe 16 | # set Soft-RoCE, change 'ens33' to your NIC name 17 | sudo rdma link add rxe_0 type rxe netdev ens33 18 | # check if ACTIVE 19 | sudo rdma link 20 | # show the rdma(virtual) device 21 | ibv_devices 22 | ``` 23 | 24 | ### How to use 25 | 26 | - Set up the environment: 27 | 28 | ```bash 29 | . setup_env.sh 30 | ``` 31 | 32 | - Build the lib and example: 33 | 34 | ```bash 35 | mkdir build && cd build 36 | cmake .. 37 | make 38 | 39 | # tree of build 40 | . 41 | ├── app # example 42 | ├── CMakeCache.txt 43 | ├── CMakeFiles 44 | ├── cmake_install.cmake 45 | ├── Makefile 46 | └── src # lib 47 | ``` 48 | 49 | - Use the lib, for example: 50 | 51 | ```bash 52 | # bash-1 53 | ./app/server 54 | # bash-2 55 | ./app/client 56 | # bash-N 57 | ./app/client # same as above 58 | ``` 59 | 60 | - Use the lib, for yourself: 61 | 62 | ```cpp 63 | // Server 64 | Server s(const char* /*host*/, const char* /*port*/); 65 | s.run(); 66 | // Client 67 | Client c; 68 | c.connect(const char* /*server host*/, const char* /*server port*/); 69 | c.sendRequest(string /*msg*/); 70 | ``` 71 | 72 | - Customize your own RPC Handler: 73 | 74 | ```cpp 75 | // Implement a subclass of Handler if you'd like customize the rpc 76 | class Handler { 77 | public: 78 | Message handlerRequest(Message* req); 79 | } 80 | ``` 81 | 82 | ### Design 83 | 84 | #### Resources Management 85 | 86 | - All RDMA resources(pd, mr, qp, cp, etc.) are encapsulated in the Connection class, where each connection corresponds to a link. 87 | - A Client allows only one Connection, whereas a Server allows multiple. 88 | - The Connections are managed and polled through Poller. 89 | - Server uses a single-thread Poller to poll all Connections. 90 | 91 | #### RPC Procedure 92 | 93 | A RCP procedure for one Connection is depicted in the following diagram: 94 | 95 | ![image-20230911212756277](README/image-20230911212756277.png) 96 | 97 | ### Q&A 98 | 99 | - Why not use RDMA Write ? 100 | 101 | > Since this is an RPC, the client needs to be aware of the server's response, so there is no need to use "Write". If "Write" is used, the client will have to continuously poll the memory for the server response. 102 | 103 | - If I wanna connect multi Server, how can I do this ? 104 | 105 | > Just start multi Client objects, each Client connects to a Server. 106 | 107 | ### To do 108 | 109 | - [ ] add the bench to test performance. 110 | - [ ] accelerate the RPC of large request. 111 | - [ ] optimize memory management. 112 | 113 | ### Reference 114 | 115 | - [RDMA mojo](http://www.rdmamojo.com/) 116 | - [Hammer Li's rdma-based-rpc](https://github.com/PDSL-DPU-KV/rdma-based-rpc/tree/master) 117 | - [rdma-example](https://github.com/animeshtrivedi/rdma-example) -------------------------------------------------------------------------------- /README/image-20230911212756277.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sakura-ysy/rdma-rpc/364b21e795c30110ddb62a4b313c93e1a57558b3/README/image-20230911212756277.png -------------------------------------------------------------------------------- /app/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(app) 2 | file(GLOB_RECURSE SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.cc) 3 | add_executable(server server.cc) 4 | add_executable(client client.cc) 5 | 6 | target_include_directories(server PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include) 7 | target_link_libraries(server PUBLIC rdma-lib) 8 | 9 | target_include_directories(client PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../include) 10 | target_link_libraries(client PUBLIC rdma-lib) -------------------------------------------------------------------------------- /app/client.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | std::string rand_str(const int len) 7 | { 8 | 9 | std::string str; 10 | char c; 11 | for(int i = 0; i < len; i++) 12 | { 13 | c = 'a' + rand()%26; 14 | str.push_back(c); 15 | } 16 | return str; 17 | } 18 | 19 | int main([[gnu::unused]] int argc, char *argv[]) { 20 | Client c; 21 | c.connect(argv[1], argv[2]); 22 | 23 | info("connect successfully"); 24 | info("start to send request"); 25 | 26 | int len = 10; 27 | int cnt = 100; 28 | for (int i = 0; i < cnt; i++) 29 | { 30 | c.sendRequest(rand_str(len)); 31 | } 32 | 33 | sleep(3); 34 | } -------------------------------------------------------------------------------- /app/server.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | int main([[gnu::unused]] int argc, char *argv[]) { 6 | Server s(argv[1], argv[2]); 7 | s.run(); 8 | } -------------------------------------------------------------------------------- /cmake/modules/Findrdmacm.cmake: -------------------------------------------------------------------------------- 1 | # - Find rdma cm 2 | # Find the rdma cm library and includes 3 | # 4 | # RDMACM_INCLUDE_DIR - where to find cma.h, etc. 5 | # RDMACM_LIBRARIES - List of libraries when using rdmacm. 6 | # RDMACM_FOUND - True if rdmacm found. 7 | 8 | find_path(RDMACM_INCLUDE_DIR rdma/rdma_cma.h) 9 | find_library(RDMACM_LIBRARIES rdmacm) 10 | 11 | include(FindPackageHandleStandardArgs) 12 | find_package_handle_standard_args(rdmacm DEFAULT_MSG RDMACM_LIBRARIES RDMACM_INCLUDE_DIR) 13 | 14 | if(RDMACM_FOUND) 15 | if(NOT TARGET RDMA::RDMAcm) 16 | add_library(RDMA::RDMAcm UNKNOWN IMPORTED) 17 | endif() 18 | set_target_properties(RDMA::RDMAcm PROPERTIES 19 | INTERFACE_INCLUDE_DIRECTORIES "${RDMACM_INCLUDE_DIR}" 20 | IMPORTED_LINK_INTERFACE_LANGUAGES "C" 21 | IMPORTED_LOCATION "${RDMACM_LIBRARIES}") 22 | endif() 23 | 24 | mark_as_advanced( 25 | RDMACM_LIBRARIES 26 | ) 27 | -------------------------------------------------------------------------------- /cmake/modules/Findverbs.cmake: -------------------------------------------------------------------------------- 1 | # - Find rdma verbs 2 | # Find the rdma verbs library and includes 3 | # 4 | # VERBS_INCLUDE_DIR - where to find ibverbs.h, etc. 5 | # VERBS_LIBRARIES - List of libraries when using ibverbs. 6 | # VERBS_FOUND - True if ibverbs found. 7 | # HAVE_IBV_EXP - True if experimental verbs is enabled. 8 | 9 | find_path(VERBS_INCLUDE_DIR infiniband/verbs.h) 10 | find_library(VERBS_LIBRARIES ibverbs) 11 | 12 | include(FindPackageHandleStandardArgs) 13 | find_package_handle_standard_args(verbs DEFAULT_MSG VERBS_LIBRARIES VERBS_INCLUDE_DIR) 14 | 15 | if(VERBS_FOUND) 16 | include(CheckCXXSourceCompiles) 17 | CHECK_CXX_SOURCE_COMPILES(" 18 | #include 19 | int main() { 20 | struct ibv_context* ctxt; 21 | struct ibv_exp_gid_attr gid_attr; 22 | ibv_exp_query_gid_attr(ctxt, 1, 0, &gid_attr); 23 | return 0; 24 | } " HAVE_IBV_EXP) 25 | if(NOT TARGET IBVerbs::verbs) 26 | add_library(IBVerbs::verbs UNKNOWN IMPORTED) 27 | endif() 28 | set_target_properties(IBVerbs::verbs PROPERTIES 29 | INTERFACE_INCLUDE_DIRECTORIES "${VERBS_INCLUDE_DIR}" 30 | IMPORTED_LINK_INTERFACE_LANGUAGES "C" 31 | IMPORTED_LOCATION "${VERBS_LIBRARIES}") 32 | endif() 33 | 34 | mark_as_advanced( 35 | VERBS_LIBRARIES 36 | ) 37 | -------------------------------------------------------------------------------- /include/client.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | class ClientPoller { 19 | public: 20 | ClientPoller(); 21 | ~ClientPoller(); 22 | 23 | void registerConn(Connection* conn); 24 | void deregisterConn(); 25 | void sendRequest(Message req); 26 | 27 | void run(); 28 | void stop(); 29 | void poll(); 30 | 31 | private: 32 | std::atomic_bool running_{false}; 33 | Spinlock lock_{}; 34 | Connection* conn_; 35 | std::thread poll_thread_; 36 | }; 37 | 38 | 39 | // one client can only connect one server 40 | class Client { 41 | public: 42 | Client(); 43 | ~Client(); 44 | 45 | void connect(const char* host, const char* port); 46 | rdma_cm_event* waitEvent(rdma_cm_event_type expected); 47 | void setupConnection(rdma_cm_id* cm_id, uint32_t n_buffer_page); 48 | 49 | void sendRequest(std::string msg); 50 | 51 | private: 52 | rdma_cm_id* cm_id_; // only one qp, so only one cm_id 53 | addrinfo* dst_addr_{nullptr}; 54 | rdma_event_channel* cm_event_channel_{nullptr}; 55 | 56 | // event-driven, to avoid the block 57 | event_base* base_{nullptr}; 58 | event* conn_event_{nullptr}; 59 | event* exit_event_{nullptr}; 60 | 61 | // connection related 62 | ClientPoller poller_{}; 63 | }; 64 | -------------------------------------------------------------------------------- /include/connection.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class Server; 13 | 14 | enum Role : int32_t { 15 | Error, 16 | ServerConn, 17 | ClientConn, 18 | }; 19 | 20 | enum State : int32_t { 21 | Vacant, // Server & Client 22 | WaitingForRequest, // Server 23 | WaitingForResponse, // Client 24 | HandlingRequest, // Server 25 | }; 26 | 27 | 28 | class Connection { 29 | public: 30 | 31 | public: 32 | 33 | // for client, the cm_id is local, 34 | // for server, the cm_id is remote 35 | Connection(Role role, rdma_cm_id* cm_id, uint32_t n_buffer_page); 36 | ~Connection(); 37 | 38 | rdma_conn_param copyConnParam(); 39 | void setRkey(uint32_t rkey); 40 | rdma_cm_id* getCmId(); 41 | uint32_t getLKey(); 42 | uint32_t getRKey(); 43 | void* getMRAddr(); 44 | 45 | void postSend(void* local_addr, uint32_t length, uint32_t lkey, bool need_inline); 46 | void postRecv(void* local_addr, uint32_t length, uint32_t lkey); 47 | void lock(); 48 | void unlock(); 49 | 50 | void fillMR(void* data, uint32_t size); 51 | 52 | void prepare(); 53 | void poll(); 54 | void serverAdvance(const ibv_wc &wc); 55 | void clientAdvance(const ibv_wc &wc); 56 | 57 | private: 58 | static ibv_qp_init_attr defaultQpInitAttr(); 59 | 60 | Role role_{Role::Error}; 61 | State state_{State::Vacant}; 62 | // for client, it presents the local cm_id, 63 | // for server, it presents the remote(client) cm_id 64 | rdma_cm_id* cm_id_; 65 | ibv_pd* local_pd_; 66 | ibv_cq* local_cq_; 67 | ibv_qp* local_qp_; 68 | uint32_t n_buffer_page_; 69 | void* buffer_; 70 | ibv_mr* buffer_mr_; 71 | rdma_conn_param param_; 72 | uint32_t rkey_; 73 | uint32_t lkey_; 74 | Spinlock lock_{}; 75 | 76 | // hander; 77 | Handler handler_{}; 78 | }; 79 | -------------------------------------------------------------------------------- /include/const.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | constexpr uint32_t DEFAULT_BACK_LOG = 8; 5 | constexpr uint32_t MAX_CONNECTION_NUM = 8; 6 | constexpr uint32_t MAX_QUEUE_SIZE = 256; 7 | constexpr uint32_t MAX_WORKER_NUM = 2; 8 | constexpr uint32_t DEFAULT_CQ_CAPACITY = 64; 9 | constexpr uint32_t MAX_SEND_WR_NUM = 64; 10 | constexpr uint32_t MAX_RECV_WR_NUM = 64; 11 | constexpr uint32_t BUFFER_PAGE_SIZE = 65536; 12 | constexpr uint32_t DEFAULT_CONNECTION_TIMEOUT = 3000; 13 | constexpr uint32_t MESSAGE_BUF_SIZE = 64; -------------------------------------------------------------------------------- /include/context.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | class Context { 5 | public: 6 | Context(void* addr, uint32_t len); 7 | ~Context(); 8 | 9 | void* addr(); 10 | uint32_t length(); 11 | 12 | private: 13 | void* addr_; 14 | uint32_t length_; 15 | }; -------------------------------------------------------------------------------- /include/handler.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | class Handler { 4 | public: 5 | Handler(); 6 | ~Handler(); 7 | 8 | Message handlerRequest(Message* req); 9 | 10 | private: 11 | }; -------------------------------------------------------------------------------- /include/message.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | class [[gnu::packed]] BufferMeta { 7 | public: 8 | char buf_[MESSAGE_BUF_SIZE]{}; 9 | }; 10 | 11 | enum MessageType { 12 | Dummy, 13 | Request, 14 | ImmRequest, 15 | Response, 16 | }; 17 | 18 | class [[gnu::packed]] Header { 19 | public: 20 | uint32_t data_len_{}; 21 | MessageType type_{Dummy}; 22 | }; 23 | 24 | 25 | class [[gnu::packed]] Message { 26 | public: 27 | explicit Message(char* buf, uint32_t len, MessageType type); 28 | ~Message(); 29 | 30 | char* dataAddr(); 31 | uint32_t dataLen(); 32 | MessageType msgType(); 33 | 34 | private: 35 | Header header_{}; 36 | BufferMeta meta_; 37 | }; 38 | 39 | -------------------------------------------------------------------------------- /include/misc.h: -------------------------------------------------------------------------------- 1 | #ifndef __RDMA_EXAMPLE_MISC__ 2 | #define __RDMA_EXAMPLE_MISC__ 3 | 4 | #include 5 | 6 | #if defined(__x86_64__) 7 | 8 | #include 9 | #define PAUSE _mm_pause() 10 | 11 | #elif defined(__aarch64__) 12 | 13 | #define PAUSE asm volatile("yield" ::: "memory") 14 | 15 | #else 16 | 17 | #define PAUSE 18 | 19 | #endif 20 | 21 | 22 | // TTAS Lock 23 | class Spinlock { 24 | public: 25 | Spinlock() = default; 26 | ~Spinlock() = default; 27 | 28 | public: 29 | auto lock() -> void { 30 | for (;;) { 31 | if (not b_.exchange(true, std::memory_order_acquire)) { 32 | return; 33 | } 34 | while (b_.load(std::memory_order_relaxed)) { 35 | PAUSE; 36 | } 37 | } 38 | } 39 | 40 | auto tryLock() -> bool { 41 | return not b_.load(std::memory_order_relaxed) and 42 | not b_.exchange(true, std::memory_order_acquire); 43 | } 44 | 45 | auto unlock() -> void { b_.store(false, std::memory_order_release); } 46 | 47 | private: 48 | std::atomic_bool b_{false}; 49 | }; 50 | 51 | 52 | #undef PAUSE 53 | 54 | #endif -------------------------------------------------------------------------------- /include/server.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | class Connection; 18 | 19 | 20 | class ServerPoller { 21 | public: 22 | ServerPoller(); 23 | ~ServerPoller(); 24 | 25 | void registerConn(Connection* conn); 26 | void deregisterConn(Connection* conn); 27 | 28 | void run(); 29 | void stop(); 30 | void poll(); 31 | 32 | private: 33 | std::atomic_bool running_{false}; 34 | Spinlock lock_{}; 35 | std::list conn_list_; 36 | std::thread poll_thread_; 37 | }; 38 | 39 | 40 | // one server can connect multiple clients; 41 | class Server { 42 | public: 43 | Server(const char* host, const char* port); 44 | ~Server(); 45 | 46 | void run(); 47 | void handleConnectionEvent(); 48 | void handleExitEvent(); 49 | 50 | void setupConnection(rdma_cm_event* cm_event, uint32_t n_buffer_page); 51 | 52 | private: 53 | static void onConnectionEvent(evutil_socket_t fd, short what, void* arg); 54 | 55 | addrinfo* addr_{nullptr}; 56 | rdma_event_channel* cm_event_channel_{nullptr}; 57 | rdma_cm_id* listen_cm_id_{nullptr}; 58 | 59 | // event-driven, to avoid the block 60 | event_base* base_{nullptr}; 61 | event* conn_event_{nullptr}; 62 | event* exit_event_{nullptr}; 63 | 64 | // poller 65 | ServerPoller poller_{}; 66 | }; 67 | -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | inline auto die(const char* err_msg) { 5 | throw std::runtime_error(err_msg); 6 | } 7 | 8 | inline void info(const char* msg) { 9 | fprintf(stderr, "%s\n", msg); 10 | } 11 | 12 | template 13 | static inline auto info(const char *fmt, Args... args) -> void { 14 | fprintf(stderr, fmt, args...); 15 | fprintf(stderr, "\n"); 16 | } 17 | 18 | // Fail if ret != type 19 | template 20 | inline void checkEqual(Type ret, Type cmp, const char* err_msg) { 21 | if (ret != cmp) 22 | die(err_msg); 23 | } 24 | 25 | // Fail if ret == type 26 | template 27 | inline void checkNotEqual(Type ret, Type cmp, const char* err_msg) { 28 | if (ret == cmp) 29 | die(err_msg); 30 | } 31 | 32 | // Warn if ret != type 33 | template 34 | inline void wCheckEqual(Type ret, Type cmp, const char* err_msg) { 35 | if (ret != cmp) 36 | info(err_msg); 37 | } 38 | 39 | // Warn if ret == type 40 | template 41 | inline void wCheckNotEqual(Type ret, Type cmp, const char* err_msg) { 42 | if (ret == cmp) 43 | info(err_msg); 44 | } 45 | -------------------------------------------------------------------------------- /setup_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cmd="apt-get update" 3 | echo "start: $cmd" 4 | sudo $cmd 5 | 6 | # rdma lib 7 | cmd="apt-get install libibverbs-dev librdmacm-dev libibverbs1 ibverbs-utils librdmacm1 libibumad3 ibverbs-providers infiniband-diags rdma-core -y" 8 | echo "start: $cmd" 9 | sudo $cmd 10 | 11 | # show the devicexs 12 | cmd="ibv_devices" 13 | echo "start: $cmd" 14 | sudo $cmd 15 | 16 | # iproute2 17 | cmd="apt-get install iproute2 -y" 18 | echo "start: $cmd" 19 | sudo $cmd 20 | 21 | # perftest 22 | cmd="apt-get install perftest -y" 23 | echo "start: $cmd" 24 | sudo $cmd 25 | 26 | # load rxe 27 | cmd="modprobe rdma_rxe" 28 | echo "start: $cmd" 29 | sudo $cmd 30 | 31 | # set Soft-RoCE 32 | # change 'ens33' to your NIC name 33 | # 'rxe_0' is the new name, you can change it to whatever you want 34 | cmd="rdma link add rxe_0 type rxe netdev ens33" 35 | echo "start: $cmd" 36 | sudo $cmd 37 | 38 | # check if ACTIVE 39 | cmd="rdma link" 40 | echo "start: $cmd" 41 | sudo $cmd 42 | 43 | 44 | # install libevent 45 | echo "start: install libevent" 46 | git clone https://github.com/libevent/libevent.git 47 | cd libevent 48 | mkdir build && cd build 49 | cmake .. 50 | make 51 | sudo make install 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(rdma-lib) 2 | file(GLOB_RECURSE SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.cc) 3 | add_library(${PROJECT_NAME} ${SOURCE_FILES}) 4 | target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../include) 5 | target_link_libraries(${PROJECT_NAME} PUBLIC IBVerbs::verbs RDMA::RDMAcm pthread PkgConfig::libevent) 6 | -------------------------------------------------------------------------------- /src/client.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | Client::Client() { 9 | cm_event_channel_ = rdma_create_event_channel(); 10 | checkNotEqual(cm_event_channel_, static_cast(nullptr), "rdma_create_event_channel() failed"); 11 | } 12 | 13 | Client::~Client() { 14 | // rdma_disconnect will generate an event and a IBV_WC_SEND 15 | poller_.stop(); 16 | int ret = rdma_disconnect(cm_id_); 17 | wCheckEqual(ret, 0, "rdma_disconnect() failed to disconnect"); 18 | rdma_cm_event* cm_event = waitEvent(RDMA_CM_EVENT_DISCONNECTED); 19 | wCheckNotEqual(cm_event, static_cast(nullptr), "failed to get disconnection event"); 20 | ret = rdma_ack_cm_event(cm_event); 21 | wCheckEqual(ret, 0, "rdma_ack_cm_event() failed to send ack"); 22 | freeaddrinfo(dst_addr_); 23 | rdma_destroy_event_channel(cm_event_channel_); 24 | poller_.deregisterConn(); 25 | info("client disconnect"); 26 | } 27 | 28 | void Client::connect(const char* host, const char* port) { 29 | 30 | int ret = 0; 31 | ret = rdma_create_id(cm_event_channel_, &cm_id_, nullptr, RDMA_PS_TCP); 32 | checkEqual(ret, 0, "rdma_create_id() failed"); 33 | 34 | ret = getaddrinfo(host, port, nullptr, &dst_addr_); 35 | checkEqual(ret, 0, "getaddrinfo() failed"); 36 | 37 | // When the resolution is completed, the cm_event_channel_ will generate an cm_event. 38 | ret = rdma_resolve_addr(cm_id_, nullptr, dst_addr_->ai_addr, DEFAULT_CONNECTION_TIMEOUT); // non-block 39 | checkEqual(ret, 0, "rdma_resolve_addr() failed"); 40 | rdma_cm_event* cm_event = waitEvent(RDMA_CM_EVENT_ADDR_RESOLVED); // block 41 | checkNotEqual(cm_event, static_cast(nullptr), "failed to resolve the address"); 42 | ret = rdma_ack_cm_event(cm_event); 43 | checkEqual(ret, 0, "rdma_ack_cm_event() failed to send ack"); 44 | info("address resolution is completed"); 45 | 46 | // same as above, resolve the route 47 | ret = rdma_resolve_route(cm_id_, DEFAULT_CONNECTION_TIMEOUT); 48 | checkEqual(ret, 0, "rdma_resolve_route failed"); 49 | cm_event = waitEvent(RDMA_CM_EVENT_ROUTE_RESOLVED); 50 | checkNotEqual(cm_event, static_cast(nullptr), "failed to resolve the route"); 51 | ret = rdma_ack_cm_event(cm_event); 52 | checkEqual(ret, 0, "rdma_ack_cm_event() failed to send ack"); 53 | info("route resolution is completed"); 54 | 55 | setupConnection(cm_id_, 64); 56 | 57 | // run poller 58 | poller_.run(); 59 | } 60 | 61 | rdma_cm_event* Client::waitEvent(rdma_cm_event_type expected) { 62 | rdma_cm_event* cm_event = nullptr; 63 | int ret = rdma_get_cm_event(cm_event_channel_, &cm_event); 64 | if (ret != 0) { 65 | info("fail to get cm event"); 66 | return nullptr; 67 | } 68 | if (cm_event->status != 0) { 69 | info("get a bad cm event"); 70 | return nullptr; 71 | } 72 | else if (cm_event->event != expected) { 73 | info("got: %s, expected: %s", rdma_event_str(cm_event->event), 74 | rdma_event_str(expected)); 75 | return nullptr; 76 | } 77 | else 78 | return cm_event; 79 | } 80 | 81 | void Client::setupConnection(rdma_cm_id* client_id, uint32_t n_buffer_page) { 82 | Connection* conn = new Connection(Role::ClientConn, client_id, n_buffer_page); 83 | rdma_conn_param param = conn->copyConnParam(); 84 | int ret = rdma_connect(client_id, ¶m); 85 | checkEqual(ret, 0, "rdma_connect() failed"); 86 | rdma_cm_event* ev = waitEvent(RDMA_CM_EVENT_ESTABLISHED); 87 | checkNotEqual(ev, static_cast(nullptr), "fail to establish the connection"); 88 | info("connection is established"); 89 | 90 | poller_.registerConn(conn); 91 | } 92 | 93 | void Client::sendRequest(std::string msg) { 94 | Message req((char*)msg.c_str(), msg.length(), MessageType::ImmRequest); 95 | poller_.sendRequest(req); 96 | } 97 | 98 | 99 | // /* ClientPoller */ 100 | ClientPoller::ClientPoller() { 101 | 102 | } 103 | 104 | ClientPoller::~ClientPoller() { 105 | 106 | } 107 | 108 | void ClientPoller::registerConn(Connection* conn) { 109 | std::lock_guard lock(lock_); 110 | conn_ = conn; 111 | } 112 | 113 | void ClientPoller::deregisterConn() { 114 | std::lock_guard lock(lock_); 115 | delete conn_; 116 | } 117 | 118 | void ClientPoller::sendRequest(Message req) { 119 | conn_->lock(); // unlock when receive response; 120 | conn_->fillMR((void*)&req, sizeof(req)); 121 | info("post send reqeust, req data is: %s", req.dataAddr()); 122 | conn_->postSend(conn_->getMRAddr(), sizeof(req), conn_->getLKey(), false); 123 | } 124 | 125 | 126 | void ClientPoller::run() { 127 | running_.store(true, std::memory_order_release); 128 | poll_thread_ = std::thread(&ClientPoller::poll, this); 129 | info("start running client poller"); 130 | } 131 | 132 | void ClientPoller::stop() { 133 | if (running_.load(std::memory_order_acquire)) { 134 | running_.store(false, std::memory_order_release); 135 | poll_thread_.join(); 136 | info("connection poller stopped"); 137 | } 138 | } 139 | 140 | void ClientPoller::poll() { 141 | while (running_.load(std::memory_order_acquire)) { 142 | std::lock_guard lock(lock_); 143 | conn_->poll(); 144 | } 145 | } -------------------------------------------------------------------------------- /src/connection.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | /* Connection */ 11 | Connection::Connection(Role role, rdma_cm_id* cm_id, uint32_t n_buffer_page) 12 | : role_(role), 13 | cm_id_(cm_id), 14 | n_buffer_page_(n_buffer_page) { 15 | 16 | info("start new connection"); 17 | 18 | // create pd 19 | int ret = 0; 20 | local_pd_ = ibv_alloc_pd(cm_id_->verbs); 21 | checkNotEqual(local_pd_, static_cast(nullptr), "ibv_alloc_pd() failed, server_pd_ == nullptr"); 22 | 23 | // create cq 24 | local_cq_ = ibv_create_cq(cm_id_->verbs, DEFAULT_CQ_CAPACITY, this, nullptr, 0); 25 | checkNotEqual(local_cq_, static_cast(nullptr), "ibv_create_cq() failed, server_cq_ == nullptr"); 26 | cm_id_->recv_cq = local_cq_; 27 | cm_id_->send_cq = local_cq_; 28 | info("create protection domain(pd) and completion queue(cq)"); 29 | 30 | ibv_qp_init_attr init_attr = defaultQpInitAttr(); 31 | init_attr.send_cq = local_cq_; 32 | init_attr.recv_cq = local_cq_; 33 | 34 | // create qp 35 | // rdma_create_qp() will create a qp and store it's address to cm_id->qp. 36 | // we should record the qp in Connection. 37 | ret = rdma_create_qp(cm_id_, local_pd_, &init_attr); 38 | local_qp_ = cm_id_->qp; 39 | checkEqual(ret, 0, "rdma_create_qp() failed"); 40 | info("create queue pair(qp)"); 41 | 42 | // create mr 43 | size_t size = n_buffer_page * BUFFER_PAGE_SIZE; 44 | buffer_ = malloc(size); 45 | checkNotEqual(buffer_, static_cast(nullptr), "malloc() failed to alloc buffer"); 46 | 47 | int access = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; 48 | buffer_mr_ = ibv_reg_mr(local_pd_, buffer_, size, access); 49 | checkNotEqual(buffer_mr_, static_cast(nullptr), "ibv_reg_mr() falied, buffer_mr_ == nullptr"); 50 | info("create memory region(mr), size is %zu", size); 51 | 52 | // set param 53 | memset(¶m_, 0, sizeof(rdma_conn_param)); 54 | param_.private_data = reinterpret_cast(&(buffer_mr_->rkey)); 55 | param_.private_data_len = sizeof(buffer_mr_->rkey); 56 | param_.responder_resources = 16; 57 | param_.initiator_depth = 16; 58 | param_.rnr_retry_count = 7; 59 | info("initialize connection parameters"); 60 | 61 | // prepare recv 62 | prepare(); 63 | info("connection prepares"); 64 | } 65 | 66 | Connection::~Connection() { 67 | 68 | int ret = 0; 69 | 70 | //clear the memory 71 | memset(buffer_mr_->addr, 0 ,buffer_mr_->length); 72 | 73 | ret = ibv_destroy_qp(local_qp_); 74 | wCheckEqual(ret, 0, "fail to destroy qp"); 75 | 76 | ret = rdma_destroy_id(cm_id_); 77 | wCheckEqual(ret, 0, "fail to destroy cm_id"); 78 | 79 | // warnning! you must ack all events before destroy cq, 80 | // otherwise, the func would never end. 81 | ret = ibv_destroy_cq(local_cq_); 82 | wCheckEqual(ret, 0, "fail to destroy cq"); 83 | 84 | ret = ibv_dereg_mr(buffer_mr_); 85 | wCheckEqual(ret, 0, "fail to deregister buffer memory region"); 86 | 87 | ret = ibv_dealloc_pd(local_pd_); 88 | wCheckEqual(ret, 0, "fail to deallocate pd"); 89 | 90 | free(buffer_); 91 | 92 | info("clean up connection resources"); 93 | } 94 | 95 | ibv_qp_init_attr Connection::defaultQpInitAttr() { 96 | ibv_qp_init_attr init_attr; 97 | init_attr.qp_context = nullptr; 98 | init_attr.send_cq = nullptr; 99 | init_attr.recv_cq = nullptr; 100 | init_attr.srq = nullptr; 101 | init_attr.cap = ibv_qp_cap { 102 | MAX_SEND_WR_NUM, // max_send_wr 103 | MAX_RECV_WR_NUM, // max_recv_wr 104 | 1, // max_send_sge 105 | 1, // max_recv_sge 106 | 0, // max_inline_data 107 | }; 108 | init_attr.qp_type = IBV_QPT_RC; 109 | init_attr.sq_sig_all = 0; 110 | return init_attr; 111 | } 112 | 113 | rdma_conn_param Connection::copyConnParam() { 114 | return param_; 115 | } 116 | 117 | void Connection::setRkey(uint32_t rkey) { 118 | rkey_ = rkey; 119 | } 120 | 121 | rdma_cm_id* Connection::getCmId() { 122 | return cm_id_; 123 | } 124 | 125 | uint32_t Connection::getLKey() { 126 | return buffer_mr_->lkey; 127 | } 128 | 129 | uint32_t Connection::getRKey() { 130 | return buffer_mr_->rkey; 131 | } 132 | 133 | void* Connection::getMRAddr() { 134 | return buffer_mr_->addr; 135 | } 136 | 137 | 138 | void Connection::fillMR(void* data, uint32_t size) { 139 | assert(size <= buffer_mr_->length); 140 | for (int i = 0; i < size; i++) 141 | { 142 | *((char*)(buffer_mr_->addr) + i) = *((char*)(data) + i); 143 | } 144 | } 145 | 146 | void Connection::prepare() { 147 | postRecv(buffer_mr_->addr, buffer_mr_->length, buffer_mr_->lkey); 148 | switch (role_) { 149 | case Role::ServerConn : { 150 | state_ = State::WaitingForRequest; 151 | break; 152 | } 153 | case Role::ClientConn : { 154 | state_ = State::WaitingForResponse; 155 | break; 156 | } 157 | default : { 158 | checkNotEqual(0, 0, "Connection has wrong role"); 159 | } 160 | } 161 | } 162 | 163 | void Connection::poll() { 164 | static ibv_wc wc[DEFAULT_CQ_CAPACITY]; 165 | int ret = ibv_poll_cq(local_cq_, DEFAULT_CQ_CAPACITY, wc); 166 | if (ret < 0) { 167 | info("poll cq error"); 168 | return; 169 | } else if (ret == 0) { 170 | //info("get no wc"); 171 | return; 172 | } 173 | 174 | for (int i = 0; i < ret; i++) { 175 | switch (role_){ 176 | case Role::ServerConn: { 177 | serverAdvance(wc[i]); 178 | break; 179 | } 180 | case Role::ClientConn: { 181 | clientAdvance(wc[i]); 182 | break; 183 | } 184 | } 185 | } 186 | } 187 | 188 | void Connection::serverAdvance(const ibv_wc &wc) { 189 | switch (wc.opcode) { 190 | case IBV_WC_RECV: { 191 | assert(state_ == WaitingForRequest); 192 | Context* ctx = reinterpret_cast(wc.wr_id); 193 | Message* req = reinterpret_cast(ctx->addr()); 194 | state_ = HandlingRequest; 195 | info("recive from client, start handling the request"); 196 | Message resp = handler_.handlerRequest(req); 197 | info("handle over"); 198 | // send resp 199 | fillMR((void*)&resp, sizeof(resp)); 200 | postSend(getMRAddr(), sizeof(resp), getLKey(), false); 201 | delete ctx; 202 | break; 203 | } 204 | case IBV_WC_SEND: { 205 | assert(state_ == HandlingRequest); 206 | prepare(); 207 | info("response send completed, waiting for next request"); 208 | state_ = WaitingForRequest; 209 | break; 210 | } 211 | case IBV_WC_RDMA_READ: { 212 | // todo 213 | break; 214 | } 215 | case IBV_WC_RDMA_WRITE: { 216 | // todo 217 | break; 218 | } 219 | default: { 220 | info("unexpected wc opcode: %d", wc.opcode); 221 | break; 222 | } 223 | } 224 | } 225 | 226 | 227 | void Connection::clientAdvance(const ibv_wc &wc) { 228 | switch (wc.opcode) { 229 | case IBV_WC_SEND: { 230 | Context* ctx = reinterpret_cast(wc.wr_id); 231 | delete ctx; 232 | state_ = WaitingForResponse; 233 | info("request send completed, waiting for response"); 234 | break; 235 | } 236 | case IBV_WC_RECV: { 237 | prepare(); 238 | assert(state_ == WaitingForResponse); 239 | Context* ctx = reinterpret_cast(wc.wr_id); 240 | Message* resp = reinterpret_cast(ctx->addr()); 241 | if(resp->msgType() == Response) { 242 | info("receive response from server, resp data is: %s", resp->dataAddr()); 243 | state_ = Vacant; 244 | unlock(); 245 | } 246 | break; 247 | } 248 | case IBV_WC_RDMA_READ: { 249 | break; 250 | } 251 | case IBV_WC_RDMA_WRITE: { 252 | break; 253 | } 254 | default: { 255 | info("unexpected wc opcode: %d", wc.opcode); 256 | break; 257 | } 258 | } 259 | } 260 | 261 | void Connection::postSend(void* local_addr, uint32_t length, uint32_t lkey, bool need_inline){ 262 | ibv_sge sge { 263 | (uint64_t) local_addr, // addr 264 | length, // length 265 | lkey, // lkey 266 | }; 267 | ibv_send_wr wr { 268 | (uint64_t)(new Context(local_addr, length)), // wr_id 269 | nullptr, // next 270 | &sge, // sg_list 271 | 1, // num_sge 272 | IBV_WR_SEND, // opcode 273 | IBV_SEND_SIGNALED, // send_flags 274 | {}, 275 | {}, 276 | {}, 277 | {}, 278 | }; 279 | 280 | if (need_inline) { 281 | wr.send_flags |= IBV_SEND_INLINE; 282 | } 283 | 284 | ibv_send_wr* bad_wr = nullptr; 285 | 286 | int ret = ibv_post_send(local_qp_, &wr, &bad_wr); 287 | checkEqual(ret, 0, "ibv_post_send() failed"); 288 | } 289 | 290 | 291 | void Connection::postRecv(void* local_addr, uint32_t length, uint32_t lkey) { 292 | ibv_sge sge { 293 | (uint64_t) local_addr, // addr 294 | length, // length 295 | lkey, // lkey 296 | }; 297 | ibv_recv_wr wr { 298 | (uint64_t)(new Context(local_addr, length)), // wr_id 299 | nullptr, // next 300 | &sge, // sg_list 301 | 1, // num_sge 302 | }; 303 | 304 | //std::cout << "postRecv local_addr: " << local_addr << std::endl; 305 | 306 | ibv_recv_wr* bad_wr = nullptr; 307 | int ret = ibv_post_recv(local_qp_, &wr, &bad_wr); 308 | checkEqual(ret, 0, "ibv_post_recv() failed"); 309 | } 310 | 311 | void Connection::lock() { 312 | lock_.lock(); 313 | } 314 | 315 | void Connection::unlock() { 316 | lock_.unlock(); 317 | } -------------------------------------------------------------------------------- /src/context.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | Context::Context(void* addr, uint32_t len) 4 | : addr_(addr), 5 | length_(len) { 6 | } 7 | 8 | Context::~Context() { 9 | 10 | } 11 | 12 | void* Context::addr(){ 13 | return addr_; 14 | } 15 | uint32_t Context::length() { 16 | return length_; 17 | } 18 | -------------------------------------------------------------------------------- /src/handler.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | Handler::Handler(){ 9 | 10 | } 11 | 12 | Handler::~Handler(){ 13 | 14 | } 15 | 16 | Message Handler::handlerRequest(Message* req) { 17 | assert(req->msgType() == ImmRequest || req->msgType() == Request); 18 | char data[req->dataLen()]; 19 | memcpy(data, req->dataAddr(), req->dataLen()); 20 | std::sort(data, data + sizeof(data)); 21 | Message resp(data, sizeof(data), Response); 22 | return resp; 23 | } -------------------------------------------------------------------------------- /src/message.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | Message::Message(char* buf, uint32_t len, MessageType type) { 5 | memcpy(meta_.buf_, buf, len); 6 | header_.data_len_ = len; 7 | header_.type_ = type; 8 | }; 9 | 10 | Message::~Message() { 11 | 12 | }; 13 | 14 | char* Message::dataAddr() { 15 | return meta_.buf_; 16 | } 17 | 18 | uint32_t Message::dataLen() { 19 | return header_.data_len_; 20 | } 21 | 22 | MessageType Message::msgType() { 23 | return header_.type_; 24 | } 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/server.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | Server::Server(const char* host, const char* port) { 8 | 9 | int ret = 0; 10 | ret = getaddrinfo(host, port, nullptr, &addr_); 11 | checkEqual(ret, 0, "getaddrinfo() failed"); 12 | 13 | // Open a channel used to report asynchronous communication event 14 | cm_event_channel_ = rdma_create_event_channel(); 15 | checkNotEqual(cm_event_channel_, static_cast(nullptr), "rdma_create_event_channel() failed, cm_event_channel_ == nullptr"); 16 | 17 | // rdma_cm_id is the connection identifier (like socket) which is used to define an RDMA connection. 18 | ret = rdma_create_id(cm_event_channel_, &listen_cm_id_, nullptr, RDMA_PS_TCP); 19 | checkEqual(ret, 0, "rdma_create_id() failed"); 20 | 21 | // Explicit binding of rdma cm id to the socket credentials 22 | ret = rdma_bind_addr(listen_cm_id_, addr_->ai_addr); 23 | checkEqual(ret, 0 ,"rdma_bind_addr() failed"); 24 | 25 | // Now we start to listen on the passed IP and port. However unlike 26 | // normal TCP listen, this is a non-blocking call. When a new client is 27 | // connected, a new connection management (CM) event is generated on the 28 | // RDMA CM event channel from where the listening id was created. Here we 29 | // have only one channel, so it is easy. 30 | ret = rdma_listen(listen_cm_id_, DEFAULT_BACK_LOG); // backlog is the max clients num, same as tcp, see 'man listen' 31 | checkEqual(ret, 0, "rdma_listen() failed"); 32 | info("start listening, address: %s:%s", host, port); 33 | 34 | // Now, register a conn_event 35 | // In order to avoid block 36 | base_ = event_base_new(); 37 | checkNotEqual(base_, static_cast(nullptr), "event_base_new() failed"); 38 | conn_event_ = event_new(base_, cm_event_channel_->fd, EV_READ | EV_PERSIST, &Server::onConnectionEvent, this); // bind the event to channel fd 39 | checkNotEqual(conn_event_, static_cast(nullptr), "event_new() failed to create conn_event"); 40 | ret = event_add(conn_event_, nullptr); // register the event 41 | checkEqual(ret, 0, "event_add() failed to register conn_event"); 42 | } 43 | 44 | Server::~Server() { 45 | event_base_free(base_); 46 | event_free(conn_event_); 47 | event_free(exit_event_); 48 | int ret = event_base_loopbreak(base_); 49 | rdma_destroy_event_channel(cm_event_channel_); 50 | freeaddrinfo(addr_); 51 | info("clean up the server resources"); 52 | } 53 | 54 | 55 | void Server::onConnectionEvent([[gnu::unused]]evutil_socket_t fd, [[gnu::unused]]short what, void* arg) { 56 | reinterpret_cast(arg) -> handleConnectionEvent(); 57 | } 58 | 59 | void Server::handleConnectionEvent() { 60 | rdma_cm_event* cm_ev; 61 | // wait the client connect, blocking 62 | // But! must get value immediately because conn_event, so it's like non-blocking 63 | int ret = rdma_get_cm_event(cm_event_channel_, &cm_ev); 64 | checkEqual(ret, 0, "rdma_get_cm_event() failed"); 65 | 66 | if (cm_ev->status != 0) { 67 | info("got a bad cm_event"); 68 | return; 69 | } 70 | 71 | switch(cm_ev->event) { 72 | case RDMA_CM_EVENT_CONNECT_REQUEST: { 73 | info("start to handle a connection request"); 74 | setupConnection(cm_ev, 64); 75 | break; 76 | } 77 | case RDMA_CM_EVENT_ESTABLISHED: { 78 | int ret = rdma_ack_cm_event(cm_ev); 79 | wCheckEqual(ret, 0, "rdma_ack_cm_event() failed to ack event"); 80 | info("establish the connection"); 81 | break; 82 | } 83 | case RDMA_CM_EVENT_DISCONNECTED: { 84 | Connection* conn = reinterpret_cast(cm_ev->id->context); 85 | int ret = rdma_ack_cm_event(cm_ev); 86 | info("send disconnect ack"); 87 | wCheckEqual(ret, 0, "rdma_ack_cm_event() failed to ack event"); 88 | poller_.deregisterConn(conn); 89 | info("delete the connection"); 90 | break; 91 | } 92 | default: { 93 | info("unexpected event: %s", rdma_event_str(cm_ev->event)); 94 | break; 95 | } 96 | } 97 | 98 | } 99 | 100 | 101 | void Server::run() { 102 | // run poller 103 | poller_.run(); 104 | // listen connection 105 | info("start event loop for conection"); 106 | info("=============================="); 107 | event_base_dispatch(base_); 108 | } 109 | 110 | void Server::setupConnection(rdma_cm_event* cm_event, uint32_t n_buffer_page) { 111 | 112 | rdma_cm_id* client_id = cm_event->id; 113 | Connection* conn = new Connection(Role::ServerConn, client_id, n_buffer_page); 114 | rdma_conn_param param = conn->copyConnParam(); 115 | int ret = rdma_accept(client_id, ¶m); 116 | checkEqual(ret, 0, "rdma_accept() failed"); 117 | uint32_t rkey = *(uint32_t*)(cm_event->param.conn.private_data); 118 | conn->setRkey(rkey); 119 | 120 | ret = rdma_ack_cm_event(cm_event); 121 | wCheckEqual(ret, 0, "rdma_ack_cm_event() failed to ack event"); 122 | info("accept the connection"); 123 | 124 | poller_.registerConn(conn); 125 | 126 | client_id->context = reinterpret_cast(conn); // in order to release when client disconnect 127 | } 128 | 129 | 130 | /* ServerPoller */ 131 | ServerPoller::ServerPoller() { 132 | 133 | } 134 | 135 | ServerPoller::~ServerPoller() { 136 | std::lock_guard lock(lock_); 137 | conn_list_.clear(); 138 | } 139 | 140 | void ServerPoller::registerConn(Connection* conn) { 141 | std::lock_guard lock(lock_); 142 | conn_list_.emplace_back(conn); 143 | } 144 | 145 | void ServerPoller::deregisterConn(Connection* conn) { 146 | std::lock_guard lock(lock_); 147 | for (auto it = conn_list_.begin(); it != conn_list_.end(); it++) { 148 | if ((*it) == conn) { 149 | conn_list_.erase(it); 150 | delete conn; 151 | break; 152 | } 153 | } 154 | } 155 | 156 | void ServerPoller::run() { 157 | running_.store(true, std::memory_order_release); 158 | poll_thread_ = std::thread(&ServerPoller::poll, this); 159 | info("start running server poller"); 160 | } 161 | 162 | void ServerPoller::stop() { 163 | if (running_.load(std::memory_order_acquire)) { 164 | running_.store(false, std::memory_order_release); 165 | poll_thread_.join(); 166 | info("connection poller stopped"); 167 | } 168 | } 169 | 170 | void ServerPoller::poll() { 171 | while (running_.load(std::memory_order_acquire)) { 172 | std::lock_guard lock(lock_); 173 | for (auto conn : conn_list_) { 174 | conn->poll(); 175 | } 176 | } 177 | } 178 | --------------------------------------------------------------------------------