├── README.md ├── makefile ├── .gitignore ├── client.cpp ├── server.cpp ├── LICENSE └── rdma_socket.hpp /README.md: -------------------------------------------------------------------------------- 1 | # rdma 2 | C++ RDMA Wrappers 3 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | CXX=g++ -std=c++11 2 | LIBS=-lrdmacm -libverbs 3 | 4 | all: client server 5 | 6 | client: client.cpp rdma_socket.hpp 7 | $(CXX) -o client client.cpp $(LIBS) 8 | 9 | server: server.cpp rdma_socket.hpp 10 | $(CXX) -o server server.cpp $(LIBS) 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | -------------------------------------------------------------------------------- /client.cpp: -------------------------------------------------------------------------------- 1 | #include "rdma_socket.hpp" 2 | 3 | int main(int argc, char* argv[]) { 4 | rdma_socket client(RDMA_PS_TCP); 5 | 6 | client.resolve("192.168.3.3", 8000); 7 | client.create_qp(); 8 | 9 | char* mem = new char[4096]; 10 | client.post_recv(mem, 4096); 11 | 12 | client.connect(); 13 | 14 | struct ibv_wc wc = client.recv_comp(); 15 | printf("%s\n", reinterpret_cast(wc.wr_id)); 16 | return 0; 17 | } 18 | -------------------------------------------------------------------------------- /server.cpp: -------------------------------------------------------------------------------- 1 | #include "rdma_socket.hpp" 2 | 3 | int main(int argc, char* argv[]) { 4 | rdma_socket server(RDMA_PS_TCP); 5 | 6 | server.bind("192.168.3.3", 8000); 7 | server.listen(); 8 | rdma_socket rclient = server.accept(); 9 | 10 | char* data = new char[4096]; 11 | std::memcpy(data, "Hello World!", 13); 12 | 13 | rclient.post_send(data, 4096); 14 | 15 | struct ibv_wc wc = rclient.send_comp(); 16 | return 0; 17 | } 18 | 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Ian Forbes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /rdma_socket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | struct rdma_error : public std::exception { 22 | const char* _what; 23 | 24 | rdma_error(const char* what) : _what(what) {} 25 | 26 | virtual const char* what() const noexcept override { 27 | return _what; 28 | } 29 | }; 30 | 31 | void print_id(struct rdma_cm_id* id) { 32 | printf("%p\n", id->verbs); 33 | } 34 | 35 | class rdma_socket { 36 | struct rdma_cm_id* _id; 37 | struct rdma_event_channel* _events; 38 | struct ibv_pd* _protect; 39 | 40 | rdma_socket() 41 | : _id(nullptr), _events(rdma_create_event_channel()), _protect(nullptr) {} 42 | 43 | public: 44 | struct ibv_qp_init_attr _qp_attr; 45 | 46 | public: 47 | rdma_socket(enum rdma_port_space type) { 48 | _events = rdma_create_event_channel(); 49 | int rcode = rdma_create_id(_events, &_id, nullptr, type); 50 | assert(!rcode); 51 | } 52 | 53 | // non-copyable 54 | rdma_socket(const rdma_socket&) = delete; 55 | rdma_socket& operator=(const rdma_socket&) = delete; 56 | 57 | // moveable 58 | rdma_socket(rdma_socket&& s) 59 | : _id(s._id), _events(s._events), _protect(s._protect) 60 | { 61 | s._id = nullptr; 62 | s._events = nullptr; 63 | s._protect = nullptr; 64 | } 65 | 66 | // dtor 67 | ~rdma_socket() { 68 | if(_id) 69 | rdma_destroy_id(_id); 70 | if(_events) 71 | rdma_destroy_event_channel(_events); 72 | if(_protect) 73 | ibv_dealloc_pd(_protect); 74 | } 75 | 76 | 77 | void bind() { 78 | this->bind("0.0.0.0", 0); 79 | } 80 | 81 | void bind(const char* address, short port) { 82 | struct sockaddr_in addrin; 83 | addrin.sin_family = AF_INET; 84 | addrin.sin_port = htons(port); 85 | addrin.sin_addr.s_addr = inet_addr(address); 86 | bzero(&addrin.sin_zero, 8); 87 | this->bind((struct sockaddr*) &addrin); 88 | } 89 | 90 | void bind(struct sockaddr* address) { 91 | int rcode = rdma_bind_addr(_id, address); 92 | assert(!rcode); 93 | } 94 | 95 | void listen(int backlog = 16) { 96 | int rcode = rdma_listen(_id, backlog); 97 | assert(!rcode); 98 | } 99 | 100 | rdma_socket accept() { 101 | struct rdma_cm_event* ev; 102 | int rcode = rdma_get_cm_event(_events, &ev); 103 | if(ev->event == RDMA_CM_EVENT_CONNECT_REQUEST) { 104 | // GOOD 105 | } 106 | else { 107 | // BAD 108 | } 109 | rdma_socket new_sock; 110 | new_sock._id = ev->id; 111 | new_sock.create_qp(); 112 | 113 | struct rdma_conn_param params; 114 | bzero(¶ms, sizeof(params)); 115 | rcode = rdma_accept(new_sock._id, ¶ms); 116 | rcode = rdma_ack_cm_event(ev); // ack conn request 117 | 118 | rcode = rdma_get_cm_event(_events, &ev); 119 | rcode = rdma_ack_cm_event(ev); // ack 120 | 121 | return new_sock; 122 | } 123 | 124 | void resolve(const char* address, short port) { 125 | struct sockaddr_in addrin; 126 | addrin.sin_family = AF_INET; 127 | addrin.sin_port = htons(port); 128 | addrin.sin_addr.s_addr = inet_addr(address); 129 | bzero(&addrin.sin_zero, 8); 130 | this->resolve((struct sockaddr*) &addrin); 131 | } 132 | 133 | void resolve(struct sockaddr* address) { 134 | const int timeout = 100; 135 | int rcode = rdma_resolve_addr(_id, nullptr, address, timeout); 136 | struct rdma_cm_event* ev; 137 | rcode = rdma_get_cm_event(_events, &ev); 138 | if(ev->event == RDMA_CM_EVENT_ADDR_RESOLVED) { 139 | // GOOD 140 | } 141 | else if(ev->event == RDMA_CM_EVENT_ADDR_ERROR) { 142 | // BAD 143 | } 144 | else { 145 | throw rdma_error("Unexpected"); 146 | } 147 | rcode = rdma_ack_cm_event(ev); 148 | } 149 | 150 | void connect() { 151 | int rcode; 152 | const int timeout = 100; 153 | rcode = rdma_resolve_route(_id, timeout); 154 | 155 | struct rdma_cm_event* ev; 156 | rcode = rdma_get_cm_event(_events, &ev); 157 | rcode = rdma_ack_cm_event(ev); 158 | struct rdma_conn_param params; 159 | bzero(¶ms, sizeof(params)); 160 | params.private_data = nullptr; 161 | params.private_data = 0; 162 | params.responder_resources = 8; 163 | params.initiator_depth = 8; 164 | rcode = rdma_connect(_id, ¶ms); 165 | rcode = rdma_get_cm_event(_events, &ev); 166 | rcode = rdma_ack_cm_event(ev); 167 | assert(!rcode); 168 | } 169 | 170 | void disconnect() { 171 | int rcode = rdma_disconnect(_id); 172 | assert(!rcode); 173 | } 174 | 175 | void create_qp() { 176 | bzero(&_qp_attr, sizeof(_qp_attr)); 177 | _qp_attr.recv_cq = nullptr; 178 | _qp_attr.send_cq = nullptr; 179 | _qp_attr.srq = nullptr; 180 | _qp_attr.qp_context = NULL; 181 | _qp_attr.qp_type = IBV_QPT_RC; 182 | _qp_attr.cap.max_send_wr = 8; 183 | _qp_attr.cap.max_recv_wr = 8; 184 | _qp_attr.cap.max_send_sge = 8; 185 | _qp_attr.cap.max_recv_sge = 8; 186 | _qp_attr.cap.max_inline_data = 8; 187 | _protect = ibv_alloc_pd(_id->verbs); 188 | int rcode = rdma_create_qp(_id, _protect, &_qp_attr); 189 | assert(!rcode); 190 | } 191 | 192 | sockaddr get_peer_addr() { 193 | return *rdma_get_peer_addr(_id); 194 | } 195 | 196 | sockaddr get_local_addr() { 197 | return *rdma_get_local_addr(_id); 198 | } 199 | 200 | void post_recv(void * addr, size_t length, struct ibv_mr* mr = nullptr, void * user_data = nullptr) { 201 | if(mr == nullptr) 202 | mr = ibv_reg_mr(_protect, addr, length, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE); 203 | if(user_data == nullptr) 204 | user_data = addr; 205 | int rcode = rdma_post_recv(_id, user_data, addr, length, mr); 206 | assert(!rcode); 207 | } 208 | 209 | void post_send(void * addr, size_t length, struct ibv_mr* mr = nullptr, void * user_data = nullptr) { 210 | if(mr == nullptr) 211 | mr = ibv_reg_mr(_protect, addr, length, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE); 212 | if(user_data == nullptr) 213 | user_data = addr; 214 | int rcode = rdma_post_send(_id, user_data, addr, length, mr, 0); 215 | assert(!rcode); 216 | } 217 | 218 | struct ibv_wc recv_comp() { 219 | struct ibv_wc wc; 220 | int rcode = rdma_get_recv_comp(_id, &wc); 221 | return wc; 222 | } 223 | 224 | struct ibv_wc send_comp() { 225 | struct ibv_wc wc; 226 | int rcode = rdma_get_send_comp(_id, &wc); 227 | return wc; 228 | } 229 | }; 230 | --------------------------------------------------------------------------------