├── .gitignore ├── LICENSE ├── README.md ├── rdma_write_client.c └── rdma_write_server.c /.gitignore: -------------------------------------------------------------------------------- 1 | **/.DS_store 2 | .vscode 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019, THE 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RDMA programming example 2 | 3 | [![BSD license](https://img.shields.io/badge/License-BSD-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) 4 | 5 | Send 2 numbers to server then send back the sum 6 | 7 | ## System required: 8 | 9 | RDMA capable NIC, OFED driver with libverbs and librdmacm. 10 | 11 | ## How to use: 12 | 13 | Git clone this repository at both client and server 14 | 15 | # git clone https://github.com/w180112/RDMA-example.git 16 | 17 | Set iptables to make UDP port 4791 open 18 | 19 | On client side, 20 | 21 | # gcc -o client rdma_write_client.c -lrdmacm -libverbs 22 | # ./client 23 | e.g. 24 | 25 | # ./client 192.168.0.168 123 456 26 | 27 | On server side, 28 | 29 | # gcc -o server rdma_write_server.c -lrdmacm -libverbs 30 | # ./server 31 | 32 | ## Test environment 33 | 34 | 1. Mellanox Connectx-4 Lx with SRIOV enable 35 | 2. Rocky Linux 9.2, with kernel 5.14.0-284.11.1.el9_2.x86_64 36 | -------------------------------------------------------------------------------- /rdma_write_client.c: -------------------------------------------------------------------------------- 1 | /* 2 | * modified from http://www.digitalvampire.org/rdma-tutorial-2007/notes.pdf 3 | * 4 | * build: 5 | * gcc -o client rdma_write_client.c -lrdmacm -libverbs 6 | * 7 | * usage: 8 | * ./client 9 | * 10 | * connect to server, send two integers, and waits for server to send back the sum. 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | enum { 27 | RESOLVE_TIMEOUT_MS = 5000, 28 | }; 29 | struct pdata { 30 | uint64_t buf_va; 31 | uint32_t buf_rkey; 32 | }; 33 | 34 | int prepare_send_notify_after_rdma_write(struct rdma_cm_id *cm_id, struct ibv_pd *pd) 35 | { 36 | struct ibv_sge sge; 37 | struct ibv_send_wr send_wr = { }; 38 | struct ibv_send_wr *bad_send_wr; 39 | 40 | uint8_t *buf = calloc(1, sizeof(uint8_t)); 41 | struct ibv_mr *mr = ibv_reg_mr(pd, buf, sizeof(uint8_t), IBV_ACCESS_LOCAL_WRITE); 42 | if (!mr) 43 | return 1; 44 | 45 | sge.addr = (uintptr_t)buf; 46 | sge.length = sizeof(uint8_t); 47 | sge.lkey = mr->lkey; 48 | 49 | memset(&send_wr, 0, sizeof(send_wr)); 50 | send_wr.wr_id = 2; 51 | send_wr.opcode = IBV_WR_SEND; 52 | send_wr.sg_list = &sge; 53 | send_wr.num_sge = 1; 54 | 55 | if (ibv_post_send(cm_id->qp,&send_wr,&bad_send_wr)) 56 | return 1; 57 | 58 | return 0; 59 | } 60 | 61 | int main(int argc, char *argv[]) 62 | { 63 | struct pdata server_pdata; 64 | 65 | struct rdma_event_channel *cm_channel; 66 | struct rdma_cm_id *cm_id; 67 | struct rdma_cm_event *event; 68 | struct rdma_conn_param conn_param = { }; 69 | 70 | struct ibv_pd *pd; 71 | struct ibv_comp_channel *comp_chan; 72 | struct ibv_cq *cq; 73 | struct ibv_cq *evt_cq; 74 | struct ibv_mr *mr; 75 | struct ibv_qp_init_attr qp_attr = { }; 76 | struct ibv_sge sge; 77 | struct ibv_send_wr send_wr = { }; 78 | struct ibv_send_wr *bad_send_wr; 79 | struct ibv_recv_wr recv_wr = { }; 80 | struct ibv_recv_wr *bad_recv_wr; 81 | struct ibv_wc wc; 82 | void *cq_context; 83 | struct addrinfo *res, *t; 84 | struct addrinfo hints = { 85 | .ai_family = AF_INET, 86 | .ai_socktype = SOCK_STREAM 87 | }; 88 | int n; 89 | uint32_t *buf; 90 | int err; 91 | 92 | /* We use rdmacm lib to establish rdma connection and ibv lib to write, read, send, receive data here. */ 93 | 94 | /* In RDMA programming, transmission is a "asychronize" procedure, all the "events" were generated on NIC. 95 | * Programmer should "get" those event and than ack and process them. 96 | */ 97 | 98 | /* In rdmacm lib, each event will generated by NIC and we should "get" these events from event channel, 99 | * so we should create an event channel first. 100 | */ 101 | cm_channel = rdma_create_event_channel(); 102 | if (!cm_channel) 103 | return 1; 104 | 105 | /* Like socket fd in socket porgramming, we need to acquire a rdmacm id. 106 | */ 107 | err = rdma_create_id(cm_channel, &cm_id, NULL, RDMA_PS_TCP); 108 | if (err) 109 | return err; 110 | 111 | /* Note: port 20000 doesn't equal to the socket port in TCP/IP, 112 | * in RoCEv2, all of the packets use port 4791, 113 | * port 20000 here indicates a higher level abstraction port 114 | */ 115 | n = getaddrinfo(argv[1], "20000", &hints, &res); 116 | if (n < 0) 117 | return 1; 118 | 119 | /* Resolve addr. */ 120 | err = rdma_resolve_addr(cm_id, NULL, res->ai_addr, RESOLVE_TIMEOUT_MS); 121 | if (err) 122 | return err; 123 | /* We need to "get" rdmacm event to acquire event occured on NIC. */ 124 | err = rdma_get_cm_event(cm_channel, &event); 125 | if (err) 126 | return err; 127 | if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) 128 | return 1; 129 | /* Each rdmacm event should be acked. */ 130 | rdma_ack_cm_event(event); 131 | 132 | err = rdma_resolve_route(cm_id, RESOLVE_TIMEOUT_MS); 133 | if (err) 134 | return err; 135 | err = rdma_get_cm_event(cm_channel, &event); 136 | if (err) 137 | return err; 138 | if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) 139 | return 1; 140 | rdma_ack_cm_event(event); 141 | 142 | /* Allocate protection domain, each pd can be used to create queue pair, 143 | * register memory regien, etc. 144 | * Each pd is a protection of a group of objects, 145 | * it means you can't use these objects between different pd. 146 | */ 147 | pd = ibv_alloc_pd(cm_id->verbs); 148 | if (!pd) 149 | return 1; 150 | /* A completion event channel like rdma_create_event_channel in libibverbs */ 151 | comp_chan = ibv_create_comp_channel(cm_id->verbs); 152 | if (!comp_chan) 153 | return 1; 154 | /* create a completion queue, a cq contains a completion work request. 155 | * All the events about NIC, transmission will be in the cq 156 | * Since libibverbs is thread-safe, use multiple cqs to 1 or many completion channels is avaliable. 157 | */ 158 | cq = ibv_create_cq(cm_id->verbs,2,NULL,comp_chan,0); 159 | if (!cq) 160 | return 1; 161 | /* Requests create compiletion notification when any work completion is add to the cq, 162 | * therefore work completion can be "get" by using ibv_get_cq_event() 163 | */ 164 | if (ibv_req_notify_cq(cq,0)) 165 | return 1; 166 | 167 | buf = calloc(2,sizeof(uint32_t)); 168 | if (!buf) 169 | return 1; 170 | /* register a memory region with a specific pd */ 171 | mr = ibv_reg_mr(pd, buf,2 * sizeof(uint32_t), IBV_ACCESS_LOCAL_WRITE); 172 | if (!mr) 173 | return 1; 174 | 175 | qp_attr.cap.max_send_wr = 4; 176 | qp_attr.cap.max_send_sge = 1; 177 | qp_attr.cap.max_recv_wr = 1; 178 | qp_attr.cap.max_recv_sge = 1; 179 | 180 | qp_attr.send_cq = cq; 181 | qp_attr.recv_cq = cq; 182 | qp_attr.qp_type = IBV_QPT_RC; 183 | /* create a queue pair, a qp is for post send/receive. 184 | * If pd is NULL, rdma_create_qp will use default pd on RDMA device 185 | */ 186 | err = rdma_create_qp(cm_id,pd,&qp_attr); 187 | if (err) 188 | return err; 189 | 190 | conn_param.initiator_depth = 1; 191 | conn_param.retry_count = 7; 192 | 193 | err = rdma_connect(cm_id,&conn_param); 194 | if (err) 195 | return err; 196 | 197 | err = rdma_get_cm_event(cm_channel,&event); 198 | if (err) 199 | return err; 200 | 201 | if (event->event != RDMA_CM_EVENT_ESTABLISHED) 202 | return 1; 203 | /* event->param.conn.private_data includes the memory info at server */ 204 | memcpy(&server_pdata,event->param.conn.private_data,sizeof(server_pdata)); 205 | rdma_ack_cm_event(event); 206 | 207 | /* We prepare ibv_post_recv() first */ 208 | 209 | sge.addr = (uintptr_t)buf; 210 | sge.length = sizeof(uint32_t); 211 | sge.lkey = mr->lkey; 212 | 213 | /* wr_id is used to identify the recv data when get ibv event */ 214 | recv_wr.wr_id = 0; 215 | recv_wr.sg_list = &sge; 216 | recv_wr.num_sge = 1; 217 | 218 | if (ibv_post_recv(cm_id->qp,&recv_wr,&bad_recv_wr)) 219 | return 1; 220 | 221 | buf[0] = strtoul(argv[2],NULL,0); 222 | buf[1] = strtoul(argv[3],NULL,0); 223 | buf[0] = htonl(buf[0]); 224 | buf[1] = htonl(buf[1]); 225 | 226 | sge.addr = (uintptr_t)buf; 227 | sge.length = sizeof(buf); 228 | sge.lkey = mr->lkey; 229 | 230 | send_wr.wr_id = 1; 231 | send_wr.opcode = IBV_WR_RDMA_WRITE; 232 | /* set IBV_SEND_SIGNALED flag will cause an ibv event recv at sender when data transmit from memory to NIC */ 233 | send_wr.send_flags = IBV_SEND_SIGNALED; 234 | send_wr.sg_list = &sge; 235 | send_wr.num_sge = 1; 236 | send_wr.wr.rdma.rkey = ntohl(server_pdata.buf_rkey); 237 | send_wr.wr.rdma.remote_addr = bswap_64(server_pdata.buf_va); 238 | 239 | if (ibv_post_send(cm_id->qp,&send_wr,&bad_send_wr)) 240 | return 1; 241 | 242 | int end_loop = 0; 243 | while (!end_loop) { 244 | if (ibv_get_cq_event(comp_chan,&evt_cq,&cq_context)) 245 | return 1; 246 | if (ibv_req_notify_cq(cq,0)) 247 | return 1; 248 | if (ibv_poll_cq(cq,1,&wc) != 1) 249 | return 1; 250 | if (wc.status != IBV_WC_SUCCESS) 251 | return 1; 252 | switch (wc.wr_id) { 253 | case 0: 254 | printf("server ans : %d\n", ntohl(buf[0])); 255 | end_loop = 1; 256 | break; 257 | case 1: 258 | /* due to server side doesn't know when the IBV_WR_RDMA_WRITE is done, 259 | * we need to send a notification to tell server side the IBV_WR_RDMA_WRITE is alreay sent 260 | */ 261 | if (prepare_send_notify_after_rdma_write(cm_id, pd)) 262 | return 1; 263 | break; 264 | default: 265 | end_loop = 1; 266 | break; 267 | } 268 | } 269 | ibv_ack_cq_events(cq,2); 270 | rdma_disconnect(cm_id); 271 | err = rdma_get_cm_event(cm_channel,&event); 272 | if (err) 273 | return err; 274 | 275 | rdma_ack_cm_event(event); 276 | rdma_destroy_qp(cm_id); 277 | ibv_dereg_mr(mr); 278 | free(buf); 279 | err = rdma_destroy_id(cm_id); 280 | if (err) { 281 | perror("rdma_destroy_id"); 282 | return err; 283 | } 284 | rdma_destroy_event_channel(cm_channel); 285 | return 0; 286 | } -------------------------------------------------------------------------------- /rdma_write_server.c: -------------------------------------------------------------------------------- 1 | /* 2 | * modified from http://www.digitalvampire.org/rdma-tutorial-2007/notes.pdf 3 | * 4 | * build: 5 | * gcc -o server rdma_write_server.c -lrdmacm -libverbs 6 | * 7 | * usage: 8 | * ./server 9 | * 10 | * waits for client to connect, receives two integers, and sends their sum back to the client. 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | 21 | enum { 22 | RESOLVE_TIMEOUT_MS = 5000, 23 | }; 24 | 25 | struct pdata { 26 | uint64_t buf_va; 27 | uint32_t buf_rkey; 28 | }; 29 | 30 | int prepare_recv_notify_before_using_rdma_write(struct rdma_cm_id *cm_id, struct ibv_pd *pd) 31 | { 32 | uint8_t *buf = calloc(1, sizeof(uint8_t)); 33 | struct ibv_mr *mr = ibv_reg_mr(pd, buf, sizeof(uint8_t), IBV_ACCESS_LOCAL_WRITE); 34 | if (!mr) 35 | return 1; 36 | struct ibv_sge notify_sge = { 37 | .addr = (uintptr_t)buf, 38 | .length = sizeof(uint32_t), 39 | .lkey = mr->lkey, 40 | }; 41 | struct ibv_recv_wr notify_wr = { 42 | .wr_id = 0, 43 | .sg_list = ¬ify_sge, 44 | .num_sge = 1, 45 | .next = NULL, 46 | }; 47 | struct ibv_recv_wr *bad_recv_wr; 48 | if (ibv_post_recv(cm_id->qp,¬ify_wr,&bad_recv_wr)) 49 | return 1; 50 | 51 | return 0; 52 | } 53 | 54 | int check_notify_before_using_rdma_write(struct ibv_comp_channel *comp_chan, struct ibv_cq *cq) 55 | { 56 | struct ibv_wc wc; 57 | struct ibv_cq *evt_cq; 58 | void *cq_context; 59 | 60 | if (ibv_get_cq_event(comp_chan,&evt_cq,&cq_context)) 61 | return 1; 62 | if (ibv_req_notify_cq(cq,0)) 63 | return 1; 64 | if (ibv_poll_cq(cq,1,&wc) != 1) 65 | return 1; 66 | if (wc.status != IBV_WC_SUCCESS) 67 | return 1; 68 | 69 | return 0; 70 | } 71 | 72 | int main(int argc, char *argv[]) 73 | { 74 | struct pdata rep_pdata; 75 | 76 | struct rdma_event_channel *cm_channel; 77 | struct rdma_cm_id *listen_id; 78 | struct rdma_cm_id *cm_id; 79 | struct rdma_cm_event *event; 80 | struct rdma_conn_param conn_param = { }; 81 | 82 | struct ibv_pd *pd; 83 | struct ibv_comp_channel *comp_chan; 84 | struct ibv_cq *cq; 85 | struct ibv_cq *evt_cq; 86 | struct ibv_mr *mr; 87 | struct ibv_qp_init_attr qp_attr = { }; 88 | struct ibv_sge sge; 89 | struct ibv_send_wr send_wr = { }; 90 | struct ibv_send_wr *bad_send_wr; 91 | struct ibv_wc wc; 92 | void *cq_context; 93 | struct sockaddr_in sin; 94 | uint32_t *buf; 95 | int err; 96 | 97 | /* We use rdmacm lib to establish rdma connection and ibv lib to write, read, send, receive data here. */ 98 | 99 | /* In RDMA programming, transmission is an "asychronize" procedure, all the "events" were generated on NIC. 100 | * Programmer should "get" those event and than ack and process them. 101 | */ 102 | 103 | /* In rdmacm lib, each event will generated by NIC and we should "get" these events from event channel, 104 | * so we should create an event channel first. 105 | */ 106 | cm_channel = rdma_create_event_channel(); 107 | if (!cm_channel) 108 | return 1; 109 | 110 | /* Like socket fd in socket porgramming, we need to acquire a rdmacm id. 111 | */ 112 | err = rdma_create_id(cm_channel,&listen_id,NULL,RDMA_PS_TCP); 113 | if (err) 114 | return err; 115 | 116 | /* Note: port 20000 doesn't equal to the socket port in TCP/IP, 117 | * in RoCEv2, all of the packets use port 4791, 118 | * port 20000 here indicates a higher level abstraction port 119 | */ 120 | sin.sin_family = AF_INET; 121 | sin.sin_port = htons(20000); 122 | sin.sin_addr.s_addr = INADDR_ANY; 123 | 124 | 125 | /* Bind to local port and listen for connection request */ 126 | /* Binding an indicated addr means rdmacm id will be bound to that RDMA device, 127 | * we bind wildcard addr here to make rdma_listen listen to all RDMA device 128 | */ 129 | err = rdma_bind_addr(listen_id,(struct sockaddr *)&sin); 130 | if (err) 131 | return 1; 132 | err = rdma_listen(listen_id,1); 133 | if (err) 134 | return 1; 135 | 136 | while(1) { 137 | /* We need to "get" rdmacm event to acquire event occured on NIC. */ 138 | err = rdma_get_cm_event(cm_channel,&event); 139 | if (err) 140 | return err; 141 | if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) 142 | return 1; 143 | cm_id = event->id; 144 | /* Each rdmacm event should be acked. */ 145 | rdma_ack_cm_event(event); 146 | 147 | /* Create verbs objects now that we know which device to use */ 148 | 149 | /* Allocate protection domain, each pd can be used to create queue pair, 150 | * register memory regien, etc. 151 | * Each pd is a protection of a group of objects, 152 | * it means you can't use these objects between different pd. 153 | */ 154 | pd = ibv_alloc_pd(cm_id->verbs); 155 | if (!pd) 156 | return 1; 157 | 158 | /* A completion event channel like rdma_create_event_channel in libibverbs */ 159 | comp_chan = ibv_create_comp_channel(cm_id->verbs); 160 | if (!comp_chan) 161 | return 1; 162 | 163 | /* create a completion queue, a cq contains a completion work request. 164 | * All the events about NIC, transmission will be in the cq 165 | * Since libibverbs is thread-safe, use multiple cqs to 1 or many completion channels is avaliable. 166 | */ 167 | cq = ibv_create_cq(cm_id->verbs,1,NULL,comp_chan,0); 168 | if (!cq) 169 | return 1; 170 | 171 | /* Requests create compiletion notification when any work completion is add to the cq, 172 | * therefore work completion can be "get" by using ibv_get_cq_event() 173 | */ 174 | if (ibv_req_notify_cq(cq,0)) 175 | return 1; 176 | 177 | buf = calloc(2,sizeof(uint32_t)); 178 | if (!buf) 179 | return 1; 180 | 181 | /* register a memory region with a specific pd */ 182 | mr = ibv_reg_mr(pd,buf,2*sizeof(uint32_t), 183 | IBV_ACCESS_LOCAL_WRITE | 184 | IBV_ACCESS_REMOTE_READ | 185 | IBV_ACCESS_REMOTE_WRITE); 186 | if (!mr) 187 | return 1; 188 | 189 | memset(&qp_attr,0,sizeof(qp_attr)); 190 | qp_attr.cap.max_send_wr = 1; 191 | qp_attr.cap.max_send_sge = 1; 192 | qp_attr.cap.max_recv_wr = 1; 193 | qp_attr.cap.max_recv_sge = 1; 194 | 195 | qp_attr.send_cq = cq; 196 | qp_attr.recv_cq = cq; 197 | 198 | qp_attr.qp_type = IBV_QPT_RC; 199 | /* create a queue pair, a qp is for post send/receive. 200 | * If pd is NULL, rdma_create_qp will use default pd on RDMA device 201 | */ 202 | err = rdma_create_qp(cm_id,pd,&qp_attr); 203 | if (err) { 204 | perror("rdma cm create qp error"); 205 | return err; 206 | } 207 | rep_pdata.buf_va = bswap_64((uintptr_t)buf); 208 | /* we need to prepare remote key to give to client */ 209 | rep_pdata.buf_rkey = htonl(mr->rkey); 210 | conn_param.responder_resources = 1; 211 | conn_param.private_data = &rep_pdata; 212 | conn_param.private_data_len = sizeof(rep_pdata); 213 | 214 | /* Accept connection, at this point, server will send the mr info and buffer addr to client to let client directly write data to it, 215 | * our example here is rep_pdata and conn_param 216 | */ 217 | err = rdma_accept(cm_id,&conn_param); 218 | if (err) 219 | return 1; 220 | err = rdma_get_cm_event(cm_channel,&event); 221 | if (err) 222 | return err; 223 | if (event->event != RDMA_CM_EVENT_ESTABLISHED) 224 | return 1; 225 | rdma_ack_cm_event(event); 226 | 227 | /* we need to check IBV_WR_RDMA_WRITE is done, so we post_recv at first */ 228 | if (prepare_recv_notify_before_using_rdma_write(cm_id, pd)) 229 | return 1; 230 | /* we need to check we already receive RDMA_WRITE done notification */ 231 | if (check_notify_before_using_rdma_write(comp_chan, cq)) 232 | return 1; 233 | 234 | /* Add two integers and send reply back */ 235 | printf("client write %d and %d\n", ntohl(buf[0]), ntohl(buf[1])); 236 | buf[0] = htonl(ntohl(buf[0]) + ntohl(buf[1])); 237 | 238 | /* register post send, here we use IBV_WR_SEND */ 239 | sge.addr = (uintptr_t)buf; 240 | sge.length = sizeof(uint32_t); 241 | sge.lkey = mr->lkey; 242 | 243 | send_wr.opcode = IBV_WR_SEND; 244 | send_wr.send_flags = IBV_SEND_SIGNALED; 245 | send_wr.sg_list = &sge; 246 | send_wr.num_sge = 1; 247 | 248 | if (ibv_post_send(cm_id->qp,&send_wr,&bad_send_wr)) 249 | return 1; 250 | 251 | /* Like rdmacm, ibv events should be "get" than "ack". 252 | * Note that the "ack" procedure use mutex to make sure "ack" successfully, 253 | * so "ack" many events simultaneously can enhance performence 254 | */ 255 | if (ibv_get_cq_event(comp_chan,&evt_cq,&cq_context)) 256 | return 1; 257 | ibv_ack_cq_events(cq,1); 258 | if (ibv_req_notify_cq(cq,0)) 259 | return 1; 260 | if (ibv_poll_cq(cq,1,&wc) != 1) 261 | return 1; 262 | if (wc.status != IBV_WC_SUCCESS) 263 | return 1; 264 | 265 | err = rdma_get_cm_event(cm_channel,&event); 266 | if (err) 267 | return err; 268 | rdma_ack_cm_event(event); 269 | 270 | if (event->event == RDMA_CM_EVENT_DISCONNECTED) { 271 | rdma_destroy_qp(cm_id); 272 | ibv_dereg_mr(mr); 273 | free(buf); 274 | err = rdma_destroy_id(cm_id); 275 | if (err != 0) 276 | perror("destroy cm id fail."); 277 | } 278 | } 279 | rdma_destroy_event_channel(cm_channel); 280 | return 0; 281 | } 282 | --------------------------------------------------------------------------------