├── README.md ├── _config.yml ├── client.c └── server.c /README.md: -------------------------------------------------------------------------------- 1 | # RDMA-example-application 2 | Writing RDMA applications on Linux Example programs by Roland Dreier. 3 | 4 | source: http://www.digitalvampire.org/rdma-tutorial-2007/notes.pdf 5 | 6 | # use 7 | 8 | please make sure you have rdma and libibverbs library. 9 | 10 | ## compiler 11 | client : 12 | 13 | $ cc -o client client.c -lrdmacm -libverbs 14 | 15 | server : 16 | 17 | $ cc -o server server.c -lrdmacm -libverbs 18 | 19 | | @shrebhan thanks! 20 | 21 | ## run 22 | server : 23 | 24 | $ ./server 25 | 26 | client : ( syntax: client [servername] [val1] [val2] ) 27 | 28 | $./client 192.168.1.2 123 567 29 | 30 | 123 + 567 = 690 31 | 32 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /client.c: -------------------------------------------------------------------------------- 1 | /* 2 | * http://www.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.rdma/client_active_example.htm?lang=zh-tw 3 | * 建置： 4 | * cc -o client client.c -lrdmacm -libverbs 5 | * 6 | * 用法： 7 | * client 8 | * 9 | * 連接至伺服器，透過「RDMA 寫入」傳送 val1 並透過「RDMA 傳送」傳送 val2， 10 | * 然後從伺服器接收回 val1+val2。 11 | */ 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | enum { 24 | RESOLVE_TIMEOUT_MS = 5000, 25 | }; 26 | struct pdata { 27 | uint64_t buf_va; 28 | uint32_t buf_rkey; 29 | }; 30 | 31 | int main(int argc, char *argv[ ]) 32 | { 33 | struct pdata server_pdata; 34 | 35 | struct rdma_event_channel *cm_channel; 36 | struct rdma_cm_id *cm_id; 37 | struct rdma_cm_event *event; 38 | struct rdma_conn_param conn_param = { }; 39 | 40 | struct ibv_pd *pd; 41 | struct ibv_comp_channel *comp_chan; 42 | struct ibv_cq *cq; 43 | struct ibv_cq *evt_cq; 44 | struct ibv_mr *mr; 45 | struct ibv_qp_init_attr qp_attr = { }; 46 | struct ibv_sge sge; 47 | struct ibv_send_wr send_wr = { }; 48 | struct ibv_send_wr *bad_send_wr; 49 | struct ibv_recv_wr recv_wr = { }; 50 | struct ibv_recv_wr *bad_recv_wr; 51 | struct ibv_wc wc; 52 | void *cq_context; 53 | struct addrinfo *res, *t; 54 | struct addrinfo hints = { 55 | .ai_family = AF_INET, 56 | .ai_socktype = SOCK_STREAM 57 | }; 58 | int n; 59 | uint32_t *buf; 60 | int err; 61 | 62 | /* 設定 RDMA CM 結構 */ 63 | cm_channel = rdma_create_event_channel(); 64 | if (!cm_channel) 65 | return 1; 66 | 67 | err = rdma_create_id(cm_channel, &cm_id, NULL, RDMA_PS_TCP); 68 | if (err) 69 | return err; 70 | 71 | n = getaddrinfo(argv[1], "20079", &hints, &res); 72 | if (n < 0) 73 | return 1; 74 | 75 | /* 解析伺服器位址及路徑 */ 76 | 77 | for (t = res; t; t = t->ai_next) { 78 | err = rdma_resolve_addr(cm_id, NULL, t->ai_addr, RESOLVE_TIMEOUT_MS); 79 | if (!err) 80 | break; 81 | } 82 | if (err) 83 | return err; 84 | 85 | err = rdma_get_cm_event(cm_channel, &event); 86 | if (err) 87 | return err; 88 | 89 | if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) 90 | return 1; 91 | 92 | rdma_ack_cm_event(event); 93 | 94 | err = rdma_resolve_route(cm_id, RESOLVE_TIMEOUT_MS); 95 | if (err) 96 | return err; 97 | 98 | err = rdma_get_cm_event(cm_channel, &event); 99 | if (err) 100 | return err; 101 | 102 | if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) 103 | return 1; 104 | 105 | rdma_ack_cm_event(event); 106 | 107 | /* 建立動詞物件，因為我們知道要使用哪一個裝置 */ 108 | 109 | pd = ibv_alloc_pd(cm_id->verbs); 110 | if (!pd) 111 | return 1; 112 | 113 | comp_chan = ibv_create_comp_channel(cm_id->verbs); 114 | if (!comp_chan) 115 | return 1; 116 | 117 | cq = ibv_create_cq(cm_id->verbs, 2,NULL, comp_chan, 0); 118 | if (!cq) 119 | return 1; 120 | 121 | if (ibv_req_notify_cq(cq, 0)) 122 | return 1; 123 | 124 | buf = calloc(2, sizeof (uint32_t)); 125 | if (!buf) 126 | return 1; 127 | 128 | mr = ibv_reg_mr(pd, buf,2 * sizeof(uint32_t), IBV_ACCESS_LOCAL_WRITE); 129 | if (!mr) 130 | return 1; 131 | 132 | qp_attr.cap.max_send_wr = 2; 133 | qp_attr.cap.max_send_sge = 1; 134 | qp_attr.cap.max_recv_wr = 1; 135 | qp_attr.cap.max_recv_sge = 1; 136 | 137 | qp_attr.send_cq = cq; 138 | qp_attr.recv_cq = cq; 139 | qp_attr.qp_type = IBV_QPT_RC; 140 | 141 | err = rdma_create_qp(cm_id, pd, &qp_attr); 142 | if (err) 143 | return err; 144 | 145 | conn_param.initiator_depth = 1; 146 | conn_param.retry_count = 7; 147 | 148 | /* 連接至伺服器 */ 149 | 150 | err = rdma_connect(cm_id, &conn_param); 151 | if (err) 152 | return err; 153 | 154 | err = rdma_get_cm_event(cm_channel,&event); 155 | if (err) 156 | return err; 157 | 158 | if (event->event != RDMA_CM_EVENT_ESTABLISHED) 159 | return 1; 160 | 161 | memcpy(&server_pdata, event->param.conn.private_data, sizeof server_pdata); 162 | rdma_ack_cm_event(event); 163 | 164 | /* Prepost 接收 */ 165 | 166 | sge.addr = (uintptr_t) buf; 167 | sge.length = sizeof (uint32_t); 168 | sge.lkey = mr->lkey; 169 | 170 | recv_wr.wr_id = 0; 171 | recv_wr.sg_list = &sge; 172 | recv_wr.num_sge = 1; 173 | 174 | if (ibv_post_recv(cm_id->qp, &recv_wr, &bad_recv_wr)) 175 | return 1; 176 | 177 | /* 寫入/傳送要新增的兩個整數 */ 178 | 179 | buf[0] = strtoul(argv[2], NULL, 0); 180 | buf[1] = strtoul(argv[3], NULL, 0); 181 | printf("%d + %d = ", buf[0], buf[1]); 182 | buf[0] = htonl(buf[0]); 183 | buf[1] = htonl(buf[1]); 184 | 185 | sge.addr = (uintptr_t) buf; 186 | sge.length = sizeof (uint32_t); 187 | sge.lkey = mr->lkey; 188 | 189 | send_wr.wr_id = 1; 190 | send_wr.opcode = IBV_WR_RDMA_WRITE; 191 | send_wr.sg_list = &sge; 192 | send_wr.num_sge = 1; 193 | send_wr.wr.rdma.rkey = ntohl(server_pdata.buf_rkey); 194 | send_wr.wr.rdma.remote_addr = ntohl(server_pdata.buf_va); 195 | 196 | if (ibv_post_send(cm_id->qp, &send_wr, &bad_send_wr)) 197 | return 1; 198 | 199 | sge.addr = (uintptr_t) buf + sizeof (uint32_t); 200 | sge.length = sizeof (uint32_t); 201 | sge.lkey = mr->lkey; 202 | send_wr.wr_id = 2; 203 | send_wr.opcode = IBV_WR_SEND; 204 | send_wr.send_flags = IBV_SEND_SIGNALED; 205 | send_wr.sg_list =&sge; 206 | send_wr.num_sge = 1; 207 | 208 | if (ibv_post_send(cm_id->qp, &send_wr,&bad_send_wr)) 209 | return 1; 210 | 211 | /* 等待接收完成 */ 212 | 213 | while (1) { 214 | if (ibv_get_cq_event(comp_chan,&evt_cq, &cq_context)) 215 | return 1; 216 | 217 | if (ibv_req_notify_cq(cq, 0)) 218 | return 1; 219 | 220 | if (ibv_poll_cq(cq, 1, &wc) != 1) 221 | return 1; 222 | 223 | if (wc.status != IBV_WC_SUCCESS) 224 | return 1; 225 | 226 | if (wc.wr_id == 0) { 227 | printf("%d\n", ntohl(buf[0])); 228 | return 0; 229 | } 230 | } 231 | return 0; 232 | } 233 | -------------------------------------------------------------------------------- /server.c: -------------------------------------------------------------------------------- 1 | /* 2 | * build: 3 | * cc -o server server.c -lrdmacm 4 | * 5 | * usage: 6 | * server 7 | * 8 | * waits for client to connect, receives two integers, and sends their 9 | * sum back to the client. 10 | * lib:libibverbs-dev 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | enum { 21 | RESOLVE_TIMEOUT_MS = 5000, 22 | }; 23 | 24 | struct pdata { 25 | uint64_t buf_va; 26 | uint32_t buf_rkey; 27 | }; 28 | 29 | int main(int argc, char *argv[]) 30 | { 31 | struct pdata rep_pdata; 32 | 33 | struct rdma_event_channel *cm_channel; 34 | struct rdma_cm_id *listen_id; 35 | struct rdma_cm_id *cm_id; 36 | struct rdma_cm_event *event; 37 | struct rdma_conn_param conn_param = { }; 38 | 39 | struct ibv_pd *pd; 40 | struct ibv_comp_channel *comp_chan; 41 | struct ibv_cq *cq; 42 | struct ibv_cq *evt_cq; 43 | struct ibv_mr *mr; 44 | struct ibv_qp_init_attr qp_attr = { }; 45 | struct ibv_sge sge; 46 | struct ibv_send_wr send_wr = { }; 47 | struct ibv_send_wr *bad_send_wr; 48 | struct ibv_recv_wr recv_wr = { }; 49 | struct ibv_recv_wr *bad_recv_wr; 50 | struct ibv_wc wc; 51 | void *cq_context; 52 | 53 | struct sockaddr_in sin; 54 | 55 | uint32_t *buf; 56 | 57 | int err; 58 | 59 | /* Set up RDMA CM structures */ 60 | 61 | cm_channel = rdma_create_event_channel(); 62 | if (!cm_channel) 63 | return 1; 64 | 65 | err = rdma_create_id(cm_channel, &listen_id, NULL, RDMA_PS_TCP); 66 | if (err) 67 | return err; 68 | 69 | sin.sin_family = AF_INET; 70 | sin.sin_port = htons(20079); 71 | sin.sin_addr.s_addr = INADDR_ANY; 72 | 73 | 74 | /* Bind to local port and listen for connection request */ 75 | 76 | err = rdma_bind_addr(listen_id, (struct sockaddr *) &sin); 77 | if (err) 78 | return 1; 79 | 80 | err = rdma_listen(listen_id, 1); 81 | if (err) 82 | return 1; 83 | 84 | err = rdma_get_cm_event(cm_channel, &event); 85 | if (err) 86 | return err; 87 | 88 | if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) 89 | return 1; 90 | 91 | cm_id = event->id; 92 | 93 | rdma_ack_cm_event(event); 94 | 95 | /* Create verbs objects now that we know which device to use */ 96 | 97 | pd = ibv_alloc_pd(cm_id->verbs); 98 | if (!pd) 99 | return 1; 100 | 101 | comp_chan = ibv_create_comp_channel(cm_id->verbs); 102 | if (!comp_chan) 103 | return 1; 104 | 105 | cq = ibv_create_cq(cm_id->verbs, 2, NULL, comp_chan, 0); 106 | if (!cq) 107 | return 1; 108 | 109 | if (ibv_req_notify_cq(cq, 0)) 110 | return 1; 111 | 112 | buf = calloc(2, sizeof (uint32_t)); 113 | if (!buf) 114 | return 1; 115 | 116 | mr = ibv_reg_mr(pd, buf, 2 * sizeof (uint32_t), 117 | IBV_ACCESS_LOCAL_WRITE | 118 | IBV_ACCESS_REMOTE_READ | 119 | IBV_ACCESS_REMOTE_WRITE); 120 | if (!mr) 121 | return 1; 122 | 123 | qp_attr.cap.max_send_wr = 1; 124 | qp_attr.cap.max_send_sge = 1; 125 | qp_attr.cap.max_recv_wr = 1; 126 | qp_attr.cap.max_recv_sge = 1; 127 | 128 | qp_attr.send_cq = cq; 129 | qp_attr.recv_cq = cq; 130 | 131 | qp_attr.qp_type = IBV_QPT_RC; 132 | 133 | err = rdma_create_qp(cm_id, pd, &qp_attr); 134 | if (err) 135 | return err; 136 | 137 | /* Post receive before accepting connection */ 138 | sge.addr = (uintptr_t) buf + sizeof (uint32_t); 139 | sge.length = sizeof (uint32_t); 140 | sge.lkey = mr->lkey; 141 | 142 | recv_wr.sg_list = &sge; 143 | recv_wr.num_sge = 1; 144 | 145 | if (ibv_post_recv(cm_id->qp, &recv_wr, &bad_recv_wr)) 146 | return 1; 147 | 148 | rep_pdata.buf_va = htonl((uintptr_t) buf); 149 | rep_pdata.buf_rkey = htonl(mr->rkey); 150 | 151 | conn_param.responder_resources = 1; 152 | conn_param.private_data = &rep_pdata; 153 | conn_param.private_data_len = sizeof rep_pdata; 154 | 155 | /* Accept connection */ 156 | 157 | err = rdma_accept(cm_id, &conn_param); 158 | if (err) 159 | return 1; 160 | 161 | err = rdma_get_cm_event(cm_channel, &event); 162 | if (err) 163 | return err; 164 | 165 | if (event->event != RDMA_CM_EVENT_ESTABLISHED) 166 | return 1; 167 | 168 | rdma_ack_cm_event(event); 169 | 170 | /* Wait for receive completion */ 171 | 172 | if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context)) 173 | return 1; 174 | 175 | if (ibv_req_notify_cq(cq, 0)) 176 | return 1; 177 | 178 | if (ibv_poll_cq(cq, 1, &wc) < 1) 179 | return 1; 180 | 181 | if (wc.status != IBV_WC_SUCCESS) //spark error 182 | return 1; 183 | 184 | /* Add two integers and send reply back */ 185 | 186 | buf[0] = htonl(ntohl(buf[0]) + ntohl(buf[1])); 187 | 188 | sge.addr = (uintptr_t) buf; 189 | sge.length = sizeof (uint32_t); 190 | sge.lkey = mr->lkey; 191 | 192 | send_wr.opcode = IBV_WR_SEND; 193 | send_wr.send_flags = IBV_SEND_SIGNALED; 194 | send_wr.sg_list = &sge; 195 | send_wr.num_sge = 1 ; 196 | 197 | if (ibv_post_send(cm_id->qp, &send_wr, &bad_send_wr)) 198 | return 1; 199 | 200 | /* Wait for send completion */ 201 | 202 | if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context)) 203 | return 1; 204 | 205 | if (ibv_poll_cq(cq, 1, &wc) < 1) 206 | return 1; 207 | 208 | if (wc.status != IBV_WC_SUCCESS) 209 | return 1; 210 | 211 | ibv_ack_cq_events(cq, 2); 212 | return 0; 213 | } 214 | --------------------------------------------------------------------------------