├── AUTHORS
├── src
    ├── main
    │   └── java
    │   │   └── com
    │   │       └── ibm
    │   │           └── darpc
    │   │               ├── DaRPCProtocol.java
    │   │               ├── DaRPCMessage.java
    │   │               ├── DaRPCService.java
    │   │               ├── DaRPCCluster.java
    │   │               ├── DaRPCServerEvent.java
    │   │               ├── DaRPCInstance.java
    │   │               ├── DaRPCEndpointGroup.java
    │   │               ├── DaRPCStream.java
    │   │               ├── DaRPCClientGroup.java
    │   │               ├── DaRPCResourceManager.java
    │   │               ├── DaRPCServerEndpoint.java
    │   │               ├── DaRPCFuture.java
    │   │               ├── DaRPCClientEndpoint.java
    │   │               ├── DaRPCServerGroup.java
    │   │               └── DaRPCEndpoint.java
    └── test
    │   └── java
    │       └── com
    │           └── ibm
    │               └── darpc
    │                   └── examples
    │                       ├── protocol
    │                           ├── RdmaRpcProtocol.java
    │                           ├── RdmaRpcRequest.java
    │                           └── RdmaRpcResponse.java
    │                       ├── server
    │                           ├── RdmaRpcService.java
    │                           └── DaRPCServer.java
    │                       └── client
    │                           └── DaRPCClient.java
├── pom.xml
├── README.md
└── LICENSE


/AUTHORS:
--------------------------------------------------------------------------------
1 | Patrick Stuedi <stu@zurich.ibm.com>
2 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCProtocol.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | public interface DaRPCProtocol<R extends DaRPCMessage, T extends DaRPCMessage> {
25 | 	public abstract R createRequest();
26 | 	public abstract T createResponse();	
27 | }
28 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCMessage.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | import java.nio.ByteBuffer;
26 | 
27 | public interface DaRPCMessage {
28 | 	public int write(ByteBuffer buffer) throws IOException;
29 | 	public void update(ByteBuffer buffer) throws IOException;
30 | 	public int size();	
31 | 
32 | }
33 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCService.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | 
26 | public interface DaRPCService <R extends DaRPCMessage, T extends DaRPCMessage> extends DaRPCProtocol<R,T> {
27 | 	public void processServerEvent(DaRPCServerEvent<R,T> event) throws IOException;
28 | 	public void open(DaRPCServerEndpoint<R,T> rpcClientEndpoint);
29 | 	public void close(DaRPCServerEndpoint<R,T> rpcClientEndpoint);
30 | }
31 | 


--------------------------------------------------------------------------------
/src/test/java/com/ibm/darpc/examples/protocol/RdmaRpcProtocol.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc.examples.protocol;
23 | 
24 | import com.ibm.darpc.DaRPCProtocol;
25 | 
26 | public class RdmaRpcProtocol implements DaRPCProtocol<RdmaRpcRequest, RdmaRpcResponse> {
27 | 	public static final int FUNCTION_FOO = 1;
28 | 	public static final int FUNCTION_BAR = 2;	
29 | 	
30 | 	@Override
31 | 	public RdmaRpcRequest createRequest() {
32 | 		return new RdmaRpcRequest();
33 | 	}
34 | 
35 | 	@Override
36 | 	public RdmaRpcResponse createResponse() {
37 | 		return new RdmaRpcResponse();
38 | 	}
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCCluster.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | import com.ibm.disni.verbs.*;
26 | import com.ibm.disni.*;
27 | 
28 | public class DaRPCCluster<R extends DaRPCMessage, T extends DaRPCMessage> extends RdmaCqProcessor<DaRPCEndpoint<R,T>>{
29 | 	public DaRPCCluster(IbvContext context, int cqSize, int wrSize, long affinity, int clusterId,
30 | 			int timeout, boolean polling) throws IOException {
31 | 		super(context, cqSize, wrSize, affinity, clusterId, timeout, polling);
32 | 	}
33 | 	
34 | 	@Override
35 | 	public void dispatchCqEvent(DaRPCEndpoint<R,T> endpoint, IbvWC wc) throws IOException {
36 | 		endpoint.dispatchCqEvent(wc);
37 | 	}	
38 | }
39 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCServerEvent.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | 
26 | public class DaRPCServerEvent<R extends DaRPCMessage, T extends DaRPCMessage> {
27 | 	private DaRPCServerEndpoint<R,T> endpoint;
28 | 	private R request;
29 | 	private T response;
30 | 	private int ticket;
31 | 	
32 | 	DaRPCServerEvent(DaRPCServerEndpoint<R,T> endpoint, R request, T response){
33 | 		this.endpoint = endpoint;
34 | 		this.request = request;
35 | 		this.response = response;
36 | 		this.ticket = 0;
37 | 	}
38 | 	
39 | 	public R getReceiveMessage() {
40 | 		return request;
41 | 	}
42 | 
43 | 	public T getSendMessage() {
44 | 		return response;
45 | 	}
46 | 	
47 | 	public void triggerResponse() throws IOException {
48 | 		endpoint.sendResponse(this);
49 | 	}
50 | 	
51 | 	public int getTicket(){
52 | 		return ticket;
53 | 	}
54 | 	
55 | 	void stamp(int ticket){
56 | 		this.ticket = ticket;
57 | 	}
58 | 	
59 | 
60 | }
61 | 


--------------------------------------------------------------------------------
/src/test/java/com/ibm/darpc/examples/protocol/RdmaRpcRequest.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc.examples.protocol;
23 | 
24 | import java.nio.ByteBuffer;
25 | 
26 | import com.ibm.darpc.DaRPCMessage;
27 | 
28 | public class RdmaRpcRequest implements DaRPCMessage {
29 | 	public static int SERIALIZED_SIZE = 16;
30 | 	
31 | 	private int cmd;
32 | 	private int param;
33 | 	private long time;
34 | 	
35 | 	public RdmaRpcRequest(){
36 | 	}
37 | 
38 | 	public int size() {
39 | 		return SERIALIZED_SIZE;
40 | 	}
41 | 
42 | 	@Override
43 | 	public void update(ByteBuffer buffer) {
44 | 		cmd = buffer.getInt();
45 | 		param = buffer.getInt();
46 | 		time = buffer.getLong();
47 | 	}
48 | 
49 | 	@Override
50 | 	public int write(ByteBuffer buffer) {
51 | 		buffer.putInt(cmd);
52 | 		buffer.putInt(param);
53 | 		buffer.putLong(time);
54 | 		
55 | 		return SERIALIZED_SIZE;
56 | 	}
57 | 
58 | 	public int getCmd() {
59 | 		return cmd;
60 | 	}
61 | 
62 | 	public void setCmd(int cmd) {
63 | 		this.cmd = cmd;
64 | 	}
65 | 
66 | 	public int getParam() {
67 | 		return param;
68 | 	}
69 | 
70 | 	public void setParam(int param) {
71 | 		this.param = param;
72 | 	}
73 | }


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCInstance.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | import java.util.concurrent.ConcurrentHashMap;
26 | 
27 | import org.slf4j.Logger;
28 | import org.slf4j.LoggerFactory;
29 | 
30 | import com.ibm.disni.verbs.*;
31 | 
32 | public class DaRPCInstance<R extends DaRPCMessage, T extends DaRPCMessage> {
33 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
34 | 	
35 | 	private DaRPCCluster<R,T>[] processorArray;
36 | 	protected ConcurrentHashMap<RdmaCmId, Object> ids;
37 | 	private int poolsize;
38 | 	
39 | 	@SuppressWarnings("unchecked")
40 | 	public DaRPCInstance(IbvContext context, int cqSize, int wrSize, long[] affinities, int timeout, boolean polling) throws IOException {
41 | 		this.poolsize = affinities.length;
42 | 		logger.info("new cq pool, size " + poolsize);
43 | 		processorArray = new DaRPCCluster[affinities.length];
44 | 		for (int i = 0; i < affinities.length; i++){
45 | 			processorArray[i] = new DaRPCCluster<R,T>(context, cqSize, wrSize, affinities[i], i, timeout, polling);
46 | 			processorArray[i].start();
47 | 		}
48 | 		this.ids = new ConcurrentHashMap<RdmaCmId, Object>();
49 | 	}
50 | 	
51 | 	
52 | 	public DaRPCCluster<R,T> getProcessor(int clusterId) {
53 | 		return processorArray[clusterId];
54 | 	}
55 | 	
56 | 	public void close() throws IOException, InterruptedException {
57 | 		for (int i = 0; i < processorArray.length; i++){
58 | 			processorArray[i].close();
59 | 		}
60 | 	}	
61 | }
62 | 


--------------------------------------------------------------------------------
/src/test/java/com/ibm/darpc/examples/protocol/RdmaRpcResponse.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc.examples.protocol;
23 | 
24 | import java.nio.ByteBuffer;
25 | 
26 | import com.ibm.darpc.DaRPCMessage;
27 | 
28 | public class RdmaRpcResponse implements DaRPCMessage {
29 | 	public static int SERIALIZED_SIZE = 24;
30 | 	
31 | 	private int error;
32 | 	private int name;
33 | 	private long time;
34 | 	private int type;
35 | 	
36 | 	public RdmaRpcResponse(){
37 | 	}
38 | 	
39 | 	public int size() {
40 | 		return SERIALIZED_SIZE;
41 | 	}
42 | 
43 | 	@Override
44 | 	public void update(ByteBuffer buffer) {
45 | 		error = buffer.getInt();
46 | 		name = buffer.getInt();
47 | 		time = buffer.getLong();
48 | 		type = buffer.getInt();
49 | 	}
50 | 
51 | 	@Override
52 | 	public int write(ByteBuffer buffer) {
53 | 		buffer.putInt(error);
54 | 		buffer.putInt(name);
55 | 		buffer.putLong(time);
56 | 		buffer.putInt(type);
57 | 		
58 | 		return SERIALIZED_SIZE;
59 | 	}
60 | 
61 | 	public int getError() {
62 | 		return error;
63 | 	}
64 | 
65 | 	public int getName() {
66 | 		return name;
67 | 	}
68 | 
69 | 	public long getTime() {
70 | 		return time;
71 | 	}
72 | 
73 | 	public int getType() {
74 | 		return type;
75 | 	}
76 | 
77 | 	public void setError(int error) {
78 | 		this.error = error;
79 | 	}
80 | 
81 | 	public void setName(int name) {
82 | 		this.name = name;
83 | 	}
84 | 
85 | 	public void setTime(long time) {
86 | 		this.time = time;
87 | 	}
88 | 
89 | 	public void setType(int type) {
90 | 		this.type = type;
91 | 	}
92 | 
93 | 	@Override
94 | 	public String toString() {
95 | 		return "" + name;
96 | 	}
97 | }


--------------------------------------------------------------------------------
/src/test/java/com/ibm/darpc/examples/server/RdmaRpcService.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc.examples.server;
23 | 
24 | import java.io.IOException;
25 | 
26 | import com.ibm.darpc.DaRPCServerEndpoint;
27 | import com.ibm.darpc.DaRPCServerEvent;
28 | import com.ibm.darpc.DaRPCService;
29 | import com.ibm.darpc.examples.protocol.RdmaRpcProtocol;
30 | import com.ibm.darpc.examples.protocol.RdmaRpcRequest;
31 | import com.ibm.darpc.examples.protocol.RdmaRpcResponse;
32 | 
33 | public class RdmaRpcService extends RdmaRpcProtocol implements DaRPCService<RdmaRpcRequest, RdmaRpcResponse> {
34 | 	private int servicetimeout;
35 | 	
36 | 	public RdmaRpcService(int servicetimeout){
37 | 		this.servicetimeout = servicetimeout;
38 | 	}
39 | 	
40 | 	public void processServerEvent(DaRPCServerEvent<RdmaRpcRequest, RdmaRpcResponse> event) throws IOException {
41 | 		RdmaRpcRequest request = event.getReceiveMessage();
42 | 		RdmaRpcResponse response = event.getSendMessage();
43 | 		response.setName(request.getParam() + 1);
44 | 		if (servicetimeout > 0){
45 | 			try {
46 | 				Thread.sleep(servicetimeout);
47 | 			} catch (InterruptedException e) {
48 | 				e.printStackTrace();
49 | 			}
50 | 		}
51 | 		event.triggerResponse();
52 | 	}
53 | 
54 | 	@Override
55 | 	public void open(DaRPCServerEndpoint<RdmaRpcRequest, RdmaRpcResponse> endpoint) {
56 | 		System.out.println("new connection " + endpoint.getEndpointId() + ", cluster " + endpoint.clusterId());
57 | 	}
58 | 
59 | 	@Override
60 | 	public void close(DaRPCServerEndpoint<RdmaRpcRequest, RdmaRpcResponse> endpoint) {
61 | 		System.out.println("disconnecting " + endpoint.getEndpointId() + ", cluster " + endpoint.clusterId());
62 | 	}
63 | }
64 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCEndpointGroup.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | import org.slf4j.Logger;
26 | import org.slf4j.LoggerFactory;
27 | 
28 | import com.ibm.disni.verbs.*;
29 | import com.ibm.disni.*;
30 | 
31 | 
32 | public abstract class DaRPCEndpointGroup<E extends DaRPCEndpoint<R,T>, R extends DaRPCMessage, T extends DaRPCMessage> extends RdmaEndpointGroup<E> {
33 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
34 | 	private static int DARPC_VERSION = 50;
35 | 	
36 | 	private int recvQueueSize;
37 | 	private int sendQueueSize;
38 | 	private int timeout;
39 | 	private int bufferSize;
40 | 	private int maxInline;
41 | 	
42 | 	public static int getVersion(){
43 | 		return DARPC_VERSION;
44 | 	}	
45 | 	
46 | 	protected DaRPCEndpointGroup(DaRPCProtocol<R,T> protocol, int timeout, int maxinline, int recvQueue, int sendQueue) throws Exception {
47 | 		super(timeout);
48 | 		this.recvQueueSize = recvQueue;
49 | 		this.sendQueueSize = Math.max(recvQueue, sendQueue);
50 | 		this.timeout = timeout;
51 | 		this.bufferSize = Math.max(protocol.createRequest().size(), protocol.createResponse().size());
52 | 		this.maxInline = maxinline;
53 | 	}	
54 | 	
55 | 	protected synchronized IbvQP createQP(RdmaCmId id, IbvPd pd, IbvCQ cq) throws IOException{
56 | 		IbvQPInitAttr attr = new IbvQPInitAttr();
57 | 		attr.cap().setMax_recv_wr(recvQueueSize);
58 | 		attr.cap().setMax_send_wr(sendQueueSize);
59 | 		attr.cap().setMax_recv_sge(1);
60 | 		attr.cap().setMax_send_sge(1);
61 | 		attr.cap().setMax_inline_data(maxInline);
62 | 		attr.setQp_type(IbvQP.IBV_QPT_RC);
63 | 		attr.setRecv_cq(cq);
64 | 		attr.setSend_cq(cq);		
65 | 		IbvQP qp = id.createQP(pd, attr);
66 | 		return qp;
67 | 	}
68 | 	
69 | 	public int getTimeout() {
70 | 		return timeout;
71 | 	}
72 | 	
73 | 	public int getBufferSize() {
74 | 		return bufferSize;
75 | 	}	
76 | 
77 | 	public void close() throws IOException, InterruptedException {
78 | 		super.close();
79 | 		logger.info("rpc group down");
80 | 	}	
81 | 	
82 | 	public int recvQueueSize() {
83 | 		return recvQueueSize;
84 | 	}
85 | 	
86 | 	public int sendQueueSize() {
87 | 		return sendQueueSize;
88 | 	}		
89 | 	
90 | 	public int getMaxInline() {
91 | 		return maxInline;
92 | 	}
93 | }
94 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCStream.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | import java.util.concurrent.LinkedBlockingDeque;
26 | 
27 | import org.slf4j.Logger;
28 | import org.slf4j.LoggerFactory;
29 | 
30 | public class DaRPCStream<R extends DaRPCMessage,T extends DaRPCMessage> {
31 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
32 | 	
33 | 	private DaRPCClientEndpoint<R,T> endpoint;
34 | 	private LinkedBlockingDeque<DaRPCFuture<R, T>> completedList;
35 | 	
36 | 	DaRPCStream(DaRPCClientEndpoint<R,T> endpoint, int streamId) throws IOException{
37 | 		logger.info("new direct rpc stream");
38 | 		this.endpoint = endpoint;
39 | 		this.completedList = new LinkedBlockingDeque<DaRPCFuture<R, T>>();
40 | 	}	
41 | 	
42 | 	public DaRPCFuture<R, T> request(R request, T response, boolean streamLogged) throws IOException {
43 | 		DaRPCFuture<R, T> future = new DaRPCFuture<R, T>(this, endpoint, request, response, streamLogged);
44 | 		endpoint.sendRequest(future);
45 | 		return future;
46 | 	}
47 | 	
48 | 	public DaRPCFuture<R, T> take() throws IOException {
49 | 		try {
50 | 			DaRPCFuture<R, T> future = completedList.poll();
51 | 			while (future == null){
52 | 				endpoint.pollOnce();
53 | 				future = completedList.poll();
54 | 			}
55 | 			return future;
56 | 		} catch(Exception e){
57 | 			throw new IOException(e);
58 | 		}
59 | 	}
60 | 	
61 | 	public DaRPCFuture<R, T> take(int timeout) throws IOException {
62 | 		try {
63 | 			DaRPCFuture<R, T> future = completedList.poll();
64 | 			long sumtime = 0;
65 | 			while (future == null && sumtime < timeout){
66 | 				endpoint.pollOnce();
67 | 				future = completedList.poll();
68 | 			}
69 | 			return future;
70 | 		} catch (Exception e){
71 | 			throw new IOException(e);
72 | 		}
73 | 		
74 | 	}
75 | 	
76 | 	public DaRPCFuture<R, T> poll() throws IOException {
77 | 		DaRPCFuture<R, T> future = completedList.poll();
78 | 		if (future == null){
79 | 			endpoint.pollOnce();
80 | 			future = completedList.poll();			
81 | 		}
82 | 		return future;
83 | 	}
84 | 	
85 | 	public void clear(){
86 | 		completedList.clear();
87 | 	}
88 | 
89 | 	void addFuture(DaRPCFuture<R, T> future){
90 | 		completedList.add(future);
91 | 	}
92 | 
93 | 	public boolean isEmpty() {
94 | 		return completedList.isEmpty();
95 | 	}
96 | }
97 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCClientGroup.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * DaRPC: Data Center Remote Procedure Call
 3 |  *
 4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
 5 |  *
 6 |  * Copyright (C) 2016-2018, IBM Corporation
 7 |  *
 8 |  * Licensed under the Apache License, Version 2.0 (the "License");
 9 |  * you may not use this file except in compliance with the License.
10 |  * You may obtain a copy of the License at
11 |  *
12 |  *     http://www.apache.org/licenses/LICENSE-2.0
13 |  *
14 |  * Unless required by applicable law or agreed to in writing, software
15 |  * distributed under the License is distributed on an "AS IS" BASIS,
16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 |  * See the License for the specific language governing permissions and
18 |  * limitations under the License.
19 |  *
20 |  */
21 | 
22 | package com.ibm.darpc;
23 | 
24 | import java.io.IOException;
25 | 
26 | import com.ibm.disni.RdmaCqProvider;
27 | import com.ibm.disni.RdmaEndpointFactory;
28 | import com.ibm.disni.verbs.IbvCQ;
29 | import com.ibm.disni.verbs.IbvQP;
30 | import com.ibm.disni.verbs.RdmaCmId;
31 | 
32 | public class DaRPCClientGroup<R extends DaRPCMessage, T extends DaRPCMessage> extends DaRPCEndpointGroup<DaRPCClientEndpoint<R,T>, R, T> {
33 | 	public static <R extends DaRPCMessage, T extends DaRPCMessage> DaRPCClientGroup<R, T> createClientGroup(DaRPCProtocol<R, T> protocol, int timeout, int maxinline, int recvQueue, int sendQueue) throws Exception {
34 | 		DaRPCClientGroup<R,T> group = new DaRPCClientGroup<R,T>(protocol, timeout, maxinline, recvQueue, sendQueue);
35 | 		group.init(new RpcClientFactory<R,T>(group));
36 | 		return group;
37 | 	}	
38 | 	
39 | 	private DaRPCClientGroup(DaRPCProtocol<R, T> protocol, int timeout, int maxinline, int recvQueue, int sendQueue)
40 | 			throws Exception {
41 | 		super(protocol, timeout, maxinline, recvQueue, sendQueue);
42 | 	}
43 | 	
44 | 
45 | 	@Override
46 | 	public void allocateResources(DaRPCClientEndpoint<R, T> endpoint) throws Exception {
47 | 		endpoint.allocateResources();
48 | 	}
49 | 
50 | 	@Override
51 | 	public RdmaCqProvider createCqProvider(DaRPCClientEndpoint<R, T> endpoint) throws IOException {
52 | 		return new RdmaCqProvider(endpoint.getIdPriv().getVerbs(), recvQueueSize() + sendQueueSize());
53 | 	}
54 | 
55 | 	@Override
56 | 	public IbvQP createQpProvider(DaRPCClientEndpoint<R, T> endpoint) throws IOException {
57 | 		RdmaCqProvider cqProvider = endpoint.getCqProvider();
58 | 		IbvCQ cq = cqProvider.getCQ();
59 | 		IbvQP qp = this.createQP(endpoint.getIdPriv(), endpoint.getPd(), cq);
60 | 		return qp;
61 | 	}
62 | 	
63 | 	public static class RpcClientFactory<R extends DaRPCMessage, T extends DaRPCMessage> implements RdmaEndpointFactory<DaRPCClientEndpoint<R,T>> {
64 | 		private DaRPCClientGroup<R, T> group;
65 | 		
66 | 		public RpcClientFactory(DaRPCClientGroup<R, T> group){
67 | 			this.group = group;
68 | 		}
69 | 		
70 | 		@Override
71 | 		public DaRPCClientEndpoint<R,T> createEndpoint(RdmaCmId id, boolean serverSide) throws IOException {
72 | 			return new DaRPCClientEndpoint<R,T>(group, id, serverSide);
73 | 		}
74 | 	}	
75 | 
76 | }
77 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCResourceManager.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc;
 23 | 
 24 | import java.io.IOException;
 25 | import java.util.concurrent.LinkedBlockingQueue;
 26 | import java.util.concurrent.TimeUnit;
 27 | 
 28 | import org.slf4j.Logger;
 29 | import org.slf4j.LoggerFactory;
 30 | 
 31 | import com.ibm.disni.util.*;
 32 | 
 33 | public class DaRPCResourceManager {
 34 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
 35 | 	private RpcResourceAllocator[] allocaters;
 36 | 	
 37 | 	public DaRPCResourceManager(long[] affinities, int timeout) throws Exception {
 38 | 		this.allocaters = new RpcResourceAllocator[affinities.length];
 39 | 		for (int i = 0; i < affinities.length; i++){
 40 | 			allocaters[i] = new RpcResourceAllocator(affinities[i], i, timeout);
 41 | 			allocaters[i].start();
 42 | 		}
 43 | 	}
 44 | 	
 45 | 	public void allocateResources(DaRPCServerEndpoint<?,?> endpoint) throws Exception {
 46 | 		logger.info("dispatching resource, clusterid " + endpoint.clusterId());
 47 | 		allocaters[endpoint.clusterId()].initResource(endpoint);
 48 | 	}
 49 | 	
 50 | 	public void close() throws IOException, InterruptedException {
 51 | 		for (int i = 0; i < allocaters.length; i++){
 52 | 			allocaters[i].close();
 53 | 		}		
 54 | 	}
 55 | 
 56 | 	public static class RpcResourceAllocator implements Runnable {
 57 | 		private static Logger logger = LoggerFactory.getLogger("com.ibm.zac.darpc");
 58 | 		private LinkedBlockingQueue<DaRPCEndpoint<?,?>> requestQueue;
 59 | 		private long affinity;
 60 | 		private int index;
 61 | 		private boolean running;
 62 | 		private Thread thread;
 63 | 		private int timeout;
 64 | 		
 65 | 		public RpcResourceAllocator(long affinity, int index, int timeout) throws Exception {
 66 | 			this.affinity = affinity;
 67 | 			this.index = index;
 68 | 			this.requestQueue = new LinkedBlockingQueue<DaRPCEndpoint<?,?>>();
 69 | 			this.running = false;
 70 | 			this.timeout = timeout;
 71 | 			if (timeout <= 0){
 72 | 				this.timeout = Integer.MAX_VALUE;
 73 | 			}
 74 | 			this.thread = new Thread(this);
 75 | 		}
 76 | 		
 77 | 		public void initResource(DaRPCEndpoint<?,?> endpoint) {
 78 | 			requestQueue.add(endpoint);
 79 | 		}	
 80 | 		
 81 | 		public synchronized void start(){
 82 | 			running = true;
 83 | 			thread.start();
 84 | 		}		
 85 | 		
 86 | 		public void run() {
 87 | 			NativeAffinity.setAffinity(affinity);
 88 | 			logger.info("running resource management, index " + index + ", affinity " + affinity + ", timeout " + timeout);
 89 | 			while(running){
 90 | 				try {
 91 | 					DaRPCEndpoint<?,?> endpoint = requestQueue.poll(timeout, TimeUnit.MILLISECONDS);
 92 | 					if (endpoint != null){
 93 | 						logger.info("allocating resources, cluster " + index + ", endpoint " + endpoint.getEndpointId());
 94 | 						endpoint.allocateResources();
 95 | 					}
 96 | 				} catch(Exception e){
 97 | 					e.printStackTrace();
 98 | 				}
 99 | 			}
100 | 		}
101 | 		
102 | 		public void close() throws IOException, InterruptedException {
103 | 			running = false;
104 | 			thread.join();
105 | 			logger.info("resource management closed");
106 | 		}		
107 | 	}
108 | 
109 | 
110 | }
111 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCServerEndpoint.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc;
 23 | 
 24 | import java.io.IOException;
 25 | import java.nio.ByteBuffer;
 26 | import java.util.concurrent.ArrayBlockingQueue;
 27 | 
 28 | import org.slf4j.Logger;
 29 | import org.slf4j.LoggerFactory;
 30 | 
 31 | import com.ibm.disni.verbs.RdmaCmEvent;
 32 | import com.ibm.disni.verbs.RdmaCmId;
 33 | 
 34 | public class DaRPCServerEndpoint<R extends DaRPCMessage, T extends DaRPCMessage> extends DaRPCEndpoint<R,T> {
 35 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
 36 | 	
 37 | 	private DaRPCServerGroup<R, T> group;
 38 | 	private ArrayBlockingQueue<DaRPCServerEvent<R,T>> eventPool;
 39 | 	private ArrayBlockingQueue<DaRPCServerEvent<R,T>> lazyEvents;
 40 | 	private int getClusterId;
 41 | 	
 42 | 	public DaRPCServerEndpoint(DaRPCServerGroup<R, T> group, RdmaCmId idPriv, boolean serverSide) throws IOException {
 43 | 		super(group, idPriv, serverSide);
 44 | 		this.group = group;
 45 | 		this.getClusterId = group.newClusterId();
 46 | 		this.eventPool = new ArrayBlockingQueue<DaRPCServerEvent<R,T>>(group.recvQueueSize());
 47 | 		this.lazyEvents = new ArrayBlockingQueue<DaRPCServerEvent<R,T>>(group.recvQueueSize());
 48 | 	}
 49 | 
 50 | 	public void init() throws IOException {
 51 | 		super.init();
 52 | 		for(int i = 0; i < group.recvQueueSize(); i++){
 53 | 			DaRPCServerEvent<R,T> event = new DaRPCServerEvent<R,T>(this, group.createRequest(), group.createResponse());
 54 | 			this.eventPool.add(event);
 55 | 			
 56 | 		}
 57 | 	}
 58 | 	
 59 | 	void sendResponse(DaRPCServerEvent<R,T> event) throws IOException {
 60 | 		if (sendMessage(event.getSendMessage(), event.getTicket())){
 61 | 			eventPool.add(event);
 62 | 		} else {
 63 | 			lazyEvents.add(event);
 64 | 		}
 65 | 	}	
 66 | 	
 67 | 	public synchronized void dispatchCmEvent(RdmaCmEvent cmEvent) throws IOException {
 68 | 		super.dispatchCmEvent(cmEvent);
 69 | 		try {
 70 | 			int eventType = cmEvent.getEvent();
 71 | 			if (eventType == RdmaCmEvent.EventType.RDMA_CM_EVENT_ESTABLISHED.ordinal()) {
 72 | 				logger.info("new RPC connection, eid " + this.getEndpointId());
 73 | 				group.open(this);
 74 | 			} else if (eventType == RdmaCmEvent.EventType.RDMA_CM_EVENT_DISCONNECTED.ordinal()) {
 75 | 				logger.info("RPC disconnection, eid " + this.getEndpointId());
 76 | 				group.close(this);
 77 | 			} 
 78 | 		} catch (Exception e) {
 79 | 			e.printStackTrace();
 80 | 		}
 81 | 	}	
 82 | 
 83 | 	public int clusterId() {
 84 | 		return getClusterId;
 85 | 	}
 86 | 	
 87 | 	public void dispatchReceive(ByteBuffer recvBuffer, int ticket, int recvIndex) throws IOException {
 88 | 		DaRPCServerEvent<R,T> event = eventPool.poll();
 89 | 		if (event == null){
 90 | 			logger.info("no free events, must be overrunning server.. ");
 91 | 			throw new IOException("no free events, must be overrunning server.. ");
 92 | 		}
 93 | 		event.getReceiveMessage().update(recvBuffer);
 94 | 		event.stamp(ticket);
 95 | 		postRecv(recvIndex);
 96 | 		group.processServerEvent(event);			
 97 | 	}
 98 | 	
 99 | 	public void dispatchSend(int ticket) throws IOException {
100 | 		freeSend(ticket);		
101 | 		DaRPCServerEvent<R,T> event = lazyEvents.poll();
102 | 		if (event != null){
103 | 			sendResponse(event);
104 | 		}
105 | 	}
106 | }
107 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCFuture.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc;
 23 | 
 24 | import java.util.concurrent.ExecutionException;
 25 | import java.util.concurrent.Future;
 26 | import java.util.concurrent.TimeUnit;
 27 | import java.util.concurrent.atomic.AtomicInteger;
 28 | 
 29 | import org.slf4j.Logger;
 30 | import org.slf4j.LoggerFactory;
 31 | 
 32 | public class DaRPCFuture<R extends DaRPCMessage, T extends DaRPCMessage> implements Future<T> {
 33 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
 34 | 	
 35 | 	protected static int RPC_PENDING = 0;
 36 | 	protected static int RPC_DONE = 1;
 37 | 	protected static int RPC_ERROR = 2;	
 38 | 	
 39 | 	private int ticket;
 40 | 	private R request;
 41 | 	private T response;	
 42 | 	
 43 | 	private DaRPCStream<R, T> stream;
 44 | 	private DaRPCClientEndpoint<R,T> endpoint;
 45 | 	private boolean streamLogged;
 46 | 	private AtomicInteger status;
 47 | 	private AtomicInteger recvStatus;
 48 | 	
 49 | 	public DaRPCFuture(DaRPCStream<R,T> stream, DaRPCClientEndpoint<R,T> endpoint, R request, T response, boolean streamLogged){
 50 | 		this.request = request;
 51 | 		this.response = response;		
 52 | 		this.ticket = -1;		
 53 | 		
 54 | 		this.stream = stream;
 55 | 		this.endpoint = endpoint;
 56 | 		this.status = new AtomicInteger(RPC_PENDING);
 57 | 		this.streamLogged = streamLogged;
 58 | 		this.recvStatus = new AtomicInteger(0);
 59 | 	}	
 60 | 	
 61 | 	public int getTicket() {
 62 | 		return this.ticket;
 63 | 	}
 64 | 	
 65 | 	public R getSendMessage(){
 66 | 		return request;
 67 | 	}
 68 | 	
 69 | 	public T getReceiveMessage(){
 70 | 		return response;
 71 | 	}
 72 | 	
 73 | 	public void stamp(int ticket) {
 74 | 		this.ticket = ticket;
 75 | 	}	
 76 | 	
 77 | 	@Override
 78 | 	public T get() throws InterruptedException, ExecutionException {
 79 | 		if (status.get() == RPC_PENDING){
 80 | 			try {
 81 | 				endpoint.pollUntil(status, Long.MAX_VALUE);
 82 | 			} catch(Exception e){
 83 | 				status.set(RPC_ERROR);
 84 | 				throw new InterruptedException(e.getMessage());
 85 | 			}
 86 | 		}
 87 | 		
 88 | 		if (status.get() == RPC_DONE){
 89 | 			return this.getReceiveMessage();
 90 | 		} else if (status.get() == RPC_PENDING){
 91 | 			throw new InterruptedException("RPC timeout");
 92 | 		} else {
 93 | 			throw new InterruptedException("RPC error");
 94 | 		}
 95 | 	}	
 96 | 	
 97 | 	@Override
 98 | 	public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException {
 99 | 		if (status.get() == RPC_PENDING){
100 | 			try {
101 | 				endpoint.pollUntil(status, timeout);
102 | 			} catch(Exception e){
103 | 				status.set(RPC_ERROR);
104 | 				throw new InterruptedException(e.getMessage());
105 | 			}
106 | 		}
107 | 		
108 | 		if (status.get() == RPC_DONE){
109 | 			return this.getReceiveMessage();
110 | 		} else if (status.get() == RPC_PENDING){
111 | 			throw new InterruptedException("RPC timeout");
112 | 		} else {
113 | 			throw new InterruptedException("RPC error");
114 | 		}
115 | 	}	
116 | 	
117 | 	@Override
118 | 	public boolean isDone() {
119 | 		if (status.get() == 0) {
120 | 			try {
121 | 				endpoint.pollOnce();
122 | 			} catch(Exception e){
123 | 				status.set(RPC_ERROR);
124 | 				logger.info(e.getMessage());
125 | 			}
126 | 		}
127 | 		return status.get() > 0;
128 | 	}
129 | 
130 | 	public synchronized void signal(int wcstatus) {
131 | 		if (status.get() == 0){
132 | 			if (wcstatus == 0){
133 | 				status.set(RPC_DONE);
134 | 			} else {
135 | 				status.set(RPC_ERROR);
136 | 			}
137 | 			if (streamLogged){
138 | 				stream.addFuture(this);
139 | 			}
140 | 		}
141 | 	}
142 | 	
143 | 	@Override
144 | 	public boolean cancel(boolean mayInterruptIfRunning) {
145 | 		return false;
146 | 	}
147 | 
148 | 	@Override
149 | 	public boolean isCancelled() {
150 | 		return false;
151 | 	}
152 | 	
153 | 	public boolean isStreamLogged() {
154 | 		return streamLogged;
155 | 	}
156 | 	
157 | 	public boolean touch(){
158 | 		if (recvStatus.incrementAndGet() == 2){
159 | 			recvStatus.set(0);
160 | 			return true;
161 | 		} else {
162 | 			return false;
163 | 		}
164 | 	}
165 | }
166 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCClientEndpoint.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc;
 23 | 
 24 | import java.io.IOException;
 25 | import java.nio.ByteBuffer;
 26 | import java.util.concurrent.ConcurrentHashMap;
 27 | import java.util.concurrent.atomic.AtomicInteger;
 28 | import java.util.concurrent.locks.ReentrantLock;
 29 | import org.slf4j.Logger;
 30 | import org.slf4j.LoggerFactory;
 31 | import com.ibm.disni.verbs.IbvCQ;
 32 | import com.ibm.disni.verbs.IbvWC;
 33 | import com.ibm.disni.verbs.RdmaCmId;
 34 | import com.ibm.disni.verbs.SVCPollCq;
 35 | 
 36 | public class DaRPCClientEndpoint<R extends DaRPCMessage, T extends DaRPCMessage> extends DaRPCEndpoint<R,T> {
 37 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
 38 | 	
 39 | 	private ConcurrentHashMap<Integer, DaRPCFuture<R,T>> pendingFutures;
 40 | 	private AtomicInteger ticketCount;
 41 | 	private int streamCount;
 42 | 	private IbvWC[] wcList;
 43 | 	private SVCPollCq poll;	
 44 | 	private ReentrantLock lock;	
 45 | 
 46 | 	public DaRPCClientEndpoint(DaRPCEndpointGroup<? extends DaRPCClientEndpoint<R, T>, R, T> group, RdmaCmId idPriv, boolean serverSide) throws IOException {
 47 | 		super(group, idPriv, serverSide);
 48 | 		this.pendingFutures = new ConcurrentHashMap<Integer, DaRPCFuture<R,T>>();
 49 | 		this.ticketCount = new AtomicInteger(0);
 50 | 		this.streamCount = 1;
 51 | 		this.lock = new ReentrantLock();		
 52 | 	}
 53 | 	
 54 | 	@Override
 55 | 	public void init() throws IOException {
 56 | 		super.init();
 57 | 		IbvCQ cq = getCqProvider().getCQ();
 58 | 		this.wcList = new IbvWC[getCqProvider().getCqSize()];
 59 | 		for (int i = 0; i < wcList.length; i++){
 60 | 			wcList[i] = new IbvWC();
 61 | 		}		
 62 | 		this.poll = cq.poll(wcList, wcList.length);			
 63 | 	}	
 64 | 	
 65 | 	public DaRPCStream<R, T> createStream() throws IOException {
 66 | 		int streamId = this.streamCount;
 67 | 		DaRPCStream<R,T> stream = new DaRPCStream<R,T>(this, streamId);
 68 | 		streamCount++;
 69 | 		return stream;
 70 | 	}	
 71 | 	
 72 | 	int sendRequest(DaRPCFuture<R,T> future) throws IOException {
 73 | 		int ticket = getAndIncrement();
 74 | 		future.stamp(ticket);
 75 | 		pendingFutures.put(future.getTicket(), future);
 76 | 		while (!sendMessage(future.getSendMessage(), future.getTicket())){
 77 | 			pollOnce();
 78 | 		}
 79 | 		return ticket;
 80 | 	}		
 81 | 
 82 | 	@Override
 83 | 	public void dispatchReceive(ByteBuffer recvBuffer, int ticket, int recvIndex) throws IOException {
 84 | 		DaRPCFuture<R, T> future = pendingFutures.get(ticket);
 85 | 		if (future == null){
 86 | 			logger.info("no pending future (receive) for ticket " + ticket);
 87 | 			throw new IOException("no pending future (receive) for ticket " + ticket);
 88 | 		}		
 89 | 		future.getReceiveMessage().update(recvBuffer);
 90 | 		postRecv(recvIndex);
 91 | 		if (future.touch()){
 92 | 			pendingFutures.remove(ticket);
 93 | 			freeSend(ticket);
 94 | 		}
 95 | 		future.signal(0);
 96 | 	}
 97 | 
 98 | 	@Override
 99 | 	public void dispatchSend(int ticket) throws IOException {
100 | 		DaRPCFuture<R, T> future = pendingFutures.get(ticket);
101 | 		if (future == null){
102 | 			logger.info("no pending future (send) for ticket " + ticket);
103 | 			throw new IOException("no pending future (send) for ticket " + ticket);
104 | 		}		
105 | 		if (future.touch()){
106 | 			pendingFutures.remove(ticket);
107 | 			freeSend(ticket);
108 | 		}
109 | 	}
110 | 	
111 | 	private int getAndIncrement() {
112 | 		return ticketCount.getAndIncrement() & Integer.MAX_VALUE;
113 | 	}	
114 | 
115 | 	public void pollOnce() throws IOException {
116 | 		if (!lock.tryLock()){
117 | 			return;
118 | 		}
119 | 		
120 | 		try {
121 | 			_pollOnce();
122 | 		} finally {
123 | 			lock.unlock();
124 | 		}
125 | 	}
126 | 	
127 | 	public void pollUntil(AtomicInteger future, long timeout) throws IOException {
128 | 		boolean locked = false;
129 | 		while(true){
130 | 			locked = lock.tryLock();
131 | 			if (future.get() > 0 || locked){
132 | 				break;
133 | 			}
134 | 		}
135 | 
136 | 		try {
137 | 			if (future.get() == 0){
138 | 				_pollUntil(future, timeout);
139 | 			}
140 | 		} finally {
141 | 			if (locked){
142 | 				lock.unlock();
143 | 			}
144 | 		}
145 | 	}	
146 | 	
147 | 	private int _pollOnce() throws IOException {
148 | 		int res = poll.execute().getPolls();
149 | 		if (res > 0) {
150 | 			for (int i = 0; i < res; i++){
151 | 				IbvWC wc = wcList[i];
152 | 				dispatchCqEvent(wc);
153 | 			}
154 | 			
155 | 		} 
156 | 		return res;
157 | 	}
158 | 	
159 | 	private int _pollUntil(AtomicInteger future, long timeout) throws IOException {
160 | 		long count = 0;
161 | 		final long checkTimeOut = 1 << 14 /* 16384 */;
162 | 		long startTime = System.nanoTime();
163 | 		while(future.get() == 0){
164 | 			int res = poll.execute().getPolls();
165 | 			if (res > 0){
166 | 				for (int i = 0; i < res; i++){
167 | 					IbvWC wc = wcList[i];
168 | 					dispatchCqEvent(wc);
169 | 				}
170 | 			}
171 | 			if (count == checkTimeOut) {
172 | 				count = 0;
173 | 				if ((System.nanoTime() - startTime) / 1e6 > timeout) {
174 | 					break;
175 | 				}
176 | 			}			
177 | 			count++;			
178 | 		}
179 | 		return 1;
180 | 	}	
181 | 
182 | }
183 | 


--------------------------------------------------------------------------------
/src/test/java/com/ibm/darpc/examples/server/DaRPCServer.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc.examples.server;
 23 | 
 24 | import java.net.InetSocketAddress;
 25 | import org.apache.commons.cli.CommandLine;
 26 | import org.apache.commons.cli.CommandLineParser;
 27 | import org.apache.commons.cli.DefaultParser;
 28 | import org.apache.commons.cli.HelpFormatter;
 29 | import org.apache.commons.cli.Option;
 30 | import org.apache.commons.cli.Options;
 31 | import org.apache.commons.cli.ParseException;
 32 | 
 33 | import com.ibm.darpc.DaRPCServerEndpoint;
 34 | import com.ibm.darpc.DaRPCServerGroup;
 35 | import com.ibm.darpc.examples.protocol.RdmaRpcRequest;
 36 | import com.ibm.darpc.examples.protocol.RdmaRpcResponse;
 37 | import com.ibm.disni.*;
 38 | 
 39 | public class DaRPCServer {
 40 | 	private String host; 
 41 | 	private int poolsize = 3;
 42 | 	private int recvQueue = 16;
 43 | 	private int sendQueue = 16;
 44 | 	private int wqSize = recvQueue;
 45 | 	private int servicetimeout = 0;
 46 | 	private boolean polling = false;
 47 | 	private int maxinline = 0;
 48 | //	private int connections = 16;
 49 | 	
 50 | 	public void run() throws Exception{
 51 | 		long[] clusterAffinities = new long[poolsize];
 52 | 		for (int i = 0; i < poolsize; i++){
 53 | 			long cpu = 1L << i;
 54 | 			clusterAffinities[i] = cpu;
 55 | 		}
 56 | 		System.out.println("running...server " + host + ", poolsize " + poolsize + ", maxinline " + maxinline + ", polling " + polling + ", recvQueue " + recvQueue + ", sendQueue " + sendQueue + ", wqSize " + wqSize + ", rpcservice-timeout " + servicetimeout);
 57 | 		RdmaRpcService rpcService = new RdmaRpcService(servicetimeout);
 58 | 		DaRPCServerGroup<RdmaRpcRequest, RdmaRpcResponse> group = DaRPCServerGroup.createServerGroup(rpcService, clusterAffinities, -1, maxinline, polling, recvQueue, sendQueue, wqSize, 32); 
 59 | 		RdmaServerEndpoint<DaRPCServerEndpoint<RdmaRpcRequest, RdmaRpcResponse>> serverEp = group.createServerEndpoint();
 60 | 		InetSocketAddress address = new InetSocketAddress(host, 1919);
 61 | 		serverEp.bind(address, 100);
 62 | 		while(true){
 63 | 			serverEp.accept();
 64 | 		}		
 65 | 	}
 66 | 	
 67 | 	public void launch(String[] args) throws Exception {
 68 | 		Option addressOption = Option.builder("a").required().desc("server address").hasArg().build();
 69 | 		Option poolsizeOption = Option.builder("p").desc("pool size").hasArg().build();
 70 | 		Option servicetimeoutOption = Option.builder("t").desc("service timeout").hasArg().build();
 71 | 		Option pollingOption = Option.builder("d").desc("if polling, default false").build();
 72 | 		Option maxinlineOption = Option.builder("i").desc("max inline data").hasArg().build();
 73 | 		Option wqSizeOption = Option.builder("w").desc("wq size").hasArg().build();
 74 | 		Option recvQueueOption = Option.builder("r").desc("receive queue").hasArg().build();
 75 | 		Option sendQueueOption = Option.builder("s").desc("send queue").hasArg().build();
 76 | 		Option serializedSizeOption = Option.builder("l").desc("serialized size").hasArg().build();
 77 | 		Options options = new Options();
 78 | 		options.addOption(addressOption);
 79 | 		options.addOption(poolsizeOption);
 80 | 		options.addOption(servicetimeoutOption);
 81 | 		options.addOption(pollingOption);
 82 | 		options.addOption(maxinlineOption);
 83 | 		options.addOption(wqSizeOption);
 84 | 		options.addOption(recvQueueOption);
 85 | 		options.addOption(sendQueueOption);
 86 | 		options.addOption(serializedSizeOption);
 87 | 		CommandLineParser parser = new DefaultParser();
 88 | 
 89 | 		try {
 90 | 			CommandLine line = parser.parse(options, args);
 91 | 			host = line.getOptionValue(addressOption.getOpt());
 92 | 
 93 | 			if (line.hasOption(poolsizeOption.getOpt())) {
 94 | 				poolsize = Integer.parseInt(line.getOptionValue(poolsizeOption.getOpt()));
 95 | 			}
 96 | 			if (line.hasOption(servicetimeoutOption.getOpt())) {
 97 | 				servicetimeout = Integer.parseInt(line.getOptionValue(servicetimeoutOption.getOpt()));
 98 | 			}
 99 | 			if (line.hasOption(pollingOption.getOpt())) {
100 | 				polling = true;
101 | 			}
102 | 			if (line.hasOption(maxinlineOption.getOpt())) {
103 | 				maxinline = Integer.parseInt(line.getOptionValue(maxinlineOption.getOpt()));
104 | 			}
105 | 			if (line.hasOption(wqSizeOption.getOpt())) {
106 | 				wqSize = Integer.parseInt(line.getOptionValue(wqSizeOption.getOpt()));
107 | 			}
108 | 			if (line.hasOption(recvQueueOption.getOpt())) {
109 | 				recvQueue = Integer.parseInt(line.getOptionValue(recvQueueOption.getOpt()));
110 | 			}
111 | 			if (line.hasOption(sendQueueOption.getOpt())) {
112 | 				sendQueue = Integer.parseInt(line.getOptionValue(sendQueueOption.getOpt()));
113 | 			}
114 | 			if (line.hasOption(serializedSizeOption.getOpt())) {
115 | 				RdmaRpcRequest.SERIALIZED_SIZE = Integer.parseInt(line.getOptionValue(serializedSizeOption.getOpt()));
116 | 				RdmaRpcResponse.SERIALIZED_SIZE = RdmaRpcRequest.SERIALIZED_SIZE;
117 | 			}
118 | 		} catch (ParseException e) {
119 | 			HelpFormatter formatter = new HelpFormatter();
120 | 			formatter.printHelp("DaRPCServer", options);
121 | 			System.exit(-1);
122 | 		}
123 | 		this.run();
124 | 	}
125 | 	
126 | 	public static void main(String[] args) throws Exception { 
127 | 		DaRPCServer rpcServer = new DaRPCServer();
128 | 		rpcServer.launch(args);		
129 | 	}	
130 | }
131 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  3 |   <modelVersion>4.0.0</modelVersion>
  4 | 
  5 |   <groupId>com.ibm.darpc</groupId>
  6 |   <artifactId>darpc</artifactId>
  7 |   <packaging>jar</packaging>
  8 |   <version>1.9</version>
  9 |   <name>darpc</name>
 10 |   <description>DaRPC (Data Center RPC) is a Java library for low latency Remote Procedure Call (RPC)</description>
 11 |   <url>http://github.com/zrlio/darpc</url>
 12 | 
 13 |   <licenses>
 14 |     <license>
 15 |       <name>The Apache License, Version 2.0</name>
 16 |       <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
 17 |     </license>
 18 |   </licenses>
 19 | 
 20 |   <developers>
 21 |     <developer>
 22 |       <name>Patrick Stuedi</name>
 23 |       <email>stu@zurich.ibm.com</email>
 24 |     </developer>
 25 |     <developer>
 26 |       <name>Jonas Pfefferle</name>
 27 |       <email>jpf@zurich.ibm.com</email>
 28 |     </developer>
 29 |     <developer>
 30 |       <name>Animesh Trivedi</name>
 31 |       <email>atr@zurich.ibm.com</email>
 32 |     </developer>
 33 |   </developers>
 34 | 
 35 |   <scm>
 36 |     <connection>scm:git:git://github.com/zrlio/darpc.git</connection>
 37 |     <developerConnection>scm:git:ssh://github.com:zrlio/darpc.git</developerConnection>
 38 |     <url>http://github.com/zrlio/darpc/tree/master</url>
 39 |   </scm>
 40 | 
 41 |   <distributionManagement>
 42 |     <snapshotRepository>
 43 |       <id>ossrh</id>
 44 |       <url>https://oss.sonatype.org/content/repositories/snapshots</url>
 45 |     </snapshotRepository>
 46 |   </distributionManagement>
 47 | 
 48 |   <dependencies>
 49 |     <dependency>
 50 |       <groupId>junit</groupId>
 51 |       <artifactId>junit</artifactId>
 52 |       <version>3.8.1</version>
 53 |       <scope>test</scope>
 54 |     </dependency>
 55 |     <dependency>
 56 |       <groupId>org.slf4j</groupId>
 57 |       <artifactId>slf4j-log4j12</artifactId>
 58 |       <version>1.7.12</version>
 59 |     </dependency>
 60 |     <dependency>
 61 |       <groupId>com.ibm.disni</groupId>
 62 |       <artifactId>disni</artifactId>
 63 |       <version>2.1</version>
 64 |     </dependency>
 65 |   </dependencies>
 66 | 
 67 |   <profiles>
 68 |     <profile>
 69 |         <id>release-sign-artifacts</id>
 70 |         <activation>
 71 |             <property>
 72 |                 <name>performRelease</name>
 73 |                 <value>true</value>
 74 |             </property>
 75 |         </activation>
 76 |         <build>
 77 |             <plugins>
 78 |                 <plugin>
 79 |                     <groupId>org.apache.maven.plugins</groupId>
 80 |                     <artifactId>maven-gpg-plugin</artifactId>
 81 |                     <version>1.5</version>
 82 |                     <executions>
 83 |                         <execution>
 84 |                             <id>sign-artifacts</id>
 85 |                             <phase>verify</phase>
 86 |                             <goals>
 87 |                                 <goal>sign</goal>
 88 |                             </goals>
 89 |                         </execution>
 90 |                     </executions>
 91 |                 </plugin>
 92 |             </plugins>
 93 |         </build>
 94 |     </profile>
 95 |   </profiles>
 96 | 
 97 | 	<build>
 98 |     <resources>
 99 |       <resource>
100 |         <directory>${project.basedir}</directory>
101 |         <targetPath>META-INF/</targetPath>
102 |         <includes>
103 |           <include>LICENSE</include>
104 |         </includes>
105 |       </resource>
106 |     </resources>
107 |     <plugins>
108 |       <plugin>
109 |         <groupId>org.apache.maven.plugins</groupId>
110 |         <artifactId>maven-jar-plugin</artifactId>
111 |         <version>2.6</version>
112 |         <executions>
113 |           <execution>
114 |             <goals>
115 |               <goal>test-jar</goal>
116 |             </goals>
117 |           </execution>
118 |         </executions>
119 |       </plugin>
120 |       <plugin>
121 |         <artifactId>maven-assembly-plugin</artifactId>
122 |         <executions>
123 |           <execution>
124 |             <id>jar-with-dependencies</id>
125 |             <phase>package</phase>
126 |             <goals>
127 |               <goal>single</goal>
128 |             </goals>
129 |             <configuration>
130 |               <descriptorRefs>
131 |                 <descriptorRef>jar-with-dependencies</descriptorRef>
132 |               </descriptorRefs>
133 |             </configuration>
134 |           </execution>
135 |         </executions>
136 |       </plugin>
137 |       <plugin>
138 |         <groupId>org.sonatype.plugins</groupId>
139 |         <artifactId>nexus-staging-maven-plugin</artifactId>
140 |         <version>1.6.7</version>
141 |         <extensions>true</extensions>
142 |         <configuration>
143 |           <serverId>ossrh</serverId>
144 |           <nexusUrl>https://oss.sonatype.org/</nexusUrl>
145 |           <autoReleaseAfterClose>false</autoReleaseAfterClose>
146 |         </configuration>
147 |       </plugin>
148 |       <plugin>
149 |         <groupId>org.apache.maven.plugins</groupId>
150 |         <artifactId>maven-source-plugin</artifactId>
151 |         <version>2.2.1</version>
152 |         <executions>
153 |           <execution>
154 |             <id>attach-sources</id>
155 |             <goals>
156 |               <goal>jar-no-fork</goal>
157 |             </goals>
158 |           </execution>
159 |         </executions>
160 |       </plugin>
161 |       <plugin>
162 |         <groupId>org.apache.maven.plugins</groupId>
163 |         <artifactId>maven-javadoc-plugin</artifactId>
164 |         <configuration>
165 |           <additionalparam>-Xdoclint:none</additionalparam>
166 |         </configuration>
167 |         <version>2.9.1</version>
168 |         <executions>
169 |           <execution>
170 |             <id>attach-javadocs</id>
171 |             <goals>
172 |               <goal>jar</goal>
173 |             </goals>
174 |           </execution>
175 |         </executions>
176 | 			</plugin>
177 |       <plugin>
178 |         <groupId>org.apache.rat</groupId>
179 |         <artifactId>apache-rat-plugin</artifactId>
180 |         <version>0.12</version>
181 |         <configuration>
182 |           <licenses>
183 |             <license implementation="org.apache.rat.analysis.license.SimplePatternBasedLicense">
184 |               <licenseFamilyCategory>AL20</licenseFamilyCategory>
185 |               <licenseFamilyName>Apache License, 2.0</licenseFamilyName>
186 |             </license>
187 |           </licenses>
188 |        </configuration>
189 |       </plugin>
190 |     </plugins>
191 |   </build>
192 | 
193 | </project>
194 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <!--
  2 | {% comment %}
  3 | 
  4 | Copyright (C) 2016-2018, IBM Corporation
  5 | 
  6 | Licensed to the Apache Software Foundation (ASF) under one or more
  7 | contributor license agreements.  See the NOTICE file distributed with
  8 | this work for additional information regarding copyright ownership.
  9 | The ASF licenses this file to You under the Apache License, Version 2.0
 10 | (the "License"); you may not use this file except in compliance with
 11 | the License.  You may obtain a copy of the License at
 12 | 
 13 |    http://www.apache.org/licenses/LICENSE-2.0
 14 | 
 15 | Unless required by applicable law or agreed to in writing, software
 16 | distributed under the License is distributed on an "AS IS" BASIS,
 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 18 | See the License for the specific language governing permissions and
 19 | limitations under the License.
 20 | {% endcomment %}
 21 | -->
 22 | 
 23 | # DaRPC: Data Center Remote Procedure Call
 24 | 
 25 | DaRPC is a Java library that provides ultra-low latency RPC for RDMA capable network interfaces. 
 26 | The unique features of DaRPC include:
 27 | 
 28 | * Ultra-low RPC latencies close to the raw hardware roundtrip latencies. 
 29 |   (Depending on the networking hardware and the CPU intensity of the RPC call, RPC latencies of 5-6 microseconds are possible)
 30 | * Efficient asynchronous and non-blocking RPC interfaces
 31 | * Fine grained control over network trade-offs (interrupts, polling, CPU affinities, etc.)
 32 | 
 33 | ## Building DaRPC
 34 | 
 35 | DaRPC is built using [Apache Maven](http://maven.apache.org/) and requires Java version 8 or higher.
 36 | To build DaRPC and its example programs, execute the following steps:
 37 | 
 38 | 1. Obtain a copy of DaRPC from [Github](https://github.com/zrlio/darpc)
 39 | 2. Make sure your local maven repo contains DiSNI, if not build DiSNI from [Github](https://github.com/zrlio/disni)
 40 | 2. Compile and package the source using: mvn -DskipTests install
 41 | 
 42 | ## How to Run a Simple Example
 43 | 
 44 | 1. After building DaRPC, make sure DaRPC and its dependencies are in the classpath (e.g., darpc-1.9-jar-with-dependencies.jar). Also add the DaRPC test jar (darpc-1.0-tests.jar) which includes the examples.
 45 | 2. Make sure libdisni is part of the LD_LIBRARY_PATH
 46 | 3. Make sure the RDMA network interface is configured and up on the test machines (run ibv\_devices to see the list of RDMA NICs). If your machine does not have RDMA hardware, you can also use SoftiWARP from [Github](https://github.com/zrlio/softiwarp). 
 47 | 4. Run the server\: java com.ibm.darpc.examples.server.DaRPCServer -a \<server IP\>
 48 | 5. Run the client\: java com.ibm.darpc.examples.client.DaRPCClient -a \<server IP\> 
 49 | 
 50 | ## Maven Integration
 51 | 
 52 | To use DaRPC in your maven application use the following snippet in your pom.xml file:
 53 | 
 54 |     <dependency>
 55 |       <groupId>com.ibm.darpc</groupId>
 56 |       <artifactId>darpc</artifactId>
 57 |       <version>1.9</version>
 58 |     </dependency>
 59 | 
 60 | ## Basic Steps Required to Implement your own RPC Service
 61 | 
 62 | Define the RPC request and response messages
 63 | 
 64 | ```
 65 | 	public class RdmaRpcRequest implements RdmaRpcMessage {
 66 | 		public static int SERIALIZED_SIZE = 16;
 67 | 		
 68 | 		private int cmd;
 69 | 		private int param;
 70 | 		private long time;
 71 | 		
 72 | 		public RdmaRpcRequest(){
 73 | 		}
 74 | 
 75 | 		@Override
 76 | 		public void update(ByteBuffer buffer) {
 77 | 			cmd = buffer.getInt();
 78 | 			param = buffer.getInt();
 79 | 			time = buffer.getLong();
 80 | 		}
 81 | 
 82 | 		@Override
 83 | 		public int write(ByteBuffer buffer) {
 84 | 			buffer.putInt(cmd);
 85 | 			buffer.putInt(param);
 86 | 			buffer.putLong(time);
 87 | 			
 88 | 			return SERIALIZED_SIZE;
 89 | 		}
 90 | 	}
 91 | ```
 92 | Define the protocol
 93 | ```
 94 | 	public class RdmaRpcProtocol extends RdmaRpcService<RdmaRpcRequest, RdmaRpcResponse> {
 95 | 		@Override
 96 | 		public RdmaRpcRequest createRequest() {
 97 | 			return new RdmaRpcRequest();
 98 | 		}
 99 | 
100 | 		@Override
101 | 		public RdmaRpcResponse createResponse() {
102 | 			return new RdmaRpcResponse();
103 | 		}
104 | 	}
105 | ```
106 | Define the actual RPC service
107 | ```
108 | 	public class RdmaRpcService extends RdmaRpcProtocol {
109 | 		public void processServerEvent(RpcServerEvent<RdmaRpcRequest, RdmaRpcResponse> event) throws IOException {
110 | 			RdmaRpcProtocol.RdmaRpcRequest request = event.getRequest();
111 | 			RdmaRpcProtocol.RdmaRpcResponse response = event.getResponse();
112 | 			response.setName(request.getParam() + 1);
113 | 			event.triggerResponse();
114 | 		}
115 | 	}
116 | ```
117 | Start the server
118 | ```
119 | 	RdmaRpcService rpcService = new RdmaRpcService();
120 | 	RpcActiveEndpointGroup<RdmaRpcRequest, RdmaRpcResponse> rpcGroup = RpcActiveEndpointGroup.createDefault(rpcService);
121 | 	RdmaServerEndpoint<RpcClientEndpoint<RdmaRpcRequest, RdmaRpcResponse>> rpcEndpoint = rpcGroup.createServerEndpoint();
122 | 	rpcEndpoint.bind(addr);
123 | 	while(true){
124 | 		serverEp.accept();
125 | 	}
126 | ```	
127 | Call the RPC service from a client
128 | ```
129 | 	RpcEndpointGroup rpcGroup = RpcPassiveEndpointGroup.createDefault();
130 | 	RpcEndpoint rpcEndpoint = rpcGroup.createEndpoint();
131 | 	rpcEndpoint.connect(address);
132 | 	RpcStream<RdmaRpcRequest, RdmaRpcResponse> stream = rpcEndpoint.createStream();
133 | 	RdmaRpcRequest request = new RdmaRpcRequest();
134 | 	request.setParam(77);
135 | 	RdmaRpcResponse response = new RdmaRpcResponse();
136 | 	RpcFuture<RdmaRpcRequest, RdmaRpcResponse> future = stream.request(request, response);
137 | 	...
138 | 	response = future.get();
139 | 	System.out.println("RPC completed, return value " + response.getReturnValue()); //should print 78
140 | ```
141 | ## Choosing the RpcEndpointGroup 
142 | 
143 | RpcEndpointGroups are containers and factories for RPC connections (RpcEndpoint). There are two types of groups available and which type works best depends on the application. The RpcActiveEndpointGroup actively processes network events caused by RDMA messages being transmitted or received. The RpcPassiveEndpointGroup processes network events directly in the process context of the application. As such, the passive mode has typically lower latency but may suffer from contention to per-connection hardware resources in case of large numbers of threads. The active mode, on the other hand, is more robust under large numbers of threads, but has higher latencies. Applications can control the cores and the NUMA affinities that are used for event processing. For server-side RPC processing the active mode is always used. The DaRPC paper discusses the trade-offs between active and passive RPC processing in more detail. 
144 | 
145 | ## Contributions
146 | 
147 | PRs are always welcome. Please fork, and make necessary modifications 
148 | you propose, and let us know. 
149 | 
150 | ## Contact 
151 | 
152 | If you have questions or suggestions, feel free to post at:
153 | 
154 | https://groups.google.com/forum/#!forum/zrlio-users
155 | 
156 | or email: zrlio-users@googlegroups.com
157 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCServerGroup.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc;
 23 | 
 24 | import java.io.IOException;
 25 | import java.util.concurrent.ConcurrentHashMap;
 26 | 
 27 | import org.slf4j.Logger;
 28 | import org.slf4j.LoggerFactory;
 29 | 
 30 | import com.ibm.disni.RdmaCqProvider;
 31 | import com.ibm.disni.RdmaEndpointFactory;
 32 | import com.ibm.disni.verbs.IbvCQ;
 33 | import com.ibm.disni.verbs.IbvContext;
 34 | import com.ibm.disni.verbs.IbvQP;
 35 | import com.ibm.disni.verbs.RdmaCmId;
 36 | 
 37 | public class DaRPCServerGroup<R extends DaRPCMessage, T extends DaRPCMessage> extends DaRPCEndpointGroup<DaRPCServerEndpoint<R,T>, R, T> {
 38 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
 39 | 	
 40 | 	private ConcurrentHashMap<Integer, DaRPCInstance<R,T>> deviceInstance;
 41 | 	private DaRPCResourceManager resourceManager;
 42 | 	private long[] computeAffinities;
 43 | 	private long[] resourceAffinities;
 44 | 	private int currentCluster;
 45 | 	private int nbrOfClusters;
 46 | 	private DaRPCService<R, T> rpcService;
 47 | 	private boolean polling;
 48 | 	private int pollSize;
 49 | 	private int clusterSize;
 50 | 	
 51 | 	public static <R extends DaRPCMessage, T extends DaRPCMessage> DaRPCServerGroup<R, T> createServerGroup(DaRPCService<R, T> rpcService, long[] clusterAffinities, int timeout, int maxinline, boolean polling, int recvQueue, int sendQueue, int pollSize, int clusterSize) throws Exception {
 52 | 		DaRPCServerGroup<R,T> group = new DaRPCServerGroup<R,T>(rpcService, clusterAffinities, timeout, maxinline, polling, recvQueue, sendQueue, pollSize, clusterSize);
 53 | 		group.init(new RpcServerFactory<R,T>(group));
 54 | 		return group;
 55 | 	}
 56 | 
 57 | 	private DaRPCServerGroup(DaRPCService<R, T> rpcService, long[] clusterAffinities, int timeout, int maxinline, boolean polling, int recvQueue, int sendQueue, int pollSize, int clusterSize) throws Exception {
 58 | 		super(rpcService, timeout, maxinline, recvQueue, sendQueue);
 59 | 		
 60 | 		this.rpcService = rpcService;
 61 | 		deviceInstance = new ConcurrentHashMap<Integer, DaRPCInstance<R,T>>();
 62 | 		this.computeAffinities = clusterAffinities;
 63 | 		this.resourceAffinities = clusterAffinities;		
 64 | 		this.nbrOfClusters = computeAffinities.length;
 65 | 		this.currentCluster = 0;
 66 | 		resourceManager = new DaRPCResourceManager(resourceAffinities, timeout);
 67 | 		this.polling = polling;
 68 | 		this.pollSize = pollSize;
 69 | 		this.clusterSize = clusterSize;
 70 | 	}
 71 | 	
 72 | 	public RdmaCqProvider createCqProvider(DaRPCServerEndpoint<R,T> endpoint) throws IOException {
 73 | 		logger.info("setting up cq processor (multicore)");
 74 | 		IbvContext context = endpoint.getIdPriv().getVerbs();
 75 | 		if (context == null) {
 76 | 			throw new IOException("setting up cq processor, no context found");
 77 | 		}
 78 | 		DaRPCInstance<R,T> rpcInstance = null;
 79 | 		int key = context.getCmd_fd();
 80 | 		if (!deviceInstance.containsKey(key)) {
 81 | 			int cqSize = (this.recvQueueSize() + this.sendQueueSize())*clusterSize;
 82 | 			rpcInstance = new DaRPCInstance<R,T>(context, cqSize, this.pollSize, computeAffinities, this.getTimeout(), polling);
 83 | 			deviceInstance.put(context.getCmd_fd(), rpcInstance);
 84 | 		}
 85 | 		rpcInstance = deviceInstance.get(context.getCmd_fd());
 86 | 		DaRPCCluster<R,T> cqProcessor = rpcInstance.getProcessor(endpoint.clusterId());
 87 | 		return cqProcessor;
 88 | 	}	
 89 | 	
 90 | 	public IbvQP createQpProvider(DaRPCServerEndpoint<R,T> endpoint) throws IOException{
 91 | 		logger.info("setting up QP");
 92 | 		DaRPCCluster<R,T>  cqProcessor = this.lookupCqProcessor(endpoint);
 93 | 		IbvCQ cq = cqProcessor.getCQ();
 94 | 		IbvQP qp = this.createQP(endpoint.getIdPriv(), endpoint.getPd(), cq);	
 95 | 		cqProcessor.registerQP(qp.getQp_num(), endpoint);
 96 | 		return qp;
 97 | 	}		
 98 | 	
 99 | 	public void allocateResources(DaRPCServerEndpoint<R,T> endpoint) throws Exception {
100 | 		resourceManager.allocateResources(endpoint);
101 | 	}
102 | 	
103 | 	synchronized int newClusterId() {
104 | 		int newClusterId = currentCluster;
105 | 		currentCluster = (currentCluster + 1) % nbrOfClusters;
106 | 		return newClusterId;
107 | 	}
108 | 	
109 | 	protected synchronized DaRPCCluster<R,T> lookupCqProcessor(DaRPCServerEndpoint<R,T> endpoint) throws IOException{
110 | 		IbvContext context = endpoint.getIdPriv().getVerbs();
111 | 		if (context == null) {
112 | 			throw new IOException("setting up cq processor, no context found");
113 | 		}
114 | 		DaRPCInstance<R,T> rpcInstance = null;
115 | 		int key = context.getCmd_fd();
116 | 		if (!deviceInstance.containsKey(key)) {
117 | 			return null;
118 | 		} else {
119 | 			rpcInstance = deviceInstance.get(context.getCmd_fd());
120 | 			DaRPCCluster<R,T> cqProcessor = rpcInstance.getProcessor(endpoint.clusterId());
121 | 			return cqProcessor;
122 | 		}
123 | 	}	
124 | 	
125 | 	public void close() throws IOException, InterruptedException {
126 | 		super.close();
127 | 		for (DaRPCInstance<R,T> rpcInstance : deviceInstance.values()){
128 | 			rpcInstance.close();
129 | 		}		
130 | 		resourceManager.close();
131 | 		logger.info("rpc group down");
132 | 	}	
133 | 	
134 | 	public R createRequest() {
135 | 		return rpcService.createRequest();
136 | 	}
137 | 
138 | 	public T createResponse() {
139 | 		return rpcService.createResponse();
140 | 	}
141 | 	
142 | 	public void processServerEvent(DaRPCServerEvent<R,T> event) throws IOException {
143 | 		rpcService.processServerEvent(event);
144 | 	}
145 | 	
146 | 	public void open(DaRPCServerEndpoint<R,T> endpoint){
147 | 		rpcService.open(endpoint);
148 | 	}	
149 | 	
150 | 	public void close(DaRPCServerEndpoint<R,T> endpoint){
151 | 		rpcService.close(endpoint);
152 | 	}
153 | 	
154 | 	public DaRPCService<? extends DaRPCMessage, ? extends DaRPCMessage> getRpcService() {
155 | 		return rpcService;
156 | 	}
157 | 	
158 | 	public static class RpcServerFactory<R extends DaRPCMessage, T extends DaRPCMessage> implements RdmaEndpointFactory<DaRPCServerEndpoint<R,T>> {
159 | 		private DaRPCServerGroup<R, T> group;
160 | 		
161 | 		public RpcServerFactory(DaRPCServerGroup<R, T> group){
162 | 			this.group = group;
163 | 		}
164 | 		
165 | 		@Override
166 | 		public DaRPCServerEndpoint<R,T> createEndpoint(RdmaCmId id, boolean serverSide) throws IOException {
167 | 			return new DaRPCServerEndpoint<R,T>(group, id, serverSide);
168 | 		}
169 | 	}	
170 | }
171 | 


--------------------------------------------------------------------------------
/src/main/java/com/ibm/darpc/DaRPCEndpoint.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc;
 23 | 
 24 | import java.io.IOException;
 25 | import java.nio.ByteBuffer;
 26 | import java.util.ArrayList;
 27 | import java.util.LinkedList;
 28 | import java.util.concurrent.ArrayBlockingQueue;
 29 | import java.util.concurrent.ConcurrentHashMap;
 30 | import java.util.concurrent.atomic.AtomicLong;
 31 | 
 32 | import org.slf4j.Logger;
 33 | import org.slf4j.LoggerFactory;
 34 | 
 35 | import com.ibm.disni.verbs.*;
 36 | import com.ibm.disni.util.MemoryUtils;
 37 | import com.ibm.disni.*;
 38 | 
 39 | public abstract class DaRPCEndpoint<R extends DaRPCMessage, T extends DaRPCMessage> extends RdmaEndpoint {
 40 | 	private static final Logger logger = LoggerFactory.getLogger("com.ibm.darpc");
 41 | 	private static final int headerSize = 4;
 42 | 	
 43 | 	public abstract void dispatchReceive(ByteBuffer buffer, int ticket, int recvIndex) throws IOException;
 44 | 	public abstract void dispatchSend(int ticket) throws IOException;
 45 | 	
 46 | 	private DaRPCEndpointGroup<? extends DaRPCEndpoint<R,T>, R, T> rpcGroup;
 47 | 	private ByteBuffer dataBuffer;
 48 | 	private IbvMr dataMr;
 49 | 	private ByteBuffer receiveBuffer;
 50 | 	private ByteBuffer sendBuffer;
 51 | 	private ByteBuffer[] recvBufs;
 52 | 	private ByteBuffer[] sendBufs;
 53 | 	private SVCPostRecv[] recvCall;
 54 | 	private SVCPostSend[] sendCall;
 55 | 	private ConcurrentHashMap<Integer, SVCPostSend> pendingPostSend;
 56 | 	private ArrayBlockingQueue<SVCPostSend> freePostSend;
 57 | 	private AtomicLong ticketCount;
 58 | 	private int pipelineLength;
 59 | 	private int payloadSize;
 60 | 	private int rawBufferSize;
 61 | 	private int maxinline;
 62 | 	private AtomicLong messagesSent;
 63 | 	private AtomicLong messagesReceived;
 64 | 	
 65 | 	
 66 | 	public DaRPCEndpoint(DaRPCEndpointGroup<? extends DaRPCEndpoint<R,T>, R, T> endpointGroup, RdmaCmId idPriv, boolean serverSide) throws IOException {
 67 | 		super(endpointGroup, idPriv, serverSide);
 68 | 		this.rpcGroup = endpointGroup;
 69 | 		this.maxinline = rpcGroup.getMaxInline();
 70 | 		this.payloadSize = rpcGroup.getBufferSize();
 71 | 		this.rawBufferSize = headerSize + this.payloadSize;
 72 | 		this.pipelineLength = rpcGroup.recvQueueSize();
 73 | 		this.freePostSend = new ArrayBlockingQueue<SVCPostSend>(pipelineLength);
 74 | 		this.pendingPostSend = new ConcurrentHashMap<Integer, SVCPostSend>();
 75 | 		this.recvBufs = new ByteBuffer[pipelineLength];
 76 | 		this.sendBufs = new ByteBuffer[pipelineLength];
 77 | 		this.recvCall = new SVCPostRecv[pipelineLength];
 78 | 		this.sendCall = new SVCPostSend[pipelineLength];
 79 | 		this.ticketCount = new AtomicLong(0);
 80 | 		this.messagesSent = new AtomicLong(0);
 81 | 		this.messagesReceived = new AtomicLong(0);
 82 | 		logger.info("RPC client endpoint, with payload buffer size = " + payloadSize + ", pipeline " + pipelineLength);
 83 | 	}
 84 | 	
 85 | 	public void init() throws IOException {
 86 | 		int sendBufferOffset = pipelineLength * rawBufferSize;
 87 | 
 88 | 		/* Main data buffer for sends and receives. Will be split into two regions,
 89 | 		 * one for sends and one for receives.
 90 | 		 */
 91 | 		dataBuffer = ByteBuffer.allocateDirect(pipelineLength * rawBufferSize * 2);
 92 | 		/* Only do one memory registration with the IB card. */
 93 | 		dataMr = registerMemory(dataBuffer).execute().free().getMr();
 94 | 
 95 | 		/* Receive memory region is the first half of the main buffer. */
 96 | 		dataBuffer.limit(dataBuffer.position() + sendBufferOffset);
 97 | 		receiveBuffer = dataBuffer.slice();
 98 | 
 99 | 		/* Send memory region is the second half of the main buffer. */
100 | 		dataBuffer.position(sendBufferOffset);
101 | 		dataBuffer.limit(dataBuffer.position() + sendBufferOffset);
102 | 		sendBuffer = dataBuffer.slice();
103 | 
104 | 		for(int i = 0; i < pipelineLength; i++) {
105 | 			/* Create single receive buffers within the receive region in form of slices. */
106 | 			receiveBuffer.position(i * rawBufferSize);
107 | 			receiveBuffer.limit(receiveBuffer.position() + rawBufferSize);
108 | 			recvBufs[i] = receiveBuffer.slice();
109 | 
110 | 			/* Create single send buffers within the send region in form of slices. */
111 | 			sendBuffer.position(i * rawBufferSize);
112 | 			sendBuffer.limit(sendBuffer.position() + rawBufferSize);
113 | 			sendBufs[i] = sendBuffer.slice();
114 | 
115 | 			this.recvCall[i] = setupRecvTask(i);
116 | 			this.sendCall[i] = setupSendTask(i);
117 | 			freePostSend.add(sendCall[i]);
118 | 			recvCall[i].execute();
119 | 		}
120 | 	}
121 | 
122 | 	@Override
123 | 	public synchronized void close() throws IOException, InterruptedException {
124 | 		super.close();
125 | 		deregisterMemory(dataMr);
126 | 	}	
127 | 	
128 | 	public long getMessagesSent() {
129 | 		return messagesSent.get();
130 | 	}
131 | 	
132 | 	public long getMessagesReceived() {
133 | 		return messagesReceived.get();
134 | 	}
135 | 	
136 | 	protected boolean sendMessage(DaRPCMessage message, int ticket) throws IOException {
137 | 		SVCPostSend postSend = freePostSend.poll();
138 | 		if (postSend != null){
139 | 			int index = (int) postSend.getWrMod(0).getWr_id();
140 | 			sendBufs[index].putInt(0, ticket);
141 | 			sendBufs[index].position(4);
142 | 			int written = 4 + message.write(sendBufs[index]);
143 | 			postSend.getWrMod(0).getSgeMod(0).setLength(written);
144 | 			postSend.getWrMod(0).setSend_flags(IbvSendWR.IBV_SEND_SIGNALED);
145 | 			if (written <= maxinline) {
146 | 				postSend.getWrMod(0).setSend_flags(postSend.getWrMod(0).getSend_flags() | IbvSendWR.IBV_SEND_INLINE);
147 | 			} 
148 | 			pendingPostSend.put(ticket, postSend);
149 | 			postSend.execute();
150 | 			messagesSent.incrementAndGet();
151 | 			return true;
152 | 		} else {
153 | 			return false;
154 | 		}
155 | 	}
156 | 	
157 | 	protected void postRecv(int index) throws IOException {
158 | 		recvCall[index].execute();
159 | 	}	
160 | 	
161 | 	public void freeSend(int ticket) throws IOException {
162 | 		SVCPostSend sendOperation = pendingPostSend.remove(ticket);
163 | 		if (sendOperation == null) {
164 | 			throw new IOException("no pending ticket " + ticket + ", current ticket count " + ticketCount.get());
165 | 		}
166 | 		this.freePostSend.add(sendOperation);
167 | 	}	
168 | 	
169 | 	public void dispatchCqEvent(IbvWC wc) throws IOException {
170 | 		if (wc.getStatus() == 5){
171 | 			//flush
172 | 			return;
173 | 		} else if (wc.getStatus() != 0){
174 | 			throw new IOException("Faulty operation! wc.status " + wc.getStatus());
175 | 		} 	
176 | 		
177 | 		if (wc.getOpcode() == 128){
178 | 			//receiving a message
179 | 			int index = (int) wc.getWr_id();
180 | 			ByteBuffer recvBuffer = recvBufs[index];
181 | 			int ticket = recvBuffer.getInt(0);
182 | 			recvBuffer.position(4);
183 | 			dispatchReceive(recvBuffer, ticket, index);
184 | 		} else if (wc.getOpcode() == 0) {
185 | 			//send completion
186 | 			int index = (int) wc.getWr_id();
187 | 			ByteBuffer sendBuffer = sendBufs[index];
188 | 			int ticket = sendBuffer.getInt(0);
189 | 			dispatchSend(ticket);
190 | 		} else {
191 | 			throw new IOException("Unkown opcode " + wc.getOpcode());
192 | 		}		
193 | 	}	
194 | 	
195 | 	private SVCPostSend setupSendTask(int wrid) throws IOException {
196 | 		ArrayList<IbvSendWR> sendWRs = new ArrayList<IbvSendWR>(1);
197 | 		LinkedList<IbvSge> sgeList = new LinkedList<IbvSge>();
198 | 
199 | 		IbvSge sge = new IbvSge();
200 | 		sge.setAddr(MemoryUtils.getAddress(sendBufs[wrid]));
201 | 		sge.setLength(rawBufferSize);
202 | 		sge.setLkey(dataMr.getLkey());
203 | 		sgeList.add(sge);
204 | 
205 | 		IbvSendWR sendWR = new IbvSendWR();
206 | 		sendWR.setSg_list(sgeList);
207 | 		sendWR.setWr_id(wrid);
208 | 		sendWRs.add(sendWR);
209 | 		sendWR.setSend_flags(IbvSendWR.IBV_SEND_SIGNALED);
210 | 		sendWR.setOpcode(IbvSendWR.IbvWrOcode.IBV_WR_SEND.ordinal());
211 | 
212 | 		return postSend(sendWRs);
213 | 	}
214 | 
215 | 	private SVCPostRecv setupRecvTask(int wrid) throws IOException {
216 | 		ArrayList<IbvRecvWR> recvWRs = new ArrayList<IbvRecvWR>(1);
217 | 		LinkedList<IbvSge> sgeList = new LinkedList<IbvSge>();
218 | 
219 | 		IbvSge sge = new IbvSge();
220 | 		sge.setAddr(MemoryUtils.getAddress(recvBufs[wrid]));
221 | 		sge.setLength(rawBufferSize);
222 | 		sge.setLkey(dataMr.getLkey());
223 | 		sgeList.add(sge);
224 | 
225 | 		IbvRecvWR recvWR = new IbvRecvWR();
226 | 		recvWR.setWr_id(wrid);
227 | 		recvWR.setSg_list(sgeList);
228 | 		recvWRs.add(recvWR);
229 | 
230 | 		return postRecv(recvWRs);
231 | 	}
232 | }
233 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/src/test/java/com/ibm/darpc/examples/client/DaRPCClient.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DaRPC: Data Center Remote Procedure Call
  3 |  *
  4 |  * Author: Patrick Stuedi <stu@zurich.ibm.com>
  5 |  *
  6 |  * Copyright (C) 2016-2018, IBM Corporation
  7 |  *
  8 |  * Licensed under the Apache License, Version 2.0 (the "License");
  9 |  * you may not use this file except in compliance with the License.
 10 |  * You may obtain a copy of the License at
 11 |  *
 12 |  *     http://www.apache.org/licenses/LICENSE-2.0
 13 |  *
 14 |  * Unless required by applicable law or agreed to in writing, software
 15 |  * distributed under the License is distributed on an "AS IS" BASIS,
 16 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 17 |  * See the License for the specific language governing permissions and
 18 |  * limitations under the License.
 19 |  *
 20 |  */
 21 | 
 22 | package com.ibm.darpc.examples.client;
 23 | 
 24 | import java.io.FileOutputStream;
 25 | import java.net.InetSocketAddress;
 26 | import java.nio.ByteBuffer;
 27 | import java.nio.channels.FileChannel;
 28 | import java.util.concurrent.ArrayBlockingQueue;
 29 | import java.util.concurrent.TimeUnit;
 30 | 
 31 | import org.apache.commons.cli.CommandLine;
 32 | import org.apache.commons.cli.CommandLineParser;
 33 | import org.apache.commons.cli.DefaultParser;
 34 | import org.apache.commons.cli.HelpFormatter;
 35 | import org.apache.commons.cli.Option;
 36 | import org.apache.commons.cli.Options;
 37 | import org.apache.commons.cli.ParseException;
 38 | 
 39 | import com.ibm.darpc.DaRPCClientEndpoint;
 40 | import com.ibm.darpc.DaRPCClientGroup;
 41 | import com.ibm.darpc.DaRPCEndpoint;
 42 | import com.ibm.darpc.DaRPCFuture;
 43 | import com.ibm.darpc.DaRPCStream;
 44 | import com.ibm.darpc.examples.protocol.RdmaRpcProtocol;
 45 | import com.ibm.darpc.examples.protocol.RdmaRpcRequest;
 46 | import com.ibm.darpc.examples.protocol.RdmaRpcResponse;
 47 | import com.ibm.disni.util.*;
 48 | 
 49 | public class DaRPCClient {
 50 | 	public static enum BenchmarkType {
 51 | 		UNDEFINED
 52 | 	};	
 53 | 	
 54 | 	public static class ClientThread implements Runnable {
 55 | 		public static final int FUTURE_POLL = 0;
 56 | 		public static final int STREAM_POLL = 1;
 57 | 		public static final int FUTURE_TAKE = 2;
 58 | 		public static final int STREAM_TAKE = 3;
 59 | 		public static final int BATCH_STREAM_TAKE = 4;
 60 | 		public static final int BATCH_STREAM_POLL = 5;
 61 | 		
 62 | 		private DaRPCClientEndpoint<RdmaRpcRequest, RdmaRpcResponse> clientEp;
 63 | 		private int loop;
 64 | 		private int queryMode;
 65 | 		private int clienttimeout;
 66 | 		private ArrayBlockingQueue<RdmaRpcResponse> freeResponses;
 67 | 		
 68 | 		protected double throughput;
 69 | 		protected double latency;
 70 | 		protected double readOps;
 71 | 		protected double writeOps;
 72 | 		protected double errorOps;		
 73 | 		
 74 | 		public ClientThread(DaRPCClientEndpoint<RdmaRpcRequest, RdmaRpcResponse> clientEp, int loop, InetSocketAddress address, int mode, int rpcpipeline, int clienttimeout){
 75 | 			this.clientEp = clientEp;
 76 | 			this.loop = loop;
 77 | 			this.queryMode = mode;
 78 | 			this.clienttimeout = clienttimeout;
 79 | 			this.freeResponses = new ArrayBlockingQueue<RdmaRpcResponse>(rpcpipeline);
 80 | 			for (int i = 0; i < rpcpipeline; i++){
 81 | 				RdmaRpcResponse response = new RdmaRpcResponse();
 82 | 				freeResponses.add(response);
 83 | 			}	
 84 | 		}
 85 | 		
 86 | 		@Override
 87 | 		public void run() {
 88 | 			try {
 89 | 				DaRPCStream<RdmaRpcRequest, RdmaRpcResponse> stream = clientEp.createStream();
 90 | 				RdmaRpcRequest request = new RdmaRpcRequest();
 91 | 				boolean streamMode = (queryMode == STREAM_POLL) || (queryMode == STREAM_TAKE) || (queryMode == BATCH_STREAM_TAKE) || (queryMode == BATCH_STREAM_POLL);
 92 | 				int issued = 0;
 93 | 				int consumed = 0;
 94 | 				for (;  issued < loop; issued++) {
 95 | 					while(freeResponses.isEmpty()){
 96 | 						DaRPCFuture<RdmaRpcRequest, RdmaRpcResponse> future = stream.poll();
 97 | 						if (future != null){
 98 | 							freeResponses.add(future.getReceiveMessage());						
 99 | 							consumed++;
100 | 						}
101 | 					}
102 | 					
103 | 					request.setParam(issued);
104 | 					RdmaRpcResponse response = freeResponses.poll();
105 | 					DaRPCFuture<RdmaRpcRequest, RdmaRpcResponse> future = stream.request(request, response, streamMode);
106 | 					
107 | 					switch (queryMode) {
108 | 					case FUTURE_POLL:
109 | 						while (!future.isDone()) {
110 | 						}
111 | 						consumed++;
112 | //						System.out.println("i " + issued + ", response " + future.getReceiveMessage().toString());
113 | 						freeResponses.add(future.getReceiveMessage());
114 | 						break;
115 | 					case STREAM_POLL:
116 | 						future = stream.poll();
117 | 						while (future == null) {
118 | 							future = stream.poll();
119 | 						}
120 | 						consumed++;
121 | 						freeResponses.add(future.getReceiveMessage());
122 | 						break;		
123 | 					case FUTURE_TAKE:
124 | 						future.get(clienttimeout, TimeUnit.MILLISECONDS);
125 | 						consumed++;
126 | 						freeResponses.add(future.getReceiveMessage());
127 | 						break;						
128 | 					case STREAM_TAKE:
129 | 						future = stream.take(clienttimeout);
130 | 						consumed++;
131 | 						freeResponses.add(future.getReceiveMessage());
132 | 						break;
133 | 					case BATCH_STREAM_TAKE:
134 | 						break;
135 | 					case BATCH_STREAM_POLL:
136 | 						break;						
137 | 					}
138 | 				}
139 | 				while (consumed < issued){
140 | 					DaRPCFuture<RdmaRpcRequest, RdmaRpcResponse> future = stream.take();
141 | //					System.out.println("response " + future.getReceiveMessage().toString());
142 | 					consumed++;
143 | 					freeResponses.add(future.getReceiveMessage());
144 | 				}
145 | 			} catch(Exception e){
146 | 				e.printStackTrace();
147 | 			}
148 | 		}
149 | 
150 | 		public void close() throws Exception {
151 | 			clientEp.close();
152 | 		}
153 | 		
154 | 		public double getThroughput() {
155 | 			return throughput;
156 | 		}
157 | 
158 | 		public double getLatency() {
159 | 			return this.latency;
160 | 		}
161 | 
162 | 		public double getReadOps() {
163 | 			return this.readOps;
164 | 		}
165 | 
166 | 		public double getWriteOps() {
167 | 			return this.writeOps;
168 | 		}
169 | 
170 | 		public double getErrorOps() {
171 | 			return this.errorOps;
172 | 		}	
173 | 		
174 | 		public double getOps(){
175 | 			return loop;
176 | 		}		
177 | 	}
178 | 	
179 | 	public void launch(String[] args) throws Exception {
180 | 		String host = ""; 
181 | 		int size = 24;
182 | 		int loop = 100;
183 | 		int threadCount = 1;
184 | 		int mode = ClientThread.FUTURE_POLL;
185 | 		int batchSize = 16;
186 | 		int connections = 1;
187 | 		int clienttimeout = 3000;
188 | 		int maxinline = 0;
189 | 		int recvQueue = batchSize;
190 | 		int sendQueue = batchSize;
191 | 
192 | 		Option addressOption = Option.builder("a").required().desc("server address").hasArg().build();
193 | 		Option loopOption = Option.builder("k").desc("loop count").hasArg().build();
194 | 		Option threadCountOption = Option.builder("n").desc("thread count").hasArg().build();
195 | 		Option modeOption = Option.builder("m").desc("mode").hasArg().build();
196 | 		Option batchSizeOption = Option.builder("b").desc("batch size").hasArg().build();
197 | 		Option connectionsOption = Option.builder("c").desc("number of connections").hasArg().build();
198 | 		Option clienttimeoutOption = Option.builder("t").desc("client timeout").hasArg().build();
199 | 		Option maxinlineOption = Option.builder("i").desc("max inline data").hasArg().build();
200 | 		Option sendQueueOption = Option.builder("s").desc("send queue").hasArg().build();
201 | 		Option recvQueueOption = Option.builder("r").desc("receive queue").hasArg().build();
202 | 		Option serializedSizeOption = Option.builder("l").desc("serialized size").hasArg().build();
203 | 		Options options = new Options();
204 | 		options.addOption(addressOption);
205 | 		options.addOption(loopOption);
206 | 		options.addOption(threadCountOption);
207 | 		options.addOption(modeOption);
208 | 		options.addOption(batchSizeOption);
209 | 		options.addOption(connectionsOption);
210 | 		options.addOption(clienttimeoutOption);
211 | 		options.addOption(maxinlineOption);
212 | 		options.addOption(sendQueueOption);
213 | 		options.addOption(recvQueueOption);
214 | 		options.addOption(serializedSizeOption);
215 | 		CommandLineParser parser = new DefaultParser();
216 | 		
217 | 		try {
218 | 			CommandLine line = parser.parse(options, args);
219 | 			host = line.getOptionValue(addressOption.getOpt());
220 | 
221 | 			if (line.hasOption(loopOption.getOpt())) {
222 | 				loop = Integer.parseInt(line.getOptionValue(loopOption.getOpt()));
223 | 			}
224 | 			if (line.hasOption(threadCountOption.getOpt())) {
225 | 				threadCount = Integer.parseInt(line.getOptionValue(threadCountOption.getOpt()));
226 | 			}
227 | 			if (line.hasOption(modeOption.getOpt())) {
228 | 				String _mode = line.getOptionValue(modeOption.getOpt());
229 | 				if (_mode.equalsIgnoreCase("future-poll")) {
230 | 					mode = ClientThread.FUTURE_POLL;
231 | 				} else if (_mode.equalsIgnoreCase("stream-poll")) {
232 | 					mode = ClientThread.STREAM_POLL;
233 | 				} else if (_mode.equalsIgnoreCase("future-take")) {
234 | 					mode = ClientThread.FUTURE_TAKE;
235 | 				} else if (_mode.equalsIgnoreCase("stream-take")) {
236 | 					mode = ClientThread.STREAM_TAKE;
237 | 				} else if (_mode.equalsIgnoreCase("batch-stream-take")) {
238 | 					mode = ClientThread.BATCH_STREAM_TAKE;
239 | 				} else if (_mode.equalsIgnoreCase("batch-stream-poll")) {
240 | 					mode = ClientThread.BATCH_STREAM_POLL;
241 | 				}
242 | 			}
243 | 			if (line.hasOption(batchSizeOption.getOpt())) {
244 | 				batchSize = Integer.parseInt(line.getOptionValue(batchSizeOption.getOpt()));
245 | 			}
246 | 			if (line.hasOption(connectionsOption.getOpt())) {
247 | 				connections = Integer.parseInt(line.getOptionValue(connectionsOption.getOpt()));
248 | 			}
249 | 			if (line.hasOption(clienttimeoutOption.getOpt())) {
250 | 				clienttimeout = Integer.parseInt(line.getOptionValue(clienttimeoutOption.getOpt()));
251 | 			}
252 | 			if (line.hasOption(maxinlineOption.getOpt())) {
253 | 				maxinline = Integer.parseInt(line.getOptionValue(maxinlineOption.getOpt()));
254 | 			}
255 | 			if (line.hasOption(recvQueueOption.getOpt())) {
256 | 				recvQueue = Integer.parseInt(line.getOptionValue(recvQueueOption.getOpt()));
257 | 			}
258 | 			if (line.hasOption(sendQueueOption.getOpt())) {
259 | 				sendQueue = Integer.parseInt(line.getOptionValue(sendQueueOption.getOpt()));
260 | 			}
261 | 			if (line.hasOption(serializedSizeOption.getOpt())) {
262 | 				RdmaRpcRequest.SERIALIZED_SIZE = Integer.parseInt(line.getOptionValue(serializedSizeOption.getOpt()));
263 | 				RdmaRpcResponse.SERIALIZED_SIZE = RdmaRpcRequest.SERIALIZED_SIZE;
264 | 			}
265 | 		} catch (ParseException e) {
266 | 			HelpFormatter formatter = new HelpFormatter();
267 | 			formatter.printHelp("DaRPCClient", options);
268 | 			System.exit(-1); // undefined option
269 | 		}
270 | 
271 | 		if ((threadCount % connections) != 0){
272 | 			throw new Exception("thread count needs to be a multiple of connections");
273 | 		}
274 | 		
275 | 		int threadsperconnection = threadCount / connections;
276 | 		DaRPCEndpoint<?,?>[] rpcConnections = new DaRPCEndpoint[connections];
277 | 		Thread[] workers = new Thread[threadCount];
278 | 		ClientThread[] benchmarkTask = new ClientThread[threadCount];
279 | 		
280 | 		RdmaRpcProtocol rpcProtocol = new RdmaRpcProtocol();
281 | 		System.out.println("starting.. threads " + threadCount + ", connections " + connections + ", server " + host + ", recvQueue " + recvQueue + ", sendQueue" + sendQueue + ", batchSize " + batchSize + ", mode " + mode);
282 | 		DaRPCClientGroup<RdmaRpcRequest, RdmaRpcResponse> group = DaRPCClientGroup.createClientGroup(rpcProtocol, 100, maxinline, recvQueue, sendQueue);
283 | 		InetSocketAddress address = new InetSocketAddress(host, 1919);
284 | 		
285 | 		int k = 0;
286 | 		for (int i = 0; i < rpcConnections.length; i++){
287 | 			DaRPCClientEndpoint<RdmaRpcRequest, RdmaRpcResponse> clientEp = group.createEndpoint();
288 | 			clientEp.connect(address, 1000);
289 | 			rpcConnections[i] = clientEp;
290 | 			for (int j = 0; j < threadsperconnection; j++){
291 | 				benchmarkTask[k] = new ClientThread(clientEp, loop, address, mode, batchSize, clienttimeout);
292 | 				k++;
293 | 			}
294 | 		}
295 | 
296 | 		StopWatch stopWatchThroughput = new StopWatch();
297 | 		stopWatchThroughput.start();		
298 | 		for(int i = 0; i < threadCount;i++){
299 | 			workers[i] = new Thread(benchmarkTask[i]);
300 | 			workers[i].start();
301 | 		}
302 | 		for(int i = 0; i < threadCount;i++){
303 | 			workers[i].join();
304 | 			System.out.println("finished joining worker " + i);
305 | 		}
306 | 		double executionTime = (double) stopWatchThroughput.getExecutionTime() / 1000.0;
307 | 		System.out.println("executionTime " + executionTime);
308 | 		double ops = 0;
309 | 		for (int i = 0; i < threadCount; i++) {
310 | 			ops += benchmarkTask[i].getOps();
311 | 		}
312 | 		System.out.println("ops " + ops);
313 | 		double throughput = 0.0;
314 | 		double latency = 0.0;
315 | 		if (executionTime > 0) {
316 | 			throughput = ops / executionTime;
317 | 			double _threadcount = (double) threadCount;
318 | 			double throughputperclient = throughput / _threadcount;
319 | 			double norm = 1.0;
320 | 			latency = norm / throughputperclient * 1000000.0;
321 | 		}	
322 | 		System.out.println("throughput " + throughput);
323 | 
324 | 		String dataFilename = "datalog-client.dat";
325 | 		FileOutputStream dataStream = new FileOutputStream(dataFilename, true);
326 | 		FileChannel dataChannel = dataStream.getChannel();
327 | 		String _bechmarkType = "";
328 | 		String logdata = _bechmarkType + "\t\t"
329 | 				+ 0 + "\t" + loop + "\t" + size
330 | 				+ "\t\t" + 0 + "\t\t"
331 | 				+ 0 + "\t\t"
332 | 				+ throughput + "\t\t"
333 | 				+ 9 + "\t\t"
334 | 				+ 0 + "\t\t" + latency
335 | 				+ "\n";
336 | 		ByteBuffer buffer = ByteBuffer.wrap(logdata.getBytes());
337 | 		dataChannel.write(buffer);
338 | 		dataChannel.close();		
339 | 		dataStream.close();
340 | 		
341 | 		for (int i = 0; i < rpcConnections.length; i++){
342 | 			rpcConnections[i].close();
343 | 		}
344 | 		group.close();
345 | 	}
346 | 	
347 | 	public static void main(String[] args) throws Exception { 
348 | 		DaRPCClient rpcClient = new DaRPCClient();
349 | 		rpcClient.launch(args);		
350 | 		System.exit(0);
351 | 	}
352 | }
353 | 


--------------------------------------------------------------------------------