├── .gitattributes ├── .gitignore ├── Makefile ├── README.md ├── basic.proto ├── homa_client.cc ├── homa_client.h ├── homa_incoming.cc ├── homa_incoming.h ├── homa_listener.cc ├── homa_listener.h ├── homa_socket.cc ├── homa_socket.h ├── homa_stream.cc ├── homa_stream.h ├── java ├── .gitattributes ├── .gitignore ├── .project ├── README.md ├── grpcHoma │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ ├── cz │ │ │ └── adamh │ │ │ │ └── utils │ │ │ │ └── NativeUtils.java │ │ │ ├── grpcHoma │ │ │ ├── HomaClient.java │ │ │ ├── HomaClientStream.java │ │ │ ├── HomaClientTransport.java │ │ │ ├── HomaIncoming.java │ │ │ ├── HomaServer.java │ │ │ ├── HomaServerBuilder.java │ │ │ ├── HomaServerStream.java │ │ │ ├── HomaSocket.java │ │ │ ├── HomaWire.java │ │ │ └── StreamId.java │ │ │ └── io │ │ │ └── grpc │ │ │ └── HomaMetadata.java │ │ └── test │ │ └── java │ │ └── grpcHoma │ │ ├── HomaIncomingTest.java │ │ └── HomaSocketTest.java ├── homaJni │ ├── build.gradle │ └── src │ │ └── main │ │ └── cpp │ │ ├── homa.cpp │ │ ├── homa.h │ │ └── homa_api.cpp ├── rsync_exclude.txt ├── settings.gradle ├── testApp │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── grpcHoma │ │ │ ├── BasicClient.java │ │ │ ├── BasicServer.java │ │ │ ├── HomaChannelBuilder.java │ │ │ └── Main.java │ │ └── proto │ │ └── basic.proto └── updateRemote ├── mergedep.pl ├── mock.cc ├── mock.h ├── notes ├── rsync_exclude.txt ├── stream_id.cc ├── stream_id.h ├── stress.cc ├── tcp_test.cc ├── test.proto ├── test_client.cc ├── test_incoming.cc ├── test_listener.cc ├── test_server.cc ├── test_socket.cc ├── test_stream.cc ├── time_trace.cc ├── time_trace.h ├── tt ├── ttgrep.py ├── ttmerge.py ├── ttoffset.py ├── ttrange.py └── ttsum.py ├── update_remote ├── util.cc ├── util.h ├── wire.cc └── wire.h /.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # These are explicitly windows files and should use crlf 5 | *.bat text eol=crlf 6 | 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | \#*# 3 | *.pyc 4 | *.o 5 | *.hi 6 | *.dump 7 | *.log 8 | *.rej 9 | *.orig 10 | *.patch 11 | *.diff 12 | .tags* 13 | .deps 14 | *.pdf 15 | 16 | # Ignore IDE files 17 | /.idea/ 18 | /nbproject/ 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for grpc_homa, which allows C++ applications based on gRPC 2 | # to use Homa for message transport. 3 | # 4 | # Configuration: right now you'll need to modify this Makefile by hand 5 | # to configure for your site. To do that, set the following variables 6 | # below: 7 | # 8 | # HOMA_DIR: location of the top-level directory for the HomaModule repo 9 | # GRPC_INSTALL_DIR: location where gRPC binaries are installed (note 10 | # separate values for debugand release builds). 11 | # DEBUG: can be set to "yes" to enable compilation with debugging 12 | # enabled (must "make clean" after changing this). 13 | 14 | HOMA_DIR := /users/ouster/homaModule 15 | DEBUG := no 16 | ifeq ($(DEBUG),no) 17 | GRPC_INSTALL_DIR := /ouster/install.release 18 | DEBUG_FLAGS := -O3 -DNDEBUG 19 | else 20 | GRPC_INSTALL_DIR := /ouster/install.debug 21 | DEBUG_FLAGS := -g 22 | endif 23 | 24 | GRPC_LIBS := $(shell export PKG_CONFIG_PATH=$(GRPC_INSTALL_DIR)/lib/pkgconfig; \ 25 | pkg-config --libs protobuf grpc++) -lupb -lcares -lre2 -lz \ 26 | -laddress_sorting -lssl -lcrypto -lsystemd 27 | ifeq ($(DEBUG),yes) 28 | GRPC_LIBS := $(subst -lprotobuf,-lprotobufd,$(GRPC_LIBS)) 29 | endif 30 | 31 | GTEST_INCLUDE_PATH = ../googletest/googletest/include 32 | GTEST_LIB_PATH = ../googletest/build/lib 33 | export PKG_LIB_PATH = $(GRPC_INSTALL_DIR)/lib/pkgconfig 34 | GRPC_DIR = ../grpc 35 | PROTOC = $(GRPC_INSTALL_DIR)/bin/protoc 36 | CXX = g++ 37 | INCLUDES = -I $(GRPC_INSTALL_DIR)/include \ 38 | -I /users/ouster/homaModule \ 39 | -I $(GRPC_DIR) \ 40 | -I $(GRPC_DIR)/third_party/abseil-cpp \ 41 | -I $(GTEST_INCLUDE_PATH) 42 | CXXFLAGS += $(DEBUG_FLAGS) -std=c++17 -Wall -Werror -Wno-comment \ 43 | -fno-strict-aliasing $(INCLUDES) -MD 44 | CFLAGS = -Wall -Werror -fno-strict-aliasing $(DEBUG_FLAGS) -MD 45 | 46 | OBJS = homa_client.o \ 47 | homa_incoming.o \ 48 | homa_listener.o \ 49 | homa_socket.o \ 50 | homa_stream.o \ 51 | stream_id.o \ 52 | time_trace.o \ 53 | util.o \ 54 | wire.o 55 | 56 | HOMA_OBJS = homa_api.o 57 | 58 | TEST_OBJS = mock.o \ 59 | test_incoming.o \ 60 | test_listener.o \ 61 | test_socket.o \ 62 | test_stream.o 63 | 64 | LDFLAGS += -L/usr/local/lib $(GRPC_LIBS)\ 65 | -pthread\ 66 | -Wl,--no-as-needed -lgrpc++_reflection -Wl,--as-needed\ 67 | -ldl 68 | 69 | GRPC_CPP_PLUGIN = $(GRPC_INSTALL_DIR)/bin/grpc_cpp_plugin 70 | PROTOS_PATH = . 71 | 72 | all: libhoma.a stress test_client test_server tcp_test 73 | 74 | libhoma.a: $(OBJS) $(HOMA_OBJS) 75 | ar rcs libhoma.a $(OBJS) $^ 76 | 77 | stress: basic.grpc.pb.o basic.pb.o stress.o libhoma.a 78 | $(CXX) $^ $(LDFLAGS) -o $@ 79 | 80 | test_client: test.grpc.pb.o test.pb.o test_client.o libhoma.a 81 | $(CXX) $^ $(LDFLAGS) -o $@ 82 | 83 | test_server: test.grpc.pb.o test.pb.o test_server.o libhoma.a 84 | $(CXX) $^ $(LDFLAGS) -o $@ 85 | 86 | tcp_test: test.grpc.pb.o test.pb.o tcp_test.o time_trace.o 87 | $(CXX) $^ $(LDFLAGS) -o $@ 88 | 89 | unit: $(OBJS) $(TEST_OBJS) $(GTEST_LIB_PATH)/libgtest_main.a \ 90 | $(GTEST_LIB_PATH)/libgtest.a 91 | $(CXX) $^ $(LDFLAGS) -o $@ 92 | 93 | test: unit 94 | ./unit --gtest_brief=1 95 | 96 | homa_api.o: $(HOMA_DIR)/homa_api.c 97 | cc -c $(CFLAGS) $< -o $@ 98 | 99 | clean: 100 | rm -f test_client test_server unit tcp_test *.a *.o *.pb.* .deps 101 | 102 | install: all 103 | rsync -avz stress test_client test_server node-1: 104 | rsync -avz stress test_client test_server node-2: 105 | rsync -avz stress test_client test_server node-3: 106 | rsync -avz stress test_client test_server node-4: 107 | 108 | %.o: %.cc 109 | $(CXX) -c $(CXXFLAGS) $< -o $@ 110 | 111 | %.o: %.c 112 | cc -c $(CFLAGS) $< -o $@ 113 | 114 | %.grpc.pb.cc %.grpc.pb.h: %.proto %.pb.cc 115 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN) $< 116 | 117 | %.pb.cc %.pb.h: %.proto 118 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 119 | 120 | .PHONY: test clean 121 | .PRECIOUS: test.grpc.pb.h test.grpc.pb.cc test.pb.h test.pb.cc 122 | 123 | test_client.o test_server.o tcp_test.o : test.grpc.pb.h test.pb.h 124 | 125 | stress.o: basic.grpc.pb.h basic.pb.h 126 | 127 | # This magic (along with the -MD gcc option) automatically generates makefile 128 | # dependencies for header files included from source files we compile, 129 | # and keeps those dependencies up-to-date every time we recompile. 130 | # See 'mergedep.pl' for more information. 131 | .deps: $(wildcard *.d) 132 | @mkdir -p $(@D) 133 | perl mergedep.pl $@ $^ 134 | -include .deps 135 | 136 | # The following target is useful for debugging Makefiles; it 137 | # prints the value of a make variable. 138 | print-%: 139 | @echo $* = $($*) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This project adds Homa support to gRPC, so that gRPC applications can use 2 | Homa instead of TCP for transport. 3 | 4 | - This project is still in a relatively early stage of development, but C++ 5 | support is functional as of November 2021. Next up is Java support. 6 | 7 | - Please contact me if you have any problems; I'm happy to provide 8 | advice and support. 9 | 10 | - The head is currently based on gRPC v. 1.57.0 (and there is a branch 11 | named grpc-1.57.0 that will track this version of gRPC). There are 12 | also branches grpc-1.43.0 and grpc-1.41.0, which represent the 13 | most recent code to work on those branches. Older branches are not 14 | actively maintained. 15 | 16 | - Known limitations: 17 | - grpc_homa currently supports only insecure channels. 18 | 19 | - Initial performance measurements show that short RPCs complete about 20 | 40% faster with Homa than with TCP (about 55 us round trip for Homa, 21 | vs. 90 us for TCP). 22 | 23 | ### How to use grpc_homa 24 | - You will need to download the 25 | [Linux kernel driver for Homa](https://github.com/PlatformLab/HomaModule). 26 | Compile it as described in that repo and install it on all the machines 27 | where you plan to use gRPC. 28 | 29 | - Configure the Makefile as described in the comments at the top (sorry... 30 | I know this shouldn't need to be done manually). 31 | 32 | - Type `make`. This will build `libhoma.a`, which you should link with 33 | your applications. 34 | 35 | - When compiling your applications, use `-I` to specify this directory, 36 | then `#include homa_client.h` as needed for clients and 37 | `#include homa_listener.h` as needed for servers. 38 | 39 | - On clients, pass `HomaClient::insecureChannelCredentials()` to 40 | `grpc::CreateChannel` instead of `grpc::InsecureChannelCredentials()` 41 | to create a channel that uses Homa. 42 | For an example of a simple but complete client, see `test_client.cc`. 43 | 44 | - On servers, pass `HomaListener::insecureCredentials()` to 45 | `grpc::AddListeningPort` instead of `grpc::InsecureServerCredentials()`. 46 | For an example of a simple but complete server, see `test_server.cc`. 47 | 48 | - Once you have done this, all your existing gRPC-based code should 49 | work just fine. 50 | -------------------------------------------------------------------------------- /basic.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffer definitions used for testing grpcHoma; they 2 | // make basic use of each of the RPC types (streaming in either 3 | // or both directions) 4 | 5 | syntax = "proto3"; 6 | 7 | package basic; 8 | 9 | service Basic { 10 | rpc Ping(Request) returns (Response) {} 11 | rpc StreamOut(stream StreamOutRequest) returns (Response) {} 12 | rpc StreamIn(StreamInRequest) returns (stream Response) {} 13 | rpc Stream2Way(stream StreamOutRequest) returns (stream Response) {} 14 | rpc PrintLog(Empty) returns (Empty) {} 15 | } 16 | 17 | message Request { 18 | sfixed32 requestItems = 1; 19 | sfixed32 replyItems = 2; 20 | repeated sfixed32 data = 3; 21 | } 22 | 23 | message Response { 24 | repeated sfixed32 data = 1; 25 | } 26 | 27 | message StreamOutRequest { 28 | sfixed32 done = 1; 29 | sfixed32 requestItems = 2; 30 | sfixed32 replyItems = 3; 31 | repeated sfixed32 data = 4; 32 | } 33 | 34 | message StreamInRequest { 35 | repeated sfixed32 sizes = 1; 36 | } 37 | 38 | message Empty {} 39 | -------------------------------------------------------------------------------- /homa_client.h: -------------------------------------------------------------------------------- 1 | #ifndef HOMA_CLIENT_H 2 | #define HOMA_CLIENT_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include "src/core/ext/filters/client_channel/client_channel.h" 11 | 12 | #include "homa_socket.h" 13 | #include "homa_stream.h" 14 | 15 | /** 16 | * This structure stores all of the shared information for gRPC 17 | * clients using Homa. There is just one of these structures for 18 | * each application. 19 | */ 20 | class HomaClient { 21 | public: 22 | static std::shared_ptr 23 | insecureChannelCredentials(); 24 | 25 | protected: 26 | HomaClient(bool ipv6); 27 | ~HomaClient(); 28 | static void init(); 29 | static grpc_channel* createChannel(const char* target, 30 | const grpc_channel_args* args); 31 | 32 | /** 33 | * This class provides credentials used to create Homa channels. 34 | */ 35 | class InsecureCredentials final : public grpc::ChannelCredentials { 36 | public: 37 | std::shared_ptr CreateChannelImpl( 38 | const std::string& target, const grpc::ChannelArguments& args) 39 | override; 40 | grpc::SecureChannelCredentials* AsSecureCredentials() override 41 | { 42 | return nullptr; 43 | } 44 | }; 45 | 46 | /** 47 | * This class is invoked by gRPC to create subchannels for a channel. 48 | * There is typically only one instance. 49 | */ 50 | class SubchannelFactory : public grpc_core::ClientChannelFactory { 51 | public: 52 | grpc_core::RefCountedPtr CreateSubchannel( 53 | const grpc_resolved_address& address, 54 | const grpc_core::ChannelArgs& args) override; 55 | }; 56 | 57 | /** 58 | * An instance of this class creates "connections" for subchannels 59 | * of a given channel. It doesn't do much, since Homa doesn't have 60 | * connections. 61 | */ 62 | class Connector : public grpc_core::SubchannelConnector { 63 | public: 64 | void Connect(const Args& args, Result* status, grpc_closure* notify) 65 | override; 66 | void Shutdown(grpc_error_handle error) override; 67 | }; 68 | 69 | /** 70 | * An instance of this class contains information specific to a 71 | * peer. These objects are used as "transports" in gRPC, but unlike 72 | * transports for TCP, there's no network connection info here, 73 | * since Homa is connectionless. 74 | */ 75 | struct Peer { 76 | // Contains a virtual function table for use by the rest of gRPC. 77 | // gRPC uses this as a transport handle. 78 | grpc_transport transport; 79 | 80 | // Shared client state. 81 | HomaClient *hc; 82 | 83 | // Linux struct sockaddr containing the IP address and port of the peer. 84 | grpc_resolved_address addr; 85 | 86 | Peer(HomaClient *hc, grpc_resolved_address addr); 87 | }; 88 | 89 | static void destroy(grpc_transport* gt); 90 | static void destroy_stream(grpc_transport* gt, grpc_stream* gs, 91 | grpc_closure* then_schedule_closure); 92 | static int init_stream(grpc_transport* gt, grpc_stream* gs, 93 | grpc_stream_refcount* refcount, 94 | const void* server_data, grpc_core::Arena* arena); 95 | static void onRead(void* arg, grpc_error_handle error); 96 | static void perform_op(grpc_transport* gt, grpc_transport_op* op); 97 | static void perform_stream_op(grpc_transport* gt, grpc_stream* gs, 98 | grpc_transport_stream_op_batch* op); 99 | static void set_pollset(grpc_transport* gt, grpc_stream* gs, 100 | grpc_pollset* pollset); 101 | static void set_pollset_set(grpc_transport* gt, grpc_stream* gs, 102 | grpc_pollset_set* pollset_set); 103 | static grpc_endpoint* 104 | get_endpoint(grpc_transport* gt); 105 | 106 | // Used by gRPC to invoke transport-specific functions on all 107 | // HomaPeer objects associated with this HomaClient. 108 | struct grpc_transport_vtable vtable; 109 | 110 | // Holds all streams with outstanding requests. 111 | std::unordered_map streams; 113 | 114 | // Id to use for the next outgoing RPC. 115 | int nextId; 116 | 117 | // Must be held when accessing @streams or @nextId. Must not be 118 | // acquired while holding a stream lock. 119 | grpc_core::Mutex mutex; 120 | 121 | // Holds information about the socket used for Homa communication, 122 | // such as information about receive buffers. 123 | HomaSocket sock; 124 | 125 | // Used to call us back when fd is readable. 126 | grpc_closure readClosure; 127 | 128 | // Number of peers that exist for this object. 129 | int numPeers; 130 | 131 | // Single shared HomaClient used for all channels. Nullptr means 132 | // not created yet. 133 | static HomaClient *sharedClient; 134 | 135 | // Held when creating or deleting sharedClient and when updating numPeers. 136 | static grpc_core::Mutex refCountMutex; 137 | 138 | // Used to create subchannels for all Homa channels. 139 | static SubchannelFactory factory; 140 | }; 141 | 142 | #endif // HOMA_CLIENT_H 143 | -------------------------------------------------------------------------------- /homa_incoming.h: -------------------------------------------------------------------------------- 1 | #ifndef HOMA_INCOMING_H 2 | #define HOMA_INCOMING_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include "homa.h" 9 | #include "homa_socket.h" 10 | #include "stream_id.h" 11 | #include "wire.h" 12 | 13 | /** 14 | * An instance of this class describes one incoming Homa message 15 | * (either a request or a response). 16 | */ 17 | class HomaIncoming { 18 | public: 19 | // This class is used as the deleter for Incoming::UniquePtr, 20 | // so that sliceRefs can count a std::unique_ptr along with other 21 | // refs to the Incoming. 22 | struct UnrefIncoming { 23 | void operator()(HomaIncoming* msg) 24 | { 25 | msg->sliceRefs.Unref({}); 26 | } 27 | }; 28 | 29 | // This struct is used to return multiple results from HomaIncoming::read. 30 | struct ReadResults { 31 | // Returns success/failure information about the call. 32 | grpc_error_handle error; 33 | 34 | // Homa's RPC id associated with the incoming message. This may 35 | // be nonzero even when !error.ok(), in which case the error 36 | // was related to a specific Homa RPC. 37 | uint64_t homaId; 38 | 39 | // Identifies the gRPC stream that this message belongs to. If 40 | // homaId is nonzero after an error, then this can be used to 41 | // identify a stream to abort. 42 | StreamId streamId; 43 | }; 44 | 45 | typedef std::unique_ptr UniquePtr; 46 | 47 | explicit HomaIncoming(HomaSocket *sock); 48 | explicit HomaIncoming(HomaSocket *sock, int sequence, bool initMd, 49 | size_t bodyLength, int firstValue, 50 | bool messageComplete, bool trailMd); 51 | ~HomaIncoming(); 52 | size_t addMetadata(size_t offset, ...); 53 | void copyOut(void *dst, size_t offset, size_t length); 54 | void deserializeMetadata(size_t offset, size_t length, 55 | grpc_metadata_batch* batch); 56 | grpc_core::Slice getSlice(size_t offset, size_t length); 57 | 58 | static UniquePtr read(HomaSocket *sock, int flags, ReadResults *results); 59 | 60 | /** 61 | * Return a count of the number of contiguous bytes available at a 62 | * given offset in the message (or zero if the offset is outside 63 | * the range of the message). 64 | * \param offset 65 | * Offset of a byte within the message. 66 | */ 67 | size_t contiguous(size_t offset) 68 | { 69 | if (offset >= length) { 70 | return 0; 71 | } 72 | if ((offset >> HOMA_BPAGE_SHIFT) == (recvArgs.num_bpages-1)) { 73 | return length - offset; 74 | } 75 | return HOMA_BPAGE_SIZE - (offset & (HOMA_BPAGE_SIZE-1)); 76 | } 77 | 78 | /** 79 | * Make a range of bytes from a message addressable in a contiguous 80 | * chunk. 81 | * \param offset 82 | * Offset within the message of the first byte of the desired 83 | * object. 84 | * \param auxSpace 85 | * If the object is split across buffers in msg, it will 86 | * be copied here to make it contiguous. If you know that the 87 | * requested information will be contiguous in the message, 88 | * this parameter can be specified as nullptr. 89 | * \tparam T 90 | * Type of the desired object. 91 | * \return 92 | * A pointer to contiguous memory containing the desired object, 93 | * or nullptr if the desired object extends beyond the end of the 94 | * message. 95 | */ 96 | template 97 | T *get(size_t offset, T *auxSpace) 98 | { 99 | if ((offset + sizeof(T)) > length) { 100 | return nullptr; 101 | } 102 | size_t bufIndex = offset >> HOMA_BPAGE_SHIFT; 103 | size_t offsetInBuf = offset & (HOMA_BPAGE_SIZE-1); 104 | uint8_t *start = sock->getBufRegion() + recvArgs.bpage_offsets[bufIndex] 105 | + offsetInBuf; 106 | size_t cbytes = contiguous(offset); 107 | if (cbytes >= sizeof(T)) { 108 | return reinterpret_cast(start); 109 | } 110 | 111 | // Must copy the object to make it contiguous; it could span any 112 | // number of buffers. 113 | uint8_t *p = reinterpret_cast(auxSpace); 114 | memcpy(p, start, cbytes); 115 | for (size_t offsetInObj = cbytes; offsetInObj < sizeof(T); 116 | offsetInObj += cbytes) { 117 | bufIndex++; 118 | cbytes = ((sizeof(T) - offsetInObj) > HOMA_BPAGE_SIZE) 119 | ? HOMA_BPAGE_SIZE : (sizeof(T) - offsetInObj); 120 | memcpy(p + offsetInObj, 121 | sock->getBufRegion() + recvArgs.bpage_offsets[bufIndex], 122 | cbytes); 123 | } 124 | return auxSpace; 125 | } 126 | 127 | 128 | // Keeps track of all outstanding references to this message 129 | // (such as a std::unique_ptr for the entire message, and metadata 130 | // keys and values). 131 | grpc_slice_refcount sliceRefs; 132 | 133 | // Information about the Homa socket from which the message was read. 134 | HomaSocket *sock; 135 | 136 | // Passed as msg_control argument to recvmsg; contains information about 137 | // the incoming message (such as where its bytes are stored). Note that 138 | // buffers referenced here must eventually be returned to Homa. 139 | struct homa_recvmsg_args recvArgs; 140 | 141 | // Total length of the message. 142 | size_t length; 143 | 144 | // Sequence number for this message (extracted from hdr). 145 | int sequence; 146 | 147 | // Bytes of initial metadata in the message (extracted from hdr). 148 | uint32_t initMdLength; 149 | 150 | // Bytes of gRPC message payload. Set to 0 once message data has 151 | // been transferred to gRPC. 152 | uint32_t bodyLength; 153 | 154 | // Bytes of trailing metadata in the message (extracted from hdr). 155 | uint32_t trailMdLength; 156 | 157 | // If non-null, the target is incremented when this object is destroyed. 158 | int *destroyCounter; 159 | 160 | // If the value for a metadata item is longer than this, it will be 161 | // stored as a refcounted pointer into the message, rather than a 162 | // static slice. This is a variable so it can be modified for testing. 163 | size_t maxStaticMdLength; 164 | 165 | static void destroyer(grpc_slice_refcount *sliceRefs); 166 | 167 | Wire::Header *hdr() 168 | { 169 | return reinterpret_cast( 170 | sock->getBufRegion() + recvArgs.bpage_offsets[0]); 171 | } 172 | }; 173 | 174 | #endif // HOMA_INCOMING_H 175 | -------------------------------------------------------------------------------- /homa_listener.h: -------------------------------------------------------------------------------- 1 | #ifndef HOMA_LISTENER_H 2 | #define HOMA_LISTENER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include "src/core/lib/surface/server.h" 10 | 11 | #include "homa_socket.h" 12 | #include "homa_stream.h" 13 | #include "wire.h" 14 | 15 | #ifndef __UNIT_TEST__ 16 | #define PROTECTED protected 17 | #else 18 | #define PROTECTED public 19 | #endif 20 | 21 | /** 22 | * Stores all the state needed to serve Homa requests on a particular 23 | * port. 24 | */ 25 | class HomaListener : public grpc_core::Server::ListenerInterface { 26 | public: 27 | static HomaListener *Get(grpc_server* server, int *port, bool ipv6); 28 | static std::shared_ptr insecureCredentials(void); 29 | 30 | PROTECTED: 31 | /** 32 | * This class provides credentials to create Homa listeners. 33 | */ 34 | class InsecureCredentials final : public grpc::ServerCredentials { 35 | public: 36 | int AddPortToServer(const std::string& addr, grpc_server* server); 37 | void SetAuthMetadataProcessor( 38 | const std::shared_ptr& processor) 39 | override { 40 | (void)processor; 41 | GPR_ASSERT(0); // Should not be called on insecure credentials. 42 | } 43 | }; 44 | 45 | /** 46 | * This class manages the Homa socket associated with the listener, and 47 | * contains most of the listener functionality. In the TCP world, the 48 | * listener corresponds to the listen socket and the transport corresponds 49 | * to all of the individual data connections; in the Homa world, a single 50 | * Homa socket serves both purposes, and it is managed here in the 51 | * transport. Thus there isn't much left in the listener. 52 | */ 53 | class Transport { 54 | public: 55 | Transport(grpc_server* server, int *port, bool ipv6); 56 | ~Transport(); 57 | HomaStream * getStream(StreamId *streamId, 58 | std::optional& streamLock, 59 | bool create); 60 | void shutdown(); 61 | void start(grpc_core::Server* server, 62 | const std::vector* pollsets); 63 | static void destroy(grpc_transport* gt); 64 | static void destroy_stream(grpc_transport* gt, grpc_stream* gs, 65 | grpc_closure* then_schedule_closure); 66 | static grpc_endpoint* 67 | get_endpoint(grpc_transport* gt); 68 | static int init_stream(grpc_transport* gt, grpc_stream* gs, 69 | grpc_stream_refcount* refcount, 70 | const void* init_info, grpc_core::Arena* arena); 71 | static void onRead(void* arg, grpc_error_handle error); 72 | static void perform_op(grpc_transport* gt, grpc_transport_op* op); 73 | static void perform_stream_op(grpc_transport* gt, grpc_stream* gs, 74 | grpc_transport_stream_op_batch* op); 75 | static void set_pollset(grpc_transport* gt, grpc_stream* gs, 76 | grpc_pollset* pollset); 77 | static void set_pollset_set(grpc_transport* gt, grpc_stream* gs, 78 | grpc_pollset_set* pollset_set); 79 | 80 | PROTECTED: 81 | /** 82 | * This structure is used to pass data down through callbacks to 83 | * init_stream and back up again. 84 | */ 85 | struct StreamInit { 86 | // Identifying information from incoming RPC. 87 | StreamId *streamId; 88 | 89 | // Used to return the HomaStream address back through callbacks. 90 | HomaStream *stream; 91 | }; 92 | 93 | // Points to a virtual function table for use by the rest of gRPC to 94 | // treat this object as a transport. gRPC uses a pointer to this field 95 | // as a generic handle for the object. 96 | grpc_transport vtable; 97 | 98 | // Associated gRPC server. Not owned by this object 99 | grpc_core::Server *server; 100 | 101 | // Keeps track of all RPCs currently in some stage of processing; 102 | // used to look up the Stream for an RPC based on its id. 103 | std::unordered_map activeRpcs; 105 | 106 | typedef std::unordered_map::iterator ActiveIterator; 108 | 109 | // Must be held when accessing @activeRpcs. Must not be acquired while 110 | // holding a stream lock. 111 | grpc_core::Mutex mutex; 112 | 113 | // Manages the Homa socket, including buffer space. 114 | HomaSocket sock; 115 | 116 | // Used to call us back when fd is readable. 117 | grpc_closure read_closure; 118 | 119 | grpc_core::ConnectivityStateTracker state_tracker; 120 | 121 | // Used to notify gRPC of new incoming requests. 122 | void (*accept_stream_cb)(void* user_data, grpc_transport* transport, 123 | const void* server_data); 124 | void* accept_stream_data; 125 | 126 | friend class TestListener; 127 | }; 128 | 129 | HomaListener(grpc_server* server, int *port, bool ipv6); 130 | ~HomaListener(); 131 | void Orphan() override ; 132 | void SetOnDestroyDone(grpc_closure* on_destroy_done) override; 133 | void Start(grpc_core::Server* server, 134 | const std::vector* pollsets) override; 135 | grpc_core::channelz::ListenSocketNode* channelz_listen_socket_node() 136 | const override; 137 | 138 | // Transport associated with the listener; its lifetime is managed 139 | // outside this class. 140 | Transport *transport; 141 | 142 | // Homa port number associated with this listener. 143 | int port; 144 | 145 | grpc_closure* on_destroy_done; 146 | 147 | /** 148 | * Information that is shared across all HomaListener/Transport objects. 149 | */ 150 | struct Shared { 151 | // Contains pointers to all open Homa ports: keys are port numbers. 152 | std::unordered_map ports; 153 | 154 | // Synchronizes access to this structure. 155 | grpc_core::Mutex mutex; 156 | 157 | // Function table shared across all HomaListeners. 158 | struct grpc_transport_vtable vtable; 159 | Shared() : ports(), mutex() {} 160 | }; 161 | // Singleton object with common info. 162 | 163 | static std::optional shared; 164 | static gpr_once shared_once; 165 | static void InitShared(void); 166 | 167 | friend class TestListener; 168 | }; 169 | 170 | #endif // HOMA_LISTENER_H 171 | -------------------------------------------------------------------------------- /homa_socket.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "src/core/lib/transport/transport_impl.h" 4 | 5 | #include "homa.h" 6 | 7 | #include "homa_socket.h" 8 | 9 | /** 10 | * Constructor for HomaSockets; opens the socket and sets up buffer space. 11 | * If an error occurs in setting up the socket then winter information will 12 | * be logged and getFd() will return -1. 13 | * \param domain 14 | * Communication domain for the socket; must be AF_INET for IPv4 or 15 | * AF_INET6 for IPv6. 16 | * \param port 17 | * Homa port to bind to the socket. Must be either a Homa port number 18 | * less than HOMA_MIN_DEFAULT_PORT or 0 (in which case Homa will assign 19 | * an unused port number). 20 | */ 21 | HomaSocket::HomaSocket(int domain, int port) 22 | : fd(-1) 23 | , gfd(nullptr) 24 | , port(0) 25 | , bufRegion(nullptr) 26 | , bufSize(0) 27 | , savedBuffers() 28 | , mutex() 29 | { 30 | sockaddr_in_union addr{}; 31 | socklen_t addr_size = sizeof(addr); 32 | int status; 33 | 34 | fd = socket(domain, SOCK_DGRAM | SOCK_CLOEXEC, IPPROTO_HOMA); 35 | if (fd < 0) { 36 | gpr_log(GPR_ERROR, "Couldn't open Homa socket: %s\n", strerror(errno)); 37 | goto error; 38 | } 39 | gfd = grpc_fd_create(fd, "homa-socket", true); 40 | 41 | // Bind to port, if needed, and retrieve port number. 42 | if (port > 0) { 43 | if (domain == AF_INET6) { 44 | addr.in6.sin6_family = AF_INET6; 45 | addr.in6.sin6_port = htons(port); 46 | } else { 47 | addr.in4.sin_family = AF_INET; 48 | addr.in4.sin_port = htons(port); 49 | } 50 | if (bind(fd, &addr.sa, sizeof(addr)) != 0) { 51 | gpr_log(GPR_ERROR, "Couldn't bind Homa socket to port %d: %s\n", 52 | port, strerror(errno)); 53 | goto error; 54 | } 55 | } 56 | getsockname(fd, &addr.sa, &addr_size); 57 | this->port = ntohs((domain == AF_INET6) ? addr.in6.sin6_port 58 | : addr.in4.sin_port); 59 | 60 | // Set up the buffer region. 61 | bufSize = 1000*HOMA_BPAGE_SIZE; 62 | bufRegion = (uint8_t *) mmap(NULL, bufSize, PROT_READ|PROT_WRITE, 63 | MAP_PRIVATE|MAP_ANONYMOUS, 0, 0); 64 | if (bufRegion == MAP_FAILED) { 65 | gpr_log(GPR_ERROR, 66 | "Couldn't mmap buffer region for server on port %d: %s\n", 67 | port, strerror(errno)); 68 | bufRegion = nullptr; 69 | bufSize = 0; 70 | goto error; 71 | } 72 | struct homa_set_buf_args setBufArgs; 73 | setBufArgs.start = bufRegion; 74 | setBufArgs.length = bufSize; 75 | status = setsockopt(fd, IPPROTO_HOMA, SO_HOMA_SET_BUF, &setBufArgs, 76 | sizeof(setBufArgs)); 77 | if (status < 0) { 78 | gpr_log(GPR_ERROR, 79 | "Error in setsockopt(SO_HOMA_SET_BUF) for port %d: %s\n", 80 | port, strerror(errno)); 81 | goto error; 82 | } 83 | return; 84 | 85 | error: 86 | cleanup(); 87 | } 88 | 89 | /** 90 | * HomaSocket constructor used during unit tests; doesn't actually open 91 | * a socket. 92 | * \param bufRegion 93 | * Buffer region to use for the socket. 94 | */ 95 | HomaSocket::HomaSocket(uint8_t *bufRegion) 96 | : fd(-1) 97 | , gfd(nullptr) 98 | , port(0) 99 | , bufRegion(bufRegion) 100 | , bufSize(0) 101 | , savedBuffers() 102 | , mutex() 103 | { 104 | } 105 | 106 | /** 107 | * Destructor for HomaSockets. Closes the socket and releases buffer space. 108 | */ 109 | HomaSocket::~HomaSocket() 110 | { 111 | cleanup(); 112 | } 113 | 114 | /** 115 | * Release all resources associated with the socket, including closing the 116 | * socket itself. 117 | */ 118 | void HomaSocket::cleanup() 119 | { 120 | if (bufRegion) { 121 | if (munmap(bufRegion, bufSize) != 0) { 122 | gpr_log(GPR_ERROR, 123 | "Munmap failed for Homa socket with fd %d, port %d: %s\n", 124 | fd, port, strerror(errno)); 125 | } 126 | bufRegion = nullptr; 127 | } 128 | if (gfd) { 129 | // Note: grpc_fd_shutdown will close the fd. 130 | grpc_fd_shutdown(gfd, GRPC_ERROR_CREATE("Homa socket destroyed")); 131 | grpc_fd_orphan(gfd, nullptr, nullptr, "goodbye"); 132 | grpc_core::ExecCtx::Get()->Flush(); 133 | gfd = nullptr; 134 | } else if (fd >= 0) { 135 | if (close(fd) < 0) { 136 | gpr_log(GPR_ERROR, 137 | "close failed for Homa socket with fd %d, port %d: %s\n", 138 | fd, port, strerror(errno)); 139 | } 140 | } 141 | fd = -1; 142 | port = 0; 143 | } 144 | 145 | /** 146 | * This method is called when the buffer space received in a previous 147 | * recvmsg call is no longer needed. It saves information about the 148 | * buffers, so that it can return that information in a later call to 149 | * getSavedBuffers(). 150 | * \param recvArgs 151 | * Structure in which Homa passed buffer information to the application. 152 | * recvArgs->num_bpages will be set to 0 to indicate that all buffers 153 | * have been claimed here. 154 | */ 155 | void HomaSocket::saveBuffers(struct homa_recvmsg_args *recvArgs) 156 | { 157 | grpc_core::MutexLock lock(&mutex); 158 | for (uint32_t i = 0; i < recvArgs->num_bpages; i++) { 159 | savedBuffers.emplace_back(recvArgs->bpage_offsets[i]); 160 | } 161 | recvArgs->num_bpages = 0; 162 | } 163 | 164 | /** 165 | * This method is called before invoking Homa recvmsg; it adds as many 166 | * saved buffers as possible to recvArgs->buffers, so that they will be 167 | * returned to Homa as part of recvmsg. 168 | * \param recvArgs 169 | * Struct that is about to be passed to Homa's recvmsg. May already 170 | * contain some buffers to return. 171 | */ 172 | void HomaSocket::getSavedBuffers(struct homa_recvmsg_args *recvArgs) 173 | { 174 | grpc_core::MutexLock lock(&mutex); 175 | uint32_t count = recvArgs->num_bpages; 176 | while ((count < HOMA_MAX_BPAGES) && !savedBuffers.empty()) { 177 | recvArgs->bpage_offsets[count] = savedBuffers.front(); 178 | savedBuffers.pop_front(); 179 | count++; 180 | } 181 | recvArgs->num_bpages = count; 182 | } -------------------------------------------------------------------------------- /homa_socket.h: -------------------------------------------------------------------------------- 1 | #ifndef HOMA_SOCKET_H 2 | #define HOMA_SOCKET_H 3 | 4 | #include 5 | 6 | #include "src/core/lib/iomgr/ev_posix.h" 7 | 8 | #include "homa.h" 9 | 10 | /** 11 | * An instance of this class stores state associated with an open Homa 12 | * socket. It particular, it manages the buffer region used for 13 | * incoming messages. 14 | */ 15 | class HomaSocket { 16 | public: 17 | HomaSocket(int domain, int port); 18 | HomaSocket(uint8_t *bufRegion); 19 | ~HomaSocket(void); 20 | void saveBuffers(struct homa_recvmsg_args *recvArgs); 21 | void getSavedBuffers(struct homa_recvmsg_args *recvArgs); 22 | 23 | //Returns the file descriptor associated with this object, or -1 24 | // if the constructor failed to initialize the socket. 25 | inline int getFd() const 26 | { 27 | return fd; 28 | } 29 | 30 | /** 31 | * Returns GRPCs handle for the file descriptor for this socket, or 32 | * nullptr if the socket was not successfully opened. 33 | */ 34 | inline grpc_fd *getGfd() const 35 | { 36 | return gfd; 37 | } 38 | 39 | /** 40 | * Returns the port number associated with this socket, or zero if 41 | * if the constructor failed to initialize the socket. 42 | */ 43 | inline int getPort() const 44 | { 45 | return port; 46 | } 47 | 48 | /** 49 | * Returns the base address of the receive buffer region for this 50 | * socket. If the constructor failed to initialize the socket then 51 | * nullptr is returned. 52 | * @return 53 | */ 54 | inline uint8_t *getBufRegion() const 55 | { 56 | return bufRegion; 57 | } 58 | 59 | // The info below should be treated as private; it is left public 60 | // in order to enable unit testing. 61 | 62 | // File descriptor for a Homa socket, or -1 if the constructor 63 | // failed to set up the socket properly. 64 | int fd; 65 | 66 | // GRPC token corresponding to fd, or nullptr if the constructor failed. 67 | grpc_fd *gfd; 68 | 69 | // Homa port number assigned to this socket. 70 | int port; 71 | 72 | // First byte of memory region for buffer space for incoming messages. 73 | // or nullptr if buffer space has not been allocated. This is an mmapped 74 | // region. 75 | uint8_t *bufRegion; 76 | 77 | // Size of the buffer region at *bufRegion, in bytes. 78 | size_t bufSize; 79 | 80 | // Tokens for buffers that need to be returned eventually to Homa. 81 | // Each token is an entry from the buffers array in a 82 | // struct homa_recvmsg_control object. 83 | std::deque savedBuffers; 84 | 85 | // Must be held whenever accessing savedBuffers. 86 | grpc_core::Mutex mutex; 87 | 88 | void cleanup(); 89 | }; 90 | 91 | #endif // HOMA_SOCK -------------------------------------------------------------------------------- /homa_stream.h: -------------------------------------------------------------------------------- 1 | #ifndef HOMA_STREAM_H 2 | #define HOMA_STREAM_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include "src/core/lib/transport/transport_impl.h" 9 | 10 | #include "homa_incoming.h" 11 | #include "stream_id.h" 12 | #include "wire.h" 13 | 14 | /** 15 | * This class provides common facilities used for Homa streams on both 16 | * the client and server side. Each stream object corresponds to a single 17 | * RPC, and it exists for the life of that RPC. It is used to manage the 18 | * movement of data between gRPC and Homa messages, as well as gRPC callbacks. 19 | */ 20 | class HomaStream { 21 | public: 22 | // Must be held whenever accessing info in this structure. 23 | grpc_core::Mutex mutex; 24 | 25 | // File descriptor for the Homa socket to use for I/O. 26 | int fd; 27 | 28 | // Uniquely identifies this gRPC RPC, and also provides info about 29 | // the peer (e.g. for sending responses). 30 | StreamId streamId; 31 | 32 | // Homa's identifier for the most recent request sent on this stream. 33 | uint64_t sentHomaId; 34 | 35 | // Homa's identifier for an unacknowledged Homa request received on this 36 | // stream, or 0 if none. The next time we want to send a message, we'll 37 | // send a reply to this RPC, rather than starting a new RPC. 38 | uint64_t homaRequestId; 39 | 40 | // Reference count (owned externally). 41 | grpc_stream_refcount* refs; 42 | 43 | // Small statically allocated buffer for outgoing messages; holds 44 | // header plus initial and trailing metadata, if they fit. 45 | uint8_t xmitBuffer[10000]; 46 | 47 | // If the metadata didn't completely fit in xmit_msg, extra chunks 48 | // are allocated dynamically; this vector keeps track of them all 49 | // so they can be freed. 50 | std::vector xmitOverflows; 51 | 52 | // How many bytes to allocate for each element of @xmitOverflows. 53 | // This is a variable so it can be changed for unit testing. 54 | size_t overflowChunkSize; 55 | 56 | // Contains all of the Slices (of message data) referred to by vecs; 57 | // keeps them alive and stable until the Homa request is sent. 58 | std::vector slices; 59 | 60 | // Describes all of the pieces of the current outgoing message. 61 | std::vector vecs; 62 | 63 | // Additional bytes available immediately following the last element 64 | // of vecs. 65 | size_t lastVecAvail; 66 | 67 | // Current length of output message, in bytes. 68 | size_t xmitSize; 69 | 70 | // Sequence number to use for the next outgoing message. 71 | int nextXmitSequence; 72 | 73 | // Incoming Homa messages that have not been fully processed. 74 | // Entries are sorted in increasing order of sequence number. 75 | std::vector incoming; 76 | 77 | // All incoming Homa messages with sequence numbers less than this one 78 | // have already been processed. 79 | int nextIncomingSequence; 80 | 81 | // Information saved from "receive" stream ops, so that we can 82 | // fill in message data/metadata and invoke callbacks. 83 | grpc_metadata_batch* initMd; 84 | grpc_closure* initMdClosure; 85 | bool *initMdTrailMdAvail; 86 | absl::optional* messageBody; 87 | grpc_closure* messageClosure; 88 | grpc_metadata_batch* trailMd; 89 | grpc_closure* trailMdClosure; 90 | 91 | // True means we have passed trailing metadata to gRPC, so there is 92 | // no more message data coming for this stream. 93 | bool eof; 94 | 95 | // True means this RPC has been cancelled, so we shouldn't send 96 | // any more Homa messages. 97 | bool cancelled; 98 | 99 | // True means that trailing metadata has been sent for this stream. 100 | bool trailMdSent; 101 | 102 | // True means that this stream is for the server side of an RPC; 103 | // false means client. 104 | bool isServer; 105 | 106 | // Error that has occurred on this stream, if any. 107 | grpc_error_handle error; 108 | 109 | // Maximum number of bytes to allow in a single Homa message (this 110 | // is a variable so it can be modified for unit testing). 111 | size_t maxMessageLength; 112 | 113 | HomaStream(bool isServer, StreamId streamId, int fd, 114 | grpc_stream_refcount* refcount); 115 | 116 | Wire::Header *hdr() 117 | { 118 | return reinterpret_cast(xmitBuffer); 119 | } 120 | 121 | virtual ~HomaStream(void); 122 | void addPeerToMetadata(grpc_metadata_batch *md); 123 | void cancelPeer(void); 124 | void flush(void); 125 | void handleIncoming(HomaIncoming::UniquePtr msg, uint64_t homaId); 126 | void notifyError(grpc_error_handle error); 127 | void resetXmit(void); 128 | void saveCallbacks(grpc_transport_stream_op_batch* op); 129 | void sendDummyResponse(); 130 | void serializeMetadata(const void *key, uint32_t keyLength, 131 | const void *value, uint32_t valueLength); 132 | void serializeMetadataBatch(grpc_metadata_batch *batch); 133 | void transferData(); 134 | void xmit(grpc_transport_stream_op_batch* op); 135 | 136 | static size_t metadataLength(grpc_metadata_batch* batch); 137 | }; 138 | 139 | #endif // HOMA_STREAM_H 140 | -------------------------------------------------------------------------------- /java/.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | 5 | * text eol=lf 6 | 7 | # These are explicitly windows files and should use crlf 8 | *.bat text eol=crlf 9 | 10 | -------------------------------------------------------------------------------- /java/.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | \#*# 3 | *.pyc 4 | *.o 5 | *.hi 6 | *.dump 7 | *.log 8 | *.rej 9 | *.orig 10 | *.patch 11 | *.diff 12 | .tags* 13 | .deps 14 | *.pdf 15 | 16 | # Ignore IDE files 17 | /.idea/ 18 | /nbproject/ 19 | 20 | # Ignore Gradle info 21 | gradle* 22 | .gradle 23 | build 24 | buildSrc 25 | -------------------------------------------------------------------------------- /java/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | java 4 | 5 | 6 | grpc-java 7 | 8 | 9 | 10 | org.eclipse.jdt.core.javabuilder 11 | 12 | 13 | 14 | 15 | 16 | org.eclipse.jdt.core.javanature 17 | 18 | 19 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | This directory contains Java support for Homa in gRPC. It has three 2 | main subdirectories: 3 | - homaJni: contains JNI native support for invoking Homa kernel calls. 4 | - grpcHoma: a Java library that encapsulates Homa support. Its .jar 5 | file includes homaJni, so it is complete and self-contained. 6 | - testApp: a main program that can be used to test and exercise grpcHoma. 7 | Invoke it with the --help option to find out more. 8 | -------------------------------------------------------------------------------- /java/grpcHoma/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | repositories { 6 | mavenCentral() 7 | } 8 | 9 | // Location of local gRPC files 10 | // def grpcDirectory = '../../grpc-java' 11 | def grpcDirectory = '/ouster/grpc-java' 12 | 13 | dependencies { 14 | // testImplementation 'junit:junit:4.13.2' 15 | testImplementation 'org.junit.jupiter:junit-jupiter:5.7.2' 16 | 17 | implementation('com.google.guava:guava:31.0.1-jre') 18 | implementation('io.netty:netty-all:4.1.24.Final') 19 | 20 | implementation files( 21 | "${grpcDirectory}/protobuf/build/libs/grpc-protobuf-1.41.0.jar", 22 | "${grpcDirectory}/api/build/libs/grpc-api-1.41.0.jar", 23 | "${grpcDirectory}/context/build/libs/grpc-context-1.41.0.jar", 24 | "${grpcDirectory}/core/build/libs/grpc-core-1.41.0.jar", 25 | "${grpcDirectory}/netty/shaded/build/libs/grpc-netty-shaded-1.41.0.jar", 26 | "${grpcDirectory}/stub/build/libs/grpc-stub-1.41.0.jar", 27 | "${grpcDirectory}/protobuf-lite/build/libs/grpc-protobuf-lite-1.41.0.jar", 28 | "${grpcDirectory}/benchmarks/build/install/grpc-benchmarks/lib/protobuf-java-3.17.2.jar" 29 | ) 30 | } 31 | 32 | jar { 33 | from('../homaJni/build/lib/main/debug') { 34 | include '*.so' 35 | } 36 | dependsOn(':homaJni:build') 37 | } 38 | 39 | test { 40 | testLogging.exceptionFormat = 'full' 41 | dependsOn(':homaJni:build') 42 | ignoreFailures = true 43 | 44 | // Used by HomaSocket to load the native library during testing. 45 | systemProperty 'grpcHoma.libhomaJni.so', 46 | '../homaJni/build/lib/main/debug/libhomaJni.so' 47 | 48 | // Needed so that test output to stdout/stderr will be visible 49 | outputs.upToDateWhen {false} 50 | testLogging.showStandardStreams = true 51 | 52 | // Jupiter tests won't run without this. 53 | useJUnitPlatform() 54 | } 55 | 56 | -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/cz/adamh/utils/NativeUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Class NativeUtils is published under the The MIT License: 3 | * 4 | * Copyright (c) 2012 Adam Heinrich 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | package cz.adamh.utils; 25 | 26 | import java.io.*; 27 | import java.nio.file.FileSystemNotFoundException; 28 | import java.nio.file.FileSystems; 29 | import java.nio.file.Files; 30 | import java.nio.file.ProviderNotFoundException; 31 | import java.nio.file.StandardCopyOption; 32 | 33 | /** 34 | * A simple library class which helps with loading dynamic libraries stored in the 35 | * JAR archive. These libraries usually contain implementation of some methods in 36 | * native code (using JNI - Java Native Interface). 37 | * 38 | * @see http://adamheinrich.com/blog/2012/how-to-load-native-jni-library-from-jar 39 | * @see https://github.com/adamheinrich/native-utils 40 | * 41 | */ 42 | public class NativeUtils { 43 | 44 | /** 45 | * The minimum length a prefix for a file has to have according to {@link File#createTempFile(String, String)}}. 46 | */ 47 | private static final int MIN_PREFIX_LENGTH = 3; 48 | public static final String NATIVE_FOLDER_PATH_PREFIX = "nativeutils"; 49 | 50 | /** 51 | * Temporary directory which will contain the DLLs. 52 | */ 53 | private static File temporaryDir; 54 | 55 | /** 56 | * Private constructor - this class will never be instanced 57 | */ 58 | private NativeUtils() { 59 | } 60 | 61 | /** 62 | * Loads library from current JAR archive 63 | * 64 | * The file from JAR is copied into system temporary directory and then loaded. The temporary file is deleted after 65 | * exiting. 66 | * Method uses String as filename because the pathname is "abstract", not system-dependent. 67 | * 68 | * @param path The path of file inside JAR as absolute path (beginning with '/'), e.g. /package/File.ext 69 | * @throws IOException If temporary file creation or read/write operation fails 70 | * @throws IllegalArgumentException If source file (param path) does not exist 71 | * @throws IllegalArgumentException If the path is not absolute or if the filename is shorter than three characters 72 | * (restriction of {@link File#createTempFile(java.lang.String, java.lang.String)}). 73 | * @throws FileNotFoundException If the file could not be found inside the JAR. 74 | */ 75 | public static void loadLibraryFromJar(String path) throws IOException { 76 | 77 | if (null == path || !path.startsWith("/")) { 78 | throw new IllegalArgumentException("The path has to be absolute (start with '/')."); 79 | } 80 | 81 | // Obtain filename from path 82 | String[] parts = path.split("/"); 83 | String filename = (parts.length > 1) ? parts[parts.length - 1] : null; 84 | 85 | // Check if the filename is okay 86 | if (filename == null || filename.length() < MIN_PREFIX_LENGTH) { 87 | throw new IllegalArgumentException("The filename has to be at least 3 characters long."); 88 | } 89 | 90 | // Prepare temporary file 91 | if (temporaryDir == null) { 92 | temporaryDir = createTempDirectory(NATIVE_FOLDER_PATH_PREFIX); 93 | temporaryDir.deleteOnExit(); 94 | } 95 | 96 | File temp = new File(temporaryDir, filename); 97 | 98 | try (InputStream is = NativeUtils.class.getResourceAsStream(path)) { 99 | Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); 100 | } catch (IOException e) { 101 | temp.delete(); 102 | throw e; 103 | } catch (NullPointerException e) { 104 | temp.delete(); 105 | throw new FileNotFoundException("File " + path + " was not found inside JAR."); 106 | } 107 | 108 | try { 109 | System.load(temp.getAbsolutePath()); 110 | } finally { 111 | if (isPosixCompliant()) { 112 | // Assume POSIX compliant file system, can be deleted after loading 113 | temp.delete(); 114 | } else { 115 | // Assume non-POSIX, and don't delete until last file descriptor closed 116 | temp.deleteOnExit(); 117 | } 118 | } 119 | } 120 | 121 | private static boolean isPosixCompliant() { 122 | try { 123 | return FileSystems.getDefault() 124 | .supportedFileAttributeViews() 125 | .contains("posix"); 126 | } catch (FileSystemNotFoundException 127 | | ProviderNotFoundException 128 | | SecurityException e) { 129 | return false; 130 | } 131 | } 132 | 133 | private static File createTempDirectory(String prefix) throws IOException { 134 | String tempDir = System.getProperty("java.io.tmpdir"); 135 | File generatedDir = new File(tempDir, prefix + System.nanoTime()); 136 | 137 | if (!generatedDir.mkdir()) 138 | throw new IOException("Failed to create temp directory " + generatedDir.getName()); 139 | 140 | return generatedDir; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/HomaClient.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import io.grpc.ChannelLogger; 19 | 20 | import java.util.HashMap; 21 | 22 | import static io.grpc.ChannelLogger.ChannelLogLevel; 23 | 24 | /** 25 | * A single instance of this class is used to store state shared across 26 | * all RPCs emanating from a client process. 27 | */ 28 | public class HomaClient { 29 | // Used for all Homa communication. 30 | HomaSocket homa; 31 | 32 | // Singleton instance, shared across all transports and streams. 33 | static HomaClient instance; 34 | 35 | // Identifies the stream object for each outstanding RPC. Access only 36 | // with the lock for this object. 37 | HashMap streams; 38 | 39 | // Identifier to use for next client stream. Access only when holding 40 | // the lock for this object. 41 | int nextSid; 42 | 43 | // Receives responses. 44 | ClientThread receiver; 45 | 46 | // Used for logging various messages. 47 | ChannelLogger logger; 48 | 49 | /** 50 | * Constructor for HomaClients. 51 | * @param logger 52 | * Use this for logging messages. 53 | */ 54 | HomaClient(ChannelLogger logger) { 55 | homa = new HomaSocket(); 56 | streams = new HashMap<>(); 57 | nextSid = 1; 58 | receiver = new ClientThread(this); 59 | this.logger = logger; 60 | receiver.start(); 61 | } 62 | 63 | /** 64 | * Returns the singleton instance (creates it if it doesn't already exist). 65 | * @param logger 66 | * If a new HomaClient is created, this will be used by the HomaClient 67 | * for logging during its lifetime. 68 | */ 69 | static HomaClient getInstance(ChannelLogger logger) { 70 | if (instance == null) { 71 | synchronized (HomaClient.class) { 72 | // Must recheck instance again, since it could have been 73 | // created while we were waiting for the lock. 74 | if (instance == null) { 75 | instance = new HomaClient(logger); 76 | } 77 | } 78 | } 79 | return instance; 80 | } 81 | 82 | /** 83 | * This class represents a thread that receives incoming Homa responses 84 | * and handles them appropriately. 85 | */ 86 | static class ClientThread extends Thread { 87 | // Shared client state. 88 | HomaClient client; 89 | 90 | ClientThread(HomaClient client) { 91 | this.client = client; 92 | } 93 | 94 | public void run() { 95 | try { 96 | while (true) { 97 | HomaIncoming msg = new HomaIncoming(); 98 | HomaClientStream stream; 99 | String err = msg.read(client.homa, 100 | HomaSocket.flagReceiveResponse); 101 | if (err != null) { 102 | client.logger.log(ChannelLogLevel.ERROR, err); 103 | continue; 104 | } 105 | System.out.printf("Received response for id %d from %s, " + 106 | "totalBytes %d, initMdBytes %d, messageBytes %d, " + 107 | "trailMdBytes %d, flags 0x%x\n", 108 | msg.homaId, msg.peer.getInetSocketAddress(), 109 | msg.length, msg.header.initMdBytes, 110 | msg.header.messageBytes, msg.header.trailMdBytes, 111 | msg.header.flags); 112 | synchronized(this) { 113 | StreamId streamId = new StreamId(msg.streamId.address, 114 | msg.header.sid); 115 | stream = client.streams.get(streamId); 116 | } 117 | if (stream != null) { 118 | stream.handleIncoming(msg); 119 | } 120 | } 121 | } catch (Exception e) { 122 | String message = e.getMessage(); 123 | if (message == null) { 124 | message = "no information about cause"; 125 | } 126 | client.logger.log(ChannelLogLevel.ERROR, String.format( 127 | "HomaClient.ClientThread crashed with %s: %s", 128 | e.getClass().getName(), message)); 129 | } 130 | } 131 | } 132 | } -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/HomaClientTransport.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import java.io.InputStream; 19 | import java.net.InetSocketAddress; 20 | import java.net.SocketAddress; 21 | import java.util.concurrent.Executor; 22 | import java.util.concurrent.ScheduledExecutorService; 23 | 24 | import io.grpc.Attributes; 25 | import io.grpc.CallOptions; 26 | import io.grpc.ChannelLogger; 27 | import io.grpc.ClientStreamTracer; 28 | import io.grpc.InternalLogId; 29 | import io.grpc.Metadata; 30 | import io.grpc.MethodDescriptor; 31 | import io.grpc.Status; 32 | import io.grpc.internal.ClientStream; 33 | import io.grpc.internal.ClientTransportFactory.ClientTransportOptions; 34 | import io.grpc.internal.ConnectionClientTransport; 35 | import io.grpc.InternalChannelz.SocketStats; 36 | 37 | import com.google.common.util.concurrent.ListenableFuture; 38 | 39 | /** 40 | * An instance of this class is used to create streams that can communicate 41 | * with a specific gRPC server. 42 | */ 43 | public class HomaClientTransport implements ConnectionClientTransport { 44 | // The following are copies of construtor arguments. 45 | HomaClient client; 46 | InetSocketAddress serverAddress; 47 | ClientTransportOptions options; 48 | ChannelLogger logger; 49 | 50 | // Unique identifier used for logging. 51 | InternalLogId logId; 52 | 53 | // Used to notify gRPC of various interesting things happening on 54 | // this transport. Null means the transport hasn't been started yet. 55 | Listener listener; 56 | 57 | /** 58 | * Constructor for HomaClientTransports; normally invoked indirectly 59 | * though a HomaChannelBuilder. 60 | * @param client 61 | * Shared client state. 62 | * @param serverAddress 63 | * Location of the server for requests. 64 | * @param options 65 | * Various parameters that may be used to configure the channel. 66 | * @param channelLogger 67 | * Used for any log messages related to this transport and its streams. 68 | */ 69 | HomaClientTransport(HomaClient client, SocketAddress serverAddress, 70 | ClientTransportOptions options, 71 | ChannelLogger channelLogger) { 72 | System.out.printf("Constructing HomaClientTransport for %s, authority %s\n", 73 | serverAddress.toString(), options.getAuthority()); 74 | this.client = client; 75 | this.serverAddress = (InetSocketAddress) serverAddress; 76 | this.options = options; 77 | logId = InternalLogId.allocate(getClass(), serverAddress.toString()); 78 | listener = null; 79 | logger = channelLogger; 80 | } 81 | 82 | @Override 83 | public ListenableFuture getStats() { 84 | System.out.printf("HomaClientTransport.getStats invoked\n"); 85 | return null; 86 | } 87 | 88 | @Override 89 | public InternalLogId getLogId() { 90 | return logId; 91 | } 92 | 93 | @Override 94 | public ClientStream newStream(MethodDescriptor method, 95 | Metadata headers, CallOptions callOptions, 96 | ClientStreamTracer[] tracers) { 97 | System.out.printf("HomaClientTransport.newStream invoked\n"); 98 | System.out.printf("Creating HomaClientStream for %s", 99 | serverAddress); 100 | return new HomaClientStream(this, method, headers, 101 | callOptions, tracers); 102 | } 103 | 104 | @Override 105 | public void ping(PingCallback callback, Executor executor) { 106 | System.out.printf("HomaClientTransport.ping invoked\n"); 107 | } 108 | 109 | @Override 110 | public Attributes getAttributes() { 111 | System.out.printf("HomaClientTransport.getAttributes invoked\n"); 112 | return null; 113 | } 114 | 115 | @Override 116 | public Runnable start(Listener listener) { 117 | System.out.printf("HomaClientTransport.start invoked\n"); 118 | this.listener = listener; 119 | listener.transportReady(); 120 | return null; 121 | } 122 | 123 | @Override 124 | public void shutdown(Status reason) { 125 | System.out.printf("HomaClientTransport.shutdown invoked\n"); 126 | } 127 | 128 | @Override 129 | public void shutdownNow(Status reason) { 130 | System.out.printf("HomaClientTransport.shutdownNow invoked\n"); 131 | } 132 | } -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/HomaIncoming.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2022 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import java.io.IOException; 19 | import java.io.InputStream; 20 | import java.net.InetSocketAddress; 21 | import java.nio.ByteBuffer; 22 | import java.nio.ByteOrder; 23 | import java.util.logging.Logger; 24 | 25 | import io.grpc.ChannelLogger; 26 | import io.grpc.Metadata; 27 | 28 | /** 29 | * Each instance of this class stores information about one incoming Homa 30 | * message (either a request or a response). A gRPC message may require 31 | * more than one Homa message. 32 | */ 33 | class HomaIncoming { 34 | // The first part of the Homa message. This will be large enough to hold 35 | // most messages in their entirety; at minimum, it is guaranteed to be large 36 | // enough to hold the entire HomaWire::Header. 37 | static final int initialPayloadSize = 10000; 38 | ByteBuffer initialPayload; 39 | 40 | // If the message is longer than initialPayloadSize, holds all of the 41 | // bytes after those in initialPayload. 42 | ByteBuffer tail; 43 | 44 | // Homa's identifier for the incoming message (or a negative errno if 45 | // there was an error receiving the message; in this case, no other 46 | // fields are valid except peer). 47 | long homaId; 48 | 49 | // Total number of bytes in the Homa message. 50 | int length; 51 | 52 | // Total number of bytes stored at initialPayload. 53 | int baseLength; 54 | 55 | // Header info extracted from initialPayload. 56 | HomaWire.Header header; 57 | 58 | // Information about the peer from which the message was received, plus 59 | // the Homa RPC's id. 60 | HomaSocket.RpcSpec peer; 61 | 62 | // Unique identifier for this stream. 63 | StreamId streamId; 64 | 65 | // Raw initial metadata extracted from the message (alternating keys and 66 | // values); null means there were none, or they have already been passed 67 | // to gRPC. 68 | byte[][] headers; 69 | 70 | // Raw trailing metadata extracted from the message (alternating keys and 71 | // values); null means there were none, or they have already been passed 72 | // to gRPC. 73 | byte[][] trailers; 74 | 75 | // The message payload from the message, or null if none. 76 | MessageStream message; 77 | 78 | HomaIncoming() { 79 | initialPayload = ByteBuffer.allocateDirect(initialPayloadSize); 80 | initialPayload.order(ByteOrder.BIG_ENDIAN); 81 | length = 0; 82 | baseLength = 0; 83 | header = new HomaWire.Header(); 84 | peer = new HomaSocket.RpcSpec(); 85 | headers = null; 86 | trailers = null; 87 | message = null; 88 | } 89 | 90 | /** 91 | * Read an incoming Homa request or response message and populate the 92 | * structure with information about that message. This method also 93 | * takes care of sending automatic responses for streaming RPCs and 94 | * discarding those responses. 95 | * @param homa 96 | * Used to receive an incoming message. 97 | * @param flags 98 | * The flags value to pass to HomaSocket.receive. 99 | * @return 100 | * Null (for success) or a string containing an error message (for 101 | * logging) if there was a problem. 102 | */ 103 | String read(HomaSocket homa, int flags) { 104 | // This loop takes care of discarding automatic responses. 105 | while (true) { 106 | peer.reset(); 107 | length = homa.receive(initialPayload, 108 | flags | HomaSocket.flagReceivePartial, peer); 109 | homaId = peer.getId(); 110 | if (length < 0) { 111 | return String.format("Error receiving Homa id %d from %s: %s", 112 | homaId, peer.getInetSocketAddress().toString(), 113 | HomaSocket.strerror(-length)); 114 | } 115 | header.deserialize(initialPayload); 116 | if ((header.flags & HomaWire.Header.emptyResponse) == 0) { 117 | break; 118 | } 119 | } 120 | baseLength = initialPayload.limit(); 121 | streamId = new StreamId(peer.getInetSocketAddress(), header.sid); 122 | int expected = HomaWire.Header.length + header.initMdBytes 123 | + header.messageBytes + header.trailMdBytes; 124 | if (length != expected) { 125 | return String.format("Bad message length %d (expected " + 126 | "%d); initMdLength %d, messageLength %d, trailMdLength %d " + 127 | "header length %d", length, expected, header.initMdBytes, 128 | header.messageBytes, header.trailMdBytes, 129 | HomaWire.Header.length); 130 | } 131 | 132 | // Read the tail of the message, if needed. 133 | if (length != baseLength) { 134 | tail = ByteBuffer.allocateDirect(length - baseLength); 135 | tail.order(ByteOrder.BIG_ENDIAN); 136 | int tailLength = homa.receive(initialPayload, flags, peer); 137 | if (tailLength < 0) { 138 | return String.format("Error while receiving tail of Homa id " + 139 | "%d from %s: %s", homaId, 140 | peer.getInetSocketAddress().toString(), 141 | HomaSocket.strerror(-tailLength)); 142 | } 143 | if (tailLength != (length - baseLength)) { 144 | return String.format("Tail of Homa message has wrong length: " + 145 | "expected %d bytes, got %d bytes", length - baseLength, 146 | tailLength); 147 | } 148 | } 149 | 150 | // Separate the three major sections of payload (headers, message, 151 | // trailers). 152 | if ((header.flags & HomaWire.Header.initMdPresent) != 0) { 153 | headers = HomaWire.deserializeMetadata( 154 | getBuffer(header.initMdBytes), header.initMdBytes); 155 | } 156 | if ((header.flags & HomaWire.Header.trailMdPresent) != 0) { 157 | trailers = HomaWire.deserializeMetadata( 158 | getBuffer(header.trailMdBytes), header.trailMdBytes); 159 | } 160 | if ((header.flags & HomaWire.Header.messageComplete) != 0) { 161 | message = new MessageStream(this); 162 | } 163 | return null; 164 | } 165 | 166 | /** 167 | * Returns a ByteBuffer that can be used to extract the next numBytes of 168 | * data from the message. It normally returns either initialPayload or 169 | * tail, but if the desired range crosses the boundary between these 170 | * two then it creates a new ByteBuffer by copying data from these two. 171 | * @param numBytes 172 | * The caller wants this many bytes contiguous in a single 173 | * ByteBuffer. The caller must ensure that at least this many 174 | * bytes are available, between initialPayload and tail together. 175 | */ 176 | ByteBuffer getBuffer(int numBytes) { 177 | int initSize = initialPayload.remaining(); 178 | if (numBytes <= initSize) { 179 | return initialPayload; 180 | } 181 | if (initSize == 0) { 182 | return tail; 183 | } 184 | ByteBuffer spliced = ByteBuffer.allocate(numBytes); 185 | spliced.order(ByteOrder.BIG_ENDIAN); 186 | spliced.put(initialPayload); 187 | int oldLimit = tail.limit(); 188 | tail.limit(numBytes - initSize); 189 | spliced.put(tail); 190 | tail.limit(oldLimit); 191 | spliced.flip(); 192 | return spliced; 193 | } 194 | 195 | /** 196 | * An instance of this class represents an incoming gRPC message; it is 197 | * passed up to gRPC and used by gRPC to read the contents of the 198 | * message. 199 | */ 200 | static private class MessageStream extends InputStream { 201 | 202 | // Information about the incoming message. 203 | HomaIncoming incoming; 204 | 205 | // Keeps track of whether the two parts of the payload have 206 | // already been read. 207 | boolean initialSent = false; 208 | boolean tailSent = false; 209 | 210 | /** 211 | * Constructor for MessageStream. 212 | * @param incoming 213 | * The actual message data is stored here. 214 | */ 215 | public MessageStream(HomaIncoming incoming) { 216 | this.incoming = incoming; 217 | } 218 | 219 | @Override 220 | public int available() throws IOException { 221 | return incoming.initialPayload.limit() 222 | - incoming.initialPayload.position(); 223 | } 224 | 225 | @Override 226 | public int read() throws IOException { 227 | if (incoming.initialPayload.position() 228 | >= incoming.initialPayload.limit()) { 229 | return -1; 230 | } 231 | return incoming.initialPayload.get(); 232 | } 233 | } 234 | } -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/HomaServer.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2022 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import io.grpc.*; 19 | import io.grpc.internal.InternalServer; 20 | import io.grpc.internal.ServerListener; 21 | import io.grpc.internal.ServerTransport; 22 | import io.grpc.internal.ServerTransportListener; 23 | import io.grpc.InternalChannelz.SocketStats; 24 | 25 | import java.io.IOException; 26 | import java.net.SocketAddress; 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | import java.util.concurrent.ScheduledExecutorService; 30 | import java.util.logging.Level; 31 | import java.util.logging.Logger; 32 | 33 | import com.google.common.util.concurrent.ListenableFuture; 34 | 35 | /** 36 | * An instance of this class represents a server that will accept gRPC 37 | * requests arriving via Homa RPCs on a specific Homa port. 38 | */ 39 | public class HomaServer implements InternalServer, InternalWithLogId { 40 | // Port on which this server receives incoming requests. 41 | int port; 42 | 43 | // Homa socket corresponding to port. 44 | HomaSocket homa; 45 | 46 | // Used for logging interesting events. 47 | Logger logger = Logger.getLogger(InternalServer.class.getName()); 48 | 49 | // Receives requests. 50 | ServerThread receiver; 51 | 52 | // Used to report certain events up to gRPC. 53 | ServerListener listener; 54 | 55 | // Only used to fetch transportListener. 56 | HomaTransport transport; 57 | 58 | // Used to report certain events up to gRPC. 59 | ServerTransportListener transportListener; 60 | 61 | // Needed by HomaServerStream.statsTraceContext 62 | List streamTracerFactories; 63 | 64 | /** 65 | * Construct a Homa server. 66 | * @param port 67 | * Homa port number on which to listen for incoming requests. 68 | */ 69 | HomaServer(int port, List 70 | streamTracerFactories) { 71 | this.port = port; 72 | this.homa = null; 73 | logger = Logger.getLogger(HomaServer.class.getName()); 74 | receiver = new ServerThread(this); 75 | listener = null; 76 | transport = null; 77 | transportListener = null; 78 | this.streamTracerFactories = streamTracerFactories; 79 | } 80 | 81 | @Override 82 | public InternalLogId getLogId() { 83 | System.out.printf("HomaServer.getLogId invoked\n"); 84 | return null; 85 | } 86 | 87 | @Override 88 | public void start(ServerListener listener) throws IOException { 89 | homa = new HomaSocket(port); 90 | receiver.start(); 91 | System.out.printf("HomaServer.start opened Homa socket on port %d " + 92 | "and started receiver thread\n", port); 93 | this.listener = listener; 94 | transport = new HomaTransport(); 95 | transportListener = listener.transportCreated(transport); 96 | transportListener.transportReady(Attributes.newBuilder().build()); 97 | } 98 | 99 | @Override 100 | public void shutdown() { 101 | System.out.printf("HomaServer.shutdown invoked\n"); 102 | } 103 | 104 | @Override 105 | public SocketAddress getListenSocketAddress() { 106 | System.out.printf("HomaServer.getListenSocketAddress invoked\n"); 107 | return null; 108 | } 109 | 110 | @Override 111 | public InternalInstrumented getListenSocketStats() { 112 | System.out.printf("HomaServer.getListenSocketStats invoked\n"); 113 | return null; 114 | } 115 | 116 | @Override 117 | public List getListenSocketAddresses() { 118 | System.out.printf("HomaServer.getListenSocketAddresses invoked\n"); 119 | return new ArrayList(); 120 | } 121 | 122 | @Override 123 | public List> getListenSocketStatsList() { 124 | System.out.printf("HomaServer.getListenSocketStatsList invoked\n"); 125 | return null; 126 | } 127 | 128 | /** 129 | * This class represents a thread that receives incoming Homa requests 130 | * and handles them appropriately. 131 | */ 132 | static class ServerThread extends Thread { 133 | // Shared client state. 134 | HomaServer server; 135 | 136 | ServerThread(HomaServer server) { 137 | this.server = server; 138 | } 139 | 140 | public void run() { 141 | try { 142 | while (true) { 143 | HomaIncoming msg = new HomaIncoming(); 144 | String err = msg.read(server.homa, 145 | HomaSocket.flagReceiveRequest); 146 | if (err != null) { 147 | server.logger.log(Level.SEVERE, err); 148 | continue; 149 | } 150 | System.out.printf("Received request with Homa id %d from %s, " + 151 | "totalBytes %d, initMdBytes %d, messageBytes %d, " + 152 | "trailMdBytes %d, flags 0x%x\n", 153 | msg.homaId, msg.peer.getInetSocketAddress(), 154 | msg.length, msg.header.initMdBytes, 155 | msg.header.messageBytes, msg.header.trailMdBytes, 156 | msg.header.flags); 157 | HomaServerStream stream = new HomaServerStream(msg, server, 158 | server.transportListener); 159 | 160 | } 161 | } catch (Exception e) { 162 | String message = e.getMessage(); 163 | if (message == null) { 164 | message = "no information about cause"; 165 | } 166 | server.logger.log(Level.SEVERE, 167 | String.format("HomaServer.ServerThread crashed with " + 168 | "%s exception: %s", e.getClass().getName(), 169 | message)); 170 | } 171 | } 172 | } 173 | 174 | /** 175 | * In the HTTP implementation of gRPC, there is one instance of this class 176 | * for each incoming connection. Since Homa is connectionless, a single 177 | * instance of this class is used for all incoming requests; the only 178 | * reason this class exists at all is because it's needed to retrieve a 179 | * ServerTransportListener. 180 | */ 181 | static class HomaTransport implements ServerTransport { 182 | InternalLogId logId; 183 | 184 | HomaTransport() { 185 | logId = InternalLogId.allocate(getClass(), ""); 186 | } 187 | 188 | @Override 189 | public ListenableFuture getStats() { 190 | System.out.printf("HomaTransport.getStats invoked\n"); 191 | return null; 192 | } 193 | 194 | @Override 195 | public InternalLogId getLogId() { 196 | System.out.printf("HomaTransport.getLogId invoked\n"); 197 | return logId; 198 | } 199 | 200 | @Override 201 | public void shutdown() { 202 | System.out.printf("HomaTransport.shutdown invoked\n"); 203 | } 204 | 205 | @Override 206 | public void shutdownNow(Status reason) { 207 | System.out.printf("HomaTransport.shutdownNow invoked\n"); 208 | } 209 | 210 | @Override 211 | public ScheduledExecutorService getScheduledExecutorService() { 212 | System.out.printf("HomaTransport.getScheduledExecutorService invoked\n"); 213 | return null; 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/HomaServerBuilder.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import io.grpc.ServerBuilder; 19 | import io.grpc.ServerStreamTracer; 20 | import io.grpc.internal.AbstractServerImplBuilder; 21 | import io.grpc.internal.InternalServer; 22 | import io.grpc.internal.ServerImplBuilder; 23 | 24 | import java.util.List; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | /** 28 | * This class contains boilerplate used to create Homa servers. It is 29 | * based heavily on NettyServerBuilder from gRPC; it's not documented very 30 | * well because I'm not sure I understand the mechanism well enough to 31 | * document it :-(. 32 | */ 33 | public class HomaServerBuilder 34 | extends AbstractServerImplBuilder { 35 | ServerImplBuilder serverImplBuilder; 36 | int port; 37 | 38 | /** 39 | * Constructor for HomaServerBuilder. 40 | * 41 | * @param port Homa port number on which to listen for incoming requests. 42 | */ 43 | public HomaServerBuilder(int port) { 44 | this.port = port; 45 | serverImplBuilder = new ServerImplBuilder( 46 | new HomaClientTransportServersBuilder()); 47 | 48 | // Without this, gRPC will attempt to use the result of 49 | // HomaTransport.getScheduledExecutorService to schedule a handshake 50 | // timeout (which is irrelevant for Homa). 51 | serverImplBuilder.handshakeTimeout(Long.MAX_VALUE, 52 | TimeUnit.MILLISECONDS); 53 | } 54 | 55 | /** 56 | * Create a new HomaServerBuilder for a given port. 57 | * @param port 58 | * The Homa port on which the resulting server should listen for 59 | * incoming Homa RPCs. 60 | * @return 61 | * The new builder. 62 | */ 63 | public static HomaServerBuilder forPort(int port) { 64 | return new HomaServerBuilder(port); 65 | } 66 | 67 | @Override 68 | protected ServerBuilder delegate() { 69 | System.out.printf("HomaServerBuilder.delegate invoked\n"); 70 | return serverImplBuilder; 71 | } 72 | 73 | private class HomaClientTransportServersBuilder 74 | implements ServerImplBuilder.ClientTransportServersBuilder { 75 | @Override 76 | public InternalServer buildClientTransportServers( 77 | List streamTracerFactories) { 78 | return new HomaServer(port, streamTracerFactories); 79 | } 80 | } 81 | } 82 | 83 | -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/HomaWire.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import java.nio.ByteBuffer; 19 | import java.nio.charset.StandardCharsets; 20 | import java.util.ArrayList; 21 | 22 | import io.grpc.Metadata; 23 | import io.grpc.HomaMetadata; 24 | 25 | /** 26 | * This class encapsulates information about how to serialize information into 27 | * Homa messages and deserialize information out of Homa messages. Note: this 28 | * class must be consistent with the definitions in the C++ class "wire.h" in 29 | * the C++ implementation of Homa support for gRPC, in order to maintain 30 | * interoperability between the C++ and Java implementations. 31 | */ 32 | class HomaWire { 33 | /** 34 | * Defines the fields of a message header. 35 | */ 36 | static public class Header { 37 | // Unique identifier for this stream (all messages in the stream will 38 | // contain this value). 39 | int sid = 0; 40 | 41 | // Position of this Homa message among all of those sent on the 42 | // stream. Used on the other end to make sure that messages are 43 | // processed in order. The first number for each stream is 1. 44 | int sequence = 0; 45 | 46 | // Number of bytes of initial metadata (may be zero), which follows 47 | // the header in the Homa message. 48 | int initMdBytes = 0; 49 | 50 | // Number of bytes of trailing metadata (may be zero), which follows 51 | // the initial metadata. 52 | int trailMdBytes = 0; 53 | 54 | // Number of bytes of gRPC message data (may be zero), which follows 55 | // the trailing metadata. 56 | int messageBytes = 0; 57 | 58 | // OR-ed combination of various flag bits; see below for definitions. 59 | byte flags = 0; 60 | 61 | // Currently supported flag bits: 62 | 63 | // Indicates that this message contains all available initial metadata 64 | // (possibly none). 65 | static final byte initMdPresent = 1; 66 | 67 | // Indicates that, as of this Homa RPC, all message data has been sent. 68 | // If the message data is too long to fit in a single message, only the 69 | // last message has this bit set. 70 | static final byte messageComplete = 2; 71 | 72 | // Indicates that this message contains all available trailing metadata 73 | // (possibly none). 74 | static final byte trailMdPresent = 4; 75 | 76 | // Indicates that this message is a Homa request (meaning it that it 77 | // requires an eventual response). 0 means this is a Homa response. 78 | static final byte isRequest = 8; 79 | 80 | // Indicates that there is no useful information in this message; 81 | // it is a dummy Homa response sent by the other side. 82 | static final byte emptyResponse = 16; 83 | 84 | // Indicates that the sender has cancelled this RPC, and the receiver 85 | // should do the same. 86 | static final byte cancelled = 32; 87 | 88 | // Number of bytes occupied by a Header. 89 | static final int length = 21; 90 | 91 | /** 92 | * Construct a new Header 93 | * @param sid 94 | * Identifer for the client stream this message belongs to. 95 | * @param sequence 96 | * Sequence number of this message within the stream. 97 | * @param flags 98 | * Initial value for the flags field. 99 | */ 100 | Header(int sid, int sequence, byte flags) { 101 | this.sid = sid; 102 | this.sequence = sequence; 103 | initMdBytes = 0; 104 | trailMdBytes = 0; 105 | messageBytes = 0; 106 | this.flags = flags; 107 | } 108 | 109 | /** 110 | * Construct a new Header, leaving fields undefined (intended for 111 | * incoming messages). 112 | */ 113 | Header() { 114 | } 115 | 116 | /** 117 | * Serializes the contents of this object into an outgoing message. 118 | * @param buf 119 | * Used to build and transmit the outgoing message; must have 120 | * big-endian byte order. The header will be written at the 121 | * current position in the buffer (presumably the beginning?) 122 | */ 123 | void serialize(ByteBuffer buf) { 124 | // Note: the order here must match the order of fields in 125 | // the C++ header wire.h. 126 | buf.putInt(sid); 127 | System.out.printf("serialized streamId %d in header: initMdBytes " + 128 | "%d, trailMdBytes %s, messageBytes %d\n", sid, initMdBytes, 129 | trailMdBytes, messageBytes); 130 | buf.putInt(sequence); 131 | buf.putInt(initMdBytes); 132 | buf.putInt(trailMdBytes); 133 | buf.putInt(messageBytes); 134 | buf.put(flags); 135 | } 136 | 137 | /** 138 | * Extracts header information from incoming Homa message. 139 | * @param buf 140 | * Holds the contents of an incoming message. Buf will be 141 | * positioned at the first byte of data after the header 142 | * when this method returns. 143 | */ 144 | void deserialize(ByteBuffer buf) { 145 | buf.position(0); 146 | sid = buf.getInt(); 147 | sequence = buf.getInt(); 148 | initMdBytes = buf.getInt(); 149 | trailMdBytes = buf.getInt(); 150 | messageBytes = buf.getInt(); 151 | flags = buf.get(); 152 | } 153 | } 154 | 155 | /** 156 | * Serialize metadata values into an outgoing Homa message. 157 | * @param md 158 | * Metadata to serialize; contains an even number of elements, 159 | * consisting of alternating keys and values. 160 | * @param buf 161 | * Used to build and transmit the outgoing message; must have 162 | * big-endian byte order and enough space to hold the metadata. 163 | * The metadata is written at the current position in the buffer 164 | * (and that position is advanced). 165 | * @return 166 | * The total number of bytes of data added to buf. 167 | */ 168 | static int serializeMetadata(byte[][] md, ByteBuffer buf) { 169 | // See Mdata definition in wire.h (C++) for wire format of metadata. 170 | int totalBytes = 0; 171 | 172 | for (int i = 0; i < md.length; i+= 2) { 173 | buf.putInt(md[i].length); 174 | buf.putInt(md[i+1].length); 175 | buf.put(md[i]); 176 | buf.put(md[i+1]); 177 | totalBytes += 8 + md[i].length + md[i+1].length; 178 | } 179 | return totalBytes; 180 | } 181 | 182 | /** 183 | * Extract metadata from an incoming Homa message. 184 | * @param buf 185 | * Contains the raw message. Must be positioned at the first byte of 186 | * metadata within the message. Upon return, the position will be 187 | * the first byte of data after the metadata. 188 | * @param numBytes 189 | * Total bytes of metadata in the message. 190 | * @return 191 | * The extracted metadata, in an array of byte arrays; at the 192 | * outer array, even elements contain keys and odd elements the 193 | * corresponding values. 194 | */ 195 | static byte[][] deserializeMetadata(ByteBuffer buf, int numBytes) { 196 | // See Mdata definition in wire.h (C++) for wire format of metadata. 197 | ArrayList keysAndValues = new ArrayList(); 198 | int bytesLeft = numBytes; 199 | while (bytesLeft > 0) { 200 | int keyLength = buf.getInt(); 201 | int valueLength = buf.getInt(); 202 | if (bytesLeft < (8 + keyLength + valueLength)) { 203 | System.out.printf("Bogus incoming metadata\n"); 204 | break; 205 | } 206 | byte[] key = new byte[keyLength]; 207 | for (int i = 0; i < keyLength; i++) { 208 | key[i] = buf.get(); 209 | } 210 | keysAndValues.add(key); 211 | byte[] value = new byte[valueLength]; 212 | for (int i = 0; i < valueLength; i++) { 213 | value[i] = buf.get(); 214 | } 215 | keysAndValues.add(value); 216 | System.out.printf("Incoming metadata: key %s, value %s\n", 217 | new String(key, StandardCharsets.US_ASCII), 218 | new String(value, StandardCharsets.US_ASCII)); 219 | bytesLeft = bytesLeft - keyLength - valueLength - 8; 220 | } 221 | byte[][] result = new byte[keysAndValues.size()][]; 222 | for (int i = 0; i < keysAndValues.size(); i++) { 223 | result[i] = keysAndValues.get(i); 224 | } 225 | return result; 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/grpcHoma/StreamId.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import java.net.InetSocketAddress; 19 | 20 | /** 21 | * Holds information identifying a stream (which represents a gRPC RPC) in a 22 | * form that can be used as a key in HashMap. 23 | */ 24 | public class StreamId { 25 | // Address of the other end of this stream (could be either client 26 | // or server). 27 | public InetSocketAddress address; 28 | 29 | // Uniquely identifies this gRPC RPC among all those from this client. 30 | public int sid; 31 | 32 | /** 33 | * Constructor for StreamId 34 | * @param address 35 | * Address of the other end of this stream. 36 | * @param sid 37 | * Unique sequence number generated by client. 38 | */ 39 | public StreamId(InetSocketAddress address, int sid) { 40 | this.address = address; 41 | this.sid = sid; 42 | } 43 | 44 | /** 45 | * Returns a hash value representing the contents of the object (so 46 | * it can be used in HashMaps). 47 | */ 48 | @Override 49 | public int hashCode() { 50 | return address.hashCode() ^ sid; 51 | } 52 | 53 | /** 54 | * Tests whether this object is logically equal to some other object. 55 | * @param o 56 | * Other object to test against. 57 | * @return 58 | * True means equal, false means not equal. 59 | */ 60 | @Override 61 | public boolean equals(Object o) { 62 | if (this == o) { 63 | return true; 64 | } 65 | if ((o == null) || (getClass() != o.getClass())) { 66 | return false; 67 | } 68 | StreamId other = (StreamId) o; 69 | return (sid == other.sid) && (address.equals(other.address)); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /java/grpcHoma/src/main/java/io/grpc/HomaMetadata.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package io.grpc; 17 | 18 | /** 19 | * This class exists only to provide access to gRPC's Metadata constructor, 20 | * which is currently private. 21 | */ 22 | public class HomaMetadata { 23 | public static Metadata newMetadata(int numValues, Object[] keysAndValues) { 24 | return new Metadata(numValues, keysAndValues); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /java/grpcHoma/src/test/java/grpcHoma/HomaIncomingTest.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2022 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import java.nio.ByteBuffer; 19 | import java.net.*; 20 | 21 | import org.junit.jupiter.api.AfterEach; 22 | import org.junit.jupiter.api.BeforeEach; 23 | import org.junit.jupiter.api.Test; 24 | import static org.junit.jupiter.api.Assertions.*; 25 | 26 | import java.nio.ByteBuffer; 27 | import java.nio.ByteOrder; 28 | 29 | public class HomaIncomingTest { 30 | HomaIncoming msg; 31 | 32 | @BeforeEach 33 | void setUp() { 34 | msg = new HomaIncoming(); 35 | } 36 | 37 | @Test 38 | public void test_HomaIncoming_getBuffer_fromInitialData() { 39 | msg.initialPayload.put((byte) 1); 40 | msg.initialPayload.put((byte) 2); 41 | msg.initialPayload.putInt(12345); 42 | msg.initialPayload.flip(); 43 | msg.initialPayload.position(2); 44 | ByteBuffer b = msg.getBuffer(4); 45 | assertEquals(b, msg.initialPayload); 46 | assertEquals(12345, b.getInt()); 47 | assertEquals(0, msg.initialPayload.remaining()); 48 | } 49 | @Test 50 | public void test_HomaIncoming_getBuffer_fromTail() { 51 | msg.initialPayload.limit(0); 52 | msg.tail = ByteBuffer.allocate(100); 53 | msg.tail.order(ByteOrder.BIG_ENDIAN); 54 | msg.tail.put((byte) 1); 55 | msg.tail.put((byte) 2); 56 | msg.tail.putInt(12345); 57 | msg.tail.put((byte) 3); 58 | msg.tail.flip(); 59 | msg.tail.position(2); 60 | ByteBuffer b = msg.getBuffer(4); 61 | assertEquals(b, msg.tail); 62 | assertEquals(12345, b.getInt()); 63 | assertEquals(1, msg.tail.remaining()); 64 | } 65 | @Test 66 | public void test_HomaIncoming_getBuffer_mustSplice() { 67 | msg.initialPayload.put((byte) 1); 68 | msg.initialPayload.put((byte) 2); 69 | msg.initialPayload.putInt(12345); 70 | msg.initialPayload.flip(); 71 | msg.tail = ByteBuffer.allocate(100); 72 | msg.tail.order(ByteOrder.BIG_ENDIAN); 73 | msg.tail.put((byte) 1); 74 | msg.tail.put((byte) 2); 75 | msg.tail.putInt(12345); 76 | msg.tail.flip(); 77 | msg.initialPayload.position(2); 78 | ByteBuffer b = msg.getBuffer(6); 79 | assertNotSame(b, msg.initialPayload); 80 | assertNotSame(b, msg.tail); 81 | assertEquals(12345, b.getInt()); 82 | assertEquals(1, b.get()); 83 | assertEquals(2, b.get()); 84 | assertEquals(0, msg.initialPayload.remaining()); 85 | assertEquals(4, msg.tail.remaining()); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /java/grpcHoma/src/test/java/grpcHoma/HomaSocketTest.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021-2022 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import java.nio.ByteBuffer; 19 | import java.net.*; 20 | 21 | import org.junit.jupiter.api.AfterEach; 22 | import org.junit.jupiter.api.Test; 23 | import static org.junit.jupiter.api.Assertions.*; 24 | 25 | public class HomaSocketTest { 26 | HomaSocket serverSock; 27 | HomaSocket clientSock; 28 | 29 | @AfterEach 30 | void tearDown() { 31 | if (serverSock != null) { 32 | serverSock.close(); 33 | serverSock = null; 34 | } 35 | if (clientSock != null) { 36 | clientSock.close(); 37 | clientSock = null; 38 | } 39 | } 40 | 41 | @Test 42 | public void test_HomaSocket_constructor__bad_port() { 43 | HomaSocket.HomaError error = assertThrows( 44 | HomaSocket.HomaError.class, () -> { 45 | new HomaSocket(100000); 46 | }); 47 | assertEquals("Couldn't create Homa socket for port 100000: " 48 | + "Invalid argument", error.getMessage()); 49 | } 50 | 51 | @Test 52 | public void test_HomaSocket__basic_roundtrip() 53 | throws UnknownHostException { 54 | serverSock = new HomaSocket(5555); 55 | clientSock = new HomaSocket(0); 56 | HomaSocket.RpcSpec outgoingSpec = new HomaSocket.RpcSpec( 57 | new InetSocketAddress("localhost", 5555)); 58 | HomaSocket.RpcSpec incomingSpec = new HomaSocket.RpcSpec(); 59 | ByteBuffer clientBuffer = ByteBuffer.allocateDirect(100); 60 | ByteBuffer serverBuffer = ByteBuffer.allocateDirect(100); 61 | clientBuffer.putInt(11111); 62 | clientBuffer.putInt(22222); 63 | 64 | assertTrue(clientSock.send(outgoingSpec, clientBuffer) > 0); 65 | int result = serverSock.receive(serverBuffer, 66 | HomaSocket.flagReceiveRequest, incomingSpec); 67 | assertEquals(8, result); 68 | assertEquals(11111, serverBuffer.getInt(0)); 69 | assertEquals(22222, serverBuffer.getInt(4)); 70 | 71 | serverBuffer.clear(); 72 | serverBuffer.putInt(3333); 73 | serverBuffer.putInt(4444); 74 | serverBuffer.putInt(5555); 75 | assertEquals(0, serverSock.reply(incomingSpec, serverBuffer)); 76 | result = clientSock.receive(clientBuffer, 0, outgoingSpec); 77 | assertEquals(12, result); 78 | assertEquals(3333, clientBuffer.getInt()); 79 | assertEquals(4444, clientBuffer.getInt()); 80 | assertEquals(5555, clientBuffer.getInt()); 81 | } 82 | 83 | @Test 84 | public void test_HomaSocket_send_to_unconnected_port() 85 | throws UnknownHostException { 86 | clientSock = new HomaSocket(0); 87 | HomaSocket.RpcSpec spec = new HomaSocket.RpcSpec( 88 | new InetSocketAddress("localhost", 9876)); 89 | ByteBuffer buffer = ByteBuffer.allocateDirect(100); 90 | buffer.putInt(12345); 91 | 92 | clientSock.send(spec, buffer); 93 | spec.reset(); 94 | int result = clientSock.receive(buffer, 95 | HomaSocket.flagReceiveResponse, spec); 96 | assertEquals("Transport endpoint is not connected", 97 | HomaSocket.strerror(-result)); 98 | } 99 | 100 | 101 | @Test 102 | public void test_HomaSocket_receive_bogus_id() 103 | throws UnknownHostException { 104 | clientSock = new HomaSocket(0); 105 | HomaSocket.RpcSpec spec = new HomaSocket.RpcSpec( 106 | new InetSocketAddress("localhost", 9876)); 107 | ByteBuffer buffer = ByteBuffer.allocateDirect(100); 108 | buffer.putInt(12345); 109 | 110 | clientSock.send(spec, buffer); 111 | spec.spec.putLong(HomaSocket.RpcSpec.idOffset, spec.getId()+1); 112 | int result = clientSock.receive(buffer, 0, spec); 113 | assertEquals("Invalid argument", HomaSocket.strerror(-result)); 114 | } 115 | 116 | @Test 117 | public void test_HomaSocket__receive_in_pieces() 118 | throws UnknownHostException { 119 | serverSock = new HomaSocket(5555); 120 | clientSock = new HomaSocket(0); 121 | HomaSocket.RpcSpec outgoingSpec = new HomaSocket.RpcSpec( 122 | new InetSocketAddress("localhost", 5555)); 123 | ByteBuffer sendBuffer = ByteBuffer.allocateDirect(100); 124 | sendBuffer.putInt(11111); 125 | sendBuffer.putInt(22222); 126 | sendBuffer.putInt(33333); 127 | sendBuffer.putInt(44444); 128 | assertTrue(clientSock.send(outgoingSpec, sendBuffer) > 0); 129 | 130 | HomaSocket.RpcSpec incomingSpec = new HomaSocket.RpcSpec(); 131 | ByteBuffer receiveBuffer = ByteBuffer.allocateDirect(100); 132 | receiveBuffer.limit(12); 133 | int result = serverSock.receive(receiveBuffer, 134 | HomaSocket.flagReceiveRequest|HomaSocket.flagReceivePartial, 135 | incomingSpec); 136 | assertEquals(16, result); 137 | assertEquals(12, receiveBuffer.limit()); 138 | assertEquals(11111, receiveBuffer.getInt(0)); 139 | assertEquals(22222, receiveBuffer.getInt(4)); 140 | assertEquals(33333, receiveBuffer.getInt(8)); 141 | 142 | receiveBuffer.clear(); 143 | result = serverSock.receive(receiveBuffer, HomaSocket.flagReceivePartial, 144 | incomingSpec); 145 | assertEquals(16, result); 146 | assertEquals(4, receiveBuffer.limit()); 147 | assertEquals(44444, receiveBuffer.getInt(0)); 148 | } 149 | 150 | @Test 151 | public void test_strerror() { 152 | assertEquals("No such file or directory", HomaSocket.strerror(2)); 153 | assertEquals("Broken pipe", HomaSocket.strerror(32)); 154 | } 155 | } -------------------------------------------------------------------------------- /java/homaJni/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This module contains C++ code implementing a JNI interface 3 | * for Homa system calls. 4 | */ 5 | 6 | plugins { 7 | id 'cpp-library' 8 | } 9 | 10 | library { 11 | targetMachines = [machines.linux.x86_64] 12 | 13 | tasks.withType(CppCompile).configureEach { 14 | compilerArgs.add '-Wall' 15 | compilerArgs.add '-Werror' 16 | compilerArgs.add '-fno-strict-aliasing' 17 | compilerArgs.add '-fPIC' 18 | compilerArgs.add '-I' + System.getenv("JAVA_HOME") + '/include' 19 | compilerArgs.add '-I' + System.getenv("JAVA_HOME") + '/include/linux' 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/homaJni/src/main/cpp/homa.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * This file implements JNI functions for invoking Homa kernel calls. 3 | * It can be accessed from Java through the HomaSocket class; the APIs 4 | * are all documented in that class. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | #include "homa.h" 15 | 16 | /** 17 | * This class defines the layout of the @spec ByteBuffer in Java RpcSpecs. 18 | * Structural changes here must be reflected in HomaSocket.RpcSpec 19 | * and vice versa. 20 | */ 21 | struct Spec { 22 | // Peer address, network byte order. 23 | int addr; 24 | 25 | // Port number on peer. 26 | int port; 27 | 28 | // RPC sequence number. 29 | uint64_t id; 30 | 31 | // Number of bytes received in the most recent call to receive. 32 | int length; 33 | }; 34 | 35 | /** 36 | * JNI implementation of HomaSocket.socketNative: opens a socket. 37 | * \param env 38 | * Used to access information in the Java environment (not used). 39 | * \param jHomaSocket. 40 | * Info about the HomaSocket class from which this method was invoked 41 | * (not used). 42 | * \param jPort 43 | * Port number to use for the socket; 0 means that Homa should 44 | * assign a number automatically. 45 | * \return 46 | * The fd assigned to the socket, or a negative errno value. 47 | */ 48 | extern "C" JNIEXPORT jint Java_grpcHoma_HomaSocket_socketNative(JNIEnv *env, 49 | jclass jHomaSocket, jint jPort) { 50 | int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_HOMA); 51 | if ((fd >= 0) && (jPort > 0)) { 52 | struct sockaddr_in addr; 53 | memset(&addr, 0, sizeof(addr)); 54 | addr.sin_family = AF_INET; 55 | addr.sin_port = htons(jPort); 56 | if (bind(fd, reinterpret_cast(&addr), 57 | sizeof(addr)) != 0) { 58 | return -errno; 59 | } 60 | } 61 | return fd; 62 | } 63 | 64 | /** 65 | * JNI implementation of HomaSocket.closeNative: closes a socket. 66 | * \param env 67 | * Used to access information in the Java environment (not used). 68 | * \param jHomaSocket. 69 | * Info about the HomaSocket class from which this method was invoked 70 | * (not used). 71 | * \param fFd 72 | * Homa socket to close. 73 | */ 74 | extern "C" JNIEXPORT void Java_grpcHoma_HomaSocket_closeNative(JNIEnv *env, 75 | jclass jHomaSocket, jint jFd) { 76 | close(jFd); 77 | } 78 | 79 | /** 80 | * JNI implementation of HomaSocket.strerror: return a human-readable 81 | * string for a given errno. 82 | * \param env 83 | * Used to access information in the Java environment (not used). 84 | * \param jHomaSocket. 85 | * Info about the HomaSocket class from which this method was invoked 86 | * (not used). 87 | * \param jErrno 88 | * Errno value returned by some other method. 89 | */ 90 | extern "C" JNIEXPORT jstring Java_grpcHoma_HomaSocket_strerrorNative( 91 | JNIEnv *env, jclass jHomaSocket, jint jErrno) { 92 | return env->NewStringUTF(strerror(jErrno)); 93 | } 94 | 95 | /** 96 | * JNI implementation of HomaSocket.sendNative: sends the request 97 | * message for a new outgoing Homa RPC. 98 | * \param env 99 | * Used to access information in the Java environment (not used). 100 | * \param jHomaSocket. 101 | * Info about the HomaSocket class from which this method was invoked 102 | * (not used). 103 | * \param fFd 104 | * Homa socket to use for sending the message (previous return value 105 | * from socket). 106 | * \param jBuffer 107 | * ByteBuffer containing the contents of the message (none of the 108 | * metadata for this buffer is modified). 109 | * \param jLength 110 | * Number of bytes in the message in jBuffer. 111 | * \param jSpec 112 | * Reference to the a HomaSpec.spec ByteBuffer: address and port 113 | * identify the target for the RPC, and the id will be filled in 114 | * by this method. 115 | * \return 116 | * Returns the id of the new request on success, or a negative errno value 117 | * if there was an error. 118 | */ 119 | extern "C" JNIEXPORT jlong Java_grpcHoma_HomaSocket_sendNative(JNIEnv *env, 120 | jclass jHomaSocket, jint jFd, jobject jBuffer, jint jLength, 121 | jobject jSpec) { 122 | void *buffer = env->GetDirectBufferAddress(jBuffer); 123 | Spec *spec = static_cast(env->GetDirectBufferAddress(jSpec)); 124 | if ((buffer == NULL) || (spec == NULL)) { 125 | return -EINVAL; 126 | } 127 | struct sockaddr_in addr; 128 | addr.sin_family = AF_INET; 129 | addr.sin_addr.s_addr = spec->addr; 130 | addr.sin_port = htons(spec->port); 131 | 132 | struct sockaddr *saddr = reinterpret_cast (&addr); 133 | if (homa_send(jFd, buffer, jLength, saddr, sizeof(addr), &spec->id) == 0) { 134 | return spec->id; 135 | } else { 136 | return -errno; 137 | } 138 | } 139 | 140 | /** 141 | * JNI implementation of HomaSocket.replyNative: sends the response 142 | * message for an RPC. 143 | * \param env 144 | * Used to access information in the Java environment (not used). 145 | * \param jHomaSocket. 146 | * Info about the HomaSocket class from which this method was invoked 147 | * (not used). 148 | * \param fFd 149 | * Homa socket to use for sending the response (previous return value 150 | * from socket). 151 | * \param jBuffer 152 | * ByteBuffer containing the response message (none of the metadata 153 | * for this buffer is modified). 154 | * \param jLength 155 | * Number of bytes in the message in jBuffer. 156 | * \param jSpec 157 | * Reference to the a HomaSpec.spec ByteBuffer: identifies the client 158 | * that issued the RPC (as set by receiveNative). 159 | * \return 160 | * Returns zero for success or a negative errno value if there 161 | * was an error. 162 | */ 163 | extern "C" JNIEXPORT jlong Java_grpcHoma_HomaSocket_replyNative(JNIEnv *env, 164 | jclass jHomaSocket, jint jFd, jobject jBuffer, jint jLength, 165 | jobject jSpec) { 166 | void *buffer = env->GetDirectBufferAddress(jBuffer); 167 | Spec *spec = static_cast(env->GetDirectBufferAddress(jSpec)); 168 | if ((buffer == NULL) || (spec == NULL)) { 169 | return -EINVAL; 170 | } 171 | struct sockaddr_in addr; 172 | addr.sin_family = AF_INET; 173 | addr.sin_addr.s_addr = spec->addr; 174 | addr.sin_port = htons(spec->port); 175 | 176 | struct sockaddr *saddr = reinterpret_cast (&addr); 177 | if (homa_reply(jFd, buffer, jLength, saddr, sizeof(addr), spec->id) 178 | == 0) { 179 | return 0; 180 | } else { 181 | return -errno; 182 | } 183 | } 184 | 185 | /** 186 | * JNI implementation of HomaSocket.receiveNative: receives an incoming 187 | * message. 188 | * \param env 189 | * Used to access information in the Java environment (not used). 190 | * \param jHomaSocket. 191 | * Info about the HomaSocket class from which this method was invoked 192 | * (not used). 193 | * \param fFd 194 | * Homa socket to use for receiving the message (previous return value 195 | * from socket). 196 | * \param flags 197 | * OR-ed collection of bits that control receive operation; same 198 | * as the @flag passed to homa_recv. 199 | * \param jBuffer 200 | * ByteBuffer in which to store the incoming message (none of the 201 | * metadata for this buffer is modified). 202 | * \param jLength 203 | * Number of bytes available in jBuffer. 204 | * \param jSpec 205 | * Reference to the a HomaSpec.spec ByteBuffer: if the id in this 206 | * structure is nonzero, then this structure describes a specific 207 | * RPC, which may be received even if not selected by @flags. 208 | * This will be modified to describe the incoming message (including 209 | * the length field). 210 | * \return 211 | * The total number of bytes in the message (may be more than the 212 | * number actually received), or a negative errno value if there 213 | * was an error. 214 | */ 215 | extern "C" JNIEXPORT jlong Java_grpcHoma_HomaSocket_receiveNative(JNIEnv *env, 216 | jclass jHomaSocket, jint jFd, jint jFlags, jobject jBuffer, 217 | jint jLength, jobject jSpec) { 218 | void *buffer = env->GetDirectBufferAddress(jBuffer); 219 | Spec *spec = static_cast(env->GetDirectBufferAddress(jSpec)); 220 | if ((buffer == NULL) || (spec == NULL)) { 221 | return -EINVAL; 222 | } 223 | struct sockaddr_in addr; 224 | addr.sin_family = AF_INET; 225 | addr.sin_addr.s_addr = spec->addr; 226 | addr.sin_port = ntohs(spec->port); 227 | size_t addrLength = sizeof(addr); 228 | size_t msgLength; 229 | 230 | struct sockaddr *saddr = reinterpret_cast (&addr); 231 | int result = homa_recv(jFd, buffer, jLength, jFlags, saddr, &addrLength, 232 | &spec->id, &msgLength); 233 | if (result < 0) { 234 | return -errno; 235 | } 236 | spec->addr = addr.sin_addr.s_addr; 237 | spec->port = ntohs(addr.sin_port); 238 | spec->length = result; 239 | return msgLength; 240 | } 241 | -------------------------------------------------------------------------------- /java/homaJni/src/main/cpp/homa.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019-2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and/or distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | /* This file defines the kernel call interface for the Homa 17 | * transport protocol. 18 | */ 19 | 20 | #ifndef _HOMA_H 21 | #define _HOMA_H 22 | 23 | #include 24 | #ifndef __KERNEL__ 25 | #include 26 | #include 27 | #endif 28 | 29 | #ifdef __cplusplus 30 | extern "C" 31 | { 32 | #endif 33 | 34 | /* Homa's protocol number within the IP protocol space (this is not an 35 | * officially allocated slot). 36 | */ 37 | #define IPPROTO_HOMA 140 38 | 39 | /** 40 | * define HOMA_MAX_MESSAGE_LENGTH - Maximum bytes of payload in a Homa 41 | * request or response message. 42 | */ 43 | #define HOMA_MAX_MESSAGE_LENGTH 1000000 44 | 45 | /** 46 | * define HOMA_MIN_DEFAULT_PORT - The 16-bit port space is divided into 47 | * two nonoverlapping regions. Ports 1-32767 are reserved exclusively 48 | * for well-defined server ports. The remaining ports are used for client 49 | * ports; these are allocated automatically by Homa. Port 0 is reserved. 50 | */ 51 | #define HOMA_MIN_DEFAULT_PORT 0x8000 52 | 53 | /** 54 | * I/O control calls on Homa sockets. These particular values were 55 | * chosen somewhat randomly, and probably need to be reconsidered to 56 | * make sure they don't conflict with anything else. 57 | */ 58 | 59 | #define HOMAIOCSEND 1003101 60 | #define HOMAIOCRECV 1003102 61 | #define HOMAIOCREPLY 1003103 62 | #define HOMAIOCABORT 1003104 63 | #define HOMAIOCFREEZE 1003105 64 | 65 | extern int homa_send(int sockfd, const void *request, size_t reqlen, 66 | const struct sockaddr *dest_addr, size_t addrlen, 67 | uint64_t *id); 68 | extern int homa_sendv(int sockfd, const struct iovec *iov, int iovcnt, 69 | const struct sockaddr *dest_addr, size_t addrlen, 70 | uint64_t *id); 71 | extern ssize_t homa_recv(int sockfd, void *buf, size_t len, int flags, 72 | struct sockaddr *src_addr, size_t *addrlen, 73 | uint64_t *id, size_t *msglen); 74 | extern ssize_t homa_recvv(int sockfd, const struct iovec *iov, int iovcnt, 75 | int flags, struct sockaddr *src_addr, size_t *addrlen, 76 | uint64_t *id, size_t *msglen); 77 | extern ssize_t homa_reply(int sockfd, const void *response, size_t resplen, 78 | const struct sockaddr *dest_addr, size_t addrlen, 79 | uint64_t id); 80 | extern ssize_t homa_replyv(int sockfd, const struct iovec *iov, int iovcnt, 81 | const struct sockaddr *dest_addr, size_t addrlen, 82 | uint64_t id); 83 | extern int homa_abort(int sockfd, uint64_t id); 84 | 85 | /** 86 | * define homa_args_send_ipv4 - Structure that passes arguments and results 87 | * betweeen homa_send and the HOMAIOCSEND ioctl. Assumes IPV4 addresses. 88 | */ 89 | struct homa_args_send_ipv4 { 90 | // Exactly one of request and iovec will be non-null. 91 | void *request; 92 | const struct iovec *iovec; 93 | 94 | // The number of bytes at *request, or the number of elements at *iovec. 95 | size_t length; 96 | struct sockaddr_in dest_addr; 97 | __u64 id; 98 | }; 99 | 100 | /** 101 | * define homa_args_recv_ipv4 - Structure that passes arguments and results 102 | * betweeen homa_recv and the HOMAIOCRECV ioctl. Assumes IPV4 addresses. 103 | */ 104 | struct homa_args_recv_ipv4 { 105 | // Exactly one of buf and iovec will be non-null. 106 | void *buf; 107 | const struct iovec *iovec; 108 | 109 | // Initially holds length of @buf or @iovec; modified to return total 110 | // message length. 111 | size_t len; 112 | struct sockaddr_in source_addr; 113 | int flags; 114 | __u64 requestedId; 115 | __u64 actualId; 116 | int type; 117 | }; 118 | 119 | /* Flag bits for homa_recv (see man page for documentation): 120 | */ 121 | #define HOMA_RECV_REQUEST 0x01 122 | #define HOMA_RECV_RESPONSE 0x02 123 | #define HOMA_RECV_NONBLOCKING 0x04 124 | #define HOMA_RECV_PARTIAL 0x08 125 | 126 | /** 127 | * define homa_args_reply_ipv4 - Structure that passes arguments and results 128 | * betweeen homa_reply and the HOMAIOCREPLY ioctl. Assumes IPV4 addresses. 129 | */ 130 | struct homa_args_reply_ipv4 { 131 | // Exactly one of response and iovec will be non-null. 132 | void *response; 133 | const struct iovec *iovec; 134 | 135 | // The number of bytes at *response, or the number of elements at *iovec. 136 | size_t length; 137 | struct sockaddr_in dest_addr; 138 | __u64 id; 139 | }; 140 | 141 | /** 142 | * Meanings of the bits in Homa's flag word, which can be set using 143 | * "sysctl /net/homa/flags". 144 | */ 145 | 146 | /** 147 | * Disable the output throttling mechanism: always send all packets 148 | * immediately. 149 | */ 150 | #define HOMA_FLAG_DONT_THROTTLE 2 151 | 152 | #ifdef __cplusplus 153 | } 154 | #endif 155 | 156 | #endif /* _HOMA_H */ -------------------------------------------------------------------------------- /java/rsync_exclude.txt: -------------------------------------------------------------------------------- 1 | # This file lists directories and files that should not be copied when 2 | # updating a remote machine with grpc_homa java info. 3 | .git 4 | .gradle 5 | buildSrc 6 | gradle* 7 | build 8 | -------------------------------------------------------------------------------- /java/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/7.3/userguide/multi_project_builds.html 8 | */ 9 | 10 | rootProject.name = 'grpc_homa' 11 | include('homaJni', 'grpcHoma', 'testApp') 12 | -------------------------------------------------------------------------------- /java/testApp/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'application' 3 | id 'com.google.protobuf' version '0.8.18' 4 | } 5 | 6 | repositories { 7 | mavenCentral() 8 | } 9 | 10 | // Location of local gRPC files 11 | // def grpcDirectory = '../../grpc-java' 12 | def grpcDirectory = '/ouster/grpc-java' 13 | 14 | dependencies { 15 | implementation project(':grpcHoma') 16 | // implementation "io.grpc:grpc-protobuf:1.41.0" 17 | // implementation "io.grpc:grpc-stub:1.41.0" 18 | implementation "javax.annotation:javax.annotation-api:1.3.2" 19 | implementation("com.google.guava:guava:31.0.1-jre") 20 | 21 | // Use local copy of gRPC rather than distributions (for debugging). 22 | // runtimeOnly "io.grpc:grpc-netty-shaded:1.41.0" 23 | implementation files( 24 | "${grpcDirectory}/protobuf/build/libs/grpc-protobuf-1.41.0.jar", 25 | "${grpcDirectory}/api/build/libs/grpc-api-1.41.0.jar", 26 | "${grpcDirectory}/context/build/libs/grpc-context-1.41.0.jar", 27 | "${grpcDirectory}/core/build/libs/grpc-core-1.41.0.jar", 28 | "${grpcDirectory}/netty/shaded/build/libs/grpc-netty-shaded-1.41.0.jar", 29 | //"${grpcDirectory}/netty/build/libs/grpc-netty-1.41.0.jar", 30 | "${grpcDirectory}/stub/build/libs/grpc-stub-1.41.0.jar", 31 | "${grpcDirectory}/protobuf-lite/build/libs/grpc-protobuf-lite-1.41.0.jar", 32 | "${grpcDirectory}/benchmarks/build/install/grpc-benchmarks/lib/protobuf-java-3.17.2.jar" 33 | ) 34 | runtimeOnly "io.perfmark:perfmark-api:0.23.0" 35 | } 36 | 37 | protobuf { 38 | protoc { artifact = "com.google.protobuf:protoc:3.19.1" } 39 | plugins { 40 | grpc { artifact = "io.grpc:protoc-gen-grpc-java:1.41.0" } 41 | } 42 | generateProtoTasks { 43 | all()*.plugins { grpc {} } 44 | } 45 | } 46 | 47 | application { 48 | // Define the main class for the application. 49 | mainClass = 'grpcHoma.Main' 50 | } 51 | -------------------------------------------------------------------------------- /java/testApp/src/main/java/grpcHoma/BasicClient.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package basic; 17 | 18 | import io.grpc.*; 19 | import io.grpc.stub.StreamObserver; 20 | 21 | /** 22 | * An instance of this class can be used to issue RPCs from basic.proto 23 | * to a particular target (server and port). 24 | */ 25 | public class BasicClient { 26 | public BasicClient(Channel channel) { 27 | stub = BasicGrpc.newBlockingStub(channel); 28 | } 29 | 30 | /** 31 | * Issue a simple RPC that sends a block of data to the server 32 | * and receives another block in response. 33 | * @param requestLength 34 | * Number of bytes in the request message. 35 | * @param replyLength 36 | * Number of bytes in the response message. 37 | * @throws InterruptedException 38 | */ 39 | public void ping(int requestLength, int replyLength) 40 | throws InterruptedException { 41 | BasicProto.Request.Builder builder = BasicProto.Request.newBuilder(); 42 | builder.setRequestItems(requestLength); 43 | builder.setReplyItems(replyLength); 44 | for (int i = 0; i < requestLength; i++) { 45 | builder.addData(i); 46 | } 47 | try { 48 | BasicProto.Response response = stub.ping(builder.build()); 49 | if (response.getDataCount() != replyLength) { 50 | System.out.printf("Ping returned %d bytes, expected %d\n", 51 | response.getDataCount(), replyLength); 52 | } 53 | } catch (StatusRuntimeException e) { 54 | System.out.printf("Ping RPC failed: %s\n", e.getStatus()); 55 | Thread.sleep(1000); 56 | } 57 | } 58 | 59 | protected BasicGrpc.BasicBlockingStub stub; 60 | } -------------------------------------------------------------------------------- /java/testApp/src/main/java/grpcHoma/BasicServer.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package basic; 17 | 18 | import java.io.IOException; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | import grpcHoma.HomaServerBuilder; 22 | import io.grpc.Server; 23 | import io.grpc.ServerBuilder; 24 | import io.grpc.stub.StreamObserver; 25 | 26 | /** 27 | * An object of this class represents a server that listens on a 28 | * given port and handles requests with the interface specified 29 | * by basic.proto. 30 | */ 31 | public class BasicServer { 32 | /** 33 | * Construct a BasicServer. 34 | * @param port 35 | * Port number that the service should listen on. 36 | * @param useHoma 37 | * True means that Homa should be used for transport; false means TCP. 38 | */ 39 | public BasicServer(int port, boolean useHoma) throws IOException { 40 | if (useHoma) { 41 | server = HomaServerBuilder.forPort(port) 42 | .addService(new BasicService()) 43 | .build(); 44 | } else { 45 | server = ServerBuilder.forPort(port) 46 | .addService(new BasicService()) 47 | .build(); 48 | } 49 | server.start(); 50 | } 51 | 52 | /** 53 | * Shut down the server, so that it no longer responds to requests. 54 | */ 55 | void close() { 56 | server.shutdownNow(); 57 | try { 58 | server.awaitTermination(5, TimeUnit.SECONDS); 59 | } catch (InterruptedException e) { 60 | // Ignore exception and return. 61 | } 62 | } 63 | 64 | protected static class BasicService extends BasicGrpc.BasicImplBase { 65 | @Override 66 | public void ping(BasicProto.Request request, 67 | StreamObserver responseObserver) { 68 | System.out.printf("BasicService.ping invoked\n"); 69 | if (request.getDataCount() != request.getRequestItems()) { 70 | System.out.printf("Expected %d items in Ping request, got %d\n", 71 | request.getRequestItems(), request.getDataCount()); 72 | } 73 | BasicProto.Response.Builder builder = 74 | BasicProto.Response.newBuilder(); 75 | int length = request.getReplyItems(); 76 | for (int i = 0; i < length; i++) { 77 | builder.addData(i); 78 | } 79 | responseObserver.onNext(builder.build()); 80 | responseObserver.onCompleted(); 81 | } 82 | } 83 | 84 | // gRPC server associated with this object. 85 | Server server; 86 | } -------------------------------------------------------------------------------- /java/testApp/src/main/java/grpcHoma/HomaChannelBuilder.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | package grpcHoma; 17 | 18 | import io.grpc.ChannelCredentials; 19 | import io.grpc.ChannelLogger; 20 | import io.grpc.ManagedChannelBuilder; 21 | import io.grpc.internal.*; 22 | 23 | import java.net.SocketAddress; 24 | import java.util.concurrent.ScheduledExecutorService; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | /** 28 | * This class just contains boilerplate used to create Homa channels. It is 29 | * based heavily on InProcessChannelBuilder and NettyChannelBuilder from gRPC; 30 | * it's not documented very well because the whole mechanism doesn't really 31 | * make much sense to me :-(. 32 | */ 33 | public final class HomaChannelBuilder 34 | extends AbstractManagedChannelImplBuilder { 35 | 36 | private final ManagedChannelImplBuilder managedChannelImplBuilder; 37 | 38 | /** 39 | * Constructor for HomaChannelBuilders. 40 | * @param target 41 | * Server associated with this channel, in the form host:port. 42 | */ 43 | public HomaChannelBuilder(String target) { 44 | managedChannelImplBuilder = new ManagedChannelImplBuilder(target, 45 | new HomaChannelTransportFactoryBuilder(), 46 | null); 47 | } 48 | 49 | /** 50 | * Creates a new channel builder for a given target. 51 | * @param target 52 | * Identifies the server associated with this builder (host:port). 53 | */ 54 | public static HomaChannelBuilder forTarget(String target) { 55 | return new HomaChannelBuilder(target); 56 | } 57 | 58 | @Override 59 | protected ManagedChannelBuilder delegate() { 60 | return managedChannelImplBuilder; 61 | } 62 | 63 | ClientTransportFactory buildTransportFactory() { 64 | return new HomaChannelBuilder.HomaClientTransportFactory(); 65 | } 66 | 67 | private final class HomaChannelTransportFactoryBuilder 68 | implements ManagedChannelImplBuilder.ClientTransportFactoryBuilder { 69 | @Override 70 | public ClientTransportFactory buildClientTransportFactory() { 71 | return buildTransportFactory(); 72 | } 73 | } 74 | 75 | /** 76 | * Creates Homa transports. 77 | */ 78 | static final class HomaClientTransportFactory implements ClientTransportFactory { 79 | // True means this factory can no longer be used to create channels. 80 | private boolean closed; 81 | 82 | private ScheduledExecutorService schedExecService; 83 | 84 | public HomaClientTransportFactory() { 85 | closed = false; 86 | schedExecService = SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE); 87 | } 88 | 89 | @Override 90 | public ConnectionClientTransport newClientTransport(SocketAddress addr, 91 | ClientTransportOptions options, ChannelLogger channelLogger) { 92 | if (closed) { 93 | throw new IllegalStateException( 94 | "The Homa transport factory is closed."); 95 | } 96 | return new HomaClientTransport(HomaClient.getInstance(channelLogger), 97 | addr, options, channelLogger); 98 | } 99 | 100 | @Override 101 | public ScheduledExecutorService getScheduledExecutorService() { 102 | return schedExecService; 103 | } 104 | 105 | @Override 106 | public SwapChannelCredentialsResult swapChannelCredentials( 107 | ChannelCredentials channelCreds) { 108 | return null; 109 | } 110 | 111 | @Override 112 | public void close() { 113 | closed = true; 114 | SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, schedExecService); 115 | schedExecService = null; 116 | } 117 | } 118 | } -------------------------------------------------------------------------------- /java/testApp/src/main/proto/basic.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffer definitions used by the testApp for gRPC Homa 2 | // support in Java. Allows testing all the variations combinations 3 | // of streaming in and out. 4 | 5 | syntax = "proto3"; 6 | 7 | package basic; 8 | 9 | option java_outer_classname = "BasicProto"; 10 | 11 | service Basic { 12 | rpc Ping(Request) returns (Response) {} 13 | rpc StreamOut(stream StreamOutRequest) returns (Response) {} 14 | rpc StreamIn(StreamInRequest) returns (stream Response) {} 15 | rpc Stream2Way(stream StreamOutRequest) returns (stream Response) {} 16 | rpc PrintLog(Empty) returns (Empty) {} 17 | } 18 | 19 | message Request { 20 | sfixed32 requestItems = 1; 21 | sfixed32 replyItems = 2; 22 | repeated sfixed32 data = 3; 23 | } 24 | 25 | message Response { 26 | repeated sfixed32 data = 1; 27 | } 28 | 29 | message StreamOutRequest { 30 | sfixed32 done = 1; 31 | sfixed32 requestItems = 2; 32 | sfixed32 replyItems = 3; 33 | repeated sfixed32 data = 4; 34 | } 35 | 36 | message StreamInRequest { 37 | repeated sfixed32 sizes = 1; 38 | } 39 | 40 | message Empty {} 41 | -------------------------------------------------------------------------------- /java/updateRemote: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This script copies modified information from this directory to a 4 | # CloudLab machine. 5 | 6 | target=`cat $HOME/.cloudlabNode` 7 | rsync -rtv --exclude-from=rsync_exclude.txt ./ $target:/ouster/java/ -------------------------------------------------------------------------------- /mergedep.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | # Copyright 2003 Bryan Ford 3 | # Distributed under the GNU General Public License. 4 | # 5 | # Usage: mergedep [ ...] 6 | # 7 | # This script merges the contents of all specified 8 | # on the command line into the single file , 9 | # which may or may not previously exist. 10 | # Dependencies in the will override 11 | # any existing dependencies for the same targets in . 12 | # The are deleted after is updated. 13 | # 14 | # The are typically generated by GCC with the -MD option, 15 | # and the is typically included from a Makefile, 16 | # as shown here for GNU 'make': 17 | # 18 | # .deps: $(wildcard *.d) 19 | # perl mergedep $@ $^ 20 | # -include .deps 21 | # 22 | # This script properly handles multiple dependencies per , 23 | # including dependencies having no target, 24 | # so it is compatible with GCC3's -MP option. 25 | # 26 | 27 | sub readdeps { 28 | my $filename = shift; 29 | 30 | open(DEPFILE, $filename) or return 0; 31 | while () { 32 | if (/([^:]*):([^\\:]*)([\\]?)$/) { 33 | my $target = $1; 34 | my $deplines = $2; 35 | my $slash = $3; 36 | while ($slash ne '') { 37 | $_ = ; 38 | defined($_) or die 39 | "Unterminated dependency in $filename"; 40 | /(^[ \t][^\\]*)([\\]?)$/ or die 41 | "Bad continuation line in $filename"; 42 | $deplines = "$deplines\\\n$1"; 43 | $slash = $2; 44 | } 45 | #print "DEPENDENCY [[$target]]: [[$deplines]]\n"; 46 | $dephash{$target} = $deplines; 47 | } elsif (/^[#]?[ \t]*$/) { 48 | # ignore blank lines and comments 49 | } else { 50 | die "Bad dependency line in $filename: $_"; 51 | } 52 | } 53 | close DEPFILE; 54 | return 1; 55 | } 56 | 57 | 58 | if ($#ARGV < 0) { 59 | print "Usage: mergedep [ ..]\n"; 60 | exit(1); 61 | } 62 | 63 | %dephash = (); 64 | 65 | # Read the main dependency file 66 | $maindeps = $ARGV[0]; 67 | readdeps($maindeps); 68 | 69 | # Read and merge in the new dependency files 70 | foreach $i (1 .. $#ARGV) { 71 | readdeps($ARGV[$i]) or die "Can't open $ARGV[$i]"; 72 | } 73 | 74 | # Update the main dependency file 75 | open(DEPFILE, ">$maindeps.tmp") or die "Can't open output file $maindeps.tmp"; 76 | foreach $target (keys %dephash) { 77 | print DEPFILE "$target:$dephash{$target}"; 78 | } 79 | close DEPFILE; 80 | rename("$maindeps.tmp", "$maindeps") or die "Can't overwrite $maindeps"; 81 | 82 | # Finally, delete the new dependency files 83 | foreach $i (1 .. $#ARGV) { 84 | unlink($ARGV[$i]) or print "Error removing $ARGV[$i]\n"; 85 | } 86 | 87 | -------------------------------------------------------------------------------- /mock.h: -------------------------------------------------------------------------------- 1 | #ifndef MOCK_H 2 | #define MOCK_H 3 | 4 | #include 5 | #include 6 | 7 | #include "grpc/impl/codegen/log.h" 8 | 9 | #include "gtest/gtest.h" 10 | 11 | #include "homa.h" 12 | 13 | #include "wire.h" 14 | 15 | /* This class defines additional variables and functions that can be used 16 | * by unit tests as part of mocking out system features. 17 | */ 18 | class Mock { 19 | public: 20 | // Used by some methods as the errno to return after a simulated failure. 21 | static int errorCode; 22 | 23 | // The variables below can be set to non-zero values by unit tests in order 24 | // to simulate error returns from various functions. If bit 0 is set to 1, 25 | // the next call to the function will fail; bit 1 corresponds to the next 26 | // call after that, and so on. 27 | static int homaReplyErrors; 28 | static int homaReplyvErrors; 29 | static int homaSendvErrors; 30 | static int recvmsgErrors; 31 | 32 | static int buffersReturned; 33 | 34 | // Holds all messages sent by homa_sendv and homa_replyv. 35 | static std::deque> homaMessages; 36 | 37 | // Return info for upcoming invocations of homa_recv. 38 | static std::deque recvmsgHeaders; 39 | static std::deque recvmsgLengths; 40 | static std::deque recvmsgReturns; 41 | 42 | // Accumulates various information over the course of a test, which 43 | // can then be queried. 44 | static std::string log; 45 | 46 | // Buffer region for the Homa socket. 47 | static uint8_t *bufRegion; 48 | 49 | static int checkError(int *errorMask); 50 | static grpc_core::Slice 51 | dataSlice(size_t length, int firstValue); 52 | static void gprLog(gpr_log_func_args* args); 53 | static void logData(const char *separator, const void *data, 54 | int length); 55 | static void logMetadata(const char *separator, 56 | const grpc_metadata_batch *batch); 57 | static void logPrintf(const char *separator, const char* format, ...); 58 | static void logSliceBuffer(const char *separator, 59 | const grpc_core::SliceBuffer *sliceBuffer); 60 | static void metadataBatchAppend(grpc_metadata_batch* batch, 61 | const char *key, const char *value); 62 | static void setUp(void); 63 | static ::testing::AssertionResult 64 | substr(const std::string& s, 65 | const std::string& substring); 66 | }; 67 | 68 | #define EXPECT_SUBSTR(sub, str) EXPECT_TRUE(Mock::substr((str), (sub))) 69 | #define EXPECT_NSUBSTR(sub, str) EXPECT_FALSE(Mock::substr((str), (sub))) 70 | 71 | #endif // MOCK_H 72 | -------------------------------------------------------------------------------- /rsync_exclude.txt: -------------------------------------------------------------------------------- 1 | # This file lists directories and files that should not be copied 2 | # to a remote machine by rsync. 3 | .git 4 | nbproject 5 | java 6 | *.pb.h -------------------------------------------------------------------------------- /stream_id.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "stream_id.h" 4 | 5 | /** 6 | * Construct an StreamId from a gRPC address. 7 | * \param gaddr 8 | * The address of the peer for the RPC. 9 | * \param id 10 | * Unique id assigned to this RPC by the client. 11 | */ 12 | StreamId::StreamId(grpc_resolved_address *gaddr, uint32_t id) 13 | : addr() 14 | , id(id) 15 | { 16 | GPR_ASSERT(gaddr->len <= sizeof(addr)); 17 | memset(&addr, 0, sizeof(addr)); 18 | memcpy(&addr, gaddr->addr, gaddr->len); 19 | } 20 | 21 | /** 22 | * Constructor for testing. 23 | * \param id 24 | * Identifier for this particular RPC. 25 | */ 26 | StreamId::StreamId(uint32_t id) 27 | : addr() 28 | , id(id) 29 | { 30 | addr.in6.sin6_family = AF_INET6; 31 | addr.in6.sin6_addr = in6addr_any; 32 | addr.in6.sin6_port = htons(40); 33 | } 34 | 35 | StreamId::StreamId(const StreamId &other) 36 | { 37 | addr = other.addr; 38 | id = other.id; 39 | } 40 | 41 | StreamId& StreamId::operator=(const StreamId &other) 42 | { 43 | addr = other.addr; 44 | id = other.id; 45 | return *this; 46 | } 47 | 48 | /** 49 | * Comparison operator for StreamIds. 50 | * \param other 51 | * StreamId to compare against 52 | * \return 53 | * True means the StreamIds match, false means they don't. 54 | */ 55 | bool StreamId::operator==(const StreamId& other) const 56 | { 57 | if (addr.in6.sin6_family == AF_INET6) { 58 | return (id == other.id) && 59 | (bcmp(&addr.in6, &other.addr.in6, sizeof(addr.in6)) == 0); 60 | } else { 61 | return (id == other.id) && 62 | (bcmp(&addr.in4, &other.addr.in4, sizeof(addr.in4)) == 0); 63 | } 64 | } 65 | 66 | /** 67 | * Return a string containing all of the information in this struct. 68 | */ 69 | std::string StreamId::toString() { 70 | char buf[INET6_ADDRSTRLEN]; 71 | const char *ntop_result = nullptr; 72 | if (addr.in6.sin6_family == AF_INET6) { 73 | ntop_result = inet_ntop(AF_INET6, &addr.in6.sin6_addr, buf, sizeof(buf)); 74 | } else if (addr.in4.sin_family == AF_INET) { 75 | ntop_result = inet_ntop(AF_INET, &addr.in4.sin_addr, buf, sizeof(buf)); 76 | } 77 | if (!ntop_result) { 78 | return "invalid address"; 79 | } 80 | char buf2[50]; 81 | snprintf(buf2, sizeof(buf2), ":%d, stream id %d", port(), id); 82 | std::string result(buf); 83 | result.append(buf2); 84 | return result; 85 | } -------------------------------------------------------------------------------- /stream_id.h: -------------------------------------------------------------------------------- 1 | #ifndef STREAM_ID_H 2 | #define STREAM_ID_H 3 | 4 | #include 5 | #include 6 | 7 | #include "homa.h" 8 | 9 | #include "src/core/lib/iomgr/resolve_address.h" 10 | 11 | /** 12 | * Holds information identifying a stream (which represents a gRPC RPC) 13 | * in a form that can be used as a key in std::unordered_map. 14 | */ 15 | struct StreamId { 16 | // Specifies the address and port of the other machine. 17 | sockaddr_in_union addr; 18 | 19 | // Unique id for this RPC among all those from its client. 20 | uint32_t id; 21 | 22 | StreamId() {} 23 | StreamId(uint32_t id); 24 | StreamId(const StreamId &other); 25 | StreamId(grpc_resolved_address *gaddr, uint32_t id); 26 | StreamId& operator=(const StreamId &other); 27 | bool operator==(const StreamId& other) const; 28 | 29 | std::string toString(); 30 | 31 | int port() 32 | { 33 | if (addr.in6.sin6_family == AF_INET6) { 34 | return htons(addr.in6.sin6_port); 35 | } else if (addr.in4.sin_family == AF_INET) { 36 | return htons(addr.in4.sin_port); 37 | } 38 | return -1; 39 | } 40 | 41 | /** 42 | * This class computes a hash of an StreamId, so that StreamIds can 43 | * be used as keys in unordered_maps. 44 | */ 45 | struct Hasher { 46 | std::size_t operator()(const StreamId& streamId) const 47 | { 48 | std::size_t hash = 0; 49 | if (streamId.addr.in6.sin6_family == AF_INET6) { 50 | hash ^= std::hash()(streamId.addr.in6.sin6_addr.s6_addr32[0]); 51 | hash ^= std::hash()(streamId.addr.in6.sin6_addr.s6_addr32[1]); 52 | hash ^= std::hash()(streamId.addr.in6.sin6_addr.s6_addr32[2]); 53 | hash ^= std::hash()(streamId.addr.in6.sin6_addr.s6_addr32[3]); 54 | hash ^= std::hash()(streamId.addr.in6.sin6_port); 55 | } else if (streamId.addr.in4.sin_family == AF_INET) { 56 | hash ^= std::hash()(streamId.addr.in4.sin_addr.s_addr); 57 | hash ^= std::hash()(streamId.addr.in4.sin_port); 58 | } 59 | return hash ^ streamId.id; 60 | } 61 | }; 62 | 63 | struct Pred { 64 | bool operator()(const StreamId& a, const StreamId& b) const 65 | { 66 | if (a.id != b.id) { 67 | return false; 68 | } 69 | if (a.addr.in6.sin6_family != b.addr.in6.sin6_family) { 70 | return false; 71 | } 72 | if (a.addr.in6.sin6_family == AF_INET6) { 73 | return (a.addr.in6.sin6_addr.s6_addr32[0] 74 | == b.addr.in6.sin6_addr.s6_addr32[0]) 75 | && (a.addr.in6.sin6_addr.s6_addr32[1] 76 | == b.addr.in6.sin6_addr.s6_addr32[1]) 77 | && (a.addr.in6.sin6_addr.s6_addr32[2] 78 | == b.addr.in6.sin6_addr.s6_addr32[2]) 79 | && (a.addr.in6.sin6_addr.s6_addr32[3] 80 | == b.addr.in6.sin6_addr.s6_addr32[3]) 81 | && (a.addr.in6.sin6_port == b.addr.in6.sin6_port); 82 | } else if (a.addr.in4.sin_family == AF_INET) { 83 | return (a.addr.in4.sin_addr.s_addr 84 | == b.addr.in4.sin_addr.s_addr) 85 | && (a.addr.in4.sin_port == b.addr.in4.sin_port); 86 | } 87 | return false; 88 | } 89 | }; 90 | }; 91 | 92 | #endif // STREAM_ID_H 93 | -------------------------------------------------------------------------------- /tcp_test.cc: -------------------------------------------------------------------------------- 1 | // This file contains both the client and server code for a simple 2 | // gRPC application. 3 | // 4 | // Usage: 5 | // tcp_test server port 6 | // tcp test client port@host 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include "test.grpc.pb.h" 14 | 15 | // Server-side code to implement the service. 16 | class SumService : public test::Test::Service { 17 | public: 18 | grpc::Status Sum(grpc::ServerContext*context, const test::SumArgs *args, 19 | test::SumResult *result) { 20 | result->set_sum(args->op1() + args->op2()); 21 | return grpc::Status::OK; 22 | } 23 | }; 24 | 25 | // Client-side wrapper class. 26 | class SumClient{ 27 | public: 28 | SumClient(const std::shared_ptr channel) 29 | : stub(test::Test::NewStub(channel)) {} 30 | 31 | std::unique_ptr stub; 32 | 33 | int Sum(int op1, int op2) 34 | { 35 | test::SumArgs args; 36 | test::SumResult result; 37 | grpc::ClientContext context; 38 | 39 | args.set_op1(op1); 40 | args.set_op2(op2); 41 | grpc::Status status = stub->Sum(&context, args, &result); 42 | if (!status.ok()) { 43 | printf("Sum RPC failed!\n"); 44 | return -1; 45 | } 46 | return result.sum(); 47 | } 48 | }; 49 | 50 | int main(int argc, char** argv) { 51 | if (argc < 2) { 52 | goto usage; 53 | } 54 | if (strcmp(argv[1], "client") == 0) { 55 | std::string target = "node1:4000"; 56 | if (argc == 3) { 57 | target = argv[2]; 58 | } else if (argc != 2) { 59 | goto usage; 60 | } 61 | SumClient client(grpc::CreateChannel(target, 62 | grpc::InsecureChannelCredentials())); 63 | int sum = client.Sum(22, 33); 64 | printf("Sum of 22 and 33 is %d\n", sum); 65 | return 0; 66 | } else if (strcmp(argv[1], "server") == 0) { 67 | std::string port = "0.0.0.0:4000"; 68 | if (argc == 3) { 69 | port = "0.0.0.0:"; 70 | port += argv[2]; 71 | } else if (argc != 2) { 72 | goto usage; 73 | } 74 | SumService service; 75 | grpc::ServerBuilder builder; 76 | builder.AddListeningPort(port, grpc::InsecureServerCredentials()); 77 | builder.RegisterService(&service); 78 | std::unique_ptr server(builder.BuildAndStart()); 79 | if (server == nullptr) 80 | exit(1); 81 | server->Wait(); 82 | } else { 83 | goto usage; 84 | } 85 | exit(0); 86 | 87 | usage: 88 | fprintf(stderr, "Usage: %s client server:port or %s server port\n", 89 | argv[0], argv[0]); 90 | exit(1); 91 | } -------------------------------------------------------------------------------- /test.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffer definitions used for testing grpcHoma. 2 | 3 | syntax = "proto3"; 4 | 5 | package test; 6 | 7 | service Test { 8 | rpc Sum(SumArgs) returns (SumResult) {} 9 | rpc SumMany(stream Value) returns (SumResult) {} 10 | rpc GetValues(Value) returns (stream Value) {} 11 | rpc IncMany(stream Value) returns (stream Value) {} 12 | rpc PrintTrace(String) returns (Empty) {} 13 | } 14 | 15 | message SumArgs { 16 | int32 op1 = 1; 17 | int32 op2 = 2; 18 | } 19 | 20 | message SumResult { 21 | int32 sum = 1; 22 | } 23 | 24 | message Value { 25 | int32 value = 1; 26 | } 27 | 28 | message String { 29 | string s = 1; 30 | } 31 | 32 | message Empty {} 33 | -------------------------------------------------------------------------------- /test_client.cc: -------------------------------------------------------------------------------- 1 | // Simple gRPC client for testing. 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include "test.grpc.pb.h" 13 | 14 | #include "homa.h" 15 | #include "homa_client.h" 16 | #include "time_trace.h" 17 | #include "util.h" 18 | 19 | const char *ttFile = "homa.tt"; 20 | const char *ttServerFile = "homaServer.tt"; 21 | 22 | class TestClient{ 23 | public: 24 | TestClient(const std::shared_ptr channel) 25 | : stub(test::Test::NewStub(channel)) {} 26 | 27 | std::unique_ptr stub; 28 | 29 | /** 30 | * Adds two numbers together. 31 | * \param op1 32 | * First number to add. 33 | * \param op2 34 | * Second number to add. 35 | * \return 36 | * Sum of the two numbers. 37 | */ 38 | int Sum(int op1, int op2) 39 | { 40 | test::SumArgs args; 41 | test::SumResult result; 42 | grpc::ClientContext context; 43 | 44 | context.AddMetadata("md1", "md1_value"); 45 | args.set_op1(op1); 46 | args.set_op2(op2); 47 | grpc::Status status = stub->Sum(&context, args, &result); 48 | if (!status.ok()) { 49 | printf("Sum RPC failed: %s\n", stringForStatus(&status).c_str()); 50 | return -1; 51 | } 52 | return result.sum(); 53 | } 54 | 55 | /** 56 | * Issues 3 concurrent asynchronous Sum RPCs, waits for all results, 57 | * and prints results. 58 | */ 59 | void Sum3Async() 60 | { 61 | test::SumArgs args[3]; 62 | test::SumResult results[3]; 63 | grpc::ClientContext contexts[3]; 64 | grpc::CompletionQueue cq; 65 | grpc::Status statuses[3]; 66 | 67 | args[0].set_op1(100); 68 | args[0].set_op2(200); 69 | args[1].set_op1(300); 70 | args[1].set_op2(400); 71 | args[2].set_op1(500); 72 | args[2].set_op2(600); 73 | 74 | std::unique_ptr> rpc0( 75 | stub->AsyncSum(&contexts[0], args[0], &cq)); 76 | std::unique_ptr> rpc1( 77 | stub->AsyncSum(&contexts[1], args[1], &cq)); 78 | std::unique_ptr> rpc2( 79 | stub->AsyncSum(&contexts[2], args[2], &cq)); 80 | 81 | rpc0->Finish(&results[0], &statuses[0], (void *) 1); 82 | rpc1->Finish(&results[1], &statuses[1], (void *) 2); 83 | rpc2->Finish(&results[2], &statuses[2], (void *) 3); 84 | 85 | for (int i = 0; i < 3; i++) { 86 | uint64_t got_tag; 87 | bool ok = false; 88 | 89 | if (!cq.Next(reinterpret_cast(&got_tag), &ok) || !ok) { 90 | printf("Sum3Async failed: couldn't get event from " 91 | "completion queue\n"); 92 | return; 93 | } 94 | 95 | if ((got_tag < 1) || (got_tag > 3)) { 96 | printf("Sum3Async received bad tag %lu\n", got_tag); 97 | return; 98 | } 99 | 100 | printf("Sum3Async operation %lu completed with result %d\n", 101 | got_tag, results[got_tag-1].sum()); 102 | } 103 | } 104 | 105 | /** 106 | * Adds many numbers together. 107 | * \param op 108 | * The first number to add. 109 | * \param ... 110 | * A number of additional values, terminated by a -1 value. 111 | * \return 112 | * The sum of all of the numbers (except the terminating one). 113 | */ 114 | int SumMany(int op, ...) { 115 | test::SumResult result; 116 | grpc::ClientContext context; 117 | std::unique_ptr> writer( 118 | stub->SumMany(&context, &result)); 119 | va_list ap; 120 | va_start(ap, op); 121 | while (op != -1) { 122 | test::Value value; 123 | value.set_value(op); 124 | if (!writer->Write(value)) { 125 | break; 126 | } 127 | op = va_arg(ap, int); 128 | } 129 | writer->WritesDone(); 130 | grpc::Status status = writer->Finish(); 131 | if (!status.ok()) { 132 | printf("SumMany RPC failed: %s\n", stringForStatus(&status).c_str()); 133 | return -1; 134 | } 135 | return result.sum(); 136 | } 137 | 138 | /** 139 | * Sends a seed value to the server, then prints all the individual 140 | * values that the server returns. 141 | * \param seed 142 | * Server will use this value to determine how many values to 143 | * return. 144 | */ 145 | void PrintValues(int seed) 146 | { 147 | grpc::ClientContext context; 148 | test::Value value; 149 | test::Value response; 150 | value.set_value(seed); 151 | int numResponses = 0; 152 | 153 | std::unique_ptr > reader( 154 | stub->GetValues(&context, value)); 155 | while (reader->Read(&response)) { 156 | printf("PrintValues received value %d\n", response.value()); 157 | numResponses++; 158 | } 159 | printf("PrintValues received %d response messages\n", numResponses); 160 | grpc::Status status = reader->Finish(); 161 | if (!status.ok()) { 162 | printf("PrintValues RPC failed: %s\n", 163 | stringForStatus(&status).c_str()); 164 | } 165 | } 166 | 167 | /** 168 | * Sends an initial value to the server, checks to see that the server responds 169 | * with incremented value. Repeats this many times. 170 | * \param initial 171 | * Additional value to send to the server. 172 | * \param count 173 | * How many times to have the server increment the value. 174 | */ 175 | void IncMany(int initial, int count) 176 | { 177 | int current = initial; 178 | grpc::ClientContext context; 179 | std::shared_ptr> 180 | stream(stub->IncMany(&context)); 181 | test::Value request; 182 | test::Value response; 183 | 184 | for (int i = 0; i < count; i++) { 185 | request.set_value(current); 186 | if (!stream->Write(request)) { 187 | break; 188 | } 189 | if (!stream->Read(&response)) { 190 | break; 191 | } 192 | printf("IncMany sent %d, got %d\n", current, response.value()); 193 | current = response.value(); 194 | } 195 | stream->WritesDone(); 196 | grpc::Status status = stream->Finish(); 197 | if (!status.ok()) { 198 | printf("IncMany RPC failed: %s\n", 199 | stringForStatus(&status).c_str()); 200 | } 201 | } 202 | /** 203 | * Have the peer print its timetrace to a file. 204 | * \param name 205 | * Name of the file in which to print the trace. 206 | */ 207 | void PrintTrace(const char *name) 208 | { 209 | test::String args; 210 | test::Empty result; 211 | grpc::ClientContext context; 212 | 213 | args.set_s(name); 214 | grpc::Status status = stub->PrintTrace(&context, args, &result); 215 | if (!status.ok()) { 216 | printf("PrintTrace RPC failed: %s\n", 217 | stringForStatus(&status).c_str()); 218 | } 219 | } 220 | }; 221 | 222 | void measureRtt(TestClient *client) 223 | { 224 | #define NUM_REQUESTS 50000 225 | for (int i = 0; i < 5; i++) { 226 | client->Sum(1, 2); 227 | } 228 | uint64_t start = TimeTrace::rdtsc(); 229 | for (int i = 0; i < NUM_REQUESTS; i++) { 230 | tt("Starting request"); 231 | client->Sum(1, 2); 232 | tt("Request %d finished", i); 233 | tt("TimeTrace::record completed"); 234 | tt("Second TimeTrace::record completed"); 235 | } 236 | uint64_t end = TimeTrace::rdtsc(); 237 | printf("Round-trip time for Sum requests: %.1f us\n", 238 | TimeTrace::toSeconds(end-start)*1e06/NUM_REQUESTS); 239 | client->PrintTrace(ttServerFile); 240 | TimeTrace::printToFile(ttFile); 241 | } 242 | 243 | int main(int argc, char** argv) 244 | { 245 | const char *server = "node1:4000"; 246 | bool useHoma = true; 247 | std::vector args; 248 | unsigned nextArg; 249 | 250 | for (int i = 0; i < argc; i++) { 251 | args.emplace_back(argv[i]); 252 | } 253 | 254 | for (nextArg = 1; nextArg < args.size(); nextArg++) { 255 | const char *option = args[nextArg].c_str(); 256 | if (strcmp(option, "--tcp") == 0) { 257 | useHoma = false; 258 | continue; 259 | } 260 | break; 261 | } 262 | if (nextArg != args.size()) { 263 | if (nextArg != (args.size() - 1)) { 264 | fprintf(stderr, "Usage: test_client [--tcp] [host::port]\n"); 265 | exit(1); 266 | } 267 | server = args.back().c_str(); 268 | } 269 | 270 | std::optional client; 271 | if (useHoma) { 272 | client.emplace(grpc::CreateChannel(server, 273 | HomaClient::insecureChannelCredentials())); 274 | } else { 275 | ttFile = "tcp.tt"; 276 | ttServerFile = "tcpServer.tt"; 277 | client.emplace(grpc::CreateChannel(server, 278 | grpc::InsecureChannelCredentials())); 279 | } 280 | // int sum; 281 | // sum = client->Sum(22, 33); 282 | // printf("Sum of 22 and 33 is %d\n", sum); 283 | // client->Sum3Async(); 284 | // printf("SumMany of 1..5 is %d\n", client->SumMany(1, 2, 3, 4, 5, -1)); 285 | // client->PrintValues(21); 286 | // client->IncMany(3, 4); 287 | measureRtt(&client.value()); 288 | 289 | return 0; 290 | } -------------------------------------------------------------------------------- /test_listener.cc: -------------------------------------------------------------------------------- 1 | #define __UNIT_TEST__ 1 2 | #include "homa_listener.h" 3 | #include "mock.h" 4 | 5 | #include "src/core/lib/resource_quota/resource_quota.h" 6 | 7 | // This file contains unit tests for homa_listener.cc and homa_listener.h. 8 | 9 | class TestListener : public ::testing::Test { 10 | public: 11 | HomaListener *lis; 12 | HomaListener::Transport *trans; 13 | std::vector streams; 14 | grpc_stream_refcount refcount; 15 | HomaSocket sock; 16 | HomaIncoming msg; 17 | uint32_t msgStreamId; 18 | grpc_closure closure1; 19 | 20 | static void closureFunc1(void* arg, grpc_error_handle error) { 21 | int64_t value = reinterpret_cast(arg); 22 | if (!error.ok()) { 23 | Mock::logPrintf("; ", "closure1 invoked with %ld, error %s", 24 | value, error.ToString().c_str()); 25 | } else { 26 | Mock::logPrintf("; ", "closure1 invoked with %ld", value); 27 | } 28 | } 29 | 30 | TestListener() 31 | : lis(nullptr) 32 | , trans(nullptr) 33 | , streams() 34 | , refcount() 35 | , sock(Mock::bufRegion) 36 | , msg(&sock, 2, true, 100, 0, true, true) 37 | , msgStreamId() 38 | , closure1() 39 | { 40 | int port = 4000; 41 | gpr_once_init(&HomaListener::shared_once, HomaListener::InitShared); 42 | Mock::setUp(); 43 | lis = new HomaListener(nullptr, &port, false); 44 | trans = lis->transport; 45 | trans->accept_stream_cb = acceptStreamCallback; 46 | trans->accept_stream_data = this; 47 | GRPC_CLOSURE_INIT(&closure1, closureFunc1, 48 | reinterpret_cast(123), dummy); 49 | } 50 | 51 | ~TestListener() 52 | { 53 | grpc_core::ExecCtx exec_ctx; 54 | for (HomaStream *stream: streams) { 55 | stream->~HomaStream(); 56 | free(stream); 57 | } 58 | delete lis; 59 | } 60 | 61 | static void acceptStreamCallback(void* fixture, grpc_transport* transport, 62 | const void* initInfo) 63 | { 64 | TestListener *test = reinterpret_cast(fixture); 65 | HomaListener::Transport::StreamInit *init = 66 | static_cast( 67 | const_cast(initInfo)); 68 | 69 | // This contortion is needed to be consistent with other code that 70 | // creates HomaStreams with placement new. 71 | init->stream = reinterpret_cast(malloc(sizeof(HomaStream))); 72 | new (init->stream) HomaStream(false, *init->streamId, 73 | test->trans->sock.getFd(), &test->refcount); 74 | test->streams.push_back(init->stream); 75 | } 76 | }; 77 | 78 | TEST_F(TestListener, getStream_basics) { 79 | std::optional lockGuard; 80 | StreamId streamId(100); 81 | 82 | // Id 100: add new stream 83 | HomaStream *stream1 = lis->transport->getStream(&streamId, lockGuard, true); 84 | EXPECT_EQ(1U, trans->activeRpcs.size()); 85 | EXPECT_EQ(100U, stream1->streamId.id); 86 | lockGuard.reset(); 87 | 88 | // Id 200: add new stream 89 | streamId.id = 200; 90 | HomaStream *stream2 = trans->getStream(&streamId, lockGuard, true); 91 | EXPECT_EQ(2U, trans->activeRpcs.size()); 92 | EXPECT_EQ(200U, stream2->streamId.id); 93 | lockGuard.reset(); 94 | 95 | // Id 100 again 96 | streamId.id = 100; 97 | HomaStream *stream3 = trans->getStream(&streamId, lockGuard, true); 98 | EXPECT_EQ(2U, trans->activeRpcs.size()); 99 | EXPECT_EQ(100U, stream3->streamId.id); 100 | EXPECT_EQ(stream1, stream3); 101 | } 102 | TEST_F(TestListener, getStream_dontCreate) { 103 | std::optional lockGuard; 104 | StreamId streamId(100); 105 | 106 | // Id 100: add new stream 107 | HomaStream *stream = lis->transport->getStream(&streamId, lockGuard, false); 108 | EXPECT_EQ(nullptr, stream); 109 | EXPECT_EQ(0U, trans->activeRpcs.size()); 110 | } 111 | TEST_F(TestListener, getStream_noCallback) { 112 | std::optional lockGuard; 113 | StreamId streamId(100); 114 | trans->accept_stream_cb = nullptr; 115 | HomaStream *stream1 = trans->getStream(&streamId, lockGuard, true); 116 | EXPECT_EQ(0U, trans->activeRpcs.size()); 117 | EXPECT_EQ(nullptr, stream1); 118 | } 119 | 120 | TEST_F(TestListener, destroy_stream) { 121 | HomaStream *stream; 122 | grpc_core::ExecCtx execCtx; 123 | StreamId streamId(100); 124 | { 125 | std::optional lockGuard; 126 | stream = trans->getStream(&streamId, lockGuard, true); 127 | EXPECT_EQ(1U, trans->activeRpcs.size()); 128 | EXPECT_EQ(100U, streamId.id); 129 | ASSERT_EQ(1U, streams.size()); 130 | ASSERT_EQ(stream, streams[0]); 131 | } 132 | 133 | HomaListener::Transport::destroy_stream(&trans->vtable, 134 | reinterpret_cast (stream), &closure1); 135 | free(stream); 136 | streams.clear(); 137 | execCtx.Flush(); 138 | EXPECT_EQ(0U, trans->activeRpcs.size()); 139 | EXPECT_STREQ("closure1 invoked with 123", Mock::log.c_str()); 140 | } -------------------------------------------------------------------------------- /test_server.cc: -------------------------------------------------------------------------------- 1 | // gRPC server for testing. 2 | 3 | #include 4 | #include 5 | 6 | #include "test.grpc.pb.h" 7 | #include "homa.h" 8 | #include "homa_listener.h" 9 | #include "time_trace.h" 10 | #include "util.h" 11 | 12 | class TestImpl : public test::Test::Service { 13 | public: 14 | grpc::Status Sum(grpc::ServerContext* context, const test::SumArgs *args, 15 | test::SumResult *status) override 16 | { 17 | // printf("Sum invoked with arguments %d and %d\n", 18 | // args->op1(), args->op2()); 19 | tt("Sum service method invoked"); 20 | // printf("%lu metadata values from client\n", 21 | // context->client_metadata().size()); 22 | // for (auto md: context->client_metadata()) { 23 | // printf("Incoming metadata: name '%.*s', value '%.*s'\n", 24 | // static_cast(md.first.length()), md.first.data(), 25 | // static_cast(md.second.length()), md.second.data()); 26 | // } 27 | status->set_sum(args->op1() + args->op2()); 28 | return grpc::Status::OK; 29 | } 30 | 31 | grpc::Status SumMany(grpc::ServerContext* context, 32 | grpc::ServerReader* reader, test::SumResult *status) 33 | override 34 | { 35 | test::Value value; 36 | int sum = 0; 37 | while (reader->Read(&value)) { 38 | printf("SumMany received value %d\n", value.value()); 39 | sum += value.value(); 40 | } 41 | status->set_sum(sum); 42 | printf("Returning result: %d\n", sum); 43 | return grpc::Status::OK; 44 | } 45 | 46 | grpc::Status GetValues(grpc::ServerContext* context, const test::Value *arg, 47 | grpc::ServerWriter* writer) override 48 | { 49 | test::Value response; 50 | for (int i = 1; i <= arg->value(); i += 2) { 51 | response.set_value(i); 52 | printf("Writing response with %d\n", i); 53 | writer->Write(response); 54 | } 55 | printf("GetValues finished (input %d)\n", arg->value()); 56 | return grpc::Status::OK; 57 | } 58 | 59 | grpc::Status IncMany(grpc::ServerContext* context, 60 | grpc::ServerReaderWriter* stream) override 61 | { 62 | test::Value request; 63 | test::Value response; 64 | while (stream->Read(&request)) { 65 | printf("IncMany got value %d\n", request.value()); 66 | response.set_value(request.value() + 1); 67 | stream->Write(response); 68 | } 69 | printf("IncMany finished\n"); 70 | return grpc::Status::OK; 71 | } 72 | 73 | grpc::Status PrintTrace(grpc::ServerContext*context, 74 | const test::String *args, test::Empty *status) override 75 | { 76 | TimeTrace::printToFile(args->s().c_str()); 77 | return grpc::Status::OK; 78 | } 79 | }; 80 | 81 | int main(int argc, char** argv) { 82 | recordFunc = TimeTrace::record2; 83 | std::string serverAddress; 84 | bool useHoma = true; 85 | std::vector args; 86 | unsigned nextArg; 87 | 88 | for (int i = 0; i < argc; i++) { 89 | args.emplace_back(argv[i]); 90 | } 91 | 92 | for (nextArg = 1; nextArg < args.size(); nextArg++) { 93 | const char *option = args[nextArg].c_str(); 94 | if (strcmp(option, "--tcp") == 0) { 95 | useHoma = false; 96 | continue; 97 | } 98 | break; 99 | } 100 | if (nextArg != args.size()) { 101 | if (nextArg != (args.size() - 1)) { 102 | fprintf(stderr, "Usage: test_server [--tcp] [homa:port]\n"); 103 | exit(1); 104 | } 105 | serverAddress = args.back(); 106 | } 107 | if (serverAddress.empty()) { 108 | if (useHoma) { 109 | serverAddress = "homa:4000"; 110 | } else { 111 | serverAddress = "0.0.0.0:4000"; 112 | } 113 | } 114 | 115 | 116 | TestImpl service; 117 | grpc::ServerBuilder builder; 118 | int actualPort; 119 | if (useHoma) { 120 | builder.AddListeningPort(serverAddress, 121 | HomaListener::insecureCredentials(), &actualPort); 122 | } else { 123 | builder.AddListeningPort(serverAddress, 124 | grpc::InsecureServerCredentials(), &actualPort); 125 | } 126 | builder.RegisterService(&service); 127 | std::unique_ptr server(builder.BuildAndStart()); 128 | if (server == nullptr) 129 | exit(1); 130 | std::cout << "Server listening on port " << actualPort << std::endl; 131 | server->Wait(); 132 | 133 | return 0; 134 | } -------------------------------------------------------------------------------- /test_socket.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "homa_socket.h" 4 | #include "mock.h" 5 | #include "util.h" 6 | 7 | // This file contains unit tests for homa_socket.cc and homa_socket.h. 8 | 9 | class TestSocket : public ::testing::Test { 10 | public: 11 | HomaSocket sock; 12 | 13 | TestSocket() 14 | : sock(nullptr) 15 | { 16 | Mock::setUp(); 17 | } 18 | 19 | ~TestSocket() 20 | { 21 | } 22 | 23 | /** 24 | * Return a human-readable string describing all of the saved buffer 25 | * info in a HomaSocket. 26 | * \param sock 27 | * Socket whose saved buffers should be scanned. 28 | */ 29 | std::string getSaved(HomaSocket *sock) 30 | { 31 | std::string result; 32 | for (uint32_t offset: sock->savedBuffers) { 33 | char buffer[100]; 34 | if (!result.empty()) { 35 | result.append(" "); 36 | } 37 | snprintf(buffer, sizeof(buffer), "%d", offset); 38 | result.append(buffer); 39 | } 40 | return result; 41 | } 42 | }; 43 | 44 | TEST_F(TestSocket, saveBuffers) { 45 | struct homa_recvmsg_args recvArgs; 46 | recvArgs.bpage_offsets[0] = 0; 47 | recvArgs.bpage_offsets[1] = 1; 48 | recvArgs.bpage_offsets[2] = 2; 49 | recvArgs.num_bpages = 3; 50 | sock.saveBuffers(&recvArgs); 51 | EXPECT_STREQ("0 1 2", getSaved(&sock).c_str()); 52 | 53 | // Should be idempotent. 54 | sock.saveBuffers(&recvArgs); 55 | EXPECT_STREQ("0 1 2", getSaved(&sock).c_str()); 56 | } 57 | 58 | TEST_F(TestSocket, getSavedBuffers) { 59 | struct homa_recvmsg_args recvArgs; 60 | for (uint32_t i = 0; i < 20; i++) { 61 | recvArgs.bpage_offsets[0] = 2*i + 1; 62 | recvArgs.num_bpages = 1; 63 | sock.saveBuffers(&recvArgs); 64 | } 65 | EXPECT_EQ(20U, sock.savedBuffers.size()); 66 | 67 | // First attempt: limited by space in control. 68 | recvArgs.num_bpages = 0; 69 | sock.getSavedBuffers(&recvArgs); 70 | EXPECT_EQ(static_cast(HOMA_MAX_BPAGES), recvArgs.num_bpages); 71 | EXPECT_EQ(1U, recvArgs.bpage_offsets[0]); 72 | EXPECT_EQ(3U, recvArgs.bpage_offsets[1]); 73 | EXPECT_EQ(static_cast(2*HOMA_MAX_BPAGES - 1), 74 | recvArgs.bpage_offsets[HOMA_MAX_BPAGES-1]); 75 | 76 | // Second attempt: limited by available buffers. 77 | recvArgs.num_bpages = 0; 78 | sock.getSavedBuffers(&recvArgs); 79 | EXPECT_EQ(static_cast(20 - HOMA_MAX_BPAGES), recvArgs.num_bpages); 80 | EXPECT_EQ(static_cast(2*HOMA_MAX_BPAGES + 1), recvArgs.bpage_offsets[0]); 81 | EXPECT_EQ(static_cast(2*HOMA_MAX_BPAGES + 3), recvArgs.bpage_offsets[1]); 82 | 83 | // Third attempt: no available buffers. 84 | recvArgs.num_bpages = 0; 85 | sock.getSavedBuffers(&recvArgs); 86 | EXPECT_EQ(0U, recvArgs.num_bpages); 87 | } 88 | TEST_F(TestSocket, getSavedBuffers_argsAlreadyHasSomeBuffers) { 89 | struct homa_recvmsg_args control; 90 | sock.savedBuffers.push_back(10); 91 | sock.savedBuffers.push_back(11); 92 | control.num_bpages = 3; 93 | sock.getSavedBuffers(&control); 94 | EXPECT_EQ(5U, control.num_bpages); 95 | EXPECT_EQ(10U, control.bpage_offsets[3]); 96 | EXPECT_EQ(11U, control.bpage_offsets[4]); 97 | } -------------------------------------------------------------------------------- /time_trace.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2020-2021 Stanford University 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR(S) DISCLAIM ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL AUTHORS BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | */ 15 | 16 | #ifndef TIMETRACE_H 17 | #define TIMETRACE_H 18 | 19 | #include 20 | #include 21 | 22 | // Change 1 -> 0 in the following line to disable time tracing globally. 23 | #define ENABLE_TIME_TRACE 0 24 | 25 | /** 26 | * This class implements a circular buffer of entries, each of which 27 | * consists of a fine-grain timestamp, a short descriptive string, and 28 | * a few additional values. It's typically used to record times at 29 | * various points in an operation, in order to find performance bottlenecks. 30 | * It can record a trace relatively efficiently (< 10ns as of 7/2020), 31 | * and then either return the trace either as a string or print it to 32 | * a file. 33 | * 34 | * This class is thread-safe. By default, trace information is recorded 35 | * separately for each thread in order to avoid synchronization and cache 36 | * consistency overheads; the thread-local traces are merged when the 37 | * timetrace is printed, so the existence of multiple trace buffers is 38 | * normally invisible. 39 | * 40 | * The TimeTrace class should never be constructed; it offers only 41 | * static methods. 42 | * 43 | * If you want to use a single trace buffer rather than per-thread 44 | * buffers, see the subclass TimeTrace::Buffer below. 45 | */ 46 | class TimeTrace { 47 | public: 48 | static void freeze(); 49 | static double getCyclesPerSec(); 50 | static std::string getTrace(); 51 | static int printToFile(const char *name); 52 | static void record2(const char* format, uint64_t arg0 = 0, 53 | uint64_t arg1 = 0, uint64_t arg2 = 0, uint64_t arg3 = 0); 54 | static double toSeconds(uint64_t cycles); 55 | 56 | // Nonzero means that the timetrace is already frozen. 57 | static int frozen; 58 | 59 | protected: 60 | /** 61 | * Holds one entry in a TimeTrace::Buffer. 62 | */ 63 | struct Event { 64 | /* See documentation for record method. */ 65 | uint64_t timestamp; 66 | const char* format; 67 | uint64_t arg0; 68 | uint64_t arg1; 69 | uint64_t arg2; 70 | uint64_t arg3; 71 | }; 72 | 73 | /** 74 | * Represents a sequence of events generated by a single thread. Has 75 | * a fixed capacity, so slots are re-used on a circular basis. This 76 | * class is not thread-safe. 77 | */ 78 | class Buffer { 79 | public: 80 | Buffer(); 81 | ~Buffer(); 82 | void record(uint64_t timestamp, const char* format, 83 | uint64_t arg0 = 0, uint64_t arg1 = 0, 84 | uint64_t arg2 = 0, uint64_t arg3 = 0); 85 | void reset(); 86 | 87 | public: 88 | // Name that identifies this buffer/thread. 89 | std::string name; 90 | 91 | // Determines the number of events we can retain, as an exponent of 2. 92 | static const uint8_t BUFFER_SIZE_EXP = 16; 93 | 94 | // Total number of events that we can retain at any given time. 95 | static const uint32_t BUFFER_SIZE = 1 << BUFFER_SIZE_EXP; 96 | 97 | // Bit mask used to implement a circular event buffer. 98 | static const uint32_t BUFFER_MASK = BUFFER_SIZE - 1; 99 | 100 | // Index within events of the slot to use for the next call to @record. 101 | int nextIndex; 102 | 103 | // Number of thread_buffer objects that reference this buffer. When 104 | // this count becomes 0, the buffer can be deleted in the next call 105 | // to time_trace::cleanup. 106 | int refCount; 107 | 108 | // Holds information from the most recent calls to record. 109 | TimeTrace::Event events[BUFFER_SIZE]; 110 | 111 | friend class TimeTrace; 112 | }; 113 | 114 | /** 115 | * Stores a pointer to a Buffer; used to automatically allocate 116 | * buffers on thread creation and recycle them on thread exit. 117 | */ 118 | class BufferPtr { 119 | public: 120 | BufferPtr(); 121 | ~BufferPtr(); 122 | Buffer* operator->() 123 | { 124 | return buffer; 125 | } 126 | 127 | Buffer *buffer; 128 | }; 129 | 130 | // Points to a private per-thread TimeTrace::Buffer. 131 | static thread_local BufferPtr tb; 132 | 133 | // Holds pointers to all of the existing thread-private buffers. 134 | // Entries get deleted only by free_unused. 135 | static std::vector threadBuffers; 136 | 137 | // Buffers that have been previously used by a thread, but the 138 | // thread has exited so the buffer is available for re-use. 139 | static std::vector freeBuffers; 140 | 141 | public: 142 | /** 143 | * Record an event in a thread-local buffer. 144 | * \param timestamp 145 | * The time at which the event occurred (as returned by rdtsc()). 146 | * \param format 147 | * A format string for snprintf that will be used when the time trace 148 | * is printed, along with arg0..arg3, to generate a human-readable 149 | * message describing what happened. The message is generated by 150 | * calling snprintf as follows: snprintf(buffer, size, format, arg0, 151 | * arg1, arg2, arg3) where format and arg0..arg3 are the corresponding 152 | * arguments to this method. This pointer is stored in the time trace, 153 | * so the caller must ensure that its contents will not change over 154 | * its lifetime in the trace. 155 | * \param arg0 156 | * Argument to use when printing a message about this event. 157 | * \param arg1 158 | * Argument to use when printing a message about this event. 159 | * \param arg2 160 | * Argument to use when printing a message about this event. 161 | * \param arg3 162 | * Argument to use when printing a message about this event. 163 | */ 164 | static inline void record(uint64_t timestamp, const char* format, 165 | uint64_t arg0 = 0, uint64_t arg1 = 0, 166 | uint64_t arg2 = 0, uint64_t arg3 = 0) { 167 | #if ENABLE_TIME_TRACE 168 | tb->record(timestamp, format, arg0, arg1, arg2, arg3); 169 | #endif 170 | } 171 | static inline void record(const char* format, uint64_t arg0 = 0, 172 | uint64_t arg1 = 0, uint64_t arg2 = 0, uint64_t arg3 = 0) { 173 | #if ENABLE_TIME_TRACE 174 | record(rdtsc(), format, arg0, arg1, arg2, arg3); 175 | #endif 176 | } 177 | 178 | /** 179 | * Return the current value of the fine-grain CPU cycle counter 180 | * (accessed via the RDTSC instruction). 181 | */ 182 | inline static uint64_t rdtsc(void) 183 | { 184 | uint32_t lo, hi; 185 | __asm__ __volatile__("rdtsc" : "=a" (lo), "=d" (hi)); 186 | return (((uint64_t)hi << 32) | lo); 187 | } 188 | 189 | protected: 190 | TimeTrace(); 191 | static void printInternal(std::string* s, FILE *f); 192 | }; 193 | 194 | extern void (*recordFunc)(const char* format, uint64_t arg0, 195 | uint64_t arg1, uint64_t arg2, uint64_t arg3); 196 | 197 | #define tt TimeTrace::record 198 | 199 | /** 200 | * Records time trace record indirectly through recordFunc; used to 201 | * add time tracing to the gRPC core. 202 | */ 203 | inline void tt2(const char* format, uint64_t arg0 = 0, 204 | uint64_t arg1 = 0, uint64_t arg2 = 0, uint64_t arg3 = 0) 205 | { 206 | (*recordFunc)(format, arg0, arg1, arg2, arg3); 207 | } 208 | 209 | #endif // TIMETRACE_H 210 | 211 | -------------------------------------------------------------------------------- /tt/ttgrep.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Copyright (c) 2019-2020 Stanford University 4 | # 5 | # Permission to use, copy, modify, and distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR(S) DISCLAIM ALL WARRANTIES 10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL AUTHORS BE LIABLE FOR 12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | 17 | """ 18 | Scan the time trace data in a log file; find all records containing 19 | a given string, and output only those records. If the --rebase argument 20 | is present, times are offset so the first event is at time 0. If the file 21 | is omitted, standard input is used. 22 | Usage: ttgrep.py [--rebase] string [file] 23 | """ 24 | 25 | from __future__ import division, print_function 26 | from glob import glob 27 | from optparse import OptionParser 28 | import math 29 | import os 30 | import re 31 | import string 32 | import sys 33 | 34 | rebase = False; 35 | 36 | def scan(f, string): 37 | """ 38 | Scan the log file given by 'f' (handle for an open file) and output 39 | all-time trace records containing string. 40 | """ 41 | global rebase 42 | startTime = 0.0 43 | prevTime = 0.0 44 | writes = 0 45 | for line in f: 46 | match = re.match(' *([0-9.]+) us \(\+ *([0-9.]+) us\) (.*)', 47 | line) 48 | if not match: 49 | continue 50 | time = float(match.group(1)) 51 | interval = float(match.group(2)) 52 | event = match.group(3) 53 | if (string not in event) and ("Freez" not in event): 54 | continue 55 | if startTime == 0.0: 56 | startTime = time 57 | prevTime = time 58 | if rebase: 59 | printTime = time - startTime 60 | else: 61 | printTime = time 62 | print("%9.3f us (+%8.3f us) %s" % (printTime, 63 | time - prevTime, event)) 64 | prevTime = time 65 | 66 | if (len(sys.argv) > 1) and (sys.argv[1] == "--rebase"): 67 | rebase = True 68 | del sys.argv[1] 69 | 70 | f = sys.stdin 71 | if len(sys.argv) == 3: 72 | f = open(sys.argv[2]) 73 | elif len(sys.argv) != 2: 74 | print("Usage: %s [--rebase] string [logFile]" % (sys.argv[0])) 75 | sys.exit(1) 76 | 77 | scan(f, sys.argv[1]) -------------------------------------------------------------------------------- /tt/ttmerge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Copyright (c) 2019-2020 Stanford University 4 | # 5 | # Permission to use, copy, modify, and distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR(S) DISCLAIM ALL WARRANTIES 10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL AUTHORS BE LIABLE FOR 12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | 17 | """ 18 | Merge two or more timetraces into a single trace. All of the traces 19 | must use the same time source. 20 | Usage: ttmerge.py file file file ... 21 | """ 22 | 23 | from __future__ import division, print_function 24 | from glob import glob 25 | import math 26 | from optparse import OptionParser 27 | import os 28 | import re 29 | import string 30 | import sys 31 | 32 | # Each entry in the following list describes one file; it is a dictionary 33 | # with the following fields: 34 | # name: Name of the file 35 | # f: Open file for reading 36 | # ghz: Clock rate assumed for this file 37 | # first: Timestamp of first entry 38 | # offset: How much to add to times in this file so they align 39 | # with times in the other files 40 | # time: Time of the current line, adjusted by offset 41 | # suffix: Everything on the current line after the times 42 | files = [] 43 | 44 | # Earliest first timestamp from all the files. 45 | first = 0 46 | 47 | # Reference ghz (taken from input file with the earliest start time; 48 | # used for output). Used to compensate for the fact that different 49 | # traces may have assumed slightly different conversion rates from 50 | # ticks to microseconds. 51 | ghz = 0.0 52 | 53 | def next_line(info): 54 | """ 55 | Read information from a file. The info argument is one of the 56 | entries in files. 57 | """ 58 | while True: 59 | line = info["f"].readline() 60 | if not line: 61 | info["f"].close() 62 | info["f"] = None 63 | return 64 | match = re.match(' *([0-9.]+) us \(\+ *([0-9.]+) us\) (.*)', line) 65 | if not match: 66 | continue 67 | info["time"] = (float(match.group(1)) * ghz / info["ghz"]) + info["offset"] 68 | info["suffix"] = match.group(3).rstrip() 69 | return 70 | 71 | # Open each of the files and initialize information for them. 72 | for file in sys.argv[1:]: 73 | f = open(file, newline='\n') 74 | line = f.readline() 75 | if not line: 76 | continue 77 | info = {"f": f} 78 | match = re.match(' *([0-9.]+) us \(\+ *([0-9.]+) us\) .* ' 79 | 'First event has timestamp ([0-9]+) ' 80 | '\(cpu_ghz ([0-9.]+)\)', line) 81 | if not match: 82 | continue 83 | info = {"name": file, 84 | "f": f, 85 | "ghz": float(match.group(4)), 86 | "first": int(match.group(3)), 87 | "offset": 0.0} 88 | files.append(info) 89 | 90 | # Find the earliest timestamp and set offsets. 91 | for info in files: 92 | if (first == 0) or info["first"] < first: 93 | first = info["first"] 94 | ghz = info["ghz"] 95 | for info in files: 96 | info["offset"] = ((info["first"] - first)/ghz)/1000.0 97 | # print("file %s has offset %.2f us (difference: %d)" % (info["name"], 98 | # info["offset"], info["first"] - first)) 99 | 100 | # Prime the info with the first real trace entry. 101 | next_line(info) 102 | 103 | # Repeatedly output the earliest line until there are no lines left to output. 104 | prevTime = 0.0 105 | while True: 106 | best = None 107 | best_time = 0.0 108 | for info in files: 109 | if info["f"] and ((best_time == 0.0) or (info["time"] < best_time)): 110 | best_time = info["time"] 111 | best = info 112 | if not best: 113 | break 114 | time = best["time"] 115 | print("%9.3f us (+%8.3f us) %s" % (time, time - prevTime, best["suffix"])) 116 | prev_time = time 117 | next_line(best) -------------------------------------------------------------------------------- /tt/ttoffset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Copyright (c) 2019-2020 Stanford University 4 | # 5 | # Permission to use, copy, modify, and distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR(S) DISCLAIM ALL WARRANTIES 10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL AUTHORS BE LIABLE FOR 12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | 17 | """ 18 | Rewrite a time trace with all of the times offset by a fixed amount 19 | (typically used to align the times in two timetraces) 20 | Usage: ttoffset.py old_time new_time [tt_file] 21 | 22 | The existing timetrace is in tt_file (or stdin in tt_file is omitted); a new 23 | timetrace will be written to standard output, with (new_time - old_time) 24 | added to each timestamp. 25 | """ 26 | 27 | from __future__ import division, print_function 28 | from glob import glob 29 | from optparse import OptionParser 30 | import math 31 | import os 32 | import re 33 | import string 34 | import sys 35 | 36 | if len(sys.argv) == 4: 37 | f = open(sys.argv[3]) 38 | elif len(sys.argv) == 3: 39 | f = sys.stdin 40 | else: 41 | print("Usage: %s old_time new_time [tt_file]" % (sys.argv[0])) 42 | sys.exit(1) 43 | 44 | delta = float(sys.argv[2]) - float(sys.argv[1]) 45 | 46 | for line in f: 47 | match = re.match(' *([0-9.]+) us (.*)', line) 48 | if not match: 49 | print(line) 50 | continue 51 | time = float(match.group(1)) 52 | print("%9.3f us %s" % (time + delta, match.group(2))) -------------------------------------------------------------------------------- /tt/ttrange.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Copyright (c) 2019-2020 Stanford University 4 | # 5 | # Permission to use, copy, modify, and distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR(S) DISCLAIM ALL WARRANTIES 10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL AUTHORS BE LIABLE FOR 12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | 17 | """ 18 | Extract entries from a timetrace that For any particular time range. 19 | Usage: ttrange.py start_time end_time [tt_file] 20 | 21 | The existing timetrace is in tt_file (or stdin in tt_file is omitted); a new 22 | timetrace will be written to standard output containing all entries whose 23 | timestamps fall between start_time and end_time, inclusive. 24 | """ 25 | 26 | from __future__ import division, print_function 27 | from glob import glob 28 | from optparse import OptionParser 29 | import math 30 | import os 31 | import re 32 | import string 33 | import sys 34 | 35 | if len(sys.argv) == 4: 36 | f = open(sys.argv[3]) 37 | elif len(sys.argv) == 3: 38 | f = sys.stdin 39 | else: 40 | print("Usage: %s start_time end_time [tt_file]" % (sys.argv[0])) 41 | sys.exit(1) 42 | 43 | start_time = float(sys.argv[1]) 44 | end_time = float(sys.argv[2]) 45 | 46 | for line in f: 47 | match = re.match(' *([0-9.]+) us (.*)', line) 48 | if not match: 49 | continue 50 | time = float(match.group(1)) 51 | if (time >= start_time) and (time <= end_time): 52 | print(line.rstrip('\n')) -------------------------------------------------------------------------------- /tt/ttsum.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Copyright (c) 2019-2020 Stanford University 4 | # 5 | # Permission to use, copy, modify, and distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR(S) DISCLAIM ALL WARRANTIES 10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL AUTHORS BE LIABLE FOR 12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | 17 | """ 18 | This program reads a log file and generates summary information for 19 | time trace information in the file. 20 | """ 21 | 22 | from __future__ import division, print_function 23 | from glob import glob 24 | from optparse import OptionParser 25 | import math 26 | import os 27 | import re 28 | import string 29 | import sys 30 | 31 | # This variable collects all the times for all events, individually. It is 32 | # a dictionary that maps from key names to a list containing all of the 33 | # intervals for that event name (each interval is the elapsed time between 34 | # the most recent previous event and this event). 35 | eventIntervals = {} 36 | 37 | # This variable collects information for all events relative to a given 38 | # starting event (the --from command line option). 39 | # 40 | # relativeEvents: 41 | # dictionary (event name => OccurrenceList) 42 | # 43 | # The same event may sometimes happen multiple times for single occurrence 44 | # of the starting event. An OccurrenceList is a list, where the nth entry 45 | # describes all the events that occurred after n prior occurrences of the 46 | # event. 47 | # OccurrenceList: 48 | # list (OccurrenceInfo) 49 | # 50 | # OccurrenceInfo: 51 | # dictionary ( 52 | # times: list() One entry for each event: elapsed ns between 53 | # the starting event and this event 54 | # intervals: list() One entry for each event: elapsed ns between 55 | # immediately preceding event and this event 56 | # ) 57 | 58 | relativeEvents = {} 59 | 60 | # This variable contains a count of the number of times each event has 61 | # occurred since the last time the starting event occurred. 62 | 63 | eventCount = {} 64 | 65 | def scan(f, startingEvent): 66 | """ 67 | Scan the log file given by 'f' (handle for an open file) and collect 68 | information from time trace records as described by the arguments. 69 | If 'startingEvent' isn't None, it specifies an event indicating the 70 | beginning of a related sequence of event; times are accumulated for all 71 | other events, relative to the most recent occurrence of the starting event. 72 | """ 73 | 74 | foundStart = False 75 | startTime = 0.0 76 | lastTime = -1.0 77 | for line in f: 78 | match = re.match('(^|.* )([0-9.]+) us \(\+ *([0-9.]+) us\) (.+)', line) 79 | if not match: 80 | continue 81 | thisEventTime = float(match.group(2))*1000.0 82 | thisEventInterval = float(match.group(3))*1000.0 83 | thisEvent = match.group(4) 84 | if options.noNumbers: 85 | thisEvent = re.sub('0x[0-9a-f]+', '?', thisEvent) 86 | thisEvent = re.sub('[0-9]+', '?', thisEvent) 87 | if (thisEventTime < lastTime): 88 | print('Time went backwards at the following line:\n%s' % (line)) 89 | lastTime = thisEventTime 90 | if thisEventInterval != 0.0: 91 | if not thisEvent in eventIntervals: 92 | eventIntervals[thisEvent] = [] 93 | eventIntervals[thisEvent].append(thisEventInterval) 94 | # print('%s %s %s' % (thisEventTime, thisEventInterval, thisEvent)) 95 | if startingEvent: 96 | if startingEvent in thisEvent: 97 | # Reset variables to indicate that we are starting a new 98 | # sequence of events from the starting event. 99 | startTime = thisEventTime 100 | foundStart = True 101 | eventCount = {} 102 | 103 | if not foundStart: 104 | continue 105 | 106 | # If we get here, it means that we have found an event that 107 | # is not the starting event, and startTime indicates the time of 108 | # the starting event. First, see how many times this event has 109 | # occurred since the last occurrence of the starting event. 110 | relativeTime = thisEventTime - startTime 111 | # print('%.1f %.1f %s' % (relativeTime, thisEventInterval, thisEvent)) 112 | if thisEvent in eventCount: 113 | count = eventCount[thisEvent] + 1 114 | else: 115 | count = 1 116 | eventCount[thisEvent] = count 117 | # print("Count for '%s': %d" % (thisEvent, count)) 118 | if not thisEvent in relativeEvents: 119 | relativeEvents[thisEvent] = [] 120 | occurrences = relativeEvents[thisEvent] 121 | while len(occurrences) < count: 122 | occurrences.append({'times': [], 'intervals': []}) 123 | occurrences[count-1]['times'].append(relativeTime) 124 | occurrences[count-1]['intervals'].append(thisEventInterval) 125 | 126 | # Parse command line options 127 | parser = OptionParser(description= 128 | 'Read one or more log files and summarize the time trace information ' 129 | 'present in the file(s) as specified by the arguments.', 130 | usage='%prog [options] file file ...', 131 | conflict_handler='resolve') 132 | parser.add_option('-a', '--alt', action='store_true', default=False, 133 | dest='altFormat', 134 | help='use alternate output format if -f is specified (print min, ' 135 | 'max, etc. for cumulative time, not delta)') 136 | parser.add_option('-f', '--from', type='string', dest='startEvent', 137 | help='measure times for other events relative to FROM; FROM contains a ' 138 | 'substring of an event') 139 | parser.add_option('-n', '--numbers', action='store_false', default=True, 140 | dest='noNumbers', 141 | help='treat numbers in event names as significant; if this flag ' 142 | 'is not specified, all numbers are replaced with ? (events will be ' 143 | 'considered the same if they differ only in numeric fields)') 144 | 145 | (options, files) = parser.parse_args() 146 | if len(files) == 0: 147 | print("No log files given") 148 | sys.exit(1) 149 | for name in files: 150 | scan(open(name), options.startEvent) 151 | 152 | # Print information about all events, unless --from was specified. 153 | if not options.startEvent: 154 | # Do this in 2 passes. First, generate a string describing each 155 | # event; then sort the list of messages and print. 156 | 157 | # Each entry in the following variable will contain a list with 158 | # 2 elements: time to use for sorting, and string to print. 159 | outputInfo = [] 160 | 161 | # Compute the length of the longest event name. 162 | nameLength = 0; 163 | for event in eventIntervals.keys(): 164 | nameLength = max(nameLength, len(event)) 165 | 166 | # Each iteration through the following loop processes one event name 167 | for event in eventIntervals.keys(): 168 | intervals = eventIntervals[event] 169 | intervals.sort() 170 | medianTime = intervals[len(intervals)//2] 171 | message = '%-*s %6.0f %6.0f %6.0f %6.0f %7d' % (nameLength, 172 | event, medianTime, intervals[0], intervals[-1], 173 | sum(intervals)/len(intervals), len(intervals)) 174 | outputInfo.append([medianTime, message]) 175 | 176 | # Pass 2: sort in order of median interval length, then print. 177 | outputInfo.sort(key=lambda item: item[0], reverse=True) 178 | print('%-*s Median Min Max Avg Count' % (nameLength, 179 | "Event")) 180 | print('%s---------------------------------------------' % 181 | ('-' * nameLength)) 182 | for message in outputInfo: 183 | print(message[1]) 184 | 185 | # Print output for the --from option. First, process each event occurrence, 186 | # then sort them by elapsed time from the starting event. 187 | if options.startEvent: 188 | # Each entry in the following variable will contain a list with 189 | # 2 elements: time to use for sorting, and string to print. 190 | outputInfo = [] 191 | 192 | # Compute the length of the longest event name. 193 | nameLength = 0; 194 | for event in relativeEvents.keys(): 195 | occurrences = relativeEvents[event] 196 | thisLength = len(event) 197 | if len(occurrences) > 1: 198 | thisLength += len(' (#%d)' % (len(occurrences))) 199 | nameLength = max(nameLength, thisLength) 200 | 201 | # Each iteration through the following loop processes one event name 202 | for event in relativeEvents.keys(): 203 | occurrences = relativeEvents[event] 204 | 205 | # Each iteration through the following loop processes the nth 206 | # occurrence of this event. 207 | for i in range(len(occurrences)): 208 | eventName = event 209 | if i != 0: 210 | eventName = '%s (#%d)' % (event, i+1) 211 | times = occurrences[i]['times'] 212 | intervals = occurrences[i]['intervals'] 213 | times.sort() 214 | medianTime = times[len(times)//2] 215 | intervals.sort() 216 | medianInterval = intervals[len(intervals)//2] 217 | if options.altFormat: 218 | message = '%-*s %6.0f %6.0f %6.0f %6.0f %6.0f %7d' % ( 219 | nameLength, eventName, medianTime, times[0], times[-1], 220 | sum(times)/len(times), intervals[len(intervals)//2], 221 | len(times)) 222 | else: 223 | message = '%-*s %6.0f %6.0f %6.0f %6.0f %6.0f %7d' % ( 224 | nameLength, eventName, medianTime, medianInterval, 225 | intervals[0], intervals[-1], sum(intervals)/len(intervals), 226 | len(intervals)) 227 | outputInfo.append([medianTime, message]) 228 | 229 | outputInfo.sort(key=lambda item: item[0]) 230 | if options.altFormat: 231 | print('%-*s Median Min Max Avg Delta Count' % (nameLength, 232 | "Event")) 233 | print('%s--------------------------------------------' % 234 | ('-' * nameLength)) 235 | else: 236 | print('%-*s Cum. ---------------Delta---------------' % 237 | (nameLength, "")) 238 | print('%-*s Median Median Min Max Avg Count' % 239 | (nameLength, "Event")) 240 | print('%s--------------------------------------------' % 241 | ('-' * nameLength)) 242 | for message in outputInfo: 243 | print(message[1]) -------------------------------------------------------------------------------- /update_remote: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This script copies modified information from this directory to a 4 | # CloudLab machine. 5 | 6 | target=`cat $HOME/.cloudlabNode` 7 | # rsync -rtv --exclude-from=rsync-exclude.txt ./ $target:/ouster/grpc/ 8 | rsync -rtv --exclude-from=rsync_exclude.txt ./ $target:/ouster/grpc_homa/ -------------------------------------------------------------------------------- /util.cc: -------------------------------------------------------------------------------- 1 | #include "homa.h" 2 | 3 | #include "util.h" 4 | 5 | /** 6 | * Returns a human-readable string containing the bpage indexes in a 7 | * homa_recvmsg_args structure. 8 | * \param recvArgs 9 | * Structure whose buffers entries should be returned. 10 | */ 11 | std::string bpagesToString(struct homa_recvmsg_args *recvArgs) 12 | { 13 | std::string result; 14 | for (uint32_t i = 0; i < recvArgs->num_bpages; i++) { 15 | char buffer[100]; 16 | if (!result.empty()) { 17 | result.append(" "); 18 | } 19 | snprintf(buffer, sizeof(buffer), "%u", recvArgs->bpage_offsets[i]); 20 | result.append(buffer); 21 | } 22 | return result; 23 | } 24 | 25 | /** 26 | * Fill in a block of memory with predictable values that can be checked 27 | * later by Mock::log_data. 28 | * \param data 29 | * Address of first byte of data. 30 | * \param length 31 | * Total amount of data, in bytes. 32 | * \param firstValue 33 | * Value to store in first 4 bytes of data. Each successive 4 bytes 34 | * of data will have a value 4 greater than the previous. 35 | */ 36 | void fillData(void *data, int length, int firstValue) 37 | { 38 | int i; 39 | uint8_t *p = static_cast(data); 40 | for (i = 0; i <= length-4; i += 4) { 41 | *reinterpret_cast(p + i) = firstValue + i; 42 | } 43 | 44 | /* Fill in extra bytes with a special value. */ 45 | for ( ; i < length; i += 1) { 46 | p[i] = 0xaa; 47 | } 48 | } 49 | 50 | /** 51 | * Generate log messages describing a batch of metadata. 52 | * \param mdBatch 53 | * Metadata for which to generate log messages. 54 | * \param info 55 | * Additional inforation about the nature of this metadata (included 56 | * in each log message). 57 | */ 58 | void logMetadata(const grpc_metadata_batch* mdBatch, const char *info) 59 | { 60 | if (mdBatch->empty()) { 61 | gpr_log(GPR_INFO, "%s: metadata empty", info); 62 | } 63 | mdBatch->Log([info] (absl::string_view key, absl::string_view value) { 64 | gpr_log(GPR_INFO, "%s: %.*s -> %.*s", info, 65 | static_cast(key.length()), key.data(), 66 | static_cast(value.length()), value.data()); 67 | }); 68 | } 69 | 70 | /** 71 | * Generate a string using printf-style arguments. 72 | * \param format 73 | * Standard printf-style format string. 74 | * \param ... 75 | * Values as needed to plug into the formula. 76 | */ 77 | std::string stringPrintf(const char* format, ...) 78 | { 79 | std::string result; 80 | va_list ap; 81 | va_start(ap, format); 82 | 83 | // We're not really sure how big of a buffer will be necessary. 84 | // Try 1K, if not the return value will tell us how much is necessary. 85 | int buf_size = 1024; 86 | while (true) { 87 | char buf[buf_size]; 88 | // vsnprintf trashes the va_list, so copy it first 89 | va_list aq; 90 | __va_copy(aq, ap); 91 | int length = vsnprintf(buf, buf_size, format, aq); 92 | assert(length >= 0); // old glibc versions returned -1 93 | if (length < buf_size) { 94 | result.append(buf, length); 95 | break; 96 | } 97 | buf_size = length + 1; 98 | } 99 | va_end(ap); 100 | return result; 101 | } 102 | 103 | /** 104 | * Generate a string describing all the useful information in a gRPC 105 | * Status value. 106 | */ 107 | std::string stringForStatus(grpc::Status *status) 108 | { 109 | std::string message = status->error_message(); 110 | if (message.empty()) { 111 | return symbolForCode(status->error_code()); 112 | } 113 | return stringPrintf("%s (%s)", symbolForCode(status->error_code()), 114 | message.c_str()); 115 | } 116 | 117 | /** 118 | * Return a printable string corresponding to a gRPC status code. 119 | */ 120 | const char *symbolForCode(grpc::StatusCode code) 121 | { 122 | static char buffer[100]; 123 | switch (code) { 124 | case grpc::OK: 125 | return "OK"; 126 | case grpc::CANCELLED: 127 | return "CANCELLED"; 128 | case grpc::UNKNOWN: 129 | return "UNKNOWN"; 130 | case grpc::INVALID_ARGUMENT: 131 | return "INVALID_ARGUMENT"; 132 | case grpc::DEADLINE_EXCEEDED: 133 | return "DEADLINE_EXCEEDED"; 134 | case grpc::NOT_FOUND: 135 | return "NOT_FOUND"; 136 | case grpc::ALREADY_EXISTS: 137 | return "ALREADY_EXISTS"; 138 | case grpc::PERMISSION_DENIED: 139 | return "PERMISSION_DENIED"; 140 | case grpc::RESOURCE_EXHAUSTED: 141 | return "RESOURCE_EXHAUSTED"; 142 | case grpc::FAILED_PRECONDITION: 143 | return "FAILED_PRECONDITION"; 144 | case grpc::ABORTED: 145 | return "ABORTED"; 146 | case grpc::OUT_OF_RANGE: 147 | return "OUT_OF_RANGE"; 148 | case grpc::UNIMPLEMENTED: 149 | return "UNIMPLEMENTED"; 150 | case grpc::INTERNAL: 151 | return "INTERNAL"; 152 | case grpc::UNAVAILABLE: 153 | return "UNAVAILABLE"; 154 | case grpc::DATA_LOSS: 155 | return "DATA_LOSS"; 156 | case grpc::UNAUTHENTICATED: 157 | return "UNAUTHENTICATED"; 158 | default: 159 | snprintf(buffer, sizeof(buffer), "Unknown status %d", code); 160 | return buffer; 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /util.h: -------------------------------------------------------------------------------- 1 | #ifndef UTIL_H 2 | #define UTIL_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include "src/core/lib/surface/channel.h" 12 | 13 | // This file contains miscellaneous small facilities that are useful 14 | // in the Homa gRPC driver. 15 | 16 | template 17 | size_t offsetOf(const M P::*member) 18 | { 19 | return reinterpret_cast(&( reinterpret_cast(0)->*member)); 20 | } 21 | 22 | template 23 | P* containerOf(M* ptr, const M P::*member) 24 | { 25 | return reinterpret_cast(reinterpret_cast(ptr) 26 | - offsetOf(member)); 27 | } 28 | 29 | inline void parseType(const char *s, char **end, int *value) 30 | { 31 | *value = strtol(s, end, 0); 32 | } 33 | 34 | inline void parseType(const char *s, char **end, int64_t *value) 35 | { 36 | *value = strtoll(s, end, 0); 37 | } 38 | 39 | inline void parseType(const char *s, char **end, double *value) 40 | { 41 | *value = strtod(s, end); 42 | } 43 | 44 | /** 45 | * Parse a value of a particular type from an argument word. 46 | * \param words 47 | * Words of a command being parsed. 48 | * \param i 49 | * Index within words of a word expected to contain an integer value 50 | * (may be outside the range of words, in which case an error message 51 | * is printed). 52 | * \param value 53 | * The parsed value corresponding to @words[i] is stored here, if the 54 | * function completes successfully. 55 | * \param option 56 | * Name of option being parsed (for use in error messages). 57 | * \param typeName 58 | * Human-readable name for ValueType (for use in error messages). 59 | * 60 | * \return 61 | * Nonzero means success, zero means an error occurred (and a 62 | * message was printed). 63 | */ 64 | template 65 | int parse(std::vector &words, unsigned i, ValueType *value, 66 | const char *option, const char *typeName) 67 | { 68 | ValueType num; 69 | char *end; 70 | 71 | if (i >= words.size()) { 72 | printf("No value provided for %s\n", option); 73 | return 0; 74 | } 75 | parseType(words[i].c_str(), &end, &num); 76 | if (*end != 0) { 77 | printf("Bad value '%s' for %s; must be %s\n", 78 | words[i].c_str(), option, typeName); 79 | return 0; 80 | } 81 | *value = num; 82 | return 1; 83 | } 84 | 85 | extern std::string bpagesToString( 86 | struct homa_recvmsg_args *control); 87 | extern void fillData(void *data, int length, int firstValue); 88 | extern void logMetadata(const grpc_metadata_batch* mdBatch, 89 | const char *info); 90 | extern std::string stringForStatus(grpc::Status *status); 91 | extern std::string stringPrintf(const char* format, ...) 92 | __attribute__((format(printf, 1, 2))); 93 | extern const char * symbolForCode(grpc::StatusCode code); 94 | 95 | #endif // UTIL_H -------------------------------------------------------------------------------- /wire.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include "util.h" 8 | #include "wire.h" 9 | 10 | /** 11 | * Print to the log the contents of a block of metadata, as serialized 12 | * by appendMetadata. 13 | * \param buffer 14 | * First byte of serialized data. 15 | * \param length 16 | * Total amount of serialized data. 17 | * \param severity 18 | * Log severity level to use for messages. 19 | */ 20 | void Wire::dumpMetadata(void *buffer, size_t length, gpr_log_severity severity) 21 | { 22 | size_t remaining = length; 23 | uint8_t *src = static_cast(buffer); 24 | 25 | // Each iteration prints one metadata value 26 | while (remaining > 0) { 27 | Wire::Mdata* msgMd = reinterpret_cast(src); 28 | if (remaining < sizeof(*msgMd)) { 29 | gpr_log(__FILE__, __LINE__, severity, "Not enough bytes for " 30 | "metadata header: need %lu, have %lu", 31 | sizeof(*msgMd), remaining); 32 | return; 33 | } 34 | uint32_t keyLength = ntohl(msgMd->keyLength); 35 | uint32_t valueLength = ntohl(msgMd->valueLength); 36 | remaining -= sizeof(*msgMd); 37 | src += sizeof(*msgMd); 38 | if (remaining < (keyLength + valueLength)) { 39 | gpr_log(__FILE__, __LINE__, severity, "Not enough bytes for " 40 | "key and value: need %u, have %lu", 41 | keyLength + valueLength, remaining); 42 | return; 43 | } 44 | gpr_log(__FILE__, __LINE__, severity, 45 | "Key: %.*s, value: %.*s", keyLength, src, valueLength, 46 | src+keyLength); 47 | remaining -= keyLength + valueLength; 48 | src += keyLength + valueLength; 49 | } 50 | } 51 | 52 | /** 53 | * Log information about the contents of a Homa message header. 54 | * \param msg 55 | * Address of the first byte of a Homa message (expected to contain 56 | * a valid header). 57 | * \param severity 58 | * Log severity level to use for messages. 59 | */ 60 | void Wire::dumpHeader(void *msg, gpr_log_severity severity) 61 | { 62 | Wire::Header *hdr = static_cast(msg); 63 | std::string s; 64 | char buffer[100]; 65 | 66 | snprintf(buffer, sizeof(buffer), "id: %u, sequence %u", 67 | ntohl(hdr->streamId), ntohl(hdr->sequenceNum)); 68 | s.append(buffer); 69 | if (hdr->initMdBytes) { 70 | snprintf(buffer, sizeof(buffer), ", initMdBytes %u", 71 | ntohl(hdr->initMdBytes)); 72 | s.append(buffer); 73 | } 74 | if (hdr->messageBytes) { 75 | snprintf(buffer, sizeof(buffer), ", messageBytes %u", 76 | ntohl(hdr->messageBytes)); 77 | s.append(buffer); 78 | } 79 | if (hdr->trailMdBytes) { 80 | snprintf(buffer, sizeof(buffer), ", trailMdBytes %u", 81 | ntohl(hdr->trailMdBytes)); 82 | s.append(buffer); 83 | } 84 | if (hdr->flags & Header::initMdPresent) { 85 | s.append(", initMdPresent"); 86 | } 87 | if (hdr->flags & Header::messageComplete) { 88 | s.append(", messageComplete"); 89 | } 90 | if (hdr->flags & Header::trailMdPresent) { 91 | s.append(", trailMdPresent"); 92 | } 93 | if (hdr->flags & Header::request) { 94 | s.append(", request"); 95 | } 96 | if (hdr->flags & Header::emptyResponse) { 97 | s.append(", emptyResponse"); 98 | } 99 | if (hdr->flags & Header::cancelled) { 100 | s.append(", cancelled"); 101 | } 102 | gpr_log(__FILE__, __LINE__, severity, "Header: %s", s.c_str()); 103 | } -------------------------------------------------------------------------------- /wire.h: -------------------------------------------------------------------------------- 1 | #ifndef WIRE_H 2 | #define WIRE_H 3 | 4 | #include "src/core/lib/transport/transport.h" 5 | 6 | /* This file defines the on-the-wire format of messages used to implement 7 | * gRPC over Homa. 8 | */ 9 | 10 | // Values that are stored in network byte order ("big endian"). 11 | typedef int16_t be16; 12 | typedef int32_t be32; 13 | typedef int64_t be64; 14 | 15 | /** 16 | * This class defines the on-the-wire format of messages used to implement 17 | * gRPC over Homa, and also provides methods for serializing and 18 | * deserializing messages. Note: if you make any changes to this class, 19 | * you may also need to make changes to HomaWire.java in the Java 20 | * implementation of gRPC over Homa, in order to maintain interoperability 21 | * between the two implementations. 22 | */ 23 | class Wire { 24 | public: 25 | /** 26 | * Every Homa RPC (whether request or response) starts with this 27 | * information. 28 | */ 29 | struct Header { 30 | // Unique identifier for this stream (all messages for this RPC 31 | // will use the same identifier). 32 | be32 streamId; 33 | 34 | // Position of this Homa message among all of those sent on 35 | // this stream. Used on the other end to make sure that messages 36 | // are processed in order. The first number for each stream is 1. 37 | be32 sequenceNum; 38 | 39 | // Number of bytes of initial metadata (may be zero), which 40 | // follows this header in the Homa message. 41 | be32 initMdBytes; 42 | 43 | // Number of bytes of trailing metadata (may be zero), which 44 | // follows the initial metadata. 45 | be32 trailMdBytes; 46 | 47 | // Number of bytes of gRPC message data (may be zero), which follows 48 | // the trailing metadata. 49 | be32 messageBytes; 50 | 51 | // ORed combination of one or more flag bits defined below. 52 | uint8_t flags; 53 | 54 | // Flag bit indicating that this message contains all available 55 | // initial metadata (possibly none). 56 | static const int initMdPresent = 1; 57 | 58 | // Flag bit indicating that, as of this Homa RPC, all message 59 | // data has been sent. If the message data is too long to fit 60 | // in a single message, only the last message has this bit set. 61 | static const int messageComplete = 2; 62 | 63 | // Flag bit indicating that this message contains all available 64 | // trailing metadata (possibly none). 65 | static const int trailMdPresent = 4; 66 | 67 | // Flag bit indicating that this message is a Homa request 68 | // (meaning it that it requires an eventual response). 69 | static const int request = 8; 70 | 71 | // Indicates that there is no useful information in this message; 72 | // it was a dummy Homa response sent by the other side. 73 | static const int emptyResponse = 16; 74 | 75 | // Flag bit indicating that the sender has cancelled this 76 | // RPC, and the receiver should do the same. 77 | static const int cancelled = 32; 78 | 79 | Header(int streamId, int sequence, int initMdBytes, int trailMdBytes, 80 | int messageBytes) 81 | : streamId(htonl(streamId)) 82 | , sequenceNum(htonl(sequence)) 83 | , initMdBytes(htonl(initMdBytes)) 84 | , trailMdBytes(htonl(trailMdBytes)) 85 | , messageBytes(htonl(messageBytes)) 86 | , flags(0) 87 | { } 88 | 89 | Header(int streamId, int sequence) 90 | : streamId(htonl(streamId)) 91 | , sequenceNum(htonl(sequence)) 92 | , initMdBytes(0) 93 | , trailMdBytes(0) 94 | , messageBytes(0) 95 | , flags(0) 96 | { } 97 | 98 | Header() 99 | : streamId() 100 | , sequenceNum() 101 | , initMdBytes() 102 | , trailMdBytes() 103 | , messageBytes() 104 | , flags() 105 | { } 106 | 107 | } __attribute__((packed)); 108 | 109 | /** Each metadata value has the following format. */ 110 | struct Mdata { 111 | // Number of bytes in the key for this item. 112 | be32 keyLength; 113 | 114 | // Number of bytes in the value for this item. 115 | be32 valueLength; 116 | 117 | // The key is stored starting here, followed by the value. 118 | char data[0]; 119 | } __attribute__((packed)); 120 | 121 | static void dumpHeader(void *msg, gpr_log_severity severity); 122 | static void dumpMetadata(void *buffer, size_t length, 123 | gpr_log_severity severity = GPR_LOG_SEVERITY_INFO); 124 | }; 125 | 126 | #endif // WIRE_H --------------------------------------------------------------------------------