├── .gitignore ├── LICENSE ├── README.md ├── benchmark-gym-http.py ├── benchmark-gym-uds.py ├── binding-cpp ├── Makefile ├── include │ └── gym-uds.h └── src │ ├── gym-uds-client.cc │ └── gym-uds.cc ├── gym-uds-server.py ├── gym-uds-test-client.py └── gym-uds.proto /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | binding-cpp/bin/ 3 | gym_http_client.py 4 | gym_http_server.py 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Francesco Cagnin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gym-uds-api 2 | This project presents a basic API for interacting with [OpenAI Gym](https://github.com/openai/gym) environments in languages other than Python through the [gRPC](https://grpc.io) framework and local Unix domain sockets. 3 | 4 | The API comes with example C++ bindings supporting one-dimensional observation spaces of type `Box` and action spaces of type `Discrete` (suitable, for instance, for the CartPole-v0 environment). 5 | 6 | ## Requisites 7 | Example instructions are provided for macOS (tested on macOS Catalina 10.15.3). 8 | 9 | 0. Install [Homebrew](https://brew.sh). 10 | 11 | 1. Install pkg-config: 12 | 13 | ~$ brew install pkg-config 14 | 15 | 1. Install [protobuf](https://github.com/protocolbuffers/protobuf): 16 | 17 | ~$ brew install protobuf 18 | 19 | 1. Install gRPC: 20 | 21 | ~$ brew install grpc 22 | 23 | 1. Install the grpcio_tools, [NumPy](https://numpy.org) and [OpenAI Gym](https://github.com/openai/gym) pip packages (for Python 3): 24 | 25 | ~$ pip install grpcio_tools numpy gym 26 | 27 | Tested with: 28 | ``` 29 | ~$ brew list --versions pkg-config protobuf grpc 30 | grpc 1.27.3 31 | pkg-config 0.29.2 32 | protobuf 3.11.4 33 | ``` 34 | ``` 35 | ~$ pip list | egrep "grpcio|numpy|gym" 36 | grpcio 1.27.2 37 | grpcio-tools 1.27.2 38 | gym 0.17.0 39 | numpy 1.18.1 40 | ``` 41 | 42 | ## Installation 43 | 1. Clone this repository: 44 | 45 | ~$ git clone https://github.com/integeruser/gym-uds-api 46 | 47 | 1. `cd` to the `gym-uds-api` directory and generate the gRPC headers and sources for the Python server and client: 48 | 49 | gym-uds-api$ python -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. ./gym-uds.proto 50 | 51 | 1. To build the C++ client, generate the gRPC headers and sources for C++ and move them to the `binding-cpp` directory: 52 | 53 | gym-uds-api$ protoc -I=. --cpp_out=. --grpc_out=. --plugin=protoc-gen-grpc=$(which grpc_cpp_plugin) ./gym-uds.proto 54 | gym-uds-api$ mv ./gym-uds.pb.h binding-cpp/include/ 55 | gym-uds-api$ mv ./gym-uds.pb.cc binding-cpp/src/ 56 | gym-uds-api$ mv ./gym-uds.grpc.pb.h binding-cpp/include/ 57 | gym-uds-api$ mv ./gym-uds.grpc.pb.cc binding-cpp/src/ 58 | 59 | ## Usage 60 | 1. Start the Python server: 61 | 62 | gym-uds-api$ python ./gym-uds-server.py CartPole-v0 63 | 64 | 1. On a second terminal, execute the dummy Python client: 65 | 66 | gym-uds-api$ python ./gym-uds-test-client.py 67 | Ep. 1: 15.00 68 | Ep. 2: 12.00 69 | Ep. 3: 20.00 70 | 71 | 1. Alternatively, on a second terminal, `cd` to the `binding-cpp` directory, then build and execute the dummy C++ client: 72 | 73 | gym-uds-api/binding-cpp$ make 74 | gym-uds-api/binding-cpp$ bin/gym-uds-client 75 | Ep. 1: 19 76 | Ep. 2: 13 77 | Ep. 3: 10 78 | -------------------------------------------------------------------------------- /benchmark-gym-http.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | 4 | import gym_http_client # https://github.com/openai/gym-http-api 5 | 6 | 7 | class HttpEnvironment: 8 | def __init__(self, env_id): 9 | self.client = gym_http_client.Client('http://127.0.0.1:5000') 10 | self.instance_id = self.client.env_create(env_id) 11 | 12 | def reset(self): 13 | return self.client.env_reset(self.instance_id) 14 | 15 | def sample(self): 16 | return self.client.env_action_space_sample(self.instance_id) 17 | 18 | def step(self, action): 19 | return self.client.env_step(self.instance_id, action) 20 | 21 | 22 | if __name__ == '__main__': 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument('env_id') 25 | args = parser.parse_args() 26 | 27 | env = HttpEnvironment(args.env_id) 28 | benchmark_gym_uds = __import__('benchmark-gym-uds') 29 | benchmark_gym_uds.benchmark(env) 30 | -------------------------------------------------------------------------------- /benchmark-gym-uds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import timeit 3 | 4 | 5 | def benchmark(env, MAX_NUM_STEPS=1): 6 | start = timeit.default_timer() 7 | 8 | num_steps = 0 9 | benchmark_is_over = False 10 | while not benchmark_is_over: 11 | env.reset() 12 | 13 | done = False 14 | while not done: 15 | action = env.sample() 16 | env.step(action) 17 | 18 | num_steps += 1 19 | if num_steps == MAX_NUM_STEPS: 20 | benchmark_is_over = True 21 | break 22 | 23 | end = timeit.default_timer() 24 | print('%d steps in %f seconds' % (num_steps, end - start)) 25 | 26 | 27 | class UdsEnvironment: 28 | def __init__(self, sock_filepath): 29 | self.env = __import__('gym-uds-test-client').EnvironmentClient(sock_filepath) 30 | 31 | def reset(self): 32 | return self.env.reset() 33 | 34 | def sample(self): 35 | return self.env.action_space.sample() 36 | 37 | def step(self, action): 38 | return self.env.step(action) 39 | 40 | 41 | if __name__ == '__main__': 42 | env = UdsEnvironment('unix:///tmp/gym-uds-socket') 43 | benchmark(env) 44 | -------------------------------------------------------------------------------- /binding-cpp/Makefile: -------------------------------------------------------------------------------- 1 | CXXFLAGS=-std=c++11 -O2 -march=native -Wall 2 | ifeq ($(shell uname | cut -f 1 -d_),Darwin) 3 | LDFLAGS += -L/usr/local/lib `pkg-config --libs protobuf grpc++`\ 4 | -pthread\ 5 | -lgrpc++_reflection\ 6 | -ldl 7 | else 8 | LDFLAGS += -L/usr/local/lib `pkg-config --libs protobuf grpc++`\ 9 | -pthread\ 10 | -Wl,--no-as-needed -lgrpc++_reflection -Wl,--as-needed\ 11 | -ldl 12 | endif 13 | 14 | default: 15 | mkdir -p bin 16 | $(CXX) $(CXXFLAGS) $(LDFLAGS) -o bin/gym-uds-client \ 17 | -I include \ 18 | src/gym-uds.pb.cc src/gym-uds-client.cc src/gym-uds.grpc.pb.cc src/gym-uds.cc 19 | 20 | clean: 21 | rm -rf bin/gym-uds-client 22 | -------------------------------------------------------------------------------- /binding-cpp/include/gym-uds.h: -------------------------------------------------------------------------------- 1 | #ifndef GYM_UDS_H 2 | #define GYM_UDS_H 3 | 4 | #include 5 | #include 6 | 7 | #include "gym-uds.pb.h" 8 | #include "gym-uds.grpc.pb.h" 9 | 10 | class EnvironmentClient 11 | { 12 | private: 13 | std::unique_ptr stub; 14 | 15 | public: 16 | EnvironmentClient(const std::string &); 17 | 18 | void reset(State *state); 19 | void step(const Action &action, State *state); 20 | 21 | void sample(Action *action); 22 | }; 23 | 24 | #endif // GYM_UDS_H 25 | -------------------------------------------------------------------------------- /binding-cpp/src/gym-uds-client.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "gym-uds.h" 5 | #include "gym-uds.pb.h" 6 | #include "gym-uds.grpc.pb.h" 7 | 8 | int main(int argc, char const *argv[]) 9 | { 10 | GOOGLE_PROTOBUF_VERIFY_VERSION; 11 | 12 | auto env = EnvironmentClient("unix:///tmp/gym-uds-socket"); 13 | 14 | const int num_episodes = 3; 15 | for (int episode = 1; episode <= num_episodes; ++episode) 16 | { 17 | State state; 18 | env.reset(&state); 19 | 20 | float episode_reward = 0.0f; 21 | while (!state.done()) 22 | { 23 | Action action; 24 | env.sample(&action); 25 | 26 | env.step(action, &state); 27 | episode_reward += state.reward(); 28 | } 29 | std::cout << "Ep. " << episode << ": " << episode_reward << std::endl; 30 | } 31 | 32 | return EXIT_SUCCESS; 33 | } 34 | -------------------------------------------------------------------------------- /binding-cpp/src/gym-uds.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "gym-uds.h" 5 | 6 | #include 7 | 8 | EnvironmentClient::EnvironmentClient(const std::string &sockfilepath) 9 | : stub(Environment::NewStub(grpc::CreateChannel(sockfilepath, grpc::InsecureChannelCredentials()))) 10 | { 11 | } 12 | 13 | void EnvironmentClient::reset(State *state) 14 | { 15 | grpc::ClientContext context; 16 | Empty empty; 17 | grpc::Status status = stub->Reset(&context, empty, state); 18 | assert(status.ok()); 19 | } 20 | void EnvironmentClient::step(const Action &action, State *state) 21 | { 22 | grpc::ClientContext context; 23 | grpc::Status status = stub->Step(&context, action, state); 24 | assert(status.ok()); 25 | } 26 | 27 | void EnvironmentClient::sample(Action *action) 28 | { 29 | grpc::ClientContext context; 30 | Empty empty; 31 | grpc::Status status = stub->Sample(&context, empty, action); 32 | assert(status.ok()); 33 | } 34 | -------------------------------------------------------------------------------- /gym-uds-server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import os 4 | import time 5 | from concurrent import futures 6 | 7 | import gym 8 | import numpy as np 9 | 10 | import grpc 11 | import gym_uds_pb2 12 | import gym_uds_pb2_grpc 13 | 14 | 15 | class EnvironmentServicer(gym_uds_pb2_grpc.EnvironmentServicer): 16 | def __init__(self, env_id): 17 | self.env = gym.make(env_id) 18 | 19 | def Reset(self, empty_request, context): 20 | observation = self.env.reset() 21 | observation_pb = gym_uds_pb2.Observation(data=observation.ravel(), shape=observation.shape) 22 | return gym_uds_pb2.State(observation=observation_pb, reward=0.0, done=False) 23 | 24 | def Step(self, action_request, context): 25 | observation, reward, done, _ = self.env.step(action_request.value) 26 | observation_pb = gym_uds_pb2.Observation(data=observation.ravel(), shape=observation.shape) 27 | return gym_uds_pb2.State(observation=observation_pb, reward=reward, done=done) 28 | 29 | def Sample(self, empty_request, context): 30 | action = self.env.action_space.sample() 31 | return gym_uds_pb2.Action(value=action) 32 | 33 | 34 | if __name__ == "__main__": 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument("id", help="the id of the gym environment to simulate") 37 | parser.add_argument( 38 | "sockfilepath", 39 | nargs="?", 40 | default="unix:///tmp/gym-uds-socket", 41 | help="a unique filepath where the Unix domain server will bind", 42 | ) 43 | args = parser.parse_args() 44 | 45 | try: 46 | os.remove(args.sockfilepath) 47 | except FileNotFoundError: 48 | pass 49 | 50 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) 51 | gym_uds_pb2_grpc.add_EnvironmentServicer_to_server(EnvironmentServicer(args.id), server) 52 | server.add_insecure_port(args.sockfilepath) 53 | server.start() 54 | server.wait_for_termination() 55 | -------------------------------------------------------------------------------- /gym-uds-test-client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | 4 | import numpy as np 5 | 6 | import grpc 7 | import gym_uds_pb2 8 | import gym_uds_pb2_grpc 9 | 10 | 11 | class EnvironmentClient: 12 | def __init__(self, sockfilepath): 13 | channel = grpc.insecure_channel(sockfilepath) 14 | self.stub = gym_uds_pb2_grpc.EnvironmentStub(channel) 15 | self.action_space = lambda: None 16 | self.action_space.sample = self.sample 17 | 18 | def reset(self): 19 | state_pb = self.stub.Reset(gym_uds_pb2.Empty()) 20 | observation = np.asarray(state_pb.observation.data).reshape(state_pb.observation.shape) 21 | return observation 22 | 23 | def step(self, action): 24 | state_pb = self.stub.Step(gym_uds_pb2.Action(value=action)) 25 | observation = np.asarray(state_pb.observation.data).reshape(state_pb.observation.shape) 26 | return observation, state_pb.reward, state_pb.done 27 | 28 | def sample(self): 29 | action_pb = self.stub.Sample(gym_uds_pb2.Empty()) 30 | return action_pb.value 31 | 32 | 33 | if __name__ == "__main__": 34 | parser = argparse.ArgumentParser() 35 | parser.add_argument( 36 | "sockfilepath", 37 | nargs="?", 38 | default="unix:///tmp/gym-uds-socket", 39 | help="a unique filepath where the Unix domain client will connect", 40 | ) 41 | args = parser.parse_args() 42 | 43 | env = EnvironmentClient(args.sockfilepath) 44 | 45 | num_episodes = 3 46 | for episode in range(1, num_episodes + 1): 47 | observation = env.reset() 48 | 49 | episode_reward = 0 50 | done = False 51 | while not done: 52 | action = env.action_space.sample() 53 | observation, reward, done = env.step(action) 54 | episode_reward += reward 55 | print("Ep. {}: {:.2f}".format(episode, episode_reward)) 56 | -------------------------------------------------------------------------------- /gym-uds.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | service Environment { 4 | rpc Reset (Empty) returns (State) {} 5 | rpc Step (Action) returns (State) {} 6 | rpc Sample (Empty) returns (Action) {} 7 | } 8 | 9 | 10 | message Empty { 11 | } 12 | 13 | 14 | message State { 15 | Observation observation = 1; 16 | float reward = 2; 17 | bool done = 3; 18 | } 19 | 20 | message Observation { 21 | repeated float data = 1; 22 | repeated int32 shape = 2; 23 | } 24 | 25 | 26 | message Action { 27 | int32 value = 1; 28 | } 29 | --------------------------------------------------------------------------------