├── .gitignore ├── Dockerfile ├── Dockerfile-ppo ├── Instructions.md ├── README.md ├── baselines ├── __init__.py ├── agents.py └── config │ ├── ppo-agent.yaml │ └── random-agent.yaml ├── config ├── check-ground-truth.yaml ├── ground-truth.yaml └── perception-pipeline.yaml ├── doc ├── details.md ├── generate.md ├── images │ ├── depth_estimate.png │ ├── depth_ground_truth.png │ ├── fruit.jpg │ ├── goseek-scenes.jpg │ ├── gt_est_pose.png │ ├── gt_pose.png │ ├── rgb_estimate.png │ ├── rgb_ground_truth.png │ ├── segmentation_estimate.png │ ├── segmentation_ground_truth.png │ └── unity_coords.png ├── perception-pipeline.md └── ppo-baseline.md ├── docker ├── .gitignore ├── goseek-base │ └── Dockerfile └── goseek-kimera │ └── Dockerfile ├── eval.py ├── logs └── .gitkeep └── test_locally.py /.gitignore: -------------------------------------------------------------------------------- 1 | simulator/ 2 | 3 | __pycache__ 4 | 5 | *.pyc 6 | .vscode/ 7 | logs/* 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 3 | # 4 | # This material is based upon work supported by the Under Secretary of Defense for Research and 5 | # Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions 6 | # or recommendations expressed in this material are those of the author(s) and do not necessarily 7 | # reflect the views of the Under Secretary of Defense for Research and Engineering. 8 | # 9 | # (c) 2020 Massachusetts Institute of Technology. 10 | # 11 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 12 | # 13 | # The software/firmware is provided to you on an As-Is basis 14 | # 15 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 16 | # or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work 17 | # are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other 18 | # than as specifically authorized by the U.S. Government may violate any copyrights that exist in 19 | # this work. 20 | ################################################################################################### 21 | 22 | from goseek-base:latest 23 | 24 | WORKDIR /goseek-challenge 25 | 26 | COPY baselines/agents.py baselines/agents.py 27 | 28 | COPY baselines/config/random-agent.yaml agent.yaml 29 | -------------------------------------------------------------------------------- /Dockerfile-ppo: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 3 | # 4 | # This material is based upon work supported by the Under Secretary of Defense for Research and 5 | # Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions 6 | # or recommendations expressed in this material are those of the author(s) and do not necessarily 7 | # reflect the views of the Under Secretary of Defense for Research and Engineering. 8 | # 9 | # (c) 2020 Massachusetts Institute of Technology. 10 | # 11 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 12 | # 13 | # The software/firmware is provided to you on an As-Is basis 14 | # 15 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 16 | # or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work 17 | # are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other 18 | # than as specifically authorized by the U.S. Government may violate any copyrights that exist in 19 | # this work. 20 | ################################################################################################### 21 | 22 | from goseek-base:latest 23 | 24 | RUN apt-get update && \ 25 | apt-get install python-opencv -y && \ 26 | pip install stable-baselines && \ 27 | conda install tensorflow-gpu==1.13.1 28 | 29 | WORKDIR /goseek-challenge 30 | 31 | COPY baselines/agents.py baselines/agents.py 32 | 33 | COPY baselines/config/ppo-agent.yaml agent.yaml 34 | 35 | COPY ppo-weights.pkl ppo-weights.pkl 36 | -------------------------------------------------------------------------------- /Instructions.md: -------------------------------------------------------------------------------- 1 | # GOSEEK Instructions 2 | 3 | These instructions will get your local machine setup to train, test, and submit solutions for the [GOSEEK challenge](README.md). 4 | 5 | Contents: 6 | 7 | * [Prerequisites](#Prerequisites) 8 | * [Installation](#Installation) 9 | * [Usage](#Usage) 10 | * [Training](#Training) 11 | * [Local Evaluation](#Local-Evaluation) 12 | * [Prepare Docker Submission](#Prepare-Docker-Submission) 13 | * [Examples](#Examples) 14 | 15 | 16 | ## Prerequisites 17 | 18 | The competition requires that you use linux. 19 | 20 | Using [Anaconda](https://www.anaconda.com/distribution/#download-section) or [miniconda](https://docs.conda.io/en/latest/miniconda.html) is highly recommended. 21 | Python 3.7 is required. 22 | 23 | Participating in the competition requires Docker, as well. 24 | The __Perception Pipeline__ is defined in a Docker container. 25 | Participant policies are also submitted as Docker containers. 26 | Install [Docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/) and then install [`nvidia-docker`](https://github.com/NVIDIA/nvidia-docker#quickstart) on your host machine. 27 | Note that if you are behind a proxy, please [follow these instructions on configuring the docker client](https://docs.docker.com/network/proxy/#configure-the-docker-client) to use your organization's proxy settings. 28 | 29 | ## Installation 30 | 31 | 1. If using conda, create a new conda environment: 32 | 33 | ```sh 34 | conda create -n goseek python=3.7 ipython jupyter numpy scipy 35 | conda activate goseek 36 | ``` 37 | 38 | 2. Install tesse-gym. 39 | ```sh 40 | git clone https://github.com/MIT-TESSE/tesse-gym.git 41 | cd tesse-gym 42 | python setup.py develop 43 | cd .. 44 | ``` 45 | Note that we recommend installing in [development mode](https://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode) in case you wish to make any local modifications. 46 | See [below](#training) for further discussion. 47 | 48 | 49 | 3. Clone this repository and install requirements. 50 | 51 | ```sh 52 | git clone https://github.com/MIT-TESSE/goseek-challenge.git 53 | cd goseek-challenge 54 | ``` 55 | 56 | 57 | 4. Next, you need to obtain GOSEEK simulator. Execute the following: 58 | ```sh 59 | mkdir -p simulator 60 | wget https://github.com/MIT-TESSE/goseek-challenge/releases/download/0.1.0/goseek-v0.1.4.zip -P simulator 61 | unzip simulator/goseek-v0.1.4.zip -d simulator 62 | chmod +x simulator/goseek-v0.1.4.x86_64 63 | ``` 64 | 65 | This creates a new `simulator` folder, download and unzips the simulator to that folder, and makes the simulator executable. 66 | Note that if you choose to place the simulator in an alternative location, you will need to specify the location in a configuration file that overrides the default value such as in [config/check-ground-truth.yaml](config/check-ground-truth.yaml). 67 | 68 | 5. Test your installation by running a random agent. The agent receives observations and takes random actions: 69 | 70 | ```sh 71 | python eval.py --agent-config baselines/config/random-agent.yaml --episode-config config/check-ground-truth.yaml 72 | ``` 73 | 74 | 6. Next, build a docker image called `goseek-base`, which is needed to submit online solutions. 75 | 76 | ```sh 77 | cd docker/goseek-base/ 78 | docker build -t goseek-base . 79 | cd ../../ 80 | ``` 81 | 82 | Optionally, run the following to verify the docker image. It should print the usage instructions for [eval.py](eval.py). 83 | ```sh 84 | docker run --rm -it goseek-base /bin/bash -c "python eval.py --help" 85 | ``` 86 | 87 | 88 | 7. In order to run the __Perception Pipeline__, you will need another docker image with [Kimera](https://github.com/MIT-SPARK/Kimera). Directions for building this image (named `goseek-kimera`) are as follows. 89 | Please note that building this image is not required to submit a solution for the competition; it is only used for any local testing and/or training with the __Perception Pipeline__. 90 | We strongly encourage participants to review [further details](doc/perception-pipeline.md) on the perception pipeline, as well. 91 | 92 | a. First, obtain [TensorRT](https://developer.nvidia.com/nvidia-tensorrt-download). 93 | Note that you'll need to create an Nvidia account and manually download it because TensorRT is proprietary. 94 | Select `TensorRT-6.0.1.5.Ubuntu-18.04.x86_64-gnu.cuda-10.0.cudnn7.6.tar.gz` from the Nvidia TensorRT 95 | download page and **place the binary in `goseek-challenge/docker/goseek-kimera/`**. 96 | 97 | b. Build the docker image. 98 | Note that this takes quite a while. 99 | (Go watch a movie? :man_shrugging:) 100 | 101 | ```sh 102 | docker build --rm -t goseek-kimera . 103 | ``` 104 | 105 | c. And, optionally again, run the following to verify the docker image. 106 | It should return the list of directory contents in [tesse_gym_bridge](https://www.github.com/MIT-TESSE/tesse-gym-bridge). 107 | 108 | ```sh 109 | docker run --rm -it goseek-kimera /bin/bash -c "source /catkin_ws/devel/setup.bash && rosls tesse_gym_bridge" 110 | ``` 111 | 112 | 113 | ## Usage 114 | 115 | ### Training 116 | 117 | We implement the [OpenAI Gym](https://gym.openai.com/) interface in [tesse-gym](https://github.com/MIT-TESSE/tesse-gym) which can be used for reinforcement learning. 118 | 119 | Our specific implementation for this challenge can be found in the [goseek module](https://github.com/MIT-TESSE/tesse-gym/tree/master/src/tesse_gym/tasks/goseek). Participants are welcome to locally modify relevant details for developing a solution (e.g. modifying the reward function). 120 | 121 | We provide a complete [example](doc/ppo-baseline.md) demonstrating how to train a PPO agent using [tesse-gym](https://github.com/MIT-TESSE/tesse-gym). 122 | 123 | 124 | ### Local Evaluation 125 | 126 | 1. Implement the following interface in `baselines/agents.py`. 127 | 128 | 129 | ```python 130 | class Agent: 131 | """ Interface for submitting an agent for evaluation. """ 132 | 133 | def act(self, observation: np.ndarray) -> int: 134 | """ Act upon an environment observation. 135 | 136 | The observation is given as a vector of shape (384003,). 137 | The first 384000 values contain RGB, depth, and segmentation images, 138 | the last three contain pose in (x, y, heading). 139 | 140 | `tesse_gym` provides a function to decode this: 141 | 142 | >>> from tesse_gym.tasks.goseek import decode_observations 143 | >>> rgb, segmentation, depth, pose = decode_observations(observation) 144 | 145 | Providing image and pose data: 146 | 147 | >>> rgb.shape 148 | (1, 240, 320, 3) 149 | >>> segmentation.shape 150 | (1, 240, 320, 3) 151 | >>> depth.shape 152 | (1, 240, 320, 3) 153 | >>> pose.shape 154 | (1, 3) 155 | 156 | Args: 157 | observation (np.ndarray): Shape (1, 384003) array of observations as described above. 158 | 159 | Returns: 160 | int: Agent's action in the range [0,3]. 161 | """ 162 | raise NotImplementedError 163 | 164 | def reset(self) -> None: 165 | """ Called when the environment resets. """ 166 | raise NotImplementedError 167 | ``` 168 | 169 | 2. Define configuration files 170 | 171 | All configurations required by the agent must be specified by a YAML file. This file must contain the field `name`, 172 | specifying the agent class name. All other fields will be passed as keyword arguments to the agent's class constructor 173 | upon initialization. An example is below: 174 | 175 | ```yaml 176 | # example-configuration.yaml 177 | name: AgentName 178 | custom_field_1: VALUE_1 179 | ... 180 | custom_field_n: VALUE_N 181 | ``` 182 | 183 | ### Prepare Docker Submission 184 | 185 | Competition submission are submitted as docker image, which you are responsible for preparing. 186 | 187 | We will run [eval.py](eval.py) on a participant docker image, which has the following usage. 188 | ``` 189 | usage: eval.py [-h] [--episode-config EPISODE_CONFIG] 190 | [--agent-config AGENT_CONFIG] 191 | 192 | optional arguments: 193 | -h, --help show this help message and exit 194 | --episode-config EPISODE_CONFIG 195 | --agent-config AGENT_CONFIG 196 | ``` 197 | 198 | 199 | Note the following. 200 | - We will run `eval.py` with an `EPISODE_CONFIG` value that points to a file we mount on the docker image with episode configuration information. 201 | Example configuration files, which are used for local testing, can be found in [config](config). 202 | - You are responsible for updating [baselines/agents.py](baselines/agents.py) to include your agent definition. 203 | Your code changes and any dependencies or additional files must be incorporated into the docker image. 204 | - We will also run `eval.py` with `AGENT_CONFIG` defined as `agent.yaml`. 205 | You are responsible for defining this file in the docker image. 206 | Note that if your policy does not require any configuration, then an empty file is acceptable. 207 | 208 | #### Create docker image 209 | 210 | This repository has a [Dockerfile](Dockerfile) that specifies a `RandomAgent`. 211 | It copies `baselines/agents.py`, which defines the `RandomAgent`. 212 | It also copies a configuration file for the `RandomAgent` to `agent.yaml`. 213 | 214 | Update this file as appropriate for your agent. 215 | 216 | When complete, build your docker image. 217 | ```sh 218 | docker build -t submission . 219 | ``` 220 | 221 | #### Test docker image 222 | 223 | You can test your docker image locally using [test_locally.py](test_locally.py). It has the following usage. 224 | ``` 225 | usage: test_locally.py [-h] -s SIMULATOR -i IMAGE (-g | -p) 226 | 227 | optional arguments: 228 | -h, --help show this help message and exit 229 | -s SIMULATOR, --simulator SIMULATOR 230 | Path to the simulator binary 231 | -i IMAGE, --image IMAGE 232 | Name of the docker image to use for local evaluation 233 | -g, --groundtruth Use groundtruth observations 234 | -p, --perception Use realistic perception for observations 235 | ``` 236 | 237 | For example, you can run the following to test against __Ground Truth__ data source: 238 | ```sh 239 | python test_locally.py -s simulator/goseek-v0.1.4.x86_64 -i submission -g 240 | ``` 241 | 242 | #### Submitting docker image 243 | 244 | As mentioned in [competition overview](README.md), we are using the [EvalAI](https://evalai.cloudcv.org/) platform to host our submission server. To upload a solution, please follow the instructions below: 245 | 246 | 247 | 1. Install [EvalAI-CLI](https://evalai-cli.cloudcv.org/): `pip install evalai`. 248 | 249 | 2. Create on account on EvalAI's [website](https://evalai.cloudcv.org/) and sign up for the [GOSEEK-Challenge](https://evalai.cloudcv.org/web/challenges/challenge-page/607/overview). 250 | 251 | 3. Follow the instructions on the [submission](https://evalai.cloudcv.org/web/challenges/challenge-page/607/submission) tab to push your docker image. 252 | Note that we've provided four phases -- some to support development. 253 | Only the leader of the **Competition Phase with Perception Pipeline** will be declared the competition winner. 254 | ## Examples 255 | 256 | 257 | See any of the following for additional information and examples. 258 | 259 | - [Baseline Proximal Policy Optimization (PPO)](doc/ppo-baseline.md) 260 | - [Additional problem details](doc/details.md) 261 | - [Instructions for running headless on a linux server](https://github.com/MIT-TESSE/tesse-core#running-tesse-headless) 262 | - [Generating configuration files](doc/generate.md) 263 | - [Details on the Perception Pipeline](doc/perception-pipeline.md) 264 | 265 | 266 | ## Disclaimer 267 | 268 | DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 269 | 270 | This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 271 | 272 | (c) 2020 Massachusetts Institute of Technology. 273 | 274 | MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 275 | 276 | The software/firmware is provided to you on an As-Is basis 277 | 278 | Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 279 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # goseek-challenge 2 | 3 | Welcome to the GOSEEK challenge page, which is run in conjunction with the [Perception, Action, Learning Workshop](https://mit-spark.github.io/PAL-ICRA2020/) at [ICRA 2020](https://www.icra2020.org/competitions/goseek-challenge). 4 | 5 | For this competition, participants create a reinforcement learning (RL) agent that combines perception and high-level decision-making to search for objects placed within complex indoor environments from a Unity-based simulator. 6 | Simply put: like PACMAN, but in a realistic scene and with realistic perception capabilities. 7 | Several data modalities are provided from both the simulator ground truth and a perception pipeline (e.g., images, depth, agent location) to enable the participants to focus on the RL/search aspects. 8 | The contest will be hosted on the EvalAI platform, where participants will submit solutions, via docker containers run on AWS instances, for scoring. 9 | 10 | __Outline__ 11 | 1. [Task Overview](#task-overview) 12 | 2. [Logistics](#logistics) 13 | 3. [Getting Started](#getting-started) 14 | 4. [Participation](#participation) 15 | 16 | ## Task Overview 17 | 18 | The objective of this challenge is to navigate an agent through an office environment to collect randomly-spawned fruit as quickly as possible. 19 | Our teaser trailer (below) highlights several of the components of the challenge, such as the office environment, the target fruit, the perception pipeline, and our idealized robot's physical characteristics. 20 | 21 | [![GOSEEK Teaser Trailer](https://img.youtube.com/vi/KXTag0xsg28/0.jpg)](https://www.youtube.com/watch?v=KXTag0xsg28) 22 | 23 | More specifically, the agent can select from one of four actions at each decision epoch: move forward 0.5 meters, turn left 8 degrees, turn right 8 degrees, and collect fruit within 2.0 meters of the agent's current position. 24 | Our robot is equipped with stereo cameras and an Inertial Measurement Unit (IMU), from which a state-of-the-art perception pipeline estimates three pieces of information that make up the agent's observation at each decision epoch: localization information (position and heading relative to start position), pixel-wise semantic labels for objects in the robot's field of view, and pixel-wise depth in the robot's field of view. 25 | 26 | ### Data Sources 27 | 28 | We provide two data sources for training: 29 | 30 | 1. __Ground Truth__: The agent observes ground truth (i.e., error free) information that is provided directly from the simulator. 31 | 2. __Perception Pipeline__: The agent observes output of [Kimera](http://web.mit.edu/sparklab/2019/10/13/Kimera__an_Open-Source_Library_for_Real-Time_Metric-Semantic_Localization_and_Mapping.html), which is an open-source C++ library for real-time metric-semantic visual-inertial Simultaneous Localization And Mapping (SLAM). 32 | Note that the types (and dimensions) of observations provided are the same as before; however, the error characteristics are now representative of a real perception system. 33 | 34 | Participants can use either or both of these sources for training their agents. 35 | Agent interfaces are identical between the two sources. 36 | We'll accept online submissions against either source (see [below](#online-submission) for details) and maintain a leaderboard for both. 37 | However, only evaluations against the __Perception Pipeline__ will be used to declare an overall competition winner. 38 | 39 | ### Evaluation 40 | 41 | Agents are evaluated on the following criteria for each episode: 42 | 43 | 1. `r`: recall of finding target fruit when the agents selects the collect action, 44 | 1. `p`: precision of finding target fruit when the agent selects the collect action, 45 | 1. `c`: number of collisions with objects in the scene, and 46 | 1. `a`: actions taken in the episode before all target fruit are collected or time expires. 47 | 48 | A single episode score is: 49 | ``` 50 | r + 0.1p - 0.1c/l - 0.1a/l 51 | ``` 52 | where `l` is the maximum episode length (400). Note that an episode terminates early if all fruit are collected. 53 | 54 | We use Monte Carlo evaluations to estimate an average episode score for the competition. 55 | Note that evaluations occur on withheld office scenes. 56 | 57 | ## Logistics 58 | 59 | ### Timeline 60 | 61 | The current timeline for the competition is as follows: 62 | 63 | - __Until Mid-April__: Competition software available for local testing and training by participants with __Ground Truth__ data source. 64 | - __Mid-April__: __Perception Pipeline__ data source provided to participants. Instructions for online submissions also made available. 65 | - __May 25__: Online submission period ends. 66 | 67 | ### Announcements 68 | 69 | Over the course of the competition, any important announcements or updates will be listed in this section. 70 | We recommend that you follow this repository to be alerted to these announcements. 71 | 72 | 1. We've posted version 0.1.3 of the challenge simulator. 73 | This provides better support for the __Perception Pipeline__ and addresses a minor bug. 74 | ~Please download this new simulator, if you were using 0.1.0 before. 75 | The link can be found in the [instructions](Instructions.md).~ 76 | *[See 4. below]* 77 | 2. We've also updated [tesse-gym](https://github.com/MIT-TESSE/tesse-gym) since our initial release to support the __Perception Pipeline__. 78 | You should update your clone of `tesse-gym` from the [instructions](Instructions.md). 79 | Please also rebuild the `goseek-base` docker image, as well as any of your submission images. 80 | 3. We continue to track the status of [ICRA 2020](https://icra2020.org/). 81 | We do not anticipate that any future statements from the planning committee will change the timeline of this challenge. 82 | 4. **[April 24]** Unfortunately, 0.1.3 of the challenge simulator was missing a shader for the strawberry in the RGB feed -- if you looked carefully it would appear gray. 83 | We believe that most people are not directly using RGB data, so hopefully this has not negatively impacted your experience. 84 | We've posted [0.1.4](https://github.com/MIT-TESSE/goseek-challenge/releases/download/0.1.0/goseek-v0.1.4.zip) of the simulator to correct this and updated all instructions. 85 | 5. **[April 24]** We've updated the [goseek-kimera Dockerfile](docker/goseek-kimera/Dockerfile) to incorporate some recent updates to Kimera. 86 | The changes improve stability. 87 | Please rebuild if you are using this. We've also updated [tesse-gym](https://github.com/MIT-TESSE/tesse-gym) to synchronize with the aforementioned changes. Please reclone to use the latest version. 88 | 6. **[May 19]** We've changed the date for final submissions from May 20th to May 25th. 89 | 90 | ## Getting Started 91 | 92 | Complete installation instructions can be found [here](Instructions.md), which lays out prerequisites, provides a link to download the competition simulator, and describes steps to install all required competition software. 93 | Users can also find an example for training an RL agent here, as well. 94 | 95 | ## Participation 96 | 97 | Participants will upload docker containers with their agents to EvalAI in order to be evaluated for the competition. 98 | The number of submissions is limited for each user, so we highly recommend performing local evaluations prior to submitting online solutions. 99 | This sections describes how to evaluate your agent locally, then submit online for a score. 100 | 101 | Before proceeding, we recommend that you have read through and completed [these instructions](Instructions.md). 102 | 103 | ### Prepare submission 104 | 105 | 1. Modify `Dockerfile` as appropriate for your agent. 106 | See [these instructions](Instructions.md#prepare-docker-submission) for modification details. 107 | The example we've provided runs an agent that randomly selects actions at each step. 108 | 109 | 2. Build the docker image. Here we are naming the image `submission`. 110 | ``` 111 | docker build -t submission . 112 | ``` 113 | 114 | ### Test locally 115 | 116 | Use `test_locally.py` for local testing. 117 | 118 | Assume you've named your docker image `submission` as above, then evaluate your agent with __Ground Truth__ data as follows. 119 | ```sh 120 | python test_locally.py -s simulator/goseek-v0.1.4.x86_64 -i submission -g 121 | ``` 122 | 123 | Similarly, evaluate your agent with __Perception Pipeline__ data as follows. 124 | ```sh 125 | python test_locally.py -s simulator/goseek-v0.1.4.x86_64 -i submission -p 126 | ``` 127 | 128 | ### Submit online 129 | 130 | 1. Install [EvalAI-CLI](https://evalai-cli.cloudcv.org/): `pip install evalai`. 131 | 132 | 2. Create on account on EvalAI's [website](https://evalai.cloudcv.org/) and sign up for the [GOSEEK-Challenge](https://evalai.cloudcv.org/web/challenges/challenge-page/607/overview). 133 | 3. Follow the instructions on the [submission](https://evalai.cloudcv.org/web/challenges/challenge-page/607/submission) tab to push your docker image. 134 | Note that we've provided four phases -- some to support development. 135 | Only the leader of the **Competition Phase with Perception Pipeline** will be declared the competition winner. 136 | 137 | ## Citing 138 | 139 | If you participate in GOSEEK and write a paper or a report about your entry, please cite: 140 | 141 | - D. Yadav, R. Jain, H. Agrawal, P. Chattopadhyay, T. Singh, A. Jain, S. B. Singh, S. Lee, D. Batra, “EvalAI: Towards Better Evaluation Systems for AI Agents”, arXiv:1902.03570, 2019. 142 | - A. Rosinol, M. Abate, Y. Chang, and L. Carlone. Kimera: an open-source library for real-time metric-semantic localization and mapping. In IEEE Intl. Conf. on Robotics and Automation (ICRA), 2020. 143 | 144 | 145 | ## Acknowledgements 146 | 147 | First, we would like to thank [Rishabh Jain](https://rishabhjain.xyz/) and the rest of the team at [EvalAI](https://evalai.cloudcv.org/) for providing their infrastructure and personal time to support this challenge. We must also acknowledge the team behind [The Habitat challenge](https://github.com/facebookresearch/habitat-challenge) for being pathfinders of RL challenges with online submissions. Their challenge and associated infrastructure was inspiration for many of our own decisions. 148 | 149 | ## Disclaimer 150 | 151 | DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 152 | 153 | This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 154 | 155 | (c) 2020 Massachusetts Institute of Technology. 156 | 157 | MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 158 | 159 | The software/firmware is provided to you on an As-Is basis 160 | 161 | Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 162 | -------------------------------------------------------------------------------- /baselines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/baselines/__init__.py -------------------------------------------------------------------------------- /baselines/agents.py: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 3 | # 4 | # This material is based upon work supported by the Under Secretary of Defense for Research and 5 | # Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions 6 | # or recommendations expressed in this material are those of the author(s) and do not necessarily 7 | # reflect the views of the Under Secretary of Defense for Research and Engineering. 8 | # 9 | # (c) 2020 Massachusetts Institute of Technology. 10 | # 11 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 12 | # 13 | # The software/firmware is provided to you on an As-Is basis 14 | # 15 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 16 | # or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work 17 | # are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other 18 | # than as specifically authorized by the U.S. Government may violate any copyrights that exist in 19 | # this work. 20 | ################################################################################################### 21 | 22 | from typing import Any, Dict 23 | 24 | import numpy as np 25 | 26 | from tesse_gym.eval.agent import Agent 27 | 28 | 29 | class StableBaselinesPPO(Agent): 30 | """ Stable Baselines PPO agent for GOSEEK submission. """ 31 | 32 | def __init__(self, config: Dict[str, Any]) -> None: 33 | """ Initialize agent. 34 | 35 | Args: 36 | config (Dict[str, Any]): Agent configuration. 37 | """ 38 | from stable_baselines import PPO2 39 | self.model = PPO2.load(config["weights"]) 40 | self.state = None 41 | 42 | # Number of environments used to train model 43 | # to which stable-baselines input tensor size is fixed 44 | self.n_train_envs = self.model.initial_state.shape[0] 45 | 46 | def act(self, observation: np.ndarray) -> int: 47 | """ Act on an observation. 48 | 49 | args: 50 | observation (np.ndarray): observation. 51 | 52 | returns: 53 | int: an action in [0, 4) defined as follows 54 | - 0: forward 0.5m 55 | - 1: right 8 degrees 56 | - 2: left 8 degrees 57 | - 3: declare target 58 | """ 59 | observation = np.repeat(observation[np.newaxis], self.n_train_envs, 0) 60 | actions, state = self.model.predict( 61 | observation, state=self.state, deterministic=False 62 | ) 63 | self.state = state # update model state 64 | return actions[0] 65 | 66 | def reset(self) -> None: 67 | """ Reset model state. """ 68 | self.state = None 69 | 70 | 71 | class RandomAgent(Agent): 72 | """ Agent that takes random actions. """ 73 | 74 | def __init__(self, config: Dict[str, Any]) -> None: 75 | """ Initialize agent. 76 | 77 | Args: 78 | config (Dict[str, Any]): Agent configuration 79 | """ 80 | 81 | self.action_space = np.arange(0, 4) 82 | 83 | # give probability for actions in `self.action_space` 84 | self.action_probability = np.array(config["action_probability"]) 85 | self.action_probability /= self.action_probability.sum() 86 | 87 | def act(self, observation: np.ndarray) -> int: 88 | """ Take a uniformly random action. 89 | 90 | args: 91 | observation (np.ndarray): observation. 92 | 93 | returns: 94 | int: an action in [0, 4) defined as follows 95 | - 0: forward 0.5m 96 | - 1: right 8 degrees 97 | - 2: left 8 degrees 98 | - 3: declare target 99 | """ 100 | return np.random.choice(self.action_space, p=self.action_probability) 101 | 102 | def reset(self) -> None: 103 | """ Nothing required on episode reset. """ 104 | pass 105 | -------------------------------------------------------------------------------- /baselines/config/ppo-agent.yaml: -------------------------------------------------------------------------------- 1 | name: StableBaselinesPPO 2 | weights: 'ppo-weights.pkl' # PUT WEIGHTS AT GOSEEK-CHALLENGE ROOT 3 | -------------------------------------------------------------------------------- /baselines/config/random-agent.yaml: -------------------------------------------------------------------------------- 1 | name: RandomAgent 2 | action_probability: [0.25, 0.25, 0.25, 0.25] 3 | -------------------------------------------------------------------------------- /config/check-ground-truth.yaml: -------------------------------------------------------------------------------- 1 | ENV: 2 | sim_path: 'simulator/goseek-v0.1.4.x86_64' 3 | 4 | EPISODE: 5 | scenes: [3, 5] 6 | n_targets: [30, 30] 7 | episode_length: [400, 400] 8 | random_seeds: [10, 100] 9 | -------------------------------------------------------------------------------- /config/ground-truth.yaml: -------------------------------------------------------------------------------- 1 | ENV: 2 | sim_path: '' 3 | 4 | EPISODE: 5 | scenes: [3, 5] 6 | n_targets: [30, 30] 7 | episode_length: [400, 400] 8 | random_seeds: [10, 100] 9 | -------------------------------------------------------------------------------- /config/perception-pipeline.yaml: -------------------------------------------------------------------------------- 1 | ENV: 2 | sim_path: '' 3 | ground_truth_mode: false 4 | metadata_port: 9007 5 | image_port: 9008 6 | 7 | EPISODE: 8 | scenes: [3, 4, 5, 5] 9 | n_targets: [30, 30, 30, 30] 10 | episode_length: [400, 400, 400, 400] 11 | random_seeds: [10, 1, 5, 6] 12 | -------------------------------------------------------------------------------- /doc/details.md: -------------------------------------------------------------------------------- 1 | ## Additional problem details 2 | 3 | ### Scenes 4 | 5 | The Unity simulator has 5 office-like scenes. 6 | 7 | ![scenes](images/goseek-scenes.jpg) 8 | 9 | Each scene consists of or two hallways with several rooms connected to the the hallway. 10 | Types of rooms include an office, cubical space, a restroom, a break room, and a storage room. 11 | 12 | ### Target Fruit 13 | 14 | The target fruit includes an apple, a banana, a strawberry, an orange, and a cherry. 15 | 16 | ![fruit](images/fruit.jpg) 17 | 18 | Fruit are randomly spawned throughout the environment. 19 | However, their distribution is non-uniform. 20 | Fruit are spawned in offices and cubical spaces. 21 | Sometimes they spawn in a hallway just outside of an office. 22 | They don't spawn in other rooms (e.g., restroom, breakroom). 23 | 24 | ### Semantics 25 | 26 | There are 11 semantic categories for objects as follows. 27 | - 0: floor (0,171,143) 28 | - 1: ceiling (1,155,211) 29 | - 2: wall (2,222,110) 30 | - 3: monitor (3,69,227) 31 | - 4: door (4,218,221) 32 | - 5: table (5,81,38) 33 | - 6: chair (6,229,176) 34 | - 7: storage (7,106,60) 35 | - 8: couch (8,179,10) 36 | - 9: clutter (9,118,90) 37 | - 10: target fruit (10,138,80) 38 | 39 | The values in parentheses correspond to (red-green-blue) RGB values when semantic segmentation is displayed in videos or directly obtained from the simulator as a 3-channel image (instead of a single channel). 40 | 41 | 42 | 43 | ## Disclaimer 44 | 45 | DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 46 | 47 | This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 48 | 49 | (c) 2020 Massachusetts Institute of Technology. 50 | 51 | MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 52 | 53 | The software/firmware is provided to you on an As-Is basis 54 | 55 | Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 56 | -------------------------------------------------------------------------------- /doc/generate.md: -------------------------------------------------------------------------------- 1 | # Generating configuration files 2 | 3 | The follow example generates a configuration file with 10 episodes such as those in [../config](../config). 4 | If you find this useful, please adapt for your own use. 5 | 6 | ```py 7 | import random 8 | import yaml 9 | 10 | k = 10 # number of episodes to generate 11 | l = 400 # episode length 12 | f = 30 # number of fruit 13 | scenes = [1, 2, 3, 4, 5] # scenes (1 through 5 are provided) 14 | 15 | data = {} 16 | data['ENV'] = {'sim_path': ''} 17 | 18 | data['EPISODE'] = {'scenes': random.choices(scenes, k=k), 19 | 'n_targets': [f for _ in range(k)], 20 | 'episode_length': [l for _ in range(k)], 21 | 'random_seeds': [random.randint(1, 1e6) for _ in range(k)] 22 | } 23 | 24 | with open('example.yaml', 'w') as outfile: 25 | yaml.dump(data, outfile) 26 | ``` -------------------------------------------------------------------------------- /doc/images/depth_estimate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/depth_estimate.png -------------------------------------------------------------------------------- /doc/images/depth_ground_truth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/depth_ground_truth.png -------------------------------------------------------------------------------- /doc/images/fruit.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/fruit.jpg -------------------------------------------------------------------------------- /doc/images/goseek-scenes.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/goseek-scenes.jpg -------------------------------------------------------------------------------- /doc/images/gt_est_pose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/gt_est_pose.png -------------------------------------------------------------------------------- /doc/images/gt_pose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/gt_pose.png -------------------------------------------------------------------------------- /doc/images/rgb_estimate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/rgb_estimate.png -------------------------------------------------------------------------------- /doc/images/rgb_ground_truth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/rgb_ground_truth.png -------------------------------------------------------------------------------- /doc/images/segmentation_estimate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/segmentation_estimate.png -------------------------------------------------------------------------------- /doc/images/segmentation_ground_truth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/segmentation_ground_truth.png -------------------------------------------------------------------------------- /doc/images/unity_coords.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/doc/images/unity_coords.png -------------------------------------------------------------------------------- /doc/perception-pipeline.md: -------------------------------------------------------------------------------- 1 | # Perception Pipeline 2 | 3 | As stated in the challenge overview, we provide two data sources: one is the simulator **Ground Truth** and the other is a realistic **Perception Pipeline**. 4 | 5 | 1. __Ground Truth__: The agent observes ground truth (i.e., error free) information that is provided directly from the simulator. 6 | 2. __Perception Pipeline__: The agent observes output of [Kimera](http://web.mit.edu/sparklab/2019/10/13/Kimera__an_Open-Source_Library_for_Real-Time_Metric-Semantic_Localization_and_Mapping.html), which is an open-source C++ library for real-time metric-semantic visual-inertial Simultaneous Localization And Mapping (SLAM). 7 | Note that the types (and dimensions) of observations provided are the same as before; however, the error characteristics are now representative of a real perception system. 8 | 9 | This page gives an overview of [Data Conventions](#data-conventions) and the [Perception Pipeline Algorithms](#perception-pipeline-algorithms). 10 | 11 | ## Data Conventions 12 | 13 | Th agent observes several data products: monocular RGB, semantic segmentation, depth, and agent pose. 14 | Conventions related to these data products are as follows. 15 | 16 | ### Semantic Segmentation 17 | 18 | See the [details page](https://github.mit.edu/TESS/goseek-challenge/blob/perception-docs/doc/details.md#semantics) for a list of the semantic segmentation classes and corresponding RGB values. Note, the class index corresponds to the value of the first color channel. 19 | 20 | ### Depth 21 | 22 | Depth is rendered within a specified range, known as [clip planes](https://docs.unity3d.com/Manual/class-Camera.html), and then mapped to the range `[0, 1]`. The simulator uses minimum and maximum clip planes of `0.05` and `50`, so to recover depth in meters multiply the provided image by `50`. Note, depth beyond 50 meters is truncated. In the [evaluation interface](https://github.mit.edu/TESS/goseek-challenge/blob/master/Instructions.md#local-evaluation) this would look like: 23 | 24 | ```python 25 | from tesse_gym.tasks.goseek import decode_observations 26 | 27 | class Agent: 28 | """ Interface for submitting an agent for evaluation. """ 29 | 30 | def act(self, observation: np.ndarray) -> int: 31 | far_clip_plane = 50 32 | rgb, segmentation, depth, pose = decode_observations(observation) 33 | depth *= far_clip_plane # convert depth to meters 34 | ... 35 | ``` 36 | 37 | 38 | ### Coordinate System for Poses 39 | 40 | We use the left handed coordinate system native to Unity (see [Unity documentation](https://docs.unity3d.com/560/Documentation/Manual/Transforms.html)). For world coordinates, x- and z-axes are aligned with the horizontal plane, and the y-axis is aligned with up. Pose is given as the vector `(x, z, yaw)` where the z-axis is aligned with forward, the x-axis is positive to the right, and yaw is relative to the positive up y-axis. 41 | 42 | ![](images/unity_coords.png) 43 | 44 | 45 | 46 | ## Perception Pipeline Algorithms 47 | 48 | During evaluation, participants will be provided with realistic perception data from [Kimera](https://github.com/MIT-SPARK/Kimera), an open-source C++ library for real-time metric-semantic visual-inertial Simultaneous Localization And Mapping (SLAM). Realistic perception estimates are obtained by passing ground truth simulator data through this pipeline. Thus, the types and dimensions of observations will remain the same; however, the error characteristics are now representative of a real perception system. 49 | 50 | Please note that running the perception pipeline requires significantly more computation than groundtruth. 51 | Episodes will run **several times slower** than when running the groundtruth data pipeline. 52 | The simulator is run in a continuous dynamics mode (compared to discrete dynamics when running the groundtruth data pipeline), and it is outputting imagery at a higher rate. 53 | This higher-rate data is needed for estimating [pose](#pose) only, so agent policies will still receive data at the same rate as before. 54 | In addition, several perception algorithms are now running as described below. 55 | 56 | We recommend thinking carefully about how you use this pipeline. 57 | It may be less feasible to generate data with it for policy training, for example. 58 | 59 | ### Segmentation 60 | 61 | A [U-Net](https://arxiv.org/pdf/1505.04597.pdf) provides segmentation estimates for the 11 GOSEEK [semantic classes](https://github.com/MIT-TESSE/goseek-challenge/blob/master/doc/details.md#semantics). The [segmentation-models.pytorch](https://github.com/qubvel/segmentation_models.pytorch) project was used to train the model on data collected from [scenes 1-4](https://github.com/MIT-TESSE/goseek-challenge/blob/master/doc/details.md#scenes) of the GOSEEK simulator. Scene 5 was used to collect a validation set on which the model achieves an Intersection-over-Union (IoU) score of roughly 0.8. The model was then exported to an ONNX file, provided in this [release](https://github.com/MIT-TESSE/tesse-segmentation-ros/releases), for inference in [TensorRT](https://developer.nvidia.com/tensorrt). 62 | 63 | Training details can be found in this [notebook](https://github.com/MIT-TESSE/tesse-segmentation-ros/blob/master/training/train-segmentation-models.ipynb). The inference framework is implemented [here](https://github.com/MIT-TESSE/tesse-segmentation-ros/blob/master/src/tesse_segmentation_ros/models.py#L57). 64 | 65 | ### Depth 66 | Depth is estimated via stereo reconstruction. Using a stereo image pair from the simulator, we use the ROS [`stereo_image_proc`](http://wiki.ros.org/stereo_image_proc) node to generate a point cloud which is then projected into the camera plane to produce a depth image. 67 | 68 | 69 | ### Pose 70 | Pose is provided by [Kimera-VIO](https://github.com/MIT-SPARK/Kimera-VIO), a Visual Inertial Odometry pipeline for State Estimation from Stereo and IMU data. 71 | 72 | Below is a comparison of the data provided in Ground Truth and Perception modes. Pose is illustrated over a 100 step trajectory with the carrots representing position and heading every 1 step. 73 | 74 | 75 | | | Ground Truth | Perception | 76 | :----------------------:|:------------------------------------------:|:-------------------------: 77 | | Monocular RGB | ![](images/rgb_ground_truth.png) | ![](images/rgb_estimate.png) 78 | | Semantic Segmentation | ![](images/segmentation_ground_truth.png) | ![](images/segmentation_estimate.png) 79 | | Depth | ![](images/depth_ground_truth.png) | ![](images/depth_estimate.png) 80 | | Pose | ![](images/gt_pose.png) | ![](images/gt_est_pose.png) 81 | 82 | 83 | ## Disclaimer 84 | 85 | DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 86 | 87 | This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 88 | 89 | (c) 2020 Massachusetts Institute of Technology. 90 | 91 | MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 92 | 93 | The software/firmware is provided to you on an As-Is basis 94 | 95 | Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 96 | 97 | 98 | -------------------------------------------------------------------------------- /doc/ppo-baseline.md: -------------------------------------------------------------------------------- 1 | ## Baseline Proximal Policy Optimization (PPO) 2 | 3 | ### Installation 4 | 5 | To run the example PPO baseline, install [Stable Baselines](https://stable-baselines.readthedocs.io/en/master/) and a version of [Tensorflow](https://www.tensorflow.org/) between v0.8.0 and v1.14.0 (see the [Stable Baselines docs](https://stable-baselines.readthedocs.io/en/master/guide/install.html#prerequisites) for details). 6 | 7 | __Note__: Currently, we've tested Python 3.7 Tensorflow installation with Anaconda against Cuda 10.0-10.2 (run `nvcc -V` to check your Cuda version). 8 | 9 | For Cuda 10.0, we'd recommend installing `tensorflow-gpu v1.13.1`: 10 | 11 | ```sh 12 | conda activate goseek 13 | conda install tensorflow-gpu==1.13.1 14 | ``` 15 | 16 | For Cuda 10.1 and 10.2, we'd recommend installing `tensorflow-gpu v1.14`: 17 | 18 | ```sh 19 | conda activate goseek 20 | conda install tensorflow-gpu==1.14 21 | ``` 22 | 23 | Then, install [Stable Baselines](https://stable-baselines.readthedocs.io/en/master/) 24 | 25 | ```sh 26 | conda activate goseek 27 | pip install stable-baselines==2.9.0 28 | ``` 29 | 30 | ### Training 31 | 32 | See `tesse-gym/baselines/goseek-ppo.ipynb` to train a PPO agent for the GOSEEK challenge. The notebook details how to: 33 | 34 | * Configure a `tesse-gym` environment 35 | * Define a policy 36 | * Train a model 37 | * Visualize results 38 | 39 | ### Local Evaluation 40 | 41 | Once trained, you can evaluate your model with the same pipeline used for the random agent above. Simply update `goseek-challenge/baselines/config/baseline-ppo.yaml` with the path to the trained weights for your agent, this will be loaded by the `StableBaselinesPPO` agent defined in `baselines/agents.py`. Evaluate by running 42 | 43 | ```sh 44 | python eval.py --agent-config baselines/config/ppo-agent.yaml 45 | ``` 46 | 47 | 48 | ## Disclaimer 49 | 50 | DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 51 | 52 | This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 53 | 54 | (c) 2020 Massachusetts Institute of Technology. 55 | 56 | MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 57 | 58 | The software/firmware is provided to you on an As-Is basis 59 | 60 | Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 61 | -------------------------------------------------------------------------------- /docker/.gitignore: -------------------------------------------------------------------------------- 1 | *.tar.gz 2 | gym-ros-interface/ 3 | semantic-segmentation-ros/ 4 | tesse-gym/ 5 | tesse-interface/ 6 | goseek-challenge/ 7 | work/ 8 | stable-baselines-ppo-*.pkl 9 | -------------------------------------------------------------------------------- /docker/goseek-base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04 2 | 3 | # docker build -t goseek-base . 4 | # docker run --network="host" --gpus all --rm -it goseek-base /bin/bash 5 | 6 | RUN apt-get clean && apt-get update && apt-get install -y locales 7 | RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ 8 | locale-gen 9 | ENV LC_ALL en_US.UTF-8 10 | ENV LANG en_US.UTF-8 11 | ENV LANGUAGE en_US.UTF-8 12 | ENV SHELL /bin/bash 13 | ENV DEBIAN_FRONTEND noninteractive 14 | 15 | # switch to bash within the container so ROS sourcing is easy in build commands 16 | SHELL ["/bin/bash", "-c"] 17 | 18 | RUN apt-get update && \ 19 | apt-get install -y git curl && \ 20 | apt-get clean 21 | 22 | RUN curl -LO http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 23 | bash Miniconda3-latest-Linux-x86_64.sh -p /miniconda -b && \ 24 | rm Miniconda3-latest-Linux-x86_64.sh 25 | 26 | ENV PATH /miniconda/bin:$PATH 27 | 28 | RUN conda update -y conda && \ 29 | conda install -y -f python=3.7 numpy pip scipy pyyaml pillow 30 | 31 | RUN pip install -e git+https://github.com/MIT-TESSE/tesse-gym.git@master#egg=tesse-gym 32 | 33 | RUN git clone https://github.com/MIT-TESSE/goseek-challenge.git /goseek-challenge 34 | WORKDIR /goseek-challenge 35 | 36 | ######################################################################################### 37 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 38 | 39 | # This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 40 | 41 | # © 2019 Massachusetts Institute of Technology. 42 | 43 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 44 | 45 | # The software/firmware is provided to you on an As-Is basis 46 | 47 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 48 | -------------------------------------------------------------------------------- /docker/goseek-kimera/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 2 | # using devel to support installing pycuda 3 | # if you are behind a proxy: https://docs.docker.com/network/proxy/ 4 | 5 | RUN apt-get clean && apt-get update && apt-get install -y locales 6 | RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ 7 | locale-gen 8 | ENV LC_ALL en_US.UTF-8 9 | ENV LANG en_US.UTF-8 10 | ENV LANGUAGE en_US.UTF-8 11 | ENV SHELL /bin/bash 12 | ENV DEBIAN_FRONTEND noninteractive 13 | 14 | # switch to bash within the container so ROS sourcing is easy in build commands 15 | SHELL ["/bin/bash", "-c"] 16 | 17 | RUN apt-get update && \ 18 | apt-get install -y curl bzip2 wget vim ffmpeg git tmux unzip 19 | 20 | # tesse: opencv dependency workaround from https://github.com/NVIDIA/nvidia-docker/issues/864 21 | RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev 22 | 23 | # kimera: 24 | RUN apt-get update && \ 25 | apt-get install -y --no-install-recommends apt-utils 26 | 27 | ################################ 28 | ### ROS ### 29 | ################################ 30 | 31 | #Set the ROS distro 32 | ENV ROS_DISTRO melodic 33 | 34 | # Add the ROS keys and package 35 | RUN apt-get update && \ 36 | apt-get install -y \ 37 | lsb-release \ 38 | gnupg 39 | RUN sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' 40 | RUN mkdir ~/.gnupg 41 | RUN echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf 42 | 43 | RUN curl -sSL 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xC1CF6E31E6BADE8868B172B4F42ED6FBAB17C654' | apt-key add - 44 | 45 | # Install ROS 46 | RUN apt-get update && \ 47 | apt-get install -y ros-melodic-desktop-full 48 | 49 | RUN apt-get install -y \ 50 | python \ 51 | python-pip \ 52 | python-dev 53 | 54 | RUN apt-get install python-rosdep 55 | 56 | # Set up ROS 57 | RUN rosdep init 58 | RUN rosdep update 59 | 60 | RUN source /opt/ros/melodic/setup.bash && \ 61 | apt install -y \ 62 | python-rosinstall \ 63 | python-rosinstall-generator \ 64 | python-wstool build-essential \ 65 | python-catkin-tools 66 | 67 | # Setup catkin workspace 68 | RUN source /opt/ros/melodic/setup.bash && \ 69 | mkdir -p /catkin_ws/src && \ 70 | cd /catkin_ws/ && \ 71 | catkin init && \ 72 | # Change `melodic` to your ROS distro 73 | catkin config --extend /opt/ros/melodic && \ 74 | catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release && \ 75 | catkin config --merge-devel 76 | 77 | 78 | ################################ 79 | ### ** TESSE ** ### 80 | ### ------------ ### 81 | ### TESSE ROS packages ### 82 | ### TESSE Anaconda packages ### 83 | ################################ 84 | 85 | WORKDIR /catkin_ws/src 86 | 87 | # tesse-ros-bridge dependency 88 | RUN pip install scipy 89 | 90 | RUN git clone https://github.com/MIT-TESSE/tesse-interface.git -b 0.1.3-SNAPSHOT && \ 91 | cd tesse-interface && \ 92 | python setup.py install && \ 93 | cd .. 94 | 95 | WORKDIR /catkin_ws/src 96 | RUN source /opt/ros/melodic/setup.bash 97 | RUN wstool init 98 | 99 | RUN git clone https://github.com/MIT-TESSE/tesse-ros-bridge.git && \ 100 | wstool merge -y tesse-ros-bridge/install/tesse_ros_bridge.rosinstall 101 | 102 | RUN git clone https://github.com/MIT-TESSE/tesse-segmentation-ros.git && \ 103 | wstool merge -y tesse-segmentation-ros/install/tesse_segmentation_ros.rosinstall 104 | 105 | RUN git clone https://github.com/MIT-TESSE/tesse-gym-bridge.git && \ 106 | wstool merge -y tesse-gym-bridge/install/tesse_gym_bridge.rosinstall 107 | 108 | RUN wstool update 109 | RUN catkin build tesse_ros_bridge tesse_segmentation_ros tesse_gym_bridge 110 | 111 | 112 | ################################ 113 | ### **MIT-SPARK KIMERA** ### 114 | ### ---------------- ### 115 | ### KIMERA-VIO-ROS ### 116 | ### KIMERA-Semantics ### 117 | ################################ 118 | 119 | # pinned commits: 120 | # https://github.com/MIT-SPARK/Kimera-VIO-ROS @ f6181a1370677ffe8c24253554efebaf787f428e 121 | # https://github.com/MIT-SPARK/Kimera-VIO @ 42ad9ca557656ef3463b8cc3b841fe5a4a36398c 122 | # https://github.com/MIT-SPARK/Kimera-Semantics @ 0ae759f24cce9e9ffc28d0495327a3d2b37d2c99 123 | 124 | # Kimera dependencies 125 | RUN apt-get update && \ 126 | apt-get install -y \ 127 | libgoogle-glog-dev \ 128 | doxygen \ 129 | cmake build-essential pkg-config autoconf \ 130 | libboost-all-dev \ 131 | libjpeg-dev libpng-dev libtiff-dev \ 132 | libvtk6-dev libgtk-3-dev \ 133 | libatlas-base-dev gfortran \ 134 | libparmetis-dev \ 135 | libtbb-dev 136 | 137 | # suppress build warnings 138 | ENV PYTHONIOENCODING=UTF-8 139 | 140 | WORKDIR /catkin_ws/src 141 | 142 | RUN git clone https://github.com/MIT-SPARK/Kimera-VIO-ROS.git && \ 143 | cd Kimera-VIO-ROS && \ 144 | git checkout db6afe795f3bf6b88842d2c45cb2306926505051 && \ 145 | cd .. && \ 146 | wstool merge -y Kimera-VIO-ROS/install/kimera_vio_ros_https.rosinstall && \ 147 | wstool update 148 | 149 | RUN git clone https://github.com/MIT-SPARK/Kimera-Semantics.git && \ 150 | cd Kimera-Semantics && \ 151 | git checkout 0ae759f24cce9e9ffc28d0495327a3d2b37d2c99 && \ 152 | cd .. && \ 153 | wstool merge -y Kimera-Semantics/kimera/install/kimera_semantics_https.rosinstall && \ 154 | wstool update 155 | 156 | # checkout pinned commit of Kimera-VIO-ROS and Kimera-VIO 157 | RUN cd Kimera-VIO-ROS && \ 158 | git checkout db6afe795f3bf6b88842d2c45cb2306926505051 && \ 159 | cd .. && \ 160 | cd Kimera-VIO && \ 161 | git fetch && git checkout 13a52d8772305be143ebdf470c8714759e411f59 && \ 162 | cd .. 163 | 164 | RUN cd gtsam && \ 165 | git fetch && \ 166 | git checkout develop && \ 167 | git pull 168 | 169 | RUN catkin build gtsam opengv_catkin opencv3_catkin kimera_rpgo dbow2_catkin 170 | 171 | RUN catkin build 172 | 173 | ################################ 174 | ### Configs ### 175 | ################################ 176 | 177 | RUN wget https://github.com/MIT-TESSE/goseek-challenge/releases/download/evalai-beta-testing/goseek-unet-a1.onnx \ 178 | -P /catkin_ws/src/tesse-segmentation-ros/cfg/ && \ 179 | wget https://github.com/MIT-TESSE/goseek-challenge/releases/download/evalai-beta-testing/kimera-configs.zip && \ 180 | unzip kimera-configs.zip && \ 181 | mv kimera-configs/* /catkin_ws/src/Kimera-VIO/params/Tesse/ && \ 182 | rm kimera-configs.zip 183 | 184 | RUN source /catkin_ws/devel/setup.bash 185 | 186 | WORKDIR / 187 | 188 | RUN echo "source /catkin_ws/devel/setup.bash" >> run_perception.sh && \ 189 | echo "roslaunch tesse_gym_bridge run_goseek_perception.launch" >> run_perception.sh && \ 190 | chmod +x /run_perception.sh 191 | 192 | 193 | ################################ 194 | ### TensorRT ### 195 | ################################ 196 | 197 | ADD TensorRT-6.0.1.5.Ubuntu-18.04.x86_64-gnu.cuda-10.0.cudnn7.6.tar.gz / 198 | RUN mv /TensorRT-6.0.1.5 /tensorrt 199 | 200 | ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/tensorrt/lib 201 | 202 | # install tensorrt wheel into the python environment used with ROS 203 | RUN source /opt/ros/melodic/setup.bash && \ 204 | source /catkin_ws/devel/setup.bash && \ 205 | cd /tensorrt/python/ && \ 206 | pip install tensorrt-6.0.1.5-cp27-none-linux_x86_64.whl && \ 207 | pip install pycuda 208 | 209 | 210 | ######################################################################################### 211 | # Disclaimer: 212 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 213 | 214 | # This material is based upon work supported by the Under Secretary of Defense for Research and Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Under Secretary of Defense for Research and Engineering. 215 | 216 | # © 2019 Massachusetts Institute of Technology. 217 | 218 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 219 | 220 | # The software/firmware is provided to you on an As-Is basis 221 | 222 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than as specifically authorized by the U.S. Government may violate any copyrights that exist in this work. 223 | -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 3 | # 4 | # This material is based upon work supported by the Under Secretary of Defense for Research and 5 | # Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions 6 | # or recommendations expressed in this material are those of the author(s) and do not necessarily 7 | # reflect the views of the Under Secretary of Defense for Research and Engineering. 8 | # 9 | # (c) 2020 Massachusetts Institute of Technology. 10 | # 11 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 12 | # 13 | # The software/firmware is provided to you on an As-Is basis 14 | # 15 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 16 | # or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work 17 | # are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other 18 | # than as specifically authorized by the U.S. Government may violate any copyrights that exist in 19 | # this work. 20 | ################################################################################################### 21 | 22 | import argparse 23 | import pprint 24 | from typing import Dict 25 | 26 | from yacs.config import CfgNode 27 | import yaml 28 | 29 | from baselines.agents import * # specific agent is given in agent-config 30 | from tesse_gym import NetworkConfig 31 | from tesse_gym.eval.agent import Agent 32 | from tesse_gym.tasks.goseek import GoSeekBenchmark, get_goseek_cfg_defaults 33 | from tesse_gym.eval.utils import get_agent_cls 34 | 35 | 36 | def main( 37 | episode_args: CfgNode, agent_args: Dict[str, Any] 38 | ) -> Dict[str, Dict[str, float]]: 39 | """ Run GOSEEK evaluation over the specified environment and 40 | agent configurations. 41 | 42 | Args: 43 | episode_args (CfgNode): Environment configurations. 44 | agent_args (Dict[str, Any]): Agent configurations. 45 | 46 | Returns: 47 | Dict[str, Dict[str, float]]: Dictionary containing overall evaluation performance as 48 | well as a summary for each episode. 49 | """ 50 | benchmark = GoSeekBenchmark( 51 | build_path=episode_args.ENV.sim_path, 52 | scenes=episode_args.EPISODE.scenes, 53 | episode_length=episode_args.EPISODE.episode_length, 54 | n_targets=episode_args.EPISODE.n_targets, 55 | success_dist=episode_args.EPISODE.success_dist, 56 | random_seeds=episode_args.EPISODE.random_seeds, 57 | network_config=NetworkConfig( 58 | position_port=episode_args.ENV.position_port, 59 | image_port=episode_args.ENV.image_port, 60 | metadata_port=episode_args.ENV.metadata_port, 61 | step_port=episode_args.ENV.step_port 62 | ), 63 | ground_truth_mode=episode_args.ENV.ground_truth_mode, 64 | ) 65 | agent = get_agent_cls(agent_args["name"], Agent)(agent_args) 66 | return benchmark.evaluate(agent) 67 | 68 | 69 | if __name__ == "__main__": 70 | episode_cfg = get_goseek_cfg_defaults() 71 | 72 | parser = argparse.ArgumentParser() 73 | parser.add_argument("--episode-config", type=str) 74 | parser.add_argument("--agent-config", type=str) 75 | args = parser.parse_args() 76 | 77 | with open(args.agent_config) as f: 78 | agent_args = yaml.load(f, Loader=yaml.FullLoader) 79 | 80 | if args.episode_config: 81 | episode_cfg.merge_from_file(args.episode_config) 82 | 83 | episode_cfg.freeze() 84 | 85 | results = main(episode_cfg, agent_args) 86 | 87 | print("------ Environment Configuration -----") 88 | pprint.pprint(episode_cfg, depth=5) 89 | 90 | print("\n----- Agent Configuration -----") 91 | pprint.pprint(agent_args, depth=5) 92 | 93 | print("\n----- Per Episode Score -----") 94 | pprint.pprint(results, depth=5) 95 | -------------------------------------------------------------------------------- /logs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MIT-TESSE/goseek-challenge/f1c49c378990cafd1a14159be081a2ed00a1b7bd/logs/.gitkeep -------------------------------------------------------------------------------- /test_locally.py: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # DISTRIBUTION STATEMENT A. Approved for public release. Distribution is unlimited. 3 | # 4 | # This material is based upon work supported by the Under Secretary of Defense for Research and 5 | # Engineering under Air Force Contract No. FA8702-15-D-0001. Any opinions, findings, conclusions 6 | # or recommendations expressed in this material are those of the author(s) and do not necessarily 7 | # reflect the views of the Under Secretary of Defense for Research and Engineering. 8 | # 9 | # (c) 2020 Massachusetts Institute of Technology. 10 | # 11 | # MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014) 12 | # 13 | # The software/firmware is provided to you on an As-Is basis 14 | # 15 | # Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 16 | # or 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work 17 | # are defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other 18 | # than as specifically authorized by the U.S. Government may violate any copyrights that exist in 19 | # this work. 20 | ################################################################################################### 21 | 22 | import argparse 23 | import ast 24 | import hashlib 25 | import os 26 | import pprint 27 | import subprocess 28 | import time 29 | from datetime import datetime 30 | 31 | RECALL_WEIGHT = 1 32 | PRECISION_WEIGHT = 0.1 33 | COLLISIONS_WEIGHT = 0.1 34 | ACTIONS_WEIGHT = 0.1 35 | EPISODE_LENGTH = 400 36 | 37 | DOCKER_VIO_LOGDIR = "/catkin_ws/src/Kimera-VIO-ROS/output_logs" 38 | HOST_VIO_LOGDIR = "kimera_vio_logs" 39 | 40 | 41 | def _time_hash(prefix): 42 | """ Get has based on current time. """ 43 | hash = hashlib.sha1() 44 | hash.update(str(time.time()).encode()) 45 | return f"{prefix}_{hash.hexdigest()}" 46 | 47 | 48 | def summarize(results): 49 | n_episodes = len(results) - 1 50 | 51 | metrics = {} 52 | metrics["Recall"] = results["total"]["recall"] 53 | metrics["Precision"] = results["total"]["precision"] 54 | metrics["Collisions"] = ( 55 | results["total"]["collisions"] / n_episodes 56 | ) # results has total collisions 57 | metrics["Actions"] = ( 58 | results["total"]["steps"] / n_episodes 59 | ) # results has total steps 60 | 61 | metrics["Weighted Total"] = ( 62 | RECALL_WEIGHT * metrics["Recall"] 63 | + PRECISION_WEIGHT * metrics["Precision"] 64 | - COLLISIONS_WEIGHT * metrics["Collisions"] / EPISODE_LENGTH 65 | - ACTIONS_WEIGHT * metrics["Actions"] / EPISODE_LENGTH 66 | ) 67 | return metrics 68 | 69 | 70 | def start_perception_subprocess(image_name): 71 | """ Start docker container with perception server as subprocess. """ 72 | container_name = _time_hash("goseek_perception") 73 | return ( 74 | subprocess.Popen( 75 | [ 76 | "docker", 77 | "run", 78 | "--rm", 79 | "--name", 80 | f"{container_name}", 81 | "--network=host", 82 | "--gpus", 83 | "all", 84 | "-t", 85 | f"{image_name}", 86 | "/bin/bash", 87 | "-c", 88 | "source /catkin_ws/devel/setup.bash && " 89 | "roslaunch tesse_gym_bridge run_goseek_perception.launch", 90 | ], 91 | stdout=subprocess.DEVNULL, 92 | stderr=subprocess.DEVNULL, 93 | ), 94 | container_name, 95 | ) 96 | 97 | 98 | def start_submission_subprocess(config_dir, submission_image, episode_config_file): 99 | submission_name = _time_hash("submission") 100 | return ( 101 | subprocess.Popen( 102 | [ 103 | "docker", 104 | "run", 105 | "--name", 106 | f"{submission_name}", 107 | "--network=host", 108 | "--gpus", 109 | "all", 110 | "-v", 111 | # note to goseek-challenge participants: 112 | # we will use an internal config for our challenge evaluations 113 | "{}:/config".format(config_dir), 114 | "-t", 115 | "{}".format(submission_image), 116 | "/bin/bash", 117 | "-c", 118 | "python eval.py --episode-config /config/{} --agent-config agent.yaml".format( 119 | episode_config_file 120 | ), 121 | ], 122 | ), 123 | submission_name, 124 | ) 125 | 126 | 127 | def kill_docker_container(container_name): 128 | """ Kill docker container by name. """ 129 | subprocess.call( 130 | ["docker", "rm", "--force", f"{container_name}"], 131 | stdout=subprocess.PIPE, 132 | stderr=subprocess.PIPE, 133 | ) 134 | 135 | 136 | def get_docker_logs(container_name): 137 | """ Get logs of docker container. """ 138 | p = subprocess.run( 139 | ["docker", "logs", container_name], 140 | stdout=subprocess.PIPE, 141 | stderr=subprocess.PIPE, 142 | ) 143 | return p.stdout.decode(), "" # TODO parse output to stdout and stderr 144 | 145 | 146 | def docker_cp(container_name, container_dir, host_dir): 147 | subprocess.run(["docker", "cp", f"{container_name}:{container_dir}", host_dir]) 148 | 149 | 150 | def write_logs(fname, data): 151 | with open(fname, "w") as f: 152 | f.write(data) 153 | 154 | 155 | def extract_metrics(process, submission_stdout): 156 | if process.returncode == 0: 157 | results = submission_stdout.split("----- Per Episode Score -----")[-1] 158 | metrics = summarize(ast.literal_eval(results)) 159 | else: # Agent failure returns -1 weighted score 160 | metrics = { 161 | "Recall": 0, 162 | "Precision": 0, 163 | "Collisions": 0, 164 | "Actions": 0, 165 | "Weighted Total": -1, 166 | } 167 | return metrics 168 | 169 | 170 | def log_perception_data(container_name, logdir): 171 | perception_stdout, perception_stderr = get_docker_logs(container_name) 172 | docker_cp( 173 | container_name, DOCKER_VIO_LOGDIR, f"{logdir}/{HOST_VIO_LOGDIR}", 174 | ) 175 | 176 | write_logs(f"{logdir}/perception-stdout.txt", perception_stdout) 177 | write_logs(f"{logdir}/perception-stderr.txt", perception_stderr) 178 | 179 | 180 | def main( 181 | simulator: str, # path to simulator 182 | kimera: bool, # specify whether to run kimera docker 183 | image: str, # path to docker image 184 | config: str, # path to configuration, 185 | logdir: str = None, # optional log dir 186 | ): 187 | 188 | try: 189 | if logdir is None: 190 | logdir = f"logs/{datetime.now().strftime('%m-%d-%Y_%H-%M-%s')}" 191 | if not os.path.exists(logdir): 192 | os.makedirs(logdir) 193 | 194 | # run simulator 195 | sim = subprocess.Popen([simulator, "--set_resolution", "320", "240"]) 196 | 197 | # run perception server 198 | if kimera: 199 | perception, perception_container_name = start_perception_subprocess( 200 | "goseek-kimera" 201 | ) 202 | time.sleep(20) 203 | 204 | time.sleep(10) # wait for the simulator to open 205 | 206 | print("Running agent...") 207 | 208 | # Split up configuration 209 | dirname, filename = os.path.split(os.path.abspath(config)) 210 | p, submission_name = start_submission_subprocess(dirname, image, filename) 211 | p.communicate() # wait for process to stop 212 | 213 | except Exception as ex: 214 | print(f"CAUGHT EXCEPTION: {ex}") 215 | 216 | except KeyboardInterrupt: 217 | print(f"Caught keyboard interrupt, exiting...") 218 | 219 | finally: 220 | # cleanup 221 | sim.terminate() 222 | print("stopped simulator...") 223 | 224 | stdoutdata, stderrdata = get_docker_logs(submission_name) 225 | metrics = extract_metrics(p, stdoutdata) 226 | write_logs(f"{logdir}/submission-stdout.txt", stdoutdata) 227 | 228 | kill_docker_container(submission_name) 229 | print("stopped submission...") 230 | 231 | if kimera: 232 | log_perception_data(perception_container_name, logdir) 233 | kill_docker_container(perception_container_name) 234 | perception.terminate() 235 | print("stopped perception pipeline...") 236 | 237 | return stdoutdata, stderrdata, metrics, p.returncode 238 | 239 | 240 | if __name__ == "__main__": 241 | parser = argparse.ArgumentParser() 242 | parser.add_argument( 243 | "-s", "--simulator", help="Path to the simulator binary", required=True 244 | ) 245 | parser.add_argument( 246 | "-i", 247 | "--image", 248 | help="Name of the docker image to use for local evaluation", 249 | required=True, 250 | ) 251 | perception_mode_flag = parser.add_mutually_exclusive_group(required=True) 252 | perception_mode_flag.add_argument( 253 | "-g", "--groundtruth", help="Use groundtruth observations", action="store_true" 254 | ) 255 | perception_mode_flag.add_argument( 256 | "-p", 257 | "--perception", 258 | help="Use realistic perception for observations", 259 | action="store_true", 260 | ) 261 | args = parser.parse_args() 262 | 263 | if args.perception: 264 | episode_config = "{}/config/perception-pipeline.yaml".format(os.getcwd()) 265 | kimera = True 266 | else: # ground truth mode 267 | episode_config = "{}/config/ground-truth.yaml".format(os.getcwd()) 268 | kimera = False 269 | 270 | stdoutdata, stderrdata, metrics, returncode = main( 271 | args.simulator, kimera, args.image, episode_config 272 | ) 273 | 274 | print("\n***** Summary Metrics *****") 275 | pprint.pprint(metrics) 276 | --------------------------------------------------------------------------------