├── LICENSE ├── README.md ├── configs └── tasks │ └── multinav_mp3d.yaml ├── habitat ├── __init__.py ├── config │ ├── __init__.py │ └── default.py ├── core │ ├── __init__.py │ ├── agent.py │ ├── benchmark.py │ ├── challenge.py │ ├── dataset.py │ ├── embodied_task.py │ ├── env.py │ ├── logging.py │ ├── registry.py │ ├── simulator.py │ ├── spaces.py │ ├── utils.py │ └── vector_env.py ├── datasets │ ├── __init__.py │ ├── multi_nav │ │ ├── __init__.py │ │ └── multi_nav_dataset.py │ ├── object_nav │ │ ├── __init__.py │ │ └── object_nav_dataset.py │ ├── pointnav │ │ ├── __init__.py │ │ ├── pointnav_dataset.py │ │ └── pointnav_generator.py │ ├── registration.py │ └── utils.py ├── py.typed ├── sims │ ├── __init__.py │ ├── habitat_simulator │ │ ├── __init__.py │ │ ├── actions.py │ │ └── habitat_simulator.py │ ├── pyrobot │ │ ├── __init__.py │ │ └── pyrobot.py │ └── registration.py ├── tasks │ ├── __init__.py │ ├── nav │ │ ├── __init__.py │ │ ├── multi_nav_task.py │ │ ├── nav.py │ │ ├── object_nav_task.py │ │ └── shortest_path_follower.py │ ├── registration.py │ └── utils.py ├── utils │ ├── __init__.py │ ├── geometry_utils.py │ ├── test_utils.py │ └── visualizations │ │ ├── __init__.py │ │ ├── assets │ │ └── maps_topdown_agent_sprite │ │ │ └── 100x100.png │ │ ├── fog_of_war.py │ │ ├── maps.py │ │ └── utils.py └── version.py ├── habitat_baselines ├── __init__.py ├── common │ ├── base_trainer.py │ ├── baseline_registry.py │ ├── env_utils.py │ ├── environments.py │ ├── rollout_storage.py │ ├── tensorboard_utils.py │ └── utils.py ├── config │ ├── __init__.py │ ├── default.py │ └── multinav │ │ └── ppo_multinav.yaml ├── py.typed ├── rl │ ├── __init__.py │ ├── models │ │ ├── __init__.py │ │ ├── implicit_projection.py │ │ ├── implicit_representation.py │ │ ├── implicit_representation_exploration_occupancy.py │ │ ├── projection.py │ │ ├── rnn_state_encoder.py │ │ └── simple_cnn.py │ └── ppo │ │ ├── __init__.py │ │ ├── aux_losses_utils.py │ │ ├── policy.py │ │ ├── ppo.py │ │ └── ppo_trainer.py └── run.py └── images └── graphical_abstract.png /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Pierre Marza 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multi-Object Navigation with dynamically learned neural implicit representations 2 | ### [Project Page](https://pierremarza.github.io/projects/dynamic_implicit_representations/) | [Paper](https://arxiv.org/abs/2210.05129) 3 | 4 | This codebase is based on a previous [repository](https://github.com/PierreMarza/teaching_agents_how_to_map) studying the impact of auxiliary losses on MultiON performance. With this released code, any user can thus experiment with our introduced implicit representations (see [paper](https://arxiv.org/abs/2210.05129)) and auxiliary losses from our [previous work](https://pierremarza.github.io/projects/teaching_agents_how_to_map/). 5 | 6 | [Multi-Object Navigation with dynamically learned neural implicit representations](https://arxiv.org/abs/2210.05129) 7 | [Pierre Marza](https://pierremarza.github.io/)1, 8 | [Laetitia Matignon](https://perso.liris.cnrs.fr/laetitia.matignon/)2, 9 | [Olivier Simonin](http://perso.citi-lab.fr/osimonin/)1, 10 | [Christian Wolf](https://chriswolfvision.github.io/www/)3
11 | 1INSA Lyon, 2Université Lyon 1, 3Naver Labs Europe
12 | in ICCV 2023 13 | 14 | 15 | 16 | ## Setup 17 | Please follow the instructions in https://github.com/saimwani/multiON to install dependencies and download Matterport3D scenes. 18 | 19 | ## Data 20 | ### Episodes 21 | Episodes are the same as in our [previous work](https://github.com/PierreMarza/teaching_agents_how_to_map). Download *train*, *val* and *test* episodes [here](https://drive.google.com/file/d/1ubfMKD7LPYaJWqz6MKjvBHDkcS3RuYyO/view?usp=share_link). The 3 folders should be copied to *data/datasets/multinav/3_ON/*. Please be careful! In the [original MultiON codebase](https://github.com/saimwani/multiON), what was called *val* set referred to the set of data used to perform final test of the model. We re-named their *val* set into *test* set, and introduced a proper validation set (*val*) to perform hyper-parameter search and early stopping. 22 | 23 | ### Pre-trained models 24 | You can download all pre-trained models [here](https://drive.google.com/file/d/1H7Pf7FhZ6RF6MRRzPkfTQpeb8CcugNBl/view?usp=sharing). We provide a pre-trained global reader model (*global_reader.pth*), visual encoder and segmentation head for the Semantic Finder (*semantic_finder_visual_encoder.pth* and *semantic_finder_segmentation_head.pth*). Finally, we also release the weights of our SOTA agent (*policy.pth*) that was trained from scratch without any pre-training or training curriculum and with auxiliary losses, corresponding to row (k) in Table 2 in our [paper](https://arxiv.org/abs/2210.05129). In the paper, we report mean and std over different training runs (seeds). We chose the one with highest PPL performance on the validation (val) set. 25 | 26 | ## Agent training 27 | ### Implicit agent training 28 | You can modify hyperparameters related to implicit representations in *habitat_baselines/config/multinav/ppo_multinav.yaml*. The two most important are: *learn_sem_finder* (*True*/*False*) to train a Semantic Finder in real time and use its outputs as input to the agent, and *learn_impl_net_expl_occ* (*True*/*False*) to train an Exploration and Occupancy Implicit Representation in real time. If *learn_impl_net_expl_occ* is *True*, *learn_sem_finder* must also be *True*. Having only *learn_sem_finder* as *True* and *learn_impl_net_expl_occ* as *False* will lead to faster training (as only one type of implicit representation is trained in real time). Once hyperparameters are set up, you can use the following command to train an agent using implicit representations, 29 | ```console 30 | python habitat_baselines/run.py --exp-config habitat_baselines/config/multinav/ppo_multinav.yaml --agent-type implicit --run-type train 31 | ``` 32 | 33 | ### Implicit agent training with auxiliary losses 34 | As mentioned, training with auxiliary losses from our [previous work](https://pierremarza.github.io/projects/teaching_agents_how_to_map/) is also supported with this code. The following command shows you how to train an agent with implicit representations using the three auiliary losses (seen target, direction prediction, distance prediction) with the same coefficient (0.25) in the final loss, 35 | ```console 36 | python habitat_baselines/run.py --exp-config habitat_baselines/config/multinav/ppo_multinav.yaml --agent-type implicit --run-type train --seen_coef 0.25 --dir_coef 0.25 --dist_coef 0.25 37 | ``` 38 | 39 | ## Agent evaluation 40 | To evaluate a model simply change the *--run-type* flag from *train* to *eval* and specify the path to either a specific checkpoint, or a folder containing a set of checkpoints (to evaluate all checkpoints in the folder) with the *--eval_path* flag: in the example below, we evaluate our released SOTA agent on the test set. You can modify the flag value to evaluate any other checkpoint you get after training a model. You can specify the set to evaluate on (*val* or *test*) with the *--eval_split* flag. Be careful not to use more processes than scenes (*val* and *test* respectively contain episodes from 11 and 18 scenes). 41 | 42 | ```console 43 | python habitat_baselines/run.py --exp-config habitat_baselines/config/multinav/ppo_multinav.yaml --agent-type implicit --run-type eval --eval_path pretrained_models/policy.pth --eval_split test 44 | ``` 45 | 46 | ## Citation 47 | ``` 48 | @inproceedings{marza2023_dynamic_impl_repr, 49 | author = {Marza, Pierre and Matignon, Laetitia and Simonin, Olivier and Wolf, Christian}, 50 | title = {Multi-Object Navigation with Dynamically Learned Neural Implicit Representations}, 51 | booktitle = {International Conference on Computer Vision (ICCV)}, 52 | year = {2023}, 53 | } 54 | ``` 55 | -------------------------------------------------------------------------------- /configs/tasks/multinav_mp3d.yaml: -------------------------------------------------------------------------------- 1 | ENVIRONMENT: 2 | MAX_EPISODE_STEPS: 2500 3 | SIMULATOR: 4 | TURN_ANGLE: 30 5 | TILT_ANGLE: 30 6 | ACTION_SPACE_CONFIG: "v1" 7 | AGENT_0: 8 | SENSORS: ['RGB_SENSOR', 'DEPTH_SENSOR'] 9 | HEIGHT: 0.88 10 | RADIUS: 0.1 11 | HABITAT_SIM_V0: 12 | GPU_DEVICE_ID: 0 13 | ALLOW_SLIDING: False 14 | SEMANTIC_SENSOR: 15 | WIDTH: 256 16 | HEIGHT: 256 17 | HFOV: 79 18 | POSITION: [0, 0.88, 0] 19 | RGB_SENSOR: 20 | WIDTH: 256 21 | HEIGHT: 256 22 | HFOV: 79 23 | POSITION: [0, 0.88, 0] 24 | DEPTH_SENSOR: 25 | WIDTH: 256 26 | HEIGHT: 256 27 | HFOV: 79 28 | POSITION: [0, 0.88, 0] 29 | 30 | TASK: 31 | TYPE: MultiNav-v1 32 | POSSIBLE_ACTIONS: ["FOUND", "MOVE_FORWARD", "TURN_LEFT", "TURN_RIGHT"] 33 | 34 | SUCCESS_DISTANCE: 1.5 35 | 36 | SENSORS: ['MULTI_GOAL_SENSOR', 'COMPASS_SENSOR', 'GPS_SENSOR', 'POSITION_SENSOR', 'HEADING_SENSOR'] 37 | 38 | GOAL_SENSOR_UUID: multiobjectgoal 39 | 40 | MEASUREMENTS: ['DISTANCE_TO_CURR_GOAL', 'DISTANCE_TO_MULTI_GOAL', 'SUB_SUCCESS', 'SUCCESS', 'EPISODE_LENGTH', 'MSPL', 'PERCENTAGE_SUCCESS', 'RATIO', 'PSPL', 'RAW_METRICS'] 41 | SUCCESS: 42 | SUCCESS_DISTANCE: 1.5 43 | DISTANCE_TO_GOAL: 44 | DISTANCE_TO: POINT 45 | 46 | DATASET: 47 | TYPE: MultiNav-v1 48 | SPLIT: train 49 | DATA_PATH: "data/datasets/multinav/3_ON/{split}/{split}.json.gz" 50 | SCENES_DIR: "data/scene_datasets/" 51 | -------------------------------------------------------------------------------- /habitat/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.config import Config, get_config 13 | from habitat.core.agent import Agent 14 | from habitat.core.benchmark import Benchmark 15 | from habitat.core.challenge import Challenge 16 | from habitat.core.dataset import Dataset 17 | from habitat.core.embodied_task import EmbodiedTask, Measure, Measurements 18 | from habitat.core.env import Env, RLEnv 19 | from habitat.core.logging import logger 20 | from habitat.core.registry import registry 21 | from habitat.core.simulator import Sensor, SensorSuite, SensorTypes, Simulator 22 | from habitat.core.vector_env import ThreadedVectorEnv, VectorEnv 23 | from habitat.datasets import make_dataset 24 | from habitat.version import VERSION as __version__ # noqa 25 | 26 | __all__ = [ 27 | "Agent", 28 | "Benchmark", 29 | "Challenge", 30 | "Config", 31 | "Dataset", 32 | "EmbodiedTask", 33 | "Env", 34 | "get_config", 35 | "logger", 36 | "make_dataset", 37 | "Measure", 38 | "Measurements", 39 | "RLEnv", 40 | "Sensor", 41 | "SensorSuite", 42 | "SensorTypes", 43 | "Simulator", 44 | "ThreadedVectorEnv", 45 | "VectorEnv", 46 | ] 47 | -------------------------------------------------------------------------------- /habitat/config/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | from habitat.config.default import Config, get_config 12 | 13 | r"""Habitat-API Configuration 14 | ============================== 15 | 16 | Habitat-API uses [Yacs configuration system](https://github.com/rbgirshick/yacs) 17 | with the paradigm of `your code + a YACS config for experiment E (+ 18 | external dependencies + hardware + other nuisance terms ...) = 19 | reproducible experiment E`. Yacs advantages: 20 | - Checks for type consistency. 21 | - All parameters and default values are searchable in the code. 22 | - A parameter doesn't need to be set always as each parameter can have a 23 | default value. 24 | - Ability to freeze config to prevent unintended changes. 25 | 26 | ## Config usage 27 | An example of how to merge default config with 2 others configs and overwrite 28 | one parameter that could come from the command line: 29 | ``` 30 | merged_config = get_config( 31 | config_paths=["configs/tasks/pointnav.yaml", 32 | "configs/dataset/val.yaml"], 33 | opts=["ENVIRONMENT.MAX_EPISODE_STEPS", steps_limit] 34 | ) 35 | 36 | ``` 37 | 38 | ## Config structure 39 | Below is the structure of config used for Habitat: 40 | - Environment 41 | - Task 42 | - Sensors 43 | - Measurements 44 | - Simulator 45 | - Agent 46 | - Sensors 47 | - Dataset 48 | 49 | We use node names (e.g. `SENSORS: ['RGB_SENSOR', 'DEPTH_SENSOR']`) instead of list 50 | of config nodes (e.g. `SENSORS: [{TYPE = "HabitatSimDepthSensor", 51 | MIN_DEPTH = 0}, ...]`) to declare the Sensors attached to an Agent or Measures 52 | enabled for the Task . With this approach, it's still easy to overwrite a 53 | particular sensor parameter in yaml file without redefining the whole sensor 54 | config. 55 | 56 | ## Extending the config without defaults 57 | Create a YAML file and add new fields and values. Load the custom config using 58 | `habitat.get_config()` and defined fields will be merged in default Habitat config: 59 | ``` 60 | import habitat 61 | import argparse 62 | from typing import List, Optional, Union 63 | 64 | config = habitat.get_config("{path to user define yaml config}") 65 | env = habitat.Env(config) 66 | ``` 67 | 68 | ## Extending the config with default values 69 | Example of how to extend a config outside of `habtiat-api` repository. 70 | First, we create a config extending the default config in the code and re-use 71 | `habitat.get_config()`: 72 | ``` 73 | import habitat 74 | import argparse 75 | from typing import List, Optional, Union 76 | 77 | _C = habitat.get_config() 78 | _C.defrost() 79 | # Add new parameters to the config 80 | _C.TASK.EPISODE_INFO = habitat.Config() 81 | _C.TASK.EPISODE_INFO.TYPE = "EpisodeInfo" 82 | _C.TASK.EPISODE_INFO.VALUE = 5 83 | _C.TASK.MEASUREMENTS.append("EPISODE_INFO") 84 | 85 | # New function returning extended Habitat config that should be used instead 86 | # of habitat.get_config() 87 | def my_get_config( 88 | config_paths: Optional[Union[List[str], str]] = None, 89 | opts: Optional[list] = None, 90 | ) -> habitat.Config: 91 | CONFIG_FILE_SEPARATOR = "," 92 | config = _C.clone() 93 | if config_paths: 94 | if isinstance(config_paths, str): 95 | if CONFIG_FILE_SEPARATOR in config_paths: 96 | config_paths = config_paths.split(CONFIG_FILE_SEPARATOR) 97 | else: 98 | config_paths = [config_paths] 99 | 100 | for config_path in config_paths: 101 | config.merge_from_file(config_path) 102 | 103 | if opts: 104 | config.merge_from_list(opts) 105 | 106 | config.freeze() 107 | return config 108 | 109 | def main(): 110 | parser = argparse.ArgumentParser() 111 | parser.add_argument( 112 | "--task-config", 113 | type=str, 114 | default="configs/tasks/pointnav.yaml," 115 | "configs/datasets/pointnav/habitat_test.yaml", 116 | ) 117 | parser.add_argument( 118 | "opts", 119 | default=None, 120 | nargs=argparse.REMAINDER, 121 | help="Modify config options from command line", 122 | ) 123 | args = parser.parse_args() 124 | config = my_get_config(config_paths=args.task_config, opts=args.opts) 125 | env = habitat.Env(config) 126 | 127 | ```""" 128 | 129 | __all__ = ["Config", "get_config"] 130 | -------------------------------------------------------------------------------- /habitat/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | -------------------------------------------------------------------------------- /habitat/core/agent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | r"""Base implementation of agent inside habitat. To build agents inside habitat 12 | the user should subclass ``habitat.Agent`` and implement the ``act()`` 13 | and ``reset()`` methods. 14 | """ 15 | 16 | from typing import Any, Dict, Union 17 | 18 | from habitat.core.simulator import Observations 19 | 20 | 21 | class Agent: 22 | r"""Abstract class for defining agents which act inside `core.env.Env`. 23 | 24 | This abstract class standardizes agents to allow seamless benchmarking. 25 | """ 26 | 27 | def reset(self) -> None: 28 | r"""Called before starting a new episode in environment.""" 29 | raise NotImplementedError 30 | 31 | def act(self, observations: Observations) -> Union[int, str, Dict[str, Any]]: 32 | r"""Called to produce an action to perform in an environment. 33 | 34 | :param observations: observations coming in from environment to be 35 | used by agent to decide action. 36 | :return: action to be taken inside the environment and optional action 37 | arguments. 38 | """ 39 | raise NotImplementedError 40 | -------------------------------------------------------------------------------- /habitat/core/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | r"""Implements evaluation of ``habitat.Agent`` inside ``habitat.Env``. 12 | ``habitat.Benchmark`` creates a ``habitat.Env`` which is specified through 13 | the ``config_env`` parameter in constructor. The evaluation is task agnostic 14 | and is implemented through metrics defined for ``habitat.EmbodiedTask``. 15 | """ 16 | 17 | from collections import defaultdict 18 | from typing import Dict, Optional 19 | 20 | from habitat.config.default import get_config 21 | from habitat.core.agent import Agent 22 | from habitat.core.env import Env 23 | 24 | 25 | class Benchmark: 26 | r"""Benchmark for evaluating agents in environments.""" 27 | 28 | def __init__(self, config_paths: Optional[str] = None) -> None: 29 | r""".. 30 | 31 | :param config_paths: file to be used for creating the environment 32 | """ 33 | config_env = get_config(config_paths) 34 | self._env = Env(config=config_env) 35 | 36 | def evaluate( 37 | self, agent: Agent, num_episodes: Optional[int] = None 38 | ) -> Dict[str, float]: 39 | r""".. 40 | 41 | :param agent: agent to be evaluated in environment. 42 | :param num_episodes: count of number of episodes for which the 43 | evaluation should be run. 44 | :return: dict containing metrics tracked by environment. 45 | """ 46 | 47 | if num_episodes is None: 48 | num_episodes = len(self._env.episodes) 49 | else: 50 | assert num_episodes <= len(self._env.episodes), ( 51 | "num_episodes({}) is larger than number of episodes " 52 | "in environment ({})".format(num_episodes, len(self._env.episodes)) 53 | ) 54 | 55 | assert num_episodes > 0, "num_episodes should be greater than 0" 56 | 57 | agg_metrics: Dict = defaultdict(float) 58 | 59 | count_episodes = 0 60 | while count_episodes < num_episodes: 61 | agent.reset() 62 | observations = self._env.reset() 63 | 64 | while not self._env.episode_over: 65 | action = agent.act(observations) 66 | observations = self._env.step(action) 67 | 68 | metrics = self._env.get_metrics() 69 | for m, v in metrics.items(): 70 | agg_metrics[m] += v 71 | count_episodes += 1 72 | 73 | avg_metrics = {k: v / count_episodes for k, v in agg_metrics.items()} 74 | 75 | return avg_metrics 76 | -------------------------------------------------------------------------------- /habitat/core/challenge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import os 13 | 14 | from habitat.core.benchmark import Benchmark 15 | from habitat.core.logging import logger 16 | 17 | 18 | class Challenge(Benchmark): 19 | def __init__(self): 20 | config_paths = os.environ["CHALLENGE_CONFIG_FILE"] 21 | super().__init__(config_paths) 22 | 23 | def submit(self, agent): 24 | metrics = super().evaluate(agent) 25 | for k, v in metrics.items(): 26 | logger.info("{}: {}".format(k, v)) 27 | -------------------------------------------------------------------------------- /habitat/core/logging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import logging 13 | 14 | 15 | class HabitatLogger(logging.Logger): 16 | def __init__( 17 | self, 18 | name, 19 | level, 20 | filename=None, 21 | filemode="a", 22 | stream=None, 23 | format=None, 24 | dateformat=None, 25 | style="%", 26 | ): 27 | super().__init__(name, level) 28 | if filename is not None: 29 | handler = logging.FileHandler(filename, filemode) 30 | else: 31 | handler = logging.StreamHandler(stream) 32 | self._formatter = logging.Formatter(format, dateformat, style) 33 | handler.setFormatter(self._formatter) 34 | super().addHandler(handler) 35 | 36 | def add_filehandler(self, log_filename): 37 | filehandler = logging.FileHandler(log_filename) 38 | filehandler.setFormatter(self._formatter) 39 | self.addHandler(filehandler) 40 | 41 | 42 | logger = HabitatLogger( 43 | name="habitat", level=logging.INFO, format="%(asctime)-15s %(message)s" 44 | ) 45 | -------------------------------------------------------------------------------- /habitat/core/registry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | r"""Registry is central source of truth in Habitat. 13 | 14 | Taken from Pythia, it is inspired from Redux's concept of global store. 15 | Registry maintains mappings of various information to unique keys. Special 16 | functions in registry can be used as decorators to register different kind of 17 | classes. 18 | 19 | Import the global registry object using 20 | 21 | .. code:: py 22 | 23 | from habitat.core.registry import registry 24 | 25 | Various decorators for registry different kind of classes with unique keys 26 | 27 | - Register a task: ``@registry.register_task`` 28 | - Register a task action: ``@registry.register_task_action`` 29 | - Register a simulator: ``@registry.register_simulator`` 30 | - Register a sensor: ``@registry.register_sensor`` 31 | - Register a measure: ``@registry.register_measure`` 32 | - Register a dataset: ``@registry.register_dataset`` 33 | """ 34 | 35 | import collections 36 | from typing import Optional 37 | 38 | from habitat.core.utils import Singleton 39 | 40 | 41 | class Registry(metaclass=Singleton): 42 | mapping = collections.defaultdict(dict) 43 | 44 | @classmethod 45 | def _register_impl(cls, _type, to_register, name, assert_type=None): 46 | def wrap(to_register): 47 | if assert_type is not None: 48 | assert issubclass( 49 | to_register, assert_type 50 | ), "{} must be a subclass of {}".format(to_register, assert_type) 51 | register_name = to_register.__name__ if name is None else name 52 | 53 | cls.mapping[_type][register_name] = to_register 54 | return to_register 55 | 56 | if to_register is None: 57 | return wrap 58 | else: 59 | return wrap(to_register) 60 | 61 | @classmethod 62 | def register_task(cls, to_register=None, *, name: Optional[str] = None): 63 | r"""Register a task to registry with key :p:`name` 64 | 65 | :param name: Key with which the task will be registered. 66 | If :py:`None` will use the name of the class 67 | 68 | .. code:: py 69 | 70 | from habitat.core.registry import registry 71 | from habitat.core.embodied_task import EmbodiedTask 72 | 73 | @registry.register_task 74 | class MyTask(EmbodiedTask): 75 | pass 76 | 77 | 78 | # or 79 | 80 | @registry.register_task(name="MyTaskName") 81 | class MyTask(EmbodiedTask): 82 | pass 83 | 84 | """ 85 | from habitat.core.embodied_task import EmbodiedTask 86 | 87 | return cls._register_impl("task", to_register, name, assert_type=EmbodiedTask) 88 | 89 | @classmethod 90 | def register_simulator(cls, to_register=None, *, name: Optional[str] = None): 91 | r"""Register a simulator to registry with key :p:`name` 92 | 93 | :param name: Key with which the simulator will be registered. 94 | If :py:`None` will use the name of the class 95 | 96 | .. code:: py 97 | 98 | from habitat.core.registry import registry 99 | from habitat.core.simulator import Simulator 100 | 101 | @registry.register_simulator 102 | class MySimulator(Simulator): 103 | pass 104 | 105 | 106 | # or 107 | 108 | @registry.register_simulator(name="MySimName") 109 | class MySimulator(Simulator): 110 | pass 111 | 112 | """ 113 | from habitat.core.simulator import Simulator 114 | 115 | return cls._register_impl("sim", to_register, name, assert_type=Simulator) 116 | 117 | @classmethod 118 | def register_sensor(cls, to_register=None, *, name: Optional[str] = None): 119 | r"""Register a sensor to registry with key :p:`name` 120 | 121 | :param name: Key with which the sensor will be registered. 122 | If :py:`None` will use the name of the class 123 | """ 124 | from habitat.core.simulator import Sensor 125 | 126 | return cls._register_impl("sensor", to_register, name, assert_type=Sensor) 127 | 128 | @classmethod 129 | def register_measure(cls, to_register=None, *, name: Optional[str] = None): 130 | r"""Register a measure to registry with key :p:`name` 131 | 132 | :param name: Key with which the measure will be registered. 133 | If :py:`None` will use the name of the class 134 | """ 135 | from habitat.core.embodied_task import Measure 136 | 137 | return cls._register_impl("measure", to_register, name, assert_type=Measure) 138 | 139 | @classmethod 140 | def register_task_action(cls, to_register=None, *, name: Optional[str] = None): 141 | r"""Add a task action in this registry under key 'name' 142 | 143 | :param action_space: An action space that describes parameters to the 144 | task action's method. If :py:`None` then the task action's method 145 | takes no parameters. 146 | :param name: Key with which the task action will be registered. If 147 | :py:`None` will use the name of the task action's method. 148 | """ 149 | from habitat.core.embodied_task import Action 150 | 151 | return cls._register_impl("task_action", to_register, name, assert_type=Action) 152 | 153 | @classmethod 154 | def register_dataset(cls, to_register=None, *, name: Optional[str] = None): 155 | r"""Register a dataset to registry with key :p:`name` 156 | 157 | :param name: Key with which the dataset will be registered. 158 | If :py:`None` will use the name of the class 159 | """ 160 | from habitat.core.dataset import Dataset 161 | 162 | return cls._register_impl("dataset", to_register, name, assert_type=Dataset) 163 | 164 | @classmethod 165 | def register_action_space_configuration( 166 | cls, to_register=None, *, name: Optional[str] = None 167 | ): 168 | r"""Register a action space configuration to registry with key :p:`name` 169 | 170 | :param name: Key with which the action space will be registered. 171 | If :py:`None` will use the name of the class 172 | """ 173 | from habitat.core.simulator import ActionSpaceConfiguration 174 | 175 | return cls._register_impl( 176 | "action_space_config", 177 | to_register, 178 | name, 179 | assert_type=ActionSpaceConfiguration, 180 | ) 181 | 182 | @classmethod 183 | def _get_impl(cls, _type, name): 184 | return cls.mapping[_type].get(name, None) 185 | 186 | @classmethod 187 | def get_task(cls, name): 188 | return cls._get_impl("task", name) 189 | 190 | @classmethod 191 | def get_task_action(cls, name): 192 | return cls._get_impl("task_action", name) 193 | 194 | @classmethod 195 | def get_simulator(cls, name): 196 | return cls._get_impl("sim", name) 197 | 198 | @classmethod 199 | def get_sensor(cls, name): 200 | return cls._get_impl("sensor", name) 201 | 202 | @classmethod 203 | def get_measure(cls, name): 204 | return cls._get_impl("measure", name) 205 | 206 | @classmethod 207 | def get_dataset(cls, name): 208 | return cls._get_impl("dataset", name) 209 | 210 | @classmethod 211 | def get_action_space_configuration(cls, name): 212 | return cls._get_impl("action_space_config", name) 213 | 214 | 215 | registry = Registry() 216 | -------------------------------------------------------------------------------- /habitat/core/spaces.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from collections import OrderedDict 13 | from typing import Sized 14 | 15 | import gym 16 | from gym import Space 17 | 18 | 19 | class EmptySpace(Space): 20 | """ 21 | A ``gym.Space`` that reflects arguments space for action that doesn't have 22 | arguments. Needed for consistency ang always samples `None` value. 23 | """ 24 | 25 | def sample(self): 26 | return None 27 | 28 | def contains(self, x): 29 | if x is None: 30 | return True 31 | return False 32 | 33 | def __repr__(self): 34 | return "EmptySpace()" 35 | 36 | 37 | class ActionSpace(gym.spaces.Dict): 38 | """ 39 | A dictionary of ``EmbodiedTask`` actions and their argument spaces. 40 | 41 | .. code:: py 42 | 43 | self.observation_space = spaces.ActionSpace({ 44 | "move": spaces.Dict({ 45 | "position": spaces.Discrete(2), 46 | "velocity": spaces.Discrete(3) 47 | }), 48 | "move_forward": EmptySpace(), 49 | }) 50 | """ 51 | 52 | def __init__(self, spaces): 53 | if isinstance(spaces, dict): 54 | self.spaces = OrderedDict(sorted(list(spaces.items()))) 55 | if isinstance(spaces, list): 56 | self.spaces = OrderedDict(spaces) 57 | self.actions_select = gym.spaces.Discrete(len(self.spaces)) 58 | 59 | @property 60 | def n(self): 61 | return len(self.spaces) 62 | 63 | def sample(self): 64 | action_index = self.actions_select.sample() 65 | return { 66 | "action": list(self.spaces.keys())[action_index], 67 | "action_args": list(self.spaces.values())[action_index].sample(), 68 | } 69 | 70 | def contains(self, x): 71 | if not isinstance(x, dict) or "action" not in x: 72 | return False 73 | if x["action"] not in self.spaces: 74 | return False 75 | if not self.spaces[x["action"]].contains(x.get("action_args", None)): 76 | return False 77 | return True 78 | 79 | def __repr__(self): 80 | return ( 81 | "ActionSpace(" 82 | + ", ".join([k + ":" + str(s) for k, s in self.spaces.items()]) 83 | + ")" 84 | ) 85 | 86 | 87 | class ListSpace(Space): 88 | """ 89 | A ``gym.Space`` that describes a list of other Space. Used to describe 90 | list of tokens ids, vectors and etc. 91 | 92 | .. code:: py 93 | 94 | observation_space = ListSpace(spaces.Discrete( 95 | dataset.question_vocab.get_size())) 96 | """ 97 | 98 | def __init__(self, space, min_seq_length=0, max_seq_length=1 << 15): 99 | self.min_seq_length = min_seq_length 100 | self.max_seq_length = max_seq_length 101 | self.space = space 102 | self.length_select = gym.spaces.Discrete(max_seq_length - min_seq_length) 103 | 104 | def sample(self): 105 | seq_length = self.length_select.sample() + self.min_seq_length 106 | return [self.space.sample() for _ in range(seq_length)] 107 | 108 | def contains(self, x): 109 | if not isinstance(x, Sized): 110 | return False 111 | 112 | if not (self.min_seq_length <= len(x) <= self.max_seq_length): 113 | return False 114 | 115 | return all([self.space.contains(el) for el in x]) 116 | 117 | def __repr__(self): 118 | return ( 119 | f"ListSpace({self.space}, min_seq_length=" 120 | f"{self.min_seq_length}, max_seq_length={self.max_seq_length})" 121 | ) 122 | -------------------------------------------------------------------------------- /habitat/core/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import json 13 | from typing import List 14 | 15 | import numpy as np 16 | import quaternion 17 | 18 | from habitat.utils.geometry_utils import quaternion_to_list 19 | 20 | # Internals from inner json library needed for patching functionality in 21 | # DatasetFloatJSONEncoder. 22 | try: 23 | from _json import encode_basestring_ascii 24 | except ImportError: 25 | encode_basestring_ascii = None 26 | try: 27 | from _json import encode_basestring 28 | except ImportError: 29 | encode_basestring = None 30 | 31 | 32 | def tile_images(images: List[np.ndarray]) -> np.ndarray: 33 | r"""Tile multiple images into single image 34 | 35 | Args: 36 | images: list of images where each image has dimension 37 | (height x width x channels) 38 | 39 | Returns: 40 | tiled image (new_height x width x channels) 41 | """ 42 | assert len(images) > 0, "empty list of images" 43 | np_images = np.asarray(images) 44 | n_images, height, width, n_channels = np_images.shape 45 | new_height = int(np.ceil(np.sqrt(n_images))) 46 | new_width = int(np.ceil(float(n_images) / new_height)) 47 | # pad with empty images to complete the rectangle 48 | np_images = np.array( 49 | images + [images[0] * 0 for _ in range(n_images, new_height * new_width)] 50 | ) 51 | # img_HWhwc 52 | out_image = np_images.reshape(new_height, new_width, height, width, n_channels) 53 | # img_HhWwc 54 | out_image = out_image.transpose(0, 2, 1, 3, 4) 55 | # img_Hh_Ww_c 56 | out_image = out_image.reshape(new_height * height, new_width * width, n_channels) 57 | return out_image 58 | 59 | 60 | def not_none_validator(self, attribute, value): 61 | if value is None: 62 | raise ValueError(f"Argument '{attribute.name}' must be set") 63 | 64 | 65 | def try_cv2_import(): 66 | r"""The PyRobot python3 version which is a dependency of Habitat-PyRobot integration 67 | relies on ROS running in python2.7. In order to import cv2 in python3 we need to remove 68 | the python2.7 path from sys.path. To use the Habitat-PyRobot integration the user 69 | needs to export environment variable ROS_PATH which will look something like: 70 | /opt/ros/kinetic/lib/python2.7/dist-packages 71 | """ 72 | import sys 73 | import os 74 | 75 | ros_path = os.environ.get("ROS_PATH") 76 | if ros_path is not None and ros_path in sys.path: 77 | sys.path.remove(ros_path) 78 | import cv2 79 | 80 | sys.path.append(ros_path) 81 | else: 82 | import cv2 83 | 84 | return cv2 85 | 86 | 87 | class Singleton(type): 88 | _instances = {} 89 | 90 | def __call__(cls, *args, **kwargs): 91 | if cls not in cls._instances: 92 | cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) 93 | return cls._instances[cls] 94 | 95 | 96 | def center_crop(obs, new_shape): 97 | top_left = ( 98 | (obs.shape[0] // 2) - (new_shape[0] // 2), 99 | (obs.shape[1] // 2) - (new_shape[1] // 2), 100 | ) 101 | bottom_right = ( 102 | (obs.shape[0] // 2) + (new_shape[0] // 2), 103 | (obs.shape[1] // 2) + (new_shape[1] // 2), 104 | ) 105 | obs = obs[top_left[0] : bottom_right[0], top_left[1] : bottom_right[1], :] 106 | 107 | return obs 108 | 109 | 110 | class DatasetFloatJSONEncoder(json.JSONEncoder): 111 | r"""JSON Encoder that sets a float precision for a space saving purpose and 112 | encodes ndarray and quaternion. The encoder is compatible with JSON 113 | version 2.0.9. 114 | """ 115 | 116 | def default(self, object): 117 | # JSON doesn't support numpy ndarray and quaternion 118 | if isinstance(object, np.ndarray): 119 | return object.tolist() 120 | if isinstance(object, np.quaternion): 121 | return quaternion_to_list(object) 122 | quaternion 123 | return object.__dict__ 124 | 125 | # Overriding method to inject own `_repr` function for floats with needed 126 | # precision. 127 | def iterencode(self, o, _one_shot=False): 128 | 129 | if self.check_circular: 130 | markers = {} 131 | else: 132 | markers = None 133 | if self.ensure_ascii: 134 | _encoder = encode_basestring_ascii 135 | else: 136 | _encoder = encode_basestring 137 | 138 | def floatstr( 139 | o, 140 | allow_nan=self.allow_nan, 141 | _repr=lambda x: format(x, ".5f"), 142 | _inf=float("inf"), 143 | _neginf=-float("inf"), 144 | ): 145 | if o != o: 146 | text = "NaN" 147 | elif o == _inf: 148 | text = "Infinity" 149 | elif o == _neginf: 150 | text = "-Infinity" 151 | else: 152 | return _repr(o) 153 | 154 | if not allow_nan: 155 | raise ValueError( 156 | "Out of range float values are not JSON compliant: " + repr(o) 157 | ) 158 | 159 | return text 160 | 161 | _iterencode = json.encoder._make_iterencode( 162 | markers, 163 | self.default, 164 | _encoder, 165 | self.indent, 166 | floatstr, 167 | self.key_separator, 168 | self.item_separator, 169 | self.sort_keys, 170 | self.skipkeys, 171 | _one_shot, 172 | ) 173 | return _iterencode(o, 0) 174 | -------------------------------------------------------------------------------- /habitat/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.datasets.registration import make_dataset 13 | -------------------------------------------------------------------------------- /habitat/datasets/multi_nav/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.dataset import Dataset 13 | from habitat.core.registry import registry 14 | 15 | 16 | def _try_register_multinavdatasetv1(): 17 | try: 18 | from habitat.datasets.multi_nav.multi_nav_dataset import ( 19 | MultiNavDatasetV1, 20 | ) 21 | 22 | has_pointnav = True 23 | except ImportError as e: 24 | has_pointnav = False 25 | pointnav_import_error = e 26 | 27 | if has_pointnav: 28 | from habitat.datasets.multi_nav.multi_nav_dataset import ( 29 | MultiNavDatasetV1, 30 | ) 31 | else: 32 | 33 | @registry.register_dataset(name="MultiNav-v1") 34 | class MultiNavDatasetImportError(Dataset): 35 | def __init__(self, *args, **kwargs): 36 | raise pointnav_import_error 37 | -------------------------------------------------------------------------------- /habitat/datasets/multi_nav/multi_nav_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import json 13 | import os 14 | from typing import Any, Dict, List, Optional 15 | 16 | from habitat.config import Config 17 | from habitat.core.registry import registry 18 | from habitat.core.simulator import AgentState, ShortestPathPoint 19 | from habitat.core.utils import DatasetFloatJSONEncoder 20 | from habitat.datasets.pointnav.pointnav_dataset import ( 21 | CONTENT_SCENES_PATH_FIELD, 22 | DEFAULT_SCENE_PATH_PREFIX, 23 | PointNavDatasetV1, 24 | ) 25 | from habitat.tasks.nav.multi_nav_task import ( 26 | ObjectGoal, 27 | MultiGoal, 28 | MultiGoalNavEpisode, 29 | ObjectViewLocation, 30 | ) 31 | 32 | 33 | @registry.register_dataset(name="MultiNav-v1") 34 | class MultiNavDatasetV1(PointNavDatasetV1): 35 | r"""Class inherited from PointNavDataset that loads Object Navigation dataset.""" 36 | category_to_task_category_id: Dict[str, int] 37 | category_to_scene_annotation_category_id: Dict[str, int] 38 | episodes: List[MultiGoalNavEpisode] 39 | content_scenes_path: str = "{data_path}/content/{scene}.json.gz" 40 | goals_by_category: Dict[str, List[ObjectGoal]] 41 | 42 | def to_json(self) -> str: 43 | for i in range(len(self.episodes)): 44 | self.episodes[i].goals = [] 45 | 46 | result = DatasetFloatJSONEncoder().encode(self) 47 | 48 | for i in range(len(self.episodes)): 49 | self.episodes[i].goals = self.goals_by_category[self.episodes[i].goals_key] 50 | 51 | return result 52 | 53 | def __init__(self, config: Optional[Config] = None) -> None: 54 | self.goals_by_category = {} 55 | super().__init__(config) 56 | 57 | @staticmethod 58 | def __deserialize_goal(serialized_goal: Dict[str, Any]) -> ObjectGoal: 59 | g = ObjectGoal(**serialized_goal) 60 | 61 | for vidx, view in enumerate(g.view_points): 62 | view_location = ObjectViewLocation(**view) 63 | view_location.agent_state = AgentState(**view_location.agent_state) 64 | g.view_points[vidx] = view_location 65 | 66 | return g 67 | 68 | def from_json(self, json_str: str, scenes_dir: Optional[str] = None) -> None: 69 | deserialized = json.loads(json_str) 70 | if CONTENT_SCENES_PATH_FIELD in deserialized: 71 | self.content_scenes_path = deserialized[CONTENT_SCENES_PATH_FIELD] 72 | 73 | if "category_to_task_category_id" in deserialized: 74 | self.category_to_task_category_id = deserialized[ 75 | "category_to_task_category_id" 76 | ] 77 | 78 | if "category_to_scene_annotation_category_id" in deserialized: 79 | self.category_to_scene_annotation_category_id = deserialized[ 80 | "category_to_scene_annotation_category_id" 81 | ] 82 | 83 | if "category_to_mp3d_category_id" in deserialized: 84 | self.category_to_scene_annotation_category_id = deserialized[ 85 | "category_to_mp3d_category_id" 86 | ] 87 | 88 | assert len(self.category_to_task_category_id) == len( 89 | self.category_to_scene_annotation_category_id 90 | ) 91 | 92 | assert set(self.category_to_task_category_id.keys()) == set( 93 | self.category_to_scene_annotation_category_id.keys() 94 | ), "category_to_task and category_to_mp3d must have the same keys" 95 | 96 | if len(deserialized["episodes"]) == 0: 97 | return 98 | 99 | for i, episode in enumerate(deserialized["episodes"]): 100 | episode["object_index"] = 0 101 | episode["currGoalIndex"] = 0 102 | episode = MultiGoalNavEpisode(**episode) 103 | episode.episode_id = str(i) 104 | 105 | if scenes_dir is not None: 106 | if episode.scene_id.startswith(DEFAULT_SCENE_PATH_PREFIX): 107 | episode.scene_id = episode.scene_id[ 108 | len(DEFAULT_SCENE_PATH_PREFIX) : 109 | ] 110 | 111 | episode.scene_id = os.path.join(scenes_dir, episode.scene_id) 112 | 113 | episode.goals = [MultiGoal(**i) for i in episode.goals] 114 | 115 | if episode.shortest_paths is not None: 116 | for path in episode.shortest_paths: 117 | for p_index, point in enumerate(path): 118 | if point is None or isinstance(point, (int, str)): 119 | point = { 120 | "action": point, 121 | "rotation": None, 122 | "position": None, 123 | } 124 | 125 | path[p_index] = ShortestPathPoint(**point) 126 | 127 | self.episodes.append(episode) 128 | -------------------------------------------------------------------------------- /habitat/datasets/object_nav/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.dataset import Dataset 13 | from habitat.core.registry import registry 14 | 15 | 16 | def _try_register_objectnavdatasetv1(): 17 | try: 18 | from habitat.datasets.object_nav.object_nav_dataset import ( 19 | ObjectNavDatasetV1, 20 | ) 21 | 22 | has_pointnav = True 23 | except ImportError as e: 24 | has_pointnav = False 25 | pointnav_import_error = e 26 | 27 | if has_pointnav: 28 | from habitat.datasets.object_nav.object_nav_dataset import ( 29 | ObjectNavDatasetV1, 30 | ) 31 | else: 32 | 33 | @registry.register_dataset(name="ObjectNav-v1") 34 | class ObjectNavDatasetImportError(Dataset): 35 | def __init__(self, *args, **kwargs): 36 | raise pointnav_import_error 37 | -------------------------------------------------------------------------------- /habitat/datasets/object_nav/object_nav_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import json 13 | import os 14 | from typing import Any, Dict, List, Optional 15 | 16 | from habitat.config import Config 17 | from habitat.core.registry import registry 18 | from habitat.core.simulator import AgentState, ShortestPathPoint 19 | from habitat.core.utils import DatasetFloatJSONEncoder 20 | from habitat.datasets.pointnav.pointnav_dataset import ( 21 | CONTENT_SCENES_PATH_FIELD, 22 | DEFAULT_SCENE_PATH_PREFIX, 23 | PointNavDatasetV1, 24 | ) 25 | from habitat.tasks.nav.object_nav_task import ( 26 | ObjectGoal, 27 | ObjectGoalNavEpisode, 28 | ObjectViewLocation, 29 | ) 30 | 31 | 32 | @registry.register_dataset(name="ObjectNav-v1") 33 | class ObjectNavDatasetV1(PointNavDatasetV1): 34 | r"""Class inherited from PointNavDataset that loads Object Navigation dataset.""" 35 | category_to_task_category_id: Dict[str, int] 36 | category_to_scene_annotation_category_id: Dict[str, int] 37 | episodes: List[ObjectGoalNavEpisode] 38 | content_scenes_path: str = "{data_path}/content/{scene}.json.gz" 39 | goals_by_category: Dict[str, List[ObjectGoal]] 40 | 41 | @staticmethod 42 | def dedup_goals(dataset: Dict[str, Any]) -> Dict[str, Any]: 43 | if len(dataset["episodes"]) == 0: 44 | return dataset 45 | 46 | goals_by_category = dict() 47 | for i, ep in enumerate(dataset["episodes"]): 48 | dataset["episodes"][i]["object_category"] = ep["goals"][0][ 49 | "object_category" 50 | ] 51 | ep = ObjectGoalNavEpisode(**ep) 52 | 53 | goals_key = ep.goals_key 54 | if goals_key not in goals_by_category: 55 | goals_by_category[goals_key] = ep.goals 56 | 57 | dataset["episodes"][i]["goals"] = [] 58 | 59 | dataset["goals_by_category"] = goals_by_category 60 | 61 | return dataset 62 | 63 | def to_json(self) -> str: 64 | for i in range(len(self.episodes)): 65 | self.episodes[i].goals = [] 66 | 67 | result = DatasetFloatJSONEncoder().encode(self) 68 | 69 | for i in range(len(self.episodes)): 70 | self.episodes[i].goals = self.goals_by_category[self.episodes[i].goals_key] 71 | 72 | return result 73 | 74 | def __init__(self, config: Optional[Config] = None) -> None: 75 | self.goals_by_category = {} 76 | super().__init__(config) 77 | 78 | @staticmethod 79 | def __deserialize_goal(serialized_goal: Dict[str, Any]) -> ObjectGoal: 80 | g = ObjectGoal(**serialized_goal) 81 | 82 | for vidx, view in enumerate(g.view_points): 83 | view_location = ObjectViewLocation(**view) 84 | view_location.agent_state = AgentState(**view_location.agent_state) 85 | g.view_points[vidx] = view_location 86 | 87 | return g 88 | 89 | def from_json(self, json_str: str, scenes_dir: Optional[str] = None) -> None: 90 | deserialized = json.loads(json_str) 91 | if CONTENT_SCENES_PATH_FIELD in deserialized: 92 | self.content_scenes_path = deserialized[CONTENT_SCENES_PATH_FIELD] 93 | 94 | if "category_to_task_category_id" in deserialized: 95 | self.category_to_task_category_id = deserialized[ 96 | "category_to_task_category_id" 97 | ] 98 | 99 | if "category_to_scene_annotation_category_id" in deserialized: 100 | self.category_to_scene_annotation_category_id = deserialized[ 101 | "category_to_scene_annotation_category_id" 102 | ] 103 | 104 | if "category_to_mp3d_category_id" in deserialized: 105 | self.category_to_scene_annotation_category_id = deserialized[ 106 | "category_to_mp3d_category_id" 107 | ] 108 | 109 | assert len(self.category_to_task_category_id) == len( 110 | self.category_to_scene_annotation_category_id 111 | ) 112 | 113 | assert set(self.category_to_task_category_id.keys()) == set( 114 | self.category_to_scene_annotation_category_id.keys() 115 | ), "category_to_task and category_to_mp3d must have the same keys" 116 | 117 | if len(deserialized["episodes"]) == 0: 118 | return 119 | 120 | if "goals_by_category" not in deserialized: 121 | deserialized = self.dedup_goals(deserialized) 122 | 123 | for k, v in deserialized["goals_by_category"].items(): 124 | self.goals_by_category[k] = [self.__deserialize_goal(g) for g in v] 125 | 126 | for i, episode in enumerate(deserialized["episodes"]): 127 | episode = ObjectGoalNavEpisode(**episode) 128 | episode.episode_id = str(i) 129 | 130 | if scenes_dir is not None: 131 | if episode.scene_id.startswith(DEFAULT_SCENE_PATH_PREFIX): 132 | episode.scene_id = episode.scene_id[ 133 | len(DEFAULT_SCENE_PATH_PREFIX) : 134 | ] 135 | 136 | episode.scene_id = os.path.join(scenes_dir, episode.scene_id) 137 | 138 | episode.goals = self.goals_by_category[episode.goals_key] 139 | 140 | if episode.shortest_paths is not None: 141 | for path in episode.shortest_paths: 142 | for p_index, point in enumerate(path): 143 | if point is None or isinstance(point, (int, str)): 144 | point = { 145 | "action": point, 146 | "rotation": None, 147 | "position": None, 148 | } 149 | 150 | path[p_index] = ShortestPathPoint(**point) 151 | 152 | self.episodes.append(episode) 153 | -------------------------------------------------------------------------------- /habitat/datasets/pointnav/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.dataset import Dataset 13 | from habitat.core.registry import registry 14 | 15 | 16 | def _try_register_pointnavdatasetv1(): 17 | try: 18 | from habitat.datasets.pointnav.pointnav_dataset import ( 19 | PointNavDatasetV1, 20 | ) 21 | 22 | has_pointnav = True 23 | except ImportError as e: 24 | has_pointnav = False 25 | pointnav_import_error = e 26 | 27 | if has_pointnav: 28 | from habitat.datasets.pointnav.pointnav_dataset import ( 29 | PointNavDatasetV1, 30 | ) 31 | else: 32 | 33 | @registry.register_dataset(name="PointNav-v1") 34 | class PointnavDatasetImportError(Dataset): 35 | def __init__(self, *args, **kwargs): 36 | raise pointnav_import_error 37 | -------------------------------------------------------------------------------- /habitat/datasets/pointnav/pointnav_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import gzip 13 | import json 14 | import os 15 | from typing import List, Optional 16 | 17 | from habitat.config import Config 18 | from habitat.core.dataset import ALL_SCENES_MASK, Dataset 19 | from habitat.core.registry import registry 20 | from habitat.tasks.nav.nav import ( 21 | NavigationEpisode, 22 | NavigationGoal, 23 | ShortestPathPoint, 24 | ) 25 | 26 | CONTENT_SCENES_PATH_FIELD = "content_scenes_path" 27 | DEFAULT_SCENE_PATH_PREFIX = "data/scene_datasets/" 28 | 29 | 30 | @registry.register_dataset(name="PointNav-v1") 31 | class PointNavDatasetV1(Dataset): 32 | r"""Class inherited from Dataset that loads Point Navigation dataset.""" 33 | 34 | episodes: List[NavigationEpisode] 35 | content_scenes_path: str = "{data_path}/content/{scene}.json.gz" 36 | 37 | @staticmethod 38 | def check_config_paths_exist(config: Config) -> bool: 39 | return os.path.exists( 40 | config.DATA_PATH.format(split=config.SPLIT) 41 | ) and os.path.exists(config.SCENES_DIR) 42 | 43 | @classmethod 44 | def get_scenes_to_load(cls, config: Config) -> List[str]: 45 | r"""Return list of scene ids for which dataset has separate files with 46 | episodes. 47 | """ 48 | assert cls.check_config_paths_exist(config) 49 | dataset_dir = os.path.dirname(config.DATA_PATH.format(split=config.SPLIT)) 50 | 51 | cfg = config.clone() 52 | cfg.defrost() 53 | cfg.CONTENT_SCENES = [] 54 | dataset = cls(cfg) 55 | has_individual_scene_files = os.path.exists( 56 | dataset.content_scenes_path.split("{scene}")[0].format( 57 | data_path=dataset_dir 58 | ) 59 | ) 60 | if has_individual_scene_files: 61 | return cls._get_scenes_from_folder( 62 | content_scenes_path=dataset.content_scenes_path, 63 | dataset_dir=dataset_dir, 64 | ) 65 | else: 66 | # Load the full dataset, things are not split into separate files 67 | cfg.CONTENT_SCENES = [ALL_SCENES_MASK] 68 | dataset = cls(cfg) 69 | return list(map(cls.scene_from_scene_path, dataset.scene_ids)) 70 | 71 | @staticmethod 72 | def _get_scenes_from_folder(content_scenes_path, dataset_dir): 73 | scenes = [] 74 | content_dir = content_scenes_path.split("{scene}")[0] 75 | scene_dataset_ext = content_scenes_path.split("{scene}")[1] 76 | content_dir = content_dir.format(data_path=dataset_dir) 77 | if not os.path.exists(content_dir): 78 | return scenes 79 | 80 | for filename in os.listdir(content_dir): 81 | if filename.endswith(scene_dataset_ext): 82 | scene = filename[: -len(scene_dataset_ext)] 83 | scenes.append(scene) 84 | scenes.sort() 85 | return scenes 86 | 87 | def __init__(self, config: Optional[Config] = None) -> None: 88 | self.episodes = [] 89 | 90 | if config is None: 91 | return 92 | 93 | datasetfile_path = config.DATA_PATH.format(split=config.SPLIT) 94 | with gzip.open(datasetfile_path, "rt") as f: 95 | self.from_json(f.read(), scenes_dir=config.SCENES_DIR) 96 | 97 | # Read separate file for each scene 98 | dataset_dir = os.path.dirname(datasetfile_path) 99 | has_individual_scene_files = os.path.exists( 100 | self.content_scenes_path.split("{scene}")[0].format(data_path=dataset_dir) 101 | ) 102 | if has_individual_scene_files: 103 | scenes = config.CONTENT_SCENES 104 | if ALL_SCENES_MASK in scenes: 105 | scenes = self._get_scenes_from_folder( 106 | content_scenes_path=self.content_scenes_path, 107 | dataset_dir=dataset_dir, 108 | ) 109 | 110 | for scene in scenes: 111 | scene_filename = self.content_scenes_path.format( 112 | data_path=dataset_dir, scene=scene 113 | ) 114 | with gzip.open(scene_filename, "rt") as f: 115 | self.from_json(f.read(), scenes_dir=config.SCENES_DIR) 116 | 117 | else: 118 | self.episodes = list( 119 | filter(self.build_content_scenes_filter(config), self.episodes) 120 | ) 121 | 122 | def from_json(self, json_str: str, scenes_dir: Optional[str] = None) -> None: 123 | deserialized = json.loads(json_str) 124 | if CONTENT_SCENES_PATH_FIELD in deserialized: 125 | self.content_scenes_path = deserialized[CONTENT_SCENES_PATH_FIELD] 126 | 127 | for episode in deserialized["episodes"]: 128 | episode = NavigationEpisode(**episode) 129 | 130 | if scenes_dir is not None: 131 | if episode.scene_id.startswith(DEFAULT_SCENE_PATH_PREFIX): 132 | episode.scene_id = episode.scene_id[ 133 | len(DEFAULT_SCENE_PATH_PREFIX) : 134 | ] 135 | 136 | episode.scene_id = os.path.join(scenes_dir, episode.scene_id) 137 | 138 | for g_index, goal in enumerate(episode.goals): 139 | episode.goals[g_index] = NavigationGoal(**goal) 140 | if episode.shortest_paths is not None: 141 | for path in episode.shortest_paths: 142 | for p_index, point in enumerate(path): 143 | path[p_index] = ShortestPathPoint(**point) 144 | self.episodes.append(episode) 145 | -------------------------------------------------------------------------------- /habitat/datasets/pointnav/pointnav_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from typing import Optional 13 | 14 | import numpy as np 15 | 16 | from habitat.core.simulator import Simulator 17 | from habitat.datasets.utils import get_action_shortest_path 18 | from habitat.tasks.nav.nav import NavigationEpisode, NavigationGoal 19 | 20 | r"""A minimum radius of a plane that a point should be part of to be 21 | considered as a target or source location. Used to filter isolated points 22 | that aren't part of a floor. 23 | """ 24 | ISLAND_RADIUS_LIMIT = 1.5 25 | 26 | 27 | def _ratio_sample_rate(ratio: float, ratio_threshold: float) -> float: 28 | r"""Sampling function for aggressive filtering of straight-line 29 | episodes with shortest path geodesic distance to Euclid distance ratio 30 | threshold. 31 | 32 | :param ratio: geodesic distance ratio to Euclid distance 33 | :param ratio_threshold: geodesic shortest path to Euclid 34 | distance ratio upper limit till aggressive sampling is applied. 35 | :return: value between 0.008 and 0.144 for ratio [1, 1.1] 36 | """ 37 | assert ratio < ratio_threshold 38 | return 20 * (ratio - 0.98) ** 2 39 | 40 | 41 | def is_compatible_episode(s, t, sim, near_dist, far_dist, geodesic_to_euclid_ratio): 42 | euclid_dist = np.power(np.power(np.array(s) - np.array(t), 2).sum(0), 0.5) 43 | if np.abs(s[1] - t[1]) > 0.5: # check height difference to assure s and 44 | # t are from same floor 45 | return False, 0 46 | d_separation = sim.geodesic_distance(s, [t]) 47 | if d_separation == np.inf: 48 | return False, 0 49 | if not near_dist <= d_separation <= far_dist: 50 | return False, 0 51 | distances_ratio = d_separation / euclid_dist 52 | if distances_ratio < geodesic_to_euclid_ratio and ( 53 | np.random.rand() > _ratio_sample_rate(distances_ratio, geodesic_to_euclid_ratio) 54 | ): 55 | return False, 0 56 | if sim.island_radius(s) < ISLAND_RADIUS_LIMIT: 57 | return False, 0 58 | return True, d_separation 59 | 60 | 61 | def _create_episode( 62 | episode_id, 63 | scene_id, 64 | start_position, 65 | start_rotation, 66 | target_position, 67 | shortest_paths=None, 68 | radius=None, 69 | info=None, 70 | ) -> Optional[NavigationEpisode]: 71 | goals = [NavigationGoal(position=target_position, radius=radius)] 72 | return NavigationEpisode( 73 | episode_id=str(episode_id), 74 | goals=goals, 75 | scene_id=scene_id, 76 | start_position=start_position, 77 | start_rotation=start_rotation, 78 | shortest_paths=shortest_paths, 79 | info=info, 80 | ) 81 | 82 | 83 | def generate_pointnav_episode( 84 | sim: Simulator, 85 | num_episodes: int = -1, 86 | is_gen_shortest_path: bool = True, 87 | shortest_path_success_distance: float = 0.2, 88 | shortest_path_max_steps: int = 500, 89 | closest_dist_limit: float = 1, 90 | furthest_dist_limit: float = 30, 91 | geodesic_to_euclid_min_ratio: float = 1.1, 92 | number_retries_per_target: int = 10, 93 | ) -> NavigationEpisode: 94 | r"""Generator function that generates PointGoal navigation episodes. 95 | 96 | An episode is trivial if there is an obstacle-free, straight line between 97 | the start and goal positions. A good measure of the navigation 98 | complexity of an episode is the ratio of 99 | geodesic shortest path position to Euclidean distance between start and 100 | goal positions to the corresponding Euclidean distance. 101 | If the ratio is nearly 1, it indicates there are few obstacles, and the 102 | episode is easy; if the ratio is larger than 1, the 103 | episode is difficult because strategic navigation is required. 104 | To keep the navigation complexity of the precomputed episodes reasonably 105 | high, we perform aggressive rejection sampling for episodes with the above 106 | ratio falling in the range [1, 1.1]. 107 | Following this, there is a significant decrease in the number of 108 | straight-line episodes. 109 | 110 | 111 | :param sim: simulator with loaded scene for generation. 112 | :param num_episodes: number of episodes needed to generate 113 | :param is_gen_shortest_path: option to generate shortest paths 114 | :param shortest_path_success_distance: success distance when agent should 115 | stop during shortest path generation 116 | :param shortest_path_max_steps maximum number of steps shortest path 117 | expected to be 118 | :param closest_dist_limit episode geodesic distance lowest limit 119 | :param furthest_dist_limit episode geodesic distance highest limit 120 | :param geodesic_to_euclid_min_ratio geodesic shortest path to Euclid 121 | distance ratio upper limit till aggressive sampling is applied. 122 | :return: navigation episode that satisfy specified distribution for 123 | currently loaded into simulator scene. 124 | """ 125 | episode_count = 0 126 | while episode_count < num_episodes or num_episodes < 0: 127 | target_position = sim.sample_navigable_point() 128 | 129 | if sim.island_radius(target_position) < ISLAND_RADIUS_LIMIT: 130 | continue 131 | 132 | for retry in range(number_retries_per_target): 133 | source_position = sim.sample_navigable_point() 134 | 135 | is_compatible, dist = is_compatible_episode( 136 | source_position, 137 | target_position, 138 | sim, 139 | near_dist=closest_dist_limit, 140 | far_dist=furthest_dist_limit, 141 | geodesic_to_euclid_ratio=geodesic_to_euclid_min_ratio, 142 | ) 143 | if is_compatible: 144 | angle = np.random.uniform(0, 2 * np.pi) 145 | source_rotation = [0, np.sin(angle / 2), 0, np.cos(angle / 2)] 146 | 147 | shortest_paths = None 148 | if is_gen_shortest_path: 149 | shortest_paths = [ 150 | get_action_shortest_path( 151 | sim, 152 | source_position=source_position, 153 | source_rotation=source_rotation, 154 | goal_position=target_position, 155 | success_distance=shortest_path_success_distance, 156 | max_episode_steps=shortest_path_max_steps, 157 | ) 158 | ] 159 | 160 | episode = _create_episode( 161 | episode_id=episode_count, 162 | scene_id=sim.config.SCENE, 163 | start_position=source_position, 164 | start_rotation=source_rotation, 165 | target_position=target_position, 166 | shortest_paths=shortest_paths, 167 | radius=shortest_path_success_distance, 168 | info={"geodesic_distance": dist}, 169 | ) 170 | 171 | episode_count += 1 172 | yield episode 173 | -------------------------------------------------------------------------------- /habitat/datasets/registration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.logging import logger 13 | from habitat.core.registry import registry 14 | from habitat.datasets.multi_nav import _try_register_multinavdatasetv1 15 | from habitat.datasets.object_nav import _try_register_objectnavdatasetv1 16 | from habitat.datasets.pointnav import _try_register_pointnavdatasetv1 17 | 18 | 19 | def make_dataset(id_dataset, **kwargs): 20 | logger.info("Initializing dataset {}".format(id_dataset)) 21 | _dataset = registry.get_dataset(id_dataset) 22 | assert _dataset is not None, "Could not find dataset {}".format(id_dataset) 23 | 24 | return _dataset(**kwargs) 25 | 26 | 27 | _try_register_multinavdatasetv1() 28 | _try_register_objectnavdatasetv1() 29 | _try_register_pointnavdatasetv1() 30 | -------------------------------------------------------------------------------- /habitat/datasets/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import re 13 | from collections import Counter 14 | from typing import List 15 | 16 | from habitat.core.logging import logger 17 | from habitat.core.simulator import ShortestPathPoint 18 | from habitat.tasks.nav.shortest_path_follower import ShortestPathFollower 19 | from habitat.utils.geometry_utils import quaternion_to_list 20 | 21 | SENTENCE_SPLIT_REGEX = re.compile(r"([^\w-]+)") 22 | 23 | """ 24 | Tokenize and vocabulary utils originally authored by @apsdehal and are 25 | taken from Pythia. 26 | """ 27 | 28 | 29 | def tokenize(sentence, regex=SENTENCE_SPLIT_REGEX, keep=["'s"], remove=[",", "?"]): 30 | sentence = sentence.lower() 31 | 32 | for token in keep: 33 | sentence = sentence.replace(token, " " + token) 34 | 35 | for token in remove: 36 | sentence = sentence.replace(token, "") 37 | 38 | tokens = regex.split(sentence) 39 | tokens = [t.strip() for t in tokens if len(t.strip()) > 0] 40 | return tokens 41 | 42 | 43 | def load_str_list(fname): 44 | with open(fname) as f: 45 | lines = f.readlines() 46 | lines = [l.strip() for l in lines] 47 | return lines 48 | 49 | 50 | class VocabDict: 51 | UNK_TOKEN = "" 52 | PAD_TOKEN = "" 53 | START_TOKEN = "" 54 | END_TOKEN = "" 55 | 56 | def __init__(self, word_list=None, filepath=None): 57 | if word_list is not None: 58 | self.word_list = word_list 59 | self._build() 60 | 61 | elif filepath: 62 | self.word_list = load_str_list(filepath) 63 | self._build() 64 | 65 | def _build(self): 66 | if self.UNK_TOKEN not in self.word_list: 67 | self.word_list = [self.UNK_TOKEN] + self.word_list 68 | 69 | self.word2idx_dict = {w: n_w for n_w, w in enumerate(self.word_list)} 70 | 71 | # String (word) to integer (index) dict mapping 72 | self.stoi = self.word2idx_dict 73 | # Integer to string (word) reverse mapping 74 | self.itos = self.word_list 75 | self.num_vocab = len(self.word_list) 76 | 77 | self.UNK_INDEX = ( 78 | self.word2idx_dict[self.UNK_TOKEN] 79 | if self.UNK_TOKEN in self.word2idx_dict 80 | else None 81 | ) 82 | 83 | self.PAD_INDEX = ( 84 | self.word2idx_dict[self.PAD_TOKEN] 85 | if self.PAD_TOKEN in self.word2idx_dict 86 | else None 87 | ) 88 | 89 | def idx2word(self, n_w): 90 | return self.word_list[n_w] 91 | 92 | def __len__(self): 93 | return len(self.word_list) 94 | 95 | def get_size(self): 96 | return len(self.word_list) 97 | 98 | def get_unk_index(self): 99 | return self.UNK_INDEX 100 | 101 | def get_unk_token(self): 102 | return self.UNK_TOKEN 103 | 104 | def word2idx(self, w): 105 | if w in self.word2idx_dict: 106 | return self.word2idx_dict[w] 107 | elif self.UNK_INDEX is not None: 108 | return self.UNK_INDEX 109 | else: 110 | raise ValueError( 111 | "word %s not in dictionary \ 112 | (while dictionary does not contain )" 113 | % w 114 | ) 115 | 116 | def tokenize_and_index( 117 | self, 118 | sentence, 119 | regex=SENTENCE_SPLIT_REGEX, 120 | keep=["'s"], 121 | remove=[",", "?"], 122 | ) -> List[int]: 123 | inds = [ 124 | self.word2idx(w) 125 | for w in tokenize(sentence, regex=regex, keep=keep, remove=remove) 126 | ] 127 | return inds 128 | 129 | 130 | class VocabFromText(VocabDict): 131 | DEFAULT_TOKENS = [ 132 | VocabDict.PAD_TOKEN, 133 | VocabDict.UNK_TOKEN, 134 | VocabDict.START_TOKEN, 135 | VocabDict.END_TOKEN, 136 | ] 137 | 138 | def __init__( 139 | self, 140 | sentences, 141 | min_count=1, 142 | regex=SENTENCE_SPLIT_REGEX, 143 | keep=[], 144 | remove=[], 145 | only_unk_extra=False, 146 | ): 147 | token_counter = Counter() 148 | 149 | for sentence in sentences: 150 | tokens = tokenize(sentence, regex=regex, keep=keep, remove=remove) 151 | token_counter.update(tokens) 152 | 153 | token_list = [] 154 | for token in token_counter: 155 | if token_counter[token] >= min_count: 156 | token_list.append(token) 157 | 158 | extras = self.DEFAULT_TOKENS 159 | 160 | if only_unk_extra: 161 | extras = [self.UNK_TOKEN] 162 | 163 | self.word_list = extras + token_list 164 | self._build() 165 | 166 | 167 | def get_action_shortest_path( 168 | sim, 169 | source_position, 170 | source_rotation, 171 | goal_position, 172 | success_distance=0.05, 173 | max_episode_steps=500, 174 | shortest_path_mode="exact_gradient", 175 | ) -> List[ShortestPathPoint]: 176 | sim.reset() 177 | sim.set_agent_state(source_position, source_rotation) 178 | follower = ShortestPathFollower(sim, success_distance, False) 179 | follower.mode = shortest_path_mode 180 | 181 | shortest_path = [] 182 | step_count = 0 183 | action = follower.get_next_action(goal_position) 184 | while action is not None and step_count < max_episode_steps: 185 | state = sim.get_agent_state() 186 | shortest_path.append( 187 | ShortestPathPoint( 188 | state.position.tolist(), 189 | quaternion_to_list(state.rotation), 190 | action, 191 | ) 192 | ) 193 | sim.step(action) 194 | step_count += 1 195 | action = follower.get_next_action(goal_position) 196 | 197 | if step_count == max_episode_steps: 198 | logger.warning("Shortest path wasn't found.") 199 | return shortest_path 200 | -------------------------------------------------------------------------------- /habitat/py.typed: -------------------------------------------------------------------------------- 1 | # Marker file for PEP 561. This tells mypy that the package uses inline types. 2 | -------------------------------------------------------------------------------- /habitat/sims/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.sims.registration import make_sim 13 | -------------------------------------------------------------------------------- /habitat/sims/habitat_simulator/__init__.py: -------------------------------------------------------------------------------- 1 | from habitat.core.registry import registry 2 | from habitat.core.simulator import Simulator 3 | 4 | 5 | def _try_register_habitat_sim(): 6 | try: 7 | import habitat_sim 8 | 9 | has_habitat_sim = True 10 | except ImportError as e: 11 | has_habitat_sim = False 12 | habitat_sim_import_error = e 13 | 14 | if has_habitat_sim: 15 | from habitat.sims.habitat_simulator.habitat_simulator import HabitatSim 16 | from habitat.sims.habitat_simulator.actions import ( 17 | HabitatSimV1ActionSpaceConfiguration, 18 | ) 19 | else: 20 | 21 | @registry.register_simulator(name="Sim-v0") 22 | class HabitatSimImportError(Simulator): 23 | def __init__(self, *args, **kwargs): 24 | raise habitat_sim_import_error 25 | -------------------------------------------------------------------------------- /habitat/sims/habitat_simulator/actions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from enum import Enum 13 | from typing import Dict 14 | 15 | import attr 16 | 17 | import habitat_sim 18 | from habitat.core.registry import registry 19 | from habitat.core.simulator import ActionSpaceConfiguration, Config 20 | from habitat.core.utils import Singleton 21 | 22 | 23 | class _DefaultHabitatSimActions(Enum): 24 | STOP = 0 25 | MOVE_FORWARD = 1 26 | TURN_LEFT = 2 27 | TURN_RIGHT = 3 28 | LOOK_UP = 4 29 | LOOK_DOWN = 5 30 | FOUND = 6 31 | 32 | 33 | @attr.s(auto_attribs=True, slots=True) 34 | class HabitatSimActionsSingleton(metaclass=Singleton): 35 | r"""Implements an extendable Enum for the mapping of action names 36 | to their integer values. 37 | 38 | This means that new action names can be added, but old action names cannot 39 | be removed nor can their mapping be altered. This also ensures that all 40 | actions are always contigously mapped in :py:`[0, len(HabitatSimActions) - 1]` 41 | 42 | This accesible as the global singleton `HabitatSimActions` 43 | """ 44 | 45 | _known_actions: Dict[str, int] = attr.ib(init=False, factory=dict) 46 | 47 | def __attrs_post_init__(self): 48 | for action in _DefaultHabitatSimActions: 49 | self._known_actions[action.name] = action.value 50 | 51 | def extend_action_space(self, name: str) -> int: 52 | r"""Extends the action space to accomodate a new action with 53 | the name :p:`name` 54 | 55 | :param name: The name of the new action 56 | :return: The number the action is registered on 57 | 58 | Usage: 59 | 60 | .. code:: py 61 | 62 | from habitat.sims.habitat_simulator.actions import HabitatSimActions 63 | HabitatSimActions.extend_action_space("MY_ACTION") 64 | print(HabitatSimActions.MY_ACTION) 65 | """ 66 | assert name not in self._known_actions, "Cannot register an action name twice" 67 | self._known_actions[name] = len(self._known_actions) 68 | 69 | return self._known_actions[name] 70 | 71 | def has_action(self, name: str) -> bool: 72 | r"""Checks to see if action :p:`name` is already register 73 | 74 | :param name: The name to check 75 | :return: Whether or not :p:`name` already exists 76 | """ 77 | 78 | return name in self._known_actions 79 | 80 | def __getattr__(self, name): 81 | return self._known_actions[name] 82 | 83 | def __getitem__(self, name): 84 | return self._known_actions[name] 85 | 86 | def __len__(self): 87 | return len(self._known_actions) 88 | 89 | def __iter__(self): 90 | return iter(self._known_actions) 91 | 92 | 93 | HabitatSimActions: HabitatSimActionsSingleton = HabitatSimActionsSingleton() 94 | 95 | 96 | @registry.register_action_space_configuration(name="v0") 97 | class HabitatSimV0ActionSpaceConfiguration(ActionSpaceConfiguration): 98 | def get(self): 99 | return { 100 | HabitatSimActions.STOP: habitat_sim.ActionSpec("stop"), 101 | HabitatSimActions.MOVE_FORWARD: habitat_sim.ActionSpec( 102 | "move_forward", 103 | habitat_sim.ActuationSpec(amount=self.config.FORWARD_STEP_SIZE), 104 | ), 105 | HabitatSimActions.TURN_LEFT: habitat_sim.ActionSpec( 106 | "turn_left", 107 | habitat_sim.ActuationSpec(amount=self.config.TURN_ANGLE), 108 | ), 109 | HabitatSimActions.TURN_RIGHT: habitat_sim.ActionSpec( 110 | "turn_right", 111 | habitat_sim.ActuationSpec(amount=self.config.TURN_ANGLE), 112 | ), 113 | } 114 | 115 | 116 | @registry.register_action_space_configuration(name="v1") 117 | class HabitatSimV1ActionSpaceConfiguration(HabitatSimV0ActionSpaceConfiguration): 118 | def get(self): 119 | config = super().get() 120 | new_config = { 121 | HabitatSimActions.LOOK_UP: habitat_sim.ActionSpec( 122 | "look_up", 123 | habitat_sim.ActuationSpec(amount=self.config.TILT_ANGLE), 124 | ), 125 | HabitatSimActions.LOOK_DOWN: habitat_sim.ActionSpec( 126 | "look_down", 127 | habitat_sim.ActuationSpec(amount=self.config.TILT_ANGLE), 128 | ), 129 | } 130 | 131 | config.update(new_config) 132 | 133 | return config 134 | 135 | 136 | @registry.register_action_space_configuration(name="pyrobotnoisy") 137 | class HabitatSimPyRobotActionSpaceConfiguration(ActionSpaceConfiguration): 138 | def get(self): 139 | return { 140 | HabitatSimActions.STOP: habitat_sim.ActionSpec("stop"), 141 | HabitatSimActions.MOVE_FORWARD: habitat_sim.ActionSpec( 142 | "pyrobot_noisy_move_forward", 143 | habitat_sim.PyRobotNoisyActuationSpec( 144 | amount=self.config.FORWARD_STEP_SIZE, 145 | robot=self.config.NOISE_MODEL.ROBOT, 146 | controller=self.config.NOISE_MODEL.CONTROLLER, 147 | noise_multiplier=self.config.NOISE_MODEL.NOISE_MULTIPLIER, 148 | ), 149 | ), 150 | HabitatSimActions.TURN_LEFT: habitat_sim.ActionSpec( 151 | "pyrobot_noisy_turn_left", 152 | habitat_sim.PyRobotNoisyActuationSpec( 153 | amount=self.config.TURN_ANGLE, 154 | robot=self.config.NOISE_MODEL.ROBOT, 155 | controller=self.config.NOISE_MODEL.CONTROLLER, 156 | noise_multiplier=self.config.NOISE_MODEL.NOISE_MULTIPLIER, 157 | ), 158 | ), 159 | HabitatSimActions.TURN_RIGHT: habitat_sim.ActionSpec( 160 | "pyrobot_noisy_turn_right", 161 | habitat_sim.PyRobotNoisyActuationSpec( 162 | amount=self.config.TURN_ANGLE, 163 | robot=self.config.NOISE_MODEL.ROBOT, 164 | controller=self.config.NOISE_MODEL.CONTROLLER, 165 | noise_multiplier=self.config.NOISE_MODEL.NOISE_MULTIPLIER, 166 | ), 167 | ), 168 | HabitatSimActions.LOOK_UP: habitat_sim.ActionSpec( 169 | "look_up", 170 | habitat_sim.ActuationSpec(amount=self.config.TILT_ANGLE), 171 | ), 172 | HabitatSimActions.LOOK_DOWN: habitat_sim.ActionSpec( 173 | "look_down", 174 | habitat_sim.ActuationSpec(amount=self.config.TILT_ANGLE), 175 | ), 176 | } 177 | -------------------------------------------------------------------------------- /habitat/sims/pyrobot/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.registry import registry 13 | from habitat.core.simulator import Simulator 14 | 15 | 16 | def _try_register_pyrobot(): 17 | try: 18 | import pyrobot 19 | 20 | has_pyrobot = True 21 | except ImportError as e: 22 | has_pyrobot = False 23 | pyrobot_import_error = e 24 | 25 | if has_pyrobot: 26 | from habitat.sims.pyrobot.pyrobot import PyRobot 27 | else: 28 | 29 | @registry.register_simulator(name="PyRobot-v0") 30 | class PyRobotImportError(Simulator): 31 | def __init__(self, *args, **kwargs): 32 | raise pyrobot_import_error 33 | -------------------------------------------------------------------------------- /habitat/sims/pyrobot/pyrobot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from typing import Any 13 | 14 | import numpy as np 15 | import pyrobot 16 | from gym import Space, spaces 17 | 18 | from habitat.core.registry import registry 19 | from habitat.core.simulator import ( 20 | BumpSensor, 21 | Config, 22 | DepthSensor, 23 | RGBSensor, 24 | SensorSuite, 25 | Simulator, 26 | ) 27 | from habitat.core.utils import center_crop, try_cv2_import 28 | 29 | cv2 = try_cv2_import() 30 | 31 | 32 | def _locobot_base_action_space(): 33 | return spaces.Dict( 34 | { 35 | "go_to_relative": spaces.Box(low=-np.inf, high=np.inf, shape=(3,)), 36 | "go_to_absolute": spaces.Box(low=-np.inf, high=np.inf, shape=(3,)), 37 | } 38 | ) 39 | 40 | 41 | def _locobot_camera_action_space(): 42 | return spaces.Dict( 43 | { 44 | "set_pan": spaces.Box(low=-np.inf, high=np.inf, shape=(1,)), 45 | "set_tilt": spaces.Box(low=-np.inf, high=np.inf, shape=(1,)), 46 | "set_pan_tilt": spaces.Box(low=-np.inf, high=np.inf, shape=(2,)), 47 | } 48 | ) 49 | 50 | 51 | def _resize_observation(obs, observation_space, config): 52 | if obs.shape != observation_space.shape: 53 | if ( 54 | config.CENTER_CROP is True 55 | and obs.shape[0] > observation_space.shape[0] 56 | and obs.shape[1] > observation_space.shape[1] 57 | ): 58 | obs = center_crop(obs, observation_space) 59 | 60 | else: 61 | obs = cv2.resize( 62 | obs, (observation_space.shape[1], observation_space.shape[0]) 63 | ) 64 | return obs 65 | 66 | 67 | MM_IN_METER = 1000 # millimeters in a meter 68 | ACTION_SPACES = { 69 | "LOCOBOT": { 70 | "BASE_ACTIONS": _locobot_base_action_space(), 71 | "CAMERA_ACTIONS": _locobot_camera_action_space(), 72 | } 73 | } 74 | 75 | 76 | @registry.register_sensor 77 | class PyRobotRGBSensor(RGBSensor): 78 | def __init__(self, config): 79 | super().__init__(config=config) 80 | 81 | def _get_observation_space(self, *args: Any, **kwargs: Any): 82 | return spaces.Box( 83 | low=0, 84 | high=255, 85 | shape=(self.config.HEIGHT, self.config.WIDTH, 3), 86 | dtype=np.uint8, 87 | ) 88 | 89 | def get_observation(self, robot_obs, *args: Any, **kwargs: Any): 90 | obs = robot_obs.get(self.uuid, None) 91 | 92 | assert obs is not None, "Invalid observation for {} sensor".format(self.uuid) 93 | 94 | obs = _resize_observation(obs, self.observation_space, self.config) 95 | 96 | return obs 97 | 98 | 99 | @registry.register_sensor 100 | class PyRobotDepthSensor(DepthSensor): 101 | min_depth_value: float 102 | max_depth_value: float 103 | 104 | def __init__(self, config): 105 | if config.NORMALIZE_DEPTH: 106 | self.min_depth_value = 0 107 | self.max_depth_value = 1 108 | else: 109 | self.min_depth_value = config.MIN_DEPTH 110 | self.max_depth_value = config.MAX_DEPTH 111 | 112 | super().__init__(config=config) 113 | 114 | def _get_observation_space(self, *args: Any, **kwargs: Any): 115 | return spaces.Box( 116 | low=self.min_depth_value, 117 | high=self.max_depth_value, 118 | shape=(self.config.HEIGHT, self.config.WIDTH, 1), 119 | dtype=np.float32, 120 | ) 121 | 122 | def get_observation(self, robot_obs, *args: Any, **kwargs: Any): 123 | obs = robot_obs.get(self.uuid, None) 124 | 125 | assert obs is not None, "Invalid observation for {} sensor".format(self.uuid) 126 | 127 | obs = _resize_observation(obs, self.observation_space, self.config) 128 | 129 | obs = obs / MM_IN_METER # convert from mm to m 130 | 131 | obs = np.clip(obs, self.config.MIN_DEPTH, self.config.MAX_DEPTH) 132 | if self.config.NORMALIZE_DEPTH: 133 | # normalize depth observations to [0, 1] 134 | obs = (obs - self.config.MIN_DEPTH) / ( 135 | self.config.MAX_DEPTH - self.config.MIN_DEPTH 136 | ) 137 | 138 | obs = np.expand_dims(obs, axis=2) # make depth observations a 3D array 139 | 140 | return obs 141 | 142 | 143 | @registry.register_sensor 144 | class PyRobotBumpSensor(BumpSensor): 145 | def _get_observation_space(self, *args: Any, **kwargs: Any): 146 | return spaces.Box(low=False, high=True, shape=(1,), dtype=np.bool) 147 | 148 | def get_observation(self, robot_obs, *args: Any, **kwargs: Any): 149 | return np.array(robot_obs["bump"]) 150 | 151 | 152 | @registry.register_simulator(name="PyRobot-v0") 153 | class PyRobot(Simulator): 154 | r"""Simulator wrapper over PyRobot. 155 | 156 | PyRobot repo: https://github.com/facebookresearch/pyrobot 157 | To use this abstraction the user will have to setup PyRobot 158 | python3 version. Please refer to the PyRobot repository 159 | for setting it up. The user will also have to export a 160 | ROS_PATH environment variable to use this integration, 161 | please refer to `habitat.core.utils.try_cv2_import` for 162 | more details on this. 163 | 164 | This abstraction assumes that reality is a simulation 165 | (https://www.youtube.com/watch?v=tlTKTTt47WE). 166 | 167 | Args: 168 | config: configuration for initializing the PyRobot object. 169 | """ 170 | 171 | def __init__(self, config: Config) -> None: 172 | self._config = config 173 | 174 | robot_sensors = [] 175 | for sensor_name in self._config.SENSORS: 176 | sensor_cfg = getattr(self._config, sensor_name) 177 | sensor_type = registry.get_sensor(sensor_cfg.TYPE) 178 | 179 | assert sensor_type is not None, "invalid sensor type {}".format( 180 | sensor_cfg.TYPE 181 | ) 182 | robot_sensors.append(sensor_type(sensor_cfg)) 183 | self._sensor_suite = SensorSuite(robot_sensors) 184 | 185 | config_pyrobot = { 186 | "base_controller": self._config.BASE_CONTROLLER, 187 | "base_planner": self._config.BASE_PLANNER, 188 | } 189 | 190 | assert ( 191 | self._config.ROBOT in self._config.ROBOTS 192 | ), "Invalid robot type {}".format(self._config.ROBOT) 193 | self._robot_config = getattr(self._config, self._config.ROBOT.upper()) 194 | 195 | action_spaces_dict = {} 196 | 197 | self._action_space = self._robot_action_space( 198 | self._config.ROBOT, self._robot_config 199 | ) 200 | 201 | self._robot = pyrobot.Robot(self._config.ROBOT, base_config=config_pyrobot) 202 | 203 | def get_robot_observations(self): 204 | return { 205 | "rgb": self._robot.camera.get_rgb(), 206 | "depth": self._robot.camera.get_depth(), 207 | "bump": self._robot.base.base_state.bumper, 208 | } 209 | 210 | @property 211 | def sensor_suite(self) -> SensorSuite: 212 | return self._sensor_suite 213 | 214 | @property 215 | def base(self): 216 | return self._robot.base 217 | 218 | @property 219 | def camera(self): 220 | return self._robot.camera 221 | 222 | def _robot_action_space(self, robot_type, robot_config): 223 | action_spaces_dict = {} 224 | for action in robot_config.ACTIONS: 225 | action_spaces_dict[action] = ACTION_SPACES[robot_type.upper()][action] 226 | return spaces.Dict(action_spaces_dict) 227 | 228 | @property 229 | def action_space(self) -> Space: 230 | return self._action_space 231 | 232 | def reset(self): 233 | self._robot.camera.reset() 234 | 235 | observations = self._sensor_suite.get_observations( 236 | robot_obs=self.get_robot_observations() 237 | ) 238 | return observations 239 | 240 | def step(self, action, action_params): 241 | r"""Step in reality. Currently the supported 242 | actions are the ones defined in `_locobot_base_action_space` 243 | and `_locobot_camera_action_space`. For details on how 244 | to use these actions please refer to the documentation 245 | of namesake methods in PyRobot 246 | (https://github.com/facebookresearch/pyrobot). 247 | """ 248 | if action in self._robot_config.BASE_ACTIONS: 249 | getattr(self._robot.base, action)(**action_params) 250 | elif action in self._robot_config.CAMERA_ACTIONS: 251 | getattr(self._robot.camera, action)(**action_params) 252 | else: 253 | raise ValueError("Invalid action {}".format(action)) 254 | 255 | observations = self._sensor_suite.get_observations( 256 | robot_obs=self.get_robot_observations() 257 | ) 258 | 259 | return observations 260 | 261 | def render(self, mode: str = "rgb") -> Any: 262 | observations = self._sensor_suite.get_observations( 263 | robot_obs=self.get_robot_observations() 264 | ) 265 | 266 | output = observations.get(mode) 267 | assert output is not None, "mode {} sensor is not active".format(mode) 268 | 269 | return output 270 | 271 | def get_agent_state(self, agent_id: int = 0, base_state_type: str = "odom"): 272 | assert agent_id == 0, "No support of multi agent in {} yet.".format( 273 | self.__class__.__name__ 274 | ) 275 | state = { 276 | "base": self._robot.base.get_state(base_state_type), 277 | "camera": self._robot.camera.get_state(), 278 | } 279 | return state 280 | 281 | def seed(self, seed: int) -> None: 282 | raise NotImplementedError("No support for seeding in reality") 283 | -------------------------------------------------------------------------------- /habitat/sims/registration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.logging import logger 13 | from habitat.core.registry import registry 14 | from habitat.sims.habitat_simulator import _try_register_habitat_sim 15 | from habitat.sims.pyrobot import _try_register_pyrobot 16 | 17 | 18 | def make_sim(id_sim, **kwargs): 19 | logger.info("initializing sim {}".format(id_sim)) 20 | _sim = registry.get_simulator(id_sim) 21 | assert _sim is not None, "Could not find simulator with name {}".format(id_sim) 22 | return _sim(**kwargs) 23 | 24 | 25 | _try_register_habitat_sim() 26 | _try_register_pyrobot() 27 | -------------------------------------------------------------------------------- /habitat/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.tasks.registration import make_task 13 | -------------------------------------------------------------------------------- /habitat/tasks/nav/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.embodied_task import EmbodiedTask 13 | from habitat.core.registry import registry 14 | 15 | 16 | def _try_register_nav_task(): 17 | try: 18 | from habitat.tasks.nav.nav import NavigationTask 19 | 20 | has_navtask = True 21 | except ImportError as e: 22 | has_navtask = False 23 | navtask_import_error = e 24 | 25 | if has_navtask: 26 | from habitat.tasks.nav.nav import NavigationTask 27 | else: 28 | 29 | @registry.register_task(name="Nav-v0") 30 | class NavigationTaskImportError(EmbodiedTask): 31 | def __init__(self, *args, **kwargs): 32 | raise navtask_import_error 33 | -------------------------------------------------------------------------------- /habitat/tasks/nav/multi_nav_task.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import os 13 | from typing import Any, List, Optional 14 | 15 | import attr 16 | import numpy as np 17 | from gym import spaces 18 | 19 | from habitat.config import Config 20 | from habitat.core.dataset import Dataset 21 | from habitat.core.logging import logger 22 | from habitat.core.registry import registry 23 | from habitat.core.simulator import AgentState, Sensor, SensorTypes, Simulator 24 | from habitat.core.utils import not_none_validator 25 | from habitat.tasks.nav.nav import ( 26 | NavigationEpisode, 27 | NavigationGoal, 28 | NavigationTask, 29 | ) 30 | 31 | 32 | @attr.s(auto_attribs=True, kw_only=True) 33 | class MultiGoalNavEpisode(NavigationEpisode): 34 | r"""Multi ObjectGoal Navigation Episode 35 | 36 | :param object_category: Category of the obect 37 | """ 38 | object_category: Optional[List[str]] = None 39 | object_index: Optional[int] 40 | currGoalIndex: Optional[int] = 0 41 | 42 | @property 43 | def goals_key(self) -> str: 44 | r"""The key to retrieve the goals""" 45 | return [f"{os.path.basename(self.scene_id)}_{i}" for i in self.object_category] 46 | 47 | 48 | @attr.s(auto_attribs=True) 49 | class ObjectViewLocation: 50 | r"""ObjectViewLocation provides information about a position around an object goal 51 | usually that is navigable and the object is visible with specific agent 52 | configuration that episode's dataset was created. 53 | that is target for 54 | navigation. That can be specify object_id, position and object 55 | category. An important part for metrics calculation are view points that 56 | describe success area for the navigation. 57 | 58 | Args: 59 | agent_state: navigable AgentState with a position and a rotation where 60 | the object is visible. 61 | iou: an intersection of a union of the object and a rectangle in the 62 | center of view. This metric is used to evaluate how good is the object 63 | view form current position. Higher iou means better view, iou equals 64 | 1.0 if whole object is inside of the rectangle and no pixel inside 65 | the rectangle belongs to anything except the object. 66 | """ 67 | agent_state: AgentState 68 | iou: Optional[float] 69 | 70 | 71 | @attr.s(auto_attribs=True, kw_only=True) 72 | class ObjectGoal(NavigationGoal): 73 | r"""Object goal provides information about an object that is target for 74 | navigation. That can be specify object_id, position and object 75 | category. An important part for metrics calculation are view points that 76 | describe success area for the navigation. 77 | 78 | Args: 79 | object_id: id that can be used to retrieve object from the semantic 80 | scene annotation 81 | object_name: name of the object 82 | object_category: object category name usually similar to scene semantic 83 | categories 84 | room_id: id of a room where object is located, can be used to retrieve 85 | room from the semantic scene annotation 86 | room_name: name of the room, where object is located 87 | view_points: navigable positions around the object with specified 88 | proximity of the object surface used for navigation metrics calculation. 89 | The object is visible from these positions. 90 | """ 91 | 92 | object_id: str = attr.ib(default=None, validator=not_none_validator) 93 | object_name: Optional[str] = None 94 | object_category: Optional[str] = None 95 | room_id: Optional[str] = None 96 | room_name: Optional[str] = None 97 | view_points: Optional[List[ObjectViewLocation]] = None 98 | 99 | 100 | @attr.s(auto_attribs=True, kw_only=True) 101 | class MultiGoal(NavigationGoal): 102 | r"""Object goal provides information about an object that is target for 103 | navigation. That can be specify object_id, position and object 104 | category. An important part for metrics calculation are view points that 105 | describe success area for the navigation. 106 | Args: 107 | object_id: id that can be used to retrieve object from the semantic 108 | scene annotation 109 | object_name: name of the object 110 | object_category: object category name usually similar to scene semantic 111 | categories 112 | room_id: id of a room where object is located, can be used to retrieve 113 | room from the semantic scene annotation 114 | room_name: name of the room, where object is located 115 | view_points: navigable positions around the object with specified 116 | proximity of the object surface used for navigation metrics calculation. 117 | The object is visible from these positions. 118 | """ 119 | 120 | object_category: Optional[str] = None 121 | room_id: Optional[str] = None 122 | room_name: Optional[str] = None 123 | position: Optional[List[List[float]]] 124 | 125 | 126 | @registry.register_sensor 127 | class MultiGoalSensor(Sensor): 128 | r"""A sensor for Object Goal specification as observations which is used in 129 | ObjectGoal Navigation. The goal is expected to be specified by object_id or 130 | semantic category id. 131 | For the agent in simulator the forward direction is along negative-z. 132 | In polar coordinate format the angle returned is azimuth to the goal. 133 | Args: 134 | sim: a reference to the simulator for calculating task observations. 135 | config: a config for the ObjectGoalSensor sensor. Can contain field 136 | GOAL_SPEC that specifies which id use for goal specification, 137 | GOAL_SPEC_MAX_VAL the maximum object_id possible used for 138 | observation space definition. 139 | dataset: a Object Goal navigation dataset that contains dictionaries 140 | of categories id to text mapping. 141 | """ 142 | 143 | def __init__( 144 | self, sim, config: Config, dataset: Dataset, *args: Any, **kwargs: Any 145 | ): 146 | self._sim = sim 147 | self._dataset = dataset 148 | super().__init__(config=config) 149 | 150 | def _get_uuid(self, *args: Any, **kwargs: Any): 151 | return "multiobjectgoal" 152 | 153 | def _get_sensor_type(self, *args: Any, **kwargs: Any): 154 | return SensorTypes.SEMANTIC 155 | 156 | def _get_observation_space(self, *args: Any, **kwargs: Any): 157 | sensor_shape = (1,) 158 | max_value = (self.config.GOAL_SPEC_MAX_VAL - 1,) 159 | if self.config.GOAL_SPEC == "TASK_CATEGORY_ID": 160 | max_value = max(self._dataset.category_to_task_category_id.values()) 161 | 162 | return spaces.Box(low=0, high=max_value, shape=sensor_shape, dtype=np.int64) 163 | 164 | def get_observation( 165 | self, 166 | observations, 167 | *args: Any, 168 | episode: MultiGoalNavEpisode, 169 | **kwargs: Any, 170 | ) -> Optional[int]: 171 | if self.config.GOAL_SPEC == "TASK_CATEGORY_ID": 172 | if len(episode.goals) == 0: 173 | logger.error(f"No goal specified for episode {episode.episode_id}.") 174 | return None 175 | category_name = [i.object_category for i in episode.goals] 176 | goalArray = np.array( 177 | [self._dataset.category_to_task_category_id[i] for i in category_name], 178 | dtype=np.int64, 179 | ) 180 | return goalArray[ 181 | kwargs["task"].currGoalIndex : kwargs["task"].currGoalIndex + 1 182 | ] 183 | elif self.config.GOAL_SPEC == "OBJECT_ID": 184 | return np.array([episode.goals[0].object_name_id], dtype=np.int64) 185 | else: 186 | raise RuntimeError("Wrong GOAL_SPEC specified for ObjectGoalSensor.") 187 | 188 | 189 | @registry.register_task(name="MultiNav-v1") 190 | class MultiNavigationTask(NavigationTask): 191 | r"""An Object Navigation Task class for a task specific methods. 192 | Used to explicitly state a type of the task in config. 193 | """ 194 | 195 | def __init__( 196 | self, config: Config, sim: Simulator, dataset: Optional[Dataset] = None 197 | ) -> None: 198 | super().__init__(config=config, sim=sim, dataset=dataset) 199 | self.currGoalIndex = 0 200 | -------------------------------------------------------------------------------- /habitat/tasks/nav/object_nav_task.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import os 13 | from typing import Any, List, Optional 14 | 15 | import attr 16 | import numpy as np 17 | from gym import spaces 18 | 19 | from habitat.config import Config 20 | from habitat.core.dataset import Dataset 21 | from habitat.core.logging import logger 22 | from habitat.core.registry import registry 23 | from habitat.core.simulator import AgentState, Sensor, SensorTypes 24 | from habitat.core.utils import not_none_validator 25 | from habitat.tasks.nav.nav import ( 26 | NavigationEpisode, 27 | NavigationGoal, 28 | NavigationTask, 29 | ) 30 | 31 | 32 | @attr.s(auto_attribs=True, kw_only=True) 33 | class ObjectGoalNavEpisode(NavigationEpisode): 34 | r"""ObjectGoal Navigation Episode 35 | 36 | :param object_category: Category of the obect 37 | """ 38 | object_category: Optional[str] = None 39 | 40 | @property 41 | def goals_key(self) -> str: 42 | r"""The key to retrieve the goals""" 43 | return f"{os.path.basename(self.scene_id)}_{self.object_category}" 44 | 45 | 46 | @attr.s(auto_attribs=True) 47 | class ObjectViewLocation: 48 | r"""ObjectViewLocation provides information about a position around an object goal 49 | usually that is navigable and the object is visible with specific agent 50 | configuration that episode's dataset was created. 51 | that is target for 52 | navigation. That can be specify object_id, position and object 53 | category. An important part for metrics calculation are view points that 54 | describe success area for the navigation. 55 | 56 | Args: 57 | agent_state: navigable AgentState with a position and a rotation where 58 | the object is visible. 59 | iou: an intersection of a union of the object and a rectangle in the 60 | center of view. This metric is used to evaluate how good is the object 61 | view form current position. Higher iou means better view, iou equals 62 | 1.0 if whole object is inside of the rectangle and no pixel inside 63 | the rectangle belongs to anything except the object. 64 | """ 65 | agent_state: AgentState 66 | iou: Optional[float] 67 | 68 | 69 | @attr.s(auto_attribs=True, kw_only=True) 70 | class ObjectGoal(NavigationGoal): 71 | r"""Object goal provides information about an object that is target for 72 | navigation. That can be specify object_id, position and object 73 | category. An important part for metrics calculation are view points that 74 | describe success area for the navigation. 75 | 76 | Args: 77 | object_id: id that can be used to retrieve object from the semantic 78 | scene annotation 79 | object_name: name of the object 80 | object_category: object category name usually similar to scene semantic 81 | categories 82 | room_id: id of a room where object is located, can be used to retrieve 83 | room from the semantic scene annotation 84 | room_name: name of the room, where object is located 85 | view_points: navigable positions around the object with specified 86 | proximity of the object surface used for navigation metrics calculation. 87 | The object is visible from these positions. 88 | """ 89 | 90 | object_id: str = attr.ib(default=None, validator=not_none_validator) 91 | object_name: Optional[str] = None 92 | object_category: Optional[str] = None 93 | room_id: Optional[str] = None 94 | room_name: Optional[str] = None 95 | view_points: Optional[List[ObjectViewLocation]] = None 96 | 97 | 98 | @registry.register_sensor 99 | class ObjectGoalSensor(Sensor): 100 | r"""A sensor for Object Goal specification as observations which is used in 101 | ObjectGoal Navigation. The goal is expected to be specified by object_id or 102 | semantic category id. 103 | For the agent in simulator the forward direction is along negative-z. 104 | In polar coordinate format the angle returned is azimuth to the goal. 105 | Args: 106 | sim: a reference to the simulator for calculating task observations. 107 | config: a config for the ObjectGoalSensor sensor. Can contain field 108 | GOAL_SPEC that specifies which id use for goal specification, 109 | GOAL_SPEC_MAX_VAL the maximum object_id possible used for 110 | observation space definition. 111 | dataset: a Object Goal navigation dataset that contains dictionaries 112 | of categories id to text mapping. 113 | """ 114 | 115 | def __init__( 116 | self, sim, config: Config, dataset: Dataset, *args: Any, **kwargs: Any 117 | ): 118 | self._sim = sim 119 | self._dataset = dataset 120 | super().__init__(config=config) 121 | 122 | def _get_uuid(self, *args: Any, **kwargs: Any): 123 | return "objectgoal" 124 | 125 | def _get_sensor_type(self, *args: Any, **kwargs: Any): 126 | return SensorTypes.SEMANTIC 127 | 128 | def _get_observation_space(self, *args: Any, **kwargs: Any): 129 | sensor_shape = (1,) 130 | max_value = (self.config.GOAL_SPEC_MAX_VAL - 1,) 131 | if self.config.GOAL_SPEC == "TASK_CATEGORY_ID": 132 | max_value = max(self._dataset.category_to_task_category_id.values()) 133 | 134 | return spaces.Box(low=0, high=max_value, shape=sensor_shape, dtype=np.int64) 135 | 136 | def get_observation( 137 | self, 138 | observations, 139 | *args: Any, 140 | episode: ObjectGoalNavEpisode, 141 | **kwargs: Any, 142 | ) -> Optional[int]: 143 | if self.config.GOAL_SPEC == "TASK_CATEGORY_ID": 144 | if len(episode.goals) == 0: 145 | logger.error(f"No goal specified for episode {episode.episode_id}.") 146 | return None 147 | if not isinstance(episode.goals[0], ObjectGoal): 148 | logger.error( 149 | f"First goal should be ObjectGoal, episode {episode.episode_id}." 150 | ) 151 | return None 152 | category_name = episode.object_category 153 | return np.array( 154 | [self._dataset.category_to_task_category_id[category_name]], 155 | dtype=np.int64, 156 | ) 157 | elif self.config.GOAL_SPEC == "OBJECT_ID": 158 | return np.array([episode.goals[0].object_name_id], dtype=np.int64) 159 | else: 160 | raise RuntimeError("Wrong GOAL_SPEC specified for ObjectGoalSensor.") 161 | 162 | 163 | @registry.register_task(name="ObjectNav-v1") 164 | class ObjectNavigationTask(NavigationTask): 165 | r"""An Object Navigation Task class for a task specific methods. 166 | Used to explicitly state a type of the task in config. 167 | """ 168 | pass 169 | -------------------------------------------------------------------------------- /habitat/tasks/nav/shortest_path_follower.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from typing import Optional, Union 13 | 14 | import numpy as np 15 | 16 | import habitat_sim 17 | from habitat.sims.habitat_simulator.actions import HabitatSimActions 18 | from habitat.sims.habitat_simulator.habitat_simulator import HabitatSim 19 | from habitat.utils.geometry_utils import ( 20 | angle_between_quaternions, 21 | quaternion_from_two_vectors, 22 | ) 23 | 24 | EPSILON = 1e-6 25 | 26 | 27 | def action_to_one_hot(action: int) -> np.array: 28 | one_hot = np.zeros(len(HabitatSimActions), dtype=np.float32) 29 | one_hot[action] = 1 30 | return one_hot 31 | 32 | 33 | class ShortestPathFollower: 34 | r"""Utility class for extracting the action on the shortest path to the 35 | goal. 36 | Args: 37 | sim: HabitatSim instance. 38 | goal_radius: Distance between the agent and the goal for it to be 39 | considered successful. 40 | return_one_hot: If true, returns a one-hot encoding of the action 41 | (useful for training ML agents). If false, returns the 42 | SimulatorAction. 43 | """ 44 | 45 | def __init__( 46 | self, sim: HabitatSim, goal_radius: float, return_one_hot: bool = True 47 | ): 48 | assert ( 49 | getattr(sim, "geodesic_distance", None) is not None 50 | ), "{} must have a method called geodesic_distance".format(type(sim).__name__) 51 | 52 | self._sim = sim 53 | self._max_delta = self._sim.config.FORWARD_STEP_SIZE - EPSILON 54 | self._goal_radius = goal_radius 55 | self._step_size = self._sim.config.FORWARD_STEP_SIZE 56 | 57 | self._mode = ( 58 | "exact_gradient" 59 | if getattr(sim, "get_straight_shortest_path_points", None) is not None 60 | else "approximate_gradient" 61 | ) 62 | self._return_one_hot = return_one_hot 63 | 64 | def _get_return_value(self, action) -> Union[int, np.array]: 65 | if self._return_one_hot: 66 | return action_to_one_hot(action) 67 | else: 68 | return action 69 | 70 | def get_next_action(self, goal_pos: np.array) -> Optional[Union[int, np.array]]: 71 | """Returns the next action along the shortest path.""" 72 | if ( 73 | self._sim.geodesic_distance( 74 | self._sim.get_agent_state().position, [goal_pos] 75 | ) 76 | <= self._goal_radius 77 | ): 78 | return None 79 | 80 | max_grad_dir = self._est_max_grad_dir(goal_pos) 81 | if max_grad_dir is None: 82 | return self._get_return_value(HabitatSimActions.MOVE_FORWARD) 83 | return self._step_along_grad(max_grad_dir) 84 | 85 | def _step_along_grad(self, grad_dir: np.quaternion) -> Union[int, np.array]: 86 | current_state = self._sim.get_agent_state() 87 | alpha = angle_between_quaternions(grad_dir, current_state.rotation) 88 | if alpha <= np.deg2rad(self._sim.config.TURN_ANGLE) + EPSILON: 89 | return self._get_return_value(HabitatSimActions.MOVE_FORWARD) 90 | else: 91 | sim_action = HabitatSimActions.TURN_LEFT 92 | self._sim.step(sim_action) 93 | best_turn = ( 94 | HabitatSimActions.TURN_LEFT 95 | if ( 96 | angle_between_quaternions( 97 | grad_dir, self._sim.get_agent_state().rotation 98 | ) 99 | < alpha 100 | ) 101 | else HabitatSimActions.TURN_RIGHT 102 | ) 103 | self._reset_agent_state(current_state) 104 | return self._get_return_value(best_turn) 105 | 106 | def _reset_agent_state(self, state: habitat_sim.AgentState) -> None: 107 | self._sim.set_agent_state(state.position, state.rotation, reset_sensors=False) 108 | 109 | def _geo_dist(self, goal_pos: np.array) -> float: 110 | return self._sim.geodesic_distance( 111 | self._sim.get_agent_state().position, [goal_pos] 112 | ) 113 | 114 | def _est_max_grad_dir(self, goal_pos: np.array) -> np.array: 115 | 116 | current_state = self._sim.get_agent_state() 117 | current_pos = current_state.position 118 | 119 | if self.mode == "exact_gradient": 120 | points = self._sim.get_straight_shortest_path_points( 121 | self._sim.get_agent_state().position, goal_pos 122 | ) 123 | # Add a little offset as things get weird if 124 | # points[1] - points[0] is anti-parallel with forward 125 | if len(points) < 2: 126 | return None 127 | max_grad_dir = quaternion_from_two_vectors( 128 | self._sim.forward_vector, 129 | points[1] 130 | - points[0] 131 | + EPSILON * np.cross(self._sim.up_vector, self._sim.forward_vector), 132 | ) 133 | max_grad_dir.x = 0 134 | max_grad_dir = np.normalized(max_grad_dir) 135 | else: 136 | current_rotation = self._sim.get_agent_state().rotation 137 | current_dist = self._geo_dist(goal_pos) 138 | 139 | best_geodesic_delta = -2 * self._max_delta 140 | best_rotation = current_rotation 141 | for _ in range(0, 360, self._sim.config.TURN_ANGLE): 142 | sim_action = HabitatSimActions.MOVE_FORWARD 143 | self._sim.step(sim_action) 144 | new_delta = current_dist - self._geo_dist(goal_pos) 145 | 146 | if new_delta > best_geodesic_delta: 147 | best_rotation = self._sim.get_agent_state().rotation 148 | best_geodesic_delta = new_delta 149 | 150 | # If the best delta is within (1 - cos(TURN_ANGLE))% of the 151 | # best delta (the step size), then we almost certainly have 152 | # found the max grad dir and should just exit 153 | if np.isclose( 154 | best_geodesic_delta, 155 | self._max_delta, 156 | rtol=1 - np.cos(np.deg2rad(self._sim.config.TURN_ANGLE)), 157 | ): 158 | break 159 | 160 | self._sim.set_agent_state( 161 | current_pos, 162 | self._sim.get_agent_state().rotation, 163 | reset_sensors=False, 164 | ) 165 | 166 | sim_action = HabitatSimActions.TURN_LEFT 167 | self._sim.step(sim_action) 168 | 169 | self._reset_agent_state(current_state) 170 | 171 | max_grad_dir = best_rotation 172 | 173 | return max_grad_dir 174 | 175 | @property 176 | def mode(self): 177 | return self._mode 178 | 179 | @mode.setter 180 | def mode(self, new_mode: str): 181 | r"""Sets the mode for how the greedy follower determines the gradient of the geodesic_distance 182 | function (it then follows the negative of this gradient) 183 | 184 | :param new_mode: Must be one of 'exact_gradient' or 'approximate_gradient'. If 'exact_gradient', then 185 | the simulator must provide a straightend shortest path. 186 | 187 | If 'approximate_gradient', the follower approximates the gradient by turning 188 | and then stepping forward 189 | """ 190 | assert new_mode in {"exact_gradient", "approximate_gradient"} 191 | if new_mode == "exact_gradient": 192 | assert ( 193 | getattr(self._sim, "get_straight_shortest_path_points", None) 194 | is not None 195 | ) 196 | self._mode = new_mode 197 | -------------------------------------------------------------------------------- /habitat/tasks/registration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.core.logging import logger 13 | from habitat.core.registry import registry 14 | from habitat.tasks.nav import _try_register_nav_task 15 | 16 | 17 | def make_task(id_task, **kwargs): 18 | logger.info("Initializing task {}".format(id_task)) 19 | _task = registry.get_task(id_task) 20 | assert _task is not None, "Could not find task with name {}".format(id_task) 21 | 22 | return _task(**kwargs) 23 | 24 | 25 | _try_register_nav_task() 26 | -------------------------------------------------------------------------------- /habitat/tasks/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import numpy as np 13 | import quaternion # noqa # pylint: disable=unused-import 14 | 15 | 16 | def quaternion_to_rotation(q_r, q_i, q_j, q_k): 17 | r""" 18 | ref: https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation 19 | """ 20 | s = 1 # unit quaternion 21 | rotation_mat = np.array( 22 | [ 23 | [ 24 | 1 - 2 * s * (q_j**2 + q_k**2), 25 | 2 * s * (q_i * q_j - q_k * q_r), 26 | 2 * s * (q_i * q_k + q_j * q_r), 27 | ], 28 | [ 29 | 2 * s * (q_i * q_j + q_k * q_r), 30 | 1 - 2 * s * (q_i**2 + q_k**2), 31 | 2 * s * (q_j * q_k - q_i * q_r), 32 | ], 33 | [ 34 | 2 * s * (q_i * q_k - q_j * q_r), 35 | 2 * s * (q_j * q_k + q_i * q_r), 36 | 1 - 2 * s * (q_i**2 + q_j**2), 37 | ], 38 | ], 39 | dtype=np.float32, 40 | ) 41 | return rotation_mat 42 | 43 | 44 | def quaternion_rotate_vector(quat: np.quaternion, v: np.array) -> np.array: 45 | r"""Rotates a vector by a quaternion 46 | 47 | Args: 48 | quaternion: The quaternion to rotate by 49 | v: The vector to rotate 50 | 51 | Returns: 52 | np.array: The rotated vector 53 | """ 54 | vq = np.quaternion(0, 0, 0, 0) 55 | vq.imag = v 56 | return (quat * vq * quat.inverse()).imag 57 | 58 | 59 | def quaternion_from_coeff(coeffs: np.ndarray) -> np.quaternion: 60 | r"""Creates a quaternions from coeffs in [x, y, z, w] format""" 61 | quat = np.quaternion(0, 0, 0, 0) 62 | quat.real = coeffs[3] 63 | quat.imag = coeffs[0:3] 64 | return quat 65 | 66 | 67 | def cartesian_to_polar(x, y): 68 | rho = np.sqrt(x**2 + y**2) 69 | phi = np.arctan2(y, x) 70 | return rho, phi 71 | 72 | 73 | def compute_pixel_coverage(instance_seg, object_id): 74 | cand_mask = instance_seg == object_id 75 | score = cand_mask.sum().astype(np.float64) / cand_mask.size 76 | return score 77 | -------------------------------------------------------------------------------- /habitat/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | from habitat.utils import geometry_utils 12 | 13 | __all__ = ["visualizations", "geometry_utils"] 14 | -------------------------------------------------------------------------------- /habitat/utils/geometry_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import numpy as np 13 | import quaternion 14 | 15 | EPSILON = 1e-8 16 | 17 | 18 | def angle_between_quaternions(q1: np.quaternion, q2: np.quaternion) -> float: 19 | r"""Returns the angle (in radians) between two quaternions. This angle will 20 | always be positive. 21 | """ 22 | q1_inv = np.conjugate(q1) 23 | dq = quaternion.as_float_array(q1_inv * q2) 24 | 25 | return 2 * np.arctan2(np.linalg.norm(dq[1:]), np.abs(dq[0])) 26 | 27 | 28 | def quaternion_from_two_vectors(v0: np.array, v1: np.array) -> np.quaternion: 29 | r"""Computes the quaternion representation of v1 using v0 as the origin.""" 30 | v0 = v0 / np.linalg.norm(v0) 31 | v1 = v1 / np.linalg.norm(v1) 32 | c = v0.dot(v1) 33 | # Epsilon prevents issues at poles. 34 | if c < (-1 + EPSILON): 35 | c = max(c, -1) 36 | m = np.stack([v0, v1], 0) 37 | _, _, vh = np.linalg.svd(m, full_matrices=True) 38 | axis = vh.T[:, 2] 39 | w2 = (1 + c) * 0.5 40 | w = np.sqrt(w2) 41 | axis = axis * np.sqrt(1 - w2) 42 | return np.quaternion(w, *axis) 43 | 44 | axis = np.cross(v0, v1) 45 | s = np.sqrt((1 + c) * 2) 46 | return np.quaternion(s * 0.5, *(axis / s)) 47 | 48 | 49 | def quaternion_xyzw_to_wxyz(v: np.array): 50 | return np.quaternion(v[3], *v[0:3]) 51 | 52 | 53 | def quaternion_wxyz_to_xyzw(v: np.array): 54 | return np.quaternion(*v[1:4], v[0]) 55 | 56 | 57 | def quaternion_to_list(q: np.quaternion): 58 | return quaternion.as_float_array( 59 | quaternion_wxyz_to_xyzw(quaternion.as_float_array(q)) 60 | ).tolist() 61 | -------------------------------------------------------------------------------- /habitat/utils/test_utils.py: -------------------------------------------------------------------------------- 1 | from habitat.tasks.nav.nav import StopAction 2 | 3 | 4 | def sample_non_stop_action(action_space, num_samples=1): 5 | samples = [] 6 | for i in range(num_samples): 7 | action = action_space.sample() 8 | while action["action"] == StopAction.name: 9 | action = action_space.sample() 10 | samples.append({"action": action}) 11 | 12 | if num_samples == 1: 13 | return samples[0]["action"] 14 | else: 15 | return samples 16 | -------------------------------------------------------------------------------- /habitat/utils/visualizations/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat.utils.visualizations import maps, utils 13 | 14 | __all__ = ["maps", "utils"] 15 | -------------------------------------------------------------------------------- /habitat/utils/visualizations/assets/maps_topdown_agent_sprite/100x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreMarza/dynamic_implicit_representations/857212bef646336394adb395c52240d42ada3bbf/habitat/utils/visualizations/assets/maps_topdown_agent_sprite/100x100.png -------------------------------------------------------------------------------- /habitat/utils/visualizations/fog_of_war.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import numba 13 | import numpy as np 14 | 15 | from habitat.utils.visualizations import maps 16 | 17 | 18 | @numba.jit(nopython=True) 19 | def bresenham_supercover_line(pt1, pt2): 20 | r"""Line drawing algo based 21 | on http://eugen.dedu.free.fr/projects/bresenham/ 22 | """ 23 | 24 | ystep, xstep = 1, 1 25 | 26 | x, y = pt1 27 | dx, dy = pt2 - pt1 28 | 29 | if dy < 0: 30 | ystep *= -1 31 | dy *= -1 32 | 33 | if dx < 0: 34 | xstep *= -1 35 | dx *= -1 36 | 37 | line_pts = [[x, y]] 38 | 39 | ddx, ddy = 2 * dx, 2 * dy 40 | if ddx > ddy: 41 | errorprev = dx 42 | error = dx 43 | for i in range(int(dx)): 44 | x += xstep 45 | error += ddy 46 | 47 | if error > ddx: 48 | y += ystep 49 | error -= ddx 50 | if error + errorprev < ddx: 51 | line_pts.append([x, y - ystep]) 52 | elif error + errorprev > ddx: 53 | line_pts.append([x - xstep, y]) 54 | else: 55 | line_pts.append([x - xstep, y]) 56 | line_pts.append([x, y - ystep]) 57 | 58 | line_pts.append([x, y]) 59 | 60 | errorprev = error 61 | else: 62 | errorprev = dx 63 | error = dx 64 | for i in range(int(dy)): 65 | y += ystep 66 | error += ddx 67 | 68 | if error > ddy: 69 | x += xstep 70 | error -= ddy 71 | if error + errorprev < ddy: 72 | line_pts.append([x - xstep, y]) 73 | elif error + errorprev > ddy: 74 | line_pts.append([x, y - ystep]) 75 | else: 76 | line_pts.append([x - xstep, y]) 77 | line_pts.append([x, y - ystep]) 78 | 79 | line_pts.append([x, y]) 80 | 81 | errorprev = error 82 | 83 | return line_pts 84 | 85 | 86 | @numba.jit(nopython=True) 87 | def draw_fog_of_war_line(top_down_map, fog_of_war_mask, pt1, pt2): 88 | r"""Draws a line on the fog_of_war_mask mask between pt1 and pt2""" 89 | 90 | for pt in bresenham_supercover_line(pt1, pt2): 91 | x, y = pt 92 | 93 | if x < 0 or x >= fog_of_war_mask.shape[0]: 94 | break 95 | 96 | if y < 0 or y >= fog_of_war_mask.shape[1]: 97 | break 98 | 99 | if top_down_map[x, y] == maps.MAP_INVALID_POINT: 100 | break 101 | 102 | fog_of_war_mask[x, y] = 1 103 | 104 | 105 | @numba.jit(nopython=True) 106 | def _draw_loop( 107 | top_down_map, 108 | fog_of_war_mask, 109 | current_point, 110 | current_angle, 111 | max_line_len, 112 | angles, 113 | ): 114 | for angle in angles: 115 | draw_fog_of_war_line( 116 | top_down_map, 117 | fog_of_war_mask, 118 | current_point, 119 | current_point 120 | + max_line_len 121 | * np.array([np.cos(current_angle + angle), np.sin(current_angle + angle)]), 122 | ) 123 | 124 | 125 | def reveal_fog_of_war( 126 | top_down_map: np.ndarray, 127 | current_fog_of_war_mask: np.ndarray, 128 | current_point: np.ndarray, 129 | current_angle: float, 130 | fov: float = 90, 131 | max_line_len: float = 100, 132 | ) -> np.ndarray: 133 | r"""Reveals the fog-of-war at the current location 134 | 135 | This works by simply drawing lines from the agents current location 136 | and stopping once a wall is hit 137 | 138 | Args: 139 | top_down_map: The current top down map. Used for respecting walls when revealing 140 | current_fog_of_war_mask: The current fog-of-war mask to reveal the fog-of-war on 141 | current_point: The current location of the agent on the fog_of_war_mask 142 | current_angle: The current look direction of the agent on the fog_of_war_mask 143 | fov: The feild of view of the agent 144 | max_line_len: The maximum length of the lines used to reveal the fog-of-war 145 | 146 | Returns: 147 | The updated fog_of_war_mask 148 | """ 149 | fov = np.deg2rad(fov) 150 | 151 | # Set the angle step to a value such that delta_angle * max_line_len = 1 152 | angles = np.arange(-fov / 2, fov / 2, step=1.0 / max_line_len, dtype=np.float32) 153 | 154 | fog_of_war_mask = current_fog_of_war_mask.copy() 155 | _draw_loop( 156 | top_down_map, 157 | fog_of_war_mask, 158 | current_point, 159 | current_angle, 160 | max_line_len, 161 | angles, 162 | ) 163 | 164 | return fog_of_war_mask 165 | -------------------------------------------------------------------------------- /habitat/utils/visualizations/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import os 13 | import textwrap 14 | from typing import Dict, List, Optional, Tuple 15 | 16 | import imageio 17 | import numpy as np 18 | import tqdm 19 | 20 | from habitat.core.logging import logger 21 | from habitat.core.utils import try_cv2_import 22 | from habitat.utils.visualizations import maps 23 | 24 | cv2 = try_cv2_import() 25 | 26 | 27 | def paste_overlapping_image( 28 | background: np.ndarray, 29 | foreground: np.ndarray, 30 | location: Tuple[int, int], 31 | mask: Optional[np.ndarray] = None, 32 | ): 33 | r"""Composites the foreground onto the background dealing with edge 34 | boundaries. 35 | Args: 36 | background: the background image to paste on. 37 | foreground: the image to paste. Can be RGB or RGBA. If using alpha 38 | blending, values for foreground and background should both be 39 | between 0 and 255. Otherwise behavior is undefined. 40 | location: the image coordinates to paste the foreground. 41 | mask: If not None, a mask for deciding what part of the foreground to 42 | use. Must be the same size as the foreground if provided. 43 | Returns: 44 | The modified background image. This operation is in place. 45 | """ 46 | assert mask is None or mask.shape[:2] == foreground.shape[:2] 47 | foreground_size = foreground.shape[:2] 48 | min_pad = ( 49 | max(0, foreground_size[0] // 2 - location[0]), 50 | max(0, foreground_size[1] // 2 - location[1]), 51 | ) 52 | 53 | max_pad = ( 54 | max( 55 | 0, 56 | (location[0] + (foreground_size[0] - foreground_size[0] // 2)) 57 | - background.shape[0], 58 | ), 59 | max( 60 | 0, 61 | (location[1] + (foreground_size[1] - foreground_size[1] // 2)) 62 | - background.shape[1], 63 | ), 64 | ) 65 | 66 | background_patch = background[ 67 | (location[0] - foreground_size[0] // 2 + min_pad[0]) : ( 68 | location[0] + (foreground_size[0] - foreground_size[0] // 2) - max_pad[0] 69 | ), 70 | (location[1] - foreground_size[1] // 2 + min_pad[1]) : ( 71 | location[1] + (foreground_size[1] - foreground_size[1] // 2) - max_pad[1] 72 | ), 73 | ] 74 | foreground = foreground[ 75 | min_pad[0] : foreground.shape[0] - max_pad[0], 76 | min_pad[1] : foreground.shape[1] - max_pad[1], 77 | ] 78 | if foreground.size == 0 or background_patch.size == 0: 79 | # Nothing to do, no overlap. 80 | return background 81 | 82 | if mask is not None: 83 | mask = mask[ 84 | min_pad[0] : foreground.shape[0] - max_pad[0], 85 | min_pad[1] : foreground.shape[1] - max_pad[1], 86 | ] 87 | 88 | if foreground.shape[2] == 4: 89 | # Alpha blending 90 | foreground = ( 91 | background_patch.astype(np.int32) * (255 - foreground[:, :, [3]]) 92 | + foreground[:, :, :3].astype(np.int32) * foreground[:, :, [3]] 93 | ) // 255 94 | if mask is not None: 95 | background_patch[mask] = foreground[mask] 96 | else: 97 | background_patch[:] = foreground 98 | return background 99 | 100 | 101 | def images_to_video( 102 | images: List[np.ndarray], 103 | output_dir: str, 104 | video_name: str, 105 | fps: int = 10, 106 | quality: Optional[float] = 5, 107 | **kwargs, 108 | ): 109 | r"""Calls imageio to run FFMPEG on a list of images. For more info on 110 | parameters, see https://imageio.readthedocs.io/en/stable/format_ffmpeg.html 111 | Args: 112 | images: The list of images. Images should be HxWx3 in RGB order. 113 | output_dir: The folder to put the video in. 114 | video_name: The name for the video. 115 | fps: Frames per second for the video. Not all values work with FFMPEG, 116 | use at your own risk. 117 | quality: Default is 5. Uses variable bit rate. Highest quality is 10, 118 | lowest is 0. Set to None to prevent variable bitrate flags to 119 | FFMPEG so you can manually specify them using output_params 120 | instead. Specifying a fixed bitrate using ‘bitrate’ disables 121 | this parameter. 122 | """ 123 | assert 0 <= quality <= 10 124 | if not os.path.exists(output_dir): 125 | os.makedirs(output_dir) 126 | video_name = video_name.replace(" ", "_").replace("\n", "_") + ".mp4" 127 | writer = imageio.get_writer( 128 | os.path.join(output_dir, video_name), 129 | fps=fps, 130 | quality=quality, 131 | **kwargs, 132 | ) 133 | logger.info(f"Video created: {os.path.join(output_dir, video_name)}") 134 | for im in tqdm.tqdm(images): 135 | writer.append_data(im) 136 | writer.close() 137 | 138 | 139 | def draw_collision(view: np.ndarray, alpha: float = 0.4) -> np.ndarray: 140 | r"""Draw translucent red strips on the border of input view to indicate 141 | a collision has taken place. 142 | Args: 143 | view: input view of size HxWx3 in RGB order. 144 | alpha: Opacity of red collision strip. 1 is completely non-transparent. 145 | Returns: 146 | A view with collision effect drawn. 147 | """ 148 | strip_width = view.shape[0] // 20 149 | mask = np.ones(view.shape) 150 | mask[strip_width:-strip_width, strip_width:-strip_width] = 0 151 | mask = mask == 1 152 | view[mask] = (alpha * np.array([255, 0, 0]) + (1.0 - alpha) * view)[mask] 153 | return view 154 | 155 | 156 | def draw_subsuccess(view: np.ndarray, alpha: float = 0.6) -> np.ndarray: 157 | r"""Draw translucent blue strips on the border of input view to indicate 158 | a subsuccess event has taken place. 159 | Args: 160 | view: input view of size HxWx3 in RGB order. 161 | alpha: Opacity of blue collision strip. 1 is completely non-transparent. 162 | Returns: 163 | A view with collision effect drawn. 164 | """ 165 | strip_width = view.shape[0] // 20 166 | mask = np.ones(view.shape) 167 | mask[strip_width:-strip_width, strip_width:-strip_width] = 0 168 | mask = mask == 1 169 | view[mask] = (alpha * np.array([0, 0, 255]) + (1.0 - alpha) * view)[mask] 170 | return view 171 | 172 | 173 | def draw_found(view: np.ndarray, alpha: float = 1) -> np.ndarray: 174 | r"""Draw translucent blue strips on the border of input view to indicate 175 | that a found action has been called. 176 | Args: 177 | view: input view of size HxWx3 in RGB order. 178 | alpha: Opacity of blue collision strip. 1 is completely non-transparent. 179 | Returns: 180 | A view with found action effect drawn. 181 | """ 182 | strip_width = view.shape[0] // 20 183 | mask = np.ones(view.shape) 184 | mask[strip_width:-strip_width, strip_width:-strip_width] = 0 185 | mask = mask == 1 186 | view[mask] = (alpha * np.array([0, 0, 255]) + (1.0 - alpha) * view)[mask] 187 | return view 188 | 189 | 190 | def observations_to_image( 191 | observation: Dict, 192 | info: Dict, 193 | action: np.ndarray, 194 | ) -> np.ndarray: 195 | r"""Generate image of single frame from observation and info 196 | returned from a single environment step(). 197 | 198 | Args: 199 | observation: observation returned from an environment step(). 200 | info: info returned from an environment step(). 201 | action: action returned from an environment step(). 202 | 203 | Returns: 204 | generated image of a single frame. 205 | """ 206 | egocentric_view = [] 207 | if "rgb" in observation: 208 | observation_size = observation["rgb"].shape[0] 209 | rgb = observation["rgb"] 210 | if not isinstance(rgb, np.ndarray): 211 | rgb = rgb.cpu().numpy() 212 | 213 | egocentric_view.append(rgb) 214 | 215 | # draw depth map if observation has depth info 216 | if "depth" in observation: 217 | observation_size = observation["depth"].shape[0] 218 | depth_map = observation["depth"].squeeze() * 255.0 219 | if not isinstance(depth_map, np.ndarray): 220 | depth_map = depth_map.cpu().numpy() 221 | 222 | depth_map = depth_map.astype(np.uint8) 223 | depth_map = np.stack([depth_map for _ in range(3)], axis=2) 224 | egocentric_view.append(depth_map) 225 | 226 | assert len(egocentric_view) > 0, "Expected at least one visual sensor enabled." 227 | egocentric_view = np.concatenate(egocentric_view, axis=1) 228 | 229 | # draw collision 230 | if "collisions" in info and info["collisions"]["is_collision"]: 231 | egocentric_view = draw_collision(egocentric_view) 232 | 233 | if action[0] == 0: 234 | egocentric_view = draw_found(egocentric_view) 235 | 236 | frame = egocentric_view 237 | 238 | if "top_down_map" in info: 239 | top_down_map = info["top_down_map"]["map"] 240 | top_down_map = maps.colorize_topdown_map( 241 | top_down_map, info["top_down_map"]["fog_of_war_mask"] 242 | ) 243 | map_agent_pos = info["top_down_map"]["agent_map_coord"] 244 | top_down_map = maps.draw_agent( 245 | image=top_down_map, 246 | agent_center_coord=map_agent_pos, 247 | agent_rotation=info["top_down_map"]["agent_angle"], 248 | agent_radius_px=top_down_map.shape[0] // 16, 249 | ) 250 | 251 | if top_down_map.shape[0] > top_down_map.shape[1]: 252 | top_down_map = np.rot90(top_down_map, 1) 253 | 254 | # scale top down map to align with rgb view 255 | old_h, old_w, _ = top_down_map.shape 256 | top_down_height = observation_size 257 | top_down_width = int(float(top_down_height) / old_h * old_w) 258 | # cv2 resize (dsize is width first) 259 | top_down_map = cv2.resize( 260 | top_down_map, 261 | (top_down_width, top_down_height), 262 | interpolation=cv2.INTER_CUBIC, 263 | ) 264 | return frame 265 | -------------------------------------------------------------------------------- /habitat/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | VERSION = "0.1.4" 13 | -------------------------------------------------------------------------------- /habitat_baselines/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat_baselines.common.base_trainer import ( 13 | BaseRLTrainerNonOracle, 14 | BaseRLTrainerOracle, 15 | BaseTrainer, 16 | ) 17 | from habitat_baselines.rl.ppo.ppo_trainer import ( 18 | PPOTrainerO, 19 | PPOTrainerNO, 20 | RolloutStorageOracle, 21 | RolloutStorageNonOracle, 22 | ) 23 | 24 | __all__ = [ 25 | "BaseTrainer", 26 | "BaseRLTrainerNonOracle", 27 | "BaseRLTrainerOracle", 28 | "PPOTrainerO", 29 | "PPOTrainerNO", 30 | "RolloutStorage", 31 | "RolloutStorageOracle", 32 | "RolloutStorageNonOracle", 33 | ] 34 | -------------------------------------------------------------------------------- /habitat_baselines/common/baseline_registry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | r"""BaselineRegistry is extended from habitat.Registry to provide 13 | registration for trainer and environments, while keeping Registry 14 | in habitat core intact. 15 | 16 | Import the baseline registry object using 17 | 18 | ``from habitat_baselines.common.baseline_registry import baseline_registry`` 19 | 20 | Various decorators for registry different kind of classes with unique keys 21 | 22 | - Register a environment: ``@registry.register_env`` 23 | - Register a trainer: ``@registry.register_trainer`` 24 | """ 25 | 26 | from typing import Optional 27 | 28 | from habitat.core.registry import Registry 29 | 30 | 31 | class BaselineRegistry(Registry): 32 | @classmethod 33 | def register_trainer(cls, to_register=None, *, name: Optional[str] = None): 34 | r"""Register a RL training algorithm to registry with key 'name'. 35 | 36 | Args: 37 | name: Key with which the trainer will be registered. 38 | If None will use the name of the class. 39 | 40 | """ 41 | from habitat_baselines.common.base_trainer import BaseTrainer 42 | 43 | return cls._register_impl("trainer", to_register, name, assert_type=BaseTrainer) 44 | 45 | @classmethod 46 | def get_trainer(cls, name): 47 | return cls._get_impl("trainer", name) 48 | 49 | @classmethod 50 | def register_env(cls, to_register=None, *, name: Optional[str] = None): 51 | r"""Register a environment to registry with key 'name' 52 | currently only support subclass of RLEnv. 53 | 54 | Args: 55 | name: Key with which the env will be registered. 56 | If None will use the name of the class. 57 | 58 | """ 59 | from habitat import RLEnv 60 | 61 | return cls._register_impl("env", to_register, name, assert_type=RLEnv) 62 | 63 | @classmethod 64 | def get_env(cls, name): 65 | return cls._get_impl("env", name) 66 | 67 | 68 | baseline_registry = BaselineRegistry() 69 | -------------------------------------------------------------------------------- /habitat_baselines/common/env_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import numpy as np 13 | import random 14 | from typing import Type, Union 15 | 16 | import habitat 17 | from habitat import Config, Env, RLEnv, VectorEnv, ThreadedVectorEnv, make_dataset 18 | 19 | 20 | def make_env_fn( 21 | config: Config, env_class: Type[Union[Env, RLEnv]] 22 | ) -> Union[Env, RLEnv]: 23 | r"""Creates an env of type env_class with specified config and rank. 24 | This is to be passed in as an argument when creating VectorEnv. 25 | 26 | Args: 27 | config: root exp config that has core env config node as well as 28 | env-specific config node. 29 | env_class: class type of the env to be created. 30 | rank: rank of env to be created (for seeding). 31 | 32 | Returns: 33 | env object created according to specification. 34 | """ 35 | dataset = make_dataset( 36 | config.TASK_CONFIG.DATASET.TYPE, config=config.TASK_CONFIG.DATASET 37 | ) 38 | env = env_class(config=config, dataset=dataset) 39 | env.seed(config.TASK_CONFIG.SEED) 40 | return env 41 | 42 | 43 | def construct_envs(config: Config, env_class: Type[Union[Env, RLEnv]]) -> VectorEnv: 44 | r"""Create VectorEnv object with specified config and env class type. 45 | To allow better performance, dataset are split into small ones for 46 | each individual env, grouped by scenes. 47 | 48 | Args: 49 | config: configs that contain num_processes as well as information 50 | necessary to create individual environments. 51 | env_class: class type of the envs to be created. 52 | 53 | Returns: 54 | VectorEnv object created according to specification. 55 | """ 56 | 57 | num_processes = config.NUM_PROCESSES 58 | configs = [] 59 | env_classes = [env_class for _ in range(num_processes)] 60 | dataset = make_dataset(config.TASK_CONFIG.DATASET.TYPE) 61 | scenes = config.TASK_CONFIG.DATASET.CONTENT_SCENES 62 | if "*" in config.TASK_CONFIG.DATASET.CONTENT_SCENES: 63 | scenes = dataset.get_scenes_to_load(config.TASK_CONFIG.DATASET) 64 | 65 | if num_processes > 1: 66 | if len(scenes) == 0: 67 | raise RuntimeError( 68 | "No scenes to load, multiple process logic relies on being able to split scenes uniquely between processes" 69 | ) 70 | random.shuffle(scenes) 71 | 72 | scene_splits = [[] for _ in range(num_processes)] 73 | 74 | if len(scenes) >= num_processes: 75 | for idx, scene in enumerate(scenes): 76 | scene_splits[idx % len(scene_splits)].append(scene) 77 | else: 78 | sc = 0 79 | for i in range(num_processes): 80 | scene_splits[i].append(scenes[sc % len(scenes)]) 81 | sc += 1 82 | 83 | for i in range(num_processes): 84 | proc_config = config.clone() 85 | proc_config.defrost() 86 | 87 | task_config = proc_config.TASK_CONFIG 88 | task_config.SEED = task_config.SEED + i 89 | if len(scenes) > 0: 90 | task_config.DATASET.CONTENT_SCENES = scene_splits[i] 91 | 92 | task_config.SIMULATOR.HABITAT_SIM_V0.GPU_DEVICE_ID = config.SIMULATOR_GPU_ID 93 | 94 | task_config.SIMULATOR.AGENT_0.SENSORS = config.SENSORS 95 | 96 | proc_config.freeze() 97 | configs.append(proc_config) 98 | 99 | envs = habitat.VectorEnv( 100 | make_env_fn=make_env_fn, 101 | env_fn_args=tuple(tuple(zip(configs, env_classes))), 102 | ) 103 | return envs 104 | -------------------------------------------------------------------------------- /habitat_baselines/common/environments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | r""" 12 | This file hosts task-specific or trainer-specific environments for trainers. 13 | All environments here should be a (direct or indirect ) subclass of Env class 14 | in habitat. Customized environments should be registered using 15 | ``@baseline_registry.register_env(name="myEnv")` for reusability 16 | """ 17 | 18 | from typing import Optional, Type 19 | 20 | import habitat 21 | from habitat import Config, Dataset 22 | from habitat_baselines.common.baseline_registry import baseline_registry 23 | 24 | 25 | def get_env_class(env_name: str) -> Type[habitat.RLEnv]: 26 | r"""Return environment class based on name. 27 | 28 | Args: 29 | env_name: name of the environment. 30 | 31 | Returns: 32 | Type[habitat.RLEnv]: env class. 33 | """ 34 | return baseline_registry.get_env(env_name) 35 | 36 | 37 | @baseline_registry.register_env(name="NavRLEnv") 38 | class NavRLEnv(habitat.RLEnv): 39 | def __init__(self, config: Config, dataset: Optional[Dataset] = None): 40 | self._rl_config = config.RL 41 | self._core_env_config = config.TASK_CONFIG 42 | self._reward_measure_name = self._rl_config.REWARD_MEASURE 43 | self._success_measure_name = self._rl_config.SUCCESS_MEASURE 44 | self._subsuccess_measure_name = self._rl_config.SUBSUCCESS_MEASURE 45 | 46 | self._previous_measure = None 47 | self._previous_action = None 48 | super().__init__(self._core_env_config, dataset) 49 | 50 | def reset(self): 51 | self._previous_action = None 52 | observations = super().reset() 53 | self._previous_measure = self._env.get_metrics()[self._reward_measure_name] 54 | return observations 55 | 56 | def step(self, *args, **kwargs): 57 | self._previous_action = kwargs["action"] 58 | return super().step(*args, **kwargs) 59 | 60 | def get_reward_range(self): 61 | return ( 62 | self._rl_config.SLACK_REWARD - 1.0, 63 | self._rl_config.SUCCESS_REWARD + 1.0, 64 | ) 65 | 66 | def get_reward(self, observations, **kwargs): 67 | reward = self._rl_config.SLACK_REWARD 68 | 69 | current_measure = self._env.get_metrics()[self._reward_measure_name] 70 | 71 | if self._episode_subsuccess(): 72 | current_measure = self._env.task.foundDistance 73 | 74 | reward += self._previous_measure - current_measure 75 | self._previous_measure = current_measure 76 | 77 | if self._episode_subsuccess(): 78 | self._previous_measure = self._env.get_metrics()[self._reward_measure_name] 79 | 80 | if self._episode_success(): 81 | reward += self._rl_config.SUCCESS_REWARD 82 | elif self._episode_subsuccess(): 83 | reward += self._rl_config.SUBSUCCESS_REWARD 84 | elif self._env.task.is_found_called and self._rl_config.FALSE_FOUND_PENALTY: 85 | reward -= self._rl_config.FALSE_FOUND_PENALTY_VALUE 86 | 87 | return reward 88 | 89 | def _episode_success(self): 90 | return self._env.get_metrics()[self._success_measure_name] 91 | 92 | def _episode_subsuccess(self): 93 | return self._env.get_metrics()[self._subsuccess_measure_name] 94 | 95 | def get_done(self, observations): 96 | done = False 97 | if self._env.episode_over or self._episode_success(): 98 | done = True 99 | return done 100 | 101 | def get_info(self, observations): 102 | return self.habitat_env.get_metrics() 103 | -------------------------------------------------------------------------------- /habitat_baselines/common/tensorboard_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from typing import Any 13 | 14 | import numpy as np 15 | import torch 16 | from torch.utils.tensorboard import SummaryWriter 17 | 18 | 19 | class TensorboardWriter: 20 | def __init__(self, log_dir: str, *args: Any, **kwargs: Any): 21 | r"""A Wrapper for tensorboard SummaryWriter. It creates a dummy writer 22 | when log_dir is empty string or None. It also has functionality that 23 | generates tb video directly from numpy images. 24 | 25 | Args: 26 | log_dir: Save directory location. Will not write to disk if 27 | log_dir is an empty string. 28 | *args: Additional positional args for SummaryWriter 29 | **kwargs: Additional keyword args for SummaryWriter 30 | """ 31 | self.writer = None 32 | if log_dir is not None and len(log_dir) > 0: 33 | self.writer = SummaryWriter(log_dir, *args, **kwargs) 34 | 35 | def __getattr__(self, item): 36 | if self.writer: 37 | return self.writer.__getattribute__(item) 38 | else: 39 | return lambda *args, **kwargs: None 40 | 41 | def __enter__(self): 42 | return self 43 | 44 | def __exit__(self, exc_type, exc_val, exc_tb): 45 | if self.writer: 46 | self.writer.close() 47 | 48 | def add_video_from_np_images( 49 | self, video_name: str, step_idx: int, images: np.ndarray, fps: int = 10 50 | ) -> None: 51 | r"""Write video into tensorboard from images frames. 52 | 53 | Args: 54 | video_name: name of video string. 55 | step_idx: int of checkpoint index to be displayed. 56 | images: list of n frames. Each frame is a np.ndarray of shape. 57 | fps: frame per second for output video. 58 | 59 | Returns: 60 | None. 61 | """ 62 | if not self.writer: 63 | return 64 | # initial shape of np.ndarray list: N * (H, W, 3) 65 | frame_tensors = [torch.from_numpy(np_arr).unsqueeze(0) for np_arr in images] 66 | video_tensor = torch.cat(tuple(frame_tensors)) 67 | video_tensor = video_tensor.permute(0, 3, 1, 2).unsqueeze(0) 68 | # final shape of video tensor: (1, n, 3, H, W) 69 | self.writer.add_video(video_name, video_tensor, fps=fps, global_step=step_idx) 70 | -------------------------------------------------------------------------------- /habitat_baselines/common/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | import glob 13 | import os 14 | from collections import defaultdict 15 | from typing import Dict, List, Optional 16 | 17 | import numpy as np 18 | import torch 19 | import torch.nn as nn 20 | 21 | from habitat.utils.visualizations.utils import images_to_video 22 | from habitat_baselines.common.tensorboard_utils import TensorboardWriter 23 | import quaternion 24 | 25 | 26 | class Flatten(nn.Module): 27 | def forward(self, x): 28 | return x.contiguous().view(x.size(0), -1) 29 | 30 | 31 | class CustomFixedCategorical(torch.distributions.Categorical): 32 | def sample(self, sample_shape=torch.Size()): 33 | return super().sample(sample_shape).unsqueeze(-1) 34 | 35 | def log_probs(self, actions): 36 | return ( 37 | super() 38 | .log_prob(actions.squeeze(-1)) 39 | .view(actions.size(0), -1) 40 | .sum(-1) 41 | .unsqueeze(-1) 42 | ) 43 | 44 | def mode(self): 45 | return self.probs.argmax(dim=-1, keepdim=True) 46 | 47 | 48 | class CategoricalNet(nn.Module): 49 | def __init__(self, num_inputs, num_outputs): 50 | super().__init__() 51 | 52 | self.linear = nn.Linear(num_inputs, num_outputs) 53 | 54 | nn.init.orthogonal_(self.linear.weight, gain=0.01) 55 | nn.init.constant_(self.linear.bias, 0) 56 | 57 | def forward(self, x): 58 | x = self.linear(x) 59 | return CustomFixedCategorical(logits=x) 60 | 61 | 62 | def linear_decay(epoch: int, total_num_updates: int) -> float: 63 | r"""Returns a multiplicative factor for linear value decay 64 | 65 | Args: 66 | epoch: current epoch number 67 | total_num_updates: total number of epochs 68 | 69 | Returns: 70 | multiplicative factor that decreases param value linearly 71 | """ 72 | return 1 - (epoch / float(total_num_updates)) 73 | 74 | 75 | def _to_tensor(v): 76 | if torch.is_tensor(v): 77 | return v 78 | elif isinstance(v, np.ndarray): 79 | return torch.from_numpy(v) 80 | else: 81 | return torch.tensor(v, dtype=torch.float) 82 | 83 | 84 | def batch_obs( 85 | observations: List[Dict], device: Optional[torch.device] = None 86 | ) -> Dict[str, torch.Tensor]: 87 | r"""Transpose a batch of observation dicts to a dict of batched 88 | observations. 89 | 90 | Args: 91 | observations: list of dicts of observations. 92 | device: The torch.device to put the resulting tensors on. 93 | Will not move the tensors if None 94 | 95 | Returns: 96 | transposed dict of lists of observations. 97 | """ 98 | batch = defaultdict(list) 99 | 100 | for obs in observations: 101 | for sensor in obs: 102 | batch[sensor].append(_to_tensor(obs[sensor])) 103 | 104 | for sensor in batch: 105 | batch[sensor] = ( 106 | torch.stack(batch[sensor], dim=0).to(device=device).to(dtype=torch.float) 107 | ) 108 | 109 | return batch 110 | 111 | 112 | def poll_checkpoint_folder( 113 | checkpoint_folder: str, previous_ckpt_ind: int 114 | ) -> Optional[str]: 115 | r"""Return (previous_ckpt_ind + 1)th checkpoint in checkpoint folder 116 | (sorted by time of last modification). 117 | 118 | Args: 119 | checkpoint_folder: directory to look for checkpoints. 120 | previous_ckpt_ind: index of checkpoint last returned. 121 | 122 | Returns: 123 | return checkpoint path if (previous_ckpt_ind + 1)th checkpoint is found 124 | else return None. 125 | """ 126 | assert os.path.isdir(checkpoint_folder), ( 127 | f"invalid checkpoint folder " f"path {checkpoint_folder}" 128 | ) 129 | models_paths = list(filter(os.path.isfile, glob.glob(checkpoint_folder + "/*"))) 130 | models_paths.sort(key=lambda x: int(x.split(".")[1])) 131 | ind = previous_ckpt_ind + 1 132 | if ind < len(models_paths): 133 | return models_paths[ind] 134 | return None 135 | 136 | 137 | def generate_video( 138 | video_option: List[str], 139 | video_dir: Optional[str], 140 | images: List[np.ndarray], 141 | episode_id: int, 142 | checkpoint_idx: int, 143 | metrics: Dict[str, float], 144 | tb_writer: TensorboardWriter, 145 | fps: int = 5, 146 | ) -> None: 147 | r"""Generate video according to specified information. 148 | 149 | Args: 150 | video_option: string list of "tensorboard" or "disk" or both. 151 | video_dir: path to target video directory. 152 | images: list of images to be converted to video. 153 | episode_id: episode id for video naming. 154 | checkpoint_idx: checkpoint index for video naming. 155 | metric_name: name of the performance metric, e.g. "spl". 156 | metric_value: value of metric. 157 | tb_writer: tensorboard writer object for uploading video. 158 | fps: fps for generated video. 159 | Returns: 160 | None 161 | """ 162 | if len(images) < 1: 163 | return 164 | 165 | metric_strs = [] 166 | for k, v in metrics.items(): 167 | metric_strs.append(f"{k}={v:.2f}") 168 | 169 | video_name = f"episode={episode_id}-ckpt={checkpoint_idx}-" + "-".join(metric_strs) 170 | if "disk" in video_option: 171 | assert video_dir is not None 172 | images_to_video(images, video_dir, video_name) 173 | if "tensorboard" in video_option: 174 | tb_writer.add_video_from_np_images( 175 | f"episode{episode_id}", checkpoint_idx, images, fps=fps 176 | ) 177 | 178 | 179 | def quat_from_angle_axis(theta: float, axis: np.ndarray) -> np.quaternion: 180 | r"""Creates a quaternion from angle axis format 181 | 182 | :param theta: The angle to rotate about the axis by 183 | :param axis: The axis to rotate about 184 | :return: The quaternion 185 | """ 186 | axis = axis.astype(np.float) 187 | axis /= np.linalg.norm(axis) 188 | return quaternion.from_rotation_vector(theta * axis) 189 | 190 | 191 | class to_grid: 192 | def __init__(self, global_map_size, coordinate_min, coordinate_max): 193 | self.global_map_size = global_map_size 194 | self.coordinate_min = coordinate_min 195 | self.coordinate_max = coordinate_max 196 | self.grid_size = (coordinate_max - coordinate_min) / global_map_size 197 | 198 | def get_grid_coords(self, positions): 199 | grid_x = ((self.coordinate_max - positions[:, 0]) / self.grid_size).round() 200 | grid_y = ((positions[:, 1] - self.coordinate_min) / self.grid_size).round() 201 | return grid_x, grid_y 202 | -------------------------------------------------------------------------------- /habitat_baselines/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreMarza/dynamic_implicit_representations/857212bef646336394adb395c52240d42ada3bbf/habitat_baselines/config/__init__.py -------------------------------------------------------------------------------- /habitat_baselines/config/default.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from typing import List, Optional, Union 13 | 14 | import numpy as np 15 | 16 | from habitat import get_config as get_task_config 17 | from habitat.config import Config as CN 18 | 19 | DEFAULT_CONFIG_DIR = "configs/" 20 | CONFIG_FILE_SEPARATOR = "," 21 | # ----------------------------------------------------------------------------- 22 | # EXPERIMENT CONFIG 23 | # ----------------------------------------------------------------------------- 24 | _C = CN() 25 | _C.BASE_TASK_CONFIG_PATH = "configs/tasks/pointnav.yaml" 26 | _C.TASK_CONFIG = CN() # task_config will be stored as a config node 27 | _C.CMD_TRAILING_OPTS = [] # store command line options as list of strings 28 | _C.TRAINER_NAME = "ppo" 29 | _C.ENV_NAME = "NavRLEnv" 30 | _C.SIMULATOR_GPU_ID = 0 31 | _C.TORCH_GPU_ID = 0 32 | _C.VIDEO_OPTION = ["disk", "tensorboard"] 33 | _C.TENSORBOARD_DIR = "tb" 34 | _C.VIDEO_DIR = "video_dir" 35 | _C.TEST_EPISODE_COUNT = 2 36 | _C.EVAL_CKPT_PATH_DIR = "data/checkpoints" # path to ckpt or path to ckpts dir 37 | _C.NUM_PROCESSES = 16 38 | _C.SENSORS = ["RGB_SENSOR", "DEPTH_SENSOR"] 39 | _C.CHECKPOINT_FOLDER = "data/checkpoints" 40 | _C.NUM_UPDATES = 10000 41 | _C.LOG_INTERVAL = 10 42 | _C.LOG_FILE = "train.log" 43 | _C.CHECKPOINT_INTERVAL = 50 44 | # ----------------------------------------------------------------------------- 45 | # EVAL CONFIG 46 | # ----------------------------------------------------------------------------- 47 | _C.EVAL = CN() 48 | # The split to evaluate on 49 | _C.EVAL.SPLIT = "val" 50 | _C.EVAL.USE_CKPT_CONFIG = True 51 | # ----------------------------------------------------------------------------- 52 | # REINFORCEMENT LEARNING (RL) ENVIRONMENT CONFIG 53 | # ----------------------------------------------------------------------------- 54 | _C.RL = CN() 55 | _C.RL.REWARD_MEASURE = "distance_to_currgoal" 56 | _C.RL.SUCCESS_MEASURE = "success" 57 | _C.RL.SUBSUCCESS_MEASURE = "sub_success" 58 | _C.RL.SUCCESS_REWARD = 10.0 59 | _C.RL.SLACK_REWARD = -0.01 60 | _C.RL.OBJECT_CATEGORY_EMBEDDING_SIZE = 32 61 | _C.RL.PREVIOUS_ACTION_EMBEDDING_SIZE = 32 62 | _C.RL.PREVIOUS_ACTION = True 63 | # ----------------------------------------------------------------------------- 64 | # PROXIMAL POLICY OPTIMIZATION (PPO) 65 | # ----------------------------------------------------------------------------- 66 | _C.RL.PPO = CN() 67 | _C.RL.PPO.clip_param = 0.2 68 | _C.RL.PPO.ppo_epoch = 4 69 | _C.RL.PPO.num_mini_batch = 16 70 | _C.RL.PPO.value_loss_coef = 0.5 71 | _C.RL.PPO.entropy_coef = 0.01 72 | _C.RL.PPO.lr = 7e-4 73 | _C.RL.PPO.eps = 1e-5 74 | _C.RL.PPO.max_grad_norm = 0.5 75 | _C.RL.PPO.num_steps = 5 76 | _C.RL.PPO.use_gae = True 77 | _C.RL.PPO.use_linear_lr_decay = False 78 | _C.RL.PPO.use_linear_clip_decay = False 79 | _C.RL.PPO.gamma = 0.99 80 | _C.RL.PPO.tau = 0.95 81 | _C.RL.PPO.reward_window_size = 50 82 | _C.RL.PPO.use_normalized_advantage = True 83 | _C.RL.PPO.hidden_size = 512 84 | # ----------------------------------------------------------------------------- 85 | # MAPS 86 | # ----------------------------------------------------------------------------- 87 | _C.RL.MAPS = CN() 88 | _C.RL.MAPS.egocentric_map_size = 3 89 | _C.RL.MAPS.global_map_size = 250 90 | _C.RL.MAPS.global_map_depth = 32 91 | _C.RL.MAPS.coordinate_min = -62.3241 - 1e-6 92 | _C.RL.MAPS.coordinate_max = 90.0399 + 1e-6 93 | # ----------------------------------------------------------------------------- 94 | # DECENTRALIZED DISTRIBUTED PROXIMAL POLICY OPTIMIZATION (DD-PPO) 95 | # ----------------------------------------------------------------------------- 96 | _C.RL.DDPPO = CN() 97 | _C.RL.DDPPO.sync_frac = 0.6 98 | _C.RL.DDPPO.distrib_backend = "GLOO" 99 | _C.RL.DDPPO.rnn_type = "LSTM" 100 | _C.RL.DDPPO.num_recurrent_layers = 2 101 | _C.RL.DDPPO.backbone = "resnet50" 102 | _C.RL.DDPPO.pretrained_weights = "data/ddppo-models/gibson-2plus-resnet50.pth" 103 | # Loads pretrained weights 104 | _C.RL.DDPPO.pretrained = False 105 | # Loads just the visual encoder backbone weights 106 | _C.RL.DDPPO.pretrained_encoder = False 107 | # Whether or not the visual encoder backbone will be trained 108 | _C.RL.DDPPO.train_encoder = True 109 | # Whether or not to reset the critic linear layer 110 | _C.RL.DDPPO.reset_critic = True 111 | # ----------------------------------------------------------------------------- 112 | # ORBSLAM2 BASELINE 113 | # ----------------------------------------------------------------------------- 114 | _C.ORBSLAM2 = CN() 115 | _C.ORBSLAM2.SLAM_VOCAB_PATH = "habitat_baselines/slambased/data/ORBvoc.txt" 116 | _C.ORBSLAM2.SLAM_SETTINGS_PATH = "habitat_baselines/slambased/data/mp3d3_small1k.yaml" 117 | _C.ORBSLAM2.MAP_CELL_SIZE = 0.1 118 | _C.ORBSLAM2.MAP_SIZE = 40 119 | _C.ORBSLAM2.CAMERA_HEIGHT = get_task_config().SIMULATOR.DEPTH_SENSOR.POSITION[1] 120 | _C.ORBSLAM2.BETA = 100 121 | _C.ORBSLAM2.H_OBSTACLE_MIN = 0.3 * _C.ORBSLAM2.CAMERA_HEIGHT 122 | _C.ORBSLAM2.H_OBSTACLE_MAX = 1.0 * _C.ORBSLAM2.CAMERA_HEIGHT 123 | _C.ORBSLAM2.D_OBSTACLE_MIN = 0.1 124 | _C.ORBSLAM2.D_OBSTACLE_MAX = 4.0 125 | _C.ORBSLAM2.PREPROCESS_MAP = True 126 | _C.ORBSLAM2.MIN_PTS_IN_OBSTACLE = get_task_config().SIMULATOR.DEPTH_SENSOR.WIDTH / 2.0 127 | _C.ORBSLAM2.ANGLE_TH = float(np.deg2rad(15)) 128 | _C.ORBSLAM2.DIST_REACHED_TH = 0.15 129 | _C.ORBSLAM2.NEXT_WAYPOINT_TH = 0.5 130 | _C.ORBSLAM2.NUM_ACTIONS = 3 131 | _C.ORBSLAM2.DIST_TO_STOP = 0.05 132 | _C.ORBSLAM2.PLANNER_MAX_STEPS = 500 133 | _C.ORBSLAM2.DEPTH_DENORM = get_task_config().SIMULATOR.DEPTH_SENSOR.MAX_DEPTH 134 | 135 | 136 | def get_config( 137 | config_paths: Optional[Union[List[str], str]] = None, 138 | opts: Optional[list] = None, 139 | ) -> CN: 140 | r"""Create a unified config with default values overwritten by values from 141 | `config_paths` and overwritten by options from `opts`. 142 | Args: 143 | config_paths: List of config paths or string that contains comma 144 | separated list of config paths. 145 | opts: Config options (keys, values) in a list (e.g., passed from 146 | command line into the config. For example, `opts = ['FOO.BAR', 147 | 0.5]`. Argument can be used for parameter sweeping or quick tests. 148 | """ 149 | config = _C.clone() 150 | if config_paths: 151 | if isinstance(config_paths, str): 152 | if CONFIG_FILE_SEPARATOR in config_paths: 153 | config_paths = config_paths.split(CONFIG_FILE_SEPARATOR) 154 | else: 155 | config_paths = [config_paths] 156 | 157 | for config_path in config_paths: 158 | config.merge_from_file(config_path) 159 | 160 | config.TASK_CONFIG = get_task_config(config.BASE_TASK_CONFIG_PATH) 161 | if opts: 162 | config.CMD_TRAILING_OPTS = opts 163 | config.merge_from_list(opts) 164 | 165 | config.freeze() 166 | return config 167 | -------------------------------------------------------------------------------- /habitat_baselines/config/multinav/ppo_multinav.yaml: -------------------------------------------------------------------------------- 1 | BASE_TASK_CONFIG_PATH: "configs/tasks/multinav_mp3d.yaml" 2 | TRAINER_NAME: "ppo" 3 | ENV_NAME: "NavRLEnv" 4 | SIMULATOR_GPU_ID: 0 5 | TORCH_GPU_ID: 0 6 | VIDEO_OPTION: [] 7 | TENSORBOARD_DIR: "tb_dir" 8 | TENSORBOARD_DIR_EVAL: "eval" 9 | VIDEO_DIR: "video_dir" 10 | TEST_EPISODE_COUNT: 1000 11 | EVAL_CKPT_PATH_DIR: "eval_ckpt" 12 | NUM_PROCESSES: 4 13 | SENSORS: ["RGB_SENSOR", "DEPTH_SENSOR"] 14 | CHECKPOINT_FOLDER: "ckpt" 15 | NUM_UPDATES: 300000 16 | LOG_INTERVAL: 1 17 | CHECKPOINT_INTERVAL: 500 18 | 19 | RL: 20 | SUCCESS_REWARD: 3.0 21 | SUBSUCCESS_REWARD: 3.0 22 | FALSE_FOUND_PENALTY: False 23 | FALSE_FOUND_PENALTY_VALUE: 2.5 24 | OBJECT_CATEGORY_EMBEDDING_SIZE: 32 25 | PREVIOUS_ACTION_EMBEDDING_SIZE: 32 26 | PREVIOUS_ACTION: True 27 | PPO: 28 | # PPO params 29 | clip_param: 0.2 30 | ppo_epoch: 2 31 | num_mini_batch: 1 32 | value_loss_coef: 0.5 33 | entropy_coef: 0.01 34 | lr: 2.5e-4 35 | eps: 1e-5 36 | max_grad_norm: 0.2 37 | num_steps: 128 38 | use_gae: True 39 | gamma: 0.99 40 | tau: 0.95 41 | use_linear_clip_decay: True 42 | use_linear_lr_decay: True 43 | reward_window_size: 50 44 | use_normalized_advantage: False 45 | hidden_size: 768 46 | 47 | # Auxiliary losses coefficients 48 | aux_loss_seen_coef: 0.25 49 | aux_loss_direction_coef: 0.25 50 | aux_loss_distance_coef: 0.25 51 | 52 | MAPS: 53 | egocentric_map_size: 13 54 | global_map_size: 275 55 | global_map_depth: 32 56 | coordinate_min: -110.0 57 | coordinate_max: 110.0 58 | 59 | IMPLICIT_CONFIG: 60 | # Semantic Finder 61 | learn_sem_finder: True 62 | sem_finder_visual_encoder_path: "pretrained_models/semantic_finder_visual_encoder.pth" 63 | sem_finder_seg_path: "pretrained_models/semantic_finder_segmentation_head.pth" 64 | batch_size: 32 65 | batches: 1 66 | num_envs: 4 67 | 68 | # Exploration and Occupancy Implicit Representation 69 | learn_impl_net_expl_occ: True 70 | expl_occ_loss_threshold: 0.3 71 | global_reader_path: "pretrained_models/global_reader.pth" 72 | -------------------------------------------------------------------------------- /habitat_baselines/py.typed: -------------------------------------------------------------------------------- 1 | # Marker file for PEP 561. This tells mypy that the package uses inline types. 2 | -------------------------------------------------------------------------------- /habitat_baselines/rl/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /habitat_baselines/rl/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreMarza/dynamic_implicit_representations/857212bef646336394adb395c52240d42ada3bbf/habitat_baselines/rl/models/__init__.py -------------------------------------------------------------------------------- /habitat_baselines/rl/models/implicit_projection.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | Q_TO_MAT = torch.Tensor( 5 | [ 6 | [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], 7 | [[0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 0, -1], [0, 0, 1, 0]], 8 | [[0, 0, -1, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, -1, 0, 0]], 9 | [[0, 0, 0, -1], [0, 0, -1, 0], [0, 1, 0, 0], [1, 0, 0, 0]], 10 | ] 11 | ) 12 | 13 | Q_TO_Q_CONJ = torch.Tensor([1, -1, -1, -1]) 14 | 15 | 16 | def q_conj(q): 17 | """ 18 | Input q: float tensor of size * x 4 19 | Output conj(q): float tensor of size * x 4 20 | """ 21 | return q * Q_TO_Q_CONJ.to("cuda:0") 22 | 23 | 24 | def q_prod(q1, q2): 25 | """ 26 | Input q1: float tensor of size * x 4 27 | q2: float tensor of size * x 4 28 | Output q1.q2: float tensor of size * x 4 29 | """ 30 | mat1 = torch.tensordot(q1, Q_TO_MAT.to("cuda:0"), 1).to("cuda:0") 31 | res = torch.matmul(mat1, q2.unsqueeze(-1)) 32 | return res.squeeze(-1) 33 | 34 | 35 | def reproject(depth, gps, head, init_rot=None, init_rot_to_reinit=[]): 36 | h = 256 37 | w = 256 38 | hfov = torch.Tensor([79.0]).to("cuda:0") 39 | min_depth = 0 40 | max_depth = 10 41 | sensor_pos = torch.Tensor([0, 0, 0.88, 0]).to("cuda:0") 42 | f = 0.5 * w / torch.tan(0.5 * torch.deg2rad(hfov)) 43 | 44 | x_rect = (torch.arange(w).view(1, 1, w) - 0.5 * w).to("cuda:0") / f 45 | y_rect = (torch.arange(h).view(1, h, 1) - 0.5 * h).to("cuda:0") / f 46 | 47 | depth = min_depth + (max_depth - min_depth) * depth.squeeze(-1) 48 | gps = gps.view(-1, 1, 1, 2) 49 | head = head.view(-1, 1, 1) 50 | 51 | q = torch.stack( 52 | (torch.zeros_like(depth), depth * x_rect, -depth * y_rect, -depth), -1 53 | ) 54 | q += sensor_pos 55 | 56 | rot = torch.stack( 57 | ( 58 | torch.cos(0.5 * head), 59 | torch.zeros_like(head), 60 | torch.sin(0.5 * head), 61 | torch.zeros_like(head), 62 | ), 63 | -1, 64 | ) 65 | 66 | if init_rot is None: 67 | init_rot = rot 68 | rot = torch.zeros_like(init_rot) 69 | rot[..., 0] = 1 70 | elif len(init_rot_to_reinit) > 0: 71 | for i in range(len(init_rot)): 72 | if i in init_rot_to_reinit: 73 | init_rot[i] = rot[i] 74 | rot[i] = 0 75 | rot[i, :, :, 0] = 1 76 | else: 77 | rot[i] = q_prod(rot[i].unsqueeze(0), q_conj(init_rot[i].unsqueeze(0))) 78 | else: 79 | rot = q_prod(rot, q_conj(init_rot)) 80 | 81 | pts = q_prod(q_prod(rot, q), q_conj(rot))[..., 1:] 82 | pts[..., 0] += gps[..., 1] 83 | pts[..., 2] -= gps[..., 0] 84 | return pts, init_rot 85 | -------------------------------------------------------------------------------- /habitat_baselines/rl/models/implicit_representation_exploration_occupancy.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from torch.nn import TransformerEncoder, TransformerEncoderLayer 6 | 7 | 8 | class MLP_expl_occ(nn.Module): 9 | def __init__(self, D=3, W=512, input_ch=2, output_ch=2): 10 | super().__init__() 11 | self.pts_linears = nn.ModuleList( 12 | [nn.Linear(input_ch, W)] + [nn.Linear(W, W) for i in range(D - 2)] 13 | ) 14 | self.out = nn.Linear(W, output_ch) 15 | 16 | def forward(self, h): 17 | for i in range(len(self.pts_linears)): 18 | h = self.pts_linears[i](h) 19 | h = F.relu(h) 20 | return self.out(h) 21 | 22 | def reset_parameters(self): 23 | for i in range(len(self.pts_linears)): 24 | self.pts_linears[i].reset_parameters() 25 | self.out.reset_parameters() 26 | 27 | 28 | def run_epoch_expl_occ( 29 | criterion, net_expl_occ, optimizer_expl_occ, xyz, labels, to_train 30 | ): 31 | xyz_pred = [] 32 | for i in range(xyz.shape[0]): 33 | if to_train[i]: 34 | xyz_pred_ = net_expl_occ[i](xyz[i]) 35 | else: 36 | xyz_pred_ = None 37 | xyz_pred.append(xyz_pred_) 38 | 39 | losses = [] 40 | for i in range(xyz.shape[0]): 41 | if xyz_pred[i] is not None: 42 | loss = criterion(xyz_pred[i].view(-1, 3), labels[i].view(-1)) 43 | optimizer_expl_occ[i].zero_grad() 44 | loss.backward() 45 | optimizer_expl_occ[i].step() 46 | losses.append(loss.item()) 47 | else: 48 | losses.append(-1) 49 | return losses 50 | 51 | 52 | class PositionalEncoding(nn.Module): 53 | # Inspired from https://pytorch.org/tutorials/beginner/transformer_tutorial.html 54 | def __init__(self, d_model, dropout=0.1, nb_mlp_neurons=64 + 3): 55 | super().__init__() 56 | self.dropout = nn.Dropout(p=dropout) 57 | 58 | position = torch.arange(nb_mlp_neurons + 1).unsqueeze(1) 59 | div_term = torch.exp( 60 | torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model) 61 | ) 62 | pe = torch.zeros(nb_mlp_neurons + 1, 1, d_model) 63 | pe[:, 0, 0::2] = torch.sin(position * div_term) 64 | pe[:, 0, 1::2] = torch.cos(position * div_term) 65 | self.register_buffer("pe", pe) 66 | 67 | def forward(self, x): 68 | x += self.pe[:, 0, :] 69 | return self.dropout(x) 70 | 71 | 72 | class TransformerModel(nn.Module): 73 | # From https://pytorch.org/tutorials/beginner/transformer_tutorial.html 74 | def __init__( 75 | self, 76 | d_in_w_1, 77 | d_in_w_2, 78 | d_in_w_3, 79 | d_model, 80 | nhead, 81 | d_hid, 82 | nlayers, 83 | dropout, 84 | device, 85 | ): 86 | super().__init__() 87 | self.pos_encoder = PositionalEncoding( 88 | d_model, dropout, nb_mlp_neurons=512 + 512 + 3 89 | ) 90 | encoder_layers = TransformerEncoderLayer( 91 | d_model, nhead, d_hid, dropout, batch_first=True 92 | ) 93 | self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) 94 | self.d_model = d_model 95 | 96 | self.encoder_w_1 = nn.Sequential( 97 | nn.Linear(d_in_w_1, d_model), 98 | ) 99 | 100 | self.encoder_w_2 = nn.Sequential( 101 | nn.Linear(d_in_w_2, d_model), 102 | ) 103 | 104 | self.encoder_w_3 = nn.Sequential( 105 | nn.Linear(d_in_w_3, d_model), 106 | ) 107 | 108 | self.cls_embedding = nn.Embedding(1, d_model) 109 | self.device = device 110 | 111 | def forward(self, w_1, w_2, w_3): 112 | w_1 = self.encoder_w_1(w_1) 113 | w_2 = self.encoder_w_2(w_2) 114 | w_3 = self.encoder_w_3(w_3) 115 | 116 | cls_token = self.cls_embedding( 117 | torch.zeros(w_1.shape[0], 1).type(torch.LongTensor).to(self.device) 118 | ) 119 | src = torch.cat([cls_token, w_1, w_2, w_3], dim=1) 120 | 121 | src = self.pos_encoder(src) 122 | output = self.transformer_encoder(src) 123 | 124 | return output[:, 0, :] # only representation of cls token 125 | 126 | 127 | class MLPReader(nn.Module): 128 | def __init__( 129 | self, 130 | device, 131 | d_in_w_1=41, 132 | d_in_w_2=513, 133 | d_in_w_3=513, 134 | d_model=3 * 3 * 64, 135 | nhead=8, 136 | d_hid=3 * 3 * 64, 137 | nlayers=4, 138 | dropout=0.1, 139 | d_gps_input=2, 140 | d_gps_output=32, 141 | heading_num_embeddings=12, 142 | d_heading=32, 143 | ): 144 | super().__init__() 145 | self.encoder = TransformerModel( 146 | d_in_w_1=d_in_w_1, 147 | d_in_w_2=d_in_w_2, 148 | d_in_w_3=d_in_w_3, 149 | d_model=d_model, 150 | nhead=nhead, 151 | d_hid=d_hid, 152 | nlayers=nlayers, 153 | dropout=dropout, 154 | device=device, 155 | ) 156 | 157 | self.fc_encoder = nn.Sequential( 158 | nn.Linear(d_model, d_model), 159 | nn.ReLU(True), 160 | ) 161 | 162 | self.fc_gps = nn.Sequential(nn.Linear(d_gps_input, d_gps_output), nn.ReLU(True)) 163 | self.embed_curr_heading = nn.Embedding( 164 | num_embeddings=heading_num_embeddings, embedding_dim=d_heading 165 | ) 166 | 167 | self.fc_encoder_shift = nn.Sequential( 168 | nn.Linear(d_model + d_gps_output, d_model * 2), nn.ReLU(True) 169 | ) 170 | self.fc_encoder2_shift = nn.Sequential( 171 | nn.Linear(d_model * 2, d_model), nn.ReLU(True) 172 | ) 173 | 174 | self.fc_encoder_rot = nn.Sequential( 175 | nn.Linear(d_model + d_heading, d_model * 2), nn.ReLU(True) 176 | ) 177 | self.fc_encoder2_rot = nn.Sequential( 178 | nn.Linear(d_model * 2, d_model), nn.ReLU(True) 179 | ) 180 | 181 | def forward(self, w_1, w_2, w_3, gps, curr_heading): 182 | x = self.encoder(w_1, w_2, w_3) 183 | x = self.fc_encoder(x) 184 | 185 | gps = self.fc_gps(gps) 186 | curr_heading = self.embed_curr_heading(curr_heading) 187 | 188 | x = torch.cat((x, gps), dim=-1) 189 | x_shift = self.fc_encoder_shift(x) 190 | x_shift = self.fc_encoder2_shift(x_shift) 191 | 192 | x = torch.cat((x_shift, curr_heading[:, 0, :]), dim=-1) 193 | x = self.fc_encoder_rot(x) 194 | x = self.fc_encoder2_rot(x) 195 | 196 | return x 197 | 198 | 199 | class DecoderModel(nn.Module): 200 | def __init__(self): 201 | super().__init__() 202 | 203 | self.unflatten = nn.Unflatten(dim=1, unflattened_size=(64, 3, 3)) 204 | 205 | self.decoder_conv = nn.Sequential( 206 | nn.ConvTranspose2d(64, 32, 3, stride=2, output_padding=0), 207 | nn.BatchNorm2d(32), 208 | nn.ReLU(True), 209 | nn.ConvTranspose2d(32, 32, 3, stride=2, output_padding=0), 210 | nn.BatchNorm2d(32), 211 | nn.ReLU(True), 212 | nn.ConvTranspose2d(32, 16, 3, stride=2, output_padding=0), 213 | nn.BatchNorm2d(16), 214 | nn.ReLU(True), 215 | nn.ConvTranspose2d(16, 8, 3, stride=2, padding=0, output_padding=0), 216 | nn.BatchNorm2d(8), 217 | nn.ReLU(True), 218 | nn.ConvTranspose2d(8, 8, 3, stride=2, padding=0, output_padding=0), 219 | nn.BatchNorm2d(8), 220 | nn.ReLU(True), 221 | nn.ConvTranspose2d(8, 3, 3, stride=2, padding=0, output_padding=1), 222 | ) 223 | 224 | def forward(self, x): 225 | x = self.unflatten(x) 226 | x = self.decoder_conv(x) 227 | return x 228 | 229 | 230 | def reading_net_expl_occ_weights(net_list_exploration, device): 231 | w_1_batch = [] 232 | w_2_batch = [] 233 | w_3_batch = [] 234 | for net in net_list_exploration: 235 | state_dict = net.state_dict() 236 | 237 | w_1_ = state_dict["pts_linears.0.weight"].detach() 238 | w_1_b = state_dict["pts_linears.0.bias"].detach() 239 | w_1 = torch.cat([w_1_, w_1_b.unsqueeze(-1)], dim=1).unsqueeze(0).to(device) 240 | w_1_batch.append(w_1) 241 | 242 | w_2_ = state_dict["pts_linears.1.weight"].detach() 243 | w_2_b = state_dict["pts_linears.1.bias"].detach() 244 | w_2 = torch.cat([w_2_, w_2_b.unsqueeze(-1)], dim=1).unsqueeze(0).to(device) 245 | w_2_batch.append(w_2) 246 | 247 | w_3_ = state_dict["out.weight"].detach() 248 | w_3_b = state_dict["out.bias"].detach() 249 | w_3 = torch.cat([w_3_, w_3_b.unsqueeze(-1)], dim=1).unsqueeze(0).to(device) 250 | w_3_batch.append(w_3) 251 | 252 | w_1_batch = torch.cat(w_1_batch, dim=0) 253 | w_2_batch = torch.cat(w_2_batch, dim=0) 254 | w_3_batch = torch.cat(w_3_batch, dim=0) 255 | 256 | return w_1_batch, w_2_batch, w_3_batch 257 | -------------------------------------------------------------------------------- /habitat_baselines/rl/models/projection.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Base code from: # 3 | # * https://github.com/facebookresearch/habitat-lab # 4 | # * https://github.com/saimwani/multiON # 5 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 6 | # # 7 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 8 | ############################################################################### 9 | 10 | from einops import rearrange 11 | import numpy as np 12 | 13 | import torch 14 | import torch.nn.functional as F 15 | import torch_scatter 16 | 17 | 18 | def get_grid(pose, grid_size, device): 19 | """ 20 | Input: 21 | `pose` FloatTensor(bs, 3) 22 | `grid_size` 4-tuple (bs, _, grid_h, grid_w) 23 | `device` torch.device (cpu or gpu) 24 | Output: 25 | `rot_grid` FloatTensor(bs, grid_h, grid_w, 2) 26 | `trans_grid` FloatTensor(bs, grid_h, grid_w, 2) 27 | """ 28 | pose = pose.float() 29 | x = pose[:, 0] 30 | y = pose[:, 1] 31 | t = pose[:, 2] 32 | 33 | bs = x.size(0) 34 | cos_t = t.cos() 35 | sin_t = t.sin() 36 | 37 | theta11 = torch.stack( 38 | [cos_t, -sin_t, torch.zeros(cos_t.shape).float().to(device)], 1 39 | ) 40 | theta12 = torch.stack( 41 | [sin_t, cos_t, torch.zeros(cos_t.shape).float().to(device)], 1 42 | ) 43 | theta1 = torch.stack([theta11, theta12], 1) 44 | 45 | theta21 = torch.stack( 46 | [torch.ones(x.shape).to(device), -torch.zeros(x.shape).to(device), x], 1 47 | ) 48 | theta22 = torch.stack( 49 | [torch.zeros(x.shape).to(device), torch.ones(x.shape).to(device), y], 1 50 | ) 51 | theta2 = torch.stack([theta21, theta22], 1) 52 | 53 | rot_grid = F.affine_grid(theta1, torch.Size(grid_size)) 54 | trans_grid = F.affine_grid(theta2, torch.Size(grid_size)) 55 | 56 | return rot_grid, trans_grid 57 | 58 | 59 | class ComputeSpatialLocs: 60 | def __init__( 61 | self, 62 | egocentric_map_size, 63 | global_map_size, 64 | device, 65 | coordinate_min, 66 | coordinate_max, 67 | ): 68 | self.device = device 69 | self.cx, self.cy = 256.0 / 2.0, 256.0 / 2.0 # Hard coded camera parameters 70 | self.fx = self.fy = (256.0 / 2.0) / np.tan(np.deg2rad(79 / 2.0)) 71 | self.egocentric_map_size = egocentric_map_size 72 | self.local_scale = float(coordinate_max - coordinate_min) / float( 73 | global_map_size 74 | ) 75 | 76 | def forward(self, depth): 77 | depth = depth.permute(0, 3, 1, 2) 78 | _, _, imh, imw = depth.shape # batchsize, 1, imh, imw 79 | x = rearrange(torch.arange(0, imw), "w -> () () () w").to(self.device) 80 | y = rearrange(torch.arange(imh, 0, step=-1), "h -> () () h ()").to(self.device) 81 | xx = (x - self.cx) / self.fx 82 | yy = (y - self.cy) / self.fy 83 | 84 | # 3D real-world coordinates (in meters) 85 | Z = depth 86 | X = xx * Z 87 | Y = yy * Z 88 | 89 | # Valid inputs 90 | valid_inputs = (depth != 0) & ((Y > -0.5) & (Y < 1)) 91 | 92 | # X ground projection and Y ground projection 93 | x_gp = ( 94 | ((X / self.local_scale) + (self.egocentric_map_size - 1) / 2).round().long() 95 | ) # (bs, imh, imw, 1) 96 | y_gp = ( 97 | (-(Z / self.local_scale) + (self.egocentric_map_size - 1) / 2) 98 | .round() 99 | .long() 100 | ) # (bs, imh, imw, 1) 101 | 102 | return torch.cat([x_gp, y_gp], dim=1), valid_inputs 103 | 104 | 105 | class ProjectToGroundPlane: 106 | def __init__(self, egocentric_map_size, device): 107 | self.egocentric_map_size = egocentric_map_size 108 | self.device = device 109 | 110 | def forward(self, conv, spatial_locs, valid_inputs): 111 | outh, outw = (self.egocentric_map_size, self.egocentric_map_size) 112 | bs, f, HbyK, WbyK = conv.shape 113 | eps = -1e16 114 | K = 256 / 28 # Hardcoded value of K 115 | # K = 1 116 | 117 | # Sub-sample spatial_locs, valid_inputs according to img_feats resolution. 118 | idxes_ss = ( 119 | (torch.arange(0, HbyK, 1) * K).long().to(self.device), 120 | (torch.arange(0, WbyK, 1) * K).long().to(self.device), 121 | ) 122 | 123 | spatial_locs_ss = spatial_locs[ 124 | :, :, idxes_ss[0][:, None], idxes_ss[1] 125 | ] # (bs, 2, HbyK, WbyK) 126 | valid_inputs_ss = valid_inputs[ 127 | :, :, idxes_ss[0][:, None], idxes_ss[1] 128 | ] # (bs, 1, HbyK, WbyK) 129 | valid_inputs_ss = valid_inputs_ss.squeeze(1) # (bs, HbyK, WbyK) 130 | invalid_inputs_ss = ~valid_inputs_ss 131 | 132 | # Filter out invalid spatial locations 133 | invalid_spatial_locs = ( 134 | (spatial_locs_ss[:, 1] >= outh) 135 | | (spatial_locs_ss[:, 1] < 0) 136 | | (spatial_locs_ss[:, 0] >= outw) 137 | | (spatial_locs_ss[:, 0] < 0) 138 | ) # (bs, H, W) 139 | 140 | invalid_writes = invalid_spatial_locs | invalid_inputs_ss 141 | 142 | # Set the idxes for all invalid locations to (0, 0) 143 | spatial_locs_ss[:, 0][invalid_writes] = 0 144 | spatial_locs_ss[:, 1][invalid_writes] = 0 145 | 146 | # Accounting for max-pooling negative feature values 147 | invalid_writes_f = rearrange(invalid_writes, "b h w -> b () h w").float() 148 | conv_masked = conv * (1 - invalid_writes_f) + eps * invalid_writes_f 149 | conv_masked = rearrange(conv_masked, "b e h w -> b e (h w)") 150 | 151 | # Linearize ground-plane indices (linear idx = y * W + x) 152 | linear_locs_ss = ( 153 | spatial_locs_ss[:, 1] * outw + spatial_locs_ss[:, 0] 154 | ) # (bs, H, W) 155 | linear_locs_ss = rearrange(linear_locs_ss, "b h w -> b () (h w)") 156 | linear_locs_ss = linear_locs_ss.expand(-1, f, -1) # .contiguous() 157 | 158 | proj_feats, _ = torch_scatter.scatter_max( 159 | conv_masked, 160 | linear_locs_ss, 161 | dim=2, 162 | dim_size=outh * outw, 163 | ) 164 | proj_feats = rearrange(proj_feats, "b e (h w) -> b e h w", h=outh) 165 | 166 | # Replace invalid features with zeros 167 | eps_mask = (proj_feats == eps).float() 168 | proj_feats = proj_feats * (1 - eps_mask) + eps_mask * (proj_feats - eps) 169 | 170 | return proj_feats 171 | 172 | 173 | class RotateTensor: 174 | def __init__(self, device): 175 | self.device = device 176 | 177 | def forward(self, x_gp, heading): 178 | sin_t = torch.sin(heading.squeeze(1)) 179 | cos_t = torch.cos(heading.squeeze(1)) 180 | A = torch.zeros(x_gp.size(0), 2, 3).to(self.device) 181 | A[:, 0, 0] = cos_t 182 | A[:, 0, 1] = sin_t 183 | A[:, 1, 0] = -sin_t 184 | A[:, 1, 1] = cos_t 185 | 186 | grid = F.affine_grid(A, x_gp.size()) 187 | rotated_x_gp = F.grid_sample(x_gp, grid) 188 | return rotated_x_gp 189 | 190 | 191 | class Projection: 192 | def __init__( 193 | self, 194 | egocentric_map_size, 195 | global_map_size, 196 | device, 197 | coordinate_min, 198 | coordinate_max, 199 | ): 200 | self.egocentric_map_size = egocentric_map_size 201 | self.global_map_size = global_map_size 202 | self.compute_spatial_locs = ComputeSpatialLocs( 203 | egocentric_map_size, global_map_size, device, coordinate_min, coordinate_max 204 | ) 205 | self.project_to_ground_plane = ProjectToGroundPlane(egocentric_map_size, device) 206 | self.rotate_tensor = RotateTensor(device) 207 | 208 | def forward(self, conv, depth, heading): 209 | spatial_locs, valid_inputs = self.compute_spatial_locs.forward(depth) 210 | x_gp = self.project_to_ground_plane.forward(conv, spatial_locs, valid_inputs) 211 | rotated_x_gp = self.rotate_tensor.forward(x_gp, heading) 212 | return rotated_x_gp 213 | -------------------------------------------------------------------------------- /habitat_baselines/rl/models/rnn_state_encoder.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Base code from: # 3 | # * https://github.com/facebookresearch/habitat-lab # 4 | # * https://github.com/saimwani/multiON # 5 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 6 | # # 7 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 8 | ############################################################################### 9 | 10 | import torch 11 | import torch.nn as nn 12 | 13 | from copy import deepcopy 14 | 15 | 16 | class RNNStateEncoder(nn.Module): 17 | def __init__( 18 | self, 19 | input_size: int, 20 | hidden_size: int, 21 | num_layers: int = 1, 22 | rnn_type: str = "GRU", 23 | ): 24 | r"""An RNN for encoding the state in RL. 25 | 26 | Supports masking the hidden state during various timesteps in the forward lass 27 | 28 | Args: 29 | input_size: The input size of the RNN 30 | hidden_size: The hidden size 31 | num_layers: The number of recurrent layers 32 | rnn_type: The RNN cell type. Must be GRU or LSTM 33 | """ 34 | 35 | super().__init__() 36 | self._num_recurrent_layers = num_layers 37 | self._rnn_type = rnn_type 38 | 39 | self.rnn = getattr(nn, rnn_type)( 40 | input_size=input_size, 41 | hidden_size=hidden_size, 42 | num_layers=num_layers, 43 | ) 44 | 45 | self.layer_init() 46 | 47 | def layer_init(self): 48 | for name, param in self.rnn.named_parameters(): 49 | if "weight" in name: 50 | nn.init.orthogonal_(param) 51 | elif "bias" in name: 52 | nn.init.constant_(param, 0) 53 | 54 | @property 55 | def num_recurrent_layers(self): 56 | return self._num_recurrent_layers * (2 if "LSTM" in self._rnn_type else 1) 57 | 58 | def _pack_hidden(self, hidden_states): 59 | if "LSTM" in self._rnn_type: 60 | hidden_states = torch.cat([hidden_states[0], hidden_states[1]], dim=0) 61 | 62 | return hidden_states 63 | 64 | def _unpack_hidden(self, hidden_states): 65 | if "LSTM" in self._rnn_type: 66 | hidden_states = ( 67 | hidden_states[0 : self._num_recurrent_layers], 68 | hidden_states[self._num_recurrent_layers :], 69 | ) 70 | 71 | return hidden_states 72 | 73 | def _mask_hidden(self, hidden_states, masks): 74 | if isinstance(hidden_states, tuple): 75 | hidden_states = tuple(v * masks for v in hidden_states) 76 | else: 77 | hidden_states = masks * hidden_states 78 | 79 | return hidden_states 80 | 81 | def single_forward(self, x, hidden_states, masks): 82 | r"""Forward for a non-sequence input""" 83 | hidden_states = self._unpack_hidden(hidden_states) 84 | x, hidden_states = self.rnn( 85 | x.unsqueeze(0), 86 | self._mask_hidden(hidden_states, masks.unsqueeze(0)), 87 | ) 88 | x = x.squeeze(0) 89 | hidden_states = self._pack_hidden(hidden_states) 90 | return x, hidden_states 91 | 92 | def seq_forward(self, x, hidden_states, masks): 93 | r"""Forward for a sequence of length T 94 | 95 | Args: 96 | x: (T, N, -1) Tensor that has been flattened to (T * N, -1) 97 | hidden_states: The starting hidden state. 98 | masks: The masks to be applied to hidden state at every timestep. 99 | A (T, N) tensor flatten to (T * N) 100 | """ 101 | # x is a (T, N, -1) tensor flattened to (T * N, -1) 102 | n = hidden_states.size(1) 103 | t = int(x.size(0) / n) 104 | 105 | # unflatten 106 | x = x.view(t, n, x.size(1)) 107 | masks = masks.view(t, n) 108 | 109 | # steps in sequence which have zero for any agent. Assume t=0 has 110 | # a zero in it. 111 | has_zeros = (masks[1:] == 0.0).any(dim=-1).nonzero().squeeze().cpu() 112 | 113 | # +1 to correct the masks[1:] 114 | if has_zeros.dim() == 0: 115 | has_zeros = [has_zeros.item() + 1] # handle scalar 116 | else: 117 | has_zeros = (has_zeros + 1).numpy().tolist() 118 | 119 | # add t=0 and t=T to the list 120 | has_zeros = [0] + has_zeros + [t] 121 | 122 | hidden_states = self._unpack_hidden(hidden_states) 123 | outputs = [] 124 | for i in range(len(has_zeros) - 1): 125 | # process steps that don't have any zeros in masks together 126 | start_idx = has_zeros[i] 127 | end_idx = has_zeros[i + 1] 128 | 129 | rnn_scores, hidden_states = self.rnn( 130 | x[start_idx:end_idx], 131 | self._mask_hidden(hidden_states, masks[start_idx].view(1, -1, 1)), 132 | ) 133 | 134 | outputs.append(rnn_scores) 135 | 136 | # x is a (T, N, -1) tensor 137 | x = torch.cat(outputs, dim=0) 138 | x = x.view(t * n, -1) # flatten 139 | 140 | hidden_states = self._pack_hidden(hidden_states) 141 | return x, hidden_states 142 | 143 | def forward(self, x, hidden_states, masks): 144 | if x.size(0) == hidden_states.size(1): 145 | return self.single_forward(x, hidden_states, masks) 146 | else: 147 | return self.seq_forward(x, hidden_states, masks) 148 | -------------------------------------------------------------------------------- /habitat_baselines/rl/ppo/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | 12 | from habitat_baselines.rl.ppo.policy import ( 13 | Net, 14 | BaselinePolicyNonOracle, 15 | PolicyNonOracle, 16 | BaselinePolicyOracle, 17 | PolicyOracle, 18 | ) 19 | from habitat_baselines.rl.ppo.ppo import PPONonOracle, PPOOracle 20 | 21 | __all__ = [ 22 | "PPONonOracle", 23 | "PPOOracle", 24 | "PolicyNonOracle", 25 | "PolicyOracle", 26 | "RolloutStorageNonOracle", 27 | "RolloutStorageOracle", 28 | "BaselinePolicyNonOracle", 29 | "BaselinePolicyOracle", 30 | ] 31 | -------------------------------------------------------------------------------- /habitat_baselines/rl/ppo/aux_losses_utils.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Base code from: # 3 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 4 | # # 5 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 6 | ############################################################################### 7 | 8 | import math 9 | import numpy as np 10 | import torch 11 | 12 | 13 | def get_obj_poses(observations): 14 | mean_i_obj = [] 15 | mean_j_obj = [] 16 | gt_seen = [] 17 | not_visible_goals = [] 18 | 19 | mapCache = observations["semMap"] 20 | for batch in range(mapCache.shape[0]): 21 | indices = ( 22 | mapCache[batch, :, :, 1] == observations["multiobjectgoal"][batch] + 2 23 | ).nonzero() 24 | i_obj = indices[:, 0].type(torch.FloatTensor) 25 | j_obj = indices[:, 1].type(torch.FloatTensor) 26 | 27 | if len(i_obj) == 0 or len(j_obj) == 0: 28 | assert len(i_obj) == 0 and len(j_obj) == 0 29 | mean_i_obj.append(-1) 30 | mean_j_obj.append(-1) 31 | not_visible_goals.append(batch) 32 | gt_seen.append(0) 33 | else: 34 | mean_i_obj.append(torch.mean(i_obj).item()) 35 | mean_j_obj.append(torch.mean(j_obj).item()) 36 | gt_seen.append(1) 37 | 38 | # Convert lists to arrays 39 | mean_i_obj = np.array(mean_i_obj) 40 | mean_j_obj = np.array(mean_j_obj) 41 | 42 | return mean_i_obj, mean_j_obj, gt_seen, not_visible_goals 43 | 44 | 45 | def compute_distance_labels(mean_i_obj, mean_j_obj, not_visible_goals, device): 46 | euclidian_distance = np.sqrt( 47 | ((25 - mean_i_obj).astype(np.float64)) ** 2 48 | + ((25 - mean_j_obj).astype(np.float64)) ** 2 49 | ) 50 | euclidian_distance = np.floor(euclidian_distance) 51 | distance_labels = euclidian_distance.astype(np.int_) 52 | 53 | if len(not_visible_goals) > 0: 54 | distance_labels[not_visible_goals] = -1 55 | distance_labels = torch.from_numpy(distance_labels).to(device) 56 | 57 | return distance_labels 58 | 59 | 60 | def compute_direction_labels(mean_i_obj, mean_j_obj, not_visible_goals, device): 61 | bin_size = 360 / 12 62 | x_diffs = (mean_i_obj - 25).astype(np.float64) 63 | y_diffs = (mean_j_obj - 25).astype(np.float64) 64 | 65 | mask = 1 - (np.where((x_diffs == 0.0), 1, 0)) * (np.where((y_diffs == 0.0), 1, 0)) 66 | atans = np.arctan2(x_diffs, y_diffs) 67 | atans = (-atans) % (2 * math.pi) 68 | 69 | angle_diff = np.degrees(atans) 70 | angle_diff *= mask 71 | angle_diff = np.floor(angle_diff) 72 | 73 | direction_labels = (angle_diff - (bin_size / 2)) % 360 // bin_size 74 | direction_labels = direction_labels.astype(np.int_) 75 | if len(not_visible_goals) > 0: 76 | direction_labels[not_visible_goals] = -1 77 | direction_labels = torch.from_numpy(direction_labels).to(device) 78 | 79 | return direction_labels 80 | -------------------------------------------------------------------------------- /habitat_baselines/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ############################################################################### 4 | # Base code from: # 5 | # * https://github.com/facebookresearch/habitat-lab # 6 | # * https://github.com/saimwani/multiON # 7 | # * https://github.com/PierreMarza/teaching_agents_how_to_map # 8 | # # 9 | # Adapted by Pierre Marza (pierre.marza@insa-lyon.fr) # 10 | ############################################################################### 11 | import os 12 | import sys 13 | 14 | sys.path.insert(0, "") 15 | import argparse 16 | import random 17 | import numpy as np 18 | from habitat_baselines.common.baseline_registry import baseline_registry 19 | from habitat_baselines.config.default import get_config 20 | 21 | 22 | def main(): 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument( 25 | "--run-type", 26 | choices=["train", "eval"], 27 | required=True, 28 | help="run type of the experiment (train or eval)", 29 | ) 30 | parser.add_argument( 31 | "--exp-config", 32 | type=str, 33 | required=True, 34 | help="path to config yaml containing info about experiment", 35 | ) 36 | 37 | parser.add_argument( 38 | "--agent-type", 39 | choices=[ 40 | "no-map", 41 | "oracle", 42 | "oracle-ego", 43 | "proj-neural", 44 | "obj-recog", 45 | "implicit", 46 | ], 47 | required=True, 48 | help="agent type: oracle, oracleego, projneural, objrecog, implicit", 49 | ) 50 | 51 | parser.add_argument( 52 | "opts", 53 | default=None, 54 | nargs=argparse.REMAINDER, 55 | help="Modify config options from command line", 56 | ) 57 | 58 | # Auxiliary losses coefficients 59 | parser.add_argument( 60 | "--seen_coef", 61 | type=float, 62 | default=0, 63 | help="Aux loss seen coef", 64 | ) 65 | parser.add_argument( 66 | "--dir_coef", 67 | type=float, 68 | default=0, 69 | help="Aux loss direction coef", 70 | ) 71 | parser.add_argument( 72 | "--dist_coef", 73 | type=float, 74 | default=0, 75 | help="Aux loss distance coef", 76 | ) 77 | 78 | # Eval params (path to ckpt and eval split) 79 | parser.add_argument( 80 | "--eval_path", 81 | type=str, 82 | help="path to eval cpt folder", 83 | ) 84 | parser.add_argument( 85 | "--eval_split", 86 | type=str, 87 | help="data split to evaluate the model on", 88 | ) 89 | 90 | args = parser.parse_args() 91 | run_exp(**vars(args)) 92 | 93 | 94 | def run_exp( 95 | exp_config: str, 96 | run_type: str, 97 | agent_type: str, 98 | seen_coef: int, 99 | dir_coef: int, 100 | dist_coef: int, 101 | eval_path: str, 102 | eval_split: str, 103 | opts=None, 104 | ) -> None: 105 | r"""Runs experiment given mode and config 106 | 107 | Args: 108 | exp_config: path to config file. 109 | run_type: "train" or "eval. 110 | opts: list of strings of additional config options. 111 | 112 | Returns: 113 | None. 114 | """ 115 | config = get_config(exp_config, opts) 116 | random.seed(config.TASK_CONFIG.SEED) 117 | np.random.seed(config.TASK_CONFIG.SEED) 118 | 119 | config.defrost() 120 | config.TRAINER_NAME = agent_type 121 | config.TASK_CONFIG.TRAINER_NAME = agent_type 122 | 123 | compute_oracle_supervision = False 124 | if seen_coef > 0: 125 | config.RL.PPO.aux_loss_seen_coef = seen_coef 126 | compute_oracle_supervision = True 127 | else: 128 | config.RL.PPO.aux_loss_seen_coef = None 129 | 130 | if dir_coef > 0: 131 | config.RL.PPO.aux_loss_direction_coef = dir_coef 132 | compute_oracle_supervision = True 133 | else: 134 | config.RL.PPO.aux_loss_direction_coef = None 135 | 136 | if dist_coef > 0: 137 | config.RL.PPO.aux_loss_distance_coef = dist_coef 138 | compute_oracle_supervision = True 139 | else: 140 | config.RL.PPO.aux_loss_distance_coef = None 141 | config.TASK_CONFIG.COMPUTE_ORACLE_SUPERVISION = compute_oracle_supervision 142 | 143 | if run_type == "eval": 144 | config.EVAL_CKPT_PATH_DIR = eval_path 145 | assert eval_split in ["val", "test"] 146 | config.TASK_CONFIG.DATASET.SPLIT = eval_split 147 | config.EVAL.SPLIT = eval_split 148 | 149 | if eval_split == "val": 150 | assert ( 151 | config.NUM_PROCESSES <= 11 152 | ), "Val set contains episodes from 11 scenes. You should not have more processes than scenes." 153 | elif eval_split == "test": 154 | assert ( 155 | config.NUM_PROCESSES <= 18 156 | ), "Test set contains episodes from 18 scenes. You should not have more processes than scenes." 157 | 158 | if agent_type == "implicit": 159 | # Changing the number of processes to 1 160 | # when evaluating a model using implicit representations 161 | config.NUM_PROCESSES = 1 162 | config.RL.IMPLICIT_CONFIG.num_envs = 1 163 | config.freeze() 164 | 165 | if agent_type in ["oracle", "oracle-ego", "no-map", "implicit"]: 166 | trainer_init = baseline_registry.get_trainer("oracle") 167 | config.defrost() 168 | config.RL.PPO.hidden_size = 512 if agent_type in ["no-map", "implicit"] else 768 169 | if agent_type != "implicit": 170 | config.TASK_CONFIG.SIMULATOR.DEPTH_SENSOR.MIN_DEPTH = 0.5 171 | config.TASK_CONFIG.SIMULATOR.DEPTH_SENSOR.MAX_DEPTH = 5.0 172 | config.TASK_CONFIG.SIMULATOR.AGENT_0.HEIGHT = 1.5 173 | 174 | if agent_type == "oracle-ego" or ( 175 | agent_type == "no-map" and compute_oracle_supervision 176 | ): 177 | config.TASK_CONFIG.TASK.MEASUREMENTS.append("FOW_MAP") 178 | config.freeze() 179 | else: 180 | trainer_init = baseline_registry.get_trainer("non-oracle") 181 | config.defrost() 182 | config.RL.PPO.hidden_size = 512 183 | 184 | if compute_oracle_supervision: 185 | config.TASK_CONFIG.TASK.MEASUREMENTS.append("FOW_MAP") 186 | config.freeze() 187 | 188 | assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported" 189 | trainer = trainer_init(config) 190 | 191 | if run_type == "train": 192 | trainer.train() 193 | elif run_type == "eval": 194 | trainer.eval() 195 | 196 | 197 | if __name__ == "__main__": 198 | main() 199 | -------------------------------------------------------------------------------- /images/graphical_abstract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PierreMarza/dynamic_implicit_representations/857212bef646336394adb395c52240d42ada3bbf/images/graphical_abstract.png --------------------------------------------------------------------------------