├── .github └── workflows │ ├── cleanup.yml │ ├── docs.yml │ └── examples-as-test-b.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── behavior ├── __init__.py ├── baselines │ ├── behavioral_cloning │ │ ├── README.md │ │ ├── base_input_utils.py │ │ ├── example_data_spec_file.txt │ │ ├── simple_bc_agent.py │ │ └── test_bc_agent.py │ ├── rl │ │ └── stable_baselines3_ppo_training.py │ └── rl_mp │ │ └── stable_baselines3_behavior_mp_example.py ├── benchmark │ ├── agents │ │ ├── agent.py │ │ ├── checkpoints │ │ │ ├── full_observability_ppo_random.zip │ │ │ └── onboard_sensing_ppo_random.zip │ │ ├── random_agent.py │ │ ├── rl_agent.py │ │ └── users_agent.py │ ├── behavior_benchmark.py │ └── scripts │ │ ├── download_dataset.sh │ │ ├── evaluate_agent.sh │ │ ├── test_dev_docker.sh │ │ ├── test_minival_docker.sh │ │ └── train_minival_docker.sh ├── configs │ ├── behavior_full_observability.yaml │ ├── behavior_full_observability_fetch.yaml │ ├── behavior_onboard_sensing.yaml │ ├── behavior_onboard_sensing_fetch.yaml │ ├── behavior_segmentation_replay.yaml │ └── behavior_vr.yaml └── examples │ ├── __init__.py │ ├── checkpoint_generation_example.py │ ├── config_selector.py │ ├── dataset_generation_il_example.py │ ├── demo_segmentation_batch.py │ ├── demo_segmentation_example.py │ ├── example_selector.py │ ├── logic_states_checking_in_demo.py │ ├── metric_computation_batch.py │ ├── metric_computation_example.py │ ├── replay_demo_with_action_primitives_batch.py │ ├── replay_demo_with_action_primitives_example.py │ ├── threed_detection_dataset_generation_example.py │ └── training_example.py ├── docker ├── Dockerfile └── Dockerfile_add_results ├── docs ├── Makefile ├── acknowledgements.md ├── activities.md ├── agents.md ├── baselines.md ├── benchmarking.md ├── conf.py ├── evaluating.md ├── examples.md ├── images │ ├── brobotframes.png │ ├── fetchrobotframes.png │ ├── fig_pull_min.jpg │ ├── mosaicgif.gif │ ├── object_collage_v2.jpg │ ├── q_example.jpg │ └── secondarymetrics.jpg ├── index.rst ├── installation.md ├── intro.md ├── issues.md ├── make.bat ├── metrics.md ├── objects.md ├── projects.md ├── setups.md └── vr_demos.md ├── pyproject.toml ├── requirements-dev.txt ├── setup.py └── tests ├── create_tests_of_examples_b.py └── test_of_example_template_b.txt /.github/workflows/cleanup.yml: -------------------------------------------------------------------------------- 1 | name: Remove old artifacts 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | # Every day at 1am 7 | - cron: '0 9 * * *' 8 | 9 | jobs: 10 | remove-old-artifacts: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 10 13 | 14 | steps: 15 | - name: Remove old artifacts 16 | uses: c-hive/gha-remove-artifacts@v1 17 | with: 18 | age: '1 minute' # Remove everything 19 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Build & deploy docs 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build-docs: 7 | name: Build documentations 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - name: Checkout code 12 | uses: actions/checkout@v2 13 | 14 | - name: Setup python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: "3.8" 18 | architecture: x64 19 | 20 | - name: Upgrade pip (avoid issues with current broken pip release) 21 | run: pip install --upgrade pip 22 | 23 | - name: Install dev requirements 24 | run: pip install -r requirements-dev.txt 25 | 26 | - name: Generate docs 27 | working-directory: ./docs 28 | run: make html 29 | 30 | - name: Upload docs 31 | uses: actions/upload-artifact@v2 32 | with: 33 | name: docs 34 | path: ./docs/_build/html 35 | 36 | deploy-docs: 37 | name: Deploy prod documentation 38 | needs: build-docs 39 | runs-on: ubuntu-latest 40 | if: github.repository == 'StanfordVL/behavior' && github.ref == 'refs/heads/main' 41 | 42 | steps: 43 | - name: Download artifact 44 | uses: actions/download-artifact@v2 45 | with: 46 | name: docs 47 | path: ./docs/_build/html 48 | 49 | - name: Deploy to gh-pages 50 | uses: peaceiris/actions-gh-pages@v3 51 | with: 52 | github_token: ${{ secrets.GITHUB_TOKEN }} 53 | publish_dir: ./docs/_build/html 54 | -------------------------------------------------------------------------------- /.github/workflows/examples-as-test-b.yml: -------------------------------------------------------------------------------- 1 | name: Examples as Test B 2 | 3 | on: 4 | schedule: 5 | # * is a special character in YAML so you have to quote this string 6 | # Format: minute hour day-of-month month day-of-week(starts on sunday) 7 | # Scheduled for 2 am, everyday 8 | - cron: '0 10 * * *' 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | test: 17 | runs-on: [self-hosted, linux, gpu] 18 | if: github.repository == 'StanfordVL/behavior-dev' 19 | 20 | steps: 21 | - name: Checkout source 22 | uses: actions/checkout@v2 23 | with: 24 | submodules: true 25 | path: behavior 26 | 27 | - name: Add CUDA to env 28 | run: echo "/usr/local/cuda/bin" >> $GITHUB_PATH 29 | 30 | - name: Setup python 31 | uses: actions/setup-python@v2 32 | with: 33 | python-version: "3.8" 34 | architecture: x64 35 | 36 | - name: Create virtual env 37 | run: python -m venv env 38 | 39 | - name: Install dev requirements 40 | working-directory: behavior 41 | run: | 42 | source ../env/bin/activate 43 | pip install -r requirements-dev.txt 44 | 45 | - name: Install 46 | working-directory: behavior 47 | run: | 48 | source ../env/bin/activate 49 | pip install -e . 50 | 51 | - name: Uninstall pip igibson 52 | working-directory: behavior 53 | run: | 54 | source ../env/bin/activate 55 | pip uninstall -y igibson 56 | 57 | - name: Checkout iGibson 58 | uses: actions/checkout@v2 59 | with: 60 | repository: StanfordVL/iGibson-dev 61 | token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # PAT is required since this is a different repo 62 | path: igibson 63 | submodules: recursive 64 | lfs: true 65 | 66 | - name: Install iGibson 67 | working-directory: igibson 68 | run: | 69 | source ../env/bin/activate 70 | pip install -e . 71 | 72 | - name: Link Dataset 73 | working-directory: igibson 74 | run: ln -s /scr/ig-data igibson/data 75 | 76 | - name: Create tests of examples 77 | working-directory: behavior 78 | run: | 79 | source ../env/bin/activate 80 | python tests/create_tests_of_examples_b.py --exhaustive 81 | 82 | - name: Run tests 83 | working-directory: behavior 84 | run: | 85 | source ../env/bin/activate 86 | pytest /tmp/tests_of_examples_b 87 | 88 | - name: Remove files 89 | if: always() 90 | run: rm -rf /tmp/tests_of_examples_b || true 91 | 92 | - name: Cleanup virtual env 93 | if: always() 94 | run: rm -rf env/ 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: stable # Replace by any tag/version: https://github.com/psf/black/tags 4 | hooks: 5 | - id: black 6 | language_version: python3 # Should be a command that runs python3.6+ 7 | - repo: https://github.com/pycqa/isort 8 | rev: 5.8.0 9 | hooks: 10 | - id: isort 11 | name: isort (python) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Stanford Vision and Learning Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BEHAVIOR Repository 2 | 3 | ![mosaicgif.gif](docs/images/mosaicgif.gif) 4 | 5 | Code to evaluate a solution in the BEHAVIOR benchmark. 6 | It includes the starter code (Docker), baselines, and the necessary classes to evaluate agents in the benchmark. 7 | 8 | This README has been kept minimal to avoid redundancy. Please, visit our documentation [here](https://stanfordvl.github.io/behavior/) for more information. 9 | -------------------------------------------------------------------------------- /behavior/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | __version__ = "1.0.0" 5 | 6 | log = logging.getLogger(__name__) 7 | _LOG_LEVEL = os.environ.get("IG_LOG_LEVEL", "INFO").upper() 8 | log.setLevel(level=_LOG_LEVEL) 9 | 10 | examples_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "examples") 11 | configs_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") 12 | 13 | log.debug("Example path: {}".format(examples_path)) 14 | log.debug("Example config path: {}".format(configs_path)) 15 | -------------------------------------------------------------------------------- /behavior/baselines/behavioral_cloning/README.md: -------------------------------------------------------------------------------- 1 | Usage: 2 | 3 | Modify example_data_spec_file.txt to include directory and data file names 4 | 5 | For training: 6 | 7 | python simple_bc_agent.py --spec=[data_spec_file.txt] 8 | 9 | For testing: 10 | 11 | python test_bc_agent.py --model=[trained_model] 12 | -------------------------------------------------------------------------------- /behavior/baselines/behavioral_cloning/base_input_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base behavior dataset class. 3 | """ 4 | import argparse 5 | import logging 6 | import time 7 | 8 | import h5py 9 | import numpy as np 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | # Constants 14 | IMG_DIM = 128 15 | ACT_DIM = 28 16 | PROPRIOCEPTION_DIM = 20 17 | TASK_OBS_DIM = 456 18 | 19 | 20 | class BHDataset(object): 21 | def __init__(self, spec_file): 22 | self.files = read_spec_file(spec_file) 23 | t1 = time.time() 24 | log.info("Reading all training data into memory...") 25 | ( 26 | self.actions, 27 | self.proprioceptions, 28 | self.rgbs, 29 | self.task_obss, 30 | ) = read_proc_parallel(self.files) 31 | self.size = len(self.actions) 32 | log.debug("Time spent to read data: %.1fs" % (time.time() - t1)) 33 | 34 | def to_device(self): 35 | log.info("Sending data to gpu...") 36 | import torch 37 | 38 | self.device = "cuda" if torch.cuda.is_available() else "cpu" 39 | self.actions = torch.tensor(self.actions, dtype=torch.float32).to(self.device) 40 | self.proprioceptions = torch.tensor(self.proprioceptions, dtype=torch.float32).to(self.device) 41 | self.rgbs = torch.tensor(self.rgbs, dtype=torch.float32).permute(0, 3, 1, 2).to(self.device) 42 | self.task_obss = torch.tensor(self.task_obss, dtype=torch.float32).to(self.device) 43 | log.debug("Done.") 44 | 45 | 46 | def read_spec_file(fname): 47 | files = [] 48 | f = open(fname, "r") 49 | for line in f: 50 | items = line.split() 51 | if len(items) == 0 or line.startswith("#"): 52 | continue 53 | elif items[0] == "DIR": 54 | dir_proc = items[1] 55 | else: 56 | files.append(dir_proc + items[0] + "_episode.hdf5") 57 | return files 58 | 59 | 60 | def read_proc_parallel(files): 61 | # read action and state information from processed files 62 | actions, proprioceptions, rgbs, task_obss = ( 63 | np.empty((0, ACT_DIM)), 64 | np.empty((0, PROPRIOCEPTION_DIM)), 65 | np.empty((0, IMG_DIM, IMG_DIM, 3)), 66 | np.empty((0, TASK_OBS_DIM)), 67 | ) 68 | for f in files: 69 | log.info("Processing file %s..." % f) 70 | hf = h5py.File(f) 71 | actions = np.append(actions, np.asarray(hf["action"]), axis=0) 72 | proprioceptions = np.append(proprioceptions, np.asarray(hf["proprioception"]), axis=0) 73 | rgbs = np.append(rgbs, np.asarray(hf["rgb"]), axis=0) 74 | task_obss = np.append(task_obss, np.asarray(hf["task_obs"]), axis=0) 75 | hf.close() 76 | return actions, proprioceptions, rgbs, task_obss 77 | 78 | 79 | if __name__ == "__main__": 80 | 81 | def parse_args(): 82 | parser = argparse.ArgumentParser(description="Dataset loader") 83 | parser.add_argument("--spec", type=str, help="spec file name") 84 | return parser.parse_args() 85 | 86 | args = parse_args() 87 | data = BHDataset(args.spec) 88 | data.to_device() 89 | -------------------------------------------------------------------------------- /behavior/baselines/behavioral_cloning/example_data_spec_file.txt: -------------------------------------------------------------------------------- 1 | # Change these to be path of your data 2 | DIR /home/iGibson/igibson/data/behavior_human_demos_v2/ 3 | # Data files to include for training here (without.hdf5) 4 | throwing_away_leftovers_0_Ihlen_1_int_2021-06-04_19-48-55 5 | throwing_away_leftovers_0_Ihlen_1_int_2021-06-04_19-52-40 6 | hrowing_away_leftovers_0_Ihlen_1_int_2021-06-05_19-10-02 7 | 8 | -------------------------------------------------------------------------------- /behavior/baselines/behavioral_cloning/simple_bc_agent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Behavioral cloning agent network architecture and training 3 | """ 4 | import argparse 5 | import logging 6 | import sys 7 | 8 | sys.path.insert(0, "../utils") 9 | import base_input_utils as BIU 10 | import torch 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | import torch.optim as optim 14 | 15 | log = logging.getLogger(__name__) 16 | 17 | 18 | class BCNet_rgbp(nn.Module): 19 | """A behavioral cloning agent that uses RGB images and proprioception as state space""" 20 | 21 | def __init__(self, img_channels=3, proprioception_dim=20, num_actions=28): 22 | super(BCNet_rgbp, self).__init__() 23 | # image feature 24 | self.features1 = nn.Sequential( 25 | nn.Conv2d(img_channels, 32, kernel_size=8, stride=4), 26 | nn.ReLU(), 27 | nn.Conv2d(32, 64, kernel_size=4, stride=2), 28 | nn.ReLU(), 29 | nn.Conv2d(64, 64, kernel_size=3, stride=1), 30 | nn.ReLU(), 31 | ) 32 | self.flatten = nn.Flatten() 33 | self.features2 = nn.Sequential(nn.Linear(proprioception_dim, 40), nn.ReLU()) 34 | self.fc4 = nn.Linear(9216 + 40, 512) 35 | self.fc5 = nn.Linear(512, num_actions) 36 | 37 | def forward(self, imgs, proprioceptions): 38 | x1 = self.features1(imgs) 39 | x1 = self.flatten(x1) 40 | x2 = self.features2(proprioceptions) 41 | x = torch.cat((x1, x2), dim=1) 42 | x = self.fc4(x) 43 | x = F.relu(x) 44 | x = self.fc5(x) 45 | return x 46 | 47 | 48 | class BCNet_taskObs(nn.Module): 49 | def __init__(self, task_obs_dim=456, proprioception_dim=20, num_actions=28): 50 | super(BCNet_taskObs, self).__init__() 51 | # image feature 52 | self.fc1 = nn.Linear(task_obs_dim + proprioception_dim, 1024) 53 | self.fc2 = nn.Linear(1024, 256) 54 | self.fc3 = nn.Linear(256, num_actions) 55 | 56 | def forward(self, task_obs, proprioceptions): 57 | x = torch.cat((task_obs, proprioceptions), dim=1) 58 | x = self.fc1(x) 59 | x = F.relu(x) 60 | x = self.fc2(x) 61 | x = F.relu(x) 62 | x = self.fc3(x) 63 | return x 64 | 65 | 66 | if __name__ == "__main__": 67 | 68 | def parse_args(): 69 | parser = argparse.ArgumentParser(description="Dataset loader") 70 | parser.add_argument("--spec", type=str, help="spec file name") 71 | return parser.parse_args() 72 | 73 | args = parse_args() 74 | data = BIU.BHDataset(args.spec) 75 | data.to_device() 76 | 77 | # Training 78 | device = "cuda" if torch.cuda.is_available() else "cpu" 79 | # bc_agent = BCNet_rgbp().to(device) 80 | bc_agent = BCNet_taskObs().to(device) 81 | optimizer = optim.Adam(bc_agent.parameters()) 82 | 83 | NUM_EPOCH = 5 84 | PATH = "trained_models/model.pth" 85 | 86 | for epoch in range(NUM_EPOCH): 87 | optimizer.zero_grad() 88 | output = bc_agent(data.task_obss, data.proprioceptions) 89 | loss_func = nn.MSELoss() 90 | loss = loss_func(output, data.actions) 91 | loss.backward() 92 | optimizer.step() 93 | 94 | log.info(loss.item()) 95 | 96 | torch.save(bc_agent, PATH) 97 | -------------------------------------------------------------------------------- /behavior/baselines/behavioral_cloning/test_bc_agent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test behavioral cloning agent's performance 3 | """ 4 | import argparse 5 | import logging 6 | import os 7 | 8 | import igibson 9 | import numpy as np 10 | import torch 11 | from igibson.envs.igibson_env import iGibsonEnv 12 | from simple_bc_agent import BCNet_rgbp, BCNet_taskObs 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | def parse_args(): 18 | parser = argparse.ArgumentParser(description="Evaluate the trained behvaior_cloning agent in behavior") 19 | parser.add_argument("--model", type=str, help="Path and filename of trained model") 20 | return parser.parse_args() 21 | 22 | 23 | device = "cuda" if torch.cuda.is_available() else "cpu" 24 | args = parse_args() 25 | bc_agent = torch.load(args.model) 26 | bc_agent.eval() 27 | 28 | config_file = "behavior_onboard_sensing.yaml" 29 | env = iGibsonEnv( 30 | config_file=os.path.join(behavior.configs_path, config_file), 31 | mode="headless", 32 | action_timestep=1 / 30.0, 33 | physics_timestep=1 / 300.0, 34 | ) 35 | 36 | obs = env.reset() 37 | total_reward = 0 38 | done = False 39 | 40 | with torch.no_grad(): 41 | while not (done): 42 | task_obs = torch.tensor(obs["task_obs"], dtype=torch.float32).unsqueeze(0).to(device) 43 | proprioception = torch.tensor(obs["proprioception"], dtype=torch.float32).unsqueeze(0).to(device) 44 | action = bc_agent(task_obs, proprioception) 45 | a = action.cpu().numpy().squeeze(0) 46 | a_no_reset = np.concatenate((a[:19], a[20:27])) # we do not allow reset action for agents here 47 | obs, reward, done, info = env.step(a_no_reset) 48 | total_reward += reward 49 | log.info("Reward {}, info {}".format(total_reward, info)) 50 | -------------------------------------------------------------------------------- /behavior/baselines/rl/stable_baselines3_ppo_training.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import Callable 4 | 5 | import igibson 6 | from igibson.envs.igibson_env import iGibsonEnv 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | try: 11 | import gym 12 | import torch as th 13 | import torch.nn as nn 14 | from stable_baselines3 import PPO 15 | from stable_baselines3.common.evaluation import evaluate_policy 16 | from stable_baselines3.common.preprocessing import maybe_transpose 17 | from stable_baselines3.common.torch_layers import BaseFeaturesExtractor 18 | from stable_baselines3.common.utils import set_random_seed 19 | from stable_baselines3.common.vec_env import SubprocVecEnv, VecMonitor 20 | 21 | except ModuleNotFoundError: 22 | log.error("stable-baselines3 is not installed. You would need to do: pip install stable-baselines3") 23 | exit(1) 24 | 25 | 26 | """ 27 | Example training code using stable-baselines3 PPO for one BEHAVIOR activity. 28 | Note that due to the sparsity of the reward, this training code will not converge and achieve task success. 29 | This only serves as a starting point that users can further build upon. 30 | """ 31 | 32 | 33 | class CustomCombinedExtractor(BaseFeaturesExtractor): 34 | def __init__(self, observation_space: gym.spaces.Dict): 35 | # We do not know features-dim here before going over all the items, 36 | # so put something dummy for now. PyTorch requires calling 37 | # nn.Module.__init__ before adding modules 38 | super(CustomCombinedExtractor, self).__init__(observation_space, features_dim=1) 39 | 40 | extractors = {} 41 | 42 | total_concat_size = 0 43 | feature_size = 128 44 | for key, subspace in observation_space.spaces.items(): 45 | if key in ["proprioception", "task_obs"]: 46 | extractors[key] = nn.Sequential(nn.Linear(subspace.shape[0], feature_size), nn.ReLU()) 47 | elif key in ["rgb", "highlight", "depth", "seg", "ins_seg"]: 48 | n_input_channels = subspace.shape[2] # channel last 49 | cnn = nn.Sequential( 50 | nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), 51 | nn.ReLU(), 52 | nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0), 53 | nn.ReLU(), 54 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0), 55 | nn.ReLU(), 56 | nn.Flatten(), 57 | ) 58 | test_tensor = th.zeros([subspace.shape[2], subspace.shape[0], subspace.shape[1]]) 59 | with th.no_grad(): 60 | n_flatten = cnn(test_tensor[None]).shape[1] 61 | fc = nn.Sequential(nn.Linear(n_flatten, feature_size), nn.ReLU()) 62 | extractors[key] = nn.Sequential(cnn, fc) 63 | elif key in ["scan"]: 64 | n_input_channels = subspace.shape[1] # channel last 65 | cnn = nn.Sequential( 66 | nn.Conv1d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), 67 | nn.ReLU(), 68 | nn.Conv1d(32, 64, kernel_size=4, stride=2, padding=0), 69 | nn.ReLU(), 70 | nn.Conv1d(64, 64, kernel_size=3, stride=1, padding=0), 71 | nn.ReLU(), 72 | nn.Flatten(), 73 | ) 74 | test_tensor = th.zeros([subspace.shape[1], subspace.shape[0]]) 75 | with th.no_grad(): 76 | n_flatten = cnn(test_tensor[None]).shape[1] 77 | fc = nn.Sequential(nn.Linear(n_flatten, feature_size), nn.ReLU()) 78 | extractors[key] = nn.Sequential(cnn, fc) 79 | else: 80 | raise ValueError("Unknown observation key: %s" % key) 81 | total_concat_size += feature_size 82 | 83 | self.extractors = nn.ModuleDict(extractors) 84 | 85 | # Update the features dim manually 86 | self._features_dim = total_concat_size 87 | 88 | def forward(self, observations) -> th.Tensor: 89 | encoded_tensor_list = [] 90 | 91 | # self.extractors contain nn.Modules that do all the processing. 92 | for key, extractor in self.extractors.items(): 93 | if key in ["rgb", "highlight", "depth", "seg", "ins_seg"]: 94 | observations[key] = observations[key].permute((0, 3, 1, 2)) 95 | elif key in ["scan"]: 96 | observations[key] = observations[key].permute((0, 2, 1)) 97 | encoded_tensor_list.append(extractor(observations[key])) 98 | # Return a (B, self._features_dim) PyTorch tensor, where B is batch dimension. 99 | return th.cat(encoded_tensor_list, dim=1) 100 | 101 | 102 | def main(): 103 | config_file = "behavior_onboard_sensing.yaml" 104 | tensorboard_log_dir = "log_dir" 105 | num_cpu = 8 106 | 107 | def make_env(rank: int, seed: int = 0) -> Callable: 108 | def _init() -> iGibsonEnv: 109 | env = iGibsonEnv( 110 | config_file=os.path.join(behavior.configs_path, config_file), 111 | mode="headless", 112 | action_timestep=1 / 30.0, 113 | physics_timestep=1 / 300.0, 114 | ) 115 | env.seed(seed + rank) 116 | return env 117 | 118 | set_random_seed(seed) 119 | return _init 120 | 121 | env = SubprocVecEnv([make_env(i) for i in range(num_cpu)]) 122 | env = VecMonitor(env) 123 | 124 | eval_env = iGibsonEnv( 125 | config_file=os.path.join(behavior.configs_path, config_file), 126 | mode="headless", 127 | action_timestep=1 / 30.0, 128 | physics_timestep=1 / 300.0, 129 | ) 130 | 131 | policy_kwargs = dict( 132 | features_extractor_class=CustomCombinedExtractor, 133 | ) 134 | os.makedirs(tensorboard_log_dir, exist_ok=True) 135 | model = PPO( 136 | "MultiInputPolicy", 137 | env, 138 | verbose=1, 139 | tensorboard_log=tensorboard_log_dir, 140 | policy_kwargs=policy_kwargs, 141 | ) 142 | log.debug(model.policy) 143 | 144 | # Random Agent, before training 145 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=10) 146 | log.debug("Before Training: Mean reward: {mean_reward} +/- {std_reward:.2f}") 147 | 148 | model.learn(1000000) 149 | 150 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=20) 151 | log.info("After Training: Mean reward: {mean_reward} +/- {std_reward:.2f}") 152 | 153 | model.save("ckpt") 154 | del model 155 | 156 | model = PPO.load("ckpt") 157 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=20) 158 | log.info("After Loading: Mean reward: {mean_reward} +/- {std_reward:.2f}") 159 | 160 | 161 | if __name__ == "__main__": 162 | main() 163 | -------------------------------------------------------------------------------- /behavior/baselines/rl_mp/stable_baselines3_behavior_mp_example.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Callable 3 | 4 | import igibson 5 | from igibson.envs.behavior_mp_env import BehaviorMPEnv 6 | from igibson.envs.igibson_env import iGibsonEnv 7 | 8 | try: 9 | import gym 10 | import torch as th 11 | import torch.nn as nn 12 | from stable_baselines3 import PPO 13 | from stable_baselines3.common.evaluation import evaluate_policy 14 | from stable_baselines3.common.preprocessing import maybe_transpose 15 | from stable_baselines3.common.torch_layers import BaseFeaturesExtractor 16 | from stable_baselines3.common.utils import set_random_seed 17 | from stable_baselines3.common.vec_env import SubprocVecEnv, VecMonitor 18 | 19 | except ModuleNotFoundError: 20 | print("stable-baselines3 is not installed. You would need to do: pip install stable-baselines3") 21 | exit(1) 22 | 23 | 24 | """ 25 | Example training code using stable-baselines3 PPO for one BEHAVIOR activity. 26 | Note that due to the sparsity of the reward, this training code will not converge and achieve task success. 27 | This only serves as a starting point that users can further build upon. 28 | """ 29 | 30 | 31 | class CustomCombinedExtractor(BaseFeaturesExtractor): 32 | def __init__(self, observation_space: gym.spaces.Dict): 33 | # We do not know features-dim here before going over all the items, 34 | # so put something dummy for now. PyTorch requires calling 35 | # nn.Module.__init__ before adding modules 36 | super(CustomCombinedExtractor, self).__init__(observation_space, features_dim=1) 37 | 38 | extractors = {} 39 | 40 | total_concat_size = 0 41 | feature_size = 128 42 | for key, subspace in observation_space.spaces.items(): 43 | if key in ["proprioception", "task_obs"]: 44 | extractors[key] = nn.Sequential(nn.Linear(subspace.shape[0], feature_size), nn.ReLU()) 45 | elif key in ["rgb", "highlight", "depth", "seg", "ins_seg"]: 46 | n_input_channels = subspace.shape[2] # channel last 47 | cnn = nn.Sequential( 48 | nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), 49 | nn.ReLU(), 50 | nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0), 51 | nn.ReLU(), 52 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0), 53 | nn.ReLU(), 54 | nn.Flatten(), 55 | ) 56 | test_tensor = th.zeros([subspace.shape[2], subspace.shape[0], subspace.shape[1]]) 57 | with th.no_grad(): 58 | n_flatten = cnn(test_tensor[None]).shape[1] 59 | fc = nn.Sequential(nn.Linear(n_flatten, feature_size), nn.ReLU()) 60 | extractors[key] = nn.Sequential(cnn, fc) 61 | elif key in ["scan"]: 62 | n_input_channels = subspace.shape[1] # channel last 63 | cnn = nn.Sequential( 64 | nn.Conv1d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), 65 | nn.ReLU(), 66 | nn.Conv1d(32, 64, kernel_size=4, stride=2, padding=0), 67 | nn.ReLU(), 68 | nn.Conv1d(64, 64, kernel_size=3, stride=1, padding=0), 69 | nn.ReLU(), 70 | nn.Flatten(), 71 | ) 72 | test_tensor = th.zeros([subspace.shape[1], subspace.shape[0]]) 73 | with th.no_grad(): 74 | n_flatten = cnn(test_tensor[None]).shape[1] 75 | fc = nn.Sequential(nn.Linear(n_flatten, feature_size), nn.ReLU()) 76 | extractors[key] = nn.Sequential(cnn, fc) 77 | else: 78 | raise ValueError("Unknown observation key: %s" % key) 79 | total_concat_size += feature_size 80 | 81 | self.extractors = nn.ModuleDict(extractors) 82 | 83 | # Update the features dim manually 84 | self._features_dim = total_concat_size 85 | 86 | def forward(self, observations) -> th.Tensor: 87 | encoded_tensor_list = [] 88 | 89 | # self.extractors contain nn.Modules that do all the processing. 90 | for key, extractor in self.extractors.items(): 91 | if key in ["rgb", "highlight", "depth", "seg", "ins_seg"]: 92 | observations[key] = observations[key].permute((0, 3, 1, 2)) 93 | elif key in ["scan"]: 94 | observations[key] = observations[key].permute((0, 2, 1)) 95 | encoded_tensor_list.append(extractor(observations[key])) 96 | # Return a (B, self._features_dim) PyTorch tensor, where B is batch dimension. 97 | return th.cat(encoded_tensor_list, dim=1) 98 | 99 | 100 | def main(): 101 | config_file = "behavior_onboard_sensing.yaml" 102 | tensorboard_log_dir = "log_dir" 103 | num_cpu = 1 104 | 105 | def make_env(rank: int, seed: int = 0) -> Callable: 106 | def _init() -> BehaviorMPEnv: 107 | env = BehaviorMPEnv( 108 | config_file=os.path.join(behavior.configs_path, config_file), 109 | mode="headless", 110 | action_timestep=1 / 300.0, 111 | physics_timestep=1 / 300.0, 112 | use_motion_planning=False, 113 | ) 114 | env.seed(seed + rank) 115 | return env 116 | 117 | set_random_seed(seed) 118 | return _init 119 | 120 | env = SubprocVecEnv([make_env(i) for i in range(num_cpu)]) 121 | env = VecMonitor(env) 122 | 123 | eval_env = BehaviorMPEnv( 124 | config_file=os.path.join(behavior.configs_path, config_file), 125 | mode="headless", 126 | action_timestep=1 / 300.0, 127 | physics_timestep=1 / 300.0, 128 | use_motion_planning=False, 129 | ) 130 | 131 | policy_kwargs = dict( 132 | features_extractor_class=CustomCombinedExtractor, 133 | ) 134 | os.makedirs(tensorboard_log_dir, exist_ok=True) 135 | model = PPO( 136 | "MultiInputPolicy", 137 | env, 138 | verbose=1, 139 | tensorboard_log=tensorboard_log_dir, 140 | policy_kwargs=policy_kwargs, 141 | ) 142 | print(model.policy) 143 | 144 | # # Random Agent, before training 145 | # mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=10) 146 | # print(f"Before Training: Mean reward: {mean_reward} +/- {std_reward:.2f}") 147 | 148 | model.learn(1000000) 149 | 150 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=20) 151 | print(f"After Training: Mean reward: {mean_reward} +/- {std_reward:.2f}") 152 | 153 | model.save("ckpt") 154 | del model 155 | 156 | model = PPO.load("ckpt") 157 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=20) 158 | print(f"After Loading: Mean reward: {mean_reward} +/- {std_reward:.2f}") 159 | 160 | 161 | if __name__ == "__main__": 162 | main() 163 | -------------------------------------------------------------------------------- /behavior/benchmark/agents/agent.py: -------------------------------------------------------------------------------- 1 | class Agent(object): 2 | """ 3 | Abstract class that an agent should implement to be used in our benchmarking pipeline 4 | """ 5 | 6 | def __init__(self): 7 | """ 8 | Constructor 9 | """ 10 | pass 11 | 12 | def reset(self): 13 | """ 14 | Reset function 15 | To be called every time that the environment resets 16 | Usually clears the state and motion 17 | """ 18 | pass 19 | 20 | def act(self, obs): 21 | """ 22 | Main function of the agent 23 | Given an observation, the agent needs to provide an action 24 | :param obs: observation from the environment to decide the next action 25 | :returns: action taken by the agent given the observation (and any internal information) 26 | """ 27 | pass 28 | -------------------------------------------------------------------------------- /behavior/benchmark/agents/checkpoints/full_observability_ppo_random.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/behavior/benchmark/agents/checkpoints/full_observability_ppo_random.zip -------------------------------------------------------------------------------- /behavior/benchmark/agents/checkpoints/onboard_sensing_ppo_random.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/behavior/benchmark/agents/checkpoints/onboard_sensing_ppo_random.zip -------------------------------------------------------------------------------- /behavior/benchmark/agents/random_agent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from igibson.robots import REGISTERED_ROBOTS 3 | 4 | from behavior.benchmark.agents.agent import Agent 5 | 6 | IMG_WIDTH = 128 7 | IMG_HEIGHT = 128 8 | PROPRIOCEPTION_DIM = 20 9 | 10 | 11 | class RandomAgent(Agent): 12 | """ 13 | Implementation of the Agent class that generates random actions given an observation 14 | """ 15 | 16 | def __init__(self, robot_type="BehaviorRobot"): 17 | # TODO: It would be better to query directly the action space dim but that requires loading all robots. We 18 | # hardcode them. This may fail if the robot uses an unexpected controller type 19 | assert robot_type in REGISTERED_ROBOTS.keys(), "The given robot_type ({}) does not exist in iGibson".format( 20 | robot_type 21 | ) 22 | if robot_type == "BehaviorRobot": 23 | self.action_dim = 28 # Maybe less work too 24 | elif robot_type == "Husky": 25 | self.action_dim = 4 26 | elif robot_type == "Locobot": 27 | self.action_dim = 2 28 | elif robot_type == "Turtlebot": 29 | self.action_dim = 2 30 | elif robot_type == "Fetch": 31 | self.action_dim = 11 32 | elif robot_type == "Freight": 33 | self.action_dim = 2 34 | elif robot_type == "JR2": 35 | self.action_dim = 8 36 | elif robot_type == "Ant": 37 | self.action_dim = 8 38 | pass 39 | 40 | def reset(self): 41 | pass 42 | 43 | def act(self, obs): 44 | action = np.random.uniform(low=-1, high=1, size=(self.action_dim,)) 45 | return action 46 | 47 | 48 | # Test to print random actions 49 | if __name__ == "__main__": 50 | obs = { 51 | "rgb": np.ones((IMG_HEIGHT, IMG_WIDTH, 3)), 52 | "depth": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 53 | "seg": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 54 | "ins_seg": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 55 | "highlight": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 56 | "proprioception": np.ones((PROPRIOCEPTION_DIM,)), 57 | } 58 | agent = RandomAgent() 59 | action = agent.act(obs) 60 | print("action", action) 61 | -------------------------------------------------------------------------------- /behavior/benchmark/agents/rl_agent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from stable_baselines3 import PPO 3 | 4 | from behavior.benchmark.agents.agent import Agent 5 | 6 | IMG_WIDTH = 128 7 | IMG_HEIGHT = 128 8 | PROPRIOCEPTION_DIM = 20 9 | 10 | 11 | class PPOAgent(Agent): 12 | """ 13 | Implementation of the Agent class that loads a checkpoint of a previously trained PPO policy and queries the policy 14 | with the received observations to obtain actions 15 | """ 16 | 17 | def __init__(self, ckpt_path="checkpoints/onboard_sensing_ppo_random"): 18 | self.agent = PPO.load(ckpt_path) 19 | 20 | def reset(self): 21 | pass 22 | 23 | def act(self, obs): 24 | return self.agent.predict(obs, deterministic=True)[0] 25 | 26 | 27 | # Test to print actions 28 | if __name__ == "__main__": 29 | obs = { 30 | "rgb": np.ones((IMG_HEIGHT, IMG_WIDTH, 3)), 31 | "depth": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 32 | "seg": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 33 | "ins_seg": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 34 | "highlight": np.ones((IMG_HEIGHT, IMG_WIDTH, 1)), 35 | "proprioception": np.ones((PROPRIOCEPTION_DIM,)), 36 | } 37 | agent = PPOAgent(ckpt_path="checkpoints/onboard_sensing_ppo_random") 38 | action = agent.act(obs) 39 | print("action", action) 40 | -------------------------------------------------------------------------------- /behavior/benchmark/agents/users_agent.py: -------------------------------------------------------------------------------- 1 | from behavior.benchmark.agents.agent import Agent 2 | 3 | 4 | class CustomAgent(Agent): 5 | """ 6 | Entry point for BEHAVIOR users 7 | """ 8 | 9 | def __init__(self): 10 | """ 11 | Constructor 12 | """ 13 | raise NotImplementedError 14 | 15 | def reset(self): 16 | """ 17 | Reset function 18 | To be called every time that the environment resets 19 | Usually clears the state and motion 20 | """ 21 | raise NotImplementedError 22 | 23 | def act(self, obs): 24 | """ 25 | Main function of the agent 26 | Given an observation, the agent needs to provide an action 27 | :param obs: observation from the environment to decide the next action 28 | :returns: action taken by the agent given the observation (and any internal information) 29 | """ 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /behavior/benchmark/behavior_benchmark.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | import shutil 6 | from collections import defaultdict 7 | 8 | import bddl 9 | import igibson 10 | import numpy as np 11 | from igibson.envs.igibson_env import iGibsonEnv 12 | from igibson.metrics.agent import RobotMetric 13 | from igibson.metrics.disarrangement import KinematicDisarrangement, LogicalDisarrangement 14 | from igibson.metrics.task import TaskMetric 15 | from igibson.utils.utils import parse_config 16 | 17 | from behavior.benchmark.agents.random_agent import RandomAgent 18 | from behavior.benchmark.agents.rl_agent import PPOAgent 19 | from behavior.benchmark.agents.users_agent import CustomAgent 20 | 21 | log = logging.getLogger(__name__) 22 | log.setLevel(logging.WARNING) 23 | 24 | 25 | def get_metrics_callbacks(): 26 | metrics = [ 27 | KinematicDisarrangement(), 28 | LogicalDisarrangement(), 29 | RobotMetric(), 30 | TaskMetric(), 31 | ] 32 | 33 | return ( 34 | [metric.start_callback for metric in metrics], 35 | [metric.step_callback for metric in metrics], 36 | [metric.end_callback for metric in metrics], 37 | [metric.gather_results for metric in metrics], 38 | ) 39 | 40 | 41 | class BehaviorBenchmark(object): 42 | def __init__(self, agent, env_config_file="", output_dir="", split="", episodes_per_instance=1): 43 | """ 44 | Constructor 45 | :param agent: Agent to evaluate 46 | :param env_config_file: Config file to set up the environment. If empty, search a system variable (for Docker) 47 | :param output_dir: Directory to output results. If empty, search a system variable (for Docker) 48 | :param split: Split of activities to benchmark the agent on. If empty, search a system variable (for Docker) 49 | :param episodes_per_instance: Number of episodes to evaluate the agent in the same conditions 50 | """ 51 | self.agent = agent 52 | self.episodes_per_instance = episodes_per_instance 53 | # Takes variables set as environment variables or passed as params in initialization 54 | if env_config_file == "": 55 | print("Environment's config file is not an argument. Obtaining it from the environmental variables.") 56 | env_config_file = os.environ["CONFIG_FILE"] 57 | print("Using environment's config file: " + env_config_file) 58 | self.env_config_file = env_config_file 59 | 60 | if output_dir == "": 61 | print("Output directory is not an argument. Obtaining it from the environmental variables.") 62 | output_dir = os.environ["OUTPUT_DIR"] 63 | print("Using output dir: " + output_dir) 64 | self.output_dir = output_dir 65 | 66 | if split == "": 67 | print("Split is not an argument. Obtaining it from the environmental variables.") 68 | split = os.environ["SPLIT"] 69 | print("Using split: " + split) 70 | self.split = split 71 | 72 | def evaluate_agent(self): 73 | """ 74 | Evaluate the agent following the official guidelines: 75 | - Nine episodes, one per activity instance: 76 | - Three activity instances with "seen" object poses, objects, furniture and scene/home 77 | - Three activity instances with "unseen" object poses, "seen" objects, furniture and scene/home 78 | - Three activity instances with "unseen" object poses, "seen" objects, "unseen" furniture, "seen" scene 79 | The evaluation is performed on the activities indicated in the split: all, a subset, a single activity 80 | :return: 81 | """ 82 | # Get list of all 100 activities 83 | activities = sorted( 84 | [ 85 | item 86 | for item in os.listdir(os.path.join(os.path.dirname(bddl.__file__), "activity_definitions")) 87 | if item != "domain_igibson.bddl" 88 | ] 89 | ) 90 | assert len(activities) == 100 91 | 92 | if self.split in ["dev", "test"]: 93 | # Evaluate all activities 94 | print("Evaluating agent on all 100 activities") 95 | elif self.split == "minival": 96 | # Only evaluate a single activity specified in the config file 97 | env_config = parse_config(self.env_config_file) 98 | activities = [env_config["task"]] 99 | print("Evaluating agent on the activity specified in the config file: {}".format(activities[0])) 100 | elif self.split in activities: 101 | # Only evaluate a single activity specified by name 102 | activities = [self.split] 103 | print("Evaluating agent on the given activity: {}".format(activities[0])) 104 | elif all(elem in activities for elem in self.split): 105 | # Split is a list of valid activities 106 | activities = self.split 107 | print("Evaluating agent on activities {}".format(activities)) 108 | 109 | # This provides a dictionary of scenes where each activity can be successfully performed 110 | scene_json = os.path.join( 111 | os.path.dirname(bddl.__file__), 112 | "activity_to_preselected_scenes.json", 113 | ) 114 | with open(scene_json) as f: 115 | activity_to_scenes = json.load(f) 116 | 117 | # Loop all the activities to evaluate 118 | for activity in activities: 119 | assert activity in activity_to_scenes.keys() 120 | scenes = sorted(set(activity_to_scenes[activity])) # Get the scenes where the activity can be performed 121 | num_scenes = len(scenes) 122 | assert num_scenes <= 3 123 | 124 | # Official benchmark: Evaluate 9 episodes per activity 125 | # instance id 0-9: seen poses 126 | # instance id 10-19: unseen poses 127 | # instance id 20-29: unseen scenes (same houses but different furniture arrangement) 128 | # If you don't have enough activity instances in ig_dataset, the behavior data bundle is outdated. 129 | # You need to follow the participant guide and download again. 130 | 131 | # Depending on the number of scenes available for this activity 132 | if num_scenes == 3: 133 | # Regular case. We use three instances per scene (one "seen poses", one "unseen poses" and one 134 | # "unseen scene" (different furniture)) 135 | scene_instance_ids = { 136 | scenes[0]: [0, 10, 20], 137 | scenes[1]: [0, 10, 20], 138 | scenes[2]: [0, 10, 20], 139 | } 140 | elif num_scenes == 2: 141 | # We use four instances of one scene (two "seen poses", one "unseen poses" and one 142 | # "unseen scene" (different furniture)) and five from another scene (two "seen poses", two 143 | # "unseen poses" and one "unseen scene" (different furniture)) 144 | scene_instance_ids = { 145 | scenes[0]: [0, 1, 10, 20], 146 | scenes[1]: [0, 1, 10, 11, 20], 147 | } 148 | else: 149 | # We use nine instances of one scene (three "seen poses", three "unseen poses" and three 150 | # "unseen scene" (different furniture)) 151 | scene_instance_ids = {scenes[0]: [0, 1, 2, 10, 11, 12, 20, 21, 22]} 152 | 153 | self.evaluate_agent_on_one_activity(activity, scene_instance_ids) 154 | 155 | def evaluate_agent_on_one_activity(self, task, scene_instance_ids): 156 | """ 157 | Evaluates the given agent in the activities and instances indicated by the config file and split 158 | The results are stored in the output directory 159 | :param task: Tasks to evaluate 160 | :param scene_instance_ids: Dictionary of scenes and instances per scene to perform evaluation 161 | :return: 162 | """ 163 | env_config = parse_config(self.env_config_file) 164 | 165 | log_file = os.path.join(self.output_dir, "per_episode_metrics.json") 166 | summary_log_file = os.path.join(self.output_dir, "aggregated_metrics.json") 167 | self_reported_log_file = os.path.join( 168 | self.output_dir, "..", "participant_reported_results", "per_episode_metrics.json" 169 | ) 170 | self_reported_summary_log_file = os.path.join( 171 | self.output_dir, "..", "participant_reported_results", "aggregated_metrics.json" 172 | ) 173 | if os.path.exists(self_reported_log_file): 174 | shutil.copyfile(self_reported_log_file, log_file) 175 | print("Per episode eval results copied from self-reported results %s" % log_file) 176 | with open(self_reported_log_file) as f: 177 | self_reported_log = json.load(f) 178 | if os.path.exists(self_reported_summary_log_file): 179 | shutil.copyfile(self_reported_summary_log_file, summary_log_file) 180 | print("Aggregated eval results copied from self-reported results %s" % summary_log_file) 181 | return 182 | 183 | # Evaluation ################################################################################################## 184 | episode = 0 185 | per_episode_metrics = {} 186 | 187 | # This provides metadata about the activities, including the time humans require to perform them 188 | with open(os.path.join(igibson.ig_dataset_path, "metadata", "behavior_activity_statistics.json")) as f: 189 | activity_metadata = json.load(f) 190 | 191 | human_demo_mean_step = activity_metadata[task]["mean"] # Mean time invested by humans in the task 192 | print("Maximum number of steps is twice the mean of human time: {}".format(human_demo_mean_step * 2)) 193 | env_config["max_step"] = human_demo_mean_step * 2 # adjust env_config['max_step'] based on the human 194 | # demonstration, we give agent 2x steps of average human demonstration across all possible scenes 195 | 196 | # The robot name is "popped" from the config when loading into iG. We need to load it again in the loop 197 | # This is because the same parsed config is not expected to be reused consecutively 198 | robot_name = env_config["robot"]["name"] 199 | 200 | for scene_id, instance_ids in scene_instance_ids.items(): 201 | env_config["scene_id"] = scene_id 202 | env_config["task"] = task 203 | env_config["task_id"] = 0 204 | for instance_id in instance_ids: 205 | for episode_id in range(self.episodes_per_instance): 206 | env_config["robot"]["name"] = robot_name 207 | per_episode_metrics[episode] = self.evaluate_episode( 208 | task, scene_id, instance_id, episode_id, env_config 209 | ) 210 | episode += 1 211 | 212 | with open(log_file, "w+") as f: 213 | json.dump(per_episode_metrics, f) 214 | print("Per episode eval results saved to %s" % log_file) 215 | 216 | aggregated_metrics = {} 217 | success_score = [] 218 | simulator_time = [] 219 | kinematic_disarrangement = [] 220 | logical_disarrangement = [] 221 | distance_navigated = [] 222 | displacement_of_hands = [] 223 | 224 | task_to_mean_success_score = defaultdict(list) 225 | task_scores = [] 226 | 227 | for episode, metric in per_episode_metrics.items(): 228 | task_to_mean_success_score[metric["task"]].append(metric["q_score"]["final"]) 229 | 230 | for task, scores in task_to_mean_success_score.items(): 231 | task_scores.append(np.mean(scores)) 232 | 233 | task_scores = sorted(task_scores, reverse=True) 234 | 235 | for episode, metric in per_episode_metrics.items(): 236 | success_score.append(metric["q_score"]["final"]) 237 | simulator_time.append(metric["time"]["simulator_time"]) 238 | kinematic_disarrangement.append(metric["kinematic_disarrangement"]["relative"]) 239 | logical_disarrangement.append(metric["logical_disarrangement"]["relative"]) 240 | distance_navigated.append(np.sum(metric["agent_distance"]["timestep"]["body"])) 241 | displacement_of_hands.append( 242 | np.sum(metric["grasp_distance"]["timestep"]["left_hand"]) 243 | + np.sum(metric["grasp_distance"]["timestep"]["right_hand"]) 244 | ) 245 | 246 | aggregated_metrics["Success Score"] = np.mean(success_score) 247 | aggregated_metrics["Success Score Top 5"] = np.mean(np.array(task_scores)[:5]) 248 | aggregated_metrics["Simulated Time"] = np.mean(simulator_time) 249 | aggregated_metrics["Kinematic Disarrangement"] = np.mean(kinematic_disarrangement) 250 | aggregated_metrics["Logical Disarrangement"] = np.mean(logical_disarrangement) 251 | aggregated_metrics["Distance Navigated"] = np.mean(distance_navigated) 252 | aggregated_metrics["Displacement of Hands"] = np.mean(displacement_of_hands) 253 | with open(summary_log_file, "w+") as f: 254 | json.dump(aggregated_metrics, f) 255 | print("Aggregated eval results saved to %s" % summary_log_file) 256 | 257 | def evaluate_episode(self, task, scene_id, instance_id, episode_id, env_config): 258 | print("New episode: task {}, scene {}, instance {}, episode {}".format(task, scene_id, instance_id, episode_id)) 259 | env_config["instance_id"] = instance_id 260 | env = iGibsonEnv( 261 | config_file=env_config, 262 | mode="headless", 263 | action_timestep=1.0 / 30.0, 264 | physics_timestep=1.0 / 120.0, 265 | ) 266 | ( 267 | start_callbacks, 268 | step_callbacks, 269 | end_callbacks, 270 | data_callbacks, 271 | ) = get_metrics_callbacks() 272 | for callback in start_callbacks: 273 | callback(env, None) 274 | self.agent.reset() 275 | state = env.reset() 276 | while True: 277 | action = self.agent.act(state) 278 | state, reward, done, info = env.step(action) 279 | for callback in step_callbacks: 280 | callback(env, None) 281 | if done: 282 | break 283 | for callback in end_callbacks: 284 | callback(env, None) 285 | metrics_summary = {} 286 | for callback in data_callbacks: 287 | print("Generating report for the episode") 288 | metrics_summary.update(callback()) 289 | 290 | metrics_summary["task"] = task 291 | 292 | env.close() 293 | return metrics_summary 294 | 295 | 296 | def get_agent(agent_class, args): 297 | if agent_class == "Random": 298 | return RandomAgent() 299 | elif agent_class == "PPO": 300 | return PPOAgent(args.ckpt_path) 301 | elif agent_class == "PPO": 302 | return CustomAgent() 303 | else: 304 | print("Agent class not recognize. Consider adding it.") 305 | exit(-1) 306 | 307 | 308 | def main(): 309 | """ 310 | Entry point to evaluate agents in BEHAVIOR activities 311 | Arguments: 312 | agent-class: the Python class the agent belongs to 313 | ckpt-path: path to the checkpoint file if the agent is implemented as a neural network policy trained before 314 | split: activities to evaluate the agent on. The split can be ["minival", "dev", "test"], a list of activity 315 | labels, or a single activity label 316 | Add your own arguments for your agent, if needed 317 | """ 318 | parser = argparse.ArgumentParser() 319 | parser.add_argument( 320 | "--agent-class", 321 | type=str, 322 | default="Random", 323 | choices=["Random", "PPO", "Custom"], 324 | help="the Python class the agent belongs to", 325 | ) 326 | parser.add_argument( 327 | "--ckpt-path", 328 | default="", 329 | type=str, 330 | help="path to the checkpoint file if the agent is implemented as a neural network policy trained before", 331 | ) 332 | parser.add_argument( 333 | "--split", 334 | default="minival", 335 | type=str, 336 | help='activities to evaluate the agent on. The split can be ["minival", "dev", "test"], a list of activity labels, or a single activity label', 337 | ) 338 | 339 | args = parser.parse_args() 340 | 341 | print("Evaluating agent of type {} on {}".format(args.agent_class, args.split)) 342 | 343 | # Create agent to be evaluated 344 | agent = get_agent(args.agent_class, args) 345 | 346 | # Create an instance of the benchmark 347 | benchmark = BehaviorBenchmark(agent, split=args.split) 348 | 349 | # Evaluate agent on the benchmark 350 | benchmark.evaluate_agent() 351 | 352 | 353 | if __name__ == "__main__": 354 | main() 355 | -------------------------------------------------------------------------------- /behavior/benchmark/scripts/download_dataset.sh: -------------------------------------------------------------------------------- 1 | wget https://storage.googleapis.com/gibsonchallenge/ig_dataset.tar.gz 2 | tar -zxf ig_dataset.tar.gz 3 | -------------------------------------------------------------------------------- /behavior/benchmark/scripts/evaluate_agent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # add "--agent-class Custom" to the following command to evaluate your own agent 4 | 5 | python -m behavior.benchmark.behavior_benchmark 6 | -------------------------------------------------------------------------------- /behavior/benchmark/scripts/test_dev_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DOCKER_NAME="my_submission" 4 | 5 | while [[ $# -gt 0 ]] 6 | do 7 | key="${1}" 8 | 9 | case $key in 10 | --docker-name) 11 | shift 12 | DOCKER_NAME="${1}" 13 | shift 14 | ;; 15 | *) # unknown arg 16 | echo unkown arg ${1} 17 | exit 18 | ;; 19 | esac 20 | done 21 | 22 | 23 | docker run -v ${DATASET_PATH}/igibson.key:/opt/iGibson/igibson/data/igibson.key -v ${DATASET_PATH}/ig_dataset:/opt/iGibson/igibson/data/ig_dataset -v $(pwd)/results:/results \ 24 | --gpus=all \ 25 | ${DOCKER_NAME} \ 26 | /bin/bash -c \ 27 | "export CONFIG_FILE=/opt/behavior/behavior/configs/behavior_onboard_sensing.yaml; export SPLIT=dev; export OUTPUT_DIR=/results; cd /opt/behavior/behavior/benchmark/scripts; bash evaluate_agent.sh" 28 | 29 | # for older docker versions, use --runtime=nvidia instead of --gpus=all -------------------------------------------------------------------------------- /behavior/benchmark/scripts/test_minival_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DOCKER_NAME="my_submission" 4 | 5 | while [[ $# -gt 0 ]] 6 | do 7 | key="${1}" 8 | 9 | case $key in 10 | --docker-name) 11 | shift 12 | DOCKER_NAME="${1}" 13 | echo ${DOCKER_NAME} 14 | shift 15 | ;; 16 | --dataset-path) 17 | shift 18 | DATASET_PATH="${1}" 19 | echo ${DATASET_PATH} 20 | shift 21 | ;; 22 | *) # unknown arg 23 | echo unknown arg ${1} 24 | exit 25 | ;; 26 | esac 27 | done 28 | 29 | 30 | docker run -v ${DATASET_PATH}/igibson.key:/opt/iGibson/igibson/data/igibson.key -v ${DATASET_PATH}/ig_dataset:/opt/iGibson/igibson/data/ig_dataset -v $(pwd)/results:/results \ 31 | --gpus=all \ 32 | ${DOCKER_NAME} \ 33 | /bin/bash -c \ 34 | "export CONFIG_FILE=/opt/behavior/behavior/configs/behavior_onboard_sensing.yaml; export SPLIT=minival; export OUTPUT_DIR=/results; cd /opt/behavior/behavior/benchmark/scripts; bash evaluate_agent.sh" 35 | 36 | # for older docker versions, use --runtime=nvidia instead of --gpus=all -------------------------------------------------------------------------------- /behavior/benchmark/scripts/train_minival_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DOCKER_NAME="my_submission" 4 | DATASET_PATH="/tmp" 5 | 6 | while [[ $# -gt 0 ]] 7 | do 8 | key="${1}" 9 | 10 | case $key in 11 | --docker-name) 12 | shift 13 | DOCKER_NAME="${1}" 14 | echo ${DOCKER_NAME} 15 | shift 16 | ;; 17 | --dataset-path) 18 | shift 19 | DATASET_PATH="${1}" 20 | echo ${DATASET_PATH} 21 | shift 22 | ;; 23 | *) # unknown arg 24 | echo unknown arg ${1} 25 | exit 26 | ;; 27 | esac 28 | done 29 | 30 | 31 | docker run -v ${DATASET_PATH}/igibson.key:/opt/iGibson/igibson/data/igibson.key -v ${DATASET_PATH}/ig_dataset:/opt/iGibson/igibson/data/ig_dataset \ 32 | \ 33 | ${DOCKER_NAME} \ 34 | /bin/bash -c \ 35 | "export CONFIG_FILE=/opt/behavior/behavior/configs/behavior_onboard_sensing.yaml; export SPLIT=minival; cd /opt/behavior/behavior/examples; python training_example.py" 36 | 37 | # for older docker versions, use --runtime=nvidia instead of --gpus=all -------------------------------------------------------------------------------- /behavior/configs/behavior_full_observability.yaml: -------------------------------------------------------------------------------- 1 | # scene 2 | scene: igibson 3 | scene_id: Beechwood_1_int 4 | clutter: false 5 | build_graph: true 6 | load_texture: true 7 | pybullet_load_texture: true 8 | should_open_all_doors: false 9 | 10 | # domain randomization 11 | texture_randomization_freq: null 12 | object_randomization_freq: null 13 | 14 | # robot 15 | robot: 16 | name: BehaviorRobot 17 | 18 | # task 19 | task: cleaning_cupboards 20 | task_id: 0 21 | instance_id: 0 22 | online_sampling: false 23 | 24 | # discount factor 25 | discount_factor: 0.99 26 | 27 | # termination condition 28 | max_step: 500 29 | 30 | # sensor spec 31 | output: [proprioception, rgb, highlight, depth, seg, ins_seg, task_obs] 32 | # image 33 | fisheye: false 34 | image_width: 128 35 | image_height: 128 36 | vertical_fov: 120 37 | # depth 38 | depth_low: 0.0 39 | depth_high: 10.0 40 | 41 | # sensor noise 42 | depth_noise_rate: 0.0 43 | scan_noise_rate: 0.0 44 | -------------------------------------------------------------------------------- /behavior/configs/behavior_full_observability_fetch.yaml: -------------------------------------------------------------------------------- 1 | # scene 2 | scene: igibson 3 | scene_id: Beechwood_1_int 4 | clutter: false 5 | build_graph: true 6 | load_texture: true 7 | pybullet_load_texture: true 8 | should_open_all_doors: false 9 | 10 | # domain randomization 11 | texture_randomization_freq: null 12 | object_randomization_freq: null 13 | 14 | # robot 15 | robot: 16 | name: Fetch 17 | action_type: continuous 18 | action_normalize: true 19 | base_name: null 20 | scale: 1.0 21 | self_collision: true 22 | rendering_params: null 23 | grasping_mode: assisted 24 | rigid_trunk: false 25 | default_trunk_offset: 0.365 26 | default_arm_pose: diagonal30 27 | controller_config: 28 | base: 29 | name: DifferentialDriveController 30 | arm_0: 31 | name: InverseKinematicsController 32 | kv: 2.0 33 | gripper_0: 34 | name: MultiFingerGripperController 35 | mode: binary 36 | camera: 37 | name: JointController 38 | use_delta_commands: False 39 | 40 | # task 41 | task: cleaning_cupboards 42 | task_id: 0 43 | instance_id: 0 44 | online_sampling: false 45 | 46 | # discount factor 47 | discount_factor: 0.99 48 | 49 | # termination condition 50 | max_step: 500 51 | 52 | # sensor spec 53 | output: [proprioception, rgb, highlight, depth, seg, ins_seg, task_obs] 54 | # image 55 | fisheye: false 56 | image_width: 128 57 | image_height: 128 58 | vertical_fov: 120 59 | # depth 60 | depth_low: 0.0 61 | depth_high: 10.0 62 | 63 | # sensor noise 64 | depth_noise_rate: 0.0 65 | scan_noise_rate: 0.0 66 | -------------------------------------------------------------------------------- /behavior/configs/behavior_onboard_sensing.yaml: -------------------------------------------------------------------------------- 1 | # scene 2 | scene: igibson 3 | scene_id: Beechwood_1_int 4 | clutter: false 5 | build_graph: true 6 | load_texture: true 7 | pybullet_load_texture: true 8 | should_open_all_doors: false 9 | 10 | # domain randomization 11 | texture_randomization_freq: null 12 | object_randomization_freq: null 13 | 14 | # robot 15 | robot: 16 | name: BehaviorRobot 17 | 18 | # task 19 | task: cleaning_cupboards 20 | task_id: 0 21 | instance_id: 0 22 | online_sampling: false 23 | 24 | # discount factor 25 | discount_factor: 0.99 26 | 27 | # termination condition 28 | max_step: 500 29 | 30 | # sensor spec 31 | output: [proprioception, rgb, highlight, depth, seg, ins_seg] 32 | # image 33 | fisheye: false 34 | image_width: 128 35 | image_height: 128 36 | vertical_fov: 120 37 | # depth 38 | depth_low: 0.0 39 | depth_high: 10.0 40 | 41 | # sensor noise 42 | depth_noise_rate: 0.0 43 | scan_noise_rate: 0.0 44 | -------------------------------------------------------------------------------- /behavior/configs/behavior_onboard_sensing_fetch.yaml: -------------------------------------------------------------------------------- 1 | # scene 2 | scene: igibson 3 | scene_id: Beechwood_1_int 4 | clutter: false 5 | build_graph: true 6 | load_texture: true 7 | pybullet_load_texture: true 8 | should_open_all_doors: false 9 | 10 | # domain randomization 11 | texture_randomization_freq: null 12 | object_randomization_freq: null 13 | 14 | # robot 15 | robot: 16 | name: Fetch 17 | action_type: continuous 18 | action_normalize: true 19 | base_name: null 20 | scale: 1.0 21 | self_collision: true 22 | rendering_params: null 23 | grasping_mode: physical 24 | rigid_trunk: false 25 | default_trunk_offset: 0.365 26 | default_arm_pose: diagonal30 27 | controller_config: 28 | base: 29 | name: DifferentialDriveController 30 | arm_0: 31 | name: InverseKinematicsController 32 | kv: 2.0 33 | gripper_0: 34 | name: MultiFingerGripperController 35 | mode: binary 36 | camera: 37 | name: JointController 38 | use_delta_commands: False 39 | 40 | # task 41 | task: cleaning_cupboards 42 | task_id: 0 43 | instance_id: 0 44 | online_sampling: false 45 | 46 | # discount factor 47 | discount_factor: 0.99 48 | 49 | # termination condition 50 | max_step: 500 51 | 52 | # sensor spec 53 | output: [proprioception, rgb, highlight, depth, seg, ins_seg] 54 | # image 55 | fisheye: false 56 | image_width: 128 57 | image_height: 128 58 | vertical_fov: 120 59 | # depth 60 | depth_low: 0.0 61 | depth_high: 10.0 62 | 63 | # sensor noise 64 | depth_noise_rate: 0.0 65 | scan_noise_rate: 0.0 66 | -------------------------------------------------------------------------------- /behavior/configs/behavior_segmentation_replay.yaml: -------------------------------------------------------------------------------- 1 | # scene 2 | scene: igibson 3 | scene_id: Beechwood_1_int 4 | clutter: false 5 | build_graph: true 6 | load_texture: true 7 | pybullet_load_texture: true 8 | should_open_all_doors: true 9 | 10 | # Ignore task, use EmptyScene instead 11 | debug: False 12 | 13 | # domain randomization 14 | texture_randomization_freq: null 15 | object_randomization_freq: null 16 | 17 | # robot 18 | robot: 19 | name: BehaviorRobot 20 | 21 | # task 22 | task: cleaning_cupboards 23 | task_id: 0 24 | instance_id: 0 25 | online_sampling: False 26 | target_dist_min: 1.0 27 | target_dist_max: 10.0 28 | goal_format: polar 29 | task_obs_dim: 4 30 | 31 | # reward 32 | reward_type: l2 33 | success_reward: 10.0 34 | potential_reward_weight: 1.0 35 | collision_reward_weight: -0.1 36 | 37 | # discount factor 38 | discount_factor: 0.99 39 | 40 | # termination condition 41 | dist_tol: 0.36 # body width 42 | max_step: 100 43 | max_collisions_allowed: 500 44 | 45 | # misc config 46 | initial_pos_z_offset: 0.1 47 | collision_ignore_link_a_ids: [0, 1, 2] # ignore collisions with these robot links 48 | 49 | # sensor spec 50 | output: [proprioception, task_obs] 51 | # image 52 | # Primesense Carmine 1.09 short-range RGBD sensor 53 | # http://xtionprolive.com/primesense-carmine-1.09 54 | fisheye: false 55 | image_width: 512 56 | image_height: 512 57 | vertical_fov: 90 58 | # depth 59 | depth_low: 0.35 60 | depth_high: 3.0 61 | # scan 62 | # SICK TIM571 scanning range finder 63 | # https://docs.fetchrobotics.com/robot_hardware.html 64 | # n_horizontal_rays is originally 661, sub-sampled 1/3 65 | n_horizontal_rays: 220 66 | n_vertical_beams: 1 67 | laser_linear_range: 25.0 68 | laser_angular_range: 220.0 69 | min_laser_dist: 0.05 70 | laser_link_name: laser_link 71 | 72 | # sensor noise 73 | depth_noise_rate: 0.0 74 | scan_noise_rate: 0.0 75 | 76 | # visual objects 77 | visible_target: true 78 | 79 | 80 | # speed limit 81 | hand_threshold: 0.4 82 | hand_speed: 0.3 83 | body_speed: 0.3 -------------------------------------------------------------------------------- /behavior/configs/behavior_vr.yaml: -------------------------------------------------------------------------------- 1 | # scene 2 | scene: igibson 3 | scene_id: Rs_int 4 | clutter: false 5 | build_graph: true 6 | load_texture: true 7 | pybullet_load_texture: true 8 | should_open_all_doors: false 9 | 10 | # domain randomization 11 | texture_randomization_freq: null 12 | object_randomization_freq: null 13 | 14 | # robot 15 | robot: 16 | name: BehaviorRobot 17 | 18 | # For the VR use case, we use the higher set of joint speed/position limits, and a different gripper action space. 19 | show_visual_head: False 20 | higher_limits: True 21 | controller_config: 22 | gripper_right_hand: 23 | name: JointController 24 | gripper_left_hand: 25 | name: JointController 26 | 27 | # task 28 | task: cleaning_kitchen_cupboard 29 | task_id: 0 30 | instance_id: 0 31 | online_sampling: false 32 | should_highlight_task_relevant_objs: false 33 | 34 | # discount factor 35 | discount_factor: 0.99 36 | 37 | # termination condition 38 | max_step: 500 39 | 40 | # sensor spec 41 | output: [proprioception] 42 | # image 43 | fisheye: false 44 | image_width: 128 45 | image_height: 128 46 | vertical_fov: 120 47 | # depth 48 | depth_low: 0.0 49 | depth_high: 10.0 50 | 51 | # sensor noise 52 | depth_noise_rate: 0.0 53 | scan_noise_rate: 0.0 54 | -------------------------------------------------------------------------------- /behavior/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/behavior/examples/__init__.py -------------------------------------------------------------------------------- /behavior/examples/checkpoint_generation_example.py: -------------------------------------------------------------------------------- 1 | """Save checkpoints from a BEHAVIOR demo.""" 2 | import logging 3 | import os 4 | 5 | import igibson 6 | from igibson.examples.learning.demo_replaying_example import replay_demo, replay_demo_with_determinism_check 7 | from igibson.utils.checkpoint_utils import save_checkpoint 8 | 9 | import behavior 10 | 11 | 12 | def create_checkpoints(demo_file, checkpoint_directory, checkpoint_every_n_steps): 13 | # Create a step callback function to feed replay steps into checkpoints. 14 | def step_callback(env, _): 15 | if not env.task.current_success and env.simulator.frame_count % checkpoint_every_n_steps == 0: 16 | save_checkpoint(env.simulator, checkpoint_directory) 17 | 18 | print("Replaying demo and saving checkpoints") 19 | check_determinism = False 20 | if check_determinism: 21 | replay_demo_with_determinism_check(demo_file, mode="headless", step_callbacks=[step_callback]) 22 | else: 23 | replay_demo(demo_file, mode="headless", step_callbacks=[step_callback]) 24 | 25 | 26 | def main(selection="user", headless=False, short_exec=False): 27 | """ 28 | Opens a demo and creates checkpoints every N steps 29 | Checkpoints can be used to initialize the simulation at those states, for example, for RL 30 | """ 31 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 32 | 33 | demo_file = os.path.join( 34 | igibson.ig_dataset_path, 35 | "tests", 36 | "cleaning_windows_example.hdf5", 37 | ) 38 | checkpoint_directory = os.path.join( 39 | behavior.examples_path, 40 | "data", 41 | "checkpoints", 42 | ) 43 | os.makedirs(checkpoint_directory, exist_ok=True) 44 | 45 | steps_between_checkpoints = 30 if not short_exec else 300 46 | create_checkpoints(demo_file, checkpoint_directory, steps_between_checkpoints) 47 | 48 | 49 | RUN_AS_TEST = False # Change to True to run this example in test mode 50 | if __name__ == "__main__": 51 | logging.basicConfig(level=logging.INFO) 52 | if RUN_AS_TEST: 53 | main(selection="random", headless=True, short_exec=True) 54 | else: 55 | main() 56 | -------------------------------------------------------------------------------- /behavior/examples/config_selector.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from sys import platform 4 | 5 | import igibson 6 | import yaml 7 | from igibson.envs.igibson_env import iGibsonEnv 8 | from igibson.render.profiler import Profiler 9 | from igibson.utils.assets_utils import folder_is_hidden 10 | from igibson.utils.utils import let_user_pick 11 | 12 | import behavior 13 | 14 | 15 | def main(selection="user", headless=False, short_exec=False): 16 | """ 17 | Prompts the user to select any available interactive scene and loads a turtlebot into it. 18 | It steps the environment 100 times with random actions sampled from the action space, 19 | using the Gym interface, resetting it 10 times. 20 | """ 21 | 22 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 23 | 24 | available_configs = get_first_options() 25 | config_id = available_configs[let_user_pick(available_configs, selection=selection) - 1] 26 | print("Using config file " + config_id) 27 | config_filename = os.path.join(behavior.configs_path, config_id) 28 | config_data = yaml.load(open(config_filename, "r"), Loader=yaml.FullLoader) 29 | # Reduce texture scale for Mac. 30 | if platform == "darwin": 31 | config_data["texture_scale"] = 0.5 32 | config_data["image_width"] = 512 33 | config_data["image_height"] = 512 34 | config_data["vertical_fov"] = 60 35 | # config_data["load_object_categories"] = [] # Uncomment this line to accelerate loading with only the building 36 | env = iGibsonEnv(config_file=config_data, mode="gui_interactive" if not headless else "headless") 37 | max_iterations = 10 if not short_exec else 1 38 | for j in range(max_iterations): 39 | print("Resetting environment") 40 | env.reset() 41 | for i in range(100): 42 | with Profiler("Environment action step"): 43 | action = env.action_space.sample() * 0.05 44 | state, reward, done, info = env.step(action) 45 | if done: 46 | print("Episode finished after {} timesteps".format(i + 1)) 47 | break 48 | env.close() 49 | 50 | 51 | def get_first_options(): 52 | config_path = behavior.configs_path 53 | available_configs = sorted( 54 | [ 55 | f 56 | for f in os.listdir(config_path) 57 | if (not folder_is_hidden(f) and os.path.isfile(os.path.join(config_path, f))) 58 | ] 59 | ) 60 | return available_configs 61 | 62 | 63 | RUN_AS_TEST = False # Change to True to run this example in test mode 64 | if __name__ == "__main__": 65 | logging.basicConfig(level=logging.INFO) 66 | if RUN_AS_TEST: 67 | main(selection="random", headless=True, short_exec=True) 68 | else: 69 | main() 70 | -------------------------------------------------------------------------------- /behavior/examples/dataset_generation_il_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to convert BEHAVIOR virtual reality demos to dataset compatible with imitation learning 3 | """ 4 | 5 | import argparse 6 | import inspect 7 | import json 8 | import logging 9 | import os 10 | import sys 11 | from pathlib import Path 12 | 13 | import h5py 14 | import igibson 15 | import numpy as np 16 | import pandas as pd 17 | from igibson.examples.learning.demo_replaying_example import replay_demo 18 | from igibson.metrics.dataset import DatasetMetric 19 | from igibson.utils.ig_logging import IGLogReader 20 | from igibson.utils.utils import parse_config 21 | 22 | import behavior 23 | 24 | 25 | def save_episode(demo_file, dataset_metric): 26 | episode_identifier = "_".join(os.path.splitext(demo_file)[0].split("_")[-2:]) 27 | episode_out_log_path = "processed_hdf5s/{}_{}_{}_{}_{}_episode.hdf5".format( 28 | activity, activity_id, scene, instance_id, episode_identifier 29 | ) 30 | hf = h5py.File(episode_out_log_path, "w") 31 | 32 | # Copy the metadata 33 | for attr in IGLogReader.get_all_metadata_attrs(demo_file): 34 | hf.attrs[attr] = IGLogReader.read_metadata_attr(demo_file, attr) 35 | 36 | for key, value in dataset_metric.gather_results().items(): 37 | if key == "action": 38 | dtype = np.float64 39 | else: 40 | dtype = np.float32 41 | hf.create_dataset(key, data=np.stack(value), dtype=dtype, compression="lzf") 42 | 43 | hf.close() 44 | 45 | 46 | def generate_il_dataset( 47 | demo_dir, 48 | demo_manifest, 49 | out_dir, 50 | config_file, 51 | skip_existing=True, 52 | save_frames=False, 53 | deactivate_logger=True, 54 | ): 55 | """ 56 | Extract pairs of (observation,action) for imitation learning from a batch of BEHAVIOR demos. 57 | 58 | @param demo_dir: Directory containing the demo files listed in the manifests. 59 | @param demo_manifest: The manifest file containing list of BEHAVIOR demos to batch over. 60 | @param out_dir: Directory to store results in. 61 | @param config_file: environment config file 62 | @param skip_existing: Whether demos with existing output logs should be skipped. 63 | @param save_frames: Whether the demo's frames should be saved alongside statistics. 64 | @param deactivate_logger: If we deactivate the logger 65 | """ 66 | 67 | if deactivate_logger: 68 | logger = logging.getLogger() 69 | logger.disabled = True 70 | 71 | demo_list = pd.read_csv(demo_manifest) 72 | 73 | config = parse_config(config_file) 74 | # should NOT activate behavior robot to be consistent with VR demo collection setup 75 | config["should_activate_behavior_robot"] = False 76 | # highlight task relevant objects because the observation includes "highlight" 77 | config["should_highlight_task_relevant_objs"] = True 78 | 79 | for idx, demo in enumerate(demo_list["demos"]): 80 | if "replay" in demo: 81 | continue 82 | 83 | demo_name = os.path.splitext(demo)[0] 84 | demo_file = os.path.join(demo_dir, demo) 85 | out_log_file = os.path.join(out_dir, demo_name + "_log.json") 86 | 87 | if skip_existing and os.path.exists(out_log_file): 88 | print( 89 | "Skipping demo because an output log file already exists (we assume it has been processed): {}, {} out of {}".format( 90 | demo, idx, len(demo_list["demos"]) 91 | ) 92 | ) 93 | continue 94 | 95 | print("Replaying demo: {}, {} out of {}".format(demo, idx, len(demo_list["demos"]))) 96 | 97 | curr_frame_save_path = None 98 | if save_frames: 99 | curr_frame_save_path = os.path.join(out_dir, demo_name + ".mp4") 100 | 101 | try: 102 | dataset = DatasetMetric() 103 | demo_information = replay_demo( 104 | in_log_path=demo_file, 105 | frame_save_path=curr_frame_save_path, 106 | mode="headless", 107 | config_file=config_file, 108 | verbose=False, 109 | image_size=(128, 128), 110 | start_callbacks=[dataset.start_callback], 111 | step_callbacks=[dataset.step_callback], 112 | end_callbacks=[dataset.end_callback], 113 | ) 114 | demo_information["failed"] = False 115 | demo_information["filename"] = Path(demo).name 116 | save_episode(demo_file, dataset) 117 | 118 | except Exception as e: 119 | logging.error("Demo failed withe error: {}".format(e)) 120 | demo_information = {"demo_id": Path(demo).name, "failed": True, "failure_reason": str(e)} 121 | 122 | print("Saving data") 123 | with open(out_log_file, "w") as file: 124 | json.dump(demo_information, file) 125 | 126 | 127 | def parse_args(defaults=False): 128 | args_dict = dict() 129 | args_dict["demo_dir"] = os.path.join(igibson.ig_dataset_path, "tests") 130 | args_dict["log_manifest"] = os.path.join(igibson.ig_dataset_path, "tests", "test_manifest.txt") 131 | args_dict["out_dir"] = os.path.join(behavior.examples_path, "data") 132 | args_dict["config"] = os.path.join(behavior.configs_path, "behavior_vr.yaml") 133 | if not defaults: 134 | parser = argparse.ArgumentParser( 135 | description="Extract pairs of (observation,action) for imitation learning from a batch of BEHAVIOR demos." 136 | ) 137 | parser.add_argument("--demo_dir", type=str, help="Directory containing demos listed in the manifest.") 138 | parser.add_argument("--log_manifest", type=str, help="Plain text file consisting of list of demos to replay.") 139 | parser.add_argument("--out_dir", type=str, help="Directory to store results in.") 140 | parser.add_argument( 141 | "--config", 142 | help="which config file to use [default: use yaml files in examples/configs]", 143 | default=args_dict["config"], 144 | ) 145 | 146 | args = parser.parse_args() 147 | args_dict["demo_dir"] = args.demo_dir 148 | args_dict["log_manifest"] = args.log_manifest 149 | args_dict["out_dir"] = args.out_dir 150 | args_dict["config"] = args.config 151 | 152 | return args_dict 153 | 154 | 155 | def main(selection="user", headless=False, short_exec=False): 156 | """ 157 | Extract pairs of (observation,action) for imitation learning from a batch of BEHAVIOR demos. 158 | """ 159 | 160 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 161 | 162 | defaults = selection == "random" and headless and short_exec 163 | args_dict = parse_args(defaults=defaults) 164 | 165 | generate_il_dataset(args_dict["demo_dir"], args_dict["log_manifest"], args_dict["out_dir"], args_dict["config"]) 166 | 167 | 168 | RUN_AS_TEST = False # Change to True to run this example in test mode 169 | if __name__ == "__main__": 170 | logging.basicConfig(level=logging.INFO) 171 | if RUN_AS_TEST: 172 | main(selection="random", headless=True, short_exec=True) 173 | else: 174 | main() 175 | -------------------------------------------------------------------------------- /behavior/examples/demo_segmentation_batch.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import inspect 3 | import logging 4 | import os 5 | 6 | import igibson 7 | from igibson.examples.learning.demo_replaying_batch import replay_demo_batch 8 | 9 | import behavior 10 | from behavior.examples.demo_segmentation_example import get_default_segmentation_processors 11 | 12 | 13 | def parse_args(defaults=False): 14 | args_dict = dict() 15 | args_dict["demo_dir"] = os.path.join(igibson.ig_dataset_path, "tests") 16 | args_dict["demo_manifest"] = os.path.join(igibson.ig_dataset_path, "tests", "test_manifest.txt") 17 | args_dict["out_dir"] = os.path.join(behavior.examples_path, "data") 18 | if not defaults: 19 | parser = argparse.ArgumentParser(description="Segment a batch of BEHAVIOR demos in manifest.") 20 | parser.add_argument("demo_dir", type=str, help="Directory containing demos listed in the manifest.") 21 | parser.add_argument("demo_manifest", type=str, help="Plain text file consisting of list of demos to replay.") 22 | parser.add_argument("out_dir", type=str, help="Directory to store results in.") 23 | args = parser.parse_args() 24 | args_dict["demo_dir"] = args.demo_dir 25 | args_dict["demo_manifest"] = args.demo_manifest 26 | args_dict["out_dir"] = args.out_dir 27 | 28 | return args_dict 29 | 30 | 31 | def main(selection="user", headless=False, short_exec=False): 32 | """ 33 | Segment a batch of demos 34 | Use a manifest file to indicate the demos to segment 35 | """ 36 | 37 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 38 | 39 | testing = selection == "random" and headless and short_exec 40 | args_dict = parse_args(defaults=testing) 41 | 42 | def get_segmentation_callbacks(**kwargs): 43 | # Create default segmentation processors. 44 | segmentation_processors = get_default_segmentation_processors() 45 | 46 | # Create a data callback that unifies the results from the 47 | # segmentation processors. 48 | def data_callback(): 49 | data_cb_ret = dict() 50 | for name, sp in segmentation_processors.items(): 51 | print("Serializing segmentation {}".format(name)) 52 | data_cb_ret[name] = sp.serialize_segments() 53 | return data_cb_ret 54 | 55 | # Return all of the callbacks for a particular demo. 56 | return ( 57 | [sp.start_callback for sp in segmentation_processors.values()], 58 | [sp.step_callback for sp in segmentation_processors.values()], 59 | [], 60 | [data_callback], 61 | ) 62 | 63 | print("Run segmentation") 64 | replay_demo_batch( 65 | args_dict["demo_dir"], 66 | args_dict["demo_manifest"], 67 | args_dict["out_dir"], 68 | get_segmentation_callbacks, 69 | skip_existing=not testing, # Do not skip when testing 70 | ignore_errors=not testing, # Do not ignore when testing 71 | ) 72 | print("Batch of demos segmented!") 73 | 74 | 75 | RUN_AS_TEST = False # Change to True to run this example in test mode 76 | if __name__ == "__main__": 77 | logging.basicConfig(level=logging.INFO) 78 | if RUN_AS_TEST: 79 | main(selection="random", headless=True, short_exec=True) 80 | else: 81 | main() 82 | -------------------------------------------------------------------------------- /behavior/examples/example_selector.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import pkgutil 3 | import signal 4 | import sys 5 | from multiprocessing import Process 6 | 7 | from igibson.utils.utils import let_user_pick 8 | 9 | import behavior.examples as examples 10 | 11 | TIMEOUT = 4 12 | 13 | 14 | def interrupted(signum, frame): 15 | raise ValueError("Time-up for keyboard input") 16 | 17 | 18 | signal.signal(signal.SIGALRM, interrupted) 19 | 20 | 21 | def timed_input(example_name): 22 | try: 23 | print("Next example: " + example_name) 24 | input("Press ENTER to skip or wait 4 secs to execute\n") 25 | return True 26 | except ValueError: 27 | # timeout 28 | return False 29 | 30 | 31 | def main(): 32 | """ 33 | Selector tool to see all available examples and pick one to run or get more information, 34 | or run all examples one after the other 35 | """ 36 | examples_list = ["help", "all", "quit"] 37 | for kk in pkgutil.walk_packages(examples.__path__, examples.__name__ + "."): 38 | # Exclude package files, the example_selector itself 39 | if not kk.ispkg and kk.name[18:] != "example_selector": 40 | examples_list += [kk.name[18:]] 41 | 42 | selected_demo = 0 43 | logo = ( 44 | " ____ ______ _ _ __ _______ ____ _____" 45 | + "\n" 46 | + " | _ \\| ____| | | | /\\ \ / /_ _/ __ \\| __ \\ " 47 | + "\n" 48 | + " | |_) | |__ | |__| | / \ \\ / / | || | | | |__) |" 49 | + "\n" 50 | + " | _ <| __| | __ | / /\\ \\ \\/ / | || | | | _ / " 51 | + "\n" 52 | + " | |_) | |____| | | |/ ____ \\ / _| || |__| | | \\ \\ " 53 | + "\n" 54 | + " |____/|______|_| |_/_/ \\_\\/ |_____\\____/|_| \\_\\" 55 | ) 56 | test_mode = False 57 | while selected_demo == 0 or selected_demo == 3: 58 | print(logo) 59 | print("Select a demo/example, 'help' for information about a specific demo, or 'all' to run all demos:") 60 | selected_demo = let_user_pick(examples_list, print_intro=False) - 1 61 | if selected_demo == 0: 62 | user_input = input("\nProvide the number of the example you need information for: ") 63 | if not user_input.isdigit(): 64 | continue 65 | else: 66 | help_demo = int(user_input) - 1 67 | if help_demo == 0: 68 | print("Print the description of a demo/example") 69 | elif help_demo == 1: 70 | print("Execute all demos/examples in order") 71 | elif help_demo == 2: 72 | print("Quit the exampler") 73 | else: 74 | module_help = importlib.import_module("behavior.examples." + examples_list[help_demo]) 75 | print(module_help.main.__doc__) 76 | input("Press enter") 77 | elif selected_demo == 1: 78 | print("Executing all examples:") 79 | for idx in range(4, len(examples_list)): 80 | print("*" * 80) 81 | print("*" * 80) 82 | print(logo) 83 | print("*" * 80) 84 | print("*" * 80) 85 | signal.alarm(TIMEOUT) 86 | s = timed_input(examples_list[idx]) 87 | # disable the alarm after success 88 | signal.alarm(0) 89 | if s: 90 | continue 91 | print("Executing " + examples_list[idx]) 92 | 93 | i = importlib.import_module("behavior.examples." + examples_list[idx]) 94 | p = Process( 95 | target=i.main, 96 | ) 97 | 98 | p.start() 99 | p.join() 100 | print("Ended " + examples_list[idx]) 101 | 102 | elif selected_demo == 2: 103 | print("Exit") 104 | return 105 | else: 106 | print("Executing " + examples_list[selected_demo]) 107 | i = importlib.import_module("behavior.examples." + examples_list[selected_demo]) 108 | i.main() 109 | 110 | 111 | if __name__ == "__main__": 112 | logging.basicConfig(level=logging.INFO) 113 | main() 114 | -------------------------------------------------------------------------------- /behavior/examples/logic_states_checking_in_demo.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import logging 3 | import os 4 | import sys 5 | 6 | import bddl 7 | import igibson 8 | from igibson import object_states 9 | from igibson.examples.learning import demo_replaying_example 10 | 11 | import behavior 12 | 13 | 14 | def robot_states_callback(env, _): 15 | window1 = (env.task.object_scope["window.n.01_1"], "kitchen") 16 | window2 = (env.task.object_scope["window.n.01_2"], "living room") 17 | windows = [window1, window2] 18 | 19 | for window, roomname in windows: 20 | print( 21 | "%s window is inFOV: %r, inSameRoom: %r, inReach: %r" 22 | % ( 23 | roomname, 24 | window.states[object_states.InFOVOfRobot].get_value(), 25 | window.states[object_states.InSameRoomAsRobot].get_value(), 26 | window.states[object_states.InReachOfRobot].get_value(), 27 | ) 28 | ) 29 | 30 | rag = env.task.object_scope["rag.n.01_1"] 31 | print("Rag is in hand: %r" % rag.states[object_states.InHandOfRobot].get_value()) 32 | 33 | agent = env.task.object_scope["agent.n.01_1"] 34 | print( 35 | "Agent is in kitchen: %r, living room: %r, bedroom: %r." 36 | % ( 37 | agent.states[object_states.IsInKitchen].get_value(), 38 | agent.states[object_states.IsInLivingRoom].get_value(), 39 | agent.states[object_states.IsInBedroom].get_value(), 40 | ) 41 | ) 42 | 43 | 44 | def main(selection="user", headless=False, short_exec=False): 45 | """ 46 | Replays a demo and prints some predefined logic states for the robot at each step 47 | This demonstrates how to check some logic states of interest at each step of a give demo (e.g. BEHAVIOR demos) 48 | """ 49 | 50 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 51 | 52 | DEMO_FILE = os.path.join( 53 | igibson.ig_dataset_path, 54 | "tests", 55 | "cleaning_windows_example.hdf5", 56 | ) 57 | 58 | demo_replaying_example.replay_demo( 59 | DEMO_FILE, disable_save=True, step_callbacks=[robot_states_callback], mode="headless" 60 | ) 61 | 62 | 63 | RUN_AS_TEST = False # Change to True to run this example in test mode 64 | if __name__ == "__main__": 65 | logging.basicConfig(level=logging.INFO) 66 | if RUN_AS_TEST: 67 | main(selection="random", headless=True, short_exec=True) 68 | else: 69 | main() 70 | -------------------------------------------------------------------------------- /behavior/examples/metric_computation_batch.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import inspect 3 | import logging 4 | import os 5 | import sys 6 | 7 | import igibson 8 | from igibson.examples.learning.demo_replaying_batch import replay_demo_batch 9 | from igibson.metrics.agent import RobotMetric 10 | from igibson.metrics.disarrangement import KinematicDisarrangement, LogicalDisarrangement 11 | from igibson.metrics.gaze import GazeMetric 12 | from igibson.metrics.task import TaskMetric 13 | 14 | import behavior 15 | 16 | 17 | def parse_args(defaults=False): 18 | args_dict = dict() 19 | args_dict["demo_dir"] = os.path.join(igibson.ig_dataset_path, "tests") 20 | args_dict["demo_manifest"] = os.path.join(igibson.ig_dataset_path, "tests", "test_manifest.txt") 21 | args_dict["out_dir"] = os.path.join(behavior.examples_path, "data") 22 | if not defaults: 23 | parser = argparse.ArgumentParser(description="Collect metrics from BEHAVIOR demos in manifest.") 24 | parser.add_argument("demo_dir", type=str, help="Directory containing demos listed in the manifest.") 25 | parser.add_argument("demo_manifest", type=str, help="Plain text file consisting of list of demos to replay.") 26 | parser.add_argument("out_dir", type=str, help="Directory to store results in.") 27 | args = parser.parse_args() 28 | args_dict["demo_dir"] = args.demo_dir 29 | args_dict["demo_manifest"] = args.demo_manifest 30 | args_dict["out_dir"] = args.out_dir 31 | return args_dict 32 | 33 | 34 | def main(selection="user", headless=False, short_exec=False): 35 | """ 36 | Compute metrics on a batch of previously recorded demos 37 | Uses a manifest file 38 | """ 39 | 40 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 41 | 42 | testing = selection == "random" and headless and short_exec 43 | args_dict = parse_args(defaults=testing) 44 | 45 | def get_metrics_callbacks(**kwargs): 46 | metrics = [ 47 | KinematicDisarrangement(), 48 | LogicalDisarrangement(), 49 | RobotMetric(), 50 | # GazeMetric(), # TODO: Test fails, probably is not including the right VR info 51 | TaskMetric(), 52 | ] 53 | 54 | return ( 55 | [metric.start_callback for metric in metrics], 56 | [metric.step_callback for metric in metrics], 57 | [metric.end_callback for metric in metrics], 58 | [metric.gather_results for metric in metrics], 59 | ) 60 | 61 | replay_demo_batch( 62 | args_dict["demo_dir"], 63 | args_dict["demo_manifest"], 64 | args_dict["out_dir"], 65 | get_metrics_callbacks, 66 | skip_existing=not testing, # Do not skip when testing 67 | ignore_errors=not testing, # Do not ignore when testing 68 | ) 69 | 70 | 71 | RUN_AS_TEST = False # Change to True to run this example in test mode 72 | if __name__ == "__main__": 73 | logging.basicConfig(level=logging.INFO) 74 | if RUN_AS_TEST: 75 | main(selection="random", headless=True, short_exec=True) 76 | else: 77 | main() 78 | -------------------------------------------------------------------------------- /behavior/examples/metric_computation_example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import json 4 | import os 5 | 6 | from igibson.envs.igibson_env import iGibsonEnv 7 | from igibson.metrics.agent import RobotMetric 8 | from igibson.metrics.disarrangement import KinematicDisarrangement, LogicalDisarrangement 9 | from igibson.metrics.task import TaskMetric 10 | 11 | import behavior 12 | 13 | 14 | def get_metrics_callbacks(config): 15 | metrics = [ 16 | KinematicDisarrangement(), 17 | LogicalDisarrangement(), 18 | TaskMetric(), 19 | ] 20 | 21 | robot_type = config["robot"] 22 | if robot_type == "Fetch" or robot_type == "BehaviorRobot": 23 | metrics.append(RobotMetric()) 24 | else: 25 | Exception("Metrics only implemented for Fetch and BehaviorRobot") 26 | 27 | return ( 28 | [metric.start_callback for metric in metrics], 29 | [metric.step_callback for metric in metrics], 30 | [metric.end_callback for metric in metrics], 31 | [metric.gather_results for metric in metrics], 32 | ) 33 | 34 | 35 | def parse_args(defaults=False): 36 | args_dict = dict() 37 | args_dict["config"] = os.path.join(behavior.configs_path, "behavior_full_observability.yaml") 38 | args_dict["mode"] = "headless" 39 | args_dict["out_metric_log_file"] = os.path.join(behavior.examples_path, "data", "metrics_log.json") 40 | if not defaults: 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument( 43 | "--config", 44 | "-c", 45 | default=args_dict["config"], 46 | help="which config file to use [default: use yaml files in examples/configs]", 47 | ) 48 | parser.add_argument( 49 | "--mode", 50 | "-m", 51 | choices=["headless", "headless_tensor", "gui_interactive", "gui_non_interactive"], 52 | default=args_dict["mode"], 53 | help="which mode for simulation (default: headless)", 54 | ) 55 | parser.add_argument( 56 | "--out_metric_log_file", 57 | help="where to save the logging results of the metric computation", 58 | ) 59 | args = parser.parse_args() 60 | 61 | args_dict["config"] = args.config 62 | args_dict["mode"] = args.mode 63 | args_dict["out_metric_log_file"] = args.out_metric_log_file 64 | return args_dict 65 | 66 | 67 | def main(selection="user", headless=False, short_exec=False): 68 | """ 69 | Compute metrics "live" on a running environment with random actions 70 | """ 71 | 72 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 73 | 74 | defaults = selection == "random" and headless and short_exec 75 | args_dict = parse_args(defaults=defaults) 76 | 77 | if headless: 78 | args_dict["mode"] = "headless" 79 | 80 | print("Create environment") 81 | env = iGibsonEnv( 82 | config_file=args_dict["config"], 83 | mode=args_dict["mode"], 84 | ) 85 | 86 | start_callbacks, step_callbacks, end_callbacks, data_callbacks = get_metrics_callbacks(env.config) 87 | 88 | per_episode_metrics = {} 89 | for callback in start_callbacks: 90 | callback(env, None) 91 | 92 | print("Starting metric example") 93 | num_resets = 10 if not short_exec else 2 94 | num_steps = ( 95 | 1000 if not short_exec else 200 96 | ) # At least 150 steps to settle the simulator and cache the initial state 97 | for episode in range(num_resets): 98 | print("Resetting environment") 99 | env.reset() 100 | for i in range(num_steps): 101 | print("Stepping with random actions") 102 | action = env.action_space.sample() 103 | state, reward, done, _ = env.step(action) 104 | for callback in step_callbacks: 105 | callback(env, None) 106 | if done: 107 | break 108 | 109 | for callback in end_callbacks: 110 | callback(env, None) 111 | 112 | metrics_summary = {} 113 | 114 | for callback in data_callbacks: 115 | metrics_summary.update(callback()) 116 | 117 | per_episode_metrics[episode] = metrics_summary 118 | 119 | print("Writing metric computation results") 120 | with open(args_dict["out_metric_log_file"], "w") as file: 121 | json.dump(per_episode_metrics, file) 122 | 123 | env.close() 124 | 125 | 126 | RUN_AS_TEST = False # Change to True to run this example in test mode 127 | if __name__ == "__main__": 128 | logging.basicConfig(level=logging.INFO) 129 | if RUN_AS_TEST: 130 | main(selection="random", headless=True, short_exec=True) 131 | else: 132 | main() 133 | -------------------------------------------------------------------------------- /behavior/examples/replay_demo_with_action_primitives_batch.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import inspect 4 | import logging 5 | import os 6 | import subprocess 7 | import sys 8 | 9 | import igibson 10 | import tqdm 11 | 12 | import behavior 13 | 14 | 15 | def parse_args(defaults=False): 16 | 17 | args_dict = dict() 18 | args_dict["demo_dir"] = os.path.join(igibson.ig_dataset_path, "tests") 19 | args_dict["segm_dir"] = os.path.join(behavior.examples_path, "data") 20 | args_dict["out_dir"] = os.path.join(behavior.examples_path, "data") 21 | 22 | if not defaults: 23 | parser = argparse.ArgumentParser(description="Script to batch-replay segmented demos using action primitives.") 24 | parser.add_argument("demo_dir", type=str, help="Path to directory containing demos") 25 | parser.add_argument("segm_dir", type=str, help="Path to directory containing demo segmentations") 26 | parser.add_argument("out_dir", type=str, help="Path to directory to store results in") 27 | args = parser.parse_args() 28 | args_dict["demo_dir"] = args.demo_dir 29 | args_dict["segm_dir"] = args.segm_dir 30 | args_dict["out_dir"] = args.out_dir 31 | 32 | return args_dict 33 | 34 | 35 | def main(selection="user", headless=False, short_exec=False): 36 | """ 37 | Replays a batch of demos using action primitives 38 | Creates threads for a batch of demos to be replayed in parallel 39 | Uses the code in replay_demo_with_action_primitives.py 40 | """ 41 | 42 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 43 | 44 | defaults = selection == "random" and headless and short_exec 45 | args_dict = parse_args(defaults=defaults) 46 | 47 | skip_if_existing = not defaults # Not skipping if testing 48 | 49 | # Find all the segmentation files by searching for json files in the segmentation directory 50 | segm_files = list(glob.glob(os.path.join(args_dict["segm_dir"], "*_segm.json"))) 51 | demo_files = list(glob.glob(os.path.join(args_dict["demo_dir"], "*.hdf5"))) 52 | 53 | print("Segmentations to replay with action primitives: {}".format(len(segm_files))) 54 | 55 | # Load the demo to get info 56 | for demo_file in tqdm.tqdm(demo_files): 57 | demo = os.path.splitext(os.path.basename(demo_file))[0] 58 | if "replay" in demo: 59 | continue 60 | 61 | for segm_file in segm_files: 62 | if demo in segm_file: 63 | segm_name = os.path.splitext(os.path.basename(segm_file))[0] 64 | ap_replay_demo_file = os.path.join(args_dict["out_dir"], segm_name + "_ap_replay.json") 65 | out_log_file = os.path.join(args_dict["out_dir"], segm_name + "_ap_replay.log") 66 | 67 | if os.path.exists(ap_replay_demo_file) and skip_if_existing: 68 | print("Skipping demo because it exists already: {}".format(ap_replay_demo_file)) 69 | continue 70 | 71 | # Batch me 72 | script_file = os.path.join(behavior.examples_path, "replay_demo_with_action_primitives_example.py") 73 | command = ["python", script_file, demo_file, segm_file, ap_replay_demo_file] 74 | 75 | with open(out_log_file, "w") as log_file: 76 | print("Launching subprocess for demo. Command: {}. Log file: {}".format(command, out_log_file)) 77 | tqdm.tqdm.write("Processing %s" % demo) 78 | subprocess.run(command, stdout=log_file, stderr=subprocess.STDOUT) 79 | 80 | 81 | RUN_AS_TEST = False # Change to True to run this example in test mode 82 | if __name__ == "__main__": 83 | logging.basicConfig(level=logging.INFO) 84 | if RUN_AS_TEST: 85 | main(selection="random", headless=True, short_exec=True) 86 | else: 87 | main() 88 | -------------------------------------------------------------------------------- /behavior/examples/replay_demo_with_action_primitives_example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import inspect 3 | import json 4 | import logging 5 | import os 6 | import sys 7 | import time 8 | 9 | import igibson 10 | import yaml 11 | from igibson.envs.behavior_mp_env import ActionPrimitives, BehaviorMPEnv 12 | from igibson.utils.ig_logging import IGLogReader 13 | 14 | import behavior 15 | 16 | 17 | def get_empty_hand(current_hands): 18 | if len(current_hands) == 0: 19 | return "right_hand" 20 | elif len(current_hands) == 1: 21 | return "left_hand" if list(current_hands.values())[0] == "right_hand" else "right_hand" 22 | 23 | raise ValueError("Both hands are full but you are trying to execute a grasp.") 24 | 25 | 26 | def get_actions_from_segmentation(demo_data): 27 | print("Conversion of demo segmentation to action primitives:") 28 | 29 | hand_by_object = {} 30 | actions = [] 31 | 32 | segmentation = demo_data["sub_segments"] 33 | 34 | # Convert the segmentation to a sequence of state changes. 35 | state_changes = [] 36 | for segment in segmentation: 37 | state_records = segment["state_records"] 38 | if len(state_records) == 0: 39 | print("Found segment with no useful state changes: {}".format(segment)) 40 | continue 41 | elif len(state_records) > 1: 42 | print("Found segment with multiple state changes, using the first: {}".format(segment)) 43 | 44 | state_change = state_records[0] 45 | state_changes.append(state_change) 46 | 47 | # Now go through the state changes and convert them to actions 48 | for i, state_change in enumerate(state_changes): 49 | # Handle the combinations that we support. 50 | state_name = state_change["name"] 51 | state_value = state_change["value"] 52 | 53 | # TODO(replayMP): Here we compute grasps based on the InHand state. Ditch this and simply do a single-hand 54 | # grasp on the object we will manipulate next. That way it will be fetch-compatible. 55 | if state_name == "Open" and state_value is True: 56 | primitive = ActionPrimitives.OPEN 57 | target_object = state_change["objects"][0] 58 | elif state_name == "Open" and state_value is False: 59 | primitive = ActionPrimitives.CLOSE 60 | target_object = state_change["objects"][0] 61 | elif state_name == "InReachOfRobot" and state_value is True: 62 | # The primitives support automatic navigation to relevant objects. 63 | continue 64 | elif state_name == "InHandOfRobot" and state_value is True: 65 | target_object = state_change["objects"][0] 66 | 67 | # Check that we do something with this object later on, otherwise don't grasp it. 68 | is_used = False 69 | for future_state_change in state_changes[i + 1 :]: 70 | # We should only grasp the moved object in these cases. 71 | if future_state_change["objects"][0] != target_object: 72 | continue 73 | 74 | if future_state_change["name"] == "InHandOfRobot" and future_state_change["value"] is True: 75 | # This object is re-grasped later. No need to look any further than that. 76 | break 77 | 78 | # We only care about Inside and OnTop use cases later. 79 | if future_state_change["name"] not in ("Inside", "OnTop") or future_state_change["value"] is False: 80 | continue 81 | 82 | # This is a supported use case so we approve the grasp. 83 | is_used = True 84 | break 85 | 86 | # If the object is not used in the future, don't grasp it. 87 | if not is_used: 88 | continue 89 | 90 | hand = get_empty_hand(hand_by_object) 91 | hand_by_object[target_object] = hand 92 | primitive = ActionPrimitives.LEFT_GRASP if hand == "left_hand" else ActionPrimitives.RIGHT_GRASP 93 | elif state_name == "Inside" and state_value is True: 94 | placed_object = state_change["objects"][0] 95 | target_object = state_change["objects"][1] 96 | if placed_object not in hand_by_object: 97 | print( 98 | "Placed object %s in segment %d not currently grasped. Maybe some sort of segmentation error?".format( 99 | placed_object, i 100 | ) 101 | ) 102 | continue 103 | hand = hand_by_object[placed_object] 104 | del hand_by_object[placed_object] 105 | primitive = ( 106 | ActionPrimitives.LEFT_PLACE_INSIDE if hand == "left_hand" else ActionPrimitives.RIGHT_PLACE_INSIDE 107 | ) 108 | elif state_name == "OnTop" and state_value is True: 109 | placed_object = state_change["objects"][0] 110 | target_object = state_change["objects"][1] 111 | if placed_object not in hand_by_object: 112 | print( 113 | "Placed object %s in segment %d not currently grasped. Maybe some sort of segmentation error?".format( 114 | placed_object, i 115 | ) 116 | ) 117 | continue 118 | hand = hand_by_object[placed_object] 119 | del hand_by_object[placed_object] 120 | primitive = ActionPrimitives.LEFT_PLACE_ONTOP if hand == "left_hand" else ActionPrimitives.RIGHT_PLACE_ONTOP 121 | else: 122 | raise ValueError("Found a state change we can't process: %r" % state_change) 123 | 124 | # Append the action. 125 | action = (primitive, target_object) 126 | actions.append(action) 127 | print("Action: {}".format(action)) 128 | 129 | print("Conversion completed") 130 | return actions 131 | 132 | 133 | def replay_demo_with_aps(demo_file, segm_file, ap_replay_log_file, config_file): 134 | task = IGLogReader.read_metadata_attr(demo_file, "/metadata/atus_activity") 135 | task_id = IGLogReader.read_metadata_attr(demo_file, "/metadata/activity_definition") 136 | scene_id = IGLogReader.read_metadata_attr(demo_file, "/metadata/scene_id") 137 | 138 | # Load the segmentation of a demo for this task. 139 | with open(segm_file, "r") as f: 140 | selected_demo_data = json.load(f) 141 | 142 | # Get the actions from the segmentation 143 | actions = get_actions_from_segmentation(selected_demo_data) 144 | 145 | # Prepare the environment 146 | with open(config_file, "r") as f: 147 | config = yaml.safe_load(f) 148 | 149 | config["task"] = task 150 | config["task_id"] = task_id 151 | config["scene_id"] = scene_id 152 | 153 | env = BehaviorMPEnv( 154 | config_file=config, 155 | mode="headless", 156 | action_timestep=1.0 / 300.0, 157 | physics_timestep=1.0 / 300.0, 158 | use_motion_planning=False, 159 | activity_relevant_objects_only=False, 160 | ) 161 | 162 | start = time.time() 163 | env.reset() 164 | 165 | # env.robots[0].set_position_orientation([0, 0, 0.7], [0, 0, 0, 1]) 166 | done = False 167 | infos = [] 168 | action_successes = [] 169 | import pybullet as p 170 | 171 | p.configureDebugVisualizer(p.COV_ENABLE_GUI, False) 172 | p.resetDebugVisualizerCamera(cameraTargetPosition=[1, -1, 0], cameraDistance=4, cameraYaw=240, cameraPitch=-45) 173 | for action_pair in actions: 174 | # try: 175 | print("Executing {}".format(action_pair)) 176 | primitive, obj_name = action_pair 177 | 178 | # Convert the action 179 | obj_id = next(i for i, obj in enumerate(env.addressable_objects) if obj.name == obj_name) 180 | action = int(primitive) * env.num_objects + obj_id 181 | 182 | # Execute. 183 | state, reward, done, info = env.step(action) 184 | print("Reward: {}, Info: {}".format(reward, info)) 185 | infos.append(info) 186 | action_successes.append(True) 187 | if done: 188 | break 189 | # except: 190 | # action_successes.append(False) 191 | 192 | # Dump the results 193 | data = {"actions": actions, "infos": infos, "action_successes": action_successes} 194 | with open(ap_replay_log_file, "w") as f: 195 | json.dump(data, f) 196 | 197 | print( 198 | "Episode finished after {} timesteps, took {} seconds. Done: {}".format( 199 | env.current_step, time.time() - start, done 200 | ) 201 | ) 202 | env.close() 203 | 204 | 205 | def parse_args(defaults=False): 206 | args_dict = dict() 207 | args_dict["demo_file"] = os.path.join( 208 | igibson.ig_dataset_path, 209 | "tests", 210 | "cleaning_windows_example.hdf5", 211 | ) 212 | args_dict["segm_file"] = os.path.join( 213 | behavior.examples_path, 214 | "data", 215 | "cleaning_windows_example_flat_segm.json", 216 | ) 217 | args_dict["ap_replay_log_file"] = os.path.splitext(args_dict["demo_file"])[0] + "_ap_replay.json" 218 | 219 | # Todo: maybe better behavior_vr.yaml? 220 | args_dict["config"] = os.path.join(behavior.configs_path, "behavior_full_observability.yaml") 221 | 222 | if not defaults: 223 | parser = argparse.ArgumentParser() 224 | parser.add_argument("demo_file", type=str, help="Path of the demo hdf5 to replay.") 225 | parser.add_argument("segm_file", type=str, help="Path of the segmentation of the demo.") 226 | parser.add_argument("ap_replay_log_file", type=str, help="Path to output the log of the replay.") 227 | parser.add_argument("--config", help="which config file to use [default: use yaml files in examples/configs]") 228 | args = parser.parse_args() 229 | args_dict["demo_file"] = args.demo_file 230 | args_dict["segm_file"] = args.segm_file 231 | args_dict["ap_replay_log_file"] = args.ap_replay_log_file 232 | args_dict["config"] = args.config 233 | 234 | return args_dict 235 | 236 | 237 | def main(selection="user", headless=False, short_exec=False): 238 | """ 239 | Replays a demo using action primitives 240 | The demo must be segmented before into a valid sequence of action primitives 241 | """ 242 | 243 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 244 | 245 | defaults = selection == "random" and headless and short_exec 246 | args_dict = parse_args(defaults=defaults) 247 | 248 | replay_demo_with_aps( 249 | args_dict["demo_file"], args_dict["segm_file"], args_dict["ap_replay_log_file"], args_dict["config"] 250 | ) 251 | 252 | 253 | RUN_AS_TEST = False # Change to True to run this example in test mode 254 | if __name__ == "__main__": 255 | logging.basicConfig(level=logging.INFO) 256 | if RUN_AS_TEST: 257 | main(selection="random", headless=True, short_exec=True) 258 | else: 259 | main() 260 | -------------------------------------------------------------------------------- /behavior/examples/threed_detection_dataset_generation_example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import inspect 3 | import itertools 4 | import logging 5 | import os 6 | import sys 7 | 8 | import h5py 9 | import igibson 10 | import numpy as np 11 | import pybullet as p 12 | import trimesh 13 | from igibson.examples.learning.demo_replaying_batch import replay_demo_batch 14 | from igibson.objects.articulated_object import URDFObject 15 | from igibson.utils import utils 16 | from igibson.utils.constants import MAX_INSTANCE_COUNT, SemanticClass 17 | 18 | import behavior 19 | 20 | FRAME_BATCH_SIZE = 100 21 | START_FRAME = 500 22 | SUBSAMPLE_EVERY_N_FRAMES = 60 23 | SAVE_CAMERA = False 24 | DEBUG_DRAW = False 25 | 26 | 27 | def is_subsampled_frame(frame_count): 28 | return frame_count >= START_FRAME and (frame_count - START_FRAME) % SUBSAMPLE_EVERY_N_FRAMES == 0 29 | 30 | 31 | def frame_to_entry_idx(frame_count): 32 | return (frame_count - START_FRAME) // SUBSAMPLE_EVERY_N_FRAMES 33 | 34 | 35 | def parse_args(defaults=False): 36 | args_dict = dict() 37 | args_dict["demo_dir"] = os.path.join(igibson.ig_dataset_path, "tests") 38 | args_dict["demo_manifest"] = os.path.join(igibson.ig_dataset_path, "tests", "test_manifest.txt") 39 | args_dict["out_dir"] = os.path.join(behavior.examples_path, "data") 40 | 41 | if not defaults: 42 | 43 | parser = argparse.ArgumentParser(description="Extract ImVoteNet training data from BEHAVIOR demos in manifest.") 44 | parser.add_argument( 45 | "--demo_dir", 46 | type=str, 47 | default=args_dict["demo_dir"], 48 | help="Directory containing demos listed in the manifest.", 49 | ) 50 | parser.add_argument( 51 | "--demo_manifest", 52 | type=str, 53 | default=args_dict["demo_manifest"], 54 | help="Plain text file consisting of list of demos to replay.", 55 | ) 56 | parser.add_argument("--out_dir", type=str, default=args_dict["out_dir"], help="Directory to store results in.") 57 | args = parser.parse_args() 58 | args = parser.parse_args() 59 | args_dict["demo_dir"] = args.demo_dir 60 | args_dict["demo_manifest"] = args.demo_manifest 61 | args_dict["out_dir"] = args.out_dir 62 | 63 | return args_dict 64 | 65 | 66 | class PointCloudExtractor(object): 67 | def __init__(self, h5py_file): 68 | self.h5py_file = h5py_file 69 | self.points = None 70 | self.colors = None 71 | self.categories = None 72 | self.instances = None 73 | 74 | def start_callback(self, env, log_reader): 75 | # Create the dataset 76 | renderer = env.simulator.renderer 77 | w = renderer.width 78 | h = renderer.height 79 | n_frames = frame_to_entry_idx(log_reader.total_frame_num) 80 | self.points = self.h5py_file.create_dataset( 81 | "/pointcloud/points", 82 | (n_frames, h, w, 4), 83 | dtype=np.float32, 84 | compression="lzf", 85 | chunks=(min(n_frames, FRAME_BATCH_SIZE), h, w, 4), 86 | ) 87 | self.colors = self.h5py_file.create_dataset( 88 | "/pointcloud/colors", 89 | (n_frames, h, w, 3), 90 | dtype=np.float32, 91 | compression="lzf", 92 | chunks=(min(n_frames, FRAME_BATCH_SIZE), h, w, 3), 93 | ) 94 | self.categories = self.h5py_file.create_dataset( 95 | "/pointcloud/categories", 96 | (n_frames, h, w), 97 | dtype=np.int32, 98 | compression="lzf", 99 | chunks=(min(n_frames, FRAME_BATCH_SIZE), h, w), 100 | ) 101 | self.instances = self.h5py_file.create_dataset( 102 | "/pointcloud/instances", 103 | (n_frames, h, w), 104 | dtype=np.int32, 105 | compression="lzf", 106 | chunks=(min(n_frames, FRAME_BATCH_SIZE), h, w), 107 | ) 108 | 109 | self.create_caches(renderer) 110 | 111 | def create_caches(self, renderer): 112 | w = renderer.width 113 | h = renderer.height 114 | 115 | self.points_cache = np.zeros((FRAME_BATCH_SIZE, h, w, 4), dtype=np.float32) 116 | self.colors_cache = np.zeros((FRAME_BATCH_SIZE, h, w, 3), dtype=np.float32) 117 | self.categories_cache = np.zeros((FRAME_BATCH_SIZE, h, w), dtype=np.int32) 118 | self.instances_cache = np.zeros((FRAME_BATCH_SIZE, h, w), dtype=np.int32) 119 | 120 | def write_to_file(self, env): 121 | frame_count = frame_to_entry_idx(env.simulator.frame_count) 122 | new_lines = frame_count % FRAME_BATCH_SIZE 123 | if new_lines == 0: 124 | return 125 | 126 | start_pos = frame_count - new_lines 127 | self.points[start_pos:frame_count] = self.points_cache[:new_lines] 128 | self.colors[start_pos:frame_count] = self.colors_cache[:new_lines] 129 | self.categories[start_pos:frame_count] = self.categories_cache[:new_lines] 130 | self.instances[start_pos:frame_count] = self.instances_cache[:new_lines] 131 | 132 | self.create_caches(env.simulator.renderer) 133 | 134 | def step_callback(self, env, _): 135 | if not is_subsampled_frame(env.simulator.frame_count): 136 | return 137 | 138 | # TODO: Check how this compares to the outputs of SUNRGBD. Currently we're just taking the robot FOV. 139 | renderer = env.simulator.renderer 140 | rgb, seg, ins_seg, threed = renderer.render_robot_cameras(modes=("rgb", "seg", "ins_seg", "3d")) 141 | 142 | # Get rid of extra dimensions on segmentations 143 | seg = seg[:, :, 0].astype(int) 144 | ins_seg = np.round(ins_seg[:, :, 0] * MAX_INSTANCE_COUNT).astype(int) 145 | id_seg = renderer.get_pb_ids_for_instance_ids(ins_seg) 146 | 147 | frame_idx = frame_to_entry_idx(env.simulator.frame_count) % FRAME_BATCH_SIZE 148 | self.points_cache[frame_idx] = threed.astype(np.float32) 149 | self.colors_cache[frame_idx] = rgb[:, :, :3].astype(np.float32) 150 | self.categories_cache[frame_idx] = seg.astype(np.int32) 151 | self.instances_cache[frame_idx] = id_seg.astype(np.int32) 152 | 153 | if frame_idx == FRAME_BATCH_SIZE - 1: 154 | self.write_to_file(env) 155 | 156 | def end_callback(self, env, _): 157 | self.write_to_file(env) 158 | 159 | 160 | class BBoxExtractor(object): 161 | def __init__(self, h5py_file): 162 | self.h5py_file = h5py_file 163 | self.bboxes = None 164 | 165 | if SAVE_CAMERA: 166 | self.cameraV = None 167 | self.cameraP = None 168 | 169 | def start_callback(self, _, log_reader): 170 | # Create the dataset 171 | n_frames = frame_to_entry_idx(log_reader.total_frame_num) 172 | # body id, category id, 2d top left, 2d extent, 3d center, 4d orientation quat, 3d extent 173 | self.bboxes = self.h5py_file.create_dataset( 174 | "/bbox2d", 175 | (n_frames, p.getNumBodies(), 16), 176 | dtype=np.float32, 177 | compression="lzf", 178 | chunks=(min(n_frames, FRAME_BATCH_SIZE), p.getNumBodies(), 16), 179 | ) 180 | 181 | if SAVE_CAMERA: 182 | self.cameraV = self.h5py_file.create_dataset( 183 | "/cameraV", 184 | (n_frames, 4, 4), 185 | dtype=np.float32, 186 | compression="lzf", 187 | chunks=(min(n_frames, FRAME_BATCH_SIZE), 4, 4), 188 | ) 189 | self.cameraP = self.h5py_file.create_dataset( 190 | "/cameraP", 191 | (n_frames, 4, 4), 192 | dtype=np.float32, 193 | compression="lzf", 194 | chunks=(min(n_frames, FRAME_BATCH_SIZE), 4, 4), 195 | ) 196 | 197 | self.create_caches() 198 | 199 | def create_caches(self): 200 | self.bboxes_cache = np.full((FRAME_BATCH_SIZE, p.getNumBodies(), 16), -1, dtype=np.float32) 201 | 202 | if SAVE_CAMERA: 203 | self.cameraV_cache = np.zeros((FRAME_BATCH_SIZE, 4, 4), dtype=np.float32) 204 | self.cameraP_cache = np.zeros((FRAME_BATCH_SIZE, 4, 4), dtype=np.float32) 205 | 206 | def write_to_file(self, env): 207 | frame_count = frame_to_entry_idx(env.simulator.frame_count) 208 | new_lines = frame_count % FRAME_BATCH_SIZE 209 | if new_lines == 0: 210 | return 211 | 212 | start_pos = frame_count - new_lines 213 | self.bboxes[start_pos:frame_count] = self.bboxes_cache[:new_lines] 214 | 215 | if SAVE_CAMERA: 216 | self.cameraV[start_pos:frame_count] = self.cameraV_cache[:new_lines] 217 | self.cameraP[start_pos:frame_count] = self.cameraP_cache[:new_lines] 218 | 219 | self.create_caches() 220 | 221 | def step_callback(self, env, _): 222 | if not is_subsampled_frame(env.simulator.frame_count): 223 | return 224 | 225 | # Clear debug drawings. 226 | if DEBUG_DRAW: 227 | p.removeAllUserDebugItems() 228 | 229 | renderer = env.simulator.renderer 230 | ins_seg = renderer.render_robot_cameras(modes="ins_seg")[0][:, :, 0] 231 | ins_seg = np.round(ins_seg * MAX_INSTANCE_COUNT).astype(int) 232 | id_seg = renderer.get_pb_ids_for_instance_ids(ins_seg) 233 | 234 | frame_idx = frame_to_entry_idx(env.simulator.frame_count) % FRAME_BATCH_SIZE 235 | filled_obj_idx = 0 236 | 237 | for body_id in np.unique(id_seg): 238 | if body_id == -1 or body_id not in env.simulator.scene.objects_by_id: 239 | continue 240 | 241 | # Get the object semantic class ID 242 | obj = env.scene.objects_by_id[body_id] 243 | if not isinstance(obj, URDFObject): 244 | # Ignore robots etc. 245 | continue 246 | 247 | class_id = env.simulator.class_name_to_class_id.get(obj.category, SemanticClass.SCENE_OBJS) 248 | 249 | # 2D bounding box 250 | this_object_pixels_positions = np.argwhere(id_seg == body_id) 251 | bb_top_left = np.min(this_object_pixels_positions, axis=0) 252 | bb_bottom_right = np.max(this_object_pixels_positions, axis=0) 253 | 254 | # 3D bounding box 255 | world_frame_center, world_frame_orientation, base_frame_extent, _ = obj.get_base_aligned_bounding_box( 256 | body_id=body_id, visual=True 257 | ) 258 | world_frame_pose = np.concatenate([world_frame_center, world_frame_orientation]) 259 | camera_frame_pose = env.simulator.renderer.transform_pose(world_frame_pose) 260 | camera_frame_center = camera_frame_pose[:3] 261 | camera_frame_orientation = camera_frame_pose[3:] 262 | 263 | # Debug-mode drawing of the bounding box. 264 | if DEBUG_DRAW and obj.category not in ("walls", "floors", "ceilings"): 265 | bbox_frame_vertex_positions = np.array(list(itertools.product((1, -1), repeat=3))) * ( 266 | base_frame_extent / 2 267 | ) 268 | bbox_transform = utils.quat_pos_to_mat(world_frame_center, world_frame_orientation) 269 | world_frame_vertex_positions = trimesh.transformations.transform_points( 270 | bbox_frame_vertex_positions, bbox_transform 271 | ) 272 | for i, from_vertex in enumerate(world_frame_vertex_positions): 273 | for j, to_vertex in enumerate(world_frame_vertex_positions): 274 | if j <= i: 275 | p.addUserDebugLine(from_vertex, to_vertex, [1.0, 0.0, 0.0], 1, 0) 276 | 277 | # Record the results. 278 | self.bboxes_cache[frame_idx, filled_obj_idx, 0] = body_id 279 | self.bboxes_cache[frame_idx, filled_obj_idx, 1] = class_id 280 | self.bboxes_cache[frame_idx, filled_obj_idx, 2:4] = bb_top_left 281 | self.bboxes_cache[frame_idx, filled_obj_idx, 4:6] = bb_bottom_right - bb_top_left 282 | self.bboxes_cache[frame_idx, filled_obj_idx, 6:9] = camera_frame_center 283 | self.bboxes_cache[frame_idx, filled_obj_idx, 9:13] = camera_frame_orientation 284 | self.bboxes_cache[frame_idx, filled_obj_idx, 13:16] = base_frame_extent 285 | filled_obj_idx += 1 286 | 287 | if SAVE_CAMERA: 288 | self.cameraV_cache[frame_idx] = renderer.V 289 | self.cameraP_cache[frame_idx] = renderer.P 290 | 291 | if frame_idx == FRAME_BATCH_SIZE - 1: 292 | self.write_to_file(env) 293 | 294 | def end_callback(self, env, _): 295 | self.write_to_file(env) 296 | 297 | 298 | def main(selection="user", headless=False, short_exec=False): 299 | """ 300 | Extract a sequence of observations (ImVoteNet format: pointclouds and bounding boxes) from a batch of BEHAVIOR demos. 301 | The set of demos is specified in a manifest file 302 | """ 303 | print("*" * 80 + "\nDescription:" + main.__doc__ + "\n" + "*" * 80) 304 | 305 | defaults = selection == "random" and headless and short_exec 306 | args_dict = parse_args(defaults=defaults) 307 | 308 | def get_imvotenet_callbacks(demo_name, out_dir): 309 | path = os.path.join(out_dir, demo_name + "_data.hdf5") 310 | h5py_file = h5py.File(path, "w") 311 | extractors = [PointCloudExtractor(h5py_file), BBoxExtractor(h5py_file)] 312 | 313 | return ( 314 | [extractor.start_callback for extractor in extractors], 315 | [extractor.step_callback for extractor in extractors], 316 | [extractor.end_callback for extractor in extractors] + [lambda a, b: h5py_file.close()], 317 | [], 318 | ) 319 | 320 | # TODO: Set resolution to match model. 321 | print("Generating dataset of observations (pointclouds and bounding boxes)") 322 | replay_demo_batch( 323 | args_dict["demo_dir"], 324 | args_dict["demo_manifest"], 325 | args_dict["out_dir"], 326 | get_imvotenet_callbacks, 327 | image_size=(480, 480), 328 | ignore_errors=True, 329 | debug_display=DEBUG_DRAW, 330 | deactivate_logger=False, 331 | ) 332 | 333 | 334 | RUN_AS_TEST = False # Change to True to run this example in test mode 335 | if __name__ == "__main__": 336 | logging.basicConfig(level=logging.INFO) 337 | if RUN_AS_TEST: 338 | main(selection="random", headless=True, short_exec=True) 339 | else: 340 | main() 341 | -------------------------------------------------------------------------------- /behavior/examples/training_example.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import Callable 4 | 5 | import igibson 6 | from igibson.envs.igibson_env import iGibsonEnv 7 | 8 | import behavior 9 | 10 | try: 11 | import gym 12 | import torch as th 13 | import torch.nn as nn 14 | from stable_baselines3 import PPO 15 | from stable_baselines3.common.evaluation import evaluate_policy 16 | from stable_baselines3.common.preprocessing import maybe_transpose 17 | from stable_baselines3.common.torch_layers import BaseFeaturesExtractor 18 | from stable_baselines3.common.utils import set_random_seed 19 | from stable_baselines3.common.vec_env import SubprocVecEnv, VecMonitor 20 | 21 | except ModuleNotFoundError: 22 | print("stable-baselines3 is not installed. You would need to do: pip install stable-baselines3") 23 | exit(1) 24 | 25 | 26 | """ 27 | Example training code using stable-baselines3 PPO for PointNav task. 28 | """ 29 | 30 | 31 | class CustomCombinedExtractor(BaseFeaturesExtractor): 32 | def __init__(self, observation_space: gym.spaces.Dict): 33 | # We do not know features-dim here before going over all the items, 34 | # so put something dummy for now. PyTorch requires calling 35 | # nn.Module.__init__ before adding modules 36 | super(CustomCombinedExtractor, self).__init__(observation_space, features_dim=1) 37 | 38 | extractors = {} 39 | 40 | total_concat_size = 0 41 | feature_size = 128 42 | for key, subspace in observation_space.spaces.items(): 43 | if key in ["proprioception", "task_obs"]: 44 | extractors[key] = nn.Sequential(nn.Linear(subspace.shape[0], feature_size), nn.ReLU()) 45 | elif key in ["rgb", "highlight", "depth", "seg", "ins_seg"]: 46 | n_input_channels = subspace.shape[2] # channel last 47 | cnn = nn.Sequential( 48 | nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), 49 | nn.ReLU(), 50 | nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0), 51 | nn.ReLU(), 52 | nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0), 53 | nn.ReLU(), 54 | nn.Flatten(), 55 | ) 56 | test_tensor = th.zeros([subspace.shape[2], subspace.shape[0], subspace.shape[1]]) 57 | with th.no_grad(): 58 | n_flatten = cnn(test_tensor[None]).shape[1] 59 | fc = nn.Sequential(nn.Linear(n_flatten, feature_size), nn.ReLU()) 60 | extractors[key] = nn.Sequential(cnn, fc) 61 | elif key in ["scan"]: 62 | n_input_channels = subspace.shape[1] # channel last 63 | cnn = nn.Sequential( 64 | nn.Conv1d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), 65 | nn.ReLU(), 66 | nn.Conv1d(32, 64, kernel_size=4, stride=2, padding=0), 67 | nn.ReLU(), 68 | nn.Conv1d(64, 64, kernel_size=3, stride=1, padding=0), 69 | nn.ReLU(), 70 | nn.Flatten(), 71 | ) 72 | test_tensor = th.zeros([subspace.shape[1], subspace.shape[0]]) 73 | with th.no_grad(): 74 | n_flatten = cnn(test_tensor[None]).shape[1] 75 | fc = nn.Sequential(nn.Linear(n_flatten, feature_size), nn.ReLU()) 76 | extractors[key] = nn.Sequential(cnn, fc) 77 | else: 78 | raise ValueError("Unknown observation key: %s" % key) 79 | total_concat_size += feature_size 80 | 81 | self.extractors = nn.ModuleDict(extractors) 82 | 83 | # Update the features dim manually 84 | self._features_dim = total_concat_size 85 | 86 | def forward(self, observations) -> th.Tensor: 87 | encoded_tensor_list = [] 88 | 89 | # self.extractors contain nn.Modules that do all the processing. 90 | for key, extractor in self.extractors.items(): 91 | if key in ["rgb", "highlight", "depth", "seg", "ins_seg"]: 92 | observations[key] = observations[key].permute((0, 3, 1, 2)) 93 | elif key in ["scan"]: 94 | observations[key] = observations[key].permute((0, 2, 1)) 95 | encoded_tensor_list.append(extractor(observations[key])) 96 | # Return a (B, self._features_dim) PyTorch tensor, where B is batch dimension. 97 | return th.cat(encoded_tensor_list, dim=1) 98 | 99 | 100 | def main(selection="user", headless=False, short_exec=False): 101 | """ 102 | Example to set a training process for BEHAVIOR with Stable Baselines 3 103 | Loads a scene and starts the training process for a BEHAVIOR activities with images using PPO 104 | Saves the checkpoint and loads it again 105 | """ 106 | print("*" * 80 + "\nDescription:" + main.__doc__ + "*" * 80) 107 | config_file = "behavior_onboard_sensing.yaml" 108 | tensorboard_log_dir = "log_dir" 109 | num_environments = 8 if not short_exec else 1 110 | 111 | # Function callback to create environments 112 | def make_env(rank: int, seed: int = 0) -> Callable: 113 | def _init() -> iGibsonEnv: 114 | env = iGibsonEnv( 115 | config_file=os.path.join(behavior.configs_path, config_file), 116 | mode="headless", 117 | action_timestep=1.0 / 30.0, 118 | physics_timestep=1.0 / 120.0, 119 | ) 120 | env.seed(seed + rank) 121 | return env 122 | 123 | set_random_seed(seed) 124 | return _init 125 | 126 | # Multiprocess 127 | env = SubprocVecEnv([make_env(i) for i in range(num_environments)]) 128 | env = VecMonitor(env) 129 | 130 | # Create a new environment for evaluation 131 | eval_env = iGibsonEnv( 132 | config_file=os.path.join(behavior.configs_path, config_file), 133 | mode="headless", 134 | action_timestep=1.0 / 30.0, 135 | physics_timestep=1.0 / 120.0, 136 | ) 137 | 138 | # Obtain the arguments/parameters for the policy and create the PPO model 139 | policy_kwargs = dict( 140 | features_extractor_class=CustomCombinedExtractor, 141 | ) 142 | os.makedirs(tensorboard_log_dir, exist_ok=True) 143 | model = PPO("MultiInputPolicy", env, verbose=1, tensorboard_log=tensorboard_log_dir, policy_kwargs=policy_kwargs) 144 | print(model.policy) 145 | 146 | # Random Agent, evaluation before training 147 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=10) 148 | print(f"Before Training: Mean reward: {mean_reward} +/- {std_reward:.2f}") 149 | 150 | # Train the model for the given number of steps 151 | total_timesteps = 100 if short_exec else 1000000 152 | model.learn(total_timesteps) 153 | 154 | # Evaluate the policy after training 155 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=20) 156 | print(f"After Training: Mean reward: {mean_reward} +/- {std_reward:.2f}") 157 | 158 | # Save the trained model and delete it 159 | model.save("ckpt") 160 | del model 161 | 162 | # Reload the trained model from file 163 | model = PPO.load("ckpt") 164 | 165 | # Evaluate the trained model loaded from file 166 | mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes=20) 167 | print(f"After Loading: Mean reward: {mean_reward} +/- {std_reward:.2f}") 168 | 169 | 170 | RUN_AS_TEST = False # Change to True to run this example in test mode 171 | if __name__ == "__main__": 172 | logging.basicConfig(level=logging.INFO) 173 | if RUN_AS_TEST: 174 | main(selection="random", headless=True, short_exec=True) 175 | else: 176 | main() 177 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM igibson/igibson:latest 2 | 3 | RUN git clone --depth 1 https://github.com/StanfordVL/behavior /opt/behavior 4 | 5 | WORKDIR /opt/behavior 6 | 7 | RUN pip install --no-cache-dir -e . 8 | 9 | WORKDIR /opt/behavior/behavior/benchmark/scripts 10 | -------------------------------------------------------------------------------- /docker/Dockerfile_add_results: -------------------------------------------------------------------------------- 1 | FROM my_submission:latest 2 | 3 | ADD results/ /participant_reported_results 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/acknowledgements.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/acknowledgements.md -------------------------------------------------------------------------------- /docs/activities.md: -------------------------------------------------------------------------------- 1 | # The BEHAVIOR Dataset of Activity Definitions and BDDL 2 | 3 | Each BEHAVIOR activity is defined as a categorized object list, an initial condition, and a goal condition using the BEHAVIOR Domain Definition Language, a language to define logic-symbolic statements. 4 | Although BDDL shares some of the syntax and characteristics of the Planning Domain Definition Language (PDDL), e.g., both represent first-order logic, they are different as BDDL does not include action symbols and cannot be used for symbolic planning. 5 | 6 | In BDDL, the initial condition is a set of logical statements that are true for any valid initial state. 7 | The statements specify positional relationships between pairs of objects (e.g., `insideOf(Apple, Fridge)`) and non-kinematic states of individual objects (e.g. `cooked`, `toggledOn`, `notDusty`). 8 | The goal condition is a logical expression that specifies the objective of the activity. 9 | Since the same logic condition may correspond to multiple simulator states (e.g., multiple poses of objects `o1` and `o2` correspond to `insideOf(o1, o2)`), BDDL covers a large variety of ways to achieve each activity. 10 | 11 | iGibson 2.0 implements functionalities to read this initial state and sample concrete simulation states (e.g., object poses, temperature, toggling state) that correspond to the logic statements. 12 | We provide a set of these concrete simulator initial states that we call activity instances or caches. 13 | We provide a set of instances for training, but you can generate their own instances as well (see how to [generate your own activity](##Add your own activity!)). 14 | 15 | For each activity instance, the simulator is instantiated following the concrete specified state, the embodied AI agents perform the activity, and the simulator iGibson 2.0 checks at each step if the logic expression in the goal condition has been fulfilled. 16 | The episode ends when the expression is fulfilled or the time is up. 17 | 18 | ## Add your own activity! 19 | 20 | Coming soon! 21 | -------------------------------------------------------------------------------- /docs/agents.md: -------------------------------------------------------------------------------- 1 | # Embodiments: actuation, sensing, grasping 2 | 3 | In principle, any robot embodiment can be used to participate in BEHAVIOR. 4 | We provide two embodiments, the `BehaviorRobot` and the `FetchRobot`, fully implemented and functional. 5 | Consider adding your own robot following the instructions [here](). 6 | 7 | ## BehaviorRobot 8 | 9 | The BehaviorRobot is composed of two hands, a torso, and a head where the cameras are mounted. 10 | It has been used as avatar for humans to collect demos in virtual reality (more info [here](vr_demos.md)), but can also be controlled by an autonomous AI agent to participate in the BEHAVIOR. 11 | 12 | ### Degrees of Freedom 13 | 14 | The embodiment has 26 degrees of freedom decomposed as follows: 15 | - Base: 6 DoF pose (position, orientation) 16 | - Head: 6 DoF pose (position, orientation) - relative to the current torso frame 17 | - Left and right hands: 6 DoF pose each - relative to the current torso frame 18 | - Left and right grippers: 1 DoF of close/open (aperture) of each gripper that correspond to the synchronized actuation of the 5 fingers 19 | 20 | Reference frames for the robot state in the BehaviorRobot: 21 | ![brobotframes.png](images/brobotframes.png) 22 | 23 | ### Actuation and Control 24 | 25 | Base, head, and hands are directly controlled in Cartesian space. 26 | We provide a control interface for agents to control the motions of the BehaviorRobot by specifying: 27 | - Base: Desired change in pose (delta pose, 6D) relative to the base frame in the previous time step. 28 | - Head: Desired change in pose (delta pose, 6D) relative to the base frame 29 | - Hands: Desired change in pose (delta pose, 2 x 6D) relative to the base frame 30 | - Grippers: Binary variable (2 x 1D) specifying if the grippers should close or open (change the gripper aperture DoF) at maximum speed 31 | 32 | ## FetchRobot 33 | 34 | Fetch is composed of a mobile base, one arm and a movable head where the cameras are mounted. 35 | It models the real robot Fetch. 36 | 37 | ### Degrees of Freedom 38 | 39 | The embodiment has 12 degrees of freedom decomposed as follows: 40 | - Base: the base moves by actuating two active wheels. However, at effects of actuation and robot state, the rotation of the wheels is not relevant to the user; we consider instead the 3 DoF corresponding to the position on the floor plane and rotation around the floor plane normal 41 | - Trunk: 1 DoF of a prismatic joint that affect to both arm and head 42 | - Head: 2 DoF of pan (horizontal) & tilt (vertical) directions 43 | - Arm: 6 DoF (revolute joints) for the arm that control the motion of the right hand 44 | - Gripper: 1 DoF of close/open (aperture) of the two-fingered gripper 45 | 46 | Reference frames for the robot state in the BehaviorRobot: 47 | ![fetchrobotframes.png](images/fetchrobotframes.png) 48 | 49 | ### Actuation and Control 50 | 51 | **This section needs to be updated** 52 | 53 | Base, head, and hands are directly controlled in Cartesian space. 54 | We provide a control interface for agents to control the motions of the BehaviorRobot by specifying: 55 | - Base: Desired change in pose (delta pose, 6D) relative to the base frame in the previous time step. 56 | - Head: Desired change in pose (delta pose, 6D) relative to the base frame 57 | - Hands: Desired change in pose (delta pose, 2 x 6D) relative to the base frame 58 | - Grippers: Binary variable (2 x 1D) specifying if the grippers should close or open (change of the gripper aperture DoF) at maximum speed 59 | 60 | ## Observations 61 | 62 | Both agent embodiments have similar sensing capabilities based on virtual sensors on-board: 63 | - Agent's vision (default: 128x128 resolution, 120 degree FoV) 64 | - RGB images 65 | - Depth images 66 | - Semantic and instance segmentation images 67 | - Activity relevant object mask (all objects included in the activity definition except the agent and the floor) 68 | - BehaviorRobot's proprioception: 69 | - Hand/end-effector pose in base frame (12-dim) 70 | - Head pose in base frame (6-dim) 71 | - Hand closing fraction (2-dim) 72 | - Whether the hand is grasping (2-dim) 73 | - FetchRobot's proprioception: 74 | - Hand/end-effector pose in base frame (6-dim) 75 | - Hand aperture state (1-dim for FetchRobot) 76 | - Joint states (position, velocity and torque x 11 joints: trunk, head, arm and fingers) 77 | 78 | ## Grasping Modes 79 | 80 | We acknowledge grasping as one of the most difficult elements in the physical interaction between the agents and the environment. 81 | To alleviate the difficulty and enable research in other areas (e.g. planning, perception,...), we propose three grasping modes for BEHAVIOR: 82 | - **Physical Grasping**: This mode does not make any simplification in the grasping process. 83 | Grasping is the result of a fully simulated interactions between the agent's hand/fingers and the objects. 84 | This mode is the most difficult but also the most realistic. 85 | 86 | - **Assistive Grasping**: We simplify the grasping process with a mechanism that creates a rigid connection between the agent's hand and an object if: 1) the gripper DoF is closed over a threshold, and 2) the object is "inside" the hand. The joint created by assistive grasping between hand and object is removed if the constraint is violated because of a large force or because the object and the hand move apart (the physics engine cannot enforce the rigid connection due to other constraints), or when the grasping DoF goes under the activation threshold. 87 | The definition of "inside the hand" depends on the embodiment: 88 | - For BehaviorRobot: an object object is "inside the hand" if it is intersected by rays projected from the palm to the fingertips of the hand, and the hand is applying a non-zero force to it. 89 | - For FetchRobot: and object "inside the hand" if it is in contact with both fingers of the end-effector, and the contact point is in the inner part of the fingers. 90 | Assistive grasping simplifies the interaction but forces the agent to move in a realistic way to succeed in the grasp by placing the objects inside the hands, between the fingers. 91 | 92 | - **Sticky Mitten**: An alternative simplified grasping that creates a rigid connection between the hand and an object if the grasping DoF goes over a threshold while the hand is in contact with the object. 93 | Using the sticky mitten, agents do not need to place the fingers around an object to grasp it. 94 | 95 | ## Add your own embodiment! 96 | 97 | You can add your own embodiment to BEHAVIOR. For that, you will need to add it to iGibson following the specific instructions [here]() (coming soon!). 98 | 99 | -------------------------------------------------------------------------------- /docs/baselines.md: -------------------------------------------------------------------------------- 1 | # Baselines 2 | 3 | We provide a set of common baselines from the Robot Learning literature to help create and develop new solutions for BEHAVIOR. 4 | Our baselines include imitation learning and reinforcement learning, both in the original low-level action space of the benchmark, and making use of our provided set of action primitives based on sampling-based motion planning. 5 | Here, we briefly describe the baselines you can find in this repository. 6 | 7 | #### Imitation Learning Baseline 8 | A simple behavioral cloning (BC) baseline can be found [here](https://github.com/StanfordVL/behavior-baselines/tree/main/behavior_baselines/behavioral_cloning). 9 | Please note that you need to download our [imitation learning dataset](https://behavior.stanford.edu/vr-demos) first. 10 | 11 | There are two BC agents in `simple_bc_agent.py`: 12 | - `BCNet_rgbp`: A BC agent that uses an RGB image (128x128) and proprioception feedback (20) as state space. 13 | - `BCNet_taskObs`: A BC agent that uses task observations (456) and proprioception feedback (20) as state space. 14 | 15 | Details about state information can be found [here](https://stanfordvl.github.io/behavior/vr_demos.html). Feel free to include additional state information such as depth, instance segmentation, etc. 16 | 17 | Both agents are based on the BehaviorRobot, you can find more details about its action space [here](https://stanfordvl.github.io/behavior/agents.html#behaviorrobot). Please note that the action vector in human VR demo contains two additional dimensions that correspond to the hand reset action in VR: an action that teleports the simulated hands to the exact pose of the VR hand controller when they have diverged. These actions are not used by the agents but are necessary to understand the demos. 18 | 19 | #### Reinforcement Learning Baseline in the Original Action Space 20 | 21 | #### Reinforcement Learning Baseline with Action Primitives 22 | -------------------------------------------------------------------------------- /docs/benchmarking.md: -------------------------------------------------------------------------------- 1 | # Training Agents 2 | 3 | Any type of robotic agent can be evaluated on the BEHAVIOR benchmark. However, in recent years, machine learning based agents have gained popularity for robotics and embodied AI tasks. We provide scripts and functionalities to help training policies for agents. 4 | 5 | ### Training after a manual installation 6 | 7 | The training procedure will strongly depend on your type of solution/policy. 8 | However, we provide an example of parallel training code with `Stable Baselines 3` that may help you get started with your own solution. You can run our example by executing: 9 | ``` 10 | python -m behavior.examples.training_example 11 | ``` 12 | Please, be aware that this code won't converge to a fully successful solution for BEHAVIOR ;). 13 | 14 | 15 | ### Training after a Docker installation 16 | 17 | To train on a `minival` split (on a single activity) use the following script: 18 | ``` 19 | ./benchmark/scripts/train_minival_locally.sh --docker-name my_submission --dataset-path my/path/to/dataset 20 | ``` 21 | where `my_submission` is the name of the docker image, and `my/path/to/dataset` corresponds to the path to the BEHAVIOR bundle (3D objects and scenes dataset) and the `igibson.key` from the [installation instructions](installation.md). 22 | 23 | Note that due to the complexity of all BEHAVIOR activities, the provided parallel training with PPO from Stable Baselines 3 is NOT expected to converge. We provide this training pipeline just as a starting point for users to start their own training. 24 | 25 | Modify the script to change the activity or activities the agent is trained on, and other parameters. -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | import os 21 | import sys 22 | 23 | import sphinx_rtd_theme 24 | from recommonmark.parser import CommonMarkParser 25 | 26 | sys.path.append(os.path.abspath("../")) 27 | import behavior 28 | 29 | project = "BEHAVIOR" 30 | copyright = "Stanford University 2018-2021" 31 | author = "Sanjana Srivastava*, Chengshu Li*, Michael Lingelbach*, Roberto Martín-Martín*, Fei Xia, Kent Vainio, Zheng Lian, Cem Gokmen, Shyamal Buch, C. Karen Liu, Silvio Savarese, Hyowon Gweon, Jiajun Wu, Li Fei-Fei (*Equal Contribution)" 32 | 33 | github_doc_root = "https://github.com/StanfordVL/behavior" 34 | 35 | # The short X.Y version 36 | version = behavior.__version__ 37 | # The full version, including alpha/beta/rc tags 38 | release = behavior.__version__ 39 | 40 | sys.path.insert(0, os.path.abspath("../")) 41 | sys.path.insert(0, os.path.abspath("../../")) 42 | 43 | # -- General configuration --------------------------------------------------- 44 | 45 | # Add any Sphinx extension module names here, as strings. They can be 46 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 47 | # ones. 48 | 49 | extensions = [ 50 | "sphinx_rtd_theme", 51 | "sphinx_markdown_tables", 52 | "sphinx.ext.autodoc", 53 | "sphinx.ext.mathjax", 54 | "myst_parser", 55 | ] 56 | 57 | html_theme = "sphinx_rtd_theme" 58 | 59 | # Add any paths that contain templates here, relative to this directory. 60 | templates_path = ["_templates"] 61 | 62 | # List of patterns, relative to source directory, that match files and 63 | # directories to ignore when looking for source files. 64 | # This pattern also affects html_static_path and html_extra_path. 65 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 66 | 67 | 68 | # -- Options for HTML output ------------------------------------------------- 69 | 70 | # The theme to use for HTML and HTML Help pages. See the documentation for 71 | # a list of builtin themes. 72 | # 73 | 74 | # Add any paths that contain custom static files (such as style sheets) here, 75 | # relative to this directory. They are copied after the builtin static files, 76 | # so a file named "default.css" will overwrite the builtin "default.css". 77 | html_static_path = ["_static"] 78 | 79 | source_suffix = [".rst", ".md"] 80 | 81 | master_doc = "index" 82 | 83 | source_parsers = { 84 | ".md": CommonMarkParser, 85 | } 86 | -------------------------------------------------------------------------------- /docs/evaluating.md: -------------------------------------------------------------------------------- 1 | # Training and Evaluating Agents 2 | 3 | Once you have installed BEHAVIOR and its dependencies, you are ready to evaluate your agents on the benchmark. To be evaluated, agents have to be instantiated as objects of the class `Agent` or derived. We have created an initial derived class `CustomAgent` in `behavior/benchmark/agents/users_agent.py` to be fulfilled for your agent. The minimal functionality necessary for your agent is a `reset` function and the `act` action that provides actions at each timestep, given an observation from the environment. 4 | 5 | Once your agent is implemented as derived from the `Agent` class, you can evaluate it in BEHAVIOR activities. The way to do it will depend on your type of installation: manual or Docker installation. In the following, we include instructions for evaluation for both installation types. 6 | 7 | ### Evaluating an agent on BEHAVIOR after a manual installation 8 | 9 | To evaluate locally, use the functionalities in `behavior/benchmark/behavior_benchmark.py`. As an example, the following code benchmarks a random agent on a single activity indicated in the specified environment config file: 10 | ``` 11 | export CONFIG_FILE=path/to/your/config/for/example/behavior/configs/behavior_onboard_sensing.yaml 12 | export OUTPUT_DIR=path/to/your/output/dir/for/example/tmp 13 | python -m behavior.benchmark.behavior_benchmark 14 | ``` 15 | Once you have implemented your agent in the class `CustomAgent`, you can evaluate it instead of the random agent by changing the last command by: 16 | ``` 17 | python -m behavior.benchmark.behavior_benchmark --agent-class Custom 18 | ``` 19 | 20 | The code in `behavior_benchmark.py` can be used to evaluate an agent following the official [setup](setups.md) benchmark rules: the agent is evaluated in the indicated activity/activities for nine instances of increasing complexity: three instances of the activity that were available for training, three instances where everything is the same as in training but the small objects change the objects initial locations, and three instances where the furniture in the scenes is also different. The code runs the benchmark metrics and saves the values on files in the `OUTPUT_DIR`. 21 | 22 | The example above evaluates the random agent in a single activity specified in the environment's config file. However, you can select the activity you want to benchmark the agent on with the option `--split` and the name of the activity (check all activities [here](https://behavior.stanford.edu/activity_list.html) and video examples [here](https://behavior.stanford.edu/behavior-gallery/activity.html)), or benchmark on the entire set of 100 activities by specifying `--split dev` or `--split test`, to use developing or test activity instances. 23 | 24 | For example, to benchmark a provided PPO agent (reinforcement learning) loading a specific policy checkpoint only for the activity `cleaning_toilet`, you can execute: 25 | ``` 26 | export CONFIG_FILE=path/to/your/config/for/example/behavior/configs/behavior_onboard_sensing.yaml 27 | export OUTPUT_DIR=path/to/your/output/dir/for/example/tmp 28 | python -m behavior.benchmark.behavior_benchmark --agent-class PPO --ckpt-path /tmp/my_checkpoint --split cleaning_toilet 29 | ``` 30 | We provide pretrained checkpoints in `behavior/benchmark/agents/checkpoints`. Due to refactoring changes in the `BehaviorRobot`, the dimensionality of the action space may have changed. 31 | 32 | 33 | #### Evaluating on a single activity instance 34 | 35 | Instead of evaluating agents following the benchmark rules (nine instances per activity), you can also evaluate in one or a custom set of activity instances by calling directly the method `BehaviorBenchmark.evaluate_agent_on_one_activity` and providing a list of instances. 36 | 37 | ### Evaluating an agent on BEHAVIOR after a Docker installation 38 | 39 | We provide several scripts to evaluate agents on `minival` and `dev` splits. The `minival` split serves to evaluate on a single activity. The following code evaluates a random agent on the `minival` split using a local docker image: 40 | ``` 41 | ./benchmark/scripts/test_minival_docker.sh --docker-name my_submission --dataset-path my/path/to/dataset 42 | ``` 43 | where `my_submission` is the name of the docker image, and `my/path/to/dataset` corresponds to the path to the iGibson and BEHAVIOR Datasets, and the `igibson.key` obtained following the [installation instructions](installation.md). 44 | 45 | You can also evaluate locally for the `dev` split (all activities) by executing: 46 | ``` 47 | ./benchmark/scripts/test_dev_docker.sh --docker-name my_submission --dataset-path my/path/to/dataset 48 | ``` 49 | 50 | Both scripts call `behavior/benchmark/scripts/evaluate_agent.sh`. You can modify this script, or the docker scripts to evaluate your agent. 51 | 52 | #### Submitting to the BEHAVIOR public leaderboard on EvalAI 53 | 54 | If you use the Docker installation, you can submit your solution to be evaluated and included in the public leaderboard. For that, you first need to register for our benchmark on EvalAI [here](https://eval.ai/web/challenges/challenge-page/1190/overview). You should follow the instructions in the `submit` tab on EvalAI that we summarize here: 55 | ``` 56 | # Installing EvalAI Command Line Interface 57 | pip install "evalai>=1.2.3" 58 | 59 | # Set EvalAI account token 60 | evalai set_token 61 | 62 | # Push docker image to EvalAI docker registry 63 | evalai push my_submission:latest --phase 64 | ``` 65 | 66 | There are two valid benchmark tracks depending if your agent uses only onboard sensing or assumes full observability: `behavior-test-onboard-sensing-1190`, `behavior-test-full-observability-1190`. 67 | Once we receive your submission, we evaluate and return the results. 68 | Due to the time and resource consuming evaluation process, each participant is restricted to submit once per week, maximum 4 times per month. -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Code Examples 2 | 3 | This is a summary of the code examples you can find in this repository and that could help you getting started. 4 | 5 | ### -------------------------------------------------------------------------------- /docs/images/brobotframes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/brobotframes.png -------------------------------------------------------------------------------- /docs/images/fetchrobotframes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/fetchrobotframes.png -------------------------------------------------------------------------------- /docs/images/fig_pull_min.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/fig_pull_min.jpg -------------------------------------------------------------------------------- /docs/images/mosaicgif.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/mosaicgif.gif -------------------------------------------------------------------------------- /docs/images/object_collage_v2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/object_collage_v2.jpg -------------------------------------------------------------------------------- /docs/images/q_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/q_example.jpg -------------------------------------------------------------------------------- /docs/images/secondarymetrics.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/images/secondarymetrics.jpg -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. BEHAVIOR documentation master file, created by 2 | sphinx-quickstart on Tue Nov 19 14:38:54 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to the documentation of BEHAVIOR 7 | ======================================== 8 | 9 | In this documentation, you will find information to use BEHAVIOR to train, develop and evaluate your solutions. 10 | You will have to install and use iGibson, our open source simulation environment, and the BEHAVIOR Dataset of Objects (more in the [installation instructions](installation.md)). 11 | You will also find details about the benchmark setup such as the [embodiments available](agents.md), the control and sensing alternatives, the metrics, and the [activities](activities.md). 12 | This documentation includes also useful information to get started with [examples](examples.md) and [baselines](baselines.md). 13 | 14 | .. toctree:: 15 | :maxdepth: 1 16 | :caption: Introduction 17 | 18 | intro.md 19 | installation.md 20 | evaluating.md 21 | benchmarking.md 22 | baselines.md 23 | 24 | .. toctree:: 25 | :maxdepth: 1 26 | :caption: Components of BEHAVIOR 27 | 28 | agents.md 29 | setups.md 30 | activities.md 31 | vr_demos.md 32 | objects.md 33 | metrics.md 34 | 35 | .. toctree:: 36 | :maxdepth: 1 37 | :caption: API 38 | 39 | apidoc/modules.rst 40 | 41 | 42 | Indices and tables 43 | ================== 44 | 45 | * :ref:`genindex` 46 | * :ref:`modindex` 47 | * :ref:`search` 48 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | Evaluating Embodied AI agents in BEHAVIOR requires to install three code blocks: the iGibson simulator, the BDDL library of logic language, and the benchmarking utilities (this repo). 4 | 5 | There are two alternatives for the installation: 6 | 1) the common and most flexible way, installing the code **manually**, from our github repositories, or with `pip`. This mode of installation is available under MacOS, Linux and Windows. 7 | 2) the fastest but less flexible way, using our **Docker** images. This second alternative is only available for Linux and Windows users. It is the preferred mode if you plan to submit to our public leaderboard on EvalAI. 8 | 9 | ### Option 1: Manual installation 10 | 11 | In the following, we summarize the steps to install iGibson, BDDL, and the BEHAVIOR code. If you have any issues with iGibson or BDDL, please refer to the extend installation instructions in the respective repositories ([docs for iGibson installation](http://svl.stanford.edu/igibson/docs/installation.html) and [docs for BDDL installation](https://github.com/StanfordVL/bddl/blob/master/README.md)). 12 | 13 | 1) Ensure you have the all prerequisites for the installation of iGibson by following the dependencies section [here](http://svl.stanford.edu/igibson/docs/installation.html#installing-dependencies) 14 | 2) Clone all necessary repositories for the benchmark: 15 | 16 | a) The iGibson simulation environment: 17 | ``` 18 | git clone git@github.com:stanfordvl/iGibson.git --recursive 19 | ``` 20 | b) The BEHAVIOR repository with the benchmarking utilities and baselines: 21 | ``` 22 | git clone git@github.com:stanfordvl/behavior.git 23 | ``` 24 | 3) Download and obtain access to the iGibson Dataset of Scenes and the BEHAVIOR Dataset of Objects (3D assets with physical and semantic annotations) 25 | 1) Fill out the license agreement in this [form](https://docs.google.com/forms/d/e/1FAIpQLScPwhlUcHu_mwBqq5kQzT2VRIRwg_rJvF0IWYBk_LxEZiJIFg/viewform). This allows you to use the assets within iGibson for free for your research. 26 | 2) After submitting the form, you will receive a key (igibson.key). Copy it into the folder that will contain the dataset, as default: `iGibson/igibson/data` 27 | 3) Download the datasets from [here](https://storage.googleapis.com/gibson_scenes/ig_dataset.tar.gz) (size ~20GB). 28 | 4) Unzip the zip file into the desired folder, as default: `iGibson/igibson/data` 29 | For more information, please visit the iGibson documentation [here](https://stanfordvl.github.io/iGibson/dataset.html#downloading-the-igibson-2-0-dataset-of-scenes-and-the-behavior-dataset-of-objects). 30 | 31 | 4) (If you haven't done it already) Create a virtual environment and activate it 32 | ``` 33 | conda create -n igibson python=3.8 34 | conda activate igibson 35 | ``` 36 | 5) Install the downloaded repositories in the environment 37 | ``` 38 | pip install -e ./iGibson 39 | pip install -e ./behavior 40 | ``` 41 | 6) Download the iGibson assets that include robot models 42 | ``` 43 | python -m igibson.utils.assets_utils --download_assets 44 | ``` 45 | 7) Run the getting started script to evaluate the installation 46 | ``` 47 | python -m behavior.examples.metric_computation_example 48 | ``` 49 | If you are working on your local machine, you can add `-m gui_interactive` to visualize the test on a GUI (you may want to change the rendering resolution). 50 | 51 | Alternatively, you could install iGibson using `pip`. In that case, you can skip the steps of cloning and installing the github repositories and do it only for the `behavior` repository. 52 | 53 | You are now ready to evaluate your solutions on BEHAVIOR. You can also create a submission to our benchmark leaderboard at EvalAI. See [section Evaluating](evaluating.md). Finally, our code can also be used to train your policies if you are using techniques such as reinforcement learning; see [section Training](benchmarking.md). 54 | 55 | If you run into any issues, refer to our [FAQ]() or [contact us](mailto:behavior.benchmark@gmail.com). 56 | 57 | ### Option 2: Using Docker 58 | 59 | The following steps assume you have already installed Docker on your machine. 60 | 61 | 1) Clone this repository 62 | ``` 63 | git clone git@github.com:stanfordvl/behavior.git 64 | ``` 65 | 66 | 2) Install nvidia-docker2 following the guidelines [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) 67 | 68 | 3) Modify the provided Dockerfile (`behavior/docker/Dockerfile`) to include any additional dependencies, and your own agent. A minimal Dockerfile presents the form: 69 | ``` 70 | FROM igibson/igibson:latest 71 | RUN git clone --depth 1 https://github.com/StanfordVL/behavior /opt/behavior 72 | WORKDIR /opt/behavior 73 | RUN pip install --no-cache-dir -e . 74 | WORKDIR /opt/behavior/behavior/benchmark/scripts 75 | ``` 76 | You may want to change the project cloned, if you create a fork of this `behavior` repository, and add other projects to clone. You may also want to add a line to copy your own agent (and dependencies) that replaces the `CustomAgent` placeholder in `behavior/benchmark/agents/user_agent.py` 77 | 78 | 4) Then build your Docker container with `docker build . -t my_submission` replacing `my_submission` with the name you want to use for the Docker image. 79 | 80 | 5) Download and obtain access to the iGibson Dataset of Scenes and the BEHAVIOR Dataset of Objects (3D assets with physical and semantic annotations) 81 | 1) Fill out the license agreement in this [form](https://docs.google.com/forms/d/e/1FAIpQLScPwhlUcHu_mwBqq5kQzT2VRIRwg_rJvF0IWYBk_LxEZiJIFg/viewform). This allows you to use the assets within iGibson for free for your research. 82 | 2) After submitting the form, you will receive a key (igibson.key). Create a folder for the dataset and copy the key into that folder. 83 | 3) Download the datasets from [here](https://storage.googleapis.com/gibson_scenes/ig_dataset.tar.gz) (size ~20GB). 84 | 4) Unzip the zip file into the created folder in step 2) 85 | For more information, please visit the iGibson documentation [here](https://stanfordvl.github.io/iGibson/dataset.html#downloading-the-igibson-2-0-dataset-of-scenes-and-the-behavior-dataset-of-objects). 86 | 87 | 6) Run the following to evaluate the installation 88 | ``` 89 | ./benchmark/scripts/test_minival_docker.sh --docker-name my_submission --dataset-path my/path/to/dataset 90 | ``` 91 | where `my_submission` is the name of the docker image created before, and `my/path/to/dataset` corresponds to the path to the iGibson and BEHAVIOR Datasets 92 | 93 | You are now ready to evaluate your solutions on BEHAVIOR. You can also create a submission to our benchmark leaderboard at EvalAI. See [section Evaluating](evaluating.md). Finally, our code can also be used to train your policies if you are using techniques such as reinforcement learning; see [section Training](benchmarking.md). 94 | 95 | If you run into any issues, [contact us](mailto:behavior.benchmark@gmail.com). 96 | -------------------------------------------------------------------------------- /docs/intro.md: -------------------------------------------------------------------------------- 1 | # The BEHAVIOR Benchmark 2 | 3 | ![mosaicgif.gif](images/mosaicgif.gif) 4 | ##### Benchmark for Everyday Household Activities in Virtual, Interactive, and Ecological Environments 5 | 6 | BEHAVIOR is a benchmark for embodied AI agents to complete 100 household activities in simulation. 7 | In BEHAVIOR, the AI agents need to control a robot's embodiment taking decisions based on acquired virtual sensor signals, and executing them with control action commands. 8 | The complexity, realism and diversity of the benchmark creates a complex challenge for modern AI solutions. 9 | 10 | In this documentation, you will find information to use BEHAVIOR to train, develop and evaluate your solutions. 11 | You will have to install and use iGibson, our open source simulation environment, and the BEHAVIOR Dataset of Objects (more in the [installation instructions](installation.md)). 12 | You will also find details about the benchmark setup such as the [embodiments available](agents.md), the control and sensing alternatives, the metrics, and the [activities](activities.md). 13 | This documentation includes also useful information to get started with [baselines](baselines.md). 14 | 15 | We hope to make BEHAVIOR a useful evaluation tool in AI and robotics. Please, [contact us](mailto:behavior.benchmark@gmail.com) if you have any questions or suggestions. 16 | 17 | 18 | ### Citation 19 | If you use BEHAVIOR, its assets and models, consider citing the following publications: 20 | 21 | ``` 22 | @inproceedings{shen2021igibson, 23 | title={BEHAVIOR: Benchmark for Everyday Household Activities in Virtual, Interactive, and Ecological Environments}, 24 | author={Sanjana Srivastava and Chengshu Li and Michael Lingelbach and Roberto Mart\'in-Mart\'in and Fei Xia and Kent Vainio and Zheng Lian and Cem Gokmen and Shyamal Buch and Karen Liu and Silvio Savarese and Hyowon Gweon and Jiajun Wu and Li Fei-Fei}, 25 | booktitle={Conference in Robot Learning (CoRL)}, 26 | year={2021}, 27 | pages={accepted} 28 | } 29 | ``` 30 | 31 | ``` 32 | @inproceedings{li2021igibson, 33 | title={iGibson 2.0: Object-Centric Simulation for Robot Learning of Everyday Household Tasks}, 34 | author={Chengshu Li and Fei Xia and Roberto Martín-Martín and Michael Lingelbach and Sanjana Srivastava and Bokui Shen and Kent Vainio and Cem Gokmen and Gokul Dharan and Tanish Jain and Andrey Kurenkov and Karen Liu and Hyowon Gweon and Jiajun Wu and Li Fei-Fei and Silvio Savarese}, 35 | booktitle={Conference in Robot Learning (CoRL)}, 36 | year={2021}, 37 | pages={accepted} 38 | } 39 | ``` 40 | 41 | ### Repositories 42 | There are three main repositories necessary to evaluate with BEHAVIOR: 43 | - The `behavior` repository ([https://github.com/StanfordVL/behavior](https://github.com/StanfordVL/behavior)) with the code to evaluate and useful baselines to get started. 44 | - The `igibson` repository ([https://github.com/StanfordVL/iGibson](https://github.com/StanfordVL/iGibson)) with the code of our simulator. 45 | - The `bddl` repository ([https://github.com/StanfordVL/bddl](https://github.com/StanfordVL/bddl)) with the library for the logic language BDDL used to define activities. 46 | 47 | ### Datasets 48 | There are three datasets necessary to evaluate agents in BEHAVIOR: 49 | - **The iGibson 2.0 Dataset of Scenes**: New versions of the fully interactive scenes, more densely populated with objects. They are downloaded as part of the installation procedure above. 50 | - **The BEHAVIOR Dataset of Objects**: Collection of object models annotated with physical and semantic properties. 51 | The 3D models are free to use within iGibson 2.0 for BEHAVIOR. However, to preserver the artists' copyright, models are encrypted and allowed only to be used within iGibson2.0. 52 | Details on how to get access to the iGibson and BEHAVIOR datasets in a bundle can be found in the [Installation Section](installation.md). 53 | - **The BEHAVIOR Dataset of Activity Definitions**: Collection of definitions in logic language (in BDDL, our BEHAVIOR Domain Definition Language) that specifies the initial configuration of the scene, and the valid final goal of the activity. 54 | This dataset is shipped with the `bddl` package ([https://github.com/StanfordVL/bddl](https://github.com/StanfordVL/bddl)) 55 | Activities cannot be defined uniquely: different people may provide different definitions. 56 | Therefore, we provide two valid definitions per activity generated by human annotators and filtered to be executable in our scenes, i.e., initial states are guaranteed be sampleable and goal states to be reacheable. 57 | These definitions have been used to create the activity instances that humans solved in virtual reality to generate the BEHAVIOR Dataset of Human Demonstrations (see below). 58 | 59 | Apart from these necessary datasets, we provide an additional dataset in BEHAVIOR: 60 | - **The BEHAVIOR Dataset of Human Demonstrations**: Collection of 500 human successful executions of the activities of BEHAVIOR in iGibson 2.0 using a virtual reality interface. 61 | Humans control the BEHAVIOR Robot embodiment (see [Embodiments Section](agents.md)). 62 | The dataset includes all state and action pairs and can be deterministically replayed to generate any new observations. 63 | We also include a first set of processed observations. 64 | More information and links in the [BEHAVIOR Dataset of Human Demos Section](vr_demos.md). 65 | 66 | 67 | ### Documentation 68 | General information about our benchmark can be found in our webpage: [http://behavior.stanford.edu](http://behavior.stanford.edu) or in our publication available [on arxiv](https://arxiv.org/abs/2108.03332). 69 | 70 | For specific documentation about the iGibson simulator, please visit [http://svl.stanford.edu/igibson/docs](http://svl.stanford.edu/igibson/docs) or refer to our publications, the [iGibson 2.0 arxiv preprint](https://arxiv.org/abs/2108.03272) and the [iGibson 1.0 arxiv preprint](https://arxiv.org/abs/2012.02924). 71 | 72 | More information about BDDL can be found at [https://github.com/StanfordVL/bddl](https://github.com/StanfordVL/bddl) and in our [BEHAVIOR publication](https://arxiv.org/abs/2108.03332). 73 | 74 | ### Examples 75 | 76 | We included multiple code examples in the folder `behavior/examples`. Take a look there to get started. Additional useful examples can be found in the folder `igibson/examples` in the iGibson repository. -------------------------------------------------------------------------------- /docs/issues.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/issues.md -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | We used a combination of metrics to measure the performance of the agent, a primary metric based on the degree of completion of the activity, and a set of secondary metrics that evaluate different aspects of the efficiency of the solution. 4 | 5 | ### Primary Metric: Degree of Success 6 | 7 | The primary metric for BEHAVIOR is the (maximum) fraction of satisfied logical predicates of the activity's goal definition (Q). 8 | In the case that a goal is multi-part (e.g., "put one apple on a table or put two pears on a table"), Q is automatically calculated by choosing the combination of goal conditions that maximize Q. 9 | 10 | ![q_example.jpg](images/q_example.jpg) 11 | 12 | ### Secondary Metrics: Efficiency 13 | 14 | The secondary metrics capture how efficiently an agent performs the activity relative to the best human demonstration. 15 | 16 | - Simulated time (**Tsim**): The total simulated (not wall-clock) time, how quickly the agent solves the activity. 17 | - Kinematic disarrangement (**DK**): Displacement caused by the agent in the environment. This can be accumulated over time, or differential i.e. computed between two timesteps, e.g. initial, final. 18 | - Logical disarrangement, **DL**: Amount of changes caused by the agent in the logical state of the environment. This can be accumulated over time or differential between two time steps. This captures kinematic state changes (putting an item on top of another item, taking an item out of a cupboard) as well as non-kinematic state changes like toggling a stove on, making an item wet, freezing an item. All kinematic state changes are grouped as 1 unit of logical disarrangement per item. 19 | - Distance navigated 20 | - **Lbody**: Accumulated distance traveled by the agent’s base body. This metric evaluates the efficiency of the agent in navigating the environment. 21 | - Displacement of hands, **Lleft** and **Lright**: Accumulated displacement of each of the agent’s hands while in contact with another object for manipulation (i.e., grasping, pushing, etc). This metric evaluates the efficiency of the agent in its interaction with the environment. 22 | 23 | ![secondarymetrics.jpg](images/secondarymetrics.jpg) 24 | 25 | If you want to compute the metrics on your own, it is recommended that you follow the example in `igibson/examples/behavior/behavior_env_metrics.py`. 26 | This example shows how to leverage the start/step/end callbacks and aggregator to collect metrics on a behavior run. 27 | 28 | Secondary metrics can be computed in three different modes: 29 | - per `timestep` -- this provides the raw value of a metric for a given timestep 30 | - `integrated` over time -- this is the sum of the absolute value of the metric over time 31 | - `relative` change between two timesteps -- this is the difference of the per timestep metric between the start and the current timestep 32 | 33 | The following is a description of the report of metrics returned at the end of an episode. 34 | For additional information, see the corresponding implementation under `iGibson/metrics`. 35 | 36 | - `q_score` -- Primary metric: Degree of Success 37 | - `timestep` 38 | - `integrated` 39 | - `kinematic_disarrangement` -- see above 40 | - `timestep` 41 | - `integrated` 42 | - `relative` 43 | - `logical_disarrangment` -- see above 44 | - `relative` 45 | - `total_objects` 46 | - `total_states` 47 | - `agent_distance` -- distance traveled by torso of the behavior_robot 48 | - `timestep` 49 | - `integrated` 50 | - `grasp_distance` -- distance traveled by manipulators of behavior_robot while grasping 51 | - `left_hand`/`right_hand`/`body` 52 | - `timestep` 53 | - `integrated` 54 | - `work` -- F*d per frame performed by all body parts 55 | - `left_hand`/`right_hand`/`body` 56 | - `timestep` 57 | - `integrated` 58 | - `pos` -- 3D position of agent parts 59 | - `left_hand`/`right_hand`/`body` 60 | - `timestep` 61 | - `integrated` 62 | - `local_pos` -- 3D position of agent parts in frame of reference of body 63 | - `left_hand`/`right_hand` 64 | - `timestep` 65 | - `integrated` 66 | - `grasping` -- Whether hand is grasping (or touching) object 67 | - `left_hand`/`right_hand` 68 | - `timestep` 69 | - `integrated` 70 | - `reset` -- Demos allow a user to reset the bullet position of a robot part to the VR controller position to avoid getting “stuck” on objects 71 | - `left_hand`/`right_hand`/`body` 72 | - `timestep` 73 | - `integrated` 74 | - `satisfied_predicates` -- the top-level satisfied goal indices in the activity goal condition 75 | - `timestep` 76 | - `integrated` 77 | - `time` -- simulator time in both simulation steps and simulated time 78 | - `simulator_steps` 79 | - `simulator_time` 80 | 81 | Note that computing some of these metrics will strongly affect your runtime performance as they query simulator state, so it is recommended to only compute them in your evaluation, not during traning. 82 | 83 | -------------------------------------------------------------------------------- /docs/objects.md: -------------------------------------------------------------------------------- 1 | # The BEHAVIOR Dataset of Objects 2 | ![object_collage_v2.jpg](images/object_collage_v2.jpg) 3 | In total, we have curated and made available 1217 object models of 391 different object categories to support 100 BEHAVIOR activities. The categories range from food items to tableware, from home decorations to office supplies, and from decoration to cleaning tools. All models in the BEHAVIOR Dataset are organized following the WordNet, associating them to synsets. This structure allows us to define properties for all models of the same categories, but it also facilitates more general sampling of activity instances fulfilling initial conditions such as onTop(fruit, table) that can be achieved using any model within the branch fruit of WordNet. 4 | 5 | To reach a high level of visual realism, all object models include material information (metallic, roughness) to be used by a physics-based renderer (PBR) such as the renderer in iGibson 2.0. Our goal is that the outcome of the physical interactions with these models is as realistic as possible: the object models have been annotated with realistic scale, mass, center of mass, moment of inertia, and stable orientations to facilitate their sampling. The collision mesh is a simplified version of the visual mesh, obtained with a convex decomposition using the VHACD algorithm. Object models with a shape close to a box are annotated with a primitive box collision mesh, much more efficient and robust for collision checking and physical simulation. 6 | 7 | This describes the file structure in the BEHAVIOR Object Dataset: 8 | ``` 9 | OBJECT_NAME 10 | │ # Unified Robot Description Format (URDF) 11 | │ # http://wiki.ros.org/urdf 12 | │ # It defines the object model (parts, articulation, dynamics properties etc.). 13 | │ OBJECT_NAME.urdf 14 | │ 15 | └───shape 16 | │ └───visual 17 | │ │ │ # Directory containing visual meshes (vm) of the object. Used for iGibson's rendering. Encrypted 18 | │ │ │ # All objs are UV mapped onto the same texture, linked by default.mtl. All faces are triangles. 19 | │ │ │ vm1.encrypted.obj 20 | │ │ │ vm2.encrypted.obj 21 | │ │ │ … 22 | │ │ │ default.mtl (links the geometry to the texture files) 23 | │ │ 24 | │ └───collision 25 | │ │ │ # Directory containing collision meshes (cm) of the objects. Used for iGibson's physics simulation. 26 | │ │ │ # Each obj represents a unique link of the object. 27 | │ │ │ # For example, link_1_cm.obj represents the collision mesh of link_1. 28 | │ │ │ cm1.obj 29 | │ │ │ cm2.obj 30 | │ │ │ … 31 | │ 32 | └───material 33 | │ │ # Contains 4 default channels: 34 | │ │ # DIFFUSE.png (RGB albedo map) 35 | │ │ # METALLIC.png (metallic map) 36 | │ │ # NORMAL.png (tangent normal map) 37 | │ │ # ROUGHNESS.png (roughness map) 38 | | | # Also contains diffuse texture maps that will be used when some object state changes happen, e.g. cooked, burnt, or soaked. 39 | │ │ DIFFUSE.encrypted.png 40 | │ │ METALLIC.encrypted.png 41 | │ │ NORMAL.encrypted.png 42 | │ │ ROUGHNESS.encrypted.png 43 | │ │ DIFFUSE_Frozen.encrypted.png 44 | │ │ DIFFUSE_Cooked.encrypted.png 45 | │ │ DIFFUSE_Burnt.encrypted.png 46 | │ │ DIFFUSE_Soaked.encrypted.png 47 | │ │ DIFFUSE_ToggledOn.encrypted.png 48 | │ 49 | └───misc 50 | │ │ # contains bounding box information of the object, its stable orientations, object state-related link annotation (e.g. toggle button, water/heat/cold source, slicer/cleaning_tool, etc) 51 | │ │ metadata.json 52 | │ │ # contains the object’s material annotation of what kinds of material each link can have. 53 | │ │ material_groups.json 54 | │ │ # contains info about the minimum volume oriented bounding box that best fits the object 55 | │ │ mvbb_meta.json 56 | │ 57 | └───visualizations 58 | │ │ # An image and a video of the object rendered with iG renderer 59 | │ │ 00.png 60 | │ │ OBJECT_NAME.mp4 61 | ``` 62 | 63 | ## Add your own object! 64 | 65 | Coming soon! -------------------------------------------------------------------------------- /docs/projects.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordVL/behavior/95f7e903a13be8bbbf0af3a645678ee84866c497/docs/projects.md -------------------------------------------------------------------------------- /docs/setups.md: -------------------------------------------------------------------------------- 1 | # Benchmark Setup 2 | 3 | ## Evaluating Generalization 4 | 5 | The goal in BEHAVIOR is to help developing general, robust and versatile AI agents. 6 | However, generalization may have two interpretations: 7 | - Versatility: the same solution is general enough to solve to multiple problems 8 | - Extrapolation: the same solution is general enough to solve to problems not seen during development/training 9 | 10 | When it comes to the household activities in BEHAVIOR, generalization can be measured along different dimensions, for example, different setups of the same activities, or different activities (what is usually studied in meta-learning and transfer learning). 11 | In increasing order of difficulty, all these are possible levels of generalization of setups within the same activities: 12 | - Generalizing to object poses 13 | - Generalizing to object instances/classes 14 | - Generalizing to furniture poses 15 | - Generalizing to furniture instances/classes 16 | - Generalizing to scene layouts 17 | 18 | Our proposed setup to evaluate generalization (versatility and exploration) is the following: 19 | - Agents should be evaluated in the 100 activities. This avoids tailored solutions for a single activity. 20 | - For each activity, the agent should be evaluated in **three different simulated houses**. 21 | - For each simulated house, the agent should be evaluated in **three levels of generalization**: 22 | - Three activity instances (furniture pieces and arrangement, object models and arrangement) that have been used/seen during development/training. 23 | - Three activity instances where the furniture and object models are the same as during development/training, but the arrangement of the objects is new. 24 | - Three activity instances where the furniture is new. 25 | 26 | ## Alternative Benchmark Setups 27 | 28 | The ultimate setup in BEHAVIOR is to perform the 100 activities in their full complexity as they should be solved in the real-world: using only onboard sensing, planning the best strategies to achieve the long horizon goals, and controlling the execution of these strategies through motion of the robot at each step that produces the right effect in the physical simulated world. 29 | However, we acknowledge the extreme complexity of this setup. 30 | Therefore, we propose a set of alternative setups to study different aspects of the BEHAVIOR activities with more emphasis in perception, planning or control. 31 | 32 | #### Original Setup 33 | 34 | - Observations: Onboard sensor signals 35 | - Actuation: Control commands at 30 Hz 36 | 37 | #### Changing Observations 38 | 39 | The setup could be simplified to an easier perceptual challenge by changing the observations available to the agent. 40 | Alternatives for the observations include: 41 | - Localization of the agent in the scene. 42 | - List of objects in the environment. 43 | - Full scene graph of the environment. The scene graph includes all objects, their properties, and the relationships to other objects. 44 | - Full observability: The observations include any information that can be queried from the simulator, e.g., full object models, their state, ... 45 | 46 | #### Changing Actuation 47 | 48 | The setup could be simplified to an easier control challenge by changing the actuation available to the agent. 49 | Alternatives for the actuation include: 50 | - Predefined set of parameterized action primitives. The agent selects one primitive and its parameters, and the primitive executes for a extended period of time. 51 | We provide an initial set of primitives with semantic outcomes (`pick`, `place`, `open`, `close`, `navigate to`). 52 | Critical for the performance of the agents here is the available set of primitives and their implementation. 53 | 54 | 55 | -------------------------------------------------------------------------------- /docs/vr_demos.md: -------------------------------------------------------------------------------- 1 | # The BEHAVIOR Dataset of Human Demonstrations in Virtual Reality 2 | 3 | The BEHAVIOR dataset includes 500 demonstrations of human participants performing BEHAVIOR-100 activities in iGibson 2.0 using a virtual reality interface. The data was collected by 5 participants using an HTC Vive Pro Eye headset with 2 HTC Vive Wand controllers and 1 torso tracker strapped to the participant’s waist. The participants were cued by a VR overlay describing activity goal conditions. Activity relevant objects corresponding to a goal condition were highlighted on the user view on-demand, guiding human execution. The text of the corresponding part of the condition switched from red to green at the completion of a given part. 4 | 5 | There are two available HDF5 datasets. The first is processed for imitation learning, containing rendered sensor modalities matching the observation of the agent, as well as the recorded action. The second is the “raw” unprocessed observations which include additional data from the VR recordings and simulator state of the activity-relevant objects. 6 | 7 | Note: Since every small change in the simulator may cause deviating changes in execution, if you want to replay the demos (run the actions stored in the demonstration file, and reproduce the exact same outcomes, e.g. states/images), this comes with some caveats. You must: 8 | 1) Be on Windows 9 | 2) Use the `behavior-replay` branch of iGibson 10 | 3) Clone the `behavior-replay` branch of bddl, uninstall any existing version of bddl by doing ```pip uninstall bddl```, then install the cloned version by running ```pip install -e .``` in the bddl directory. 11 | 4) Download the behavior-replay version of the iGibson assets via [this link](https://storage.googleapis.com/gibson_scenes/assets-behavior-replay.tar.gz), put it under ```iGibson/igibson/data```, and rename it as ```assets```. 12 | 5) Download the behavior-replay version of BEHAVIOR dataset via [this link](https://storage.googleapis.com/gibson_scenes/ig_dataset_replay.zip), put it under ```iGibson/igibson/data```, and rename it as ```ig_dataset```. 13 | 14 | Following the instructions above, we have verified that the 500 demos can be replayed and reproduced perfectly. 15 | Try command ```python -m igibson.examples.behavior.behavior_demo_replay --mode headless --vr_log_path xxx.hdf5``` 16 | 17 | ## Virtual reality demonstration dataset (VR_demos) 18 | 19 | We provide a processed dataset generated from the raw files and that includes images and proprioceptive sensing. 20 | Alternative processed datasets can be generated based on the raw files, e.g. with different image resolutions. 21 | 22 | ### HDF5 Content 23 | The following are the available keys to index into the hdf5 file. The dimensionality of each component is noted parenthetically, where `N` indicates the number of frames in the demo. 24 | 25 | - action (N x 28) -- see BehaviorRobot description in the [Embodiments section](agents.md) for details about the actuation of this robot. This vector contains two additional dimensions that correspond to the `hand reset` action in VR: an action that teleports the simulated hands to the exact pose of the VR hand controller when they have diverged. These actions are not used by AI agents but are necessary to understand the demos. 26 | - proprioception (N x 22) -- proprioceptive feedback. More details in the [Embodiments section](agents.md). 27 | - rgb (N x 128 x 128 x 3) -- rgb image from camera 28 | - depth (N x 128 x 128 x 1) -- depth map 29 | - seg (N x 128 x 128 x 1) -- segmentation of scene 30 | - ins_seg (N x 128 x 128 x 1) -- instance segmentation 31 | - highlight ( N x 128 x 128 x 1) -- activity relevant object binary mask, active for all objects included in the activity goal (except the agent and the floor) 32 | - task_obs (N x 456) -- task observations, including ground truth state of the robot, and ground truth poses and grasping state of a maximum of a fixed number of activity relevant objects 33 | 34 | ## Raw virtual reality demonstration dataset (VR_demos_raw) 35 | 36 | ### Accessing the BEHAVIOR hdf5 demo files 37 | This code snipped provides access to the data using the python's hdf5 module that is dependency of iGibson: 38 | ``` 39 | import h5py 40 | import numpy 41 | 42 | frame_number = 0 43 | hf = h5py.File(‘/path/to/behavior/demo.hdf5’) 44 | 45 | # see below for metadata description 46 | activity_name = hf.attrs[‘metadata/task_name’] 47 | print(activity_name) 48 | 49 | # see below for finding indices for array 50 | position_of_hmd = np.array(hf[‘frame_data’][‘vr_device_data’][‘hmd’][frame_number, 1:4]) 51 | 52 | ``` 53 | 54 | ### Metadata 55 | The metadata can be accessed by keying into the hdf5.attrs with the following keys: 56 | - `/metadata/start_time`: the date the demo was recorded 57 | - `/metadata/physics_timestep`: the simulated time duration of each step of the physics simulator (1/300 seconds for all our demos) 58 | - `/metadata/render_timestep`: the simulated time between each rendered image, determines the framerate (1/30 seconds). `render_timestep / physics_timestep` gives the number of physics simulation steps between two generated images (10) 59 | - `/metadata/git_info`: the git info for activity definition, `iGibson`, `ig_dataset`, and `ig_assets`. This is used to ensure participants are using a compatible version of iGibson if replaying the demo 60 | - `/metadata/task_name`: The name of the activity, e.g. `washing_dishes`, `putting_away_groceries`... 61 | - `/metadata/task_instance`: The instance of the activity that specifies the state (pose, extended state) of the sampled activity relevant objects at initialization 62 | - `/metadata/scene_id`: The scene (`Rs_int`, `Wainscott_0_int`, etc.) where the activity was recorded 63 | - `/metadata/filter_objects`: Whether only activity relevant objects were recorded in the activity 64 | - `/metadata/obj_body_id_to_name`: mapping of pybullet IDs to the semantic object name 65 | 66 | 67 | ### HDF5 Content 68 | 69 | The following are the available keys to index into the hdf5 file. The dimensionality of each component is noted parenthetically, where `N` indicates the number of frames in the demo. 70 | 71 | - `frame_data` (N x 4) 72 | - `goal_status` 73 | - `satisfied` (N x total_goals) -- Total satisfied top-level predicates, where total_goals is the number of predicates 74 | - `unsatisfied` (N x total_goals) -- Total unsatisfied top-level predicates, where total_goals is the number of predicates 75 | - `physics_data` 76 | - `string` (bullet_id: total number of activity-relevant scene objects) 77 | - `position` (N x 3) -- The 3D position of the object center of mass 78 | - `orientation` (N x 4) -- The quaternion orientation of the object 79 | - `joint_state` (N x number of object joints) -- The pybullet joint state of each object 80 | - `vr` 81 | - `vr_camera` 82 | - `right_eye_view` (N x 4 x 4) -- the view projection matrix 83 | - `right_eye_proj` (N x 4 x 4) -- the camera projection matrix 84 | - `right_camera_pos` (N x 3) -- The 3D position of the camera 85 | - `vr_device_data` 86 | - `hmd` (N x 17) -- see below 87 | - `left_controller` (N x 27) -- see below 88 | - `right_controller` (N x 27) -- see below 89 | - `vr_position_data` (N x 12) -- see below 90 | - `torso_tracker` (N x 8) -- see below 91 | - `vr_button_data` 92 | - `left_controller` (N x 3) -- see below 93 | - `right_controller` (N x 3) -- see below 94 | - `vr_eye_tracking_data` 95 | - `left_controller` (N x 9) -- see below 96 | - `vr_event_data` 97 | - `left_controller` (N x 28) -- see below 98 | - `right_controller` (N x 28) -- see below 99 | - `reset_actions` (N x 2) -- reset for left and right controller 100 | - `Agent_actions` 101 | - `vr_robot` (N x 28) -- see BEHAVIOR robot description in previous section 102 | - `action` -- unused 103 | 104 | Additional description of the dimensions of the arrays noted above: the following are not keys but correspond to indices of the associated array: 105 | 106 | - `hmd` (17) 107 | - hmd tracking data is valid (1) 108 | - translation (3) 109 | - rotation (4) 110 | - right vector (3) 111 | - up vector (3) 112 | - forward vector (3) 113 | - `left_controller`/`right_controller` (27) 114 | - controller tracking data is valid (1) 115 | - translation (3) 116 | - rotation (4) 117 | - right vector (3) 118 | - up vector (3) 119 | - forward vector (3) 120 | - base_rotation (4) 121 | - base_rotation * controller_rotation (4) 122 | - applied_force (6) -- p.getConstraintState(controller_constraint_id) 123 | - `vr_button_data` 124 | - trigger fraction (1) --- open: 0 -> closed: 1 125 | - touchpad x position (1) -- left: -1 -> right: 1 126 | - touchpad y position (1) -- bottom: -1 -> right: 1 127 | - `Vr_eye_tracking_data` 128 | - eye-tracking data is valid (1) 129 | - origin of gaze in world space (3) 130 | - direction vector of gaze in world space (3) 131 | - left pupil diameter (1) 132 | - right pupil diameter (1) 133 | - `vr_position_data` (12) 134 | - position of the system in iGibson space (3) 135 | - offset of the system from the origin (3) 136 | - applied force to vr body (6) -- p.getConstraintState(body_constraint_id) 137 | - `torso_tracker` (8) 138 | - torso tracker is valid (1) 139 | - position (3) 140 | - rotation (4) 141 | 142 | ## Add your own demos! 143 | 144 | Coming soon! 145 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 120 3 | target-version = ['py36', 'py37', 'py38'] 4 | extend-exclude = 'behavior/(data|external|render/(cpp|cryptopp|glad|glfw|glm|openvr|pybind11|sranipal))' 5 | 6 | [tool.isort] 7 | profile = "black" 8 | line_length = 120 9 | py_version = 'all' 10 | extend_skip = [ 11 | 'behavior/data', 12 | ] 13 | 14 | [tool.pyright] 15 | exclude = [ 16 | 'behavior/data', 17 | ] 18 | 19 | [tool.pytest.ini_options] 20 | testpaths = [ 21 | "tests", 22 | ] 23 | addopts = '--cov=behavior --cov-report=xml' 24 | 25 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | protobuf==3.19.0 2 | traitlets~=5.2.2 3 | sphinx-markdown-tables~=0.0.15 4 | sphinx~=3.5.3 5 | recommonmark~=0.7.1 6 | sphinx-rtd-theme~=1.0.0 7 | pytest~=6.2.3 8 | bddl>=0.0.3 9 | torch>=1.7.0 10 | torchvision>=0.8.1 11 | flask~=1.1.2 12 | scikit-image~=0.16.2 13 | requests~=2.25.1 14 | pytest-cov>=3.0.0 15 | coverage[toml]>=6.1.1 16 | myst-parser>=0.15.2 17 | printree>=0.2.0 18 | pyinstrument>=4.1.1 19 | markupsafe==2.0.1 20 | tensorboard>=2.5 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | from setuptools import find_packages 3 | 4 | with open("README.md", "r", encoding="utf-8") as fh: 5 | long_description = fh.read() 6 | 7 | setuptools.setup( 8 | name="behavior", 9 | version="1.0.0", 10 | author="Stanford University", 11 | long_description=long_description, 12 | long_description_content_type="text/markdown", 13 | url="https://github.com/StanfordVL/behavior", 14 | project_urls={ 15 | "Bug Tracker": "https://github.com/StanfordVL/behavior/issues", 16 | }, 17 | packages=find_packages(), 18 | install_requires=[ 19 | "igibson", 20 | "bddl~=1.0.1", 21 | "stable-baselines3", # Very heavy dependency, only necessary for the baselines. 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /tests/create_tests_of_examples_b.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import importlib 3 | import os 4 | import pkgutil 5 | import shutil 6 | from inspect import getmembers, isfunction 7 | from string import Template 8 | 9 | import igibson 10 | from igibson.utils.assets_utils import download_assets 11 | 12 | import behavior 13 | from behavior import examples 14 | 15 | download_assets() 16 | 17 | 18 | def main(exhaustive=False): 19 | examples_list = [] 20 | num_first_options = {} 21 | for package in pkgutil.walk_packages(examples.__path__, examples.__name__ + "."): 22 | if ( 23 | not package.ispkg 24 | and "example_selector" not in package.name[18:] 25 | # and "batch" not in package.name[18:] # we do not run the batch examples as test 26 | and "vr" not in package.name[18:] # we do not run the vr collection examples as test 27 | and "action_primitives" not in package.name[18:] # we do not run the ap examples until we fix that 28 | ): # Consider removing the last condition if we have runnable VR tests 29 | examples_list += [package.name[18:]] 30 | 31 | # Import the module and get the number of first options, if there is a function for it 32 | # We use that to create the exhaustive examples iterating over the options in the first selection point 33 | i = importlib.import_module(package.name) 34 | if "get_first_options" in [name for (name, element) in getmembers(i)]: 35 | num_first_options[package.name[18:]] = len(i.get_first_options()) 36 | else: 37 | num_first_options[package.name[18:]] = 0 38 | 39 | temp_folder_of_test = os.path.join("/", "tmp", "tests_of_examples_b") 40 | shutil.rmtree(temp_folder_of_test, ignore_errors=True) 41 | os.makedirs(temp_folder_of_test, exist_ok=True) 42 | 43 | for example in examples_list: 44 | if ( 45 | num_first_options[example] == 0 or not exhaustive 46 | ): # If we do not indicate an exhaustive test, we use random selection for all tests 47 | template_file_name = os.path.join(behavior.__path__[0], "..", "tests", "test_of_example_template_b.txt") 48 | with open(template_file_name, "r") as f: 49 | substitutes = dict() 50 | substitutes["module"] = example 51 | name = example.rsplit(".", 1)[-1] 52 | substitutes["name"] = name 53 | substitutes["selection"] = '"random"' 54 | src = Template(f.read()) 55 | dst = src.substitute(substitutes) 56 | filename = os.path.join(temp_folder_of_test, name + "_test.py") 57 | test_file = open(filename, "w") 58 | print("Writing {}".format(filename)) 59 | n = test_file.write(dst) 60 | test_file.close() 61 | else: 62 | for selection_option in range(1, num_first_options[example] + 1): 63 | template_file_name = os.path.join(behavior.__path__[0], "..", "tests", "test_of_example_template_b.txt") 64 | with open(template_file_name, "r") as f: 65 | substitutes = dict() 66 | substitutes["module"] = example 67 | name = example.rsplit(".", 1)[-1] + "_{}".format(selection_option) 68 | substitutes["name"] = name 69 | substitutes["selection"] = selection_option 70 | src = Template(f.read()) 71 | dst = src.substitute(substitutes) 72 | filename = os.path.join(temp_folder_of_test, name + "_test.py") 73 | test_file = open(filename, "w") 74 | print("Writing {}".format(filename)) 75 | n = test_file.write(dst) 76 | test_file.close() 77 | 78 | 79 | if __name__ == "__main__": 80 | parser = argparse.ArgumentParser(description="Create test files for the examples") 81 | parser.add_argument( 82 | "-e", 83 | "--exhaustive", 84 | action="store_true", 85 | help="Whether to test all options in the first decision level or select randomly.", 86 | ) 87 | args = parser.parse_args() 88 | main(args.exhaustive) 89 | -------------------------------------------------------------------------------- /tests/test_of_example_template_b.txt: -------------------------------------------------------------------------------- 1 | import behavior.examples.$module as example_module 2 | 3 | def test_$name(): 4 | example_module.main(selection=$selection, headless=True, short_exec=True) 5 | --------------------------------------------------------------------------------