├── configs ├── __init__.py ├── NARUTO │ ├── naruto │ │ ├── naruto.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ ├── jiraiya │ │ ├── naruto.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ ├── hokage_room │ │ ├── naruto.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ └── NARUTO_coslam.yaml ├── MP3D │ ├── GdvgFV5R1Z5 │ │ ├── mp3d.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ ├── HxpKQynjfin │ │ ├── mp3d.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ ├── YmJkqBEsHnH │ │ ├── mp3d.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ ├── gZ6f7yhEvPG │ │ ├── mp3d.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ ├── pLe4wQe7qrG │ │ ├── mp3d.stage_config.json │ │ ├── coslam.yaml │ │ ├── NARUTO.py │ │ └── habitat.py │ └── mp3d_coslam.yaml └── Replica │ ├── office2 │ ├── coslam.yaml │ ├── NARUTO.py │ └── habitat.py │ ├── office3 │ ├── coslam.yaml │ ├── NARUTO.py │ └── habitat.py │ ├── office4 │ ├── coslam.yaml │ ├── NARUTO.py │ └── habitat.py │ ├── room0 │ ├── coslam.yaml │ ├── NARUTO.py │ ├── habitat.py │ ├── NARUTO_rrt.py │ └── update_SLAM_data.py │ ├── office0 │ ├── coslam.yaml │ ├── NARUTO.py │ ├── habitat.py │ └── update_SLAM_data.py │ ├── office1 │ ├── coslam.yaml │ ├── NARUTO.py │ ├── habitat.py │ └── update_SLAM_data.py │ ├── room1 │ ├── coslam.yaml │ ├── NARUTO.py │ ├── habitat.py │ └── update_SLAM_data.py │ ├── room2 │ ├── coslam.yaml │ ├── NARUTO.py │ ├── habitat.py │ └── update_SLAM_data.py │ └── replica_coslam.yaml ├── data ├── __init__.py ├── Replica ├── replica_v1 ├── mp3d_sim_nvs ├── mp3d_data ├── MP3D └── visualization │ └── replica │ ├── office4_view2.json │ ├── room0_view1.json │ ├── room0_view2.json │ ├── room2_view1.json │ ├── room2_view2.json │ └── office4_view1.json ├── assets └── poster_badge.png ├── .gitmodules ├── .gitignore ├── scripts ├── data │ ├── replica_slam_download.sh │ ├── replica_update.sh │ ├── replica_download.sh │ ├── generate_replica_habitat.sh │ ├── generate_mp3d_nvs.sh │ └── generate_replica_nvs.sh ├── activegamer │ ├── update_splatam.sh │ ├── run_replica.sh │ └── run_mp3d.sh ├── installation │ ├── docker_env │ │ ├── run.sh │ │ └── build.sh │ └── conda_env │ │ └── build.sh ├── evaluation │ ├── visualize_traj.sh │ ├── visualize_mesh.sh │ ├── visualize_naruto.sh │ ├── eval_replica_activegamer_rendering.sh │ ├── eval_replica_activegamer_recon.sh │ ├── eval_mp3d_activegamer_recon.sh │ ├── eval_replica.sh │ └── eval_mp3d.sh └── naruto │ ├── run_mp3d.sh │ ├── run_replica.sh │ └── run_naruto.sh ├── src ├── slam │ ├── splatam │ │ └── modified_ver │ │ │ └── datasets │ │ │ └── gradslam_datasets │ │ │ └── __init__.py │ ├── export_pose.py │ ├── __init__.py │ ├── coslam │ │ └── model │ │ │ └── keyframe.py │ └── slam_model.py ├── visualization │ ├── default_camera_view.json │ ├── __init__.py │ └── active_gs_visualizer.py ├── planner │ ├── path_planner.py │ └── __init__.py ├── layers │ ├── transformation3d.py │ ├── projection.py │ └── backprojection.py ├── simulator │ ├── simulator.py │ └── __init__.py ├── utils │ └── config_utils.py ├── evaluation │ ├── eval_traj_length.py │ ├── eval_splatam_result.py │ └── get_splatam_summary.py └── main │ └── cfg_loader.py ├── LICENSE └── envs └── requirements.txt /configs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /data/Replica: -------------------------------------------------------------------------------- 1 | /workspace/data/Replica -------------------------------------------------------------------------------- /data/replica_v1: -------------------------------------------------------------------------------- 1 | /workspace/data/replica_v1/ -------------------------------------------------------------------------------- /data/mp3d_sim_nvs: -------------------------------------------------------------------------------- 1 | /nfs/STG/SemanticDenseMapping/data/mp3d_sim_nvs -------------------------------------------------------------------------------- /data/mp3d_data: -------------------------------------------------------------------------------- 1 | /nfs/STG/SemanticDenseMapping/hyzhan/projects/ANR/mp3d_data// -------------------------------------------------------------------------------- /data/MP3D: -------------------------------------------------------------------------------- 1 | /nfs/STG/SemanticDenseMapping/interns/2023/zyfeng/actnice-slam_20230922/MP3D -------------------------------------------------------------------------------- /assets/poster_badge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/ActiveGAMER/HEAD/assets/poster_badge.png -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/splatam"] 2 | path = third_party/splatam 3 | url = git@github.com:spla-tam/SplaTAM.git 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | data/replica_v1/ 2 | data/replica_sim_nvs/ 3 | result/ 4 | 5 | *.pyc 6 | 7 | 8 | results/ 9 | wandb/ 10 | tmp/ 11 | 12 | .vscode/ 13 | -------------------------------------------------------------------------------- /configs/NARUTO/naruto/naruto.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/NARUTO/naruto/naruto2.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /scripts/data/replica_slam_download.sh: -------------------------------------------------------------------------------- 1 | mkdir -p data 2 | cd data 3 | wget https://cvg-data.inf.ethz.ch/nice-slam/data/Replica.zip 4 | unzip Replica.zip 5 | rm Replica.zip -------------------------------------------------------------------------------- /configs/NARUTO/jiraiya/naruto.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/NARUTO/jiraiya/jiraiya.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /configs/NARUTO/hokage_room/naruto.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/NARUTO/hokage_room/hokage_room.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /configs/MP3D/GdvgFV5R1Z5/mp3d.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/MP3D/v1/tasks/mp3d/GdvgFV5R1Z5/GdvgFV5R1Z5.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /configs/MP3D/HxpKQynjfin/mp3d.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/MP3D/v1/tasks/mp3d/HxpKQynjfin/HxpKQynjfin.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /configs/MP3D/YmJkqBEsHnH/mp3d.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/MP3D/v1/tasks/mp3d/YmJkqBEsHnH/YmJkqBEsHnH.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /configs/MP3D/gZ6f7yhEvPG/mp3d.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/MP3D/v1/tasks/mp3d/gZ6f7yhEvPG/gZ6f7yhEvPG.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /configs/MP3D/pLe4wQe7qrG/mp3d.stage_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "render_asset": "../../../data/MP3D/v1/tasks/mp3d/pLe4wQe7qrG/pLe4wQe7qrG.glb", 3 | "up": [0, 1, 0], 4 | "front": [0, 0, -1] 5 | } -------------------------------------------------------------------------------- /scripts/activegamer/update_splatam.sh: -------------------------------------------------------------------------------- 1 | MOD_DIR=src/slam/splatam/modified_ver 2 | DES_DIR=third_party/splatam 3 | 4 | # Copy the modified version of SplatAM to the destination directory 5 | cp -r ${MOD_DIR}/* ${DES_DIR}/ 6 | -------------------------------------------------------------------------------- /configs/MP3D/HxpKQynjfin/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/MP3D/mp3d_coslam.yaml 2 | mapping: 3 | bound: [[-1,5],[-8.3,1.6],[-0.2,2.8]] 4 | marching_cubes_bound: [[-1,5],[-8.3,1.6],[-0.2,2.8]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/MP3D/gZ6f7yhEvPG/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/MP3D/mp3d_coslam.yaml 2 | mapping: 3 | bound: [[-4.1,3.6],[-2.8,3],[-0.5,5.3]] 4 | marching_cubes_bound: [[-4.1,3.6],[-2.8,3],[-0.5,5.3]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/MP3D/YmJkqBEsHnH/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/MP3D/mp3d_coslam.yaml 2 | mapping: 3 | bound: [[-16.2,4.1],[-5.5,1.3],[-0.5,6]] 4 | marching_cubes_bound: [[-16.2,4.1],[-5.5,1.3],[-0.5,6]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/Replica/office2/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-3.5,3.1],[-2.9,5.4],[-1.3,1.6]] 4 | marching_cubes_bound: [[-3.5,3.1],[-2.9,5.4],[-1.3,1.6]] 5 | 6 | data: 7 | trainskip: 1 -------------------------------------------------------------------------------- /configs/Replica/office3/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-5.2,3.6],[-6.0,3.3],[-1.3,1.9]] 4 | marching_cubes_bound: [[-5.2,3.6],[-6.0,3.3],[-1.3,1.9]] 5 | 6 | data: 7 | trainskip: 1 -------------------------------------------------------------------------------- /configs/Replica/office4/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-1.3,5.4],[-2.4,4.3],[-1.3,1.7]] 4 | marching_cubes_bound: [[-1.3,5.4],[-2.4,4.3],[-1.3,1.7]] 5 | 6 | data: 7 | trainskip: 1 -------------------------------------------------------------------------------- /configs/Replica/room0/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-1.0,7.0],[-1.3,3.7],[-1.7,1.4]] 4 | marching_cubes_bound: [[-1.0,7.0],[-1.3,3.7],[-1.7,1.4]] 5 | 6 | data: 7 | trainskip: 1 -------------------------------------------------------------------------------- /configs/MP3D/GdvgFV5R1Z5/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/MP3D/mp3d_coslam.yaml 2 | mapping: 3 | bound: [[-6.8,0.7],[-3.8,3.6],[-0.05,3.9]] 4 | marching_cubes_bound: [[-6.8,0.7],[-3.8,3.6],[-0.05,3.9]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/MP3D/pLe4wQe7qrG/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/MP3D/mp3d_coslam.yaml 2 | mapping: 3 | bound: [[-2.3,9.2],[-3.7,3.8],[-0.5,10.5]] 4 | marching_cubes_bound: [[-2.3,9.2],[-3.7,3.8],[-0.5,10.5]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/Replica/office0/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-2.2,2.6],[-3.4,2.1],[-1.4,2.0]] 4 | marching_cubes_bound: [[-2.2,2.6],[-3.4,2.1],[-1.4,2.0]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/Replica/office1/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-1.9,3.1],[-1.6,2.6],[-1.1,1.8]] 4 | marching_cubes_bound: [[-1.9,3.1],[-1.6,2.6],[-1.1,1.8]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/Replica/room1/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-5.6,1.4],[-3.2,2.8],[-1.6,1.8]] 4 | marching_cubes_bound: [[-5.6,1.4],[-3.2,2.8],[-1.6,1.8]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/Replica/room2/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/Replica/replica_coslam.yaml 2 | mapping: 3 | bound: [[-0.9,6.0],[-3.3,1.8],[-3.0,0.7]] 4 | marching_cubes_bound: [[-0.9,6.0],[-3.3,1.8],[-3.0,0.7]] 5 | 6 | data: 7 | trainskip: 1 8 | -------------------------------------------------------------------------------- /configs/NARUTO/hokage_room/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/NARUTO/NARUTO_coslam.yaml 2 | mapping: 3 | bound: [[-15,7.5],[-10.5,11.5],[-0.5, 5.7]] 4 | marching_cubes_bound: [[-15,7.5],[-10.5,11.5],[-0.5, 5.7]] 5 | 6 | 7 | data: 8 | trainskip: 1 9 | 10 | mesh: 11 | resolution: 512 12 | render_color: False 13 | vis: 500 14 | voxel_eval: 0.05 15 | voxel_final: 0.02 16 | visualisation: False -------------------------------------------------------------------------------- /configs/NARUTO/naruto/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/NARUTO/NARUTO_coslam.yaml 2 | mapping: 3 | bound: [[-2.6, 2.6],[-2.6,2.6],[-2.6,2.6]] 4 | marching_cubes_bound: [[-0.65,0.75],[-0.5,0.9],[-1.4, 1.65]] 5 | 6 | 7 | data: 8 | trainskip: 1 9 | 10 | mesh: 11 | resolution: 512 12 | render_color: False 13 | vis: 500 14 | voxel_eval: 0.01 15 | voxel_final: 0.005 16 | visualisation: False -------------------------------------------------------------------------------- /configs/NARUTO/jiraiya/coslam.yaml: -------------------------------------------------------------------------------- 1 | inherit_from: configs/NARUTO/NARUTO_coslam.yaml 2 | mapping: 3 | bound: [[-3.05, 3.05],[-3.05,3.05],[-3.05,3.05]] 4 | marching_cubes_bound: [[-2.20,1.95],[-2.45,2.25],[-2.45, 1.55]] 5 | 6 | 7 | data: 8 | trainskip: 1 9 | 10 | mesh: 11 | resolution: 512 12 | render_color: False 13 | vis: 500 14 | voxel_eval: 0.02 15 | voxel_final: 0.01 16 | visualisation: False -------------------------------------------------------------------------------- /scripts/installation/docker_env/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### Arguments (personalized) ### 4 | u=${1:-hyzhan} 5 | docker_name=${2:-activegamer} 6 | root_dir=$HOME 7 | g=$(id -gn) 8 | DOCKER_IMAGE=${u}/activegamer:1.0 9 | 10 | ### Run Docker ### 11 | docker run --gpus all --ipc=host \ 12 | --name ${docker_name} \ 13 | --rm \ 14 | -e ROOT_DIR=${root_dir} \ 15 | -v "/tmp/.X11-unix:/tmp/.X11-unix:rw" \ 16 | -v "/home/${u}/projects:/home/${u}/projects" \ 17 | -v "/home/${u}/Data:/home/${u}/Data" \ 18 | -it $DOCKER_IMAGE \ 19 | /bin/bash \ 20 | -------------------------------------------------------------------------------- /src/slam/splatam/modified_ver/datasets/gradslam_datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .azure import AzureKinectDataset 2 | from .basedataset import GradSLAMDataset 3 | from .dataconfig import load_dataset_config 4 | from .datautils import * 5 | from .icl import ICLDataset 6 | from .replica import ReplicaDataset, ReplicaV2Dataset 7 | from .scannet import ScannetDataset 8 | from .ai2thor import Ai2thorDataset 9 | from .realsense import RealsenseDataset 10 | from .record3d import Record3DDataset 11 | from .tum import TUMDataset 12 | from .scannetpp import ScannetPPDataset 13 | from .nerfcapture import NeRFCaptureDataset 14 | from .mp3d import MP3DDataset -------------------------------------------------------------------------------- /scripts/installation/docker_env/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "[Installation] | start building docker image ..." 4 | 5 | ### Arguments (personalized) ### 6 | PYTHON_VERSION=3.8 7 | USER_NAME=${1:-hyzhan} 8 | DOCKER_TAG=${USER_NAME}/activegamer:1.0 9 | 10 | ### Docker building ### 11 | echo "Will build docker container $DOCKER_TAG ..." 12 | docker build \ 13 | --file envs/Dockerfile \ 14 | --tag $DOCKER_TAG \ 15 | --force-rm \ 16 | --build-arg USER_ID=$(id -u) \ 17 | --build-arg GROUP_ID=$(id -g) \ 18 | --build-arg USER_NAME=$USER_NAME \ 19 | --build-arg GROUP_NAME=$(id -gn) \ 20 | --build-arg python=${PYTHON_VERSION} \ 21 | . 22 | -------------------------------------------------------------------------------- /scripts/evaluation/visualize_traj.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to visualize a trajectory on 4 | ### a given mesh. Trajectory poses are extracted 5 | ### from the given checkpoint 6 | ################################################## 7 | 8 | ### Input arguments ### 9 | mesh_file=$1 10 | traj_dir=$2 11 | cam_view=${3:-src/visualization/default_camera_view.json} 12 | out_dir=$4 13 | with_interact=${5:-0} 14 | 15 | ### visualize trajectory ### 16 | python src/visualization/vis_traj.py \ 17 | --mesh_file $mesh_file \ 18 | --traj_dir $traj_dir \ 19 | --cam_json $cam_view \ 20 | --out_dir $out_dir \ 21 | --with_interact $with_interact 22 | -------------------------------------------------------------------------------- /scripts/evaluation/visualize_mesh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to visualize a trajectory on 4 | ### a given mesh. Trajectory poses are extracted 5 | ### from the given checkpoint 6 | ################################################## 7 | 8 | ### Input arguments ### 9 | mesh_dir=$1 10 | ckpt_file=$2 11 | cam_view=$3 12 | 13 | ### get trajectory from checkpoint ### 14 | python src/slam/export_pose.py \ 15 | --ckpt $ckpt_file 16 | 17 | ### visualize trajectory ### 18 | traj_file = ${ckpt_file}.pose.npy 19 | python src/visualization/vis_mesh_evo.py \ 20 | --mesh_dir $mesh_dir \ 21 | --traj_file $traj_file \ 22 | --cam_json src/visualization/default_camera_view.json 23 | -------------------------------------------------------------------------------- /scripts/evaluation/visualize_naruto.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to visualize a trajectory on 4 | ### a given mesh. Trajectory poses are extracted 5 | ### from the given checkpoint 6 | ################################################## 7 | 8 | ### Input arguments ### 9 | vis_dir=$1 10 | cam_view=${2:-src/visualization/default_camera_view.json} 11 | save_vis=${3:-1} 12 | mesh_type=${4:-color_mesh} 13 | with_interact=${5:-0} 14 | 15 | ### visualize trajectory ### 16 | python src/visualization/naruto_o3d_visualizer.py \ 17 | --vis_dir $vis_dir \ 18 | --cam_json $cam_view \ 19 | --save_vis $save_vis \ 20 | --mesh_type $mesh_type \ 21 | --with_interact $with_interact 22 | -------------------------------------------------------------------------------- /scripts/installation/conda_env/build.sh: -------------------------------------------------------------------------------- 1 | ROOT=${PWD} 2 | 3 | ### create conda environment ### 4 | conda create -y -n activegamer python=3.8 cmake=3.14.0 5 | 6 | ### activate conda environment ### 7 | source activate activegamer 8 | 9 | ### Setup habitat-sim ### 10 | cd ${ROOT}/third_party 11 | git clone git@github.com:Huangying-Zhan/habitat-sim.git habitat_sim 12 | cd habitat_sim 13 | pip install -r requirements.txt 14 | python setup.py install --headless --bullet 15 | 16 | ### extra installation ### 17 | pip install opencv-python 18 | conda install -y ipython 19 | pip install mmcv==2.0.0 20 | pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1+cu116 -f https://download.pytorch.org/whl/cu116/torch_stable.html; \ 21 | 22 | ### activegamer installation ### 23 | pip install -r ${ROOT}/envs/requirements.txt 24 | -------------------------------------------------------------------------------- /data/visualization/replica/office4_view2.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | 0.58687126943783041, 6 | 0.59456828608744716, 7 | -0.54960955803867273, 8 | 0.0, 9 | 0.80435511535110393, 10 | -0.35039688044194933, 11 | 0.4798279635297038, 12 | 0.0, 13 | 0.092709015294856645, 14 | -0.72367850552271207, 15 | -0.68388190437199914, 16 | 0.0, 17 | -2.0352931817645414, 18 | -1.5414159189862811, 19 | 9.1070423113616314, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1024, 25 | "intrinsic_matrix" : 26 | [ 27 | 889.40808968661872, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 889.40808968661872, 32 | 0.0, 33 | 511.5, 34 | 511.5, 35 | 1.0 36 | ], 37 | "width" : 1024 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /data/visualization/replica/room0_view1.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | 0.51062267310581777, 6 | -0.57511687839154246, 7 | 0.63914400716852304, 8 | 0.0, 9 | -0.84642801761994213, 10 | -0.20562190293025118, 11 | 0.49120183633952075, 12 | 0.0, 13 | -0.15107645977532067, 14 | -0.79180818966749156, 15 | -0.59179024500007249, 16 | 0.0, 17 | 0.26382193088347605, 18 | 0.58317088260751149, 19 | 5.1696828242847017, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1024, 25 | "intrinsic_matrix" : 26 | [ 27 | 889.40808968661872, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 889.40808968661872, 32 | 0.0, 33 | 511.5, 34 | 511.5, 35 | 1.0 36 | ], 37 | "width" : 1024 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /data/visualization/replica/room0_view2.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | -0.57936254348299143, 6 | 0.52620360224805041, 7 | -0.62245386350322751, 8 | 0.0, 9 | 0.80926452866903098, 10 | 0.46235396617671803, 11 | -0.36238064600465292, 12 | 0.0, 13 | 0.097108011240114783, 14 | -0.71367960524442198, 15 | -0.69370847999080709, 16 | 0.0, 17 | 1.5457088944612591, 18 | -3.1636663036208645, 19 | 10.393931231507333, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1024, 25 | "intrinsic_matrix" : 26 | [ 27 | 889.40808968661872, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 889.40808968661872, 32 | 0.0, 33 | 511.5, 34 | 511.5, 35 | 1.0 36 | ], 37 | "width" : 1024 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /data/visualization/replica/room2_view1.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | 0.49694752393552077, 6 | -0.67537404578077254, 7 | 0.54489729008324683, 8 | 0.0, 9 | -0.85774658865085529, 10 | -0.28707870978881989, 11 | 0.42644648438439164, 12 | 0.0, 13 | -0.1315824764631508, 14 | -0.67930531623983159, 15 | -0.72196283783594206, 16 | 0.0, 17 | -1.8166312555161359, 18 | 0.1476772483707216, 19 | 5.3889565689091778, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1024, 25 | "intrinsic_matrix" : 26 | [ 27 | 889.40808968661872, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 889.40808968661872, 32 | 0.0, 33 | 511.5, 34 | 511.5, 35 | 1.0 36 | ], 37 | "width" : 1024 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /data/visualization/replica/room2_view2.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | -0.43642164189181915, 6 | 0.44472647324118153, 7 | -0.78214737389248423, 8 | 0.0, 9 | 0.8728136507081401, 10 | -0.0018267696176937909, 11 | -0.4880501962403998, 12 | 0.0, 13 | -0.21847764559784533, 14 | -0.89566457276774358, 15 | -0.38736609488027957, 16 | 0.0, 17 | 1.9894249983234749, 18 | -2.5733185604885538, 19 | 8.5607964358047965, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1024, 25 | "intrinsic_matrix" : 26 | [ 27 | 886.81001347526535, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 886.81001347526535, 32 | 0.0, 33 | 511.5, 34 | 511.5, 35 | 1.0 36 | ], 37 | "width" : 1024 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /src/visualization/default_camera_view.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | -0.58109285962713231, 6 | -0.40740572525974295, 7 | 0.70452229454854365, 8 | 0.0, 9 | -0.80796285269896329, 10 | 0.39261912219648853, 11 | -0.43937029205922923, 12 | 0.0, 13 | -0.097606952359529764, 14 | -0.82454278234136624, 15 | -0.55731686044820394, 16 | 0.0, 17 | -0.39217113324767755, 18 | 0.60337344013768113, 19 | 4.5163269031177657, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1500, 25 | "intrinsic_matrix" : 26 | [ 27 | 889.40808968661884, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 889.40808968661884, 32 | 0.0, 33 | 749.5, 34 | 749.5, 35 | 1.0 36 | ], 37 | "width" : 1500 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /data/visualization/replica/office4_view1.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_name" : "PinholeCameraParameters", 3 | "extrinsic" : 4 | [ 5 | -0.58350543755262496, 6 | -0.49482932161054771, 7 | 0.64394514271090275, 8 | 0.0, 9 | -0.80646583698727081, 10 | 0.44638334068473173, 11 | -0.38775606627306258, 12 | 0.0, 13 | -0.095573312796724957, 14 | -0.74557753160458362, 15 | -0.65953004954094063, 16 | 0.0, 17 | 1.8591288694334214, 18 | -0.26984979705444179, 19 | 7.3960324527717134, 20 | 1.0 21 | ], 22 | "intrinsic" : 23 | { 24 | "height" : 1024, 25 | "intrinsic_matrix" : 26 | [ 27 | 889.40808968661872, 28 | 0.0, 29 | 0.0, 30 | 0.0, 31 | 889.40808968661872, 32 | 0.0, 33 | 511.5, 34 | 511.5, 35 | 1.0 36 | ], 37 | "width" : 1024 38 | }, 39 | "version_major" : 1, 40 | "version_minor" : 0 41 | } -------------------------------------------------------------------------------- /scripts/data/replica_update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Define the base directory containing the SCENE directories. 4 | BASE_DIR=${1:-"data/replica_v1"} 5 | 6 | # Iterate over each SCENE directory within the base directory. 7 | for scene_dir in "$BASE_DIR"/*/; do 8 | # Define the source JSON file path. 9 | src_json="$scene_dir/habitat/replica_stage.stage_config.json" 10 | 11 | # Define the destination JSON file path. 12 | dest_json="$scene_dir/habitat/replicaSDK_stage.stage_config.json" 13 | 14 | # Check if the source JSON file exists. 15 | if [[ -f "$src_json" ]]; then 16 | # Copy the source JSON file to the destination. 17 | cp "$src_json" "$dest_json" 18 | 19 | # Replace the specified content in the destination JSON file. 20 | sed -i '2i\ "up": [0, 1, 0],\n "front": [0, 0, -1],' "$dest_json" 21 | 22 | echo "Processed $dest_json" 23 | else 24 | echo "Source file does not exist: $src_json" 25 | fi 26 | done 27 | -------------------------------------------------------------------------------- /src/planner/path_planner.py: -------------------------------------------------------------------------------- 1 | class PathPlanner(): 2 | def __init__(self, *argv, **kwargs): 3 | """ 4 | 5 | Args: 6 | *args : Variable length argument list for positional arguments. 7 | **kwargs: Arbitrary keyword arguments. 8 | 9 | Returns: 10 | 11 | """ 12 | return 13 | 14 | def start_new_plan(self): 15 | """ initialize a new planning request 16 | """ 17 | raise NotImplementedError 18 | 19 | def run_full(self): 20 | """ Run planner to approximately cover the whole map 21 | """ 22 | raise NotImplementedError 23 | 24 | def run(self): 25 | """ run planner 26 | """ 27 | raise NotImplementedError 28 | 29 | def find_path(self): 30 | """ find path 31 | """ 32 | raise NotImplementedError 33 | 34 | def get_reachable_mask(self): 35 | """get reachable/traversability mask 36 | """ 37 | raise NotImplementedError 38 | -------------------------------------------------------------------------------- /scripts/data/replica_download.sh: -------------------------------------------------------------------------------- 1 | ### This script is copied from: https://github.com/facebookresearch/Replica-Dataset/blob/main/download.sh 2 | #!/usr/bin/env bash 3 | 4 | # Abort on error 5 | set -e 6 | 7 | if [ $1 != "" ]; then 8 | echo -e "\nDownloading and decompressing Replica to $1. The script can resume\npartial downloads -- if your download gets interrupted, simply run it again.\n" 9 | else 10 | echo "Specify a path to download and decompress Replica to!" 11 | exit 1 12 | fi 13 | 14 | for p in {a..q} 15 | do 16 | # Ensure files are continued in case the script gets interrupted halfway through 17 | wget --continue https://github.com/facebookresearch/Replica-Dataset/releases/download/v1.0/replica_v1_0.tar.gz.parta$p 18 | done 19 | 20 | # Create the destination directory if it doesn't exist yet 21 | mkdir -p $1 22 | 23 | cat replica_v1_0.tar.gz.part?? | unpigz -p 32 | tar -xvC $1 24 | 25 | #download, unzip, and merge the additional habitat configs 26 | wget http://dl.fbaipublicfiles.com/habitat/Replica/additional_habitat_configs.zip -P assets/ 27 | unzip -qn assets/additional_habitat_configs.zip -d $1 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 OPPO 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts/data/generate_replica_habitat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to re-generate ReplicaSLAM 4 | ### data on Replica dataset. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-office0} 9 | EXP=${2:-update_SLAM_data} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 10 | ENABLE_VIS=${3:-0} 11 | GPU_ID=${4:-0} 12 | 13 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 14 | PROJ_DIR=${PWD} 15 | DATASET=Replica 16 | 17 | ################################################## 18 | ### Scenes 19 | ### choose one or all of the scenes 20 | ################################################## 21 | scenes=(room0 room1 room2 office0 office1 office2 office3 office4) 22 | # Check if the input argument is 'all' 23 | if [ "$scene" == "all" ]; then 24 | selected_scenes=${scenes[@]} # Copy all scenes 25 | else 26 | selected_scenes=($scene) # Assign the matching scene 27 | fi 28 | 29 | ################################################## 30 | ### Main 31 | ### Run for selected scenes for N trials 32 | ################################################## 33 | for scene in $selected_scenes 34 | do 35 | # ### run experiment ### 36 | CFG=configs/${DATASET}/${scene}/${EXP}.py 37 | python src/data/update_ReplicaSLAM_data.py \ 38 | --cfg ${CFG} \ 39 | --seed 0 \ 40 | --result_dir tmp/generate_data \ 41 | --enable_vis ${ENABLE_VIS} 42 | 43 | done 44 | -------------------------------------------------------------------------------- /scripts/data/generate_mp3d_nvs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to generate Novel View Synthesis 4 | ### data for evaluation on MP3D dataset. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-GdvgFV5R1Z5} 9 | EXP=${2:-generate_nvs_data} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 10 | ENABLE_VIS=${3:-0} 11 | GPU_ID=${4:-0} 12 | 13 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 14 | PROJ_DIR=${PWD} 15 | DATASET=MP3D 16 | 17 | ################################################## 18 | ### Scenes 19 | ### choose one or all of the scenes 20 | ################################################## 21 | scenes=(GdvgFV5R1Z5 gZ6f7yhEvPG HxpKQynjfin pLe4wQe7qrG YmJkqBEsHnH) 22 | # Check if the input argument is 'all' 23 | if [ "$scene" == "all" ]; then 24 | selected_scenes=${scenes[@]} # Copy all scenes 25 | else 26 | selected_scenes=($scene) # Assign the matching scene 27 | fi 28 | 29 | ################################################## 30 | ### Main 31 | ### Run for selected scenes for N trials 32 | ################################################## 33 | for scene in $selected_scenes 34 | do 35 | # ### run experiment ### 36 | CFG=configs/${DATASET}/${scene}/${EXP}.py 37 | python src/data/generate_nvs_data.py \ 38 | --cfg ${CFG} \ 39 | --seed 0 \ 40 | --result_dir tmp/generate_data \ 41 | --enable_vis ${ENABLE_VIS} 42 | 43 | done 44 | -------------------------------------------------------------------------------- /scripts/data/generate_replica_nvs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to generate Novel View Synthesis 4 | ### data for evaluation on Replica dataset. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-office0} 9 | EXP=${2:-generate_nvs_data} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 10 | ENABLE_VIS=${3:-0} 11 | GPU_ID=${4:-0} 12 | 13 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 14 | PROJ_DIR=${PWD} 15 | DATASET=Replica 16 | 17 | ################################################## 18 | ### Scenes 19 | ### choose one or all of the scenes 20 | ################################################## 21 | scenes=(room0 room1 room2 office0 office1 office2 office3 office4) 22 | # Check if the input argument is 'all' 23 | if [ "$scene" == "all" ]; then 24 | selected_scenes=${scenes[@]} # Copy all scenes 25 | else 26 | selected_scenes=($scene) # Assign the matching scene 27 | fi 28 | 29 | ################################################## 30 | ### Main 31 | ### Run for selected scenes for N trials 32 | ################################################## 33 | for scene in $selected_scenes 34 | do 35 | # ### run experiment ### 36 | CFG=configs/${DATASET}/${scene}/${EXP}.py 37 | python src/data/generate_Replica_NVS_data.py \ 38 | --cfg ${CFG} \ 39 | --seed 0 \ 40 | --result_dir tmp/generate_data \ 41 | --enable_vis ${ENABLE_VIS} 42 | 43 | done 44 | -------------------------------------------------------------------------------- /src/layers/transformation3d.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import torch 27 | import torch.nn as nn 28 | 29 | 30 | class Transformation3D(nn.Module): 31 | """Layer which transform 3D points 32 | """ 33 | def __init__(self): 34 | super(Transformation3D, self).__init__() 35 | 36 | def forward(self, 37 | points: torch.Tensor, 38 | T: torch.Tensor 39 | ) -> torch.Tensor: 40 | """ 41 | Args: 42 | points (torch.Tensor, [N,4,(HxW)]): 3D points in homogeneous coordinates 43 | T (torch.Tensor, [N,4,4]): transformation matrice 44 | Returns: 45 | transformed_points (torch.Tensor, [N,4,(HxW)]): 3D points in homogeneous coordinates 46 | """ 47 | transformed_points = torch.matmul(T, points) 48 | return transformed_points -------------------------------------------------------------------------------- /configs/MP3D/YmJkqBEsHnH/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "MP3D", 11 | scene = "YmJkqBEsHnH", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | SLAMData_dir = None, 43 | 44 | start_c2w = np.array([ 45 | [ 1, 0, 0, 0], 46 | [ 0, 1, 0, 0], 47 | [ 0, 0, 1, 1], 48 | [ 0, 0, 0, 1]]) 49 | ) 50 | 51 | ################################################## 52 | ### Visualization 53 | ################################################## 54 | visualizer = dict( 55 | vis_rgbd = True, # visualize RGB-D 56 | 57 | ### mesh related ### 58 | mesh_vis_freq = 500, # mesh save frequency 59 | ) 60 | 61 | -------------------------------------------------------------------------------- /configs/MP3D/gZ6f7yhEvPG/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "MP3D", 11 | scene = "gZ6f7yhEvPG", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | SLAMData_dir = None, 43 | 44 | start_c2w = np.array([ 45 | [ 1, 0, 0, 0], 46 | [ 0, 1, 0, 0], 47 | [ 0, 0, 1, 1], 48 | [ 0, 0, 0, 1]]) 49 | ) 50 | 51 | ################################################## 52 | ### Visualization 53 | ################################################## 54 | visualizer = dict( 55 | vis_rgbd = True, # visualize RGB-D 56 | 57 | ### mesh related ### 58 | mesh_vis_freq = 500, # mesh save frequency 59 | ) 60 | 61 | -------------------------------------------------------------------------------- /configs/MP3D/pLe4wQe7qrG/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "MP3D", 11 | scene = "pLe4wQe7qrG", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | SLAMData_dir = None, 43 | 44 | start_c2w = np.array([ 45 | [ 1, 0, 0, 0], 46 | [ 0, 1, 0, 0], 47 | [ 0, 0, 1, 1], 48 | [ 0, 0, 0, 1]]) 49 | ) 50 | 51 | ################################################## 52 | ### Visualization 53 | ################################################## 54 | visualizer = dict( 55 | vis_rgbd = True, # visualize RGB-D 56 | 57 | ### mesh related ### 58 | mesh_vis_freq = 500, # mesh save frequency 59 | ) 60 | 61 | -------------------------------------------------------------------------------- /scripts/naruto/run_mp3d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to run the full NARUTO system 4 | ### (active planning and active ray sampling) 5 | ### on the MP3D dataset. 6 | ################################################## 7 | 8 | # Input arguments 9 | scene=${1:-gZ6f7yhEvPG} 10 | num_run=${2:-1} 11 | EXP=${3:-NARUTO} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 12 | ENABLE_VIS=${4:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=0 15 | PROJ_DIR=${PWD} 16 | DATASET=MP3D 17 | RESULT_DIR=${PROJ_DIR}/results 18 | 19 | ################################################## 20 | ### Random Seed 21 | ### also used to initialize agent pose 22 | ### from indexing a set of predefined pose/traj 23 | ################################################## 24 | seeds=(0 1224 4869 8964 1000) 25 | seeds=("${seeds[@]:0:$num_run}") 26 | 27 | ################################################## 28 | ### Scenes 29 | ### choose one or all of the scenes 30 | ################################################## 31 | scenes=( GdvgFV5R1Z5 gZ6f7yhEvPG HxpKQynjfin pLe4wQe7qrG YmJkqBEsHnH ) 32 | # Check if the input argument is 'all' 33 | if [ "$scene" == "all" ]; then 34 | selected_scenes=${scenes[@]} # Copy all scenes 35 | else 36 | selected_scenes=($scene) # Assign the matching scene 37 | fi 38 | 39 | ################################################## 40 | ### Main 41 | ### Run for selected scenes for N trials 42 | ################################################## 43 | for scene in $selected_scenes 44 | do 45 | for i in "${!seeds[@]}"; do 46 | seed=${seeds[$i]} 47 | 48 | ### create result folder ### 49 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 50 | mkdir -p ${result_dir} 51 | 52 | ### run experiment ### 53 | CFG=configs/${DATASET}/${scene}/${EXP}.py 54 | python src/main/naruto.py --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} 55 | 56 | ### evaluation ### 57 | bash scripts/evaluation/eval_mp3d.sh ${scene} ${i} 5000 ${EXP} 58 | done 59 | done 60 | -------------------------------------------------------------------------------- /scripts/naruto/run_replica.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to run the full NARUTO system 4 | ### (active planning and active ray sampling) 5 | ### on the Replica dataset. 6 | ################################################## 7 | 8 | # Input arguments 9 | scene=${1:-office0} 10 | num_run=${2:-1} 11 | EXP=${3:-NARUTO} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 12 | ENABLE_VIS=${4:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=0 15 | PROJ_DIR=${PWD} 16 | DATASET=Replica 17 | RESULT_DIR=${PROJ_DIR}/results 18 | 19 | ################################################## 20 | ### Random Seed 21 | ### also used to initialize agent pose 22 | ### from indexing the pose in Replica SLAM 23 | ### trajectory. 24 | ################################################## 25 | seeds=(0 500 1000 1500 1999) 26 | seeds=("${seeds[@]:0:$num_run}") 27 | 28 | ################################################## 29 | ### Scenes 30 | ### choose one or all of the scenes 31 | ################################################## 32 | scenes=(room0 room1 room2 office0 office1 office2 office3 office4) 33 | # Check if the input argument is 'all' 34 | if [ "$scene" == "all" ]; then 35 | selected_scenes=${scenes[@]} # Copy all scenes 36 | else 37 | selected_scenes=($scene) # Assign the matching scene 38 | fi 39 | 40 | ################################################## 41 | ### Main 42 | ### Run for selected scenes for N trials 43 | ################################################## 44 | for scene in $selected_scenes 45 | do 46 | for i in "${!seeds[@]}"; do 47 | seed=${seeds[$i]} 48 | 49 | ### create result folder ### 50 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 51 | mkdir -p ${result_dir} 52 | 53 | ### run experiment ### 54 | CFG=configs/${DATASET}/${scene}/${EXP}.py 55 | python src/main/naruto.py --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} 56 | 57 | ### evaluation ### 58 | bash scripts/evaluation/eval_replica.sh ${scene} ${i} 2000 ${EXP} 59 | done 60 | done 61 | -------------------------------------------------------------------------------- /src/simulator/simulator.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import mmengine 27 | 28 | from src.utils.general_utils import InfoPrinter 29 | 30 | 31 | class Simulator(): 32 | def __init__(self, 33 | main_cfg: mmengine.Config, 34 | info_printer: InfoPrinter 35 | ) -> None: 36 | """ 37 | Args: 38 | main_cfg (mmengine.Config): Configuration 39 | info_printer (InfoPrinter): information printer 40 | 41 | Attributes: 42 | main_cfg (mmengine.Config): configurations 43 | sim_cfg (mmengine.Config) : simulator configurations 44 | info_printer (InfoPrinter): information printer 45 | 46 | """ 47 | self.main_cfg = main_cfg 48 | self.sim_cfg = main_cfg.sim 49 | self.info_printer = info_printer 50 | self.step = 0 51 | 52 | def update_step(self, step): 53 | """ update step information 54 | 55 | Args: 56 | step (int): step size 57 | 58 | """ 59 | self.step = step 60 | -------------------------------------------------------------------------------- /configs/Replica/replica_coslam.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'replica' 2 | 3 | data: 4 | downsample: 1 5 | sc_factor: 1 6 | translation: 0 7 | num_workers: 4 8 | 9 | mapping: 10 | sample: 2048 11 | first_mesh: True 12 | iters: 10 13 | lr_embed: 0.01 14 | lr_decoder: 0.01 15 | lr_rot: 0.001 16 | lr_trans: 0.001 17 | keyframe_every: 5 18 | map_every: 5 19 | n_pixels: 0.05 20 | first_iters: 200 21 | optim_cur: True 22 | min_pixels_cur: 100 23 | map_accum_step: 1 24 | pose_accum_step: 5 25 | map_wait_step: 0 26 | filter_depth: True 27 | active_ray: False 28 | 29 | tracking: 30 | disable: True 31 | iter: 10 32 | sample: 1024 33 | pc_samples: 40960 34 | lr_rot: 0.001 35 | lr_trans: 0.001 36 | ignore_edge_W: 20 37 | ignore_edge_H: 20 38 | iter_point: 0 39 | wait_iters: 100 40 | const_speed: True 41 | best: True 42 | 43 | grid: 44 | enc: 'HashGrid' 45 | tcnn_encoding: True 46 | hash_size: 16 47 | voxel_color: 0.08 48 | voxel_sdf: 0.02 49 | oneGrid: True 50 | 51 | pos: 52 | enc: 'OneBlob' 53 | n_bins: 16 54 | 55 | decoder: 56 | geo_feat_dim: 15 57 | hidden_dim: 32 58 | num_layers: 2 59 | num_layers_color: 2 60 | hidden_dim_color: 32 61 | tcnn_network: False 62 | pred_uncert: False 63 | uncert_grid: True 64 | 65 | cam: 66 | H: 680 67 | W: 1200 68 | fx: 600.0 69 | fy: 600.0 70 | cx: 599.5 71 | cy: 339.5 72 | png_depth_scale: 6553.5 #for depth image in png format 73 | crop_edge: 0 74 | near: 0 75 | far: 5 76 | depth_trunc: 100. 77 | 78 | training: 79 | rgb_weight: 5.0 80 | depth_weight: 0.1 81 | sdf_weight: 1000 82 | fs_weight: 10 83 | uncert_weight: 0.005 84 | eikonal_weight: 0 85 | smooth_weight: 0.000001 86 | smooth_pts: 32 87 | smooth_vox: 0.1 88 | smooth_margin: 0.05 89 | #n_samples: 256 90 | n_samples_d: 32 91 | range_d: 0.1 92 | n_range_d: 11 93 | n_importance: 0 94 | perturb: 1 95 | white_bkgd: False 96 | trunc: 0.1 97 | rot_rep: 'axis_angle' 98 | rgb_missing: 0.05 99 | 100 | mesh: 101 | resolution: 512 102 | render_color: False 103 | vis: 500 104 | voxel_eval: 0.05 105 | voxel_final: 0.02 106 | -------------------------------------------------------------------------------- /configs/MP3D/mp3d_coslam.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'mp3d' 2 | 3 | data: 4 | downsample: 1 5 | sc_factor: 1 6 | translation: 0 7 | num_workers: 4 8 | 9 | mapping: 10 | sample: 2048 11 | first_mesh: True 12 | iters: 10 13 | lr_embed: 0.01 14 | lr_decoder: 0.01 15 | lr_rot: 0.001 16 | lr_trans: 0.001 17 | keyframe_every: 5 18 | map_every: 5 19 | n_pixels: 0.05 20 | first_iters: 200 21 | optim_cur: True 22 | min_pixels_cur: 100 23 | map_accum_step: 1 24 | pose_accum_step: 5 25 | map_wait_step: 0 26 | filter_depth: True 27 | active_ray: False 28 | 29 | tracking: 30 | disable: True 31 | iter: 10 32 | sample: 1024 33 | pc_samples: 40960 34 | lr_rot: 0.001 35 | lr_trans: 0.001 36 | ignore_edge_W: 20 37 | ignore_edge_H: 20 38 | iter_point: 0 39 | wait_iters: 100 40 | const_speed: True 41 | best: True 42 | 43 | grid: 44 | enc: 'HashGrid' 45 | tcnn_encoding: True 46 | hash_size: 16 47 | voxel_color: 0.08 48 | voxel_sdf: 0.02 49 | oneGrid: True 50 | 51 | pos: 52 | enc: 'OneBlob' 53 | n_bins: 16 54 | 55 | decoder: 56 | geo_feat_dim: 15 57 | hidden_dim: 32 58 | num_layers: 2 59 | num_layers_color: 2 60 | hidden_dim_color: 32 61 | tcnn_network: False 62 | pred_uncert: False 63 | uncert_grid: True 64 | 65 | cam: 66 | H: 680 67 | W: 1200 68 | fx: 600.0 69 | fy: 600.0 70 | cx: 599.5 71 | cy: 339.5 72 | png_depth_scale: 6553.5 #for depth image in png format 73 | crop_edge: 0 74 | near: 0 75 | far: 5 76 | depth_trunc: 100. 77 | 78 | training: 79 | rgb_weight: 5.0 80 | depth_weight: 0.1 81 | sdf_weight: 1000 82 | fs_weight: 10 83 | uncert_weight: 0.005 84 | eikonal_weight: 0 85 | smooth_weight: 0.000001 86 | smooth_pts: 32 87 | smooth_vox: 0.1 88 | smooth_margin: 0.05 89 | #n_samples: 256 90 | n_samples_d: 32 91 | range_d: 0.1 92 | n_range_d: 11 93 | n_importance: 0 94 | perturb: 1 95 | white_bkgd: False 96 | trunc: 0.1 97 | rot_rep: 'axis_angle' 98 | rgb_missing: 0.05 99 | 100 | mesh: 101 | resolution: 512 102 | render_color: False 103 | vis: 500 104 | voxel_eval: 0.05 105 | voxel_final: 0.02 106 | visualisation: False 107 | -------------------------------------------------------------------------------- /configs/MP3D/GdvgFV5R1Z5/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "MP3D", 11 | scene = "GdvgFV5R1Z5", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | SLAMData_dir = None, 43 | 44 | start_c2w = np.array([ 45 | [ 1, 0, 0, 0], 46 | [ 0, 1, 0, 0], 47 | [ 0, 0, 1, 1], 48 | [ 0, 0, 0, 1]]) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/NARUTO/NARUTO_coslam.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'NARUTO' 2 | 3 | data: 4 | downsample: 1 5 | sc_factor: 1 6 | translation: 0 7 | num_workers: 4 8 | 9 | mapping: 10 | sample: 2048 11 | first_mesh: True 12 | iters: 10 13 | lr_embed: 0.01 14 | lr_decoder: 0.01 15 | lr_rot: 0.001 16 | lr_trans: 0.001 17 | keyframe_every: 5 18 | map_every: 5 19 | n_pixels: 0.05 20 | first_iters: 200 21 | optim_cur: True 22 | min_pixels_cur: 100 23 | map_accum_step: 1 24 | pose_accum_step: 5 25 | map_wait_step: 0 26 | filter_depth: True 27 | active_ray: False 28 | 29 | tracking: 30 | disable: True 31 | iter: 10 32 | sample: 1024 33 | pc_samples: 40960 34 | lr_rot: 0.001 35 | lr_trans: 0.001 36 | ignore_edge_W: 20 37 | ignore_edge_H: 20 38 | iter_point: 0 39 | wait_iters: 100 40 | const_speed: True 41 | best: True 42 | 43 | grid: 44 | enc: 'HashGrid' 45 | tcnn_encoding: True 46 | hash_size: 16 47 | voxel_color: 0.08 48 | voxel_sdf: 0.02 49 | oneGrid: True 50 | 51 | pos: 52 | enc: 'OneBlob' 53 | n_bins: 16 54 | 55 | decoder: 56 | geo_feat_dim: 15 57 | hidden_dim: 32 58 | num_layers: 2 59 | num_layers_color: 2 60 | hidden_dim_color: 32 61 | tcnn_network: False 62 | pred_uncert: False 63 | uncert_grid: True 64 | 65 | cam: 66 | H: 680 67 | W: 1200 68 | fx: 600.0 69 | fy: 600.0 70 | cx: 599.5 71 | cy: 339.5 72 | png_depth_scale: 6553.5 #for depth image in png format 73 | crop_edge: 0 74 | near: 0 75 | far: 5 76 | depth_trunc: 100. 77 | 78 | training: 79 | rgb_weight: 5.0 80 | depth_weight: 0.1 81 | sdf_weight: 1000 82 | fs_weight: 10 83 | uncert_weight: 0.005 84 | eikonal_weight: 0 85 | smooth_weight: 0.000001 86 | smooth_pts: 32 87 | smooth_vox: 0.1 88 | smooth_margin: 0.05 89 | #n_samples: 256 90 | n_samples_d: 32 91 | range_d: 0.1 92 | n_range_d: 11 93 | n_importance: 0 94 | perturb: 1 95 | white_bkgd: False 96 | trunc: 0.1 97 | rot_rep: 'axis_angle' 98 | rgb_missing: 0.05 99 | 100 | mesh: 101 | resolution: 512 102 | render_color: False 103 | vis: 500 104 | voxel_eval: 0.05 105 | voxel_final: 0.02 106 | visualisation: False 107 | 108 | planning: 109 | rrt_maxz: 100 -------------------------------------------------------------------------------- /configs/MP3D/HxpKQynjfin/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "MP3D", 11 | scene = "HxpKQynjfin", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | SLAMData_dir = None, 43 | 44 | start_c2w = np.array([ 45 | [ 1, 0, 0, 0], 46 | [ 0, 1, 0, 0], 47 | [ 0, 0, 1, 1], 48 | [ 0, 0, 0, 1]]) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | ### Collision ### 56 | invalid_region_ratio_thre = 0.8, # invalid region ratio threshold by checking ERP. HxpK is a more incomplete scene. requires higher thres 57 | ) 58 | 59 | ################################################## 60 | ### Visualization 61 | ################################################## 62 | visualizer = dict( 63 | vis_rgbd = True, # visualize RGB-D 64 | 65 | ### mesh related ### 66 | mesh_vis_freq = 500, # mesh save frequency 67 | ) 68 | 69 | -------------------------------------------------------------------------------- /src/slam/export_pose.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import argparse 27 | import numpy as np 28 | import torch 29 | 30 | 31 | def argument_parsing() -> argparse.Namespace: 32 | """parse arguments 33 | 34 | Returns: 35 | args: arguments 36 | 37 | """ 38 | parser = argparse.ArgumentParser( 39 | description="Arguments to export pose from checkpoint." 40 | ) 41 | parser.add_argument("--ckpt", type=str, help="Checkpoint path") 42 | args = parser.parse_args() 43 | return args 44 | 45 | if __name__ == "__main__": 46 | args = argument_parsing() 47 | 48 | ################################################## 49 | ### load checkpoint 50 | ################################################## 51 | ckpt_path = args.ckpt 52 | npy_path = ckpt_path + ".pose.npy" 53 | ckpt = torch.load(ckpt_path) 54 | 55 | ################################################## 56 | ### read pose and save as npy 57 | ################################################## 58 | poses_tensor = ckpt['pose'] 59 | poses = [] 60 | for i in range(len(poses_tensor)): 61 | poses.append(poses_tensor[i].detach().cpu().numpy()) 62 | poses = np.stack(poses) 63 | np.save(npy_path, poses) 64 | -------------------------------------------------------------------------------- /scripts/evaluation/eval_replica_activegamer_rendering.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to evaluate ActiveGAMER system 4 | ### on the Replica dataset for rendering. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-office0} 9 | num_run=${2:-1} 10 | EXP=${3:-ActiveGAMER} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 11 | ENABLE_VIS=${4:-0} 12 | GPU_ID=${5:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 15 | PROJ_DIR=${PWD} 16 | DATASET=Replica 17 | RESULT_DIR=${PROJ_DIR}/results/ 18 | 19 | ################################################## 20 | ### Random Seed 21 | ################################################## 22 | seeds=(0 500 1000 1500 1999) 23 | seeds=("${seeds[@]:0:$num_run}") 24 | 25 | ################################################## 26 | ### Scenes 27 | ### choose one or all of the scenes 28 | ################################################## 29 | scenes=(office0 office1 office2 office3 office4 room0 room1 room2 ) 30 | # scenes=(office3) 31 | # Check if the input argument is 'all' 32 | if [ "$scene" == "all" ]; then 33 | selected_scenes=${scenes[@]} # Copy all scenes 34 | else 35 | selected_scenes=($scene) # Assign the matching scene 36 | fi 37 | 38 | ################################################## 39 | ### Main 40 | ### Run for selected scenes for N trials 41 | ################################################## 42 | for scene in $selected_scenes 43 | do 44 | for i in "${!seeds[@]}"; do 45 | seed=${seeds[$i]} 46 | 47 | ### create result folder ### 48 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 49 | mkdir -p ${result_dir} 50 | 51 | 52 | ### Rendering evaluation ### 53 | CFG=configs/${DATASET}/${scene}/${EXP}.py 54 | 55 | python src/evaluation/eval_splatam_result.py \ 56 | --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} --stage final 57 | 58 | python src/evaluation/eval_splatam_result.py \ 59 | --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} --stage exploration_stage_0 60 | 61 | python src/evaluation/eval_splatam_result.py \ 62 | --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} --stage exploration_stage_1 63 | 64 | done 65 | done 66 | -------------------------------------------------------------------------------- /configs/Replica/room1/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room1", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/Replica/room2/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room2", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /src/visualization/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | def init_visualizer(main_cfg, info_printer): 27 | """initialize Visualizer 28 | 29 | Args: 30 | main_cfg (mmengine.Config): Configuration 31 | info_printer (InfoPrinter): information printer 32 | 33 | Returns: 34 | visualizer (Visualizer): visualizer module 35 | 36 | """ 37 | ################################################## 38 | ### HabitatSim 39 | ################################################## 40 | if main_cfg.visualizer.method == "naruto": 41 | info_printer("Initialize Visualizer...", 0, "NARUTO") 42 | from src.visualization.naruto_visualizer import NARUTOVisualizer 43 | visualizer = NARUTOVisualizer( 44 | main_cfg, 45 | info_printer 46 | ) 47 | elif main_cfg.visualizer.method in ["active_gs"]: 48 | info_printer("Initialize Visualizer...", 0, "ActiveGAMER") 49 | from src.visualization.active_gs_visualizer import ActiveGSVisualizer 50 | visualizer = ActiveGSVisualizer( 51 | main_cfg, 52 | info_printer 53 | ) 54 | else: 55 | assert False, f"Visualizer choices: [naruto, active_gs]. Current option: [{main_cfg.visualizer}]" 56 | return visualizer -------------------------------------------------------------------------------- /configs/Replica/office0/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office0", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/Replica/office1/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office1", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/Replica/office2/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office2", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/Replica/office3/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office3", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/Replica/office4/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office4", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /configs/Replica/room0/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room0", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 56 | ) 57 | 58 | ################################################## 59 | ### Visualization 60 | ################################################## 61 | visualizer = dict( 62 | vis_rgbd = True, # visualize RGB-D 63 | 64 | ### mesh related ### 65 | mesh_vis_freq = 500, # mesh save frequency 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /src/slam/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import mmengine 27 | 28 | from src.utils.general_utils import InfoPrinter 29 | from tensorboardX import SummaryWriter 30 | 31 | 32 | def init_SLAM_model(main_cfg: mmengine.Config, info_printer: InfoPrinter, logger: SummaryWriter = None): 33 | """initialize SLAM model 34 | 35 | Args: 36 | main_cfg : Configuration 37 | info_printer : information printer 38 | logger : tensorbard writer 39 | 40 | Returns: 41 | slam (SlamModel): SLAM module 42 | 43 | """ 44 | ################################################## 45 | ### Co-SLAM 46 | ################################################## 47 | if main_cfg.slam.method == "coslam": 48 | info_printer("Initialize Co-SLAM...", 0, "Co-SLAM") 49 | from src.slam.coslam.coslam import CoSLAMNaruto as CoSLAM 50 | slam = CoSLAM(main_cfg, info_printer) 51 | elif main_cfg.slam.method == "splatam": 52 | info_printer("Initialize SplaTAM...", 0, "SplaTAM") 53 | from src.slam.splatam.splatam import SplatamOurs as SplaTAM 54 | slam = SplaTAM(main_cfg, info_printer, logger) 55 | else: 56 | assert False, f"SLAM choices: [coslam, splatam]. Current option: [{main_cfg.slam.method}]" 57 | return slam -------------------------------------------------------------------------------- /src/simulator/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import mmengine 27 | 28 | from src.utils.general_utils import InfoPrinter 29 | 30 | 31 | def init_simulator(main_cfg: mmengine.Config, info_printer: InfoPrinter): 32 | """initialize Simulator 33 | 34 | Args: 35 | main_cfg (mmengine.Config): Configuration 36 | info_printer (InfoPrinter): information printer 37 | 38 | Returns: 39 | sim (Simulator): sim module 40 | 41 | """ 42 | ################################################## 43 | ### HabitatSim 44 | ################################################## 45 | if main_cfg.sim.method == "habitat": 46 | info_printer("Initialize Simulator...", 0, "HabitatSim") 47 | from src.simulator.habitat_simulator import HabitatSim 48 | sim = HabitatSim( 49 | main_cfg, 50 | info_printer 51 | ) 52 | elif main_cfg.sim.method == "habitat_v2": 53 | info_printer("Initialize Simulator...", 0, "HabitatSimV2") 54 | from src.simulator.habitat_simulator import HabitatSimV2 as HabitatSim 55 | sim = HabitatSim( 56 | main_cfg, 57 | info_printer 58 | ) 59 | else: 60 | assert False, f"Simulator choices: [habitat]. Current option: [{main_cfg.sim.method}]" 61 | return sim -------------------------------------------------------------------------------- /src/slam/coslam/model/keyframe.py: -------------------------------------------------------------------------------- 1 | """ 2 | We have reused part of CoSLAM's code in this file and include our code for NARUTO. 3 | For CoSLAM License, refer to https://github.com/HengyiWang/Co-SLAM/blob/main/LICENSE. 4 | 5 | """ 6 | 7 | 8 | ''' Differences v.s. original CoSLAM implementation 9 | - store ray in add_keyframe 10 | ''' 11 | 12 | import random 13 | import torch 14 | 15 | from third_party.coslam.model.keyframe import KeyFrameDatabase 16 | 17 | class KeyFrameDatabaseNaruto(KeyFrameDatabase): 18 | def __init__(self, config, H, W, num_kf, num_rays_to_save, device) -> None: 19 | super(KeyFrameDatabaseNaruto, self).__init__(config, H, W, num_kf, num_rays_to_save, device) 20 | 21 | def sample_single_keyframe_rays(self, rays, option='random'): 22 | ''' 23 | Sampling strategy for current keyframe rays 24 | ''' 25 | if option == 'random': 26 | idxs = random.sample(range(0, self.H*self.W), self.num_rays_to_save) 27 | elif option == 'filter_depth': 28 | valid_depth_mask = (rays[..., -1] > 0.0) & (rays[..., -1] <= self.config["cam"]["depth_trunc"]) 29 | rays_valid = rays[valid_depth_mask, :] # [n_valid, 7] 30 | num_valid = len(rays_valid) 31 | idxs = random.sample(range(0, num_valid), min(num_valid, self.num_rays_to_save)) 32 | 33 | else: 34 | raise NotImplementedError() 35 | rays = rays[:, idxs] 36 | return rays 37 | 38 | def add_keyframe(self, batch, filter_depth=False): 39 | ''' 40 | Add keyframe rays to the keyframe database 41 | ''' 42 | # batch direction (Bs=1, H*W, 3) 43 | rays = torch.cat([batch['direction'], batch['rgb'], batch['depth'][..., None]], dim=-1) 44 | rays = rays.reshape(1, -1, rays.shape[-1]) 45 | if filter_depth: 46 | rays = self.sample_single_keyframe_rays(rays, 'filter_depth') 47 | else: 48 | rays = self.sample_single_keyframe_rays(rays) 49 | 50 | if not isinstance(batch['frame_id'], torch.Tensor): 51 | batch['frame_id'] = torch.tensor([batch['frame_id']]) 52 | 53 | self.attach_ids(batch['frame_id']) 54 | 55 | # Store the rays 56 | if rays.shape[1] == 0: 57 | return 58 | while rays.shape[1] < self.num_rays_to_save: 59 | rays = torch.cat([rays, rays], dim=1) 60 | self.rays[len(self.frame_ids)-1] = rays[:, :self.num_rays_to_save, :] 61 | -------------------------------------------------------------------------------- /scripts/naruto/run_naruto.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to run the full NARUTO system 4 | ### (active planning and active ray sampling) 5 | ### on the NARUTO dataset. 6 | ################################################## 7 | 8 | # Input arguments 9 | scene=${1:-konoha_village} 10 | num_run=${2:-1} 11 | EXP=${3:-NARUTO} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 12 | ENABLE_VIS=${4:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=6 15 | PROJ_DIR=${PWD} 16 | DATASET=NARUTO 17 | RESULT_DIR=${PROJ_DIR}/results 18 | 19 | ################################################## 20 | ### Random Seed 21 | ### also used to initialize agent pose 22 | ### from indexing a set of predefined pose/traj 23 | ################################################## 24 | seeds=(0) 25 | seeds=("${seeds[@]:0:$num_run}") 26 | 27 | ################################################## 28 | ### Scenes 29 | ### choose one or all of the scenes 30 | ################################################## 31 | scenes=(nine_tailed_fox running_naruto jiraiya naruto hokage_room) 32 | # Check if the input argument is 'all' 33 | if [ "$scene" == "all" ]; then 34 | selected_scenes=${scenes[@]} # Copy all scenes 35 | else 36 | selected_scenes=($scene) # Assign the matching scene 37 | fi 38 | 39 | ################################################## 40 | ### Main 41 | ### Run for selected scenes for N trials 42 | ################################################## 43 | for scene in $selected_scenes 44 | do 45 | for i in "${!seeds[@]}"; do 46 | seed=${seeds[$i]} 47 | 48 | ### create result folder ### 49 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 50 | mkdir -p ${result_dir} 51 | 52 | ### run experiment ### 53 | CFG=configs/${DATASET}/${scene}/${EXP}.py 54 | python src/main/naruto.py --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} 55 | 56 | ### get file paths ### 57 | CKPT=$(find ${result_dir}/coslam/checkpoint/ -type f -name "ckpt_*_final.pt") 58 | INPUT_MESH=$(find ${result_dir}/coslam/mesh/ -type f -name "mesh_*_final.ply") 59 | 60 | ### Cull mesh as Co-SLAM ### 61 | python third_party/neural_slam_eval/cull_mesh.py \ 62 | --config configs/${DATASET}/${scene}/coslam.yaml \ 63 | --input_mesh $INPUT_MESH \ 64 | --ckpt_path $CKPT \ 65 | --remove_occlusion # GO-Surf strategy 66 | done 67 | done 68 | -------------------------------------------------------------------------------- /src/layers/projection.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import torch 27 | import torch.nn as nn 28 | 29 | 30 | class Projection(nn.Module): 31 | """Layer which projects 3D points into a camera view 32 | """ 33 | def __init__(self, height, width, eps=1e-7): 34 | super(Projection, self).__init__() 35 | 36 | self.height = height 37 | self.width = width 38 | self.eps = eps 39 | 40 | def forward(self, points3d, K, normalized=True): 41 | """ 42 | Args: 43 | points3d (torch.tensor, [N,4,(HxW)]: 3D points in homogeneous coordinates 44 | K (torch.tensor, [torch.tensor, (N,4,4)]: camera intrinsics 45 | normalized (bool): 46 | - True: normalized to [-1, 1] 47 | - False: [0, W-1] and [0, H-1] 48 | Returns: 49 | xy (torch.tensor, [N,H,W,2]): pixel coordinates 50 | """ 51 | # projection 52 | points2d = torch.matmul(K[:, :3, :], points3d) 53 | 54 | # convert from homogeneous coordinates 55 | xy = points2d[:, :2, :] / (points2d[:, 2:3, :] + self.eps) 56 | xy = xy.view(points3d.shape[0], 2, self.height, self.width) 57 | xy = xy.permute(0, 2, 3, 1) 58 | 59 | # normalization 60 | if normalized: 61 | xy[..., 0] /= self.width - 1 62 | xy[..., 1] /= self.height - 1 63 | xy = (xy - 0.5) * 2 64 | return xy 65 | -------------------------------------------------------------------------------- /src/slam/slam_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import mmengine 27 | from tensorboardX import SummaryWriter 28 | 29 | from src.utils.general_utils import InfoPrinter 30 | 31 | 32 | class SlamModel(): 33 | def __init__(self, 34 | main_cfg : mmengine.Config, 35 | info_printer: InfoPrinter, 36 | tb_logger : SummaryWriter = None 37 | ) -> None: 38 | """ 39 | Args: 40 | main_cfg (mmengine.Config): Configuration 41 | info_printer (InfoPrinter): information printer 42 | tb_logger (SummaryWriter): Tensorboard Writer 43 | 44 | Attributes: 45 | main_cfg (mmengine.Config): configurations 46 | slam_cfg (mmengine.Config): slam model configurations 47 | tb_logger (SummaryWriter): Tensorboard Writer 48 | 49 | """ 50 | self.main_cfg = main_cfg 51 | self.slam_cfg = main_cfg.slam 52 | self.info_printer = info_printer 53 | if tb_logger is not None: 54 | self.tb_logger = tb_logger 55 | 56 | def update_step(self, step): 57 | """ update step information 58 | 59 | Args: 60 | step (int): step size 61 | 62 | """ 63 | self.step = step 64 | 65 | def save_mesh(self): 66 | """ 67 | 68 | Args: 69 | 70 | 71 | Returns: 72 | 73 | 74 | Attributes: 75 | 76 | """ 77 | raise NotImplementedError -------------------------------------------------------------------------------- /configs/NARUTO/naruto/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "NARUTO", 11 | scene = "naruto", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | ################################################## 25 | ### Simulator 26 | ################################################## 27 | if _base_.sim["method"] == "habitat": 28 | _base_.sim.update( 29 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 30 | ) 31 | 32 | ################################################## 33 | ### SLAM 34 | ################################################## 35 | if _base_.slam["method"] == "coslam": 36 | _base_.slam.update( 37 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 38 | enable_active_planning = True, # enable/disable active planning 39 | active_ray = True, # enable/disable active ray sampling 40 | voxel_size = 0.02, # Voxel size for Uncertainty/SDF volume. Unit: meter 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = None, 44 | 45 | start_c2w = np.array([ 46 | [ 1, 0, 0, 0], 47 | [ 0, 0, -1, -2.4], 48 | [ 0, 1, 0, 0], 49 | [ 0, 0, 0, 1]]).astype(np.float32) 50 | ) 51 | 52 | ################################################## 53 | ### Planner 54 | ################################################## 55 | planner = dict( 56 | up_dir = np.array([0, 0, 1]), 57 | voxel_size = 0.02, # Uncertainty Volume voxel size. Unit: meter 58 | ) 59 | 60 | ################################################## 61 | ### Visualization 62 | ################################################## 63 | visualizer = dict( 64 | vis_rgbd = True, # visualize RGB-D 65 | 66 | ### mesh related ### 67 | mesh_vis_freq = 500, # mesh save frequency 68 | enable_all_vis = True, # enable comprehensive visualization data 69 | save_mesh_voxel_size = 0.02, # voxel size to save mesh for visualization 70 | ) 71 | 72 | -------------------------------------------------------------------------------- /configs/NARUTO/hokage_room/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "NARUTO", 11 | scene = "hokage_room", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | ################################################## 25 | ### Simulator 26 | ################################################## 27 | if _base_.sim["method"] == "habitat": 28 | _base_.sim.update( 29 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 30 | ) 31 | 32 | ################################################## 33 | ### SLAM 34 | ################################################## 35 | if _base_.slam["method"] == "coslam": 36 | _base_.slam.update( 37 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 38 | enable_active_planning = True, # enable/disable active planning 39 | active_ray = True, # enable/disable active ray sampling 40 | voxel_size = 0.1, # Voxel size for Uncertainty/SDF volume. Unit: meter 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = None, 44 | 45 | start_c2w = np.array([ 46 | [ 1, 0, 0, 0], 47 | [ 0, 0, -1, -1], 48 | [ 0, 1, 0, 2], 49 | [ 0, 0, 0, 1]]).astype(np.float32) 50 | ) 51 | 52 | ################################################## 53 | ### Planner 54 | ################################################## 55 | planner = dict( 56 | up_dir = np.array([0, 0, 1]), 57 | voxel_size = 0.1, # Uncertainty Volume voxel size. Unit: meter 58 | ) 59 | 60 | ################################################## 61 | ### Visualization 62 | ################################################## 63 | visualizer = dict( 64 | vis_rgbd = True, # visualize RGB-D 65 | 66 | ### mesh related ### 67 | mesh_vis_freq = 500, # mesh save frequency 68 | enable_all_vis = True, # enable comprehensive visualization data 69 | save_mesh_voxel_size = 0.1, # voxel size to save mesh for visualization 70 | ) 71 | 72 | -------------------------------------------------------------------------------- /scripts/activegamer/run_replica.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to run the full ActiveGAMER system 4 | ### on the Replica dataset. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-office0} 9 | num_run=${2:-1} 10 | EXP=${3:-ActiveGAMER} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 11 | ENABLE_VIS=${4:-0} 12 | GPU_ID=${5:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 15 | PROJ_DIR=${PWD} 16 | DATASET=Replica 17 | RESULT_DIR=${PROJ_DIR}/results/ 18 | 19 | ################################################## 20 | ### Random Seed 21 | ################################################## 22 | seeds=(0 500 1000 1500 1999) 23 | seeds=("${seeds[@]:0:$num_run}") 24 | 25 | ################################################## 26 | ### Scenes 27 | ### choose one or all of the scenes 28 | ################################################## 29 | scenes=(room0 room1 room2 office0 office1 office2 office3 office4) 30 | # Check if the input argument is 'all' 31 | if [ "$scene" == "all" ]; then 32 | selected_scenes=${scenes[@]} # Copy all scenes 33 | else 34 | selected_scenes=($scene) # Assign the matching scene 35 | fi 36 | 37 | ################################################## 38 | ### Main 39 | ### Run for selected scenes for N trials 40 | ################################################## 41 | for scene in $selected_scenes 42 | do 43 | for i in "${!seeds[@]}"; do 44 | seed=${seeds[$i]} 45 | 46 | ### create result folder ### 47 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 48 | mkdir -p ${result_dir} 49 | 50 | ### run experiment ### 51 | CFG=configs/${DATASET}/${scene}/${EXP}.py 52 | python src/main/activegamer.py --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} 53 | 54 | ### 3D Reconstruction evaluation ### 55 | DASHSCENE=${scene: 0: 0-1}_${scene: 0-1} 56 | GT_MESH=$PROJ_DIR/data/replica_v1/${DASHSCENE}/mesh.ply 57 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 58 | 59 | for stage in exploration_stage_0 exploration_stage_1 final; do 60 | ckpt_file="${result_dir}/splatam/${stage}/params.npz" 61 | output_dir="${result_dir}/eval_3d/${stage}" 62 | 63 | if [ -f "$ckpt_file" ]; then 64 | python src/evaluation/eval_splatam_recon.py \ 65 | --ckpt "$ckpt_file" \ 66 | --gt_mesh "${GT_MESH}" \ 67 | --transform_traj "data/Replica/${scene}/traj.txt" \ 68 | --result_dir "$output_dir" 69 | else 70 | echo "⚠️ Skipping $stage: Checkpoint not found at $ckpt_file" 71 | fi 72 | done 73 | 74 | done 75 | done 76 | -------------------------------------------------------------------------------- /scripts/evaluation/eval_replica_activegamer_recon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to evaluate ActiveGAMER system 4 | ### on the Replica dataset for 3D reconstruction. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-office0} 9 | num_run=${2:-1} 10 | EXP=${3:-ActiveGAMER} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 11 | ENABLE_VIS=${4:-0} 12 | GPU_ID=${5:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 15 | PROJ_DIR=${PWD} 16 | DATASET=Replica 17 | RESULT_DIR=${PROJ_DIR}/results/ 18 | 19 | ################################################## 20 | ### Random Seed 21 | ################################################## 22 | seeds=(0 500 1000 1500 1999) 23 | seeds=("${seeds[@]:0:$num_run}") 24 | 25 | ################################################## 26 | ### Scenes 27 | ### choose one or all of the scenes 28 | ################################################## 29 | scenes=(room0 room1 room2 office0 office1 office2 office3 office4) 30 | # Check if the input argument is 'all' 31 | if [ "$scene" == "all" ]; then 32 | selected_scenes=${scenes[@]} # Copy all scenes 33 | else 34 | selected_scenes=($scene) # Assign the matching scene 35 | fi 36 | 37 | ################################################## 38 | ### Main 39 | ### Run for selected scenes for N trials 40 | ################################################## 41 | for scene in $selected_scenes 42 | do 43 | for i in "${!seeds[@]}"; do 44 | seed=${seeds[$i]} 45 | 46 | ### create result folder ### 47 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 48 | mkdir -p ${result_dir} 49 | 50 | ### 3D Reconstruction evaluation ### 51 | DASHSCENE=${scene: 0: 0-1}_${scene: 0-1} 52 | GT_MESH=$PROJ_DIR/data/replica_v1/${DASHSCENE}/mesh.ply 53 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 54 | 55 | python src/evaluation/eval_splatam_recon.py \ 56 | --ckpt ${result_dir}/splatam/exploration_stage_0/params.npz \ 57 | --gt_mesh ${GT_MESH} \ 58 | --transform_traj data/Replica/${scene}/traj.txt \ 59 | --result_dir ${result_dir}/eval_3d/exploration_stage_0 60 | 61 | python src/evaluation/eval_splatam_recon.py \ 62 | --ckpt ${result_dir}/splatam/exploration_stage_1/params.npz \ 63 | --gt_mesh ${GT_MESH} \ 64 | --transform_traj data/Replica/${scene}/traj.txt \ 65 | --result_dir ${result_dir}/eval_3d/exploration_stage_1 66 | 67 | python src/evaluation/eval_splatam_recon.py \ 68 | --ckpt ${result_dir}/splatam/final/params.npz \ 69 | --gt_mesh ${GT_MESH} \ 70 | --transform_traj data/Replica/${scene}/traj.txt \ 71 | --result_dir ${result_dir}/eval_3d/final 72 | 73 | done 74 | done 75 | -------------------------------------------------------------------------------- /src/planner/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import mmengine 27 | 28 | from src.utils.general_utils import InfoPrinter 29 | 30 | 31 | def init_planner(main_cfg: mmengine.Config, info_printer: InfoPrinter): 32 | """initialize planner 33 | 34 | Args: 35 | cfg (mmengine.Config) : Configuration 36 | info_printer (InfoPrinter): information printer 37 | 38 | Returns: 39 | planner (Planner): Planner module 40 | 41 | """ 42 | ################################################## 43 | ### NarutoPlanner 44 | ################################################## 45 | if main_cfg.planner.method == "naruto": 46 | info_printer("Initialize NARUTO Planner...", 0, "NarutoPlanner") 47 | from src.planner.naruto_planner import NarutoPlanner 48 | planner = NarutoPlanner( 49 | main_cfg, 50 | info_printer 51 | ) 52 | elif main_cfg.planner.method == "predefined_traj": 53 | info_printer("Initialize PredefinedTraj Planner...", 0, "PredefinedPlanner") 54 | from src.planner.predefined_traj_planner import PreTrajPlanner 55 | planner = PreTrajPlanner( 56 | main_cfg, 57 | info_printer 58 | ) 59 | elif main_cfg.planner.method == "active_gs": 60 | info_printer("Initialize ActiveGS Planner...", 0, "ActiveGSPlanner") 61 | from src.planner.active_gs_planner import ActiveGSPlanner 62 | planner = ActiveGSPlanner( 63 | main_cfg, 64 | info_printer 65 | ) 66 | 67 | else: 68 | assert False, f"Planner choices: [naruto, predefined_traj, active_gs]. Current option: [{main_cfg.planner.method}]" 69 | return planner -------------------------------------------------------------------------------- /scripts/evaluation/eval_mp3d_activegamer_recon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to run the full NARUTO system 4 | ### on the Replica dataset. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-GdvgFV5R1Z5} 9 | num_run=${2:-1} 10 | EXP=${3:-ActiveGAMER} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 11 | ENABLE_VIS=${4:-0} 12 | GPU_ID=${5:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 15 | PROJ_DIR=${PWD} 16 | DATASET=MP3D 17 | RESULT_DIR=${PROJ_DIR}/results/ 18 | 19 | ################################################## 20 | ### Random Seed 21 | ################################################## 22 | seeds=(0 500 1000 1500 1999) 23 | seeds=("${seeds[@]:0:$num_run}") 24 | 25 | ################################################## 26 | ### Scenes 27 | ### choose one or all of the scenes 28 | ################################################## 29 | scenes=( GdvgFV5R1Z5 gZ6f7yhEvPG HxpKQynjfin pLe4wQe7qrG YmJkqBEsHnH ) 30 | # Check if the input argument is 'all' 31 | if [ "$scene" == "all" ]; then 32 | selected_scenes=${scenes[@]} # Copy all scenes 33 | else 34 | selected_scenes=($scene) # Assign the matching scene 35 | fi 36 | 37 | ################################################## 38 | ### Main 39 | ### Run for selected scenes for N trials 40 | ################################################## 41 | for scene in $selected_scenes 42 | do 43 | for i in "${!seeds[@]}"; do 44 | seed=${seeds[$i]} 45 | 46 | ### create result folder ### 47 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 48 | mkdir -p ${result_dir} 49 | 50 | ### run experiment ### 51 | CFG=configs/${DATASET}/${scene}/${EXP}.py 52 | 53 | ### 3D Reconstruction evaluation ### 54 | DASHSCENE=${scene: 0: 0-1}_${scene: 0-1} 55 | GT_MESH=$PROJ_DIR/data/mp3d_data/v1/scans/${scene}/mesh.obj 56 | 57 | python src/evaluation/eval_splatam_recon.py \ 58 | --ckpt ${result_dir}/splatam/exploration_stage_0/params.npz \ 59 | --gt_mesh ${GT_MESH} \ 60 | --transform_traj data/mp3d_sim_nvs/${scene}/traj.txt \ 61 | --result_dir ${result_dir}/eval_3d/exploration_stage_0 62 | 63 | python src/evaluation/eval_splatam_recon.py \ 64 | --ckpt ${result_dir}/splatam/exploration_stage_1/params.npz \ 65 | --gt_mesh ${GT_MESH} \ 66 | --transform_traj data/mp3d_sim_nvs/${scene}/traj.txt \ 67 | --result_dir ${result_dir}/eval_3d/exploration_stage_1 68 | 69 | python src/evaluation/eval_splatam_recon.py \ 70 | --ckpt ${result_dir}/splatam/final/params.npz \ 71 | --gt_mesh ${GT_MESH} \ 72 | --transform_traj data/mp3d_sim_nvs/${scene}/traj.txt \ 73 | --result_dir ${result_dir}/eval_3d/final 74 | 75 | done 76 | done 77 | -------------------------------------------------------------------------------- /src/utils/config_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import yaml 27 | from typing import Dict 28 | 29 | 30 | def load_config(path: str, default_path=None) -> Dict: 31 | """ Loads config file. 32 | 33 | Args: 34 | path (str): path to config file. 35 | default_path (str, optional): whether to use default path. Defaults to None. 36 | 37 | Returns: 38 | cfg (dict): config dict. 39 | """ 40 | ### load configuration from file itself ### 41 | with open(path, 'r') as f: 42 | cfg_special = yaml.full_load(f) 43 | 44 | ### check if we should inherit from a config ### 45 | inherit_from = cfg_special.get('inherit_from') 46 | 47 | ### if yes, load this config first as default. Otherwise, use the default_path ### 48 | if inherit_from is not None: 49 | # inherit_cfg_path = "/".join(path.split("/")[:-3]) + "/" + inherit_from 50 | cfg = load_config(inherit_from, default_path) 51 | elif default_path is not None: 52 | with open(default_path, 'r') as f: 53 | cfg = yaml.full_load(f) 54 | else: 55 | cfg = dict() 56 | 57 | ### include main configuration ### 58 | update_recursive(cfg, cfg_special) 59 | 60 | return cfg 61 | 62 | 63 | def update_recursive(dict1: Dict, dict2: Dict) -> None: 64 | """ Update two config dictionaries recursively. 65 | 66 | Args: 67 | dict1 (dict): first dictionary to be updated. 68 | dict2 (dict): second dictionary which entries should be used. 69 | """ 70 | for k, v in dict2.items(): 71 | if k not in dict1: 72 | dict1[k] = dict() 73 | if isinstance(v, dict): 74 | update_recursive(dict1[k], v) 75 | else: 76 | dict1[k] = v 77 | -------------------------------------------------------------------------------- /envs/requirements.txt: -------------------------------------------------------------------------------- 1 | addict==2.4.0 2 | attrs==24.2.0 3 | autocommand==2.2.2 4 | backports.tarfile==1.2.0 5 | blinker==1.8.2 6 | certifi==2024.8.30 7 | charset-normalizer==3.3.2 8 | click==8.1.7 9 | comm==0.2.2 10 | ConfigArgParse==1.7 11 | contourpy==1.1.1 12 | cycler==0.12.1 13 | cyclonedds==0.10.5 14 | dash==2.17.1 15 | dash-core-components==2.0.0 16 | dash-html-components==2.0.0 17 | dash-table==5.0.0 18 | docker-pycreds==0.4.0 19 | fastjsonschema==2.20.0 20 | Flask==3.0.3 21 | fonttools==4.53.1 22 | freetype-py==2.5.1 23 | gitdb==4.0.11 24 | GitPython==3.1.43 25 | # habitat_sim==0.2.3 26 | idna==3.8 27 | imageio==2.35.1 28 | imageio-ffmpeg==0.5.1 29 | importlib_metadata==8.4.0 30 | importlib_resources==6.4.4 31 | inflect==7.3.1 32 | iopath==0.1.10 33 | ipywidgets==8.1.5 34 | itsdangerous==2.2.0 35 | jaraco.context==5.3.0 36 | jaraco.functools==4.0.1 37 | jaraco.text==3.12.1 38 | Jinja2==3.1.4 39 | joblib==1.4.2 40 | jsonschema==4.23.0 41 | jsonschema-specifications==2023.12.1 42 | jupyter_core==5.7.2 43 | jupyterlab_widgets==3.0.13 44 | kiwisolver==1.4.5 45 | kornia==0.7.3 46 | kornia_rs==0.1.5 47 | lightning-utilities==0.11.6 48 | llvmlite==0.41.1 49 | lpips==0.1.4 50 | markdown-it-py==3.0.0 51 | MarkupSafe==2.1.5 52 | matplotlib==3.7.5 53 | mdurl==0.1.2 54 | mmengine==0.7.3 55 | mmcv==2.0.0 56 | more-itertools==10.3.0 57 | natsort==8.4.0 58 | nbformat==5.5.0 59 | nest-asyncio==1.6.0 60 | networkx==3.1 61 | numba==0.58.1 62 | numpy==1.24.4 63 | numpy-quaternion==2023.0.4 64 | # NumpyMarchingCubes==0.0.1 65 | open3d==0.16.0 66 | opencv-python==4.10.0.84 67 | ordered-set==4.1.0 68 | packaging==24.1 69 | pandas==2.0.3 70 | pillow==10.4.0 71 | pkgutil_resolve_name==1.3.10 72 | platformdirs==4.2.2 73 | plotly==5.24.0 74 | plyfile==0.8.1 75 | portalocker==2.10.1 76 | protobuf==5.28.0 77 | psutil==6.0.0 78 | pyglet==2.0.17 79 | PyOpenGL==3.1.0 80 | pyparsing==3.1.4 81 | pyquaternion==0.9.9 82 | pyrender==0.1.45 83 | python-dateutil==2.9.0.post0 84 | pytorch-msssim==1.0.0 85 | pytz==2024.1 86 | PyYAML==6.0.2 87 | referencing==0.35.1 88 | requests==2.32.3 89 | retrying==1.3.4 90 | rich==13.8.0 91 | rich-click==1.8.3 92 | rpds-py==0.20.0 93 | scikit-learn==1.3.2 94 | scipy==1.10.1 95 | sentry-sdk==2.13.0 96 | setproctitle==1.3.3 97 | smmap==5.0.1 98 | tenacity==9.0.0 99 | termcolor==2.4.0 100 | threadpoolctl==3.5.0 101 | tensorboardX==2.6.2.2 102 | tomli==2.0.1 103 | torch==1.12.1+cu116 104 | torchaudio==0.12.1+cu116 105 | torchmetrics==1.4.1 106 | torchvision==0.13.1+cu116 107 | tqdm==4.65.0 108 | trimesh==3.23.5 109 | typeguard==4.3.0 110 | tzdata==2024.1 111 | urllib3==2.2.2 112 | wandb==0.17.8 113 | Werkzeug==3.0.4 114 | widgetsnbextension==4.0.13 115 | yapf==0.40.1 116 | zipp==3.20.1 117 | git+https://github.com/JonathonLuiten/diff-gaussian-rasterization-w-depth.git@cb65e4b86bc3bd8ed42174b72a62e8d3a3a71110 -------------------------------------------------------------------------------- /scripts/activegamer/run_mp3d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to run the full NARUTO system 4 | ### on the Replica dataset. 5 | ################################################## 6 | 7 | # Input arguments 8 | scene=${1:-GdvgFV5R1Z5} 9 | num_run=${2:-1} 10 | EXP=${3:-ActiveGAMER} # config in configs/{DATASET}/{scene}/{EXP}.py will be loaded 11 | ENABLE_VIS=${4:-0} 12 | GPU_ID=${5:-0} 13 | 14 | export CUDA_VISIBLE_DEVICES=${GPU_ID} 15 | PROJ_DIR=${PWD} 16 | DATASET=MP3D 17 | RESULT_DIR=${PROJ_DIR}/results/ 18 | 19 | ################################################## 20 | ### Random Seed 21 | ################################################## 22 | seeds=(0 500 1000 1500 1999) 23 | seeds=("${seeds[@]:0:$num_run}") 24 | 25 | ################################################## 26 | ### Scenes 27 | ### choose one or all of the scenes 28 | ################################################## 29 | scenes=( GdvgFV5R1Z5 gZ6f7yhEvPG HxpKQynjfin pLe4wQe7qrG YmJkqBEsHnH ) 30 | # Check if the input argument is 'all' 31 | if [ "$scene" == "all" ]; then 32 | selected_scenes=${scenes[@]} # Copy all scenes 33 | else 34 | selected_scenes=($scene) # Assign the matching scene 35 | fi 36 | 37 | ################################################## 38 | ### Main 39 | ### Run for selected scenes for N trials 40 | ################################################## 41 | for scene in $selected_scenes 42 | do 43 | for i in "${!seeds[@]}"; do 44 | seed=${seeds[$i]} 45 | 46 | ### create result folder ### 47 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 48 | mkdir -p ${result_dir} 49 | 50 | ### run experiment ### 51 | CFG=configs/${DATASET}/${scene}/${EXP}.py 52 | python src/main/activegamer.py --cfg ${CFG} --seed ${seed} --result_dir ${result_dir} --enable_vis ${ENABLE_VIS} 53 | 54 | ### 3D Reconstruction evaluation ### 55 | DASHSCENE=${scene: 0: 0-1}_${scene: 0-1} 56 | GT_MESH=$PROJ_DIR/data/mp3d_data/v1/scans/${scene}/mesh.obj 57 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 58 | 59 | for stage in exploration_stage_0 exploration_stage_1 final; do 60 | ckpt_file="${result_dir}/splatam/${stage}/params.npz" 61 | output_dir="${result_dir}/eval_3d/${stage}" 62 | traj_file="data/mp3d_sim_nvs/${scene}/traj.txt" 63 | 64 | if [ -f "$ckpt_file" ]; then 65 | python src/evaluation/eval_splatam_recon.py \ 66 | --ckpt "$ckpt_file" \ 67 | --gt_mesh "${GT_MESH}" \ 68 | --transform_traj "$traj_file" \ 69 | --result_dir "$output_dir" 70 | else 71 | echo "⚠️ Skipping $stage: Checkpoint not found at $ckpt_file" 72 | fi 73 | done 74 | 75 | done 76 | done 77 | -------------------------------------------------------------------------------- /configs/NARUTO/jiraiya/NARUTO.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "NARUTO", 11 | scene = "jiraiya", 12 | num_iter = 5000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | ################################################## 25 | ### Simulator 26 | ################################################## 27 | if _base_.sim["method"] == "habitat": 28 | _base_.sim.update( 29 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 30 | ) 31 | 32 | ################################################## 33 | ### SLAM 34 | ################################################## 35 | if _base_.slam["method"] == "coslam": 36 | _base_.slam.update( 37 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 38 | enable_active_planning = True, # enable/disable active planning 39 | active_ray = True, # enable/disable active ray sampling 40 | voxel_size = 0.02, # Voxel size for Uncertainty/SDF volume. Unit: meter 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = None, 44 | 45 | start_c2w = np.array([ 46 | [ 1, 0, 0, 0], 47 | [ 0, 0, -1, -2.9], 48 | [ 0, 1, 0, 0], 49 | [ 0, 0, 0, 1]]).astype(np.float32) 50 | ) 51 | 52 | ################################################## 53 | ### Planner 54 | ################################################## 55 | planner = dict( 56 | up_dir = np.array([0, 0, 1]), 57 | # step_size = 0.001, # step size. Unit: meter 58 | voxel_size = 0.02, # Uncertainty Volume voxel size. Unit: meter 59 | # rrt_step_size = 1, 60 | 61 | ) 62 | 63 | ################################################## 64 | ### Visualization 65 | ################################################## 66 | visualizer = dict( 67 | vis_rgbd = True, # visualize RGB-D 68 | 69 | ### mesh related ### 70 | mesh_vis_freq = 500, # mesh save frequency 71 | enable_all_vis = False, # enable comprehensive visualization data 72 | save_mesh_voxel_size = 0.05, # voxel size to save mesh for visualization 73 | save_mesh_freq = 5, # frequency to save mesh for visualization 74 | ) 75 | 76 | -------------------------------------------------------------------------------- /configs/NARUTO/jiraiya/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "jiraiya" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/NARUTO_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/NARUTO/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/NARUTO/{scene_name}/naruto.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | -------------------------------------------------------------------------------- /configs/NARUTO/naruto/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "naruto" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/NARUTO_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/NARUTO/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/NARUTO/{scene_name}/naruto.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | -------------------------------------------------------------------------------- /configs/NARUTO/hokage_room/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "hokage_room" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/NARUTO_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/NARUTO/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/NARUTO/{scene_name}/naruto.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | -------------------------------------------------------------------------------- /configs/Replica/office1/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "office_1" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/office2/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "office_2" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/office3/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "office_3" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/office4/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "office_4" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/room0/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "room_0" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/room1/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "room_1" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/room2/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "room_2" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/Replica/office0/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "office_0" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/replica_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/replica_v1/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(dirs['data_dir'], "habitat/replicaSDK_stage.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ##################################################z 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | ) 48 | ) 49 | 50 | ################################################## 51 | ### Camera 52 | ################################################## 53 | ### get FoV ### 54 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 55 | 56 | camera = dict( 57 | fps = simulator['FPS'], 58 | pinhole = dict( 59 | enable = True, 60 | cam_type = ['color', 'depth'], 61 | resolution_hw = [680, 1200], 62 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 63 | horizontal = dict( 64 | num_rot = 1 65 | ), 66 | fov = (fov(680, 600), fov(1200, 600)), # h, w 67 | ), 68 | equirectangular = dict( 69 | enable = True, 70 | cam_type = ['color', 'depth'], 71 | resolution_hw = [1024, 2048], 72 | poses = [ 73 | [0, 0, 0], 74 | ] 75 | ), 76 | ) 77 | 78 | ################################################## 79 | ### Simulation output storage 80 | ################################################## 81 | sim_output = dict( 82 | save_video = True, 83 | save_frame = True, 84 | save_pose = True, 85 | save_K = True, 86 | frame_suffix = '.png', 87 | depth_png_scale = 6553.5, 88 | clear_old_output = True, 89 | force_clear = True, 90 | ) 91 | 92 | -------------------------------------------------------------------------------- /configs/MP3D/GdvgFV5R1Z5/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "GdvgFV5R1Z5" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/MP3D_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/MP3D/v1/tasks/mp3d/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/MP3D/{scene_name}/mp3d.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | predefined_traj_txt = "" # format of the pose: matrix as 16D vector. RDF c2w 48 | ) 49 | ) 50 | 51 | ################################################## 52 | ### Camera 53 | ################################################## 54 | ### get FoV ### 55 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 56 | 57 | camera = dict( 58 | fps = simulator['FPS'], 59 | pinhole = dict( 60 | enable = True, 61 | cam_type = ['color', 'depth'], 62 | resolution_hw = [680, 1200], 63 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 64 | horizontal = dict( 65 | num_rot = 1 66 | ), 67 | fov = (fov(680, 600), fov(1200, 600)), # h, w 68 | ), 69 | # DEBUG 70 | equirectangular = dict( 71 | enable = True, 72 | cam_type = ['color', 'depth'], 73 | resolution_hw = [1024, 2048], 74 | poses = [ 75 | [0, 0, 0], 76 | ] 77 | ), 78 | ) 79 | 80 | ################################################## 81 | ### Simulation output storage 82 | ################################################## 83 | sim_output = dict( 84 | save_video = True, 85 | save_frame = True, 86 | save_pose = True, 87 | save_K = True, 88 | frame_suffix = '.png', 89 | depth_png_scale = 6553.5, 90 | clear_old_output = True, 91 | force_clear = True, 92 | ) 93 | 94 | -------------------------------------------------------------------------------- /configs/MP3D/HxpKQynjfin/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "HxpKQynjfin" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/MP3D_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/MP3D/v1/tasks/mp3d/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/MP3D/{scene_name}/mp3d.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | predefined_traj_txt = "" # format of the pose: matrix as 16D vector. RDF c2w 48 | ) 49 | ) 50 | 51 | ################################################## 52 | ### Camera 53 | ################################################## 54 | ### get FoV ### 55 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 56 | 57 | camera = dict( 58 | fps = simulator['FPS'], 59 | pinhole = dict( 60 | enable = True, 61 | cam_type = ['color', 'depth'], 62 | resolution_hw = [680, 1200], 63 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 64 | horizontal = dict( 65 | num_rot = 1 66 | ), 67 | fov = (fov(680, 600), fov(1200, 600)), # h, w 68 | ), 69 | # DEBUG 70 | equirectangular = dict( 71 | enable = True, 72 | cam_type = ['color', 'depth'], 73 | resolution_hw = [1024, 2048], 74 | poses = [ 75 | [0, 0, 0], 76 | ] 77 | ), 78 | ) 79 | 80 | ################################################## 81 | ### Simulation output storage 82 | ################################################## 83 | sim_output = dict( 84 | save_video = True, 85 | save_frame = True, 86 | save_pose = True, 87 | save_K = True, 88 | frame_suffix = '.png', 89 | depth_png_scale = 6553.5, 90 | clear_old_output = True, 91 | force_clear = True, 92 | ) 93 | 94 | -------------------------------------------------------------------------------- /configs/MP3D/YmJkqBEsHnH/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "YmJkqBEsHnH" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/MP3D_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/MP3D/v1/tasks/mp3d/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/MP3D/{scene_name}/mp3d.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | predefined_traj_txt = "" # format of the pose: matrix as 16D vector. RDF c2w 48 | ) 49 | ) 50 | 51 | ################################################## 52 | ### Camera 53 | ################################################## 54 | ### get FoV ### 55 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 56 | 57 | camera = dict( 58 | fps = simulator['FPS'], 59 | pinhole = dict( 60 | enable = True, 61 | cam_type = ['color', 'depth'], 62 | resolution_hw = [680, 1200], 63 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 64 | horizontal = dict( 65 | num_rot = 1 66 | ), 67 | fov = (fov(680, 600), fov(1200, 600)), # h, w 68 | ), 69 | # DEBUG 70 | equirectangular = dict( 71 | enable = True, 72 | cam_type = ['color', 'depth'], 73 | resolution_hw = [1024, 2048], 74 | poses = [ 75 | [0, 0, 0], 76 | ] 77 | ), 78 | ) 79 | 80 | ################################################## 81 | ### Simulation output storage 82 | ################################################## 83 | sim_output = dict( 84 | save_video = True, 85 | save_frame = True, 86 | save_pose = True, 87 | save_K = True, 88 | frame_suffix = '.png', 89 | depth_png_scale = 6553.5, 90 | clear_old_output = True, 91 | force_clear = True, 92 | ) 93 | 94 | -------------------------------------------------------------------------------- /configs/MP3D/gZ6f7yhEvPG/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "gZ6f7yhEvPG" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/MP3D_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/MP3D/v1/tasks/mp3d/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/MP3D/{scene_name}/mp3d.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | predefined_traj_txt = "" # format of the pose: matrix as 16D vector. RDF c2w 48 | ) 49 | ) 50 | 51 | ################################################## 52 | ### Camera 53 | ################################################## 54 | ### get FoV ### 55 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 56 | 57 | camera = dict( 58 | fps = simulator['FPS'], 59 | pinhole = dict( 60 | enable = True, 61 | cam_type = ['color', 'depth'], 62 | resolution_hw = [680, 1200], 63 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 64 | horizontal = dict( 65 | num_rot = 1 66 | ), 67 | fov = (fov(680, 600), fov(1200, 600)), # h, w 68 | ), 69 | # DEBUG 70 | equirectangular = dict( 71 | enable = True, 72 | cam_type = ['color', 'depth'], 73 | resolution_hw = [1024, 2048], 74 | poses = [ 75 | [0, 0, 0], 76 | ] 77 | ), 78 | ) 79 | 80 | ################################################## 81 | ### Simulation output storage 82 | ################################################## 83 | sim_output = dict( 84 | save_video = True, 85 | save_frame = True, 86 | save_pose = True, 87 | save_K = True, 88 | frame_suffix = '.png', 89 | depth_png_scale = 6553.5, 90 | clear_old_output = True, 91 | force_clear = True, 92 | ) 93 | 94 | -------------------------------------------------------------------------------- /configs/MP3D/pLe4wQe7qrG/habitat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | ################################################## 5 | ### Scene specific parameters 6 | ################################################## 7 | scene_name = "pLe4wQe7qrG" 8 | split_name = "tmp" 9 | left_right = "left" 10 | location_idx = 0 11 | 12 | ################################################## 13 | ### Directory 14 | ################################################## 15 | dirs = dict( 16 | output_dir = "data/MP3D_sim/{}/{}/{}/{:02}".format( 17 | scene_name, 18 | split_name, 19 | left_right, 20 | location_idx 21 | ), 22 | data_dir = "data/MP3D/v1/tasks/mp3d/{}".format(scene_name) 23 | ) 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | simulator = dict( 29 | physics = dict( 30 | enable = True, 31 | gravity = [0.0, -10.0, 0.0] 32 | ), 33 | scene_id = os.path.join(f"configs/MP3D/{scene_name}/mp3d.stage_config.json"), 34 | duration = 100, # sec 35 | FPS = 20, 36 | ) 37 | 38 | ################################################## 39 | ### Agent 40 | ################################################## 41 | agent = dict( 42 | position = [0.0, 0.0, 0.0], 43 | rotation = [0.0, 0.0, 0.0], 44 | motion_profile = dict( 45 | radius = 1, 46 | motion_type = 'predefined', # [stationary, random, spiral_forward, spiral_forward_with_hori_rot, forward] 47 | predefined_traj_txt = "" # format of the pose: matrix as 16D vector. RDF c2w 48 | ) 49 | ) 50 | 51 | ################################################## 52 | ### Camera 53 | ################################################## 54 | ### get FoV ### 55 | fov = lambda size, focal: np.rad2deg(np.arctan((size/2)/focal))*2 56 | 57 | camera = dict( 58 | fps = simulator['FPS'], 59 | pinhole = dict( 60 | enable = True, 61 | cam_type = ['color', 'depth'], 62 | resolution_hw = [680, 1200], 63 | orientation_type = 'horizontal', # [skybox, horizontal, horizontal+UpDown] 64 | horizontal = dict( 65 | num_rot = 1 66 | ), 67 | fov = (fov(680, 600), fov(1200, 600)), # h, w 68 | ), 69 | # DEBUG 70 | equirectangular = dict( 71 | enable = True, 72 | cam_type = ['color', 'depth'], 73 | resolution_hw = [1024, 2048], 74 | poses = [ 75 | [0, 0, 0], 76 | ] 77 | ), 78 | ) 79 | 80 | ################################################## 81 | ### Simulation output storage 82 | ################################################## 83 | sim_output = dict( 84 | save_video = True, 85 | save_frame = True, 86 | save_pose = True, 87 | save_K = True, 88 | frame_suffix = '.png', 89 | depth_png_scale = 6553.5, 90 | clear_old_output = True, 91 | force_clear = True, 92 | ) 93 | 94 | -------------------------------------------------------------------------------- /src/visualization/active_gs_visualizer.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import cv2 27 | import mmengine 28 | import numpy as np 29 | import os 30 | import torch 31 | 32 | from src.visualization.visualizer import Visualizer 33 | from src.utils.general_utils import InfoPrinter 34 | 35 | class ActiveGSVisualizer(Visualizer): 36 | def __init__(self, 37 | main_cfg : mmengine.Config, 38 | info_printer: InfoPrinter 39 | ) -> None: 40 | """ 41 | Args: 42 | main_cfg (mmengine.Config): Configuration 43 | info_printer (InfoPrinter): information printer 44 | 45 | Attributes: 46 | main_cfg (mmengine.Config): configurations 47 | vis_cfg (mmengine.Config) : visualizer model configurations 48 | info_printer (InfoPrinter): information printer 49 | 50 | """ 51 | super(ActiveGSVisualizer, self).__init__(main_cfg, info_printer) 52 | 53 | def save_rgbd(self, 54 | color: torch.Tensor, 55 | depth: torch.Tensor 56 | ) -> None: 57 | """save RGB-D visualization 58 | 59 | Args: 60 | rgb (torch.Tensor, [H,W,3]): color map. Range: 0-1 61 | depth (torch.Tensor, [H,W]): depth map. 62 | 63 | """ 64 | rgbd_vis = self.visualize_rgbd(color, depth, return_vis=True) 65 | filepath = os.path.join(self.main_cfg.dirs.result_dir, "visualization", "rgbd", f"{self.step:04}.png") 66 | rgbd_vis = (rgbd_vis * 255).astype(np.uint8) 67 | cv2.imwrite(filepath, rgbd_vis) 68 | 69 | def save_pose(self, pose: np.ndarray) -> None: 70 | """ save pose 71 | 72 | Args: 73 | pose: [4,4], current pose. Format: camera-to-world, RUB system 74 | 75 | """ 76 | filepath = os.path.join(self.main_cfg.dirs.result_dir, "visualization", "pose", f"{self.step:04}.npy") 77 | np.save(filepath, pose) 78 | -------------------------------------------------------------------------------- /configs/Replica/room0/NARUTO_rrt.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### NARUTO (General) 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room0", 12 | num_iter = 2000, 13 | ) 14 | 15 | ################################################## 16 | ### Directories 17 | ################################################## 18 | dirs = dict( 19 | data_dir = "data/", 20 | result_dir = "results/", 21 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 22 | ) 23 | 24 | 25 | ################################################## 26 | ### Simulator 27 | ################################################## 28 | if _base_.sim["method"] == "habitat": 29 | _base_.sim.update( 30 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 31 | ) 32 | 33 | ################################################## 34 | ### SLAM 35 | ################################################## 36 | if _base_.slam["method"] == "coslam": 37 | _base_.slam.update( 38 | room_cfg = f"{dirs['cfg_dir']}/coslam.yaml", # Co-SLAM room configuration 39 | enable_active_planning = True, # enable/disable active planning 40 | active_ray = True, # enable/disable active ray sampling 41 | 42 | use_traj_pose = False, # use pre-defined trajectory pose 43 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 44 | dirs["data_dir"], 45 | "Replica", general['scene'] 46 | ), 47 | 48 | start_c2w = np.eye(4) 49 | ) 50 | 51 | ################################################## 52 | ### Planner 53 | ################################################## 54 | planner = dict( 55 | ### RRT ### 56 | local_planner_method = "RRT", # RRT method 57 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 58 | ) 59 | 60 | ### NARUTO Planner ### 61 | if planner["local_planner_method"] == "RRT": 62 | planner.update( 63 | rrt_step_size = _base_.planner['step_size'] / _base_.planner['voxel_size'], # Unit: voxel 64 | rrt_step_amplifier = 10, # rrt step amplifier to fast expansion 65 | rrt_maxz = 100, # Maximum Z-level to limit the RRT nodes. Unit: voxel 66 | rrt_max_iter = None, # maximum iterations for RRT 67 | rrt_z_levels = None, # Z levels for sampling RRT nodes. Unit: voxel. Min and Max level 68 | ) 69 | 70 | 71 | ################################################## 72 | ### Visualization 73 | ################################################## 74 | visualizer = dict( 75 | vis_rgbd = True, # visualize RGB-D 76 | 77 | ### mesh related ### 78 | mesh_vis_freq = 500, # mesh save frequency 79 | ) 80 | 81 | -------------------------------------------------------------------------------- /src/evaluation/eval_traj_length.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import argparse 27 | import os 28 | import sys 29 | import torch 30 | from tqdm import tqdm 31 | sys.path.append(os.getcwd()) 32 | 33 | from src.utils.general_utils import update_results_file 34 | 35 | 36 | def argument_parsing() -> argparse.Namespace: 37 | """parse arguments 38 | 39 | Returns: 40 | args: arguments 41 | 42 | """ 43 | parser = argparse.ArgumentParser( 44 | description="Arguments to calculate trajectory from the poses stored in checkpoint." 45 | ) 46 | parser.add_argument("--ckpt", type=str, help="Checkpoint path") 47 | parser.add_argument("--result_txt", type=str, help="result txt") 48 | args = parser.parse_args() 49 | return args 50 | 51 | if __name__ == "__main__": 52 | args = argument_parsing() 53 | 54 | ################################################## 55 | ### load checkpoint 56 | ################################################## 57 | print(f"==> loading checkpoint: [{args.ckpt}]") 58 | ckpt_path = args.ckpt 59 | ckpt = torch.load(ckpt_path) 60 | 61 | ################################################## 62 | ### read pose and calculate trajectory and save in result 63 | ################################################## 64 | poses_tensor = ckpt['pose'] 65 | traj_len = 0 66 | 67 | for i in tqdm(range(len(poses_tensor)), desc="==> Calculating traj. length: "): 68 | if i == 0: 69 | cur_pose = poses_tensor[i] 70 | continue 71 | rel_pose = torch.inverse(poses_tensor[i]) @ cur_pose 72 | traj_len += torch.norm(rel_pose[:3, 3]).item() 73 | cur_pose = poses_tensor[i] 74 | 75 | print(f"==> Total trajectory length: {traj_len:.2f}") 76 | 77 | ################################################## 78 | ### write/update result to the result txt 79 | ################################################## 80 | result = {"traj_len(m)": traj_len} 81 | update_results_file(result, args.result_txt) 82 | -------------------------------------------------------------------------------- /scripts/evaluation/eval_replica.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to evaluate the full NARUTO system 4 | ### (active planning and active ray sampling) 5 | ### on the Replica dataset. 6 | ################################################## 7 | 8 | # Input arguments 9 | scene=${1:-office0} 10 | trial=${2:-0} 11 | iter_num=${3:-2000} 12 | EXP=${4:-NARUTO} 13 | 14 | export CUDA_VISIBLE_DEVICES=0 15 | PROJ_DIR=${PWD} 16 | DATASET=Replica 17 | RESULT_DIR=${PROJ_DIR}/results 18 | 19 | ################################################## 20 | ### Select trial indexs 21 | ################################################## 22 | trials=(0 1 2 3 4) 23 | # Check if the input argument is 'all' 24 | if [ "$trial" == "all" ]; then 25 | selected_trials=${trials[@]} # Copy all trials 26 | else 27 | selected_trials=($trial) # Assign the matching scene 28 | fi 29 | 30 | ################################################## 31 | ### Select scenes 32 | ################################################## 33 | scenes=(room0 room1 room2 office0 office1 office2 office3 office4) 34 | # Check if the input argument is 'all' 35 | if [ "$scene" == "all" ]; then 36 | selected_scenes=${scenes[@]} # Copy all scenes 37 | else 38 | selected_scenes=($scene) # Assign the matching scene 39 | fi 40 | 41 | ################################################## 42 | ### Main 43 | ### Evaluate selected experiment 44 | ################################################## 45 | for scene in $selected_scenes 46 | do 47 | for i in $trials; do 48 | ### get result folder ### 49 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 50 | result_txt=${result_dir}/eval_result.txt 51 | echo "==> Evaluating [${result_dir}]" 52 | 53 | ### get file paths ### 54 | CKPT=$result_dir/coslam/checkpoint/ckpt_${iter_num}_final.pt 55 | INPUT_MESH=$result_dir/coslam/mesh/mesh_${iter_num}_final.ply 56 | REC_MESH=$result_dir/coslam/mesh/mesh_${iter_num}_final_cull_occlusion.ply 57 | DASHSCENE=${scene: 0: 0-1}_${scene: 0-1} 58 | GT_MESH=$PROJ_DIR/data/replica_v1/${DASHSCENE}/mesh.ply 59 | 60 | ### Cull mesh as Co-SLAM ### 61 | python third_party/neural_slam_eval/cull_mesh.py \ 62 | --config configs/${DATASET}/${scene}/coslam.yaml \ 63 | --input_mesh $INPUT_MESH \ 64 | --ckpt_path $CKPT \ 65 | --remove_occlusion # GO-Surf strategy 66 | 67 | ### Evaluate ### 68 | echo "==> Evaluating reconstruction result [accuracy, completeness, and completion ratio]" 69 | python src/evaluation/eval_recon.py \ 70 | --rec_mesh $REC_MESH \ 71 | --gt_mesh $GT_MESH \ 72 | --result_txt $result_txt 73 | 74 | echo "==> Evaluating reconstruction result [MAD]" 75 | python src/evaluation/eval_mad.py \ 76 | --cfg configs/${DATASET}/${scene}/NARUTO.py \ 77 | --ckpt $CKPT --gt_mesh $GT_MESH \ 78 | --result_txt $result_txt 79 | 80 | echo "==> Evaluating trajectory length (m)" 81 | python src/evaluation/eval_traj_length.py \ 82 | --ckpt ${CKPT} \ 83 | --result_txt $result_txt 84 | done 85 | done 86 | -------------------------------------------------------------------------------- /scripts/evaluation/eval_mp3d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################## 3 | ### This script is to evaluate the full NARUTO system 4 | ### (active planning and active ray sampling) 5 | ### on the Replica dataset. 6 | ################################################## 7 | 8 | # Input arguments 9 | scene=${1:-gZ6f7yhEvPG} 10 | trial=${2:-0} 11 | iter_num=${3:-5000} 12 | EXP=${4:-NARUTO} 13 | 14 | export CUDA_VISIBLE_DEVICES=0 15 | PROJ_DIR=${PWD} 16 | DATASET=MP3D 17 | RESULT_DIR=${PROJ_DIR}/results 18 | 19 | ################################################## 20 | ### Select trial indexs 21 | ################################################## 22 | trials=(0 1 2 3 4) 23 | # Check if the input argument is 'all' 24 | if [ "$trial" == "all" ]; then 25 | selected_trials=${trials[@]} # Copy all trials 26 | else 27 | selected_trials=($trial) # Assign the matching scene 28 | fi 29 | 30 | ################################################## 31 | ### Select scenes 32 | ################################################## 33 | scenes=(GdvgFV5R1Z5 gZ6f7yhEvPG HxpKQynjfin pLe4wQe7qrG YmJkqBEsHnH) 34 | # Check if the input argument is 'all' 35 | if [ "$scene" == "all" ]; then 36 | selected_scenes=${scenes[@]} # Copy all scenes 37 | else 38 | selected_scenes=($scene) # Assign the matching scene 39 | fi 40 | 41 | ################################################## 42 | ### Main 43 | ### Evaluate selected experiment 44 | ################################################## 45 | for scene in $selected_scenes 46 | do 47 | for i in $trials; do 48 | seed=${seeds[$i]} 49 | 50 | ### get result folder ### 51 | result_dir=${RESULT_DIR}/${DATASET}/$scene/${EXP}/run_${i} 52 | result_txt=${result_dir}/eval_result.txt 53 | echo "==> Evaluating [${result_dir}]" 54 | 55 | ### get file paths ### 56 | CKPT=$result_dir/coslam/checkpoint/ckpt_${iter_num}_final.pt 57 | INPUT_MESH=$result_dir/coslam/mesh/mesh_${iter_num}_final.ply 58 | REC_MESH=$result_dir/coslam/mesh/mesh_${iter_num}_final_cull_occlusion.ply 59 | DASHSCENE=${scene: 0: 0-1}_${scene: 0-1} 60 | GT_MESH=$PROJ_DIR/data/mp3d_data/v1/scans/${scene}/mesh.obj 61 | 62 | ### Cull mesh as Co-SLAM ### 63 | python third_party/neural_slam_eval/cull_mesh.py \ 64 | --config configs/${DATASET}/${scene}/coslam.yaml \ 65 | --input_mesh $INPUT_MESH \ 66 | --ckpt_path $CKPT \ 67 | --remove_occlusion # GO-Surf strategy 68 | 69 | ### Evaluate ### 70 | echo "==> Evaluating reconstruction result [accuracy, completeness, and completion ratio]" 71 | python src/evaluation/eval_recon.py \ 72 | --rec_mesh $REC_MESH \ 73 | --gt_mesh $GT_MESH \ 74 | --result_txt $result_txt 75 | 76 | echo "==> Evaluating reconstruction result [MAD]" 77 | python src/evaluation/eval_mad.py \ 78 | --cfg configs/${DATASET}/${scene}/NARUTO.py \ 79 | --ckpt $CKPT --gt_mesh $GT_MESH \ 80 | --result_txt $result_txt 81 | 82 | echo "==> Evaluating trajectory length (m)" 83 | python src/evaluation/eval_traj_length.py \ 84 | --ckpt ${CKPT} \ 85 | --result_txt $result_txt 86 | done 87 | done 88 | -------------------------------------------------------------------------------- /src/evaluation/eval_splatam_result.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import os 27 | import sys 28 | sys.path.append(os.getcwd()) 29 | from tensorboardX import SummaryWriter 30 | 31 | from src.main.cfg_loader import argument_parsing, load_cfg 32 | from src.slam import init_SLAM_model 33 | from src.utils.timer import Timer 34 | from src.utils.general_utils import fix_random_seed, InfoPrinter 35 | 36 | 37 | if __name__ == "__main__": 38 | info_printer = InfoPrinter("ActiveLang") 39 | timer = Timer() 40 | 41 | ################################################## 42 | ### argument parsing and load configuration 43 | ################################################## 44 | info_printer("Parsing arguments...", 0, "Initialization") 45 | args = argument_parsing() 46 | info_printer("Loading configuration...", 0, "Initialization") 47 | main_cfg = load_cfg(args) 48 | main_cfg.dump(os.path.join(main_cfg.dirs.result_dir, 'main_cfg.json')) 49 | info_printer.update_total_step(main_cfg.general.num_iter) 50 | info_printer.update_scene(main_cfg.general.dataset + " - " + main_cfg.general.scene) 51 | 52 | ################################################## 53 | ### Fix random seed 54 | ################################################## 55 | info_printer("Fix random seed...", 0, "Initialization") 56 | fix_random_seed(main_cfg.general.seed) 57 | 58 | ################################################## 59 | ### initialize logger 60 | ################################################## 61 | log_savedir = os.path.join(main_cfg.dirs.result_dir, "logger") 62 | os.makedirs(log_savedir, exist_ok=True) 63 | logger = SummaryWriter(f'{log_savedir}') 64 | 65 | ################################################## 66 | ### initialize SLAM module 67 | ################################################## 68 | slam = init_SLAM_model(main_cfg, info_printer, logger) 69 | slam.load_params(stage=args.stage) 70 | 71 | ################################################## 72 | ### Save Final Mesh and Checkpoint 73 | ################################################## 74 | slam.eval_result(eval_dir_suffix=args.stage, ignore_first_frame=True, save_frames=True) 75 | -------------------------------------------------------------------------------- /src/layers/backprojection.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import numpy as np 27 | import torch 28 | import torch.nn as nn 29 | 30 | 31 | class Backprojection(nn.Module): 32 | """Layer to backproject a depth image given the camera intrinsics 33 | 34 | Attributes 35 | xy (torch.tensor, [N,3,HxW]: homogeneous pixel coordinates on regular grid 36 | """ 37 | def __init__(self, height, width): 38 | """ 39 | Args: 40 | height (int): image height 41 | width (int): image width 42 | """ 43 | super(Backprojection, self).__init__() 44 | 45 | self.height = height 46 | self.width = width 47 | 48 | # generate regular grid 49 | meshgrid = np.meshgrid(range(self.width), range(self.height), indexing='xy') 50 | id_coords = np.stack(meshgrid, axis=0).astype(np.float32) 51 | id_coords = torch.tensor(id_coords) 52 | 53 | # generate homogeneous pixel coordinates 54 | self.ones = nn.Parameter(torch.ones(1, 1, self.height * self.width), 55 | requires_grad=False) 56 | self.xy = torch.unsqueeze( 57 | torch.stack([id_coords[0].view(-1), id_coords[1].view(-1)], 0) 58 | , 0) 59 | self.xy = torch.cat([self.xy, self.ones], 1) 60 | self.xy = nn.Parameter(self.xy, requires_grad=False) 61 | 62 | def forward(self, depth, inv_K, img_like_out=False): 63 | """ 64 | Args: 65 | depth (torch.tensor, [N,1,H,W]: depth map 66 | inv_K (torch.tensor, [N,4,4]): inverse camera intrinsics 67 | img_like_out (bool): if True, the output shape is [N,4,H,W]; else [N,4,(HxW)] 68 | Returns: 69 | points (torch.tensor, [N,4,(HxW)]): 3D points in homogeneous coordinates 70 | """ 71 | depth = depth.contiguous() 72 | 73 | xy = self.xy.repeat(depth.shape[0], 1, 1) 74 | ones = self.ones.repeat(depth.shape[0],1,1) 75 | 76 | points = torch.matmul(inv_K[:, :3, :3], xy) 77 | points = depth.view(depth.shape[0], 1, -1) * points 78 | points = torch.cat([points, ones], 1) 79 | 80 | if img_like_out: 81 | points = points.reshape(depth.shape[0], 4, self.height, self.width) 82 | return points 83 | -------------------------------------------------------------------------------- /src/main/cfg_loader.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2024 OPPO 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | 26 | import argparse 27 | import mmengine 28 | 29 | def override_cfg( 30 | args: argparse.Namespace, 31 | cfg : mmengine.Config 32 | ) -> mmengine.Config: 33 | """override configuration 34 | 35 | Args: 36 | args: arguments 37 | cfg : configuration 38 | 39 | Returns: 40 | cfg : updated configuration 41 | """ 42 | if hasattr(args, "seed") and args.seed is not None: 43 | ### random seed ### 44 | cfg.general.seed = args.seed 45 | 46 | if hasattr(args, "result_dir") and args.result_dir is not None: 47 | ### output/result directory ### 48 | cfg.dirs.result_dir = args.result_dir 49 | 50 | if hasattr(args, "enable_vis") and args.enable_vis is not None: 51 | ### output/result directory ### 52 | enable_vis = args.enable_vis == 1 53 | cfg.visualizer.vis_rgbd = enable_vis 54 | return cfg 55 | 56 | 57 | def argument_parsing() -> argparse.Namespace: 58 | """parse arguments 59 | 60 | Returns: 61 | args: arguments 62 | 63 | """ 64 | parser = argparse.ArgumentParser( 65 | description="Arguments to run MAIN." 66 | ) 67 | parser.add_argument("--cfg", type=str, default="configs/default.py", 68 | help="MAIN config") 69 | parser.add_argument("--result_dir", type=str, default=None, 70 | help="result directory") 71 | parser.add_argument("--seed", type=int, default=None, 72 | help="random seed; also used as the initial pose idx for Replica") 73 | parser.add_argument("--enable_vis", type=int, default=None, 74 | help="enable visualization. 1: True, 0: False") 75 | 76 | ### Evaluation only ### 77 | parser.add_argument("--stage", type=str, default=None, 78 | help="Running Stage for ActiveGAMER: [final, exploration_stage_0, exploration_stage_1].") 79 | args = parser.parse_args() 80 | return args 81 | 82 | 83 | def load_cfg(args: argparse.Namespace) -> mmengine.Config: 84 | """argument parsing and load configuration 85 | 86 | Args: 87 | args: arguments 88 | 89 | Returns: 90 | cfg : configuration 91 | 92 | """ 93 | cfg = mmengine.Config.fromfile(args.cfg) 94 | cfg = override_cfg(args, cfg) 95 | return cfg 96 | -------------------------------------------------------------------------------- /configs/Replica/office0/update_SLAM_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### General 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office0", 12 | num_iter = 2000, 13 | device = 'cuda' 14 | ) 15 | 16 | ################################################## 17 | ### Directories 18 | ################################################## 19 | dirs = dict( 20 | data_dir = "data/", 21 | result_dir = "results/", 22 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 23 | ) 24 | 25 | 26 | ################################################## 27 | ### Simulator 28 | ################################################## 29 | sim = dict( 30 | method = "habitat_v2" # simulator method 31 | ) 32 | 33 | if sim["method"] == "habitat_v2": 34 | sim.update( 35 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 36 | ) 37 | 38 | ################################################## 39 | ### SLAM 40 | ################################################## 41 | slam = dict( 42 | method="splatam" # SLAM backbone method 43 | ) 44 | 45 | if slam["method"] == "splatam": 46 | slam.update( 47 | room_cfg = f"{dirs['cfg_dir']}/../replica_splatam_s.py", # SplaTAM room configuration 48 | enable_active_planning = False, # enable/disable active planning 49 | 50 | ### bounding box ### 51 | bbox_bound = [[-2.1,2.5],[-3.2,2],[-1.3,2.0]], 52 | bbox_voxel_size = 0.05, 53 | 54 | surface_dist_thre = 0.5, 55 | 56 | ### Refinement step ### 57 | refine_map_iter = 5, 58 | 59 | ### override ### 60 | override = dict( 61 | map_every = 5, 62 | report_global_progress_every = 20, 63 | tracking = dict( 64 | use_gt_poses=True, # Use GT Poses for Tracking 65 | ) 66 | ) 67 | ) 68 | 69 | ################################################## 70 | ### Planner 71 | ################################################## 72 | planner = dict( 73 | # method= "active_lang", # planner method [predefined_traj, active_lang] 74 | method = "predefined_traj", 75 | 76 | ### active_lang params ### 77 | gs_z_levels = [20, 30, 40, 50], #[20,30,40], 78 | num_dir_samples = 10, # viewing direction sample number 79 | xy_sample_step = 0.5, # Unit: meter 80 | 81 | surface_dist_thre = slam['surface_dist_thre'], 82 | 83 | ### Stop Criteria ### 84 | explore_thre = 0.01, 85 | color_ig_thre = 34, 86 | depth_ig_thre = 0.01, 87 | 88 | 89 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 90 | use_traj_pose = True, # use pre-defined trajectory pose 91 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 92 | dirs["data_dir"], 93 | "Replica", general['scene'] 94 | ), 95 | ) 96 | 97 | ################################################## 98 | ### Visualization 99 | ################################################## 100 | visualizer = dict( 101 | method = "active_gs", 102 | vis_rgbd = True, # visualize RGB-D 103 | vis_rgbd_max_depth = 10 104 | ) 105 | 106 | -------------------------------------------------------------------------------- /src/evaluation/get_splatam_summary.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import numpy as np 4 | 5 | def calculate_average(filepath): 6 | """Calculate the average of values in a text file.""" 7 | try: 8 | with open(filepath, 'r') as f: 9 | values = [float(line.strip()) for line in f if line.strip()] 10 | if values: 11 | return np.mean(values) 12 | except Exception as e: 13 | print(f"Error reading file {filepath}: {e}") 14 | return "/" 15 | 16 | def read_eval_3d_results(filepath): 17 | """Read and return values from eval_3d_result.txt file.""" 18 | results = {"acc": "/", "comp": "/", "comp%": "/"} 19 | try: 20 | with open(filepath, 'r') as f: 21 | for line in f: 22 | key, value = line.strip().split(',') 23 | results[key] = float(value) 24 | except Exception as e: 25 | print(f"Error reading eval_3d file {filepath}: {e}") 26 | return results["acc"], results["comp"], results["comp%"] 27 | 28 | def summarize_experiments(base_dir, experiment_list, output_csv='experiment_summary_activeslam_mp3d.csv'): 29 | # scenes = ["room0", "room1", "room2", "office0", "office1", "office2", "office3", "office4"] 30 | scenes = ["GdvgFV5R1Z5", "gZ6f7yhEvPG", "HxpKQynjfin", "pLe4wQe7qrG", "YmJkqBEsHnH"] 31 | stages = ["eval_exploration_stage_0", "eval_exploration_stage_1", "eval_final"] 32 | metrics_files = ["psnr.txt", "ssim.txt", "lpips.txt", "l1.txt", "rmse.txt"] 33 | 34 | # Prepare CSV file 35 | with open(output_csv, mode='w', newline='') as csvfile: 36 | writer = csv.writer(csvfile) 37 | # Write header 38 | writer.writerow(["scene", "exp", "stage", "psnr", "ssim", "lpips", "l1", "rmse", "acc", "comp", "comp%"]) 39 | 40 | for scene in scenes: 41 | for exp in experiment_list: 42 | for stage in stages: 43 | # Define the path to the stage folder 44 | splatam_stage_path = os.path.join(base_dir, scene, exp, "run_0", "splatam", stage) 45 | stage_abbr = stage.replace("eval_", "") 46 | eval_3d_stage_path = os.path.join(base_dir, scene, exp, "run_0", "eval_3d", stage_abbr) 47 | 48 | # Initialize row data with defaults 49 | row = [scene, exp, stage_abbr] 50 | metric_averages = [] 51 | 52 | # Check each metric file in splatam 53 | for metric in metrics_files: 54 | metric_path = os.path.join(splatam_stage_path, metric) 55 | if os.path.exists(metric_path): 56 | average_value = calculate_average(metric_path) 57 | else: 58 | average_value = "/" 59 | metric_averages.append(average_value) 60 | 61 | # Check for eval_3d results 62 | eval_3d_results_path = os.path.join(eval_3d_stage_path, "eval_3d_result.txt") 63 | if os.path.exists(eval_3d_results_path): 64 | acc, comp, comp_percent = read_eval_3d_results(eval_3d_results_path) 65 | else: 66 | acc, comp, comp_percent = "/", "/", "/" 67 | 68 | # Add row to CSV 69 | writer.writerow(row + metric_averages + [acc, comp, comp_percent]) 70 | 71 | print(f"Experiment summary saved to {output_csv}") 72 | 73 | # Usage example: 74 | base_directory = "results/MP3D" # Adjust base directory if needed 75 | # experiment_list = ["ActiveGS", "ActiveGSFull", "ActiveLang", "ActiveGS_noGlobal"] # Add or modify experiments as required 76 | experiment_list = ["ActiveGS"] # Add or modify experiments as required 77 | summarize_experiments(base_directory, experiment_list) 78 | -------------------------------------------------------------------------------- /configs/Replica/room0/update_SLAM_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### General 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room0", 12 | num_iter = 2000, 13 | device = 'cuda' 14 | ) 15 | 16 | ################################################## 17 | ### Directories 18 | ################################################## 19 | dirs = dict( 20 | data_dir = "data/", 21 | result_dir = "results/", 22 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 23 | ) 24 | 25 | 26 | ################################################## 27 | ### Simulator 28 | ################################################## 29 | sim = dict( 30 | method = "habitat_v2" # simulator method 31 | ) 32 | 33 | if sim["method"] == "habitat_v2": 34 | sim.update( 35 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 36 | ) 37 | 38 | ################################################## 39 | ### SLAM 40 | ################################################## 41 | slam = dict( 42 | method="splatam" # SLAM backbone method 43 | ) 44 | 45 | if slam["method"] == "splatam": 46 | slam.update( 47 | room_cfg = f"{dirs['cfg_dir']}/../replica_splatam_s.py", # SplaTAM room configuration 48 | # room_cfg = f"{dirs['cfg_dir']}/../replica_splatam.py", # SplaTAM room configuration 49 | enable_active_planning = False, # enable/disable active planning 50 | 51 | ### bounding box ### 52 | # bbox_bound = [[-2.2,2.6],[-3.4,2.1],[-1.4,2.0]], 53 | bbox_bound = [[-2.1,2.5],[-3.2,2],[-1.3,2.0]], 54 | bbox_voxel_size = 0.05, 55 | 56 | surface_dist_thre = 0.5, 57 | 58 | ### Refinement step ### 59 | refine_map_iter = 5, 60 | 61 | ### override ### 62 | override = dict( 63 | map_every = 5, 64 | report_global_progress_every = 20, 65 | tracking = dict( 66 | use_gt_poses=True, # Use GT Poses for Tracking 67 | ) 68 | ) 69 | ) 70 | 71 | ################################################## 72 | ### Planner 73 | ################################################## 74 | planner = dict( 75 | # method= "active_lang", # planner method [predefined_traj, active_lang] 76 | method = "predefined_traj", 77 | 78 | ### active_lang params ### 79 | gs_z_levels = [20, 30, 40, 50], #[20,30,40], 80 | num_dir_samples = 10, # viewing direction sample number 81 | xy_sample_step = 0.5, # Unit: meter 82 | 83 | surface_dist_thre = slam['surface_dist_thre'], 84 | 85 | ### Stop Criteria ### 86 | explore_thre = 0.01, 87 | color_ig_thre = 34, 88 | depth_ig_thre = 0.01, 89 | 90 | 91 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 92 | use_traj_pose = True, # use pre-defined trajectory pose 93 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 94 | dirs["data_dir"], 95 | "Replica", general['scene'] 96 | ), 97 | ) 98 | 99 | ################################################## 100 | ### Visualization 101 | ################################################## 102 | visualizer = dict( 103 | method = "active_gs", 104 | vis_rgbd = True, # visualize RGB-D 105 | vis_rgbd_max_depth = 10 106 | 107 | ### mesh related ### 108 | # mesh_vis_freq = 500, # mesh save frequency 109 | ) 110 | 111 | -------------------------------------------------------------------------------- /configs/Replica/room1/update_SLAM_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### General 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room1", 12 | num_iter = 2000, 13 | device = 'cuda' 14 | ) 15 | 16 | ################################################## 17 | ### Directories 18 | ################################################## 19 | dirs = dict( 20 | data_dir = "data/", 21 | result_dir = "results/", 22 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 23 | ) 24 | 25 | 26 | ################################################## 27 | ### Simulator 28 | ################################################## 29 | sim = dict( 30 | method = "habitat_v2" # simulator method 31 | ) 32 | 33 | if sim["method"] == "habitat_v2": 34 | sim.update( 35 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 36 | ) 37 | 38 | ################################################## 39 | ### SLAM 40 | ################################################## 41 | slam = dict( 42 | method="splatam" # SLAM backbone method 43 | ) 44 | 45 | if slam["method"] == "splatam": 46 | slam.update( 47 | room_cfg = f"{dirs['cfg_dir']}/../replica_splatam_s.py", # SplaTAM room configuration 48 | # room_cfg = f"{dirs['cfg_dir']}/../replica_splatam.py", # SplaTAM room configuration 49 | enable_active_planning = False, # enable/disable active planning 50 | 51 | ### bounding box ### 52 | # bbox_bound = [[-2.2,2.6],[-3.4,2.1],[-1.4,2.0]], 53 | bbox_bound = [[-2.1,2.5],[-3.2,2],[-1.3,2.0]], 54 | bbox_voxel_size = 0.05, 55 | 56 | surface_dist_thre = 0.5, 57 | 58 | ### Refinement step ### 59 | refine_map_iter = 5, 60 | 61 | ### override ### 62 | override = dict( 63 | map_every = 5, 64 | report_global_progress_every = 20, 65 | tracking = dict( 66 | use_gt_poses=True, # Use GT Poses for Tracking 67 | ) 68 | ) 69 | ) 70 | 71 | ################################################## 72 | ### Planner 73 | ################################################## 74 | planner = dict( 75 | # method= "active_lang", # planner method [predefined_traj, active_lang] 76 | method = "predefined_traj", 77 | 78 | ### active_lang params ### 79 | gs_z_levels = [20, 30, 40, 50], #[20,30,40], 80 | num_dir_samples = 10, # viewing direction sample number 81 | xy_sample_step = 0.5, # Unit: meter 82 | 83 | surface_dist_thre = slam['surface_dist_thre'], 84 | 85 | ### Stop Criteria ### 86 | explore_thre = 0.01, 87 | color_ig_thre = 34, 88 | depth_ig_thre = 0.01, 89 | 90 | 91 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 92 | use_traj_pose = True, # use pre-defined trajectory pose 93 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 94 | dirs["data_dir"], 95 | "Replica", general['scene'] 96 | ), 97 | ) 98 | 99 | ################################################## 100 | ### Visualization 101 | ################################################## 102 | visualizer = dict( 103 | method = "active_gs", 104 | vis_rgbd = True, # visualize RGB-D 105 | vis_rgbd_max_depth = 10 106 | 107 | ### mesh related ### 108 | # mesh_vis_freq = 500, # mesh save frequency 109 | ) 110 | 111 | -------------------------------------------------------------------------------- /configs/Replica/room2/update_SLAM_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### General 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "room2", 12 | num_iter = 2000, 13 | device = 'cuda' 14 | ) 15 | 16 | ################################################## 17 | ### Directories 18 | ################################################## 19 | dirs = dict( 20 | data_dir = "data/", 21 | result_dir = "results/", 22 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 23 | ) 24 | 25 | 26 | ################################################## 27 | ### Simulator 28 | ################################################## 29 | sim = dict( 30 | method = "habitat_v2" # simulator method 31 | ) 32 | 33 | if sim["method"] == "habitat_v2": 34 | sim.update( 35 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 36 | ) 37 | 38 | ################################################## 39 | ### SLAM 40 | ################################################## 41 | slam = dict( 42 | method="splatam" # SLAM backbone method 43 | ) 44 | 45 | if slam["method"] == "splatam": 46 | slam.update( 47 | room_cfg = f"{dirs['cfg_dir']}/../replica_splatam_s.py", # SplaTAM room configuration 48 | # room_cfg = f"{dirs['cfg_dir']}/../replica_splatam.py", # SplaTAM room configuration 49 | enable_active_planning = False, # enable/disable active planning 50 | 51 | ### bounding box ### 52 | # bbox_bound = [[-2.2,2.6],[-3.4,2.1],[-1.4,2.0]], 53 | bbox_bound = [[-2.1,2.5],[-3.2,2],[-1.3,2.0]], 54 | bbox_voxel_size = 0.05, 55 | 56 | surface_dist_thre = 0.5, 57 | 58 | ### Refinement step ### 59 | refine_map_iter = 5, 60 | 61 | ### override ### 62 | override = dict( 63 | map_every = 5, 64 | report_global_progress_every = 20, 65 | tracking = dict( 66 | use_gt_poses=True, # Use GT Poses for Tracking 67 | ) 68 | ) 69 | ) 70 | 71 | ################################################## 72 | ### Planner 73 | ################################################## 74 | planner = dict( 75 | # method= "active_lang", # planner method [predefined_traj, active_lang] 76 | method = "predefined_traj", 77 | 78 | ### active_lang params ### 79 | gs_z_levels = [20, 30, 40, 50], #[20,30,40], 80 | num_dir_samples = 10, # viewing direction sample number 81 | xy_sample_step = 0.5, # Unit: meter 82 | 83 | surface_dist_thre = slam['surface_dist_thre'], 84 | 85 | ### Stop Criteria ### 86 | explore_thre = 0.01, 87 | color_ig_thre = 34, 88 | depth_ig_thre = 0.01, 89 | 90 | 91 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 92 | use_traj_pose = True, # use pre-defined trajectory pose 93 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 94 | dirs["data_dir"], 95 | "Replica", general['scene'] 96 | ), 97 | ) 98 | 99 | ################################################## 100 | ### Visualization 101 | ################################################## 102 | visualizer = dict( 103 | method = "active_gs", 104 | vis_rgbd = True, # visualize RGB-D 105 | vis_rgbd_max_depth = 10 106 | 107 | ### mesh related ### 108 | # mesh_vis_freq = 500, # mesh save frequency 109 | ) 110 | 111 | -------------------------------------------------------------------------------- /configs/Replica/office1/update_SLAM_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | _base_ = "../../default.py" 5 | 6 | ################################################## 7 | ### General 8 | ################################################## 9 | general = dict( 10 | dataset = "Replica", 11 | scene = "office1", 12 | num_iter = 2000, 13 | device = 'cuda' 14 | ) 15 | 16 | ################################################## 17 | ### Directories 18 | ################################################## 19 | dirs = dict( 20 | data_dir = "data/", 21 | result_dir = "results/", 22 | cfg_dir = os.path.join("configs", general['dataset'], general['scene']) 23 | ) 24 | 25 | 26 | ################################################## 27 | ### Simulator 28 | ################################################## 29 | sim = dict( 30 | method = "habitat_v2" # simulator method 31 | ) 32 | 33 | if sim["method"] == "habitat_v2": 34 | sim.update( 35 | habitat_cfg = os.path.join(dirs['cfg_dir'], "habitat.py") 36 | ) 37 | 38 | ################################################## 39 | ### SLAM 40 | ################################################## 41 | slam = dict( 42 | method="splatam" # SLAM backbone method 43 | ) 44 | 45 | if slam["method"] == "splatam": 46 | slam.update( 47 | room_cfg = f"{dirs['cfg_dir']}/../replica_splatam_s.py", # SplaTAM room configuration 48 | # room_cfg = f"{dirs['cfg_dir']}/../replica_splatam.py", # SplaTAM room configuration 49 | enable_active_planning = False, # enable/disable active planning 50 | 51 | ### bounding box ### 52 | # bbox_bound = [[-2.2,2.6],[-3.4,2.1],[-1.4,2.0]], 53 | bbox_bound = [[-2.1,2.5],[-3.2,2],[-1.3,2.0]], 54 | bbox_voxel_size = 0.05, 55 | 56 | surface_dist_thre = 0.5, 57 | 58 | ### Refinement step ### 59 | refine_map_iter = 5, 60 | 61 | ### override ### 62 | override = dict( 63 | map_every = 5, 64 | report_global_progress_every = 20, 65 | tracking = dict( 66 | use_gt_poses=True, # Use GT Poses for Tracking 67 | ) 68 | ) 69 | ) 70 | 71 | ################################################## 72 | ### Planner 73 | ################################################## 74 | planner = dict( 75 | # method= "active_lang", # planner method [predefined_traj, active_lang] 76 | method = "predefined_traj", 77 | 78 | ### active_lang params ### 79 | gs_z_levels = [20, 30, 40, 50], #[20,30,40], 80 | num_dir_samples = 10, # viewing direction sample number 81 | xy_sample_step = 0.5, # Unit: meter 82 | 83 | surface_dist_thre = slam['surface_dist_thre'], 84 | 85 | ### Stop Criteria ### 86 | explore_thre = 0.01, 87 | color_ig_thre = 34, 88 | depth_ig_thre = 0.01, 89 | 90 | 91 | up_dir = np.array([0, 0, 1]), # up direction for planning pose 92 | use_traj_pose = True, # use pre-defined trajectory pose 93 | SLAMData_dir = os.path.join( # SLAM Data directory (for passive mapping or pre-defined trajectory pose) 94 | dirs["data_dir"], 95 | "Replica", general['scene'] 96 | ), 97 | ) 98 | 99 | ################################################## 100 | ### Visualization 101 | ################################################## 102 | visualizer = dict( 103 | method = "active_gs", 104 | vis_rgbd = True, # visualize RGB-D 105 | vis_rgbd_max_depth = 10 106 | 107 | ### mesh related ### 108 | # mesh_vis_freq = 500, # mesh save frequency 109 | ) 110 | 111 | --------------------------------------------------------------------------------