├── mvl_challenge ├── config │ ├── __init__.py │ └── cfg.py ├── models │ ├── __init__.py │ └── models_utils.py ├── utils │ ├── __init__.py │ ├── vispy_utils │ │ ├── __init__.py │ │ └── vispy_utils.py │ ├── download_utils.py │ ├── check_utils.py │ ├── eval_utils.py │ ├── image_utils.py │ ├── spherical_utils.py │ ├── layout_utils.py │ ├── geometry_utils.py │ └── io_utils.py ├── data_loaders │ ├── __init__.py │ └── mvl_dataloader.py ├── datasets │ ├── __init__.py │ ├── mvl_dataset.py │ └── rgbd_datasets.py ├── data_structure │ ├── __init__.py │ ├── cam_pose.py │ ├── frame.py │ └── layout.py ├── data │ ├── configs │ │ ├── camera_height.yaml │ │ ├── load_mvl_dataset.yaml │ │ ├── trained_models.yaml │ │ └── eval_mvl_dataset.yaml │ ├── gdrive_files │ │ ├── gdrive_ids__example_scene.csv │ │ ├── gdrive_ids__warm_up_training_set_folders.csv │ │ ├── gdrive_ids__ckpt_hn_models.csv │ │ ├── gdrive_ids__pilot_labels.csv │ │ ├── gdrive_ids__pilot_set.csv │ │ └── gdrive_ids__warm_up_testing_set.csv │ └── scene_list │ │ └── scene_list__hm3d_mvl__warm_up_pilot_set.json ├── remote_data │ ├── download_gdrive_ids_file.sh │ ├── unzip_data.sh │ ├── download_gdrive_ids.sh │ ├── upload_data.sh │ ├── zip_mvl_dataset.py │ ├── download_mvl_data.py │ └── zip_rgbd_dataset.py ├── __init__.py ├── main_plot_rgbd_data.py ├── pre_processing │ ├── pre_process_rgbd_data.sh │ ├── utils │ │ ├── camera_height_per_rooms.py │ │ ├── camera_height_utils.py │ │ └── geometry_info_utils.py │ ├── fix_scene_list.py │ ├── create_rgb_files.py │ ├── create_geometry_info_files.py │ └── create_npz_labels.py ├── scene_list__get_info.py ├── mvl_data │ ├── load_mvl_dataset.py │ ├── load_labels.py │ └── load_and_eval_mvl_dataset.py ├── challenge_results │ ├── create_zip_results.py │ ├── create_npz_files.py │ └── evaluate_results.py └── scene_list__edit_info.py ├── .gitmodules ├── .gitignore ├── requirements.txt ├── CHANGELOG.md ├── setup.py ├── tutorial ├── train_horizon_net │ ├── train_hn.yaml │ ├── download_pretrained_ckpt.py │ ├── README.md │ └── train_hn.py └── train_360_mlc │ ├── create_mlc_labels.yaml │ ├── train_mlc.yaml │ ├── train_mlc.py │ ├── README.md │ └── create_mlc_labels.py ├── check_scene_list.py ├── test_mvl_toolkit.py └── download_mvl_data.py /mvl_challenge/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mvl_challenge/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mvl_challenge/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mvl_challenge/data_loaders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mvl_challenge/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mvl_challenge/data_structure/__init__.py: -------------------------------------------------------------------------------- 1 | from .layout import Layout 2 | from .cam_pose import CamPose 3 | -------------------------------------------------------------------------------- /mvl_challenge/utils/vispy_utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .vispy_utils import plot_list_ly 2 | from .vispy_utils import plot_color_plc 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "mvl_challenge/models/HorizonNet"] 2 | path = mvl_challenge/models/HorizonNet 3 | url = https://github.com/sunset1995/HorizonNet.git 4 | -------------------------------------------------------------------------------- /mvl_challenge/data/configs/camera_height.yaml: -------------------------------------------------------------------------------- 1 | #! Default parameter to estimate camera height using PCL 2 | cam_height_cfg: 3 | fit_error: 0.001 4 | xz_radius: 1 5 | min_height: 1 6 | min_samples: 200 7 | iter: 5 -------------------------------------------------------------------------------- /mvl_challenge/data/gdrive_files/gdrive_ids__example_scene.csv: -------------------------------------------------------------------------------- 1 | 1G7sgrqBsCSFbxjeuAzGKdTO7b_jcYDWI example_scene_list.json bin 1.3 KB 2023-03-17 21:39:54 2 | 139U0PHEnVPVyaSAaGME9qHYtCQsrNhiv 2t7WUuJeko7_0_room0.zip bin 3.6 MB 2023-03-17 21:39:51 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | */__pycache__/ 3 | *__pycache__/ 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | *.pyc 8 | 9 | # VS-Code 10 | *.code-workspace 11 | *.notebook 12 | *.vscode/ 13 | *logs/ 14 | *ckpt/ 15 | *.egg-info 16 | assets/ 17 | *tmp* 18 | result/ 19 | -------------------------------------------------------------------------------- /mvl_challenge/data/gdrive_files/gdrive_ids__warm_up_training_set_folders.csv: -------------------------------------------------------------------------------- 1 | 1t8-ZZp7YhicuTvaYlySiK7Xu_FAi_xMX warm_up_training_3 dir 2023-03-20 14:49:26 2 | 1E1UXU8zRDnoLEQdn-fkCUDyMOGeLkxkj warm_up_training_2 dir 2023-03-20 14:49:12 3 | 1Z6k0cOUHHv95n8TUb_qWgwSNibiqpe8w warm_up_training_1 dir 2023-03-20 14:48:56 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gdown 2 | pandas 3 | dill 4 | plyfile 5 | pyquaternion 6 | GitPython 7 | PyYAML 8 | omegaconf 9 | imageio 10 | scipy 11 | vispy 12 | config 13 | matplotlib 14 | torch 15 | tensorboard 16 | shapely 17 | torchvision 18 | PyQt5 19 | PyOpenGL-accelerate 20 | pyransac3d 21 | python-dateutil==2.8.1 22 | googledrivedownloader 23 | scikit-image -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## April 1st, 2023 by @EnriqueSolarte 4 | * Fix some typos 5 | * Add Tutorial 1 which allows the current implementation to training HorizonNet in a supervise manner. 6 | For this tutorial, `pilot_scene_list` (the only one with GT labels) is used for training. Additionally, 7 | a script to download pre-trained model is also provided. -------------------------------------------------------------------------------- /mvl_challenge/data/configs/load_mvl_dataset.yaml: -------------------------------------------------------------------------------- 1 | scene_dir: None 2 | scene_list: None 3 | 4 | runners: 5 | mvl: 6 | data_dir: ${scene_dir} 7 | scene_list: ${scene_list} 8 | batch_size: 10 9 | size: -1 10 | grid_size: 0.01 11 | min_likelihood_percent: 0.000001 12 | padding: 20 13 | max_room_factor_size: 2 14 | num_workers: 5 15 | -------------------------------------------------------------------------------- /mvl_challenge/data/gdrive_files/gdrive_ids__ckpt_hn_models.csv: -------------------------------------------------------------------------------- 1 | 1W2A-_WU9d5KAwEQiTywJud2mRO3hLXqL hn_mp3d.pth bin 326.6 MB 2023-03-17 23:11:57 2 | 1jqpgvRLNZA98XphNI1lkQXGfi2vr1wK1 hn_panos2d3d.pth bin 326.6 MB 2023-03-19 21:56:40 3 | 1GCnHYf7Lp0v5f_9lMWxTWMgQcLtpusfx hn_st3d.pth bin 326.6 MB 2023-03-19 21:57:16 4 | 1pjwqlQP5_E5bHDONf0gPWXsF1ETCfbS- hn_zind.pth bin 134.6 MB 2023-03-19 21:57:34 5 | -------------------------------------------------------------------------------- /mvl_challenge/models/models_utils.py: -------------------------------------------------------------------------------- 1 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 2 | 3 | 4 | def load_layout_model(cfg): 5 | """ 6 | Load a layout model estimator and returns an instance of it 7 | """ 8 | if cfg.model.ly_model == "HorizonNet": 9 | # ! loading HorizonNet 10 | model = WrapperHorizonNet(cfg) 11 | else: 12 | raise NotImplementedError("") 13 | 14 | return model 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | with open("./requirements.txt", "r") as f: 4 | requirements = [l.strip() for l in f.readlines() if len(l.strip()) > 0] 5 | 6 | setup( 7 | name="mvl_challenge_toolkit_dev", 8 | version="0.1", 9 | packages=find_packages(), 10 | install_requires=requirements, 11 | author="Enrique Solarte", 12 | author_email="enrique.solarte.pardo@gmail.com", 13 | description=("Toolkit to process mvl datasets"), 14 | license="BSD", 15 | ) 16 | -------------------------------------------------------------------------------- /mvl_challenge/data/configs/trained_models.yaml: -------------------------------------------------------------------------------- 1 | # ! Pretrained HN models 2 | mp3d: 3 | ckpt: "/media/NFS/justin/code/360-retraining/models/HorizonNet/ckpt/resnet50_rnn__mp3d.pth" 4 | 5 | zind: 6 | ckpt: "/media/NFS/justin/code/360-retraining/models/HorizonNet/ckpt/resnet50_rnn__zind.pth" 7 | 8 | st3d: 9 | ckpt: "/media/NFS/justin/code/360-retraining/models/HorizonNet/ckpt/resnet50_rnn__st3d.pth" 10 | 11 | panos2d3d: 12 | ckpt: "/media/NFS/justin/code/360-retraining/models/HorizonNet/ckpt/resnet50_rnn__panos2d3d.pth" -------------------------------------------------------------------------------- /mvl_challenge/remote_data/download_gdrive_ids_file.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | URL='https://www.googleapis.com/drive/v3/files/' 4 | URL_END='?alt=media' 5 | FOLDER_PATH=$2 6 | ID=$1 7 | TOKEN=ya29.a0AVvZVsrrazc3-2UxSCZGZzMRp9r-At9nEVDO17t_GrjJt6bpx2TIkhDb8cEDErIZqPCO2TfJCvEnP-vXEyLjK77ZlbCuqT_ADWepSdBVH6MB5rpLVetfKRzIvJT7h1Wg5QKdmjDomgK8nf3jjox6O8FVkLtzaCgYKATYSARASFQGbdwaI0_9SwZXaCoshZEUyg_-F9w0163 8 | 9 | URL_SCENE=$URL$ID$URL_END 10 | OUTPUT=$FOLDER_PATH 11 | CODE=$(curl -H "Authorization: Bearer $TOKEN" $URL_SCENE -o $OUTPUT) 12 | -------------------------------------------------------------------------------- /mvl_challenge/remote_data/unzip_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { echo "$0 USAGE:" && grep " .)\ #" $0; exit 0;} 4 | [ $# -eq 0 ] && usage 5 | while getopts ":hd:o:" arg; do 6 | case $arg in 7 | d) # Zip directory 8 | ZIP_DIR=${OPTARG} 9 | ;; 10 | o) # Output directory 11 | OUTPUT_DIR=${OPTARG} 12 | ;; 13 | h | *) # Display help. 14 | usage 15 | exit 0 16 | ;; 17 | esac 18 | done 19 | 20 | if [ -z "$ZIP_DIR" ] 21 | then 22 | usage 23 | fi 24 | 25 | if [ -z "$OUTPUT_DIR" ] 26 | then 27 | usage 28 | fi 29 | 30 | for filename in "$ZIP_DIR"/*.zip; do 31 | echo "$filename" 32 | unzip -o "$filename" -d "$OUTPUT_DIR" 33 | done 34 | -------------------------------------------------------------------------------- /mvl_challenge/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | ROOT_DIR = os.path.dirname(__file__) 4 | CFG_DIR = os.path.join(ROOT_DIR, "data/configs") 5 | ASSETS_DIR = os.path.join(ROOT_DIR, "assets") 6 | DATA_DIR = os.path.join(ROOT_DIR, "data") 7 | GDRIVE_DIR = os.path.join(ROOT_DIR, "data/gdrive_files") 8 | SCENE_LIST_DIR = os.path.join(ROOT_DIR, "data/scene_list") 9 | DEFAULT_DOWNLOAD_DIR = os.path.join(ASSETS_DIR, "data") 10 | DEFAULT_MVL_DIR = os.path.join(ASSETS_DIR, "data/mvl_data") 11 | DEFAULT_NPZ_DIR = os.path.join(ASSETS_DIR, "npz") 12 | DEFAULT_TRAINING_DIR = os.path.join(ASSETS_DIR, "data/mvl_training_results") 13 | DEFAULT_CKPT_DIR = os.path.join(ASSETS_DIR, "ckpt") 14 | 15 | EPILOG = "\t * MVL-Challenge - CVPR 2023 - OmniCV workshop" 16 | 17 | -------------------------------------------------------------------------------- /mvl_challenge/remote_data/download_gdrive_ids.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | URL='https://www.googleapis.com/drive/v3/files/' 4 | URL_END='?alt=media' 5 | FOLDER_PATH=$2 6 | FILE_CSV=$1 7 | TOKEN=ya29.a0AVvZVsrrazc3-2UxSCZGZzMRp9r-At9nEVDO17t_GrjJt6bpx2TIkhDb8cEDErIZqPCO2TfJCvEnP-vXEyLjK77ZlbCuqT_ADWepSdBVH6MB5rpLVetfKRzIvJT7h1Wg5QKdmjDomgK8nf3jjox6O8FVkLtzaCgYKATYSARASFQGbdwaI0_9SwZXaCoshZEUyg_-F9w0163 8 | 9 | while read line; do 10 | # echo "$line"; 11 | firstItem="$(echo $line | cut -d' ' -f1)" 12 | secondItem="$(echo $line | cut -d' ' -f2)" 13 | echo $firstItem $secondItem 14 | URL_SCENE=$URL$firstItem$URL_END 15 | OUTPUT=$FOLDER_PATH$secondItem 16 | CODE=$(curl -H "Authorization: Bearer $TOKEN" $URL_SCENE -o $OUTPUT) 17 | echo $CODE 18 | done < $FILE_CSV -------------------------------------------------------------------------------- /mvl_challenge/data/configs/eval_mvl_dataset.yaml: -------------------------------------------------------------------------------- 1 | scene_dir: None 2 | scene_list: None 3 | ckpt: None 4 | cuda_device: 0 5 | 6 | auto_loading: 7 | trained_models: ${rel_path:./trained_models.yaml} 8 | 9 | runners: 10 | mvl: 11 | data_dir: ${scene_dir} 12 | scene_list: ${scene_list} 13 | batch_size: 10 14 | size: -1 15 | grid_size: 0.01 16 | min_likelihood_percent: 0.000001 17 | padding: 20 18 | max_room_factor_size: 2 19 | num_workers: 5 20 | 21 | model: 22 | ly_model: HorizonNet 23 | ckpt: ${ckpt} 24 | optimizer: Adam 25 | loss: weighted_L1 26 | min_std: 0.001 27 | epochs: 10 28 | save_every: -1 29 | lr: 0.00005 30 | lr_decay_rate: 0.90 31 | seed: 594277 32 | bn_momentum: 0 33 | freeze_earlier_blocks: -1 34 | beta1: 0.9 35 | weight_decay: 0.0 36 | no_save: True 37 | -------------------------------------------------------------------------------- /mvl_challenge/remote_data/upload_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | usage() { echo "$0 USAGE:" && grep " .)\ #" $0; exit 0;} 3 | [ $# -eq 0 ] && usage 4 | while getopts ":hd:i:c" arg; do 5 | case $arg in 6 | d) # Zip directory 7 | ZIP_DIR=${OPTARG} 8 | ;; 9 | i) # GoogleDrive ID remote directory 10 | ID_DIR=${OPTARG} 11 | ;; 12 | c) # Check GoogleDrive ID remote directory 13 | CHECK=true 14 | ;; 15 | h | *) # Display help. 16 | usage 17 | exit 0 18 | ;; 19 | esac 20 | done 21 | 22 | if [ -z "$ID_DIR" ] 23 | then 24 | usage 25 | fi 26 | 27 | 28 | if [ "$CHECK" = true ]; then 29 | gdrive list -m 20000 --no-header --query " '$ID_DIR' in parents" 30 | exit 0 31 | fi 32 | 33 | if [ -z "$ZIP_DIR" ] 34 | then 35 | usage 36 | fi 37 | 38 | 39 | for filename in "$ZIP_DIR"/*; do 40 | echo "$filename" 41 | gdrive upload -p "$ID_DIR" "$filename" 42 | done 43 | 44 | gdrive list -m 20000 --no-header --query " '$ID_DIR' in parents" > "$ZIP_DIR"/../ids_"$ID_DIR".csv -------------------------------------------------------------------------------- /tutorial/train_horizon_net/train_hn.yaml: -------------------------------------------------------------------------------- 1 | # ! Selecting MVL dataset 2 | mvl_dir: 3 | output_dir: 4 | 5 | pilot_scene_list: 6 | 7 | ckpt: 8 | cuda_device: 9 | id_exp: 10 | 11 | runners: 12 | train: 13 | data_dir: 14 | img_dir: ${mvl_dir}/img 15 | labels_dir: ${mvl_dir}/labels/ 16 | label: gt 17 | scene_list: ${pilot_scene_list} 18 | size: -1 19 | batch_size: 5 20 | num_workers: 4 21 | 22 | valid_iou: 23 | data_dir: 24 | img_dir: ${mvl_dir}/img 25 | labels_dir: ${mvl_dir}/labels/ 26 | scene_list: ${pilot_scene_list} 27 | batch_size: 4 28 | label: gt 29 | size: -1 30 | num_workers: 4 31 | 32 | model: 33 | ly_model: HorizonNet 34 | ckpt: ${ckpt} 35 | optimizer: Adam 36 | loss: weighted_L1 37 | min_std: 0.001 38 | epochs: 25 39 | save_every: -1 40 | lr: 0.00005 41 | lr_decay_rate: 0.90 42 | seed: 594277 43 | bn_momentum: 0 44 | freeze_earlier_blocks: -1 45 | beta1: 0.9 46 | weight_decay: 0.0 -------------------------------------------------------------------------------- /tutorial/train_360_mlc/create_mlc_labels.yaml: -------------------------------------------------------------------------------- 1 | pass_args: True # If False, the below must be specified by user. 2 | # Since they won't be set by passing the arguments anymore. 3 | 4 | # ! Selecting MVL dataset 5 | mvl_dir: 6 | output_dir: 7 | 8 | scene_list: 9 | ckpt: 10 | cuda: 11 | id_exp: 12 | 13 | mlc_dir: 14 | phi_coords: ${output_dir}/${id_exp}/mlc_label 15 | std: ${output_dir}/${id_exp}/std 16 | vis: ${output_dir}/${id_exp}/mlc_vis 17 | 18 | runners: 19 | mvl: 20 | data_dir: ${mvl_dir} 21 | scene_list: ${scene_list} 22 | batch_size: 10 23 | num_workers: 5 24 | max_room_factor_size: 2 25 | std_kernel: [30, 10, 5] 26 | 27 | model: 28 | ly_model: HorizonNet 29 | ckpt: ${ckpt} 30 | optimizer: Adam 31 | loss: weighted_L1 32 | min_std: 0.001 33 | epochs: 25 34 | save_every: 1 35 | lr: 0.00005 36 | lr_decay_rate: 0.90 37 | seed: 594277 38 | bn_momentum: 0 39 | freeze_earlier_blocks: -1 40 | beta1: 0.9 41 | weight_decay: 0.0 -------------------------------------------------------------------------------- /mvl_challenge/data/gdrive_files/gdrive_ids__pilot_labels.csv: -------------------------------------------------------------------------------- 1 | 1mJq8DLIU2qhNcirAW1fDJWO5NR7-xG7E Z6MFQCViBuw_0_room8.zip bin 3.6 MB 2023-03-19 00:42:50 2 | 1KVb926i7A3tPvkqkxlesGTWv33STOh_2 VFuaQ6m2Qom_0_room1.zip bin 6.5 MB 2023-03-19 00:42:48 3 | 136IZb6Cb2V4MpDw2qyqWcv4ErR1tjtPw 7y3sRwLe3Va_0_room7.zip bin 2.6 MB 2023-03-19 00:42:45 4 | 1msbyVL4VoDprHmAvDkVonw5Wi82xVZwa 7y3sRwLe3Va_0_room3.zip bin 4.5 MB 2023-03-19 00:42:43 5 | 1AZlVTbx6sipUskHWtpzmwV7fd52mezEp 7y3sRwLe3Va_0_room1.zip bin 6.8 MB 2023-03-19 00:42:41 6 | 1r4abrkkD6GB5Gkfq1jT-l1yHB-cbK3UF wcojb4TFT35_0_room0.zip bin 3.6 MB 2023-03-19 00:41:07 7 | 1SfWsDOklIIYpT-QKwhvi7sRsGgXMR8P0 q3hn1WQ12rz_6_room0.zip bin 3.9 MB 2023-03-19 00:41:02 8 | 1GRFeiwESa9cJprWv1cF4Uwc_wlu4eSxa k1cupFYWXJ6_0_room0.zip bin 4.3 MB 2023-03-19 00:41:00 9 | 1__fV7QhYRZvpohS2UVL89cK8kBdRZpBI h1zeeAwLh9Z_3_room0.zip bin 4.9 MB 2023-03-19 00:40:58 10 | 1-6DRzfdQ6ssvAYidk8uJ5QC2RsXhOjkr HaxA7YrQdEC_1_room0.zip bin 4.8 MB 2023-03-19 00:40:55 11 | -------------------------------------------------------------------------------- /tutorial/train_360_mlc/train_mlc.yaml: -------------------------------------------------------------------------------- 1 | pass_args: True # If False, the below must be specified by user. 2 | # Since they won't be set by passing the arguments anymore. 3 | 4 | # ! Selecting MVL dataset 5 | mvl_dir: 6 | output_dir: 7 | 8 | training_scene_list: 9 | testing_scene_list: 10 | pilot_scene_list: 11 | 12 | ckpt: 13 | id_exp: 14 | 15 | runners: 16 | train: 17 | data_dir: 18 | img_dir: ${mvl_dir}/img 19 | labels_dir: ${mvl_dir}/labels/${id_exp} 20 | label: mlc_label 21 | scene_list: ${training_scene_list} 22 | size: -1 23 | batch_size: 5 24 | num_workers: 4 25 | 26 | valid_iou: 27 | data_dir: 28 | img_dir: ${mvl_dir}/img 29 | labels_dir: ${mvl_dir}/labels/ 30 | scene_list: ${pilot_scene_list} 31 | batch_size: 4 32 | label: gt 33 | size: -1 34 | num_workers: 4 35 | 36 | model: 37 | ly_model: HorizonNet 38 | ckpt: ${ckpt} 39 | optimizer: Adam 40 | loss: weighted_L1 41 | min_std: 0.001 42 | epochs: 25 43 | save_every: -1 44 | lr: 0.00005 45 | lr_decay_rate: 0.90 46 | seed: 594277 47 | bn_momentum: 0 48 | freeze_earlier_blocks: -1 49 | beta1: 0.9 50 | weight_decay: 0.0 -------------------------------------------------------------------------------- /mvl_challenge/utils/download_utils.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | 5 | def download_file_from_google_drive(id, destination): 6 | URL = "https://docs.google.com/uc?export=download" 7 | 8 | session = requests.Session() 9 | response = session.get(URL, params={"id": id}, stream=False) 10 | response._content 11 | token = get_confirm_token(response) 12 | if token: 13 | params = { 'id' : id, 'confirm' : token } 14 | response = session.get(URL, params = params, stream = True) 15 | save_response_content(response, destination) 16 | 17 | 18 | def get_confirm_token(response): 19 | for key, value in response.cookies.items(): 20 | if key.startswith("download_warning"): 21 | return value 22 | 23 | return None 24 | 25 | 26 | def save_response_content(response, destination): 27 | # CHUNK_SIZE = 32768 28 | CHUNK_SIZE = 10 29 | with open(destination, "wb") as f: 30 | data = response._content 31 | f.write(data) 32 | 33 | # for chunk in response.iter_content(CHUNK_SIZE): 34 | # if chunk: # filter out keep-alive new chunks 35 | # f.write(chunk) 36 | 37 | 38 | if __name__ == "__main__": 39 | file_id = "TAKE ID FROM SHAREABLE LINK" 40 | destination = "DESTINATION FILE ON YOUR DISK" 41 | download_file_from_google_drive(file_id, destination) 42 | -------------------------------------------------------------------------------- /mvl_challenge/data/gdrive_files/gdrive_ids__pilot_set.csv: -------------------------------------------------------------------------------- 1 | 1aDEhYs022dcDAMYN6u2jS0MdXNsZ1jh8 scene_list__mp3d_fp..._up_pilot_set.json bin 9.8 KB 2023-03-19 00:44:20 2 | 1vaoFIhWrziKdDFymZVqp2h6GofL6sHuk Z6MFQCViBuw_0_room8.zip bin 3.3 MB 2023-03-19 00:44:18 3 | 183RsvL5KGDHkf-Iy4le6NEb5YBPPtGxm VFuaQ6m2Qom_0_room1.zip bin 5.6 MB 2023-03-19 00:44:15 4 | 18IaJrC3IZRmTurANAzEis9ilzOwJcW32 7y3sRwLe3Va_0_room7.zip bin 2.2 MB 2023-03-19 00:44:13 5 | 1w4hnKWg6w5myXvCBM9oli1ufbrd36zfB 7y3sRwLe3Va_0_room3.zip bin 3.8 MB 2023-03-19 00:44:11 6 | 1iZ_1UtqLZqFc4sUHtL571IbH0BuOyMoI 7y3sRwLe3Va_0_room1.zip bin 5.7 MB 2023-03-19 00:44:08 7 | 1c3xwFGWYEByvSiYSp0pFoLVZf6We1brn wcojb4TFT35_0_room0.zip bin 3.2 MB 2023-03-19 00:40:23 8 | 1SUeyEM2z1XAMQbhuGhtsfNdZUKa5wMpZ scene_list__hm3d_mv..._up_pilot_set.json bin 7.1 KB 2023-03-19 00:40:21 9 | 1cpkp1pJvLU0vTxaJABYoTcN20N87yY-g q3hn1WQ12rz_6_room0.zip bin 3.5 MB 2023-03-19 00:40:19 10 | 14NUOCXwDW9lypltFuPiB6bi8D4BMopMr k1cupFYWXJ6_0_room0.zip bin 3.8 MB 2023-03-19 00:40:16 11 | 1K-YfbiITxIxM6kLxHQ_5bhT1sKb2PSzx h1zeeAwLh9Z_3_room0.zip bin 4.5 MB 2023-03-19 00:40:14 12 | 1rV-dvl7vxPqwz3bmtUGZrEoRIlcwhzrk HaxA7YrQdEC_1_room0.zip bin 4.3 MB 2023-03-19 00:40:12 13 | -------------------------------------------------------------------------------- /mvl_challenge/main_plot_rgbd_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import numpy as np 4 | 5 | from mvl_challenge.datasets.rgbd_datasets import HM3D_MVL, MP3D_FPE, RGBD_Dataset 6 | from mvl_challenge.utils.vispy_utils import plot_color_plc 7 | 8 | 9 | def main(args): 10 | 11 | if args.dataset_name == "": 12 | dt = RGBD_Dataset.from_args(args) 13 | if args.dataset_name == "mp3d_fpe": 14 | dt = MP3D_FPE.from_args(args) 15 | elif args.dataset_name == "hm3d_mvl": 16 | dt = HM3D_MVL.from_args(args) 17 | else: 18 | raise ValueError() 19 | 20 | list_fr = dt.get_list_frames() 21 | pcl = np.hstack([fr.get_pcl() for fr in list_fr[:50]]) 22 | plot_color_plc(points=pcl[0:3, :].T, color=pcl[3:].T) 23 | 24 | 25 | def get_args(): 26 | parser = argparse.ArgumentParser() 27 | # * Input Directory (-s) 28 | parser.add_argument( 29 | "--scene_dir", 30 | # required=True, 31 | default="/media/public_dataset/HM3D-MVL/test/BHXhpBwSMLh/0/", 32 | type=str, 33 | help="Input Directory (scene directory defined until version scene)", 34 | ) 35 | 36 | parser.add_argument( 37 | "--dataset_name", 38 | # required=True, 39 | default="", 40 | type=str, 41 | help="dataset name [mp3d_fpe or hmd3d_mvl]", 42 | ) 43 | 44 | args = parser.parse_args() 45 | return args 46 | 47 | 48 | if __name__ == "__main__": 49 | args = get_args() 50 | main(args) 51 | -------------------------------------------------------------------------------- /tutorial/train_horizon_net/download_pretrained_ckpt.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from mvl_challenge import DEFAULT_CKPT_DIR, GDRIVE_DIR 5 | from mvl_challenge.remote_data.download_mvl_data import download_google_drive_link 6 | from mvl_challenge.utils.io_utils import create_directory, read_txt_file 7 | 8 | 9 | def main(args): 10 | 11 | create_directory(args.output_dir, delete_prev=False) 12 | 13 | gdrive_ids_fn = args.gdrive_ids 14 | assert os.path.exists(gdrive_ids_fn), f"Not found {gdrive_ids_fn}" 15 | 16 | lines = read_txt_file(gdrive_ids_fn) 17 | 18 | for l in lines: 19 | gd_id, zip_fn = [l for l in l.replace(" ", ",").split(",") if l != ""][:2] 20 | output_file = os.path.join(args.output_dir, zip_fn) 21 | download_google_drive_link( 22 | gd_id, output_file, f"{lines.index(l)+1}/{lines.__len__()}" 23 | ) 24 | 25 | 26 | def get_passed_args(): 27 | parser = argparse.ArgumentParser() 28 | 29 | parser.add_argument( 30 | "-f", 31 | "--gdrive_ids", 32 | type=str, 33 | default=f"{GDRIVE_DIR}/gdrive_ids__ckpt_hn_models.csv", 34 | help=f"CKPT download info.", 35 | ) 36 | 37 | parser.add_argument( 38 | "-o", 39 | "--output_dir", 40 | type=str, 41 | default=f"{DEFAULT_CKPT_DIR}", 42 | help=f"Default CKPT directory {DEFAULT_CKPT_DIR}.", 43 | ) 44 | 45 | args = parser.parse_args() 46 | return args 47 | 48 | if __name__ == "__main__": 49 | args = get_passed_args() 50 | main(args) 51 | -------------------------------------------------------------------------------- /mvl_challenge/data_structure/cam_pose.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from mvl_challenge.utils.geometry_utils import isRotationMatrix 3 | 4 | 5 | class CAM_REF: 6 | WC = "WC" # ! World Coordinates 7 | CC = "CC" # ! Cameras Coordinates 8 | WC_SO3 = "WC_SO3" # ! World Coordinates only Rot applied 9 | ROOM = "ROOM_REF" # ! Room Coordinates (primary frame) 10 | 11 | 12 | class CamPose: 13 | def __init__(self, cfg): 14 | self.cfg = cfg 15 | self.SE3 = np.eye(4) 16 | self.vo_scale = 1 17 | self.gt_scale = 1 18 | self.idx = None 19 | 20 | @property 21 | def vo_scale(self): 22 | return self.__vo_scale 23 | 24 | @vo_scale.setter 25 | def vo_scale(self, value): 26 | assert value > 0 27 | self.__vo_scale = value 28 | 29 | @property 30 | def SE3(self): 31 | return self.__pose 32 | 33 | @SE3.setter 34 | def SE3(self, value): 35 | assert value.shape == (4, 4) 36 | self.__pose = value 37 | self.rot = value[0:3, 0:3] 38 | self.t = value[0:3, 3] 39 | 40 | @property 41 | def rot(self): 42 | return self.__rot 43 | 44 | @rot.setter 45 | def rot(self, value): 46 | assert isRotationMatrix(value) 47 | self.__rot = value 48 | self.__pose[:3, :3] = value 49 | 50 | @property 51 | def t(self): 52 | return self.__t * self.vo_scale * self.gt_scale 53 | 54 | @t.setter 55 | def t(self, value): 56 | assert value.reshape( 57 | 3, 58 | ).shape == (3,) 59 | self.__t = value.reshape( 60 | 3, 61 | ) 62 | self.__pose[:3, 3] = value 63 | 64 | def SE3_scaled(self): 65 | m = np.eye(4) 66 | m[0:3, 0:3] = self.rot 67 | m[0:3, 3] = self.t 68 | return m 69 | -------------------------------------------------------------------------------- /mvl_challenge/data_structure/frame.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | from imageio import imread 5 | 6 | from mvl_challenge.utils.geometry_utils import extend_array_to_homogeneous 7 | from mvl_challenge.utils.image_utils import load_depth_map 8 | 9 | 10 | class Frame: 11 | def __init__(self, dt): 12 | self.dt = dt 13 | self.pose = None 14 | self.idx = None 15 | self.rgb_file = None 16 | self.depth_map_file = None 17 | self.rgb_map = None 18 | self.depth_map = None 19 | self.pcl = None 20 | 21 | def get_rgb(self): 22 | if self.rgb_map is None: 23 | if not os.path.exists(self.rgb_file): 24 | raise FileNotFoundError(self.rgb_file) 25 | self.rgb_map = imread(self.rgb_file) 26 | return self.rgb_map 27 | 28 | def get_depth(self): 29 | if self.depth_map is None: 30 | if not os.path.exists(self.depth_map_file): 31 | raise FileNotFoundError(self.depth_map_file) 32 | self.depth_map = load_depth_map(self.depth_map_file) 33 | return self.depth_map 34 | 35 | def get_pcl(self): 36 | if self.pcl is None: 37 | pcl, color = self.dt.cam.project_pcl_from_depth_and_rgb_maps( 38 | color_map=self.get_rgb(), 39 | depth_map=self.get_depth(), 40 | ) 41 | self.pcl = np.vstack( 42 | (self.pose[:3, :] @ extend_array_to_homogeneous(pcl), color) 43 | ) 44 | return self.pcl 45 | 46 | def set_room_data(self, room_name, pose_ref): 47 | """ 48 | Sets the room data from where this fr belongs to 49 | """ 50 | # ! This function is mainly implemented for MP3D-FPE dataset 51 | self.room_name = room_name 52 | self.pose = np.linalg.inv(pose_ref) @ self.pose 53 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/pre_process_rgbd_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # DATA_NAME=hm3d_mvl 4 | # RGBD_DATA_DIR=/media/public_dataset/HM3D-MVL/test 5 | # SCENE_LIST="$DATA_NAME"__test__scene_list 6 | 7 | DATA_NAME=mp3d_fpe 8 | RGBD_DATA_DIR=/media/public_dataset/MP3D_360_FPE/SINGLE_ROOM_SCENES 9 | SCENE_LIST="$DATA_NAME"__single_room__scene_list 10 | 11 | OUTPUT_DIR=/media/public_dataset/mvl_challenge/$DATA_NAME 12 | 13 | # ! Create scene_room_idx files --> *__scene_list.json 14 | python mvl_challenge/pre_processing/create_scene_room_idx_list.py \ 15 | -d $RGBD_DATA_DIR \ 16 | -f $SCENE_LIST \ 17 | -o "$OUTPUT_DIR" -m 5 18 | 19 | # # ! geometry info files --> scene_room_idx.json --> cam pose + camera height 20 | # python mvl_challenge/pre_processing/create_geometry_info_files.py \ 21 | # -d $RGBD_DATA_DIR \ 22 | # --scene_list "$OUTPUT_DIR"/"$SCENE_LIST".json \ 23 | # -o "$OUTPUT_DIR"/geometry_info 24 | 25 | # # ! RGB files 26 | # python mvl_challenge/pre_processing/create_rgb_files.py \ 27 | # -d $RGBD_DATA_DIR \ 28 | # -g "$OUTPUT_DIR"/geometry_info \ 29 | # -o "$OUTPUT_DIR"/img 30 | 31 | # # ! create GT labels --> scene_room_idx.npz --> phi_coords GT 32 | # python mvl_challenge/pre_processing/create_npz_labels.py \ 33 | # -d $RGBD_DATA_DIR \ 34 | # -g "$OUTPUT_DIR"/geometry_info \ 35 | # -o "$OUTPUT_DIR"/labels/gt 36 | 37 | # # ! Zipping mvl-data 38 | # python mvl_challenge/remote_data/zip_mvl_dataset.py \ 39 | # -d $OUTPUT_DIR \ 40 | # --scene_list "$OUTPUT_DIR"/"$SCENE_LIST".json \ 41 | # -o "$OUTPUT_DIR"/zips/mvl_data 42 | 43 | # python mvl_challenge/remote_data/zip_mvl_dataset.py \ 44 | # -d "$OUTPUT_DIR" \ 45 | # --scene_list "$OUTPUT_DIR"/labels/gt_labels__scene_list.json \ 46 | # -o "$OUTPUT_DIR"/zips/labels \ 47 | # --labels -------------------------------------------------------------------------------- /tutorial/train_horizon_net/README.md: -------------------------------------------------------------------------------- 1 | ## Tutorial 1: Train your HorizonNet on our dataset 2 | 3 | In this tutorial, we will walk you through how to train your own model using scenes in the pilot split (with labels). Make sure you have downloaded HorizonNet following [here](https://github.com/mvlchallenge/mvl_toolkit/tree/main#installation). 4 | 5 | ### Step 1: download pre-trained models 6 | ```bash 7 | # use -h for more details 8 | python download_pretrained_ckpt.py 9 | # or specify download info and output directory 10 | python download_pretrained_ckpt.py -f $DOWNLOAD_INFO -o $OUTPUT_DIR 11 | ``` 12 | Afterwards, you should see all pre-trained models under `mvl_challenge/assets/ckpt`. 13 | 14 | ### Step 2: start training HorizonNet 15 | 16 | Instead of using the native [HorionNet](https://github.com/sunset1995/HorizonNet), we provide a wrapper at `mvl_challenge/models/wrapper_horizon_net.py` allowing you to easily train your model on the provided dataset. 17 | 18 | This wrapper keeps the overall structure of the original implementation, except for taking out the post-processing part, which allows estimations of non-Manhattan scenes. You can find all controlable settings and hyperparameters in `train_hn.yaml`, note that parameters marked with `` should be filled. 19 | 20 | ```bash 21 | python train_hn.py 22 | # or 23 | python train_hn.py --cfg "train_hn.yaml" --pilot_scene_list $SCENE_LIST -ckpt $CHECK_POINT 24 | ``` 25 | 26 | 27 | At the same time, you can find the best-performing weights on 2D and 3D IoU and evalution logging in `mvl_challenge/assets/data/mvl_training_results/hn_mp3d__scene_list__warm_up_pilot_set/ckpt`. While the tensorboard event files will be in `mvl_challenge/assets/data/mvl_training_results/hn_mp3d__scene_list__warm_up_pilot_set/log`. 28 | 29 | Run the following command to visualize the training status 30 | ```bash 31 | LOG_DIR="mvl_challenge/assets/data/mvl_training_results/hn_mp3d__scene_list__warm_up_pilot_set/log" 32 | tensorboard --log_dir $LOG_DIR 33 | ``` 34 | 35 | ![](https://user-images.githubusercontent.com/67839539/230726425-4d5db4a9-e495-4f29-83c2-c4b1666f7732.png) 36 | -------------------------------------------------------------------------------- /mvl_challenge/utils/check_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import json 4 | from tqdm import tqdm 5 | from mvl_challenge.utils.io_utils import get_all_frames_from_scene_list 6 | 7 | 8 | def check_geo_info_and_img(mvl_dir, list_scenes): 9 | """ 10 | Checks whether the passed list scene is define in both 11 | geometry info and img directory. 12 | 13 | Args: 14 | mvl_dir (path): Path to a mvl dataset 15 | list_scenes (list): list if scene in scene_room_idx format 16 | 17 | Returns: 18 | Bool: passed or not the checking. 19 | """ 20 | 21 | geo_info_fn = [ 22 | os.path.join(mvl_dir, "geometry_info", f"{sc}.json") for sc in list_scenes 23 | ] 24 | img_fn = [os.path.join(mvl_dir, "img", f"{sc}.jpg") for sc in list_scenes] 25 | 26 | geo_info_check = [os.path.exists(fn) for fn in geo_info_fn] 27 | img_fn_check = [os.path.exists(fn) for fn in img_fn] 28 | 29 | output = True 30 | if np.sum(geo_info_check) != geo_info_check.__len__(): 31 | [ 32 | print(f"Not found {fn}") 33 | for ckpt, fn in zip(geo_info_check, geo_info_fn) 34 | if not ckpt 35 | ] 36 | output = False 37 | 38 | if np.sum(img_fn_check) != img_fn_check.__len__(): 39 | [print(f"Not found {fn}") for ckpt, fn in zip(img_fn_check, img_fn) if not ckpt] 40 | output = False 41 | 42 | return output 43 | 44 | 45 | def check_mvl_dataset(args): 46 | """Checks a MVL dataset directory based on a list of scenes defined in scene_room_idx 47 | Args: 48 | args (parseargs): args.scene_dir, args.scene_list are need 49 | Returns: 50 | [dict, bool]: data_scene_dict, True or False 51 | """ 52 | 53 | data_scene = json.load(open(args.scene_list, "r")) 54 | print(f"scene list: {args.scene_list}") 55 | check = [ 56 | check_geo_info_and_img(args.scene_dir, scene_list) 57 | for scene_list in tqdm( 58 | data_scene.values(), desc=f"loading and checking {args.scene_dir}" 59 | ) 60 | ] 61 | return data_scene, np.sum(check) == check.__len__() 62 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/utils/camera_height_per_rooms.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge.datasets.rgbd_datasets import MP3D_FPE 3 | from mvl_challenge.config.cfg import get_empty_cfg 4 | from mvl_challenge.pre_processing.utils.camera_height_utils import ( 5 | estimate_camera_height, 6 | ) 7 | import numpy as np 8 | from mvl_challenge.utils.vispy_utils import plot_color_plc 9 | from mvl_challenge.utils.io_utils import ( 10 | save_json_dict, 11 | create_directory, 12 | get_files_given_a_pattern, 13 | ) 14 | from mvl_challenge import ASSETS_DIR, CFG_DIR 15 | from mvl_challenge.config.cfg import read_omega_cfg 16 | import logging 17 | 18 | 19 | def estimate_cam_height_per_room(cfg, dt: MP3D_FPE): 20 | list_fr2world = dt.get_list_frames() 21 | cam_height_dict = {} 22 | for list_fr in dt.iter_rooms_scenes(): 23 | if list_fr.__len__() < 5: 24 | continue 25 | #! Each fr in list_fr is wrt to room references 26 | room_name = list_fr[0].room_name 27 | init_idx = list_fr[0].idx 28 | cam_h_rc = estimate_camera_height(cfg, list_fr) 29 | 30 | room_wc = [fr.pose[1, 3] for fr in list_fr2world if fr.idx == init_idx][0] 31 | cam_h_wc = cam_h_rc + room_wc 32 | cam_height_dict[room_name] = dict(cam_h_rc=cam_h_rc, cam_h_wc=cam_h_wc) 33 | [ 34 | logging.info( 35 | f"Room: {r}\tcam_h(r): {d['cam_h_rc']:2.3f}\tcam_h(w): {d['cam_h_wc']:2.3f}" 36 | ) 37 | for r, d in cam_height_dict.items() 38 | ] 39 | return cam_height_dict 40 | 41 | 42 | def main(args): 43 | cfg = read_omega_cfg(args.cfg) 44 | cfg.dataset = dict() 45 | cfg.dataset.scene_dir = args.scene_dir 46 | dt = MP3D_FPE.from_cfg(cfg) 47 | estimate_cam_height_per_room(cfg.cam_height_cfg, dt) 48 | 49 | 50 | def get_args(): 51 | parser = argparse.ArgumentParser() 52 | 53 | parser.add_argument( 54 | "--scene_dir", 55 | # required=True, 56 | default="/media/public_dataset/MP3D_360_FPE/MULTI_ROOM_SCENES/2t7WUuJeko7/1/", 57 | type=str, 58 | help="Directory of all scene in the dataset", 59 | ) 60 | 61 | parser.add_argument( 62 | "--cfg", 63 | default=f"{CFG_DIR}/camera_height.yaml", 64 | help="Cfg tp compute camera height", 65 | ) 66 | 67 | args = parser.parse_args() 68 | return args 69 | 70 | 71 | if __name__ == "__main__": 72 | args = get_args() 73 | main(args) 74 | -------------------------------------------------------------------------------- /mvl_challenge/scene_list__get_info.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ASSETS_DIR, EPILOG 3 | from mvl_challenge.utils.io_utils import ( 4 | get_files_given_a_pattern, 5 | get_scene_room_from_scene_room_idx, 6 | ) 7 | import os 8 | import json 9 | from mvl_challenge.utils.io_utils import ( 10 | save_json_dict, 11 | create_directory, 12 | get_scene_list_from_dir, 13 | ) 14 | from mvl_challenge.config.cfg import get_empty_cfg 15 | import numpy as np 16 | from tqdm import tqdm 17 | 18 | 19 | def print_scene_data_info(scene_data): 20 | frames_list = [fr for fr in scene_data.values()] 21 | numb_frames = [fr.__len__() for fr in scene_data.values()] 22 | frames_list = [item for sublist in frames_list for item in sublist] 23 | print(f"Number Rooms: {list(scene_data.keys()).__len__()}") 24 | print(f"Number Frames: {frames_list.__len__()}") 25 | if frames_list.__len__() == 0: 26 | return 27 | print(f"Max Number Frames: {np.max(numb_frames)}") 28 | print(f"Min Number Frames: {np.min(numb_frames)}") 29 | print(f"Average Number Frames: {np.mean(numb_frames)}") 30 | 31 | 32 | def print_stats_from_scene_list(args): 33 | cfg = get_empty_cfg() 34 | if os.path.isdir(args.i): 35 | cfg.scene_dir = args.i 36 | scene_data = get_scene_list_from_dir(cfg) 37 | 38 | elif os.path.isfile(args.i): 39 | cfg.scene_list = args.i 40 | scene_data = json.load(open(cfg.scene_list)) 41 | else: 42 | raise ValueError("Wrong input argument") 43 | 44 | print_scene_data_info(scene_data) 45 | 46 | if args.output_fn != "": 47 | create_directory(os.path.dirname(args.output_fn), delete_prev=False) 48 | save_json_dict(args.output_fn, scene_data) 49 | print(f"Saved at {args.output_fn}") 50 | 51 | 52 | def get_argparse(): 53 | desc = "This script reads a scene_list.json file or a mvl directory to print the pyth number of frames and rooms in the data." 54 | 55 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 56 | 57 | parser.add_argument( 58 | "--i", required=True, type=str, help="Input Filename or directory." 59 | ) 60 | 61 | parser.add_argument( 62 | "-o", 63 | "--output_fn", 64 | default="", 65 | type=str, 66 | help="Outputs the scene_list information in the passed filename.", 67 | ) 68 | 69 | args = parser.parse_args() 70 | return args 71 | 72 | 73 | if __name__ == "__main__": 74 | args = get_argparse() 75 | print_stats_from_scene_list(args) 76 | -------------------------------------------------------------------------------- /tutorial/train_horizon_net/train_hn.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from pathlib import Path 4 | 5 | from mvl_challenge import ASSETS_DIR, DEFAULT_MVL_DIR, DEFAULT_TRAINING_DIR, SCENE_LIST_DIR 6 | from mvl_challenge.config.cfg import read_omega_cfg 7 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 8 | 9 | HN_TUTORIAL_DIR=os.path.dirname(os.path.realpath(__file__)) 10 | 11 | def get_cfg_from_args(args): 12 | cfg = read_omega_cfg(args.cfg) 13 | cfg.mvl_dir = args.scene_dir 14 | cfg.pilot_scene_list = args.pilot_scene_list 15 | cfg.output_dir = args.output_dir 16 | cfg.ckpt = args.ckpt 17 | cfg.cuda_device = args.cuda_device 18 | cfg.id_exp = f"{Path(cfg.ckpt).stem}__{Path(args.pilot_scene_list).stem}" 19 | return cfg 20 | 21 | def main(args): 22 | # ! Reading configuration 23 | cfg = get_cfg_from_args(args) 24 | 25 | model = WrapperHorizonNet(cfg) 26 | 27 | model.prepare_for_training() 28 | model.set_valid_dataloader() 29 | model.valid_iou_loop() 30 | model.save_current_scores() 31 | while model.is_training: 32 | model.train_loop() 33 | model.valid_iou_loop() 34 | model.save_current_scores() 35 | 36 | def get_passed_args(): 37 | parser = argparse.ArgumentParser() 38 | 39 | default_cfg = f"{HN_TUTORIAL_DIR}/train_hn.yaml" 40 | parser.add_argument( 41 | '--cfg', 42 | default=default_cfg, 43 | help=f'Config File. Default {default_cfg}') 44 | 45 | parser.add_argument( 46 | "--pilot_scene_list", 47 | type=str, 48 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 49 | help="Pilot scene list", 50 | ) 51 | 52 | parser.add_argument( 53 | "-o", 54 | "--output_dir", 55 | type=str, 56 | default=f"{DEFAULT_TRAINING_DIR}", 57 | help="MVL dataset directory.", 58 | ) 59 | 60 | parser.add_argument( 61 | "-d", 62 | "--scene_dir", 63 | type=str, 64 | default=f"{DEFAULT_MVL_DIR}", 65 | help="MVL dataset directory.", 66 | ) 67 | 68 | parser.add_argument( 69 | "--ckpt", 70 | default=f"{ASSETS_DIR}/ckpt/hn_mp3d.pth", 71 | help=f"Path to ckpt pretrained model (Default: {ASSETS_DIR}/ckpt/hn_mp3d.pth)", 72 | ) 73 | 74 | parser.add_argument("--cuda_device", default=0, type=int, help="Cuda device. (Default: 0)") 75 | 76 | args = parser.parse_args() 77 | return args 78 | 79 | if __name__ == "__main__": 80 | args = get_passed_args() 81 | main(args) 82 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/fix_scene_list.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ( 3 | ASSETS_DIR, 4 | EPILOG, 5 | DEFAULT_MVL_DIR, 6 | ) 7 | from mvl_challenge.utils.io_utils import ( 8 | get_files_given_a_pattern, 9 | get_scene_room_from_scene_room_idx, 10 | ) 11 | import os 12 | import yaml 13 | import json 14 | from pathlib import Path 15 | from mvl_challenge.utils.io_utils import ( 16 | save_json_dict, 17 | create_directory, 18 | get_scene_list_from_dir, 19 | ) 20 | from mvl_challenge.config.cfg import get_empty_cfg 21 | from mvl_challenge.scene_list__edit_info import prune_list_frames 22 | from mvl_challenge.config.cfg import set_loggings 23 | import numpy as np 24 | from tqdm import tqdm 25 | 26 | 27 | def fix_scene_list_file(args): 28 | 29 | assert os.path.exists(args.scene_list), f"No directory found {args.scene_list}" 30 | 31 | data = json.load(open(args.scene_list, 'r')) 32 | list_scenes = [scene for scene in data.values()] 33 | list_scenes = [item for sublist in list_scenes for item in sublist] 34 | 35 | list_rooms = np.unique( 36 | [get_scene_room_from_scene_room_idx(Path(fn).stem) for fn in list_scenes] 37 | ).tolist() 38 | data_new = {} 39 | for room in tqdm(list_rooms, desc="List rooms..."): 40 | data_new[room] = [Path(fn).stem for fn in list_scenes if f"{room}_" in fn] 41 | 42 | list_rooms = list(data.keys()) 43 | check_rooms = [room not in list_rooms for room in data_new.keys()] 44 | if np.sum(check_rooms) > 0: 45 | if args.c: 46 | print_difference(data, data_new) 47 | else: 48 | print_difference(data, data_new) 49 | save_json_dict(f'{args.scene_list}', data_new) 50 | else: 51 | print("Nothing to change...") 52 | 53 | def print_difference(data, data_new): 54 | list_rooms = list(data.keys()) 55 | print("New Rooms found:") 56 | [print(f"\t{room}") for room in data_new.keys() if room not in list_rooms ] 57 | 58 | 59 | 60 | def get_argparse(): 61 | parser = argparse.ArgumentParser(epilog=EPILOG) 62 | 63 | parser.add_argument( 64 | "-f", 65 | "--scene_list", 66 | # required=True, 67 | default="/media/NFS/kike/360_Challenge/mvl_toolkit/mvl_challenge/data/scene_list/scene_list__warm_up_training.json", 68 | type=str, 69 | help="*.json scene list file", 70 | ) 71 | 72 | 73 | parser.add_argument( 74 | "-c", 75 | action='store_true', 76 | help="Overwrite passed scene list file", 77 | ) 78 | 79 | args = parser.parse_args() 80 | return args 81 | 82 | 83 | if __name__ == "__main__": 84 | args = get_argparse() 85 | fix_scene_list_file(args) 86 | -------------------------------------------------------------------------------- /mvl_challenge/mvl_data/load_mvl_dataset.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ( 3 | DATA_DIR, 4 | ROOT_DIR, 5 | CFG_DIR, 6 | EPILOG, 7 | ASSETS_DIR, 8 | DEFAULT_MVL_DIR, 9 | SCENE_LIST_DIR, 10 | ) 11 | from mvl_challenge.config.cfg import read_omega_cfg 12 | from mvl_challenge.datasets.mvl_dataset import MVLDataset 13 | import logging 14 | from tqdm import tqdm 15 | import numpy as np 16 | from mvl_challenge.utils.image_utils import plot_image, add_caption_to_image 17 | 18 | 19 | def get_cfg_from_args(args): 20 | cfg = read_omega_cfg(args.cfg) 21 | cfg.scene_dir = args.scene_dir 22 | cfg.scene_list = args.scene_list 23 | return cfg 24 | 25 | 26 | def main(args): 27 | cfg = get_cfg_from_args(args) 28 | mvl = MVLDataset(cfg) 29 | mvl.print_mvl_data_info() 30 | # ! Loading list_ly by passing room_scene 31 | for room_scene in tqdm(mvl.list_rooms, desc="Loading room scene..."): 32 | list_ly = mvl.get_list_ly(room_scene=room_scene) 33 | for ly in list_ly: 34 | plot_image(image=ly.get_rgb(), caption=ly.idx) 35 | 36 | # ! Iterator of list_ly 37 | for list_ly in mvl.iter_list_ly(): 38 | for ly in list_ly: 39 | plot_image(image=ly.get_rgb(), caption=ly.idx) 40 | 41 | 42 | def get_argparse(): 43 | desc = ( 44 | "This script loads a MVL dataset given a passed scene directory, scene list and cfg file. " 45 | + "The scene directory is where the MVL data is stored. " 46 | + "The scene list is the list of scene in scene_room_idx format. " 47 | + "The cfg file is the yaml configuration with all hyperparameters set to default values." 48 | ) 49 | 50 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 51 | 52 | parser.add_argument( 53 | "-d", 54 | "--scene_dir", 55 | # required=True, 56 | # default=f"{ASSETS_DIR}/mvl_data/mp3d_fpe", 57 | # default=f'{ASSETS_DIR}/tmp/zip_files', 58 | # default=None, 59 | default=f"{DEFAULT_MVL_DIR}", 60 | type=str, 61 | help="MVL dataset directory.", 62 | ) 63 | 64 | parser.add_argument( 65 | "--cfg", 66 | type=str, 67 | default=f"{CFG_DIR}/load_mvl_dataset.yaml", 68 | help=f"Config file to load a MVL dataset. (default: {CFG_DIR}/load_mvl_dataset.yaml)", 69 | ) 70 | 71 | parser.add_argument( 72 | "-f", 73 | "--scene_list", 74 | type=str, 75 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 76 | help="Config file to load a MVL dataset.", 77 | ) 78 | 79 | args = parser.parse_args() 80 | return args 81 | 82 | 83 | if __name__ == "__main__": 84 | args = get_argparse() 85 | main(args) 86 | -------------------------------------------------------------------------------- /mvl_challenge/mvl_data/load_labels.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import DATA_DIR, ROOT_DIR, CFG_DIR, EPILOG, ASSETS_DIR 3 | from mvl_challenge.config.cfg import read_omega_cfg 4 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 5 | import logging 6 | import os 7 | from tqdm import tqdm 8 | from mvl_challenge.utils.vispy_utils import plot_list_ly 9 | from mvl_challenge.utils.image_utils import draw_boundaries_phi_coords 10 | from imageio import imwrite 11 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 12 | import numpy as np 13 | from mvl_challenge.utils.io_utils import ( 14 | create_directory, 15 | get_scene_room_from_scene_room_idx, 16 | load_gt_label, 17 | ) 18 | from mvl_challenge.utils.image_utils import plot_image 19 | 20 | 21 | def get_cfg_from_args(args): 22 | cfg = read_omega_cfg(args.cfg) 23 | cfg.scene_dir = args.scene_dir 24 | cfg.scene_list = args.scene_list 25 | return cfg 26 | 27 | 28 | def main(args): 29 | cfg = get_cfg_from_args(args) 30 | mvl = MVLDataset(cfg) 31 | 32 | for list_ly in mvl.iter_list_ly(): 33 | for ly in list_ly: 34 | phi_coords = load_gt_label( 35 | os.path.join(args.scene_dir, "labels", "gt", f"{ly.idx}.npz") 36 | ) 37 | img = ly.get_rgb() 38 | draw_boundaries_phi_coords(img, phi_coords=phi_coords) 39 | 40 | plot_image(image=img, caption=ly.idx + " - GT labels") 41 | 42 | 43 | def get_argparse(): 44 | desc = ( 45 | "This script loads a MVL dataset given a passed scene directory, scene list and cfg file. " 46 | + "The scene directory is where the MVL data is stored. " 47 | + "The scene list is the list of scene in scene_room_idx format. " 48 | + "The cfg file is the yaml configuration with all hyperparameters set to default values." 49 | ) 50 | 51 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 52 | 53 | parser.add_argument( 54 | "-d", 55 | "--scene_dir", 56 | type=str, 57 | # default=f'{ASSETS_DIR}/mvl_data/mp3d_fpe', 58 | help="MVL dataset directory.", 59 | ) 60 | 61 | parser.add_argument( 62 | "--cfg", 63 | type=str, 64 | default=f"{CFG_DIR}/load_mvl_dataset.yaml", 65 | help=f"Config file to load a MVL dataset. (Default: {CFG_DIR}/load_mvl_dataset.yaml)", 66 | ) 67 | 68 | parser.add_argument( 69 | "-f", 70 | "--scene_list", 71 | type=str, 72 | # default=f"{DATA_DIR}/mp3d_fpe/test__gt_labels__scene_list.json", 73 | help="Config file to load a MVL dataset.", 74 | ) 75 | 76 | args = parser.parse_args() 77 | return args 78 | 79 | 80 | if __name__ == "__main__": 81 | args = get_argparse() 82 | main(args) 83 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/create_rgb_files.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import CFG_DIR, ASSETS_DIR, EPILOG 3 | import os 4 | from mvl_challenge.config.cfg import read_omega_cfg 5 | from mvl_challenge.utils.io_utils import ( 6 | get_files_given_a_pattern, 7 | get_all_frames_from_scene_list, 8 | ) 9 | from mvl_challenge.datasets.rgbd_datasets import RGBD_Dataset 10 | from mvl_challenge.pre_processing.utils.geometry_info_utils import get_geometry_info 11 | from mvl_challenge.utils.io_utils import save_json_dict, create_directory 12 | from mvl_challenge.config.cfg import set_loggings 13 | from tqdm import tqdm 14 | import logging 15 | import json 16 | from pathlib import Path 17 | from shutil import copyfile 18 | from imageio import imread, imwrite 19 | 20 | 21 | def main(args): 22 | # ! Reading geometry info 23 | set_loggings() 24 | scene_list = get_all_frames_from_scene_list(args.scene_list) 25 | create_directory(args.output_dir, delete_prev=False) 26 | for scene_room_idx in tqdm(scene_list, desc="list geom info..."): 27 | scene, version, _, idx = Path(scene_room_idx).stem.split("_") 28 | src = os.path.join(args.scene_dir, scene, version, "rgb", f"{idx}.jpg") 29 | dst = os.path.join(args.output_dir, f"{Path(scene_room_idx).stem}.jpg") 30 | if os.path.exists(src): 31 | copyfile(src, dst) 32 | logging.info(f"Img {dst} processed...") 33 | continue 34 | else: 35 | src = os.path.join(args.scene_dir, scene, version, "rgb", f"{idx}.png") 36 | if os.path.exists(src): 37 | img = imread(src) 38 | imwrite(dst, img) 39 | logging.info(f"Img {dst} processed...") 40 | continue 41 | logging.info(f"Img {src} skipped...") 42 | 43 | 44 | def get_argparse(): 45 | desc = ( 46 | "This script creates a hard copy of the RGB images defined in the given MLV dataset. " 47 | + "The RGB images are encoded in scene_room_idx format." 48 | ) 49 | 50 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 51 | 52 | parser.add_argument( 53 | "-d", 54 | "--scene_dir", 55 | # required=True, 56 | default="/media/public_dataset/MP3D_360_FPE/MULTI_ROOM_SCENES/", 57 | # default="/media/public_dataset/HM3D-MVL/test/BHXhpBwSMLh", 58 | type=str, 59 | help="RGBD dataset directory.", 60 | ) 61 | 62 | parser.add_argument( 63 | "-f", 64 | "--scene_list", 65 | # required=True, 66 | default=f"{ASSETS_DIR}/mvl_data/geometry_info", 67 | type=str, 68 | help="Geometry information directory. (default: {ASSETS_DIR}/mvl_data/geometry_info)", 69 | ) 70 | 71 | parser.add_argument( 72 | "-o", 73 | "--output_dir", 74 | # required=True, 75 | default=f"{ASSETS_DIR}/mvl_data/img", 76 | type=str, 77 | help="Output directory for the output_file to be created.", 78 | ) 79 | 80 | args = parser.parse_args() 81 | return args 82 | 83 | 84 | if __name__ == "__main__": 85 | args = get_argparse() 86 | main(args) 87 | -------------------------------------------------------------------------------- /mvl_challenge/mvl_data/load_and_eval_mvl_dataset.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ( 3 | DATA_DIR, 4 | ROOT_DIR, 5 | CFG_DIR, 6 | EPILOG, 7 | ASSETS_DIR, 8 | DEFAULT_MVL_DIR, 9 | SCENE_LIST_DIR, 10 | ) 11 | from mvl_challenge.config.cfg import read_omega_cfg 12 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 13 | import logging 14 | from tqdm import tqdm 15 | from mvl_challenge.utils.vispy_utils import plot_list_ly 16 | from mvl_challenge.utils.image_utils import draw_boundaries_phi_coords 17 | from imageio import imwrite 18 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 19 | import numpy as np 20 | from mvl_challenge.utils.io_utils import ( 21 | create_directory, 22 | get_scene_room_from_scene_room_idx, 23 | ) 24 | from mvl_challenge.utils.image_utils import plot_image 25 | 26 | 27 | def get_cfg_from_args(args): 28 | cfg = read_omega_cfg(args.cfg) 29 | cfg.scene_dir = args.scene_dir 30 | cfg.scene_list = args.scene_list 31 | cfg.ckpt = args.ckpt 32 | cfg.cuda_device = args.cuda_device 33 | return cfg 34 | 35 | 36 | def main(args): 37 | cfg = get_cfg_from_args(args) 38 | mvl = MVLDataset(cfg) 39 | hn = WrapperHorizonNet(cfg) 40 | 41 | for list_ly in iter_mvl_room_scenes(model=hn, dataset=mvl): 42 | for ly in list_ly: 43 | img = ly.get_rgb() 44 | draw_boundaries_phi_coords( 45 | img, phi_coords=np.vstack([ly.phi_coords[0], ly.phi_coords[1]]) 46 | ) 47 | plot_image(image=img, caption=ly.idx) 48 | plot_list_ly(list_ly) 49 | 50 | 51 | def get_argparse(): 52 | desc = ( 53 | "This script loads a MVL dataset given a passed scene directory, scene list and cfg file. " 54 | + "The scene directory is where the MVL data is stored. " 55 | + "The scene list is the list of scene in scene_room_idx format. " 56 | + "The cfg file is the yaml configuration with all hyperparameters set to default values." 57 | ) 58 | 59 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 60 | 61 | parser.add_argument( 62 | "-d", 63 | "--scene_dir", 64 | type=str, 65 | default=f"{DEFAULT_MVL_DIR}", 66 | help="MVL dataset directory.", 67 | ) 68 | 69 | parser.add_argument( 70 | "--cfg", 71 | type=str, 72 | default=f"{CFG_DIR}/eval_mvl_dataset.yaml", 73 | help=f"Config file to load a MVL dataset. (Default: {CFG_DIR}/eval_mvl_dataset.yaml)", 74 | ) 75 | 76 | parser.add_argument( 77 | "-f", 78 | "--scene_list", 79 | type=str, 80 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 81 | help="Scene_list of mvl scenes in scene_room_idx format.", 82 | ) 83 | 84 | parser.add_argument( 85 | "--ckpt", 86 | default=f"{ASSETS_DIR}/ckpt/hn_mp3d.pth", 87 | help="Path to ckpt pretrained model (Default: mp3d)", 88 | ) 89 | 90 | parser.add_argument("--cuda_device", default=0, type=int, help="Cuda device. (Default: 0)") 91 | 92 | args = parser.parse_args() 93 | return args 94 | 95 | 96 | if __name__ == "__main__": 97 | args = get_argparse() 98 | main(args) 99 | -------------------------------------------------------------------------------- /mvl_challenge/utils/eval_utils.py: -------------------------------------------------------------------------------- 1 | from mvl_challenge.utils.spherical_utils import phi_coords2xyz 2 | from shapely.geometry import Polygon 3 | import torch.nn.functional as F 4 | from mvl_challenge.utils.vispy_utils import plot_color_plc 5 | 6 | 7 | def compute_L1_loss(y_est, y_ref): 8 | return F.l1_loss(y_est, y_ref) 9 | 10 | def compute_weighted_L1(y_est, y_ref, std, min_std=1E-2): 11 | return F.l1_loss(y_est/(std + min_std)**2, y_ref/(std + min_std)**2) 12 | 13 | 14 | def eval_2d3d_iuo_from_tensors(est_bon, gt_bon, losses, ch=1.6): 15 | est_bearing_ceiling = phi_coords2xyz(est_bon[:, 0, :].squeeze()) 16 | est_bearing_floor = phi_coords2xyz(est_bon[:, 1, :].squeeze()) 17 | gt_bearing_ceiling = phi_coords2xyz(gt_bon[:, 0, :].squeeze()) 18 | gt_bearing_floor = phi_coords2xyz(gt_bon[:, 1, :].squeeze()) 19 | 20 | iou2d, iou3d = get_2d3d_iou(ch, est_bearing_floor, gt_bearing_floor, est_bearing_ceiling, gt_bearing_ceiling) 21 | losses["2DIoU"].append(iou2d) 22 | losses["3DIoU"].append(iou3d) 23 | 24 | 25 | def eval_2d3d_iuo(phi_coords_est, phi_coords_gt_bon, ch=1): 26 | est_bearing_ceiling = phi_coords2xyz(phi_coords_est[0]) 27 | est_bearing_floor = phi_coords2xyz(phi_coords_est[1]) 28 | gt_bearing_ceiling = phi_coords2xyz(phi_coords_gt_bon[0]) 29 | gt_bearing_floor = phi_coords2xyz(phi_coords_gt_bon[1]) 30 | return get_2d3d_iou(ch, est_bearing_floor, gt_bearing_floor, est_bearing_ceiling, gt_bearing_ceiling) 31 | # Project bearings into a xz plane, ch: camera height 32 | 33 | def get_2d3d_iou(ch, est_bearing_floor, gt_bearing_floor, est_bearing_ceiling, gt_bearing_ceiling): 34 | est_scale_floor = ch / est_bearing_floor[1, :] 35 | est_pcl_floor = est_scale_floor * est_bearing_floor 36 | 37 | gt_scale_floor = ch / gt_bearing_floor[1, :] 38 | gt_pcl_floor = gt_scale_floor * gt_bearing_floor 39 | 40 | # Calculate height 41 | est_scale_ceiling = est_pcl_floor[2] / est_bearing_ceiling[2] 42 | est_pcl_ceiling = est_scale_ceiling * est_bearing_ceiling 43 | est_h = abs(est_pcl_ceiling[1, :].mean() - ch) 44 | 45 | gt_scale_ceiling = gt_pcl_floor[2] / gt_bearing_ceiling[2] 46 | gt_pcl_ceiling = gt_scale_ceiling * gt_bearing_ceiling 47 | gt_h = abs(gt_pcl_ceiling[1, :].mean() - ch) 48 | try: 49 | est_poly = Polygon(zip(est_pcl_floor[0], est_pcl_floor[2])) 50 | gt_poly = Polygon(zip(gt_pcl_floor[0], gt_pcl_floor[2])) 51 | 52 | if not gt_poly.is_valid: 53 | print("[ERROR] Skip ground truth invalid") 54 | return -1, -1 55 | 56 | # 2D IoU 57 | try: 58 | area_dt = est_poly.area 59 | area_gt = gt_poly.area 60 | area_inter = est_poly.intersection(gt_poly).area 61 | iou2d = area_inter / (area_gt + area_dt - area_inter) 62 | except: 63 | iou2d = 0 64 | 65 | # 3D IoU 66 | try: 67 | area3d_inter = area_inter * min(est_h, gt_h) 68 | area3d_pred = area_dt * est_h 69 | area3d_gt = area_gt * gt_h 70 | iou3d = area3d_inter / (area3d_pred + area3d_gt - area3d_inter) 71 | except: 72 | iou3d = 0 73 | except: 74 | iou2d = 0 75 | iou3d = 0 76 | 77 | return iou2d, iou3d 78 | -------------------------------------------------------------------------------- /mvl_challenge/challenge_results/create_zip_results.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ( 3 | DATA_DIR, 4 | ROOT_DIR, 5 | CFG_DIR, 6 | EPILOG, 7 | ASSETS_DIR, 8 | DEFAULT_NPZ_DIR, 9 | SCENE_LIST_DIR, 10 | ) 11 | from mvl_challenge.config.cfg import read_omega_cfg 12 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 13 | from mvl_challenge.utils.vispy_utils import plot_list_ly 14 | from mvl_challenge.utils.image_utils import draw_boundaries_phi_coords 15 | from imageio import imwrite 16 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 17 | from mvl_challenge.utils.io_utils import ( 18 | create_directory, 19 | process_arcname, 20 | get_scene_room_from_scene_room_idx, 21 | save_compressed_phi_coords, 22 | ) 23 | from mvl_challenge.utils.image_utils import plot_image 24 | import numpy as np 25 | import os 26 | from pathlib import Path 27 | from tqdm import tqdm 28 | import zipfile 29 | import json 30 | 31 | 32 | def zip_results(args): 33 | list_fn = os.listdir(args.results_dir) 34 | # ! if scene_list is passed 35 | if os.path.exists(args.scene_list): 36 | data_scene = json.load(open(args.scene_list, "r")) 37 | scene_list = [sc for sc in data_scene.values()] 38 | scene_list = [item for sublist in scene_list for item in sublist] 39 | results_fn = [ 40 | os.path.join(args.results_dir, r) 41 | for r in list_fn 42 | if Path(r).stem in scene_list 43 | ] 44 | else: 45 | results_fn = [os.path.join(args.results_dir, r) for r in list_fn] 46 | 47 | output_dir = Path(args.results_dir).parent 48 | results_name = Path(args.results_dir).stem 49 | zip_results_fn = os.path.join(output_dir, f"{results_name}.zip") 50 | with zipfile.ZipFile(file=zip_results_fn, mode="w") as zf: 51 | list_arc_fn = process_arcname(results_fn, args.results_dir) 52 | [ 53 | ( 54 | print(f"zipping {fn}"), 55 | zf.write( 56 | os.path.join(args.results_dir, fn), 57 | compress_type=zipfile.ZIP_STORED, 58 | arcname=fn, 59 | ), 60 | ) 61 | for fn in tqdm(list_arc_fn) 62 | ] 63 | 64 | 65 | def get_argparse(): 66 | desc = "This script create a results.zip file from a directory of npz files (estimation files), which can be submitted to EvalAi. " 67 | 68 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 69 | 70 | parser.add_argument( 71 | "-d", 72 | "--results_dir", 73 | type=str, 74 | default=f"{DEFAULT_NPZ_DIR}/scene_list__warm_up_pilot_set", 75 | help="Results directory where *.npz files were stored.", 76 | ) 77 | 78 | parser.add_argument( 79 | "-f", 80 | "--scene_list", 81 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 82 | help="JSON scene_list file of mvl scenes in scene_room_idx format. " 83 | + "If no scene_list is passed, then all npz files in results_dir will be used.", 84 | ) 85 | 86 | args = parser.parse_args() 87 | return args 88 | 89 | 90 | if __name__ == "__main__": 91 | args = get_argparse() 92 | zip_results(args) 93 | -------------------------------------------------------------------------------- /tutorial/train_360_mlc/train_mlc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from pathlib import Path 4 | from tqdm import tqdm 5 | import numpy as np 6 | from imageio import imwrite 7 | from mvl_challenge.models.models_utils import load_layout_model 8 | from mvl_challenge.datasets.mvl_dataset import iter_mvl_room_scenes 9 | from mvl_challenge.config.cfg import read_omega_cfg 10 | from mvl_challenge.datasets.mvl_dataset import MVLDataset 11 | from mvl_challenge import ( 12 | ASSETS_DIR, 13 | DEFAULT_MVL_DIR, 14 | SCENE_LIST_DIR, 15 | DEFAULT_TRAINING_DIR 16 | ) 17 | 18 | MLC_TUTORIAL_DIR=os.path.dirname(__file__) 19 | 20 | def get_cfg_from_args(args): 21 | cfg = read_omega_cfg(args.cfg) 22 | if cfg.pass_args: # params in the yaml will be replaced by the passed arguments 23 | cfg.mvl_dir = args.scene_dir 24 | cfg.training_scene_list = args.training_scene_list 25 | cfg.pilot_scene_list = args.pilot_scene_list 26 | cfg.output_dir = args.output_dir 27 | cfg.ckpt = args.ckpt 28 | cfg.cuda_device = args.cuda_device 29 | cfg.id_exp = f"mlc__{Path(cfg.ckpt).stem}__{Path(args.training_scene_list).stem}" 30 | return cfg 31 | 32 | def main(args): 33 | # ! Reading configuration 34 | cfg = get_cfg_from_args(args) 35 | 36 | model = load_layout_model(cfg) 37 | 38 | model.prepare_for_training() 39 | model.set_valid_dataloader() 40 | model.valid_iou_loop() 41 | model.save_current_scores() 42 | while model.is_training: 43 | model.train_loop() 44 | model.valid_iou_loop() 45 | model.save_current_scores() 46 | 47 | def get_passed_args(): 48 | parser = argparse.ArgumentParser() 49 | 50 | default_cfg = f"{MLC_TUTORIAL_DIR}/train_mlc.yaml" 51 | parser.add_argument( 52 | '--cfg', 53 | default=default_cfg, 54 | help=f'Config File. Default {default_cfg}') 55 | 56 | parser.add_argument( 57 | "--training_scene_list", 58 | type=str, 59 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_training_set.json", 60 | help="Training scene list.", 61 | ) 62 | 63 | parser.add_argument( 64 | "--pilot_scene_list", 65 | type=str, 66 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 67 | help="Pilot scene list", 68 | ) 69 | 70 | parser.add_argument( 71 | "-o", 72 | "--output_dir", 73 | type=str, 74 | default=f"{DEFAULT_TRAINING_DIR}", 75 | help="MVL dataset directory.", 76 | ) 77 | 78 | parser.add_argument( 79 | "-d", 80 | "--scene_dir", 81 | type=str, 82 | default=f"{DEFAULT_MVL_DIR}", 83 | help="MVL dataset directory.", 84 | ) 85 | 86 | parser.add_argument( 87 | "--ckpt", 88 | default=f"{ASSETS_DIR}/ckpt/hn_mp3d.pth", 89 | help="Path to ckpt pretrained model (Default: mp3d)", 90 | ) 91 | 92 | parser.add_argument("--cuda_device", default=0, type=int, help="Cuda device. (Default: 0)") 93 | 94 | args = parser.parse_args() 95 | return args 96 | 97 | if __name__ == "__main__": 98 | args = get_passed_args() 99 | main(args) 100 | -------------------------------------------------------------------------------- /check_scene_list.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import json 4 | import numpy as np 5 | from mvl_challenge import EPILOG, ASSETS_DIR, DEFAULT_MVL_DIR, SCENE_LIST_DIR 6 | 7 | 8 | def main(args): 9 | data = json.load(open(args.scene_list, "r")) 10 | list_frames = [f for f in data.values()] 11 | list_frames = [item for sublist in list_frames for item in sublist] 12 | 13 | list_imgs = [os.path.join(args.data_dir, "img", f"{fn}.jpg") for fn in list_frames] 14 | list_geom_info = [ 15 | os.path.join(args.data_dir, "geometry_info", f"{fn}.json") for fn in list_frames 16 | ] 17 | list_labels = [ 18 | os.path.join(args.data_dir, "labels", "gt", f"{fn}.npz") for fn in list_frames 19 | ] 20 | 21 | print(f" - Scene list: {args.scene_list}") 22 | print(f"* Total rooms: {list(data.keys()).__len__()}") 23 | print(f"* Total frames: {list_frames.__len__()}") 24 | if np.sum([os.path.isfile(fn) for fn in list_imgs]) == list_imgs.__len__(): 25 | print(f" - [PASSED]\tAll images were found") 26 | else: 27 | if not args.v: 28 | print(f" - [FAILED]\tNot all images were found") 29 | else: 30 | [ 31 | print(f" - [FAILED]\tNot found {fn}") 32 | for fn in list_imgs 33 | if not os.path.exists(fn) 34 | ] 35 | 36 | if ( 37 | np.sum([os.path.isfile(fn) for fn in list_geom_info]) 38 | == list_geom_info.__len__() 39 | ): 40 | print(f" - [PASSED]\tAll JSON geometry files were found") 41 | else: 42 | if not args.v: 43 | print(f" - [FAILED]\tNot all JSON geometry files were found") 44 | else: 45 | [ 46 | print(f" - [FAILED]\tNot found {fn}") 47 | for fn in list_geom_info 48 | if not os.path.exists(fn) 49 | ] 50 | 51 | if np.sum([os.path.isfile(fn) for fn in list_labels]) == list_labels.__len__(): 52 | print(f" - [PASSED]\tAll labels *.npz files were found") 53 | else: 54 | if not args.v: 55 | print(f" - [FAILED]\tNot all labels *.npz files were found") 56 | else: 57 | [ 58 | print(f" - [FAILED]\tNot found {fn}") 59 | for fn in list_labels 60 | if not os.path.exists(fn) 61 | ] 62 | 63 | 64 | def get_argparse(): 65 | desc = "This script checks the passed scene_list ensuring that all mvl-data defined at data_dir can be accessed." 66 | 67 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 68 | 69 | parser.add_argument( 70 | "-d", 71 | "--data_dir", 72 | default=f"{DEFAULT_MVL_DIR}", 73 | type=str, 74 | help=f"Output directory by default it will store at {DEFAULT_MVL_DIR}.", 75 | ) 76 | 77 | parser.add_argument( 78 | "-f", 79 | "--scene_list", 80 | type=str, 81 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 82 | help=f"Scene list as JSON file. See {SCENE_LIST_DIR}.", 83 | ) 84 | 85 | parser.add_argument("-v", action="store_true", help=f"Explicit list of failures.") 86 | 87 | args = parser.parse_args() 88 | return args 89 | 90 | 91 | if __name__ == "__main__": 92 | args = get_argparse() 93 | main(args) 94 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/utils/camera_height_utils.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import numpy as np 4 | 5 | import matplotlib.pyplot as plt 6 | 7 | import logging 8 | from mvl_challenge.datasets.rgbd_datasets import RGBD_Dataset 9 | from mvl_challenge.utils.vispy_utils import plot_color_plc 10 | from mvl_challenge.utils.geometry_utils import extend_array_to_homogeneous 11 | from mvl_challenge import CFG_DIR, ASSETS_DIR 12 | import pyransac3d as pyrsc 13 | from tqdm import tqdm 14 | from mvl_challenge.config.cfg import read_omega_cfg 15 | 16 | 17 | def get_masked_pcl(cfg, list_fr): 18 | 19 | # Skipped the first camera frame to avoid noise 20 | pcl = [fr.get_pcl() for fr in list_fr[1:10]] 21 | if pcl.__len__() == 0: 22 | logging.error("Error reading point cloud") 23 | return None, None 24 | pcl = np.hstack(pcl) 25 | color = pcl[3:, :] 26 | pcl = np.linalg.inv(list_fr[0].pose)[:3, :] @ extend_array_to_homogeneous( 27 | pcl[:3, :] 28 | ) 29 | # masking around the first camera frame 30 | mask = np.linalg.norm(pcl[(0, 2), :], axis=0) < cfg.xz_radius 31 | pcl = pcl[:, mask] 32 | color = color[:, mask] 33 | # Masking the floor 34 | mask = pcl[1, :] > cfg.min_height 35 | pcl = pcl[:, mask] 36 | color = color[:, mask] 37 | # plot_color_plc(points=pcl[0:3, :].T, color=color.T) 38 | 39 | idx = np.linspace(0, pcl.shape[1] - 1, pcl.shape[1]).astype(np.int32) 40 | np.random.shuffle(idx) 41 | return pcl[:3, idx[: cfg.min_samples]], color[:, idx[: cfg.min_samples]] 42 | 43 | 44 | def estimate_camera_height(cfg, list_fr): 45 | cam_h_hyp = [] 46 | # for _ in tqdm(range(cfg.iter), desc="Iter camera Height..."): 47 | # iter_progress = tqdm(cfg.iter, desc="Iter camera Height...") 48 | iteration = 0 49 | logging.info(f"Estimating camera hight {list_fr[0].room_name}") 50 | while True: 51 | np.random.shuffle(list_fr) 52 | logging.info(f"Cam-h estimation trial: {iteration+1}") 53 | logging.info(f"Number fr: {list_fr.__len__()}") 54 | 55 | pcl, color = get_masked_pcl(cfg, list_fr) 56 | if pcl is None: 57 | continue 58 | if pcl.size == 0: 59 | continue 60 | try: 61 | pln = pyrsc.Plane() 62 | best_eq, best_inliers = pln.fit(pcl.T, cfg.fit_error) 63 | except: 64 | logging.warning("Something went wrong with RANSAC plane estimation.") 65 | continue 66 | # plot_color_plc(points=pcl[0:3, best_inliers].T, color=color[:, best_inliers].T) 67 | 68 | # ! since each camera fr does not have the same height 69 | cam_h2room = np.abs(best_eq[-1]) + list_fr[0].pose[1, 3] 70 | cam_h_hyp.append(cam_h2room) 71 | iteration += 1 72 | # iter_progress.update(iteration) 73 | 74 | if cfg.iter == iteration: 75 | break 76 | return np.median(cam_h_hyp) 77 | 78 | 79 | def main(args): 80 | dt = RGBD_Dataset.from_args(args) 81 | list_fr = dt.get_list_frames() 82 | # ! cfg file for camera height estimation 83 | cfg_cam = read_omega_cfg(args.cfg) 84 | h = estimate_camera_height(cfg_cam, list_fr) 85 | logging.info(f"Estimated camera height: {h}") 86 | 87 | 88 | def get_args(): 89 | parser = argparse.ArgumentParser() 90 | 91 | parser.add_argument( 92 | "--scene_dir", 93 | # required=True, 94 | default="/media/public_dataset/MP3D_360_FPE/SINGLE_ROOM_SCENES/2t7WUuJeko7/0/", 95 | type=str, 96 | help="Directory of all scene in the dataset", 97 | ) 98 | 99 | parser.add_argument( 100 | "--cfg", 101 | default=f"{CFG_DIR}/camera_height.yaml", 102 | help="Cfg tp compute camera height", 103 | ) 104 | 105 | args = parser.parse_args() 106 | return args 107 | 108 | 109 | if __name__ == "__main__": 110 | args = get_args() 111 | main(args) 112 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/create_geometry_info_files.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import CFG_DIR, ASSETS_DIR, EPILOG 3 | import os 4 | from mvl_challenge.config.cfg import read_omega_cfg 5 | from mvl_challenge.utils.io_utils import get_rgbd_scenes_list 6 | from mvl_challenge.datasets.rgbd_datasets import RGBD_Dataset 7 | from mvl_challenge.pre_processing.utils.geometry_info_utils import get_geometry_info 8 | from mvl_challenge.utils.io_utils import save_json_dict, create_directory 9 | from tqdm import tqdm 10 | import logging 11 | 12 | 13 | def save_geometry_info(cfg, data_dict): 14 | for dt in data_dict: 15 | scene_name = list(dt.keys())[0] 16 | geom_info = dt[scene_name] 17 | fn = os.path.join(cfg.output_dir, f"{scene_name}.json") 18 | save_json_dict(fn, geom_info) 19 | 20 | 21 | def compute_and_save_geometry_info(cfg): 22 | logging.info(f"Saving geometric info for {cfg.dataset.scene_dir}") 23 | try: 24 | dt = RGBD_Dataset.from_cfg(cfg) 25 | except Exception as err: 26 | print(type(err)) # the exception instance 27 | print(err.args) # arguments stored in .args 28 | print(err) 29 | return 30 | # raise ValueError 31 | 32 | geom_info = get_geometry_info(dt) 33 | #! save geometry info 34 | [save_geometry_info(cfg, data) for data in geom_info] 35 | 36 | 37 | def main(args): 38 | cfg = get_cfg(args) 39 | # ! Getting all scenes path define the passed scene_dir 40 | # list_scenes_dir = get_files_given_a_pattern( 41 | # args.scene_dir, flag_file="frm_ref.txt", exclude=["rgb", 'depth']) 42 | list_scenes_dir = get_rgbd_scenes_list(args) 43 | 44 | assert list_scenes_dir.__len__() > 0, "No scenes were found at {args.scene_dir} " 45 | 46 | create_directory(cfg.output_dir, delete_prev=False) 47 | 48 | for scene_dir in tqdm(list_scenes_dir, desc="list scenes..."): 49 | cfg.dataset.scene_dir = scene_dir 50 | cfg.dataset.scene_list = args.scene_list 51 | compute_and_save_geometry_info(cfg) 52 | 53 | 54 | def get_cfg(args): 55 | cfg = read_omega_cfg(args.cfg) 56 | cfg.dataset = dict() 57 | cfg.dataset.scene_dir = args.scene_dir 58 | cfg.dataset.scene_list = args.scene_list 59 | cfg.output_dir = args.output_dir 60 | return cfg 61 | 62 | 63 | def get_argparse(): 64 | desc = ( 65 | "This script computes the geometry information per frame from a scene_list in scene_room_idx format. " 66 | + "The geometry info is the geometrical information for each frame, i.e., camera pose and camera height." 67 | ) 68 | 69 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 70 | 71 | parser.add_argument( 72 | "-d", 73 | "--scene_dir", 74 | # required=True, 75 | default="/media/public_dataset/MP3D_360_FPE/MULTI_ROOM_SCENES/", 76 | # default="/media/public_dataset/HM3D-MVL/test/", 77 | # default=None, 78 | type=str, 79 | help="RGBD dataset directory.", 80 | ) 81 | 82 | parser.add_argument( 83 | "-f", 84 | "--scene_list", 85 | # required=True, 86 | default=f"{ASSETS_DIR}/issue_omitted_frames/test_scene_list.json", 87 | type=str, 88 | help="Scene list file which contents all frames encoded in scene_room_idx format.", 89 | ) 90 | 91 | parser.add_argument( 92 | "-o", 93 | "--output_dir", 94 | # required=True, 95 | default=f"{ASSETS_DIR}/issue_omitted_frames/geometry_info", 96 | type=str, 97 | help="Output directory for the output_file to be created.", 98 | ) 99 | 100 | parser.add_argument( 101 | "--cfg", 102 | default=f"{CFG_DIR}/camera_height.yaml", 103 | help=f"Hypermeter cfg (default: {CFG_DIR}/camera_height.yaml)", 104 | ) 105 | 106 | args = parser.parse_args() 107 | return args 108 | 109 | 110 | if __name__ == "__main__": 111 | args = get_argparse() 112 | main(args) 113 | -------------------------------------------------------------------------------- /mvl_challenge/utils/image_utils.py: -------------------------------------------------------------------------------- 1 | from imageio import imread 2 | import numpy as np 3 | from mvl_challenge.utils.spherical_utils import phi_coords2xyz, xyz2uv, phi_coords2uv 4 | import matplotlib.pyplot as plt 5 | from PIL import Image 6 | from PIL import ImageDraw 7 | from PIL import ImageFont 8 | import skimage.filters 9 | 10 | COLOR_GREEN = (0, 255, 0) 11 | COLOR_MAGENTA = (255, 0, 255) 12 | COLOR_CYAN = (0, 255, 255) 13 | 14 | 15 | def get_color_array(color_map): 16 | """ 17 | returns an array (3, n) of the colors in image (H, W) 18 | """ 19 | # ! This is the same solution by flatten every channel 20 | if len(color_map.shape) > 2: 21 | return np.vstack( 22 | ( 23 | color_map[:, :, 0].flatten(), 24 | color_map[:, :, 1].flatten(), 25 | color_map[:, :, 2].flatten(), 26 | ) 27 | ) 28 | else: 29 | return np.vstack( 30 | (color_map.flatten(), color_map.flatten(), color_map.flatten()) 31 | ) 32 | 33 | 34 | def load_depth_map(fpath): 35 | """Make sure the depth map has shape (H, W) but not (H, W, 1).""" 36 | depth_map = imread(fpath) 37 | if depth_map.shape[-1] == 1: 38 | depth_map = depth_map.squeeze(-1) 39 | return depth_map 40 | 41 | 42 | def draw_boundaries_uv(image, boundary_uv, color=(0, 255, 0), size=2): 43 | if image.shape.__len__() == 3: 44 | for i in range(size): 45 | image[(boundary_uv[1] + i) % image.shape[0], boundary_uv[0], :] = np.array( 46 | color 47 | ) 48 | # image[(boundary_uv[1]-i)% 0, boundary_uv[0], :] = np.array(color) 49 | else: 50 | for i in range(size): 51 | image[(boundary_uv[1] + i) % image.shape[0], boundary_uv[0]] = 255 52 | # image[(boundary_uv[1]-i)% 0, boundary_uv[0]] = 255 53 | 54 | return image 55 | 56 | 57 | def draw_boundaries_phi_coords(image, phi_coords, color=(0, 255, 0), size=2): 58 | 59 | # ! Compute bearings 60 | bearings_ceiling = phi_coords2xyz(phi_coords=phi_coords[0, :]) 61 | bearings_floor = phi_coords2xyz(phi_coords=phi_coords[1, :]) 62 | 63 | uv_ceiling = xyz2uv(bearings_ceiling) 64 | uv_floor = xyz2uv(bearings_floor) 65 | 66 | draw_boundaries_uv(image=image, boundary_uv=uv_ceiling, color=color, size=size) 67 | draw_boundaries_uv(image=image, boundary_uv=uv_floor, color=color, size=size) 68 | 69 | return image 70 | 71 | 72 | def draw_boundaries_xyz(image, xyz, color=(0, 255, 0), size=2): 73 | uv = xyz2uv(xyz) 74 | draw_boundaries_uv(image, uv, color, size) 75 | 76 | 77 | def add_caption_to_image(image, caption): 78 | img_obj = Image.fromarray(image) 79 | img_draw = ImageDraw.Draw(img_obj) 80 | font_obj = ImageFont.truetype("FreeMono.ttf", 20) 81 | img_draw.text((20, 20), f"{caption}", font=font_obj, fill=(255, 0, 0)) 82 | return np.array(img_obj) 83 | 84 | 85 | def plot_image(image, caption, figure=0): 86 | plt.figure(figure) 87 | plt.clf() 88 | image = add_caption_to_image(image, caption) 89 | plt.imshow(image) 90 | plt.draw() 91 | plt.waitforbuttonpress(0.01) 92 | 93 | 94 | def draw_uncertainty_map(sigma_boundary, peak_boundary, shape=(512, 1024)): 95 | H, W = shape 96 | img_map = np.zeros((H, W)) 97 | for u,v, sigma in zip(peak_boundary[0, :], peak_boundary[1, :], sigma_boundary): 98 | sigma_bon = (sigma / np.pi) * shape[0] 99 | 100 | sampled_points = np.random.normal(v, sigma_bon, 512).astype(np.int16) 101 | sampled_points[sampled_points >= H] = H-1 102 | sampled_points[sampled_points <= 0] = 0 103 | 104 | u_point = (np.ones_like(sampled_points) * u).astype(np.int16) 105 | img_map[sampled_points, u_point] = 1 106 | 107 | img_map = skimage.filters.gaussian( 108 | img_map, sigma=(15, 5), 109 | truncate=5, 110 | channel_axis=True) 111 | 112 | img_map = img_map/img_map.max() 113 | return img_map -------------------------------------------------------------------------------- /tutorial/train_360_mlc/README.md: -------------------------------------------------------------------------------- 1 | ## Tutorial 2: Use 360-MLC for self-training 2 | 3 | In this tutorial, we will show you how to leverage multi-view consistency property, to create pseudo-labels and self-train a model in the dataset without any annotation. We proposed this method, [360-MLC](https://enriquesolarte.github.io/360-mlc/), in NeurIPS 2022, and the code is available [here](https://github.com/EnriqueSolarte/360-mlc). Hope this example can give you an idea of how to tackle the problem in this challenge. 4 | 5 | ### Step 1: Download 360-MLC 6 | ```bash 7 | # Under your target directory 8 | git clone --recurse-submodules git@github.com:EnriqueSolarte/360-mlc.git 9 | 10 | cd 360-mlc 11 | 12 | # Install python dependencies 13 | pip install -r requirements.txt 14 | 15 | # Install MLC library 16 | pip install . 17 | ``` 18 | 19 | ### Step 2: Create MLC pseudo labels 20 | 21 | We will use a pre-trained model, [HorizonNet](https://github.com/sunset1995/HorizonNet) for example, to predict the layouts of all the images in the same room. By using the technique described in [360-MLC](https://enriquesolarte.github.io/360-mlc/), we can then create the pseudo labels of each frame. 22 | 23 | You can find all controlable settings and hyperparameters in `create_mlc_labels.yaml`, and you have two options: 24 | 25 | 1. Set `pass_args` as True (default): you can set the parameters by passing the arguments through command line. See `get_passed_args()` method in `create_mlc_labels.py` for the default values. 26 | 27 | 2. Set `pass_args` as False: you will completely use the parameters set in the yaml file, and the ones marked with `` must be filled. 28 | 29 | To create MLC pseudo labels, run the next command: 30 | ```bash 31 | # use -h for more details 32 | python create_mlc_labels.py 33 | # or 34 | python create_mlc_labels.py -f $SCENE_LIST -o $OUTPUT_DIR -ckpt $CHECK_POINT 35 | ``` 36 | 37 | After the pseudo labels are created, you can find them in `$OUTPUT_DIR`. By default, this directory is defined at `mvl_challenge/assets/data/mvl_data/labels/mlc__{ckpt}__scene_list__{SCENE_LIST}/`. 38 | 39 | There will be three folders under it: 40 | 1. `mlc_label`: pseudo labels (phi coordinates) in npy format 41 | 2. `mlc_vis`: pseudo labels visualization 42 | 3. `std`: standard deviation of pseudo labels 43 | 44 | ### Step 3: Self-train the model using MLC pseudo labels 45 | 46 | By Using the pseudo labels created in [Step 2](#step-2-create-mlc-pseudo-labels), we can self-train the model in the similar way as in [Tutorial 1](https://github.com/mvlchallenge/mvl_toolkit/tree/mvl_chellenge_dev/tutorial/train_horizon_net), where the model is trained using GT labels. 47 | 48 | You can also find all controlable settings and hyperparameters in `train_mlc.yaml`, and the usage is the same as in [Step 2](#step-2-create-mlc-pseudo-labels). 49 | 50 | ```bash 51 | # use -h for more details 52 | python train_mlc.py 53 | # or 54 | python train_mlc.py --training_scene_list $SCENE_LIST -o $OUTPUT_DIR -ckpt $CHECK_POINT 55 | ``` 56 | 57 | By default, the model will be trained on the pseudo labels created from the training split, and validated on the pilot split, since the pilot split has GT labels for you to do IoU evaluation. 58 | 59 | At the same time, you can find the the training result in `mvl_challenge/assets/data/mvl_training_results/mlc_{ckpt}__scene_list__{split}/` 60 | 61 | The structure will be: 62 | ``` 63 | └── mlc_{ckpt}__scene_list__{split}/ 64 |    ├── ckpt/ 65 | │ └── best_score.json 66 | │ └── cfg.yaml 67 | │ └── valid_evel_{epoch}.json 68 |    └── log/ 69 | └── tensorboard event file 70 | ``` 71 | 72 | `cfg.yaml` summarizes all the parameters that are used in the whole training process. 73 | 74 | `valid_evel_{epoch}.json` shows the model evaluations on the pilot split, epoch 0 means the original pre-trained model, and the self-training will start from epoch 1. 75 | 76 | `log directory` can let you visualize the training status using tensorboard. 77 | 78 | Run the following command to visualize the training status: 79 | ```bash 80 | # Using default directory 81 | LOG_DIR="mvl_challenge/assets/data/mvl_training_results/mlc__hn_mp3d__scene_list__warm_up_training_set/log" 82 | tensorboard --logdir $LOG_DIR 83 | ``` -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/utils/geometry_info_utils.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge.config.cfg import read_omega_cfg 3 | from mvl_challenge import ASSETS_DIR, CFG_DIR, EPILOG 4 | from mvl_challenge.datasets.rgbd_datasets import MP3D_FPE, RGBD_Dataset 5 | from mvl_challenge.pre_processing.utils.camera_height_utils import ( 6 | estimate_camera_height, 7 | ) 8 | from mvl_challenge.pre_processing.utils.camera_height_per_rooms import ( 9 | estimate_cam_height_per_room, 10 | ) 11 | from mvl_challenge.utils.io_utils import get_idx_from_scene_room_idx 12 | from mvl_challenge.utils.geometry_utils import get_quaternion_from_matrix 13 | import logging 14 | import os 15 | import json 16 | 17 | 18 | def get_geometry_info(dt: RGBD_Dataset): 19 | # ! Estimate camera height per room 20 | cam_h_dict = estimate_cam_height_per_room(dt.cfg.cam_height_cfg, dt) 21 | 22 | # ! List all frames in the scene 23 | scenes_room_fn = dt.cfg.dataset.scene_list 24 | scenes_room = json.load(open(scenes_room_fn, "r")) 25 | 26 | # ! list of Frames defined (defined in wc) 27 | list_fr = dt.get_list_frames() 28 | 29 | # ! Loop for every room identified 30 | list_geom_info = [] 31 | for room_name in cam_h_dict.keys(): 32 | 33 | # ! Cam height in WC 34 | cam_h_wc = cam_h_dict[room_name]["cam_h_wc"] 35 | 36 | # ! List fr defined in this room 37 | list_fr_names = scenes_room[room_name] 38 | list_idx = sorted( 39 | [get_idx_from_scene_room_idx(fr_names) for fr_names in list_fr_names] 40 | ) 41 | list_fr_room = sorted( 42 | [fr for fr in list_fr if fr.idx in list_idx], key=lambda x: x.idx 43 | ) 44 | 45 | # ! List geometry info per fr 46 | list_geom_info.append( 47 | [ 48 | { 49 | f"{room_name}_{fr.idx}": { 50 | "translation": [t for t in fr.pose[:3, 3]], 51 | "quaternion": [q for q in get_quaternion_from_matrix(fr.pose)], 52 | "cam_h": cam_h_wc - fr.pose[1, 3], 53 | } 54 | } 55 | for fr in list_fr_room 56 | ] 57 | ) 58 | 59 | return list_geom_info 60 | 61 | 62 | def main(args): 63 | cfg = read_omega_cfg(args.cfg) 64 | cfg.dataset = dict() 65 | cfg.dataset.scene_dir = args.scene_dir 66 | dt = RGBD_Dataset.from_cfg(cfg) 67 | geo_info_dict = get_geometry_info(dt) 68 | 69 | 70 | def get_argparse(): 71 | desc = ( 72 | "This script computes the geometry information per frame from a given RGBD dataset. " 73 | + "The geometry info is defined as the geometrical information which define each frame , i.e., camera pose and camera height." 74 | ) 75 | 76 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 77 | 78 | parser.add_argument( 79 | "-d", 80 | "--scene_dir", 81 | # required=True, 82 | default="/media/public_dataset/MP3D_360_FPE/MULTI_ROOM_SCENES/", 83 | # default="/media/public_dataset/HM3D-MVL/test/BHXhpBwSMLh", 84 | type=str, 85 | help="RGBD dataset directory.", 86 | ) 87 | 88 | parser.add_argument( 89 | "-f", 90 | "--scene_list", 91 | # required=True, 92 | default=f"{ASSETS_DIR}/mvl_data/mp3d_fpe_scenes.json", 93 | type=str, 94 | help="Scene list file which contents all frames encoded in scene_room_idx format.", 95 | ) 96 | 97 | parser.add_argument( 98 | "-o", 99 | "--output_dir", 100 | # required=True, 101 | default=f"{ASSETS_DIR}/mvl_data/geometry_info", 102 | type=str, 103 | help="Output directory for the output_file to be created.", 104 | ) 105 | 106 | parser.add_argument( 107 | "--cfg", 108 | default=f"{CFG_DIR}/camera_height.yaml", 109 | help=f"Hypermeter cfg (default: {CFG_DIR}/camera_height.yaml )", 110 | ) 111 | 112 | args = parser.parse_args() 113 | return args 114 | 115 | 116 | if __name__ == "__main__": 117 | args = get_argparse() 118 | main(args) 119 | -------------------------------------------------------------------------------- /mvl_challenge/data/gdrive_files/gdrive_ids__warm_up_testing_set.csv: -------------------------------------------------------------------------------- 1 | 1kQJcyl0P6OLs484enjfOy461EcIpYqJN 6D36GQHuP8H_0_room0.zip bin 6.5 MB 2023-03-19 00:48:45 2 | 1-nA3V4M3U2p0yTuzYmxYGunSjVxFlIaW wcojb4TFT35_3_room0.zip bin 3.3 MB 2023-03-19 00:48:19 3 | 1lcVK0I-JMWKecqU-OUX0DRw3Pd4dgcUv wcojb4TFT35_2_room0.zip bin 3.9 MB 2023-03-19 00:48:17 4 | 1MkqvLHmlBT8NYwwwTyrLp58e_qciMcVm wcojb4TFT35_1_room0.zip bin 3.3 MB 2023-03-19 00:48:15 5 | 1s0ByEeEIjDYW0DyeZRAlhbhTbHmDBRrC scene_list__hm3d_mv...testing_split.json bin 29.2 KB 2023-03-19 00:48:12 6 | 1g-iutJHg1kNGRybaX7yCwjlz6nQPxVqf rJhMRvNn4DS_1_room0.zip bin 6.3 MB 2023-03-19 00:48:10 7 | 1_iBjNeuHTpfHdMzwVhJetZk3kwOueaTQ rJhMRvNn4DS_0_room0.zip bin 6.8 MB 2023-03-19 00:48:08 8 | 1BX4E936Vez8HRV8RXokRfhrFkF2FPhfT q3hn1WQ12rz_5_room0.zip bin 3.7 MB 2023-03-19 00:48:06 9 | 1zLJdQ2dS2FyuIXT3D4B9KT-7XXe84U68 mL8ThkuaVTM_0_room0.zip bin 6.4 MB 2023-03-19 00:48:03 10 | 1uJ6kA95ATuUyDewO3EEMripOGHCDfHWZ k1cupFYWXJ6_1_room0.zip bin 4.5 MB 2023-03-19 00:48:01 11 | 1UksPCe_dsBqGwXzA12CpNBS3PxF3xV-U h1zeeAwLh9Z_2_room0.zip bin 4.4 MB 2023-03-19 00:47:58 12 | 1r25AmttLh7DWGEHjH5UWpCWMeCyc8Wko eF36g7L6Z9M_0_room0.zip bin 24.0 MB 2023-03-19 00:47:56 13 | 1ohJjHQpWW0qCF31LS1ssde31YBrsSb0v BHXhpBwSMLh_2_room0.zip bin 3.0 MB 2023-03-19 00:47:53 14 | 15duemhk2fb_sOAIbZxdUPxCWSYXy1Bel BHXhpBwSMLh_10_room0.zip bin 1.4 MB 2023-03-19 00:47:51 15 | 1NvW-l6RMW_aHf0sKJiDleoOLfB_Vm2l0 6D36GQHuP8H_0_room0.zip bin 6.5 MB 2023-03-19 00:47:48 16 | 1fcuKWcgKqno_GW3Myg33wAozZusJNK2_ scene_list__mp3d_fp...p_testing_set.json bin 33.4 KB 2023-03-19 00:47:26 17 | 119aZ-Yc_P_ky8y5y4cFQNoINmlQetPgf q9vSo1VnCiC_0_room20.zip bin 2.0 MB 2023-03-19 00:47:24 18 | 1x-U_Lp0fE8xOQEI1gF3V3ZSV17xwBd49 q9vSo1VnCiC_0_room14.zip bin 446.2 KB 2023-03-19 00:47:22 19 | 1f0HIMybDppdLbev_4tAo0LGn8X6eO5Rm q9vSo1VnCiC_0_room12.zip bin 763.7 KB 2023-03-19 00:47:20 20 | 17caUOFjslKKrM6kJzQgAp5aGwuFpe9kC pa4otMbVnkk_0_room7.zip bin 1.2 MB 2023-03-19 00:47:18 21 | 18uE5nfcqtdkUw_xjQDH5OBJkoBnK_2YJ pa4otMbVnkk_0_room5.zip bin 5.9 MB 2023-03-19 00:47:16 22 | 14RNREqL7H0wvB8vgBKCwMO-Wq6Kygquw pa4otMbVnkk_0_room17.zip bin 4.6 MB 2023-03-19 00:47:13 23 | 1MR1UFbDjNm2uumTovTwfz0VxHJtCG695 pa4otMbVnkk_0_room0.zip bin 6.7 MB 2023-03-19 00:47:11 24 | 11umbveOFbqJFamylCb-S4P6x0bbRl0t3 pRbA3pwrgk9_2_room3.zip bin 1.7 MB 2023-03-19 00:47:08 25 | 1hc4TiG8eH2Eqi9mwrd1shbbo0mCqBDML jtcxE69GiFV_0_room8.zip bin 3.3 MB 2023-03-19 00:47:06 26 | 1FT4rbV7f5QpQaAaoQYgqoO8hCqCcTBlS jtcxE69GiFV_0_room6.zip bin 4.2 MB 2023-03-19 00:47:04 27 | 1gIb-45fIgA2wfa3TO56BJpbGyVTvWLSC jtcxE69GiFV_0_room5.zip bin 5.2 MB 2023-03-19 00:47:01 28 | 1Fmre9tBxW9lTap1xXKDYOglev3MIXgiI jtcxE69GiFV_0_room3.zip bin 5.5 MB 2023-03-19 00:46:59 29 | 19D64eWmiMfcxQId6OzGZOloyqlh37P4x jtcxE69GiFV_0_room2.zip bin 4.3 MB 2023-03-19 00:46:56 30 | 11HU7ZUg5B9ujsjc1_K_omtKzjXaxZJ0x jtcxE69GiFV_0_room10.zip bin 1.4 MB 2023-03-19 00:46:54 31 | 11k8p_WuNsh1bLML4o1G2vBcnnXBl2xEO e9zR4mvMWw7_0_room0.zip bin 5.4 MB 2023-03-19 00:46:52 32 | 1Zs6zwjYAouVUWGFla-XZyfq5nx4u6896 VFuaQ6m2Qom_0_room2.zip bin 2.3 MB 2023-03-19 00:46:50 33 | 1gIjH-fMTg-WqbxoSsdJKR3e44AobADbg JeFG25nYj2p_0_room7.zip bin 9.4 MB 2023-03-19 00:46:47 34 | 1iuazMVes4WP0TyIdqV4j3_Q2Cg-CqlVL JeFG25nYj2p_0_room6.zip bin 2.2 MB 2023-03-19 00:46:45 35 | 1tDlO0ZpCfaV0_cf2IWA4ZBWFNtTryvty 7y3sRwLe3Va_0_room9.zip bin 2.7 MB 2023-03-19 00:46:42 36 | 16mIJTRrf4z3uimqxkvAB94KCHK4oxjOH 2t7WUuJeko7_1_room5.zip bin 2.2 MB 2023-03-19 00:46:40 37 | -------------------------------------------------------------------------------- /mvl_challenge/remote_data/zip_mvl_dataset.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import EPILOG, CFG_DIR, ASSETS_DIR, DATA_DIR 3 | from mvl_challenge.utils.check_utils import check_mvl_dataset 4 | from mvl_challenge.utils.io_utils import create_directory, process_arcname 5 | import json 6 | import zipfile 7 | import numpy as np 8 | import os 9 | from glob import glob 10 | from tqdm import tqdm 11 | from shutil import copyfile 12 | from pathlib import Path 13 | 14 | 15 | def zip_mvl_data(args): 16 | #! Get scene list 17 | data_scene, ret = check_mvl_dataset(args) 18 | assert ret, "Failed checking MVL dataset." 19 | 20 | create_directory(args.output_dir, delete_prev=False) 21 | for room, scene_list in data_scene.items(): 22 | zip_filename = os.path.join(args.output_dir, f"{room}.zip") 23 | with zipfile.ZipFile(file=zip_filename, mode="w") as zf: 24 | geo_info_fn = [ 25 | os.path.join(args.scene_dir, "geometry_info", f"{sc}.json") 26 | for sc in scene_list 27 | ] 28 | img_fn = [ 29 | os.path.join(args.scene_dir, "img", f"{sc}.jpg") for sc in scene_list 30 | ] 31 | 32 | zip_data(args, zf, geo_info_fn) 33 | zip_data(args, zf, img_fn) 34 | 35 | copyfile( 36 | args.scene_list, 37 | os.path.join(args.output_dir, os.path.split(args.scene_list)[-1]), 38 | ) 39 | 40 | 41 | def zip_mvl_labels(args): 42 | output_dir = create_directory(args.output_dir, delete_prev=True) 43 | data_scene = json.load(open(args.scene_list, "r")) 44 | for room, scene_list in data_scene.items(): 45 | 46 | gt_label = [ 47 | os.path.join(args.scene_dir, "labels", "gt", f"{sc}.npz") 48 | for sc in scene_list 49 | ] 50 | gt_label_vis = [ 51 | os.path.join(args.scene_dir, "labels", "gt_vis", f"{sc}.jpg") 52 | for sc in scene_list 53 | ] 54 | 55 | if np.sum([os.path.exists(fn) for fn in gt_label]) != gt_label.__len__(): 56 | continue 57 | 58 | zip_filename = os.path.join(output_dir, f"{room}.zip") 59 | with zipfile.ZipFile(file=zip_filename, mode="w") as zf: 60 | zip_data(args, zf, gt_label) 61 | zip_data(args, zf, gt_label_vis) 62 | 63 | copyfile( 64 | args.scene_list, 65 | os.path.join(args.output_dir, os.path.split(args.scene_list)[-1]), 66 | ) 67 | 68 | 69 | def zip_data(args, zf, geo_info_fn): 70 | list_arc_fn = process_arcname(geo_info_fn, args.scene_dir) 71 | [ 72 | ( 73 | print(f"zipping {fn}"), 74 | zf.write( 75 | os.path.join(args.scene_dir, fn), 76 | compress_type=zipfile.ZIP_STORED, 77 | arcname=fn, 78 | ), 79 | ) 80 | for fn in tqdm(list_arc_fn) 81 | ] 82 | 83 | 84 | def get_argparse(): 85 | desc = ( 86 | "This script zip and unzip MVL dataset locally. " 87 | + "A mvl dataset is composed of geometry info, and images. Both encoded in scene_room_idx format." 88 | ) 89 | 90 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 91 | 92 | parser.add_argument( 93 | "-d", 94 | "--scene_dir", 95 | # default=f'{ASSETS_DIR}/mvl_data/mp3d_fpe', 96 | type=str, 97 | help="MVL dataset directory.", 98 | ) 99 | 100 | parser.add_argument( 101 | "-f", 102 | "--scene_list", 103 | # default=f'{DATA_DIR}/mp3d_fpe/mp3d_fpe__test__scene_list.json', 104 | type=str, 105 | help="Scene list file which contents all frames encoded in scene_room_idx format.", 106 | ) 107 | 108 | parser.add_argument( 109 | "-o", 110 | "--output_dir", 111 | # default=f"{ASSETS_DIR}/tmp/zip_files", 112 | type=str, 113 | help="Output directory for the output_file to be created.", 114 | ) 115 | 116 | parser.add_argument( 117 | "--labels", action="store_true", help="Different method to zip GT labels." 118 | ) 119 | args = parser.parse_args() 120 | return args 121 | 122 | 123 | if __name__ == "__main__": 124 | args = get_argparse() 125 | if args.labels: 126 | zip_mvl_labels(args) 127 | else: 128 | zip_mvl_data(args) 129 | -------------------------------------------------------------------------------- /mvl_challenge/data_loaders/mvl_dataloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pathlib 4 | import torch.utils.data as data 5 | from PIL import Image 6 | import json 7 | import torch 8 | import logging 9 | 10 | 11 | class ListLayout: 12 | def __init__(self, list_ly): 13 | self.data = list_ly 14 | 15 | def __len__(self): 16 | return self.data.__len__() 17 | 18 | def __getitem__(self, idx): 19 | # ! iteration per each self.data 20 | img = self.data[idx].img.transpose([2, 0, 1]).copy() 21 | x = torch.FloatTensor(np.array(img / img.max())) 22 | return dict(images=x, idx=self.data[idx].idx) 23 | 24 | 25 | class MVLDataLoader(data.Dataset): 26 | ''' 27 | Dataloader that handles MLC dataset format. 28 | ''' 29 | 30 | def __init__(self, cfg): 31 | self.cfg = cfg 32 | 33 | # ! List of scenes defined in a list file 34 | if cfg.get('scene_list', '') == '': 35 | # ! Reading from available labels data 36 | self.raw_data = os.listdir( 37 | os.path.join(self.cfg.data_dir.labels_dir, self.cfg.label) 38 | ) 39 | self.list_frames = self.raw_data 40 | self.list_rooms = None 41 | else: 42 | assert os.path.exists(self.cfg.scene_list) 43 | self.raw_data = json.load(open(self.cfg.scene_list)) 44 | self.list_rooms = list(self.raw_data.keys()) 45 | self.list_frames = [self.raw_data[room] for room in self.list_rooms] 46 | self.list_frames = [item for sublist in self.list_frames for item in sublist] 47 | 48 | if cfg.get('size', -1) < 0: 49 | self.data = self.list_frames 50 | elif cfg.size < 1: 51 | np.random.shuffle(self.list_frames) 52 | self.data = self.list_frames[:int(cfg.size * self.list_frames.__len__())] 53 | else: 54 | np.random.shuffle(self.list_frames) 55 | self.data = self.list_frames[:cfg.size] 56 | # ! By default this dataloader iterates by frames 57 | logging.info(f"Simple MLC dataloader initialized with: {self.cfg.data_dir.img_dir}") 58 | logging.info(f"Labels reading from: {os.path.join(self.cfg.data_dir.labels_dir, self.cfg.label)}") 59 | logging.info(f"Total number of frames:{self.data.__len__()}") 60 | 61 | def __len__(self): 62 | return self.data.__len__() 63 | 64 | def __getitem__(self, idx): 65 | # ! iteration per each self.data given a idx 66 | filename = os.path.splitext(self.data[idx])[0] 67 | label_fn = os.path.join(self.cfg.data_dir.labels_dir, self.cfg.label, f"{filename}") 68 | 69 | std_fn = os.path.join(self.cfg.data_dir.labels_dir, 'std', f"{filename}.npy") 70 | image_fn = os.path.join(self.cfg.data_dir.img_dir, f"{filename}") 71 | 72 | if os.path.exists(image_fn + '.jpg'): 73 | image_fn += '.jpg' 74 | elif os.path.exists(image_fn + '.png'): 75 | image_fn += '.png' 76 | 77 | img = np.array(Image.open(image_fn), np.float32)[..., :3] / 255. 78 | 79 | if os.path.exists(label_fn + ".npy"): 80 | label = np.load(label_fn + ".npy") 81 | elif os.path.exists(label_fn + ".npz"): 82 | label = np.load(label_fn + ".npz")["phi_coords"] 83 | else: 84 | raise ValueError(f"Not found {label_fn}") 85 | 86 | # Random flip 87 | if self.cfg.get('flip', False) and np.random.randint(2) == 0: 88 | img = np.flip(img, axis=1) 89 | label = np.flip(label, axis=len(label.shape) - 1) 90 | 91 | # Random horizontal rotate 92 | if self.cfg.get('rotate', False): 93 | dx = np.random.randint(img.shape[1]) 94 | img = np.roll(img, dx, axis=1) 95 | label = np.roll(label, dx, axis=len(label.shape) - 1) 96 | 97 | # Random gamma augmentation 98 | if self.cfg.get('gamma', False): 99 | p = np.random.uniform(1, 2) 100 | if np.random.randint(2) == 0: 101 | p = 1 / p 102 | img = img**p 103 | 104 | if os.path.exists(std_fn): 105 | std = np.load(std_fn) 106 | else: 107 | std = np.ones_like(label) 108 | 109 | x = torch.FloatTensor(img.transpose([2, 0, 1]).copy()) 110 | label = torch.FloatTensor(label.copy()) 111 | std = torch.FloatTensor(std.copy()) 112 | return [x, label, std] 113 | 114 | -------------------------------------------------------------------------------- /mvl_challenge/challenge_results/create_npz_files.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ( 3 | DATA_DIR, 4 | ROOT_DIR, 5 | CFG_DIR, 6 | EPILOG, 7 | ASSETS_DIR, 8 | DEFAULT_MVL_DIR, 9 | DEFAULT_NPZ_DIR, 10 | SCENE_LIST_DIR, 11 | ) 12 | from mvl_challenge.config.cfg import read_omega_cfg 13 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 14 | from mvl_challenge.utils.vispy_utils import plot_list_ly 15 | from mvl_challenge.utils.image_utils import draw_boundaries_phi_coords, add_caption_to_image 16 | from imageio import imwrite 17 | from mvl_challenge.utils.check_utils import check_mvl_dataset 18 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 19 | from mvl_challenge.utils.io_utils import ( 20 | create_directory, 21 | get_scene_room_from_scene_room_idx, 22 | save_compressed_phi_coords, 23 | ) 24 | from mvl_challenge.utils.image_utils import plot_image 25 | import numpy as np 26 | import os 27 | from pathlib import Path 28 | 29 | 30 | def get_cfg_from_args(args): 31 | cfg = read_omega_cfg(args.cfg) 32 | cfg.scene_dir = args.scene_dir 33 | cfg.scene_list = args.scene_list 34 | cfg.ckpt = args.ckpt 35 | cfg.cuda_device = args.cuda_device 36 | return cfg 37 | 38 | 39 | def main(args): 40 | cfg = get_cfg_from_args(args) 41 | _, ret = check_mvl_dataset(args) 42 | assert ret, "Failed checking MVL dataset." 43 | 44 | mvl = MVLDataset(cfg) 45 | hn = WrapperHorizonNet(cfg) 46 | 47 | # ! Join the output_dir and the scene_list 48 | output_dir = create_directory( 49 | os.path.join(args.output_dir, Path(args.scene_list).stem), delete_prev=False 50 | ) 51 | 52 | if args.vis: 53 | output_dir_vis = create_directory( 54 | os.path.join(args.output_dir, Path(args.scene_list).stem + "_vis"), delete_prev=False 55 | ) 56 | 57 | for list_ly in iter_mvl_room_scenes(model=hn, dataset=mvl): 58 | for ly in list_ly: 59 | fn = os.path.join(output_dir, ly.idx) 60 | # ! IMPORTANT: Use ALWAYS save_compressed_phi_coords() 61 | save_compressed_phi_coords(ly.phi_coords, fn) 62 | 63 | if args.vis: 64 | # ! Save visualizations 65 | img = ly.get_rgb() 66 | draw_boundaries_phi_coords( 67 | img, phi_coords=np.vstack([ly.phi_coords[0], ly.phi_coords[1]]) 68 | ) 69 | img = add_caption_to_image(img, ly.idx) 70 | 71 | fn = os.path.join(output_dir_vis, f"{ly.idx}.jpg") 72 | imwrite(fn, img) 73 | 74 | 75 | 76 | def get_argparse(): 77 | desc = ( 78 | "This script saves the evaluations (phi_coords) of HorizonNet given a MVL dataset, scene list and cfg file. " 79 | + "The scene directory is where the MVL data is stored. " 80 | + "The scene list is the list of scene in scene_room_idx format. " 81 | + "The cfg file is the yaml configuration with all hyperparameters set to default values." 82 | ) 83 | 84 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 85 | 86 | parser.add_argument( 87 | "-d", 88 | "--scene_dir", 89 | type=str, 90 | default=f"{DEFAULT_MVL_DIR}", 91 | help="MVL dataset directory.", 92 | ) 93 | 94 | parser.add_argument( 95 | "--cfg", 96 | type=str, 97 | default=f"{CFG_DIR}/eval_mvl_dataset.yaml", 98 | help=f"Config file to load a MVL dataset. For this script model cfg most be defined in the cfg file too. (Default: {CFG_DIR}/eval_mvl_dataset.yaml)", 99 | ) 100 | 101 | parser.add_argument( 102 | "-f", 103 | "--scene_list", 104 | type=str, 105 | default=f"{SCENE_LIST_DIR}/scene_list__challenge_phase_testing_set.json", 106 | help="Scene_list of mvl scenes in scene_room_idx format.", 107 | ) 108 | 109 | parser.add_argument( 110 | "--ckpt", 111 | default=f"{ASSETS_DIR}/ckpt/hn_mp3d.pth", 112 | help="Pretrained model ckpt (Default: mp3d)", 113 | ) 114 | 115 | parser.add_argument("--cuda_device", default=0, type=int, help="Cuda device. (Default: 0)") 116 | 117 | parser.add_argument( 118 | "-o", 119 | "--output_dir", 120 | default=f"{DEFAULT_NPZ_DIR}", 121 | help="Output directory where to store phi_coords estimations.", 122 | ) 123 | 124 | parser.add_argument( 125 | "--vis", 126 | action="store_true", 127 | help="Output directory where to store phi_coords estimations.", 128 | ) 129 | args = parser.parse_args() 130 | return args 131 | 132 | 133 | if __name__ == "__main__": 134 | args = get_argparse() 135 | main(args) 136 | -------------------------------------------------------------------------------- /mvl_challenge/remote_data/download_mvl_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import gdown 4 | import pandas as pd 5 | from mvl_challenge import ASSETS_DIR, ROOT_DIR 6 | from mvl_challenge.utils.io_utils import create_directory, read_txt_file 7 | from mvl_challenge.config.cfg import set_loggings 8 | from mvl_challenge import EPILOG 9 | from mvl_challenge.utils.download_utils import download_file_from_google_drive 10 | from tqdm import tqdm 11 | import threading 12 | from google_drive_downloader import GoogleDriveDownloader as gdd 13 | import subprocess 14 | 15 | def download_dirs(args): 16 | set_loggings() 17 | create_directory(args.output_dir, delete_prev=False) 18 | 19 | list_google_scenes = args.ids_file 20 | lines = read_txt_file(list_google_scenes) 21 | 22 | for l in lines: 23 | gd_id, dir_path = [l for l in l.replace(" ", ",").split(",") if l != ""][:2] 24 | output_dir = os.path.join(args.output_dir) 25 | count = f"{lines.index(l)+1}/{lines.__len__()}" 26 | print(f"Downloading...{count} {output_dir}") 27 | gdown.download_folder(id=gd_id, output=output_dir, quiet=False) 28 | 29 | def callback_curl_downloading(gd_id, output_dir): 30 | subprocess.run( 31 | [ 32 | "sh", 33 | f"{ROOT_DIR}/remote_data/download_gdrive_ids_file.sh", 34 | f"{gd_id}", 35 | f"{output_dir}"] 36 | ) 37 | 38 | def download_file_by_threads(args): 39 | set_loggings() 40 | create_directory(args.output_dir, delete_prev=False) 41 | 42 | list_threads = [] 43 | list_google_scenes = args.ids_file 44 | lines = read_txt_file(list_google_scenes) 45 | 46 | for l in lines: 47 | gd_id, zip_fn = [l for l in l.replace(" ", ",").split(",") if l != ""][:2] 48 | output_file = os.path.join(args.output_dir, zip_fn) 49 | list_threads.append( 50 | threading.Thread( 51 | # target=download_google_drive_link, 52 | # args=(gd_id, output_file, f"{lines.index(l)+1}/{lines.__len__()}")) 53 | # threading.Thread( 54 | # target=download_file_from_google_drive, 55 | # args=(gd_id, output_file)) 56 | target=callback_curl_downloading, 57 | args=(gd_id, output_file)) 58 | ) 59 | [t.start() for t in list_threads] 60 | [t.join() for t in list_threads] 61 | 62 | 63 | def download_file(args): 64 | set_loggings() 65 | create_directory(args.output_dir, delete_prev=False) 66 | 67 | list_google_scenes = args.ids_file 68 | lines = read_txt_file(list_google_scenes) 69 | 70 | for l in lines: 71 | gd_id, zip_fn = [l for l in l.replace(" ", ",").split(",") if l != ""][:2] 72 | output_file = os.path.join(args.output_dir, zip_fn) 73 | download_google_drive_link( 74 | gd_id, output_file, f"{lines.index(l)+1}/{lines.__len__()}" 75 | ) 76 | 77 | 78 | 79 | def download_google_drive_link_by_request(gd_id, output_file, count=""): 80 | print(f"Downloading...{count} {output_file}") 81 | gdd.download_file_from_google_drive(file_id=gd_id, 82 | dest_path=output_file, 83 | unzip=False) 84 | 85 | 86 | def download_google_drive_link(gd_id, output_file, count=""): 87 | print(f"Downloading...{count} {output_file}") 88 | url = f"https://drive.google.com/uc?id={gd_id}" 89 | gdown.download(url, output_file, quiet=False) 90 | # url= f"https://docs.google.com/uc?export=download&confirm=t&id={gd_id}" 91 | # wget.download(url, out=output_file) 92 | # download_file_from_google_drive(gd_id, output_file) 93 | 94 | 95 | def get_argparse(): 96 | desc = ( 97 | "This script Download a set of zip files corresponding to the mvl-data. " 98 | + "This zip files may content geometry_info files, images files, or/and gt npz labels files." 99 | ) 100 | 101 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 102 | 103 | parser.add_argument( 104 | "-o", 105 | "--output_dir", 106 | type=str, 107 | # required=True, 108 | default=f"{ASSETS_DIR}/tmp/downloaded_data", 109 | help="Output dataset directory.", 110 | ) 111 | 112 | parser.add_argument( 113 | "-f", 114 | "--ids_file", 115 | type=str, 116 | # required=True, 117 | default="/media/NFS/kike/360_Challenge/mvl_toolkit/mvl_challenge/data/gdrive_files/gdrive_ids__warm_up_training_set_folders.csv", 118 | help="lists of IDS to download from GoogleDrive", 119 | ) 120 | 121 | args = parser.parse_args() 122 | return args 123 | 124 | 125 | if __name__ == "__main__": 126 | args = get_argparse() 127 | # download_scenes(args) 128 | download_dirs(args) 129 | -------------------------------------------------------------------------------- /test_mvl_toolkit.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import argparse 4 | from mvl_challenge import ( 5 | ASSETS_DIR, 6 | GDRIVE_DIR, 7 | ROOT_DIR, 8 | EPILOG, 9 | CFG_DIR, 10 | DEFAULT_DOWNLOAD_DIR, 11 | DEFAULT_NPZ_DIR 12 | ) 13 | from mvl_challenge.config.cfg import get_empty_cfg, read_omega_cfg 14 | from mvl_challenge.remote_data.download_mvl_data import ( 15 | download_file, 16 | download_google_drive_link, 17 | ) 18 | from mvl_challenge.utils.io_utils import create_directory, save_compressed_phi_coords 19 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 20 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 21 | from mvl_challenge.challenge_results.create_zip_results import zip_results 22 | 23 | GOOGLE_IDS_EXAMPLE_SCENE = "gdrive_ids__example_scene.csv" 24 | 25 | 26 | def main(args): 27 | 28 | list_logs = [] 29 | try: 30 | #! Downloading example scene 31 | list_logs.append("Download example_scene.zip") 32 | cfg = get_empty_cfg() 33 | example_scene_zip_dir = os.path.join(args.output_dir, "zips", "example_data") 34 | create_directory(example_scene_zip_dir, delete_prev=False) 35 | cfg.output_dir = example_scene_zip_dir 36 | cfg.ids_file = os.path.join(GDRIVE_DIR, GOOGLE_IDS_EXAMPLE_SCENE) 37 | download_file(cfg) 38 | 39 | # ! Unzipping mvl-data 40 | list_logs.append("Unzip example_scene.zip as mvl-data") 41 | cfg.output_dir = os.path.join(args.output_dir, "mvl_data") 42 | create_directory(cfg.output_dir, delete_prev=False) 43 | subprocess.run( 44 | [ 45 | "bash", 46 | f"{ROOT_DIR}/remote_data/unzip_data.sh", 47 | "-d", 48 | f"{example_scene_zip_dir}", 49 | "-o", 50 | f"{cfg.output_dir}", 51 | ] 52 | ) 53 | scene_list_fn = os.path.join(example_scene_zip_dir, "example_scene_list.json") 54 | assert os.path.exists(scene_list_fn), f"Not found {scene_list_fn}" 55 | 56 | #! Download CKPT 57 | list_logs.append("Download CKPT (Pretrained model)") 58 | ckpt_mp3d_id = "1W2A-_WU9d5KAwEQiTywJud2mRO3hLXqL" 59 | fn = os.path.join(ASSETS_DIR, "ckpt") 60 | fn = create_directory(fn, delete_prev=False) 61 | ckpt_fn = os.path.join(fn, "hn_mp3d.pth") 62 | download_google_drive_link(ckpt_mp3d_id, ckpt_fn) 63 | 64 | # ! Loading List of Layout instances 65 | list_logs.append("Load mvl-data as a set of list Layouts") 66 | cfg_mvl = read_omega_cfg(f"{CFG_DIR}/eval_mvl_dataset.yaml") 67 | cfg_mvl.scene_dir = cfg.output_dir 68 | cfg_mvl.scene_list = scene_list_fn 69 | mvl = MVLDataset(cfg_mvl) 70 | [ly for ly in mvl.iter_list_ly()] 71 | 72 | # ! Evaluating HorizonNet 73 | list_logs.append("Eval list Layouts using HorizonNet") 74 | cfg_mvl.ckpt = ckpt_fn 75 | cfg_mvl.cuda = 0 76 | hn = WrapperHorizonNet(cfg_mvl) 77 | list_ly = [ly for ly in iter_mvl_room_scenes(model=hn, dataset=mvl)][0] 78 | 79 | # ! Saving npz estimations 80 | list_logs.append("Save *.npz estimations") 81 | npz_estimations_dir = os.path.join(DEFAULT_NPZ_DIR, "example_data") 82 | create_directory(npz_estimations_dir, delete_prev=False) 83 | for ly in list_ly: 84 | fn = os.path.join(npz_estimations_dir, ly.idx) 85 | # ! IMPORTANT: Use ALWAYS save_compressed_phi_coords() 86 | save_compressed_phi_coords(ly.phi_coords, fn) 87 | 88 | # ! Zip npz estimations 89 | list_logs.append("Zip *.npz estimations") 90 | cfg.results_dir = npz_estimations_dir 91 | cfg.scene_list = scene_list_fn 92 | zip_results(cfg) 93 | 94 | except Exception as err: 95 | [print(f"[PASSED]:\t{log}") for log in list_logs[:-1]] 96 | print(f"[FAILED]:\t{list_logs[-1]}") 97 | print(type(err)) # the exception instance 98 | print(err.args) # arguments stored in .args 99 | print(err) 100 | return 101 | 102 | [print(f"[PASSED]:\t{log}") for log in list_logs] 103 | 104 | print(f"*\t->>>\tSuccessfully test mvl-toolkit\t<<<-\t*") 105 | 106 | 107 | def get_argparse(): 108 | desc = "This script test automatically the present mvl-toolkit using a predefined example mvl-scene." 109 | 110 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 111 | 112 | parser.add_argument( 113 | "-o", 114 | "--output_dir", 115 | default=f"{DEFAULT_DOWNLOAD_DIR}", 116 | type=str, 117 | help=f"Output directory by default it will store at {DEFAULT_DOWNLOAD_DIR}.", 118 | ) 119 | 120 | args = parser.parse_args() 121 | return args 122 | 123 | 124 | if __name__ == "__main__": 125 | args = get_argparse() 126 | main(args) 127 | -------------------------------------------------------------------------------- /mvl_challenge/datasets/mvl_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from tqdm import tqdm 4 | from mvl_challenge.data_structure import Layout, CamPose 5 | from mvl_challenge.utils.layout_utils import filter_out_noisy_layouts 6 | import numpy as np 7 | import logging 8 | from imageio import imread 9 | from pyquaternion import Quaternion 10 | from PIL import Image, ImageFile 11 | 12 | ImageFile.LOAD_TRUNCATED_IMAGES = True 13 | from torch.utils.data import DataLoader 14 | import torch.utils.data as data 15 | import torch 16 | 17 | 18 | class MVLDataset: 19 | def __init__(self, cfg): 20 | logging.info("Initializing MVL Dataset...") 21 | self.cfg = cfg 22 | self.set_paths() 23 | logging.info("MVL Dataset successfully initialized") 24 | 25 | def set_paths(self): 26 | # * Paths for MVLDataset 27 | self.DIR_GEOM_INFO = os.path.join( 28 | self.cfg.runners.mvl.data_dir, "geometry_info" 29 | ) 30 | self.DIR_IMGS = os.path.join(self.cfg.runners.mvl.data_dir, "img") 31 | 32 | assert os.path.exists( 33 | self.DIR_GEOM_INFO 34 | ), f"Not found geometry info {self.DIR_GEOM_INFO}" 35 | assert os.path.exists(self.DIR_IMGS), f"Not found img dir {self.DIR_IMGS}" 36 | 37 | # * Set main lists of files 38 | #! List of scene names 39 | assert os.path.exists( 40 | self.cfg.runners.mvl.scene_list 41 | ), f"Not found scene list {self.cfg.runners.mvl.scene_list}" 42 | logging.info(f"Scene list: {self.cfg.runners.mvl.scene_list}") 43 | self.data_scenes = json.load(open(self.cfg.runners.mvl.scene_list)) 44 | self.list_rooms = list(self.data_scenes.keys()) 45 | self.list_frames = [fr for fr in self.data_scenes.values()] 46 | self.list_frames = [item for sublist in self.list_frames for item in sublist] 47 | 48 | def iter_list_ly(self): 49 | """Iterator for all room scene defined in this class""" 50 | for room_scene in self.list_rooms: 51 | list_ly = self.get_list_ly(room_scene) 52 | yield list_ly 53 | 54 | def get_list_ly(self, room_scene): 55 | """ 56 | Returns a list of Layout instances described by room_scene. 57 | Scene name is the room name for the scene. 58 | By default it returns all scene names. 59 | """ 60 | self.cfg._room_scene = room_scene 61 | scene_data = self.data_scenes[room_scene] 62 | 63 | self.list_ly = [] 64 | for frame in tqdm(scene_data, desc=f"Loading mvl data scene {room_scene}..."): 65 | 66 | ly = Layout(self.cfg) 67 | ly.idx = os.path.splitext(frame)[0] 68 | 69 | ly.img_fn = os.path.join(self.DIR_IMGS, f"{ly.idx}.jpg") 70 | assert os.path.exists(ly.img_fn), f"Not found {ly.img_fn}" 71 | # ! Loading geometry 72 | geom = json.load(open(os.path.join(self.DIR_GEOM_INFO, f"{ly.idx}.json"))) 73 | 74 | self.set_geom_info(layout=ly, geom=geom) 75 | 76 | # ! Setting in WC 77 | ly.cam_ref = "WC" 78 | 79 | self.list_ly.append(ly) 80 | 81 | return self.list_ly 82 | 83 | def print_mvl_data_info(self): 84 | a = [dt.__len__() for dt in self.data_scenes.values()] 85 | logging.info(f"Total number of frames: {np.sum(a)}") 86 | logging.info(f"Total number of room scenes: {a.__len__()}") 87 | 88 | @staticmethod 89 | def set_geom_info(layout: Layout, geom): 90 | 91 | layout.pose = CamPose(layout.cfg) 92 | layout.pose.t = np.array(geom["translation"]) # * geom['scale'] 93 | qx, qy, qz, qw = geom["quaternion"] 94 | q = Quaternion(qx=qx, qy=qy, qz=qz, qw=qw) 95 | layout.pose.rot = q.rotation_matrix 96 | layout.pose.idx = layout.idx 97 | layout.camera_height = geom["cam_h"] 98 | 99 | 100 | class MVImageLayout(data.Dataset): 101 | def __init__(self, list_data): 102 | self.data = list_data # [(img_fn, idx),...] 103 | 104 | def __len__(self): 105 | return self.data.__len__() 106 | 107 | def __getitem__(self, idx): 108 | image_fn = self.data[idx][0] 109 | assert os.path.exists(image_fn) 110 | img = np.array(Image.open(image_fn), np.float32)[..., :3] / 255.0 111 | x = torch.FloatTensor(img.transpose([2, 0, 1]).copy()) 112 | return dict(images=x, idx=self.data[idx][1]) 113 | 114 | 115 | def iter_mvl_room_scenes(model, dataset: MVLDataset): 116 | """ 117 | Creates a generator which yields a list of layout from a defined 118 | MVL dataset and estimates layout in it. 119 | """ 120 | 121 | dataset.print_mvl_data_info() 122 | cfg = dataset.cfg 123 | for room_scene in dataset.list_rooms: 124 | list_ly = dataset.get_list_ly(room_scene=room_scene) 125 | 126 | # ! Overwrite phi_coords within the list_ly by the estimating new layouts. 127 | model.estimate_within_list_ly(list_ly) 128 | filter_out_noisy_layouts( 129 | list_ly=list_ly, max_room_factor_size=cfg.runners.mvl.max_room_factor_size 130 | ) 131 | yield list_ly 132 | -------------------------------------------------------------------------------- /mvl_challenge/config/cfg.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | import logging 3 | import os 4 | import pathlib 5 | 6 | import git 7 | import numpy as np 8 | import yaml 9 | from omegaconf import ListConfig, OmegaConf 10 | 11 | 12 | def load_auto_loading(cfg): 13 | """ 14 | Loads automatically the yaml files described under the key "auto_loading" to the root node 15 | """ 16 | if "auto_loading" not in cfg.keys(): 17 | # No auto loading has been defined 18 | return 19 | a_cfg = cfg.auto_loading 20 | #! Iteration for each path defined in cfg.auto_loading 21 | for key, path_cfg in a_cfg.items(): 22 | with open(path_cfg, "r") as f: 23 | _cfg = yaml.safe_load(f) 24 | cfg[key] = _cfg 25 | 26 | 27 | def short_decode_id(list_keys, *, _parent_): 28 | """ 29 | Decodes (in a short way) a set of keys as a unique string 30 | """ 31 | decoded_id = "" 32 | for key in list_keys: 33 | path_keys = key.split(".") 34 | __nested_value = _parent_ 35 | for _key in path_keys: 36 | __nested_value = __nested_value[_key] 37 | assert not isinstance(__nested_value, dict) 38 | decoded_id += f"{path_keys[-1]}.{__nested_value}_" 39 | return decoded_id 40 | 41 | 42 | def decode_id(list_keys, *, _parent_): 43 | """ 44 | Decodes (in a short way) a set of keys as a unique string 45 | """ 46 | decoded_id = "" 47 | for key in list_keys: 48 | path_keys = key.split(".") 49 | __nested_value = _parent_ 50 | for _key in path_keys: 51 | __nested_value = __nested_value[_key] 52 | assert not isinstance(__nested_value, dict) 53 | 54 | decoded_id += f"{key}.{__nested_value}_" 55 | return decoded_id 56 | 57 | 58 | def rel_path(rel_path, *, _parent_): 59 | path = pathlib.Path(os.path.join(CFG_ROOT, rel_path)).resolve() 60 | return path.__str__() 61 | 62 | 63 | def encode_list(key_list, *, _parent_): 64 | data_list = _parent_[key_list] 65 | data = "" 66 | for dt in [item for sublist in data_list for item in sublist]: 67 | data += f"{dt}_" 68 | return data[:-1] 69 | 70 | 71 | def range(range_info, *, _parent_): 72 | np_range = np.arange(range_info[0], range_info[1], range_info[2]) 73 | return ListConfig([float(n) for n in np_range]) 74 | 75 | 76 | def linspace(range_info, *, _parent_): 77 | np_range = np.linspace(range_info[0], range_info[1], range_info[2]) 78 | return ListConfig([float(n) for n in np_range]) 79 | 80 | OmegaConf.register_new_resolver("decode", decode_id, replace=True) 81 | OmegaConf.register_new_resolver("short_decode", short_decode_id, replace=True) 82 | OmegaConf.register_new_resolver("rel_path", rel_path, replace=True) 83 | OmegaConf.register_new_resolver("encode_list", encode_list, replace=True) 84 | OmegaConf.register_new_resolver("range", range, replace=True) 85 | OmegaConf.register_new_resolver("linspace", linspace, replace=True) 86 | 87 | def get_default(cfg): 88 | df = OmegaConf.create(dict(default_cfg=cfg["default_cfg"])) 89 | OmegaConf.resolve(df) 90 | assert os.path.exists(df.default_cfg) 91 | # ! Reading YAML file 92 | 93 | with open(df.default_cfg, "r") as f: 94 | cfg_dict = yaml.safe_load(f) 95 | logging.info(f"Loaded default cfg from: {df.default_cfg}") 96 | return cfg_dict 97 | 98 | 99 | def add_git_commit(cfg): 100 | repo = git.Repo(search_parent_directories=True) 101 | cfg["git_commit"] = repo.head._get_commit().name_rev 102 | return cfg 103 | 104 | 105 | def update_cfg(d, u): 106 | for k, v in u.items(): 107 | if isinstance(v, collections.abc.Mapping): 108 | d[k] = update_cfg(d.get(k, {}), v) 109 | else: 110 | d[k] = v 111 | return d 112 | 113 | 114 | def set_loggings(): 115 | logging.basicConfig( 116 | format="[%(levelname)s] [%(asctime)s]: %(message)s", level=logging.INFO 117 | ) 118 | 119 | 120 | def read_omega_cfg(cfg_file): 121 | assert os.path.exists(cfg_file), f"File does not exist {cfg_file}" 122 | 123 | # ! Reading YAML file 124 | with open(cfg_file, "r") as f: 125 | cfg_dict = yaml.safe_load(f) 126 | 127 | set_loggings() 128 | # ! Saving cfg root for relative_paths 129 | global CFG_ROOT 130 | CFG_ROOT = os.path.dirname(cfg_file) 131 | 132 | # ! add git commit 133 | cfg_dict = add_git_commit(cfg_dict) 134 | if "default_cfg" in cfg_dict.keys(): 135 | df = get_default(cfg_dict) 136 | update_cfg(df, cfg_dict) 137 | cfg = OmegaConf.create(df) 138 | else: 139 | cfg = OmegaConf.create(cfg_dict) 140 | 141 | # ! Loading Auto-loadings 142 | load_auto_loading(cfg) 143 | return cfg 144 | 145 | 146 | def get_empty_cfg(): 147 | logging.basicConfig( 148 | format="[%(levelname)s] [%(asctime)s]: %(message)s", level=logging.INFO 149 | ) 150 | cfg_dict = dict() 151 | 152 | # ! add git commit 153 | cfg_dict = add_git_commit(cfg_dict) 154 | 155 | cfg = OmegaConf.create(cfg_dict) 156 | return cfg 157 | 158 | 159 | def read_config(cfg_file): 160 | cfg = read_omega_cfg(cfg_file) 161 | OmegaConf.resolve(cfg) 162 | cfg = set_cfg(cfg) 163 | logging.info(f"Config file loaded successfully from: {cfg_file}") 164 | return cfg 165 | 166 | 167 | def set_cfg(cfg): 168 | OmegaConf.set_readonly(cfg, True) 169 | OmegaConf.set_struct(cfg, True) 170 | return cfg 171 | 172 | 173 | def save_cfg(cfg_file, cfg): 174 | with open(cfg_file, "w") as fn: 175 | OmegaConf.save(config=cfg, f=fn) 176 | -------------------------------------------------------------------------------- /mvl_challenge/challenge_results/evaluate_results.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from mvl_challenge import ( 3 | DATA_DIR, 4 | DEFAULT_MVL_DIR, 5 | ROOT_DIR, 6 | CFG_DIR, 7 | EPILOG, 8 | ASSETS_DIR, 9 | SCENE_LIST_DIR, 10 | ) 11 | from mvl_challenge.config.cfg import read_omega_cfg 12 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 13 | from mvl_challenge.utils.vispy_utils import plot_list_ly 14 | from mvl_challenge.utils.image_utils import draw_boundaries_phi_coords, add_caption_to_image, draw_boundaries_uv 15 | from imageio import imwrite 16 | from mvl_challenge.utils.image_utils import COLOR_CYAN, COLOR_GREEN, xyz2uv 17 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 18 | from mvl_challenge.utils.io_utils import ( 19 | create_directory, 20 | save_json_dict, 21 | load_gt_label, 22 | get_scene_room_from_scene_room_idx, 23 | ) 24 | from mvl_challenge.utils.image_utils import plot_image 25 | from mvl_challenge.utils.eval_utils import eval_2d3d_iuo 26 | import numpy as np 27 | import os 28 | from pathlib import Path 29 | 30 | 31 | def get_cfg_from_args(args): 32 | cfg = read_omega_cfg(args.cfg) 33 | cfg.scene_dir = args.scene_dir 34 | cfg.scene_list = args.scene_list 35 | cfg.ckpt = args.ckpt 36 | cfg.cuda_device = args.cuda_device 37 | return cfg 38 | 39 | 40 | def main(args): 41 | cfg = get_cfg_from_args(args) 42 | mvl = MVLDataset(cfg) 43 | hn = WrapperHorizonNet(cfg) 44 | 45 | # ! Join the output_dir and the scene_list 46 | model = Path(cfg.ckpt).stem 47 | output_dir = create_directory(os.path.join(args.output_dir, model), delete_prev=True) 48 | output_dir_vis = create_directory(os.path.join(output_dir, "vis"), delete_prev=True) 49 | 50 | results = {} 51 | 52 | for list_ly in iter_mvl_room_scenes(model=hn, dataset=mvl): 53 | for ly in list_ly: 54 | 55 | phi_coords_gt = load_gt_label( 56 | os.path.join(args.scene_dir, "labels", "gt", f"{ly.idx}.npz") 57 | ) 58 | 59 | phi_coords_est = ly.phi_coords 60 | 61 | results_2d3d_iou = eval_2d3d_iuo( 62 | phi_coords_est=phi_coords_est, 63 | phi_coords_gt_bon=phi_coords_gt, 64 | ch=ly.camera_height, 65 | ) 66 | 67 | if np.min(results_2d3d_iou) < 0: 68 | # ! IoU evaluation failed, then IoU is 0 (the highest penalty) 69 | results[f"{ly.idx}__2dIoU"] = 0 70 | results[f"{ly.idx}__3dIoU"] = 0 71 | 72 | if args.vis: 73 | # ! Save visualizations 74 | img = ly.get_rgb() 75 | draw_boundaries_phi_coords( 76 | img, phi_coords=phi_coords_est, 77 | color=COLOR_CYAN 78 | ) 79 | 80 | draw_boundaries_phi_coords( 81 | img, phi_coords=phi_coords_gt, 82 | color=COLOR_GREEN 83 | ) 84 | 85 | img = add_caption_to_image(img, ly.idx) 86 | 87 | fn = os.path.join(output_dir_vis, f"{ly.idx}.jpg") 88 | imwrite(fn, img) 89 | 90 | 91 | results[f"{ly.idx}__2dIoU"] = results_2d3d_iou[0] 92 | results[f"{ly.idx}__3dIoU"] = results_2d3d_iou[1] 93 | 94 | _2dIoU = np.mean([value for key, value in results.items() if "2dIoU" in key]) 95 | _3dIoU = np.mean([value for key, value in results.items() if "3dIoU" in key]) 96 | 97 | results["total__m2dIoU"] = _2dIoU 98 | results["total__m3dIoU"] = _3dIoU 99 | fn = os.path.join( 100 | output_dir, f"{Path(args.scene_list).stem}__{Path(args.ckpt).stem}.json" 101 | ) 102 | save_json_dict(filename=fn, dict_data=results) 103 | 104 | print(f"2d-IoU: {_2dIoU:2.3f}\t3d-IoU: {_3dIoU:2.3f}") 105 | 106 | 107 | def get_argparse(): 108 | desc = ( 109 | "This script evaluates 2d-IoU, 3d-IoU from a set of estimated phi_coords. " 110 | + "Note that this script assumes you have access to some GT labels. " 111 | + "The passed cfg file is the yaml configuration with all hyperparameters set to default values." 112 | ) 113 | 114 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 115 | 116 | parser.add_argument( 117 | "-d", 118 | "--scene_dir", 119 | type=str, 120 | default=f"{DEFAULT_MVL_DIR}", 121 | help="MVL dataset directory.", 122 | ) 123 | 124 | parser.add_argument( 125 | "--cfg", 126 | type=str, 127 | default=f"{CFG_DIR}/eval_mvl_dataset.yaml", 128 | help=f"Config file to load a MVL dataset. For this script model cfg most be defined in the cfg file too. (Default: {CFG_DIR}/eval_mvl_dataset.yaml)", 129 | ) 130 | 131 | parser.add_argument( 132 | "-f", 133 | "--scene_list", 134 | type=str, 135 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_pilot_set.json", 136 | help="Scene_list of mvl scenes in scene_room_idx format.", 137 | ) 138 | 139 | parser.add_argument( 140 | "--ckpt", 141 | default=f"{ASSETS_DIR}/ckpt/hn_mp3d.pth", 142 | help="Pretrained model ckpt (Default: mp3d)", 143 | ) 144 | 145 | parser.add_argument( 146 | "--vis", 147 | action="store_true", 148 | help="Output directory where to store phi_coords estimations.", 149 | ) 150 | 151 | parser.add_argument("--cuda_device", default=0, type=int, help="Cuda device. (Default: 0)") 152 | 153 | parser.add_argument( 154 | "-o", 155 | "--output_dir", 156 | default=f"{ASSETS_DIR}/results", 157 | help="Output directory where to store phi_coords estimations.", 158 | ) 159 | args = parser.parse_args() 160 | return args 161 | 162 | 163 | if __name__ == "__main__": 164 | args = get_argparse() 165 | main(args) 166 | -------------------------------------------------------------------------------- /mvl_challenge/utils/spherical_utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import numpy as np 4 | 5 | 6 | class SphericalCamera: 7 | def __init__(self, shape): 8 | self.shape = shape 9 | self.compute_default_grids() 10 | 11 | def compute_default_grids(self): 12 | h, w = self.shape 13 | u = np.linspace(0, w - 1, w).astype(int) 14 | v = np.linspace(0, h - 1, h).astype(int) 15 | uu, vv = np.meshgrid(u, v) 16 | self.default_pixel = np.vstack((uu.flatten(), vv.flatten())).astype(np.int) 17 | self.default_bearings = uv2xyz(self.default_pixel, self.shape) 18 | 19 | def project_pcl_from_depth_and_rgb_maps(self, color_map, depth_map, scaler=1): 20 | from mvl_challenge.utils.image_utils import get_color_array 21 | 22 | color_pixels = get_color_array(color_map=color_map) / 255 23 | mask = depth_map.flatten() > 0 24 | pcl = ( 25 | self.default_bearings[:, mask] 26 | * scaler 27 | * get_color_array(color_map=depth_map)[0][mask] 28 | ) 29 | return pcl, color_pixels[:, mask] 30 | 31 | 32 | def uv2xyz(uv, shape): 33 | """ 34 | Projects uv vectors to xyz vectors (bearing vector) 35 | """ 36 | sph = uv2sph(uv, shape) 37 | theta = sph[0] 38 | phi = sph[1] 39 | 40 | x = np.cos(phi) * np.sin(theta) 41 | y = np.sin(phi) 42 | z = np.cos(phi) * np.cos(theta) 43 | 44 | return np.vstack((x, y, z)) 45 | 46 | 47 | def uv2sph(uv, shape): 48 | """ 49 | Projects a set of uv points into spherical coordinates (theta, phi) 50 | """ 51 | H, W = shape 52 | theta = 2 * np.pi * ((uv[0]) / W - 0.5) 53 | phi = np.pi * ((uv[1]) / H - 0.5) 54 | return np.vstack((theta, phi)) 55 | 56 | 57 | def sph2xyz(sph): 58 | """ 59 | Projects spherical coordinates (theta, phi) to euclidean space xyz 60 | """ 61 | theta = sph[:, 0] 62 | phi = sph[:, 1] 63 | 64 | x = math.cos(phi) * math.sin(theta) 65 | y = math.sin(phi) 66 | z = math.cos(phi) * math.cos(theta) 67 | 68 | return np.vstack((x, y, z)) 69 | 70 | 71 | #! Checked OK 72 | def sph2uv(sph, shape): 73 | # H, W = shape 74 | # u = W * (sph[0]/(2*np.pi) + 0.5) 75 | # v = H * (sph[1]/np.pi + 0.5) 76 | # return np.floor(np.vstack(( 77 | # np.clip(u, 0, W-1), 78 | # np.clip(v, 0, H-1) 79 | # ))).astype(int) 80 | theta_coord = sph[0] 81 | phi_coords = sph[1] 82 | u = np.clip( 83 | np.floor((0.5 * theta_coord / np.pi + 0.5) * shape[1] + 0.5), 0, shape[1] - 1 84 | ) 85 | v = np.clip(np.floor((phi_coords / np.pi + 0.5) * shape[0] + 0.5), 0, shape[0] - 1) 86 | return np.vstack([u, v]).astype(int) 87 | 88 | 89 | def sphere_normalization(xyz): 90 | norm = np.linalg.norm(xyz, axis=0) 91 | return xyz / norm 92 | 93 | 94 | #! Checked ok! 95 | def phi_coords2xyz(phi_coords): 96 | """ 97 | Returns 3D bearing vectors (on the unite sphere) from phi_coords 98 | """ 99 | W = phi_coords.__len__() 100 | u = np.linspace(0, W - 1, W) 101 | theta_coords = (2 * np.pi * u / W) - np.pi 102 | bearings_y = np.sin(phi_coords) 103 | bearings_x = np.cos(phi_coords) * np.sin(theta_coords) 104 | bearings_z = np.cos(phi_coords) * np.cos(theta_coords) 105 | return np.vstack((bearings_x, bearings_y, bearings_z)) 106 | 107 | 108 | #! Checked ok! 109 | def phi_coords2uv(phi_coords, shape=(512, 1024)): 110 | """ 111 | Converts a set of phi_coordinates (2, W), defined by ceiling and floor boundaries encoded as 112 | phi coordinates, into uv pixels 113 | """ 114 | H, W = shape 115 | u = np.linspace(0, W - 1, W) 116 | theta_coords = (2 * np.pi * u / W) - np.pi 117 | uv_c = sph2uv(np.vstack((theta_coords, phi_coords[0])), shape) 118 | uv_f = sph2uv(np.vstack((theta_coords, phi_coords[1])), shape) 119 | return uv_c, uv_f 120 | 121 | 122 | #! Checked ok! 123 | def uv2phi_coords(uv, shape=(512, 1024), type_bound="floor"): 124 | # _, idx, count = np.unique(uv[0], return_index=True, return_counts=True) 125 | u_coords = np.linspace(0, shape[1] - 1, shape[1]).astype(np.int16) 126 | v = [] 127 | for u in u_coords: 128 | v_idx = np.where(uv[0] == u)[0] 129 | if v_idx.size == 0: 130 | for i in range(5): 131 | v_idx = np.where(uv[0] == (u + i) % shape[1])[0] 132 | if v_idx.size > 0: 133 | break 134 | v_idx = np.where(uv[0] == (u - i) % shape[1])[0] 135 | if v_idx.size > 0: 136 | break 137 | # for i in range(5): 138 | # v_idx = np.where(uv[0] == (u - i)% shape[1])[0] 139 | # if v_idx.size > 0: 140 | # break 141 | # v_idx = np.where(uv[0] == (u + i)% shape[1])[0] 142 | # if v_idx.size > 0: 143 | # break 144 | if v_idx.size == 0: 145 | return None 146 | 147 | if type_bound == "floor": 148 | v.append(np.max(uv[1, v_idx])) 149 | elif type_bound == "ceiling": 150 | v.append(np.min(uv[1, v_idx])) 151 | else: 152 | raise ValueError("wrong type_bound") 153 | 154 | phi_bon = (np.array(v) / shape[0] - 0.5) * np.pi 155 | return phi_bon 156 | 157 | 158 | #! Checked ok! 159 | def xyz2uv(xyz, shape=(512, 1024)): 160 | """ 161 | Projects XYZ array into uv coord 162 | """ 163 | xyz_n = xyz / np.linalg.norm(xyz, axis=0, keepdims=True) 164 | 165 | normXZ = np.linalg.norm(xyz[(0, 2), :], axis=0, keepdims=True) 166 | 167 | phi_coords = np.arcsin(xyz_n[1, :]) 168 | theta_coord = np.sign(xyz[0, :]) * np.arccos(xyz[2, :] / normXZ) 169 | 170 | u = np.clip( 171 | np.floor((0.5 * theta_coord / np.pi + 0.5) * shape[1] + 0.5), 0, shape[1] - 1 172 | ) 173 | v = np.clip(np.floor((phi_coords / np.pi + 0.5) * shape[0] + 0.5), 0, shape[0] - 1) 174 | return np.vstack((u, v)).astype(int) 175 | -------------------------------------------------------------------------------- /tutorial/train_360_mlc/create_mlc_labels.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from pathlib import Path 4 | from tqdm import tqdm 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | from imageio import imwrite 8 | from mlc.mlc import compute_pseudo_labels 9 | from mvl_challenge.datasets.mvl_dataset import iter_mvl_room_scenes 10 | from mvl_challenge.config.cfg import read_omega_cfg 11 | from mvl_challenge import DEFAULT_MVL_DIR 12 | from mvl_challenge.utils.spherical_utils import xyz2uv 13 | from mvl_challenge.utils.io_utils import create_directory, save_compressed_phi_coords 14 | from mvl_challenge.datasets.mvl_dataset import MVLDataset 15 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 16 | from mvl_challenge.utils.image_utils import ( 17 | draw_boundaries_uv, 18 | draw_uncertainty_map, 19 | COLOR_MAGENTA, COLOR_CYAN) 20 | from mvl_challenge import ( 21 | ASSETS_DIR, 22 | DEFAULT_MVL_DIR, 23 | SCENE_LIST_DIR, 24 | ) 25 | 26 | MLC_TUTORIAL_DIR=os.path.dirname(__file__) 27 | 28 | def create_mlc_label_dirs(cfg): 29 | """ 30 | Create MLC pseudo label the directories 31 | """ 32 | create_directory(os.path.join(cfg.output_dir, cfg.id_exp), delete_prev=True) 33 | create_directory(cfg.mlc_dir.phi_coords, delete_prev=True) 34 | create_directory(cfg.mlc_dir.std, delete_prev=True) 35 | create_directory(cfg.mlc_dir.vis, delete_prev=True) 36 | 37 | def save_visualization(fn, img_boundaries, sigma_map): 38 | plt.figure(0, dpi=200) 39 | plt.clf() 40 | plt.subplot(211) 41 | plt.suptitle(Path(fn).stem) 42 | plt.imshow(img_boundaries) 43 | plt.axis('off') 44 | plt.subplot(212) 45 | plt.imshow(sigma_map) 46 | plt.axis('off') 47 | plt.tight_layout() 48 | plt.draw() 49 | plt.savefig(fn,bbox_inches='tight') 50 | plt.close() 51 | 52 | def compute_and_save_mlc_labels(list_ly): 53 | for ref in tqdm(list_ly, desc="Estimating MLC Labels"): 54 | uv_ceiling_ps, uv_floor_ps, std_ceiling, std_floor, _ = compute_pseudo_labels( 55 | list_frames=list_ly, 56 | ref_frame=ref, 57 | ) 58 | 59 | # ! Saving pseudo labels 60 | v = np.vstack((uv_ceiling_ps[1], uv_floor_ps[1])) 61 | std = np.vstack((std_ceiling, std_floor)) 62 | phi_coords = (v / 512 - 0.5) * np.pi 63 | 64 | # ! NOTE: 360-MLC expects npy files as pseudo labels 65 | fn = os.path.join(ref.cfg.mlc_dir.phi_coords, f"{ref.idx}") 66 | np.save(fn, phi_coords) 67 | fn = os.path.join(ref.cfg.mlc_dir.std, f"{ref.idx}") 68 | np.save(fn, std) 69 | 70 | uv_ceiling_hat = xyz2uv(ref.bearings_ceiling) 71 | uv_floor_hat = xyz2uv(ref.bearings_floor) 72 | 73 | img = ref.get_rgb() 74 | draw_boundaries_uv( 75 | image=img, 76 | boundary_uv=uv_ceiling_hat, 77 | color=COLOR_CYAN 78 | ) 79 | 80 | draw_boundaries_uv( 81 | image=img, 82 | boundary_uv=uv_floor_hat, 83 | color=COLOR_CYAN 84 | ) 85 | 86 | draw_boundaries_uv( 87 | image=img, 88 | boundary_uv=uv_ceiling_ps, 89 | color=COLOR_MAGENTA 90 | ) 91 | 92 | draw_boundaries_uv( 93 | image=img, 94 | boundary_uv=uv_floor_ps, 95 | color=COLOR_MAGENTA 96 | ) 97 | 98 | sigma_map = draw_uncertainty_map( 99 | peak_boundary=np.hstack((uv_ceiling_ps, uv_floor_ps)), 100 | sigma_boundary=np.hstack((std_ceiling, std_floor)) 101 | ) 102 | 103 | fn = os.path.join(ref.cfg.mlc_dir.vis, f"{ref.idx}.jpg") 104 | save_visualization(fn, img, sigma_map) 105 | 106 | def get_cfg_from_args(args): 107 | cfg = read_omega_cfg(args.cfg) 108 | if cfg.pass_args: # params in the yaml will be replaced by the passed arguments 109 | cfg.mvl_dir = args.scene_dir 110 | cfg.scene_list = args.scene_list 111 | cfg.output_dir = args.output_dir 112 | cfg.ckpt = args.ckpt 113 | cfg.cuda_device = args.cuda_device 114 | cfg.id_exp = f"mlc__{Path(cfg.ckpt).stem}__{Path(args.scene_list).stem}" 115 | return cfg 116 | 117 | def main(args): 118 | # ! Reading configuration 119 | cfg = get_cfg_from_args(args) 120 | 121 | mvl = MVLDataset(cfg) 122 | hn = WrapperHorizonNet(cfg) 123 | 124 | mvl.print_mvl_data_info() 125 | create_mlc_label_dirs(cfg) 126 | 127 | for list_ly in iter_mvl_room_scenes(model=hn, dataset=mvl): 128 | compute_and_save_mlc_labels(list_ly) 129 | 130 | def get_passed_args(): 131 | parser = argparse.ArgumentParser() 132 | 133 | parser.add_argument( 134 | '--cfg', 135 | default=f"{MLC_TUTORIAL_DIR}/create_mlc_labels.yaml", 136 | help=f'Config File. Default {MLC_TUTORIAL_DIR}/create_mlc_labels.yaml') 137 | 138 | parser.add_argument( 139 | "-f", 140 | "--scene_list", 141 | type=str, 142 | default=f"{SCENE_LIST_DIR}/scene_list__warm_up_training_set.json", 143 | help="Scene_list of mvl scenes in scene_room_idx format.", 144 | ) 145 | 146 | parser.add_argument( 147 | "-d", 148 | "--scene_dir", 149 | type=str, 150 | default=f"{DEFAULT_MVL_DIR}", 151 | help="MVL dataset directory.", 152 | ) 153 | 154 | parser.add_argument( 155 | "-o", 156 | "--output_dir", 157 | type=str, 158 | default=f"{DEFAULT_MVL_DIR}/labels", 159 | help="MVL dataset directory.", 160 | ) 161 | 162 | parser.add_argument( 163 | "--ckpt", 164 | default=f"{ASSETS_DIR}/ckpt/hn_mp3d.pth", 165 | help="Path to ckpt pretrained model (Default: mp3d)", 166 | ) 167 | 168 | parser.add_argument("--cuda_device", default=0, type=int, help="Cuda device. (Default: 0)") 169 | 170 | args = parser.parse_args() 171 | return args 172 | 173 | if __name__ == "__main__": 174 | args = get_passed_args() 175 | main(args) 176 | -------------------------------------------------------------------------------- /mvl_challenge/utils/layout_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from mvl_challenge.models.HorizonNet.misc import panostretch 3 | import logging 4 | from copy import deepcopy 5 | import os 6 | from pathlib import Path 7 | from mvl_challenge.utils.vispy_utils import plot_color_plc 8 | 9 | 10 | def label_cor2ly_phi_coord(label_cor_path, shape=(512, 1024)): 11 | """ 12 | Implementation taken from HorizonNet (CVPR 2019) 13 | https://sunset1995.github.io/HorizonNet/ 14 | """ 15 | with open(label_cor_path) as f: 16 | cor = np.array([line.strip().split() for line in f if line.strip()], np.float32) 17 | return corners_uv2ly_phi_coord(corners=cor, shape=shape) 18 | 19 | 20 | def corners_uv2ly_phi_coord(corners, shape=(512, 1024)): 21 | """ 22 | Return an npy array of 2xW points corresponded to the 23 | layout boundaries defined in spherical coords (phi only). 24 | Similar to the output estimation of HorizonNet and HoHoNet 25 | Implementation taken from HorizonNet (CVPR 2019) 26 | https://sunset1995.github.io/HorizonNet/ 27 | """ 28 | # Corner with minimum x should at the beginning 29 | corners = np.roll(corners[:, :2], -2 * np.argmin(corners[::2, 0]), 0) 30 | 31 | H, W = shape 32 | # Detect occlusion 33 | assert (np.abs(corners[0::2, 0] - corners[1::2, 0]) > W / 100).sum() == 0 34 | assert (corners[0::2, 1] > corners[1::2, 1]).sum() == 0 35 | 36 | bon = cor_2_1d(corners, H, W) 37 | 38 | return bon 39 | 40 | 41 | def sort_xy_filter_unique(xs, ys, y_small_first=True): 42 | """ 43 | Implementation taken from HorizonNet (CVPR 2019) 44 | https://sunset1995.github.io/HorizonNet/ 45 | """ 46 | xs, ys = np.array(xs), np.array(ys) 47 | idx_sort = np.argsort(xs + ys / ys.max() * (int(y_small_first) * 2 - 1)) 48 | xs, ys = xs[idx_sort], ys[idx_sort] 49 | _, idx_unique = np.unique(xs, return_index=True) 50 | xs, ys = xs[idx_unique], ys[idx_unique] 51 | assert np.all(np.diff(xs) > 0) 52 | return xs, ys 53 | 54 | 55 | def cor_2_1d(cor, H, W): 56 | """ 57 | Implementation taken from HorizonNet (CVPR 2019) 58 | https://sunset1995.github.io/HorizonNet/ 59 | """ 60 | bon_ceil_x, bon_ceil_y = [], [] 61 | bon_floor_x, bon_floor_y = [], [] 62 | n_cor = len(cor) 63 | for i in range(n_cor // 2): 64 | xys = panostretch.pano_connect_points( 65 | cor[i * 2], cor[(i * 2 + 2) % n_cor], z=-50, w=W, h=H 66 | ) 67 | bon_ceil_x.extend(xys[:, 0]) 68 | bon_ceil_y.extend(xys[:, 1]) 69 | for i in range(n_cor // 2): 70 | xys = panostretch.pano_connect_points( 71 | cor[i * 2 + 1], cor[(i * 2 + 3) % n_cor], z=50, w=W, h=H 72 | ) 73 | bon_floor_x.extend(xys[:, 0]) 74 | bon_floor_y.extend(xys[:, 1]) 75 | bon_ceil_x, bon_ceil_y = sort_xy_filter_unique( 76 | bon_ceil_x, bon_ceil_y, y_small_first=True 77 | ) 78 | bon_floor_x, bon_floor_y = sort_xy_filter_unique( 79 | bon_floor_x, bon_floor_y, y_small_first=False 80 | ) 81 | bon = np.zeros((2, W)) 82 | bon[0] = np.interp(np.arange(W), bon_ceil_x, bon_ceil_y, period=W) 83 | bon[1] = np.interp(np.arange(W), bon_floor_x, bon_floor_y, period=W) 84 | bon = ((bon + 0.5) / H - 0.5) * np.pi 85 | return bon 86 | 87 | 88 | def filter_out_noisy_layouts(list_ly, max_room_factor_size=2): 89 | """ 90 | Filters out the passed list_ly based on the cam2boundary instances defined for each ly. 91 | All layouts which have an estimation greater than max_room_factor_size x median( of all cam distances) are filtered out 92 | Args: 93 | list_ly (list): List of Layout instances 94 | max_room_factor_size (int, optional): max ly size allowed. Defaults to 2. 95 | """ 96 | # ! Filtering out noisy estimation 97 | logging.info(f"Filtering noisy layouts: initial #{list_ly.__len__()}") 98 | mean_ = np.median([ly.cam2boundary.max() for ly in list_ly]) 99 | list_ly = [ 100 | ly for ly in list_ly if ly.cam2boundary.max() < max_room_factor_size * mean_ 101 | ] 102 | logging.info(f"Filtering noisy layouts: final #{list_ly.__len__()}") 103 | 104 | 105 | def normalize_list_ly(list_ly, scale_and_center=None): 106 | """ 107 | Normalize the size of all layouts in a room between -1 to 1, 108 | centralizing all point to the median center of the room. 109 | """ 110 | 111 | if scale_and_center is None: 112 | pcl = np.hstack([ly.boundary_floor for ly in list_ly]) 113 | center = np.median(pcl, axis=1, keepdims=True) 114 | pcl = pcl - center 115 | scale = np.max(np.linalg.norm(pcl[(0, 2), :], axis=0)) 116 | else: 117 | scale, center = scale_and_center 118 | [ly.normalize_boundaries(scale, center) for ly in list_ly] 119 | return scale, center 120 | 121 | 122 | def load_pseudo_labels(list_ly, hard_copy=False): 123 | cfg = list_ly[0].cfg 124 | room_scene = list_ly[0].cfg._room_scene 125 | 126 | if hard_copy: 127 | list_ly = [deepcopy(ly) for ly in list_ly] 128 | 129 | pseudo_labels_dir = cfg.mlc_labels[cfg.mlc_data].labels_dir 130 | list_fn = os.listdir(os.path.join(pseudo_labels_dir, "mlc_label")) 131 | # Select only the pseudo labels which belong to the current room 132 | ps_labels = { 133 | Path(fn).stem: np.load(os.path.join(pseudo_labels_dir, "mlc_label", fn)) 134 | for fn in list_fn 135 | if room_scene in fn 136 | } 137 | [ly.recompute_data(phi_coords=ps_labels[ly.idx]) for ly in list_ly] 138 | return list_ly 139 | 140 | 141 | def get_boundary_from_list_corners(list_corner): 142 | boundary = [] 143 | for idx in range(list_corner.__len__()): 144 | direction = ( 145 | list_corner[(idx + 1) % list_corner.__len__()] - list_corner[idx] 146 | ).reshape(3, 1) 147 | wall_long = np.linalg.norm(direction) 148 | resolution = int(wall_long / 0.001) 149 | bound_points = list_corner[idx].reshape(3, 1) + direction * np.linspace( 150 | 0, 1, resolution 151 | ) 152 | boundary.append(bound_points) 153 | 154 | return np.hstack(boundary) 155 | -------------------------------------------------------------------------------- /mvl_challenge/utils/vispy_utils/vispy_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import vispy 3 | from functools import partial 4 | from vispy.scene import visuals 5 | from vispy.visuals.transforms import STTransform 6 | from vispy.scene.visuals import Text 7 | from vispy import app, scene, io 8 | from .data_visual import DataVisual, Visualization 9 | import sys 10 | from matplotlib.colors import hsv_to_rgb 11 | import vispy.io as vispy_file 12 | import os 13 | 14 | 15 | def plot_list_ly(list_ly=None, ceil_or_floor="", save_at=None): 16 | room_scene = list_ly[0].cfg._room_scene 17 | caption = room_scene 18 | if list_ly.__len__() < 1: 19 | return 20 | 21 | # ! Setting up vispy 22 | canvas = vispy.scene.SceneCanvas( 23 | keys="interactive", bgcolor="white", create_native=True 24 | ) 25 | res = 1024 * 2 26 | canvas.size = res, res // 2 27 | canvas.show() 28 | 29 | t1 = Text(room_scene, parent=canvas.scene, color="black") 30 | t1.font_size = 24 31 | t1.pos = canvas.size[0] // 2, canvas.size[1] // 10 32 | 33 | # Create two ViewBoxes, place side-by-side 34 | vb1 = scene.widgets.ViewBox(parent=canvas.scene) 35 | visuals.XYZAxis(parent=vb1.scene) 36 | grid = canvas.central_widget.add_grid() 37 | grid.padding = 6 38 | grid.add_widget(vb1, 0, 0) 39 | vb1.camera = vispy.scene.TurntableCamera( 40 | elevation=90, azimuth=180, roll=0, fov=0, up="-y" 41 | ) 42 | 43 | raw_ly_visual = DataVisual() 44 | ly_visual = DataVisual() 45 | 46 | raw_ly_visual.set_view(vb1) 47 | ly_visual.set_view(vb1) 48 | 49 | raw = [] 50 | # [raw.append(np.linalg.inv(ly.pose)[0:3, :] @ extend_array_to_homogeneous(ly.boundary)) for ly in list_ly] 51 | # list_pl = flatten_lists_of_lists([ly.list_planes for ly in list_obj]) 52 | if ceil_or_floor == "": 53 | [raw.append(ly.boundary_floor) for ly in list_ly] 54 | [raw.append(ly.boundary_ceiling) for ly in list_ly] 55 | elif ceil_or_floor == "ceil": 56 | [raw.append(ly.boundary_ceiling) for ly in list_ly] 57 | elif ceil_or_floor == "floor": 58 | [raw.append(ly.boundary_floor) for ly in list_ly] 59 | 60 | pcl_raw = np.hstack(raw) 61 | max_size = np.max( 62 | pcl_raw - np.mean(pcl_raw, axis=1, keepdims=True), axis=1, keepdims=True 63 | ) 64 | 65 | vb1.camera.scale_factor = np.max(max_size) * 2.5 66 | vb1.camera.center = np.mean(pcl_raw, axis=1, keepdims=True) 67 | 68 | # print(max_size) 69 | raw_ly_visual.plot_pcl(pcl_raw, raw_ly_visual.black_color, size=0.5) 70 | 71 | if save_at is not None: 72 | os.makedirs(save_at, exist_ok=True) 73 | res = canvas.render() 74 | vispy_file.write_png(os.path.join(save_at, f"{caption}.png"), res) 75 | return 76 | canvas.show() 77 | if sys.flags.interactive == 0: 78 | app.run() 79 | 80 | 81 | def plot_list_pcl(list_pcl, sizes, caption="Plotting PCL"): 82 | vis = Visualization(caption=caption) 83 | vis.set_visual_list(list_pcl.__len__()) 84 | 85 | assert list_pcl.__len__() == sizes.__len__() 86 | np.random.seed(500) 87 | color_idx = (0, vis.colors_list.shape[1], list_pcl.__len__()) 88 | 89 | for i, visual, pcl, size in zip( 90 | range(sizes.__len__()), vis.list_visuals, list_pcl, sizes 91 | ): 92 | visual.plot_pcl(pcl, vis.colors_list[:, i], size) 93 | 94 | vis.canvas.show() 95 | if sys.flags.interactive == 0: 96 | app.run() 97 | 98 | 99 | def get_color_list(array_colors=None, fr=0.1, return_list=False, number_of_colors=None): 100 | """ 101 | Returns a different color RGB for every element in the array_color 102 | """ 103 | if array_colors is not None: 104 | number_of_colors = len(array_colors) 105 | 106 | h = np.linspace(0.1, 0.8, number_of_colors) 107 | np.random.shuffle(h) 108 | # values = np.linspace(0, np.pi, number_of_colors) 109 | colors = np.ones((3, number_of_colors)) 110 | 111 | colors[0, :] = h 112 | 113 | return hsv_to_rgb(colors.T).T 114 | 115 | 116 | def setting_viewer(return_canvas=False, main_axis=True, bgcolor="black", caption=""): 117 | canvas = vispy.scene.SceneCanvas(keys="interactive", show=True, bgcolor=bgcolor) 118 | size_win = 1024 119 | canvas.size = 2 * size_win, size_win 120 | 121 | t1 = Text(caption, parent=canvas.scene, color="white") 122 | t1.font_size = 24 123 | t1.pos = canvas.size[0] // 2, canvas.size[1] // 10 124 | 125 | view = canvas.central_widget.add_view() 126 | view.camera = "arcball" # turntable / arcball / fly / perspective 127 | 128 | if main_axis: 129 | visuals.XYZAxis(parent=view.scene) 130 | 131 | if return_canvas: 132 | return view, canvas 133 | return view 134 | 135 | 136 | def setting_pcl(view, size=5, edge_width=2, antialias=0): 137 | scatter = visuals.Markers() 138 | scatter.set_gl_state( 139 | "translucent", 140 | depth_test=True, 141 | blend=True, 142 | blend_func=("src_alpha", "one_minus_src_alpha"), 143 | ) 144 | # scatter.set_gl_state(depth_test=True) 145 | scatter.antialias = 0 146 | view.add(scatter) 147 | return partial(scatter.set_data, size=size, edge_width=edge_width) 148 | 149 | 150 | def plot_color_plc( 151 | points, 152 | color=(0, 0, 0, 1), 153 | return_view=False, 154 | size=0.5, 155 | plot_main_axis=True, 156 | background="white", 157 | scale_factor=15, 158 | caption="", 159 | ): 160 | 161 | view = setting_viewer(main_axis=plot_main_axis, bgcolor=background, caption=caption) 162 | view.camera = vispy.scene.TurntableCamera( 163 | elevation=90, azimuth=90, roll=0, fov=0, up="-y" 164 | ) 165 | # view.camera = vispy.scene.TurntableCamera(elevation=90, 166 | # azimuth=0, 167 | # roll=0, 168 | # fov=0, 169 | # up='-y') 170 | view.camera.scale_factor = scale_factor 171 | draw_pcl = setting_pcl(view=view) 172 | draw_pcl(points, edge_color=color, size=size) 173 | if not return_view: 174 | vispy.app.run() 175 | else: 176 | return view 177 | -------------------------------------------------------------------------------- /mvl_challenge/data_structure/layout.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from mvl_challenge.utils.geometry_utils import extend_array_to_homogeneous 4 | from mvl_challenge.utils.spherical_utils import phi_coords2xyz 5 | 6 | from .cam_pose import CAM_REF 7 | from imageio import imread 8 | 9 | 10 | class Layout: 11 | @property 12 | def phi_coords(self): 13 | return self.__phi_coords 14 | 15 | @phi_coords.setter 16 | def phi_coords(self, value): 17 | self.__phi_coords = value 18 | if value is None: 19 | return 20 | self.recompute_ly_geometry() 21 | 22 | def set_phi_coords(self, phi_coords): 23 | self.phi_coords = phi_coords 24 | 25 | def __init__(self, cfg): 26 | self.cfg = cfg 27 | 28 | self.boundary_floor = None 29 | self.boundary_ceiling = None 30 | 31 | self.cam2boundary = None 32 | self.cam2boundary_mask = None 33 | 34 | self.bearings_floor = None 35 | self.bearings_ceiling = None 36 | 37 | self.img_fn = "" 38 | self.pose = np.eye(4) 39 | self.idx = "" 40 | 41 | self.phi_coords = None 42 | self.cam_ref = CAM_REF.CC 43 | self.ceiling_height = None # ! Must be None by default 44 | self.camera_height = 1 45 | self.primary = False 46 | self.scale = 1 47 | 48 | # ! data for normalize boundaries 49 | self.bound_scale = 1 50 | self.bound_center = np.zeros((3, 1)) 51 | 52 | def apply_vo_scale(self, scale): 53 | 54 | if self.cam_ref == CAM_REF.WC_SO3: 55 | self.boundary_floor = self.boundary_floor + ( 56 | scale / self.pose.vo_scale 57 | ) * np.ones_like(self.boundary_floor) * self.pose.t.reshape(3, 1) 58 | 59 | self.boundary_ceiling = self.boundary_ceiling + ( 60 | scale / self.pose.vo_scale 61 | ) * np.ones_like(self.boundary_ceiling) * self.pose.t.reshape(3, 1) 62 | 63 | self.cam_ref = CAM_REF.WC 64 | 65 | elif self.cam_ref == CAM_REF.WC: 66 | delta_scale = scale - self.pose.vo_scale 67 | self.boundary_floor = self.boundary_floor + ( 68 | delta_scale / self.pose.vo_scale 69 | ) * np.ones_like(self.boundary_floor) * self.pose.t.reshape(3, 1) 70 | 71 | self.boundary_ceiling = self.boundary_ceiling + ( 72 | delta_scale / self.pose.vo_scale 73 | ) * np.ones_like(self.boundary_ceiling) * self.pose.t.reshape(3, 1) 74 | 75 | self.pose.vo_scale = scale 76 | 77 | return True 78 | 79 | def estimate_height_ratio(self): 80 | """ 81 | Estimates the height ratio that describes the distance ratio of camera-ceboundary_ceiling over theboundary_ceiling-ceiling distance. This information is important to recover the 3D 82 | structure of the predicted Layout 83 | """ 84 | 85 | floor = np.abs(self.ly_data[1, :]) 86 | ceiling = np.abs(self.ly_data[0, :]) 87 | 88 | ceiling[ceiling > np.radians(80)] = np.radians(80) 89 | ceiling[ceiling < np.radians(5)] = np.radians(5) 90 | floor[floor > np.radians(80)] = np.radians(80) 91 | floor[floor < np.radians(5)] = np.radians(5) 92 | 93 | self.height_ratio = np.mean(np.tan(ceiling) / np.tan(floor)) 94 | 95 | def compute_cam2boundary(self): 96 | """ 97 | Computes the horizontal distance for every boundary point w.r.t camera pose. 98 | The boundary can be in any reference coordinates 99 | """ 100 | if self.cam_ref == CAM_REF.WC_SO3 or self.cam_ref == CAM_REF.CC: 101 | # ! Boundary reference still in camera reference 102 | self.cam2boundary = np.linalg.norm(self.boundary_floor[(0, 2), :], axis=0) 103 | 104 | else: 105 | assert self.cam_ref == CAM_REF.WC 106 | pcl = np.linalg.inv(self.pose.SE3_scaled())[ 107 | :3, : 108 | ] @ extend_array_to_homogeneous(self.boundary_floor) 109 | self.cam2boundary = np.linalg.norm(pcl[(0, 2), :], axis=0) 110 | 111 | # self.cam2boundary_mask = np.zeros_like(self.cam2boundary) 112 | # self.cam2boundary_mask = self.cam2boundary < np.quantile(self.cam2boundary, 0.25) 113 | 114 | def recompute_ly_geometry(self): 115 | 116 | # ! Compute bearings 117 | self.bearings_ceiling = phi_coords2xyz(phi_coords=self.phi_coords[0, :]) 118 | self.bearings_floor = phi_coords2xyz(phi_coords=self.phi_coords[1, :]) 119 | 120 | # ! Compute floor boundary 121 | ly_scale = self.camera_height / self.bearings_floor[1, :] 122 | pcl = ly_scale * self.bearings_floor * self.scale 123 | self.cam_ref = CAM_REF.WC 124 | self.boundary_floor = self.pose.SE3_scaled()[ 125 | :3, : 126 | ] @ extend_array_to_homogeneous(pcl) 127 | 128 | # from mlc.utils.vispy_utils.vispy_utils import plot_pcl 129 | # ! Compute ceiling boundary 130 | if self.ceiling_height is None: 131 | # ! forcing consistency between floor and ceiling 132 | scale_ceil = np.linalg.norm(pcl[(0, 2), :], axis=0) / np.linalg.norm( 133 | self.bearings_ceiling[(0, 2), :], axis=0 134 | ) 135 | pcl = scale_ceil * self.bearings_ceiling 136 | # plot_pcl(pcl) 137 | else: 138 | ly_scale = ( 139 | self.ceiling_height - self.camera_height 140 | ) / self.bearings_ceiling[1, :] 141 | pcl = ly_scale * self.bearings_ceiling * self.scale 142 | 143 | self.boundary_ceiling = self.pose.SE3_scaled()[ 144 | :3, : 145 | ] @ extend_array_to_homogeneous(pcl) 146 | self.compute_cam2boundary() 147 | 148 | def transform_to_WC_SO3(self): 149 | 150 | self.boundary_ceiling = self.boundary_ceiling - self.pose.t.reshape(3, 1) 151 | self.boundary_floor = self.boundary_floor - self.pose.t.reshape(3, 1) 152 | 153 | self.cam_ref = CAM_REF.WC_SO3 154 | 155 | def normalize_boundaries(self, scale, center): 156 | assert ( 157 | scale > 0 158 | ), "Zero or negative scale is not allowed to normalize a layout bondary" 159 | if self.bound_scale != scale: 160 | self.bound_scale = scale 161 | if np.linalg.norm(self.bound_center) != np.linalg.norm(center): 162 | self.bound_center = center 163 | 164 | self.boundary_floor = self.apply_normalization(self.bearings_floor) 165 | self.boundary_ceiling = self.apply_normalization(self.bearings_ceiling) 166 | 167 | def get_rgb(self): 168 | return imread(self.img_fn) 169 | 170 | def apply_normalization(self, xyz): 171 | return (xyz - self.bound_center)/self.bound_scale -------------------------------------------------------------------------------- /mvl_challenge/remote_data/zip_rgbd_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script allows us to zip the MP3D_FPE and HM3D-MVL dataset into several zip files separated by 3 | categories and scenes for convenience. i.e., 4 | - $SCENE_$VERSION_geometry.zip: Files which define geometries: cam poses, labels (floor plan only), selected key-frame 5 | - $SCENE_$VERSION_rgb.zip: Images defined per scene 6 | - $SCENE_$VERSION_depth.zip: Depth maps defined per scene 7 | - $SCENE_$VERSION_npy.zip: Pre-computed estimation using HorizonNet (some scene also content from HohoNet) 8 | Usage: 9 | python zip.py -s ${where the scene are stored} -o {where the zip files will be stored} -k [options: ["geom", "rgb", "depth", "npy"],] 10 | Example 11 | python mp3d_fpe/zip.py -s /MP3D_360_FPE/SINGLE_ROOM_SCENES/ -o MP3D_360_FPE/zipped_mp3d_fpe/ 12 | """ 13 | 14 | import argparse 15 | import os 16 | import zipfile 17 | 18 | from tqdm import tqdm 19 | 20 | from mvl_challenge.utils.io_utils import ( 21 | create_directory, 22 | get_files_given_a_pattern, 23 | read_csv_file, 24 | process_arcname, 25 | ) 26 | 27 | 28 | def zip_geometry_files(list_scenes, args): 29 | geometry_files = [ 30 | "room_gt_v0.0.yaml", 31 | "room_gt.yaml", 32 | "label.json", 33 | "frm_ref.txt", 34 | "keyframe_list.txt", 35 | "cam_pose_gt.csv", 36 | "cam_pose_estimated.csv", 37 | ] 38 | for scene in list_scenes: 39 | zip_filename = os.path.join(args.output_dir, f"{scene.replace('/', '_')}_geometry.zip") 40 | scene_dir = os.path.join(args.scene_dir, scene) 41 | with zipfile.ZipFile(file=zip_filename, mode="w") as zf: 42 | list_fn = [] 43 | for fn in geometry_files: 44 | list_fn += get_files_given_a_pattern( 45 | data_dir=os.path.join(args.scene_dir, scene), 46 | flag_file=fn, 47 | exclude=["depth", "hn_mp3d"], 48 | include_flag_file=True, 49 | ) 50 | list_arc_fn = process_arcname(list_fn, scene_dir) 51 | [ 52 | zf.write( 53 | os.path.join(scene_dir, fn), 54 | compress_type=zipfile.ZIP_STORED, 55 | arcname=os.path.join(scene, fn), 56 | ) 57 | for fn in tqdm(list_arc_fn, desc=f"zipping {os.path.join(scene, fn)}") 58 | ] 59 | 60 | 61 | def get_list_frames(scene_path): 62 | key_fr_fn = get_files_given_a_pattern( 63 | data_dir=scene_path, 64 | flag_file="keyframe_list.txt", 65 | exclude=["depth", "hn_mp3d"], 66 | include_flag_file=True, 67 | )[0] 68 | 69 | return read_csv_file(key_fr_fn) 70 | 71 | 72 | def zip_rgb_files(list_scenes, args): 73 | for scene in list_scenes: 74 | zip_filename = os.path.join(args.output_dir, f"{scene.replace('/', '_')}_rgb.zip") 75 | scene_dir = os.path.join(args.scene_dir, scene) 76 | with zipfile.ZipFile(file=zip_filename, mode="w") as zf: 77 | list_fn = get_list_frames(scene_dir) 78 | [ 79 | zf.write( 80 | os.path.join(scene_dir, "rgb", f"{fn}.png"), 81 | compress_type=zipfile.ZIP_STORED, 82 | arcname=os.path.join(scene, "rgb", f"{fn}.png"), 83 | ) 84 | for fn in tqdm(list_fn, desc=f"zipping {os.path.join(scene, 'rgb')}") 85 | ] 86 | 87 | 88 | def zip_depth_files(list_scenes, args): 89 | for scene in list_scenes: 90 | zip_filename = os.path.join(args.output_dir, f"{scene.replace('/', '_')}_depth.zip") 91 | scene_dir = os.path.join(args.scene_dir, scene) 92 | with zipfile.ZipFile(file=zip_filename, mode="w") as zf: 93 | list_fn = get_list_frames(scene_dir) 94 | [ 95 | zf.write( 96 | os.path.join(scene_dir, "depth", "tiff", f"{fn}.tiff"), 97 | compress_type=zipfile.ZIP_STORED, 98 | arcname=os.path.join(scene, "depth", "tiff", f"{fn}.tiff"), 99 | ) 100 | for fn in tqdm(list_fn, desc=f"zipping {os.path.join(scene, 'depth')}") 101 | ] 102 | 103 | 104 | def zip_npy_files(list_scenes, args): 105 | for scene in list_scenes: 106 | zip_filename = os.path.join(args.output_dir, f"{scene.replace('/', '_')}_npy.zip") 107 | scene_dir = os.path.join(args.scene_dir, scene) 108 | with zipfile.ZipFile(file=zip_filename, mode="w") as zf: 109 | list_fn = get_files_given_a_pattern( 110 | data_dir=scene_dir, 111 | flag_file=".npy", 112 | exclude=["depth", "rgb"], 113 | include_flag_file=True, 114 | ) 115 | 116 | [ 117 | zf.write( 118 | fn, 119 | compress_type=zipfile.ZIP_STORED, 120 | arcname=os.path.relpath(fn, start=scene_dir), 121 | ) 122 | for fn in tqdm(list_fn, desc=f"zipping npy files {scene}") 123 | ] 124 | 125 | 126 | def zip_rgbd_data(args): 127 | 128 | # ! Create output directory 129 | create_directory(args.output_dir, delete_prev=False) 130 | 131 | print(f"Identifying mvl scenes in {args.scene_dir}.") 132 | list_scenes = get_files_given_a_pattern( 133 | args.scene_dir, "minos_poses.txt", exclude=["depth", "rgb", "hn_mp3d"] 134 | ) 135 | list_arcname = process_arcname(list_scenes, base_dir=args.scene_dir) 136 | print(f"Found {list_arcname.__len__()} mvl scenes.") 137 | 138 | if "npy" in args.keys: 139 | zip_npy_files(list_arcname, args) 140 | 141 | if "rgb" in args.keys: 142 | zip_rgb_files(list_arcname, args) 143 | 144 | if "geom" in args.keys: 145 | zip_geometry_files(list_arcname, args) 146 | 147 | if "depth" in args.keys: 148 | zip_depth_files(list_arcname, args) 149 | 150 | 151 | def get_argparse(): 152 | parser = argparse.ArgumentParser() 153 | 154 | # * Input Directory (-s) 155 | parser.add_argument( 156 | "--scene_dir", 157 | # required=True, 158 | default="/media/public_dataset/MP3D_360_FPE/SINGLE_ROOM_SCENES", 159 | type=str, 160 | help="Input Directory (-source)", 161 | ) 162 | 163 | # * Output Directory (-o) 164 | parser.add_argument( 165 | "--output_dir", 166 | # required=True, 167 | default="/media/public_dataset/MP3D_360_FPE/zipped_mp3d_fpe", 168 | type=str, 169 | help="Output Directory (-o)", 170 | ) 171 | 172 | parser.add_argument( 173 | "--keys", 174 | # required=True, 175 | default=["geom", "rgb", "depth", "npy"], 176 | # default="rgb", 177 | nargs="+", 178 | help='Key argument in source ["geom", "rgb", "depth", "npy"]', 179 | ) 180 | 181 | args = parser.parse_args() 182 | return args 183 | 184 | 185 | if __name__ == "__main__": 186 | args = get_argparse() 187 | zip_rgbd_data(args) 188 | -------------------------------------------------------------------------------- /mvl_challenge/utils/geometry_utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import numpy as np 4 | from pyquaternion import Quaternion 5 | 6 | 7 | def get_quaternion_from_matrix(matrix): 8 | """ 9 | Returns the [qx, qy, qz qw] quaternion vector for the passed matrix (SE3 or SO3) 10 | """ 11 | q = Quaternion(matrix=matrix) 12 | return ( 13 | np.array((q.x, q.y, q.z, q.w)) if q.w > 0 else -np.array((q.x, q.y, q.z, q.w)) 14 | ) 15 | 16 | 17 | def tum_pose2matrix44(l, seq="xyzw"): 18 | """ 19 | Generate a 4x4 homogeneous transformation matrix from a 3D point and unit quaternion. 20 | 21 | Input: 22 | l -- tuple consisting of (stamp,tx,ty,tz,qx,qy,qz,qw) where 23 | (tx,ty,tz) is the 3D position and (qx,qy,qz,qw) is the unit quaternion. 24 | 25 | Output: 26 | matrix -- 4x4 homogeneous transformation matrix 27 | """ 28 | t = l[1:4] 29 | q = np.array(l[4:8], dtype=np.float64, copy=True) 30 | if seq == "wxyz": 31 | if q[0] < 0: 32 | q *= -1 33 | q = Quaternion(w=q[0], x=q[1], y=q[2], z=q[3]) 34 | else: 35 | if q[3] < 0: 36 | q *= -1 37 | q = Quaternion( 38 | x=q[0], 39 | y=q[1], 40 | z=q[2], 41 | w=q[3], 42 | ) 43 | transform = np.eye(4) 44 | transform[0:3, 0:3] = q.rotation_matrix 45 | transform[0:3, 3] = np.array(t) 46 | 47 | return transform 48 | 49 | 50 | def isRotationMatrix(R): 51 | """ 52 | Checks if a matrix is a valid rotation matrix. 53 | """ 54 | Rt = np.transpose(R) 55 | shouldBeIdentity = np.dot(Rt, R) 56 | I = np.identity(3, dtype=R.dtype) 57 | n = np.linalg.norm(I - shouldBeIdentity) 58 | return n < 1e-6 59 | 60 | 61 | def get_xyz_from_phi_coords(phi_coords): 62 | """ 63 | Computes the xyz PCL from the ly_data (bearings_phi / phi_coords) 64 | """ 65 | bearings_floor = get_bearings_from_phi_coords(phi_coords=phi_coords[1, :]) 66 | 67 | # ! Projecting bearing to 3D as pcl --> boundary 68 | # > Forcing ly-scale = 1 69 | ly_scale = 1 / bearings_floor[1, :] 70 | pcl_floor = ly_scale * bearings_floor 71 | return pcl_floor 72 | 73 | 74 | def get_bearings_from_phi_coords(phi_coords): 75 | """ 76 | Returns 3D bearing vectors (on the unite sphere) from phi_coords 77 | """ 78 | W = phi_coords.__len__() 79 | u = np.linspace(0, W - 1, W) 80 | theta_coords = (2 * np.pi * u / W) - np.pi 81 | bearings_y = -np.sin(phi_coords) 82 | bearings_x = np.cos(phi_coords) * np.sin(theta_coords) 83 | bearings_z = np.cos(phi_coords) * np.cos(theta_coords) 84 | return np.vstack((bearings_x, bearings_y, bearings_z)) 85 | 86 | 87 | def stack_camera_poses(list_poses): 88 | """ 89 | Stack a list of camera poses using Kronecker product 90 | https://en.wikipedia.org/wiki/Kronecker_product 91 | """ 92 | M = np.zeros((list_poses.__len__() * 3, list_poses.__len__() * 4)) 93 | for idx in range(list_poses.__len__()): 94 | aux = np.zeros((list_poses.__len__(), list_poses.__len__())) 95 | aux[idx, idx] = 1 96 | M += np.kron(aux, list_poses[idx][0:3, :]) 97 | return M 98 | 99 | 100 | def extend_array_to_homogeneous(array): 101 | """ 102 | Returns the homogeneous form of a vector by attaching 103 | a unit vector as additional dimensions 104 | Parameters 105 | ---------- 106 | array of (3, n) or (2, n) 107 | Returns (4, n) or (3, n) 108 | ------- 109 | """ 110 | try: 111 | assert array.shape[0] in (2, 3, 4) 112 | dim, samples = array.shape 113 | return np.vstack((array, np.ones((1, samples)))) 114 | 115 | except: 116 | assert array.shape[1] in (2, 3, 4) 117 | array = array.T 118 | dim, samples = array.shape 119 | return np.vstack((array, np.ones((1, samples)))).T 120 | 121 | 122 | def extend_vector_to_homogeneous_transf(vector): 123 | """ 124 | Creates a homogeneous transformation (4, 4) given a vector R3 125 | :param vector: vector R3 (3, 1) or (4, 1) 126 | :return: Homogeneous transformation (4, 4) 127 | """ 128 | T = np.eye(4) 129 | if vector.__class__.__name__ == "dict": 130 | T[0, 3] = vector["x"] 131 | T[1, 3] = vector["y"] 132 | T[2, 3] = vector["z"] 133 | elif type(vector) == np.array: 134 | T[0:3, 3] = vector[0:3, 0] 135 | else: 136 | T[0:3, 3] = vector[0:3] 137 | return T 138 | 139 | 140 | def eulerAnglesToRotationMatrix(angles): 141 | theta = np.zeros((3)) 142 | 143 | if angles.__class__.__name__ == "dict": 144 | theta[0] = angles["x"] 145 | theta[1] = angles["y"] 146 | theta[2] = angles["z"] 147 | else: 148 | theta[0] = angles[0] 149 | theta[1] = angles[1] 150 | theta[2] = angles[2] 151 | 152 | R_x = np.array( 153 | [ 154 | [1, 0, 0], 155 | [0, math.cos(theta[0]), -math.sin(theta[0])], 156 | [0, math.sin(theta[0]), math.cos(theta[0])], 157 | ] 158 | ) 159 | 160 | R_y = np.array( 161 | [ 162 | [math.cos(theta[1]), 0, math.sin(theta[1])], 163 | [0, 1, 0], 164 | [-math.sin(theta[1]), 0, math.cos(theta[1])], 165 | ] 166 | ) 167 | 168 | R_z = np.array( 169 | [ 170 | [math.cos(theta[2]), -math.sin(theta[2]), 0], 171 | [math.sin(theta[2]), math.cos(theta[2]), 0], 172 | [0, 0, 1], 173 | ] 174 | ) 175 | 176 | R = np.dot(R_z, np.dot(R_y, R_x)) 177 | return R 178 | 179 | 180 | def rotationMatrixToEulerAngles(R): 181 | """rotationMatrixToEulerAngles retuns the euler angles of a SO3 matrix""" 182 | assert isRotationMatrix(R) 183 | 184 | sy = math.sqrt(R[0, 0] * R[0, 0] + R[1, 0] * R[1, 0]) 185 | 186 | singular = sy < 1e-6 187 | 188 | if not singular: 189 | x = math.atan2(R[2, 1], R[2, 2]) 190 | y = math.atan2(-R[2, 0], sy) 191 | z = math.atan2(R[1, 0], R[0, 0]) 192 | else: 193 | x = math.atan2(-R[1, 2], R[1, 1]) 194 | y = math.atan2(-R[2, 0], sy) 195 | z = 0 196 | 197 | return np.array([x, y, z]) 198 | 199 | 200 | def vector2skew_matrix(vector): 201 | """ 202 | Converts a vector [3,] into a matrix [3, 3] for cross product operations. v x v' = [v]v' where [v] is a skew representation of v 203 | :param vector: [3,] 204 | :return: skew matrix [3, 3] 205 | """ 206 | vector = vector.ravel() 207 | assert vector.size == 3 208 | 209 | skew_matrix = np.zeros((3, 3)) 210 | skew_matrix[1, 0] = vector[2] 211 | skew_matrix[2, 0] = -vector[1] 212 | skew_matrix[0, 1] = -vector[2] 213 | skew_matrix[2, 1] = vector[0] 214 | skew_matrix[0, 2] = vector[1] 215 | skew_matrix[1, 2] = -vector[0] 216 | 217 | return skew_matrix 218 | 219 | 220 | def skew_matrix2vector(matrix): 221 | assert matrix.shape == (3, 3) 222 | 223 | vector = np.zeros((3, 1)) 224 | 225 | vector[0] = matrix[2, 1] 226 | vector[1] = -matrix[2, 0] 227 | vector[2] = matrix[1, 0] 228 | 229 | return vector 230 | -------------------------------------------------------------------------------- /mvl_challenge/pre_processing/create_npz_labels.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | from tqdm import tqdm 5 | from pathlib import Path 6 | import numpy as np 7 | import logging 8 | from imageio import imread, imwrite 9 | from mvl_challenge import EPILOG, ASSETS_DIR, CFG_DIR 10 | from mvl_challenge.utils.io_utils import create_directory 11 | from mvl_challenge.config.cfg import set_loggings, get_empty_cfg 12 | from mvl_challenge.utils.io_utils import ( 13 | get_scene_room_from_scene_room_idx, 14 | get_all_frames_from_scene_list, 15 | ) 16 | from mvl_challenge.utils.layout_utils import get_boundary_from_list_corners 17 | from mvl_challenge.utils.spherical_utils import ( 18 | xyz2uv, 19 | uv2phi_coords, 20 | phi_coords2uv, 21 | phi_coords2xyz, 22 | ) 23 | from mvl_challenge.utils.vispy_utils import plot_color_plc 24 | from mvl_challenge.utils.geometry_utils import ( 25 | tum_pose2matrix44, 26 | extend_array_to_homogeneous, 27 | ) 28 | from mvl_challenge.utils.image_utils import ( 29 | draw_boundaries_uv, 30 | draw_boundaries_phi_coords, 31 | ) 32 | from mvl_challenge.utils.image_utils import add_caption_to_image 33 | from mvl_challenge.pre_processing.create_scene_room_idx_list import ( 34 | save_scene_list_from_mvl_directory, 35 | ) 36 | 37 | 38 | def save_phi_bound(args, list_corners, scene_room_idx_list): 39 | # ! ceiling 40 | crn_ceil = [np.array(c[0]).T for c in list_corners] 41 | bound_ceil = get_boundary_from_list_corners(crn_ceil) 42 | 43 | # ! Floor 44 | crn_floor = [np.array(c[1]).T for c in list_corners] 45 | bound_floor = get_boundary_from_list_corners(crn_floor) 46 | 47 | gt_dir = create_directory(args.output_dir, delete_prev=False) 48 | gt_vis_dir = create_directory(args.output_dir + "_vis", delete_prev=False) 49 | 50 | for sc_rm_idx in scene_room_idx_list: 51 | scene_room_idx = Path(sc_rm_idx).stem 52 | geom_info_fn = os.path.join(args.geom_info_dir, f"{sc_rm_idx}.json") 53 | assert os.path.exists(geom_info_fn), f"Not found {geom_info_fn}" 54 | geom_data = json.load(open(geom_info_fn, "r")) 55 | 56 | # ! Getting SE3 transformation in geom_info 57 | cam_pose = tum_pose2matrix44( 58 | [-1] + geom_data["translation"] + geom_data["quaternion"] 59 | ) 60 | 61 | # ! Transform into camera coordinates 62 | local_xyz_ceil = np.linalg.inv(cam_pose)[:3, :] @ extend_array_to_homogeneous( 63 | bound_ceil 64 | ) 65 | local_xyz_floor = np.linalg.inv(cam_pose)[:3, :] @ extend_array_to_homogeneous( 66 | bound_floor 67 | ) 68 | 69 | # ! projection into uv coord 70 | uv_ceil = xyz2uv(local_xyz_ceil) 71 | uv_floor = xyz2uv(local_xyz_floor) 72 | 73 | img = imread( 74 | Path(args.geom_info_dir).parent.__str__() + f"/img/{scene_room_idx}.jpg" 75 | ) 76 | 77 | #! Projection into phi_coords 78 | phi_coords_ceil = uv2phi_coords(uv_ceil, type_bound="ceiling") 79 | phi_coords_floor = uv2phi_coords(uv_floor, type_bound="floor") 80 | if phi_coords_ceil is None or phi_coords_floor is None: 81 | logging.warning("phi_coords is None") 82 | continue 83 | phi_coords = np.vstack([phi_coords_ceil, phi_coords_floor]) 84 | assert phi_coords.shape == (2, 1024) 85 | img = add_caption_to_image(image=img, caption="mvl-challenge " + scene_room_idx) 86 | 87 | draw_boundaries_phi_coords(img, phi_coords) 88 | 89 | gt_vis_fn = os.path.join(gt_vis_dir, f"{scene_room_idx}.jpg") 90 | gt_fn = os.path.join(gt_dir, f"{scene_room_idx}") 91 | 92 | imwrite(gt_vis_fn, img) 93 | np.savez_compressed(gt_fn, phi_coords=phi_coords) 94 | print(f"Saving... {gt_fn}.npz", end="\r") 95 | # logging.info(f"Saved {gt_fn}") 96 | print(f"Finished {gt_dir}") 97 | 98 | 99 | def main(args): 100 | # ! Reading geometry info 101 | set_loggings() 102 | 103 | create_directory(args.output_dir, delete_prev=False) 104 | scene_room_idx_list = get_all_frames_from_scene_list(args.scene_list) 105 | 106 | all_rooms = np.unique( 107 | [get_scene_room_from_scene_room_idx(room) for room in scene_room_idx_list] 108 | ).tolist() 109 | scene_rooms = np.unique(["_".join(f.split("_")[:2]) for f in all_rooms]).tolist() 110 | for scene in tqdm(scene_rooms, desc="Loading scenes..."): 111 | list_room_per_scene = [r for r in all_rooms if scene in r] 112 | list_room_per_scene = sorted( 113 | list_room_per_scene, key=lambda x: int(x.split("room")[-1]) 114 | ) 115 | for room_name in list_room_per_scene: 116 | #! idx must coincide with {scene}_{ver}_room{idx}_{idx_fr} 117 | idx = int(room_name.split("room")[-1]) 118 | scene_name, version, room = Path(room_name).stem.split("_") 119 | scene_dir = os.path.join(args.scene_dir, scene_name, version) 120 | 121 | mvl_labels_fn = os.path.join(scene_dir, "mvl_challenge_labels.json") 122 | if os.path.exists(mvl_labels_fn): 123 | logging.info(f"Saving GT labels for {room_name}") 124 | #! We process only scenes with mvl-labels 125 | 126 | mvl_data = json.load(open(mvl_labels_fn, "r")) 127 | list_corners = mvl_data["room_corners"][idx] 128 | 129 | save_phi_bound( 130 | args, 131 | list_corners, 132 | [g for g in scene_room_idx_list if room_name in g], 133 | ) 134 | 135 | # # ! Save scene_list for GT labels 136 | # cfg = get_empty_cfg() 137 | # cfg.mvl_dir = args.output_dir 138 | # cfg.output_dir = Path(args.output_dir).parent.__str__() 139 | # cfg.output_filename = "gt_labels__scene_list" 140 | # save_scene_list_from_mvl_directory(cfg) 141 | 142 | 143 | def get_argparse(): 144 | desc = ( 145 | "This script creates the npy files from the mvl-annotation. " 146 | + "This npy files content the boundary defined for ceiling and floor." 147 | ) 148 | 149 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 150 | 151 | parser.add_argument( 152 | "-d", 153 | "--scene_dir", 154 | # required=True, 155 | default="/media/public_dataset/MP3D_360_FPE/MULTI_ROOM_SCENES/", 156 | # default="/media/public_dataset/HM3D-MVL/test", 157 | type=str, 158 | help="RGBD dataset directory.", 159 | ) 160 | 161 | parser.add_argument( 162 | "-g", 163 | "--geom_info_dir", 164 | # required=True, 165 | default=f"{ASSETS_DIR}/issue_omitted_frames/geometry_info", 166 | type=str, 167 | help="Geometry information directory.", 168 | ) 169 | 170 | parser.add_argument( 171 | "-f", 172 | "--scene_list", 173 | # required=True, 174 | default=f"{ASSETS_DIR}/issue_omitted_frames/test_scene_list.json", 175 | type=str, 176 | help="Scene List.", 177 | ) 178 | 179 | parser.add_argument( 180 | "-o", 181 | "--output_dir", 182 | # required=True, 183 | default=f"{ASSETS_DIR}/issue_omitted_frames/labels/gt", 184 | type=str, 185 | help="Output directory for the output_file to be created.", 186 | ) 187 | 188 | args = parser.parse_args() 189 | return args 190 | 191 | 192 | if __name__ == "__main__": 193 | args = get_argparse() 194 | main(args) 195 | -------------------------------------------------------------------------------- /mvl_challenge/datasets/rgbd_datasets.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import logging 4 | import os 5 | from pathlib import Path 6 | 7 | import numpy as np 8 | from tqdm import tqdm 9 | 10 | from mvl_challenge.config.cfg import get_empty_cfg 11 | from mvl_challenge.data_structure.frame import Frame 12 | from mvl_challenge.utils.io_utils import read_trajectory, get_idx_from_scene_room_idx 13 | from mvl_challenge.utils.spherical_utils import SphericalCamera 14 | from mvl_challenge.utils.vispy_utils import plot_color_plc 15 | import json 16 | 17 | 18 | class RGBD_Dataset: 19 | @classmethod 20 | def from_args(clc, args): 21 | assert os.path.exists(args.scene_dir) 22 | 23 | cfg = get_empty_cfg() 24 | cfg.dataset = dict() 25 | cfg.dataset.scene_dir = args.scene_dir 26 | return clc.from_cfg(cfg) 27 | 28 | @classmethod 29 | def from_cfg(clc, cfg): 30 | # MP3D-FPE dataset has a vo* directory 31 | vo_dir = glob.glob(os.path.join(cfg.dataset.scene_dir, "vo*")) 32 | if vo_dir.__len__() == 0: 33 | # HM3D-MVL dataset 34 | dt = HM3D_MVL(cfg) 35 | else: 36 | # MP3D-FPE 37 | dt = MP3D_FPE(cfg) 38 | 39 | return dt 40 | 41 | @classmethod 42 | def from_scene_dir(clc, scene_dir): 43 | assert os.path.exists(scene_dir) 44 | cfg = get_empty_cfg() 45 | cfg.dataset = dict() 46 | cfg.dataset.scene_dir = scene_dir 47 | return clc.from_cfg(cfg) 48 | 49 | def __init__(self, cfg): 50 | self.cfg = cfg 51 | self.set_paths() 52 | self.load_data() 53 | self.cam = SphericalCamera(shape=cfg.get("resolution", (512, 1024))) 54 | logging.info(f"RGBD dataset loaded successfully") 55 | logging.info(f"Scene directory: {self.scene_dir}") 56 | logging.info(f"Scene name: {self.scene_name}") 57 | 58 | def load_data(self): 59 | # ! List of Kf 60 | self.set_list_of_frames() 61 | 62 | # ! List of camera poses 63 | self.load_camera_poses() 64 | 65 | # ! List of files 66 | self.rgb_files = [ 67 | os.path.join(self.rgb_dir, f"{f}.{self.rgb_ext}") for f in self.kf_list 68 | ] 69 | self.depth_files = [ 70 | os.path.join(self.depth_dir, f"{f}.tiff") for f in self.kf_list 71 | ] 72 | 73 | def load_camera_poses(self): 74 | """ 75 | Load both GT camera poses 76 | """ 77 | 78 | # ! Loading GT camera poses 79 | gt_poses_file = os.path.join(self.scene_dir, "frm_ref.txt") 80 | 81 | assert os.path.isfile( 82 | gt_poses_file 83 | ), f"Cam pose file {gt_poses_file} does not exist" 84 | 85 | self.gt_poses = np.stack(list(read_trajectory(gt_poses_file).values()))[ 86 | self.idx, :, : 87 | ] 88 | 89 | def set_paths(self): 90 | self.scene_dir = Path(self.cfg.dataset.scene_dir).resolve().__str__() 91 | self.scene_name = "_".join(self.scene_dir.split("/")[-2:]) 92 | # set rgb directory 93 | self.rgb_dir = os.path.join(self.scene_dir, "rgb") 94 | if not os.path.isdir(self.rgb_dir): 95 | raise FileExistsError(f"{self.rgb_dir}") 96 | 97 | # set depth directory 98 | self.depth_dir = os.path.join(self.scene_dir, "depth", "tiff") 99 | if not os.path.isdir(self.depth_dir): 100 | raise FileExistsError(f"{self.depth_dir}") 101 | 102 | def set_list_of_frames(self): 103 | 104 | raise NotImplementedError( 105 | """ 106 | Setting the list of frames in the data depends on the dataset. 107 | MP3D-FPE uses a set of estimated key-frames, which are defined in a vo-* directory. 108 | While HM3D-MVL uses directly the available images in rgb/ directory. 109 | """ 110 | ) 111 | 112 | def get_list_frames(self): 113 | list_fr = [] 114 | for rgb_fn, depth_fn, pose in tqdm( 115 | zip(self.rgb_files, self.depth_files, self.gt_poses), 116 | desc="Loading frames...", 117 | ): 118 | fr = Frame(self) 119 | fr.rgb_file = rgb_fn 120 | fr.depth_map_file = depth_fn 121 | fr.idx = int(Path(rgb_fn).stem) 122 | fr.pose = pose 123 | list_fr.append(fr) 124 | return list_fr 125 | 126 | def iter_rooms_scenes(self): 127 | """ 128 | Create a iter obj which yield per room the list of fr on it. 129 | This function is only available for the MP3D-FPE dataset 130 | """ 131 | # ! scene_room_idx definition file 132 | scenes_room_fn = self.cfg.dataset.scene_list 133 | scenes_room = json.load(open(scenes_room_fn, "r")) 134 | 135 | #! Select frames related to the scene in this class 136 | room_names = [r for r in list(scenes_room.keys()) if self.scene_name in r] 137 | 138 | #! Select all fr in the scene 139 | list_fr = self.get_list_frames() 140 | for room in room_names: 141 | logging.info(f"Reading room data {room}") 142 | #! list of indexes defined in the room 143 | list_frm_idx = [ 144 | get_idx_from_scene_room_idx(fr_name) for fr_name in scenes_room[room] 145 | ] 146 | room_list_fr = [fr for fr in list_fr if fr.idx in list_frm_idx] 147 | if room_list_fr.__len__() == 0: 148 | continue 149 | # ! Set the room the corresponding room information 150 | initial_pose = room_list_fr[0].pose.copy() 151 | [r.set_room_data(room.replace(".", ""), initial_pose) for r in room_list_fr] 152 | 153 | yield room_list_fr 154 | 155 | 156 | class MP3D_FPE(RGBD_Dataset): 157 | def __init__(self, cfg): 158 | self.rgb_ext = "png" 159 | super().__init__(cfg) 160 | 161 | def set_list_of_frames(self): 162 | self.vo_dir = glob.glob(os.path.join(self.scene_dir, "vo*"))[0] 163 | list_keyframe_fn = os.path.join(self.vo_dir, "keyframe_list.txt") 164 | assert os.path.exists(list_keyframe_fn), f"{list_keyframe_fn}" 165 | 166 | with open(list_keyframe_fn, "r") as f: 167 | self.kf_list = sorted([int(kf) for kf in f.read().splitlines()]) 168 | self.idx = np.array(self.kf_list) - 1 169 | 170 | def __str__(self): 171 | return "MP3D_FPE" 172 | 173 | 174 | class HM3D_MVL(RGBD_Dataset): 175 | def __init__(self, cfg): 176 | self.rgb_ext = "jpg" 177 | super().__init__(cfg) 178 | 179 | def set_list_of_frames(self): 180 | self.kf_list = sorted( 181 | [int(os.path.basename(f).split(".")[0]) for f in os.listdir(self.rgb_dir)] 182 | ) 183 | self.idx = np.array(self.kf_list) 184 | 185 | def __str__(self): 186 | return "HM3D_MVL" 187 | 188 | 189 | def get_default_args(): 190 | parser = argparse.ArgumentParser() 191 | 192 | # * Input Directory (-s) 193 | parser.add_argument( 194 | "-scene_dir", 195 | # required=True, 196 | default="/media/public_dataset/MP3D_360_FPE/SINGLE_ROOM_SCENES/2t7WUuJeko7/0", 197 | type=str, 198 | help="Input Directory (scene directory defined until version scene)", 199 | ) 200 | 201 | args = parser.parse_args() 202 | 203 | return args 204 | 205 | 206 | def main(args): 207 | dt = MP3D_FPE.from_args(args) 208 | list_fr = dt.get_list_frames() 209 | pcl = np.hstack([fr.get_pcl() for fr in list_fr[:4]]) 210 | plot_color_plc(points=pcl[0:3, :].T, color=pcl[3:].T) 211 | 212 | 213 | if __name__ == "__main__": 214 | args = get_default_args() 215 | main(args) 216 | -------------------------------------------------------------------------------- /mvl_challenge/scene_list__edit_info.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import numpy as np 5 | from mvl_challenge.utils.io_utils import save_json_dict 6 | from mvl_challenge.config.cfg import get_empty_cfg 7 | from pathlib import Path 8 | from mvl_challenge.scene_list__get_info import print_scene_data_info 9 | 10 | 11 | def prune_scene_data(scene_data, max_fr): 12 | new_scene_data = {} 13 | for room_id, list_frames in scene_data.items(): 14 | new_scene_data[room_id] = prune_list_frames(list_frames, max_fr) 15 | return new_scene_data 16 | 17 | 18 | def prune_list_frames(list_fr, max_fr): 19 | print(f"Initial number of frames: {list_fr.__len__()}") 20 | while True: 21 | ratio = round(list_fr.__len__() / max_fr) 22 | if ratio <= 1: 23 | break 24 | list_fr = [f for idx, f in enumerate(list_fr) if idx % ratio == 0] 25 | if list_fr.__len__() < max_fr: 26 | break 27 | list_fr.sort(key=lambda x: int(x.split("_")[-1])) 28 | print(f"Final number of frames: {list_fr.__len__()}") 29 | return list_fr 30 | 31 | 32 | def diff_scene_lists(scene_data_1, scene_data_2): 33 | new_scene_data = {} 34 | room_ref_list = list(scene_data_1.keys()) 35 | room_list = list(scene_data_2.keys()) 36 | room_diff = [r for r in room_ref_list if r not in room_list] 37 | for room_id, scene_idx in scene_data_1.items(): 38 | if room_id in room_diff: 39 | new_scene_data[room_id] = scene_idx 40 | return new_scene_data 41 | 42 | 43 | def intersect_scene_lists(scene_data_1, scene_data_2): 44 | new_scene_data = {} 45 | room_ref_list = list(scene_data_1.keys()) 46 | room_list = list(scene_data_2.keys()) 47 | common_rooms = [r for r in room_ref_list if r in room_list] 48 | for room_id, scene_idx in scene_data_1.items(): 49 | if room_id in common_rooms: 50 | new_scene_data[room_id] = scene_idx 51 | return new_scene_data 52 | 53 | 54 | def complement_scene_lists(scene_data_1, scene_data_2): 55 | intersect__list = intersect_scene_lists(scene_data_1, scene_data_2) 56 | diff_1_list = diff_scene_lists(scene_data_1, intersect__list) 57 | diff_2_list = diff_scene_lists(scene_data_2, intersect__list) 58 | new_scene_data = merge_scene_lists(diff_1_list, diff_2_list) 59 | return new_scene_data 60 | 61 | 62 | def merge_scene_lists(scene_data_1, scene_data_2): 63 | new_scene_data = {} 64 | for data in (scene_data_1, scene_data_2): 65 | for room_id, scene_idx in data.items(): 66 | new_scene_data[room_id] = scene_idx 67 | return new_scene_data 68 | 69 | 70 | def split_scene_list(scene_data, split_list): 71 | new_scene_data = {} 72 | for split, scenes in split_list.items(): 73 | print(f"Split: {split}") 74 | new_scene_data[split] = {k: v for k, v in scene_data.items() if k in scenes} 75 | if list(new_scene_data[split].keys()).__len__() == 0: 76 | new_scene_data[split] = { 77 | k: v for k, v in scene_data.items() if k.split("_")[0] in scenes 78 | } 79 | if list(new_scene_data[split].keys()).__len__() == 0: 80 | new_scene_data[split] = { 81 | k: v for k, v in scene_data.items() if k.split("_room")[0] in scenes 82 | } 83 | 84 | return new_scene_data 85 | 86 | 87 | def sampling_scene_list(scene_data, ratio): 88 | list_rooms = list(scene_data.keys()) 89 | if ratio > 1: 90 | room_number = int(ratio) 91 | else: 92 | room_number = int(ratio * list_rooms.__len__()) 93 | np.random.shuffle(list_rooms) 94 | return {r: scene_data[r] for r in list_rooms[:room_number]} 95 | 96 | 97 | def main(args): 98 | if args.r > 0: 99 | scene_data = json.load(open(args.scene_list)) 100 | new_scene_data = sampling_scene_list(scene_data, args.r) 101 | elif args.action == "split": 102 | scene_data = json.load(open(args.scene_list)) 103 | split_list = json.load(open(args.x)) 104 | new_scene_data = split_scene_list(scene_data, split_list) 105 | for key, data in new_scene_data.items(): 106 | fn = os.path.join( 107 | os.path.dirname(args.scene_list), f"{Path(args.o).stem}__{key}.json" 108 | ) 109 | save_json_dict(fn, data) 110 | print(f"Scene List saved as {fn}") 111 | return 112 | 113 | elif args.action == "merge": 114 | scene_data = json.load(open(args.scene_list)) 115 | another_scene_data = json.load(open(args.x)) 116 | new_scene_data = merge_scene_lists(scene_data, another_scene_data) 117 | elif args.action == "diff": 118 | scene_data = json.load(open(args.scene_list)) 119 | another_scene_data = json.load(open(args.x)) 120 | new_scene_data = diff_scene_lists(scene_data, another_scene_data) 121 | elif args.action == "intersect": 122 | scene_data = json.load(open(args.scene_list)) 123 | another_scene_data = json.load(open(args.x)) 124 | new_scene_data = intersect_scene_lists(scene_data, another_scene_data) 125 | elif args.action == "complement": 126 | scene_data = json.load(open(args.scene_list)) 127 | another_scene_data = json.load(open(args.x)) 128 | new_scene_data = complement_scene_lists(scene_data, another_scene_data) 129 | 130 | elif args.prune > 0: 131 | scene_data = json.load(open(args.scene_list)) 132 | new_scene_data = prune_scene_data(scene_data, args.prune) 133 | else: 134 | print("Not action found") 135 | return 136 | 137 | if args.v: 138 | print_scene_data_info(new_scene_data) 139 | return 140 | 141 | fn = os.path.join(os.path.dirname(args.scene_list), f"{Path(args.o).stem}.json") 142 | save_json_dict(fn, new_scene_data) 143 | print(f"Scene List saved as {fn}") 144 | 145 | 146 | def get_argparse(): 147 | parser = argparse.ArgumentParser() 148 | 149 | parser.add_argument( 150 | "-f", 151 | "--scene_list", 152 | # required=True, 153 | # default=f"{ASSETS_DIR}/stats/hm3d_mvl__train__scene_list.json", 154 | # default=f"{ASSETS_DIR}/stats/mp3d_fpe__multiple_rooms__scene_list.json", 155 | # default="/media/public_dataset/mvl_challenge/hm3d_mvl/03.14.2023__all_hm3d_mvl__scene_list.json", 156 | default="/media/public_dataset/mvl_challenge/hm3d_mvl/scene_list__test.json", 157 | type=str, 158 | help="Original scene list (scene_room_idx json file).", 159 | ) 160 | 161 | parser.add_argument( 162 | "-x", 163 | "--x", 164 | # required=True, 165 | # default=f"{ASSETS_DIR}/stats/hm3d_mvl__train__scene_list.json", 166 | # default=f"{ASSETS_DIR}/stats/mp3d_fpe__multiple_rooms__scene_list.json", 167 | # default="/media/public_dataset/mvl_challenge/hm3d_mvl/03.14.2023__all_hm3d_mvl__scene_list.json", 168 | default="/media/public_dataset/mvl_challenge/hm3d_mvl/scene_list__test.json", 169 | type=str, 170 | help="extra files.", 171 | ) 172 | 173 | parser.add_argument( 174 | "-o", default="pruned_scene_list", type=str, help="Output scene_list name." 175 | ) 176 | 177 | parser.add_argument( 178 | "-a", 179 | "--action", 180 | type=str, 181 | help="Actions over the passed files [merge, split, diff, intersect, complement].", 182 | ) 183 | 184 | parser.add_argument("-r", default=-1, type=float, help="Random sampling.") 185 | 186 | parser.add_argument( 187 | "--prune", type=int, default=-1, help="Max number of fr per room." 188 | ) 189 | 190 | parser.add_argument( 191 | "-v", action="store_true", help="To visualize info without creating a file" 192 | ) 193 | 194 | args = parser.parse_args() 195 | return args 196 | 197 | 198 | if __name__ == "__main__": 199 | args = get_argparse() 200 | main(args) 201 | -------------------------------------------------------------------------------- /mvl_challenge/utils/io_utils.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | import logging 4 | import os 5 | import shutil 6 | import sys 7 | from pathlib import Path 8 | from tqdm import tqdm 9 | import dill 10 | import numpy as np 11 | from plyfile import PlyData 12 | from pyquaternion import Quaternion 13 | 14 | 15 | def get_idx_from_scene_room_idx(fr_name): 16 | return int(fr_name.split("_")[-1].split(".")[0]) 17 | 18 | 19 | def get_all_frames_from_scene_list(scene_list_fn): 20 | data = json.load(open(scene_list_fn, "r")) 21 | list_geom_info = [f for f in data.values()] 22 | list_geom_info = [item for sublist in list_geom_info for item in sublist] 23 | return list_geom_info 24 | 25 | 26 | def get_rgbd_scenes_list(args): 27 | dir_data = args.scene_dir 28 | scene_list = get_all_frames_from_scene_list(args.scene_list) 29 | return np.unique( 30 | [os.path.join(dir_data, f.split("_")[0], f.split("_")[1]) for f in scene_list] 31 | ).tolist() 32 | 33 | 34 | def get_scene_room_from_scene_room_idx(fr_name): 35 | return "_".join(fr_name.split("_")[:-1]) 36 | 37 | 38 | def get_scene_list_from_dir(args): 39 | list_mvl_fn = os.listdir(args.scene_dir) 40 | list_rooms = np.unique( 41 | [get_scene_room_from_scene_room_idx(Path(fn).stem) for fn in list_mvl_fn] 42 | ).tolist() 43 | data_dict = {} 44 | for room in tqdm(list_rooms, desc="List rooms..."): 45 | data_dict[room] = [Path(fn).stem for fn in list_mvl_fn if f"{room}_" in fn] 46 | 47 | return data_dict 48 | 49 | 50 | def save_json_dict(filename, dict_data): 51 | with open(filename, "w") as outfile: 52 | json.dump(dict_data, outfile, indent="\t") 53 | 54 | 55 | def read_txt_file(filename): 56 | 57 | with open(filename, "r") as fn: 58 | data = fn.read().splitlines() 59 | 60 | return data 61 | 62 | 63 | def read_csv_file(filename): 64 | with open(filename) as f: 65 | csvreader = csv.reader(f) 66 | 67 | lines = [] 68 | for row in csvreader: 69 | lines.append(row[0]) 70 | return lines 71 | 72 | 73 | def save_csv_file(filename, data, flag="w"): 74 | with open(filename, flag) as f: 75 | writer = csv.writer(f) 76 | for line in data: 77 | writer.writerow([l for l in line]) 78 | f.close() 79 | 80 | 81 | def load_obj(filename): 82 | return dill.load(open(filename, "rb")) 83 | 84 | 85 | def create_directory(output_dir, delete_prev=True, ignore_request=False): 86 | if os.path.exists(output_dir) and delete_prev: 87 | if not ignore_request: 88 | logging.warning(f"This directory will be deleted: {output_dir}") 89 | input("This directory will be deleted. PRESS ANY KEY TO CONTINUE...") 90 | shutil.rmtree(output_dir, ignore_errors=True) 91 | if not os.path.exists(output_dir): 92 | logging.info(f"Dir created: {output_dir}") 93 | os.makedirs(output_dir, exist_ok=True) 94 | return Path(output_dir).resolve() 95 | 96 | 97 | def save_obj(filename, obj): 98 | dill.dump(obj, open(filename, "wb")) 99 | print(f" >> OBJ saved: {filename}") 100 | 101 | 102 | def get_files_given_a_pattern( 103 | data_dir, flag_file, exclude="", include_flag_file=False, isDir=False 104 | ): 105 | """ 106 | Searches in the the @data_dir, recurrently, the sub-directories which content the @flag_file. 107 | exclude directories can be passed to speed up the searching 108 | """ 109 | scenes_paths = [] 110 | for root, dirs, files in tqdm( 111 | os.walk(data_dir), desc=f"Walking through {data_dir}..." 112 | ): 113 | dirs[:] = [d for d in dirs if d not in exclude] 114 | if not isDir: 115 | if include_flag_file: 116 | [ 117 | scenes_paths.append(os.path.join(root, f)) 118 | for f in files 119 | if flag_file in f 120 | ] 121 | else: 122 | [scenes_paths.append(root) for f in files if flag_file in f] 123 | else: 124 | [ 125 | scenes_paths.append(os.path.join(root, flag_file)) 126 | for d in dirs 127 | if flag_file in d 128 | ] 129 | 130 | return scenes_paths 131 | 132 | 133 | def mytransform44(l, seq="xyzw"): 134 | """ 135 | Generate a 4x4 homogeneous transformation matrix from a 3D point and unit quaternion. 136 | 137 | Input: 138 | l -- tuple consisting of (stamp,tx,ty,tz,qx,qy,qz,qw) where 139 | (tx,ty,tz) is the 3D position and (qx,qy,qz,qw) is the unit quaternion. 140 | 141 | Output: 142 | matrix -- 4x4 homogeneous transformation matrix 143 | """ 144 | t = l[1:4] 145 | q = np.array(l[4:8], dtype=np.float64, copy=True) 146 | if seq == "wxyz": 147 | if q[0] < 0: 148 | q *= -1 149 | q = Quaternion(w=q[0], x=q[1], y=q[2], z=q[3]) 150 | else: 151 | if q[3] < 0: 152 | q *= -1 153 | q = Quaternion( 154 | x=q[0], 155 | y=q[1], 156 | z=q[2], 157 | w=q[3], 158 | ) 159 | trasnform = np.eye(4) 160 | trasnform[0:3, 0:3] = q.rotation_matrix 161 | trasnform[0:3, 3] = np.array(t) 162 | 163 | return trasnform 164 | 165 | 166 | def read_trajectory(filename, matrix=True, traj_gt_keys_sorted=[], seq="xyzw"): 167 | """ 168 | Read a trajectory from a text file. 169 | 170 | Input: 171 | filename -- file to be read_datasets 172 | matrix -- convert poses to 4x4 matrices 173 | 174 | Output: 175 | dictionary of stamped 3D poses 176 | """ 177 | file = open(filename) 178 | data = file.read() 179 | lines = data.replace(",", " ").replace("\t", " ").split("\n") 180 | list = [ 181 | [float(v.strip()) for v in line.split(" ") if v.strip() != ""] 182 | for line in lines 183 | if len(line) > 0 and line[0] != "#" 184 | ] 185 | list_ok = [] 186 | for i, l in enumerate(list): 187 | if l[4:8] == [0, 0, 0, 0]: 188 | continue 189 | isnan = False 190 | for v in l: 191 | if np.isnan(v): 192 | isnan = True 193 | break 194 | if isnan: 195 | sys.stderr.write( 196 | "Warning: line {} of file {} has NaNs, skipping line\n".format( 197 | i, filename 198 | ) 199 | ) 200 | continue 201 | list_ok.append(l) 202 | if matrix: 203 | traj = dict([(l[0], mytransform44(l[0:], seq=seq)) for l in list_ok]) 204 | else: 205 | traj = dict([(l[0], l[1:8]) for l in list_ok]) 206 | 207 | return traj 208 | 209 | 210 | def read_json_label(fn): 211 | with open(fn, "r") as f: 212 | d = json.load(f) 213 | room_list = d["room_corners"] 214 | room_corners = [] 215 | for corners in room_list: 216 | corners = np.asarray([[float(x[0]), float(x[1])] for x in corners]) 217 | room_corners.append(corners) 218 | axis_corners = d["axis_corners"] 219 | if axis_corners.__len__() > 0: 220 | axis_corners = np.asarray( 221 | [[float(x[0]), float(x[1])] for x in axis_corners] 222 | ) 223 | return room_corners, axis_corners 224 | 225 | 226 | def read_ply(fn): 227 | plydata = PlyData.read(fn) 228 | v = np.array([list(x) for x in plydata.elements[0]]) 229 | points = np.ascontiguousarray(v[:, :3]) 230 | points[:, 0:3] = points[:, [0, 2, 1]] 231 | colors = np.ascontiguousarray(v[:, 6:9], dtype=np.float32) / 255 232 | return np.concatenate((points, colors), axis=1).T 233 | 234 | 235 | def save_compressed_phi_coords(phi_coords, filename): 236 | np.savez_compressed(filename, phi_coords=phi_coords) 237 | 238 | 239 | def process_arcname(list_fn, base_dir): 240 | return [os.path.relpath(fn, start=base_dir) for fn in list_fn] 241 | 242 | 243 | def load_gt_label(fn): 244 | assert os.path.exists(fn), f"Not found {fn}" 245 | return np.load(fn)["phi_coords"] 246 | 247 | 248 | def print_cfg_information(cfg): 249 | logging.info(f"Experiment ID: {cfg.id_exp}") 250 | logging.info(f"Output_dir: {cfg.output_dir}") 251 | -------------------------------------------------------------------------------- /download_mvl_data.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | import os 3 | import subprocess 4 | import argparse 5 | from pathlib import Path 6 | from mvl_challenge import ( 7 | ASSETS_DIR, 8 | ROOT_DIR, 9 | EPILOG, 10 | CFG_DIR, 11 | GDRIVE_DIR, 12 | DEFAULT_DOWNLOAD_DIR, 13 | ) 14 | from mvl_challenge.config.cfg import get_empty_cfg, read_omega_cfg 15 | from mvl_challenge.remote_data.download_mvl_data import download_file, download_dirs, download_file_by_threads 16 | from mvl_challenge.utils.io_utils import create_directory, save_compressed_phi_coords 17 | from mvl_challenge.datasets.mvl_dataset import MVLDataset, iter_mvl_room_scenes 18 | from mvl_challenge.models.wrapper_horizon_net import WrapperHorizonNet 19 | from mvl_challenge.challenge_results.create_zip_results import zip_results 20 | from mvl_challenge.remote_data.download_mvl_data import download_google_drive_link 21 | 22 | 23 | @dataclass 24 | class DataSplit: 25 | GDRIVE_IDS_MVL_DATA_FN: str 26 | GDRIVE_IDS_LABELS_FN: str 27 | TYPE: str 28 | GDRIVE_ID: str 29 | GDRIVE_ID_LABELS: str 30 | GT_LABELS: bool 31 | 32 | 33 | def download_data_split_by_folders(args, data_split: DataSplit): 34 | #! Downloading mvl data 35 | zip_dir = os.path.join(args.output_dir, "zips", data_split.TYPE) 36 | create_directory(zip_dir, delete_prev=False) 37 | 38 | # tmp_dir = os.path.join(zip_dir, "tmp_dir") 39 | # create_directory(tmp_dir, delete_prev=True, ignore_request=True) 40 | cfg = get_empty_cfg() 41 | cfg.output_dir = zip_dir 42 | cfg.ids_file = os.path.join(GDRIVE_DIR, data_split.GDRIVE_IDS_MVL_DATA_FN) 43 | download_dirs(cfg) 44 | 45 | # list_dir_path = os.listdir() 46 | # # ! Unzipping mvl-data 47 | output_dir = os.path.join(args.output_dir, "mvl_data") 48 | unzip(zip_dir, output_dir) 49 | 50 | 51 | def download_data_split(args, data_split: DataSplit): 52 | #! Downloading mvl data 53 | zip_dir = os.path.join(args.output_dir, "zips", f"{data_split.TYPE}") 54 | create_directory(zip_dir, delete_prev=False) 55 | zip_filename = os.path.join(args.output_dir, "zips", f"{data_split.TYPE}.zip") 56 | # download_gdrive_file(data_split.GDRIVE_IDS_MVL_DATA_FN, zip_dir) 57 | download_entire_zip_split(data_split.GDRIVE_ID, zip_filename) 58 | 59 | # # ! Unzipping mvl-data 60 | output_dir = os.path.join(args.output_dir, "mvl_data") 61 | unzip(zip_dir, output_dir) 62 | 63 | if data_split.GT_LABELS: 64 | zip_dir = os.path.join(args.output_dir, "zips", "labels", f"{data_split.TYPE}") 65 | create_directory(zip_dir, delete_prev=False) 66 | # download_gdrive_file(data_split.GDRIVE_IDS_LABELS_FN, zip_dir) 67 | zip_filename = os.path.join(args.output_dir, "zips", "labels", f"{data_split.TYPE}.zip") 68 | download_entire_zip_split(data_split.GDRIVE_ID_LABELS, zip_filename) 69 | 70 | output_dir = os.path.join(args.output_dir, "mvl_data") 71 | unzip(zip_dir, output_dir) 72 | 73 | print(f"** \tzip dir for {data_split.TYPE}:\t\t{zip_dir}") 74 | print(f"** \tmvl dir for {data_split.TYPE}:\t\t{output_dir}") 75 | print(f"*\t->>>\t{data_split.TYPE} downloaded successfully\t<<<-\t*") 76 | 77 | 78 | def download_entire_zip_split(gdrive_id, zip_filename): 79 | download_google_drive_link(gd_id=gdrive_id, output_file=zip_filename) 80 | subprocess.run( 81 | [ 82 | "unzip", 83 | f"{zip_filename}", 84 | "-d", 85 | f"{Path(zip_filename).parent}" 86 | ] 87 | ) 88 | 89 | 90 | def unzip(zip_dir, output_dir): 91 | create_directory(output_dir, delete_prev=False) 92 | subprocess.run( 93 | [ 94 | "bash", 95 | f"{ROOT_DIR}/remote_data/unzip_data.sh", 96 | "-d", 97 | f"{zip_dir}", 98 | "-o", 99 | f"{output_dir}", 100 | ] 101 | ) 102 | 103 | 104 | def download_gdrive_file(gdrive_fn, zip_dir): 105 | create_directory(zip_dir, delete_prev=False) 106 | cfg = get_empty_cfg() 107 | cfg.output_dir = zip_dir 108 | cfg.ids_file = os.path.join(GDRIVE_DIR, gdrive_fn) 109 | download_file_by_threads(cfg) 110 | 111 | 112 | def main(args): 113 | if args.split == "pilot": 114 | data_split = DataSplit( 115 | GDRIVE_IDS_MVL_DATA_FN=os.path.join( 116 | GDRIVE_DIR, "gdrive_ids__pilot_set.csv" 117 | ), 118 | GDRIVE_IDS_LABELS_FN=os.path.join( 119 | GDRIVE_DIR, "gdrive_ids__pilot_labels.csv" 120 | ), 121 | TYPE="pilot_set", 122 | GT_LABELS=True, 123 | GDRIVE_ID="13dFArf0oKznUsOZTkumjspRT8Z2sTqHb", 124 | GDRIVE_ID_LABELS="1F5QW0QpoxublJTA1yjGaMXJsZNSzJuHS" 125 | ) 126 | 127 | elif args.split == "warm_up_testing": 128 | data_split = DataSplit( 129 | GDRIVE_IDS_MVL_DATA_FN=os.path.join( 130 | GDRIVE_DIR, "gdrive_ids__warm_up_testing_set.csv" 131 | ), 132 | GDRIVE_IDS_LABELS_FN="", 133 | TYPE="warm_up_testing_set", 134 | GT_LABELS=False, 135 | GDRIVE_ID="1IE1Z7SzlQXMe9lg0CSfsVLozPvAXOJJ-", 136 | GDRIVE_ID_LABELS="" 137 | ) 138 | 139 | elif args.split == "warm_up_training": 140 | data_split = DataSplit( 141 | # GDRIVE_IDS_MVL_DATA_FN=os.path.join(GDRIVE_DIR, 'gdrive_ids__warm_up_training_set.csv'), 142 | GDRIVE_IDS_MVL_DATA_FN=os.path.join( 143 | # GDRIVE_DIR, "gdrive_ids__warm_up_training_set_folders.csv" 144 | GDRIVE_DIR, "gdrive_ids__warm_up_training_set.csv" 145 | 146 | ), 147 | GDRIVE_IDS_LABELS_FN="", 148 | TYPE="warm_up_training_set", 149 | GT_LABELS=False, 150 | GDRIVE_ID="19rQ3YrhHHYGiDSjeGp2wFKyD8df7py90", 151 | GDRIVE_ID_LABELS="" 152 | ) 153 | # download_data_split_by_folders(args, data_split) 154 | # return 155 | elif args.split == "challenge_training": 156 | data_split = DataSplit( 157 | # GDRIVE_IDS_MVL_DATA_FN=os.path.join(GDRIVE_DIR, 'gdrive_ids__warm_up_training_set.csv'), 158 | GDRIVE_IDS_MVL_DATA_FN=os.path.join( 159 | # GDRIVE_DIR, "gdrive_ids__warm_up_training_set_folders.csv" 160 | GDRIVE_DIR, "gdrive_ids__challenge_phase_training_set.csv" 161 | 162 | ), 163 | GDRIVE_IDS_LABELS_FN="", 164 | TYPE="challenge_phase__training_set", 165 | GT_LABELS=False, 166 | GDRIVE_ID="1bnTTzTsc547DVRSccDQCr16LWwdzXkbG", 167 | GDRIVE_ID_LABELS="" 168 | ) 169 | elif args.split == "challenge_testing": 170 | data_split = DataSplit( 171 | # GDRIVE_IDS_MVL_DATA_FN=os.path.join(GDRIVE_DIR, 'gdrive_ids__warm_up_training_set.csv'), 172 | GDRIVE_IDS_MVL_DATA_FN=os.path.join( 173 | # GDRIVE_DIR, "gdrive_ids__warm_up_training_set_folders.csv" 174 | GDRIVE_DIR, "gdrive_ids__challenge_phase_testing_set.csv" 175 | 176 | ), 177 | GDRIVE_IDS_LABELS_FN="", 178 | TYPE="challenge_phase__testing_set", 179 | GT_LABELS=False, 180 | GDRIVE_ID="1VoTifgI8_sIfN324UOtlgB16jxHybKYC", 181 | GDRIVE_ID_LABELS="" 182 | ) 183 | else: 184 | raise ValueError(f"Not implemented split: {args.split}") 185 | 186 | download_data_split(args, data_split) 187 | 188 | 189 | def get_argparse(): 190 | desc = "This script helps you to automatically download mvl-dataset in a passed output dir." 191 | 192 | parser = argparse.ArgumentParser(description=desc, epilog=EPILOG) 193 | 194 | parser.add_argument( 195 | "-o", 196 | "--output_dir", 197 | default=f"{DEFAULT_DOWNLOAD_DIR}", 198 | type=str, 199 | help=f"Output directory by default it will store at {DEFAULT_DOWNLOAD_DIR}.", 200 | ) 201 | 202 | parser.add_argument( 203 | "-split", 204 | default="challenge_training", 205 | type=str, 206 | help="Defines the split data you want to download. Options: 'pilot', 'warm_up_testing', 'warm_up_training', 'challenge_testing', 'challenge_training' ", 207 | ) 208 | 209 | args = parser.parse_args() 210 | return args 211 | 212 | 213 | if __name__ == "__main__": 214 | args = get_argparse() 215 | main(args) 216 | -------------------------------------------------------------------------------- /mvl_challenge/data/scene_list/scene_list__hm3d_mvl__warm_up_pilot_set.json: -------------------------------------------------------------------------------- 1 | { 2 | "HaxA7YrQdEC_1_room0": [ 3 | "HaxA7YrQdEC_1_room0_0", 4 | "HaxA7YrQdEC_1_room0_3", 5 | "HaxA7YrQdEC_1_room0_6", 6 | "HaxA7YrQdEC_1_room0_9", 7 | "HaxA7YrQdEC_1_room0_12", 8 | "HaxA7YrQdEC_1_room0_15", 9 | "HaxA7YrQdEC_1_room0_18", 10 | "HaxA7YrQdEC_1_room0_21", 11 | "HaxA7YrQdEC_1_room0_24", 12 | "HaxA7YrQdEC_1_room0_27", 13 | "HaxA7YrQdEC_1_room0_30", 14 | "HaxA7YrQdEC_1_room0_33", 15 | "HaxA7YrQdEC_1_room0_36", 16 | "HaxA7YrQdEC_1_room0_39", 17 | "HaxA7YrQdEC_1_room0_46", 18 | "HaxA7YrQdEC_1_room0_49", 19 | "HaxA7YrQdEC_1_room0_52", 20 | "HaxA7YrQdEC_1_room0_55", 21 | "HaxA7YrQdEC_1_room0_58", 22 | "HaxA7YrQdEC_1_room0_61", 23 | "HaxA7YrQdEC_1_room0_64", 24 | "HaxA7YrQdEC_1_room0_67", 25 | "HaxA7YrQdEC_1_room0_70", 26 | "HaxA7YrQdEC_1_room0_73", 27 | "HaxA7YrQdEC_1_room0_76", 28 | "HaxA7YrQdEC_1_room0_79", 29 | "HaxA7YrQdEC_1_room0_82", 30 | "HaxA7YrQdEC_1_room0_85", 31 | "HaxA7YrQdEC_1_room0_88", 32 | "HaxA7YrQdEC_1_room0_91", 33 | "HaxA7YrQdEC_1_room0_94", 34 | "HaxA7YrQdEC_1_room0_97", 35 | "HaxA7YrQdEC_1_room0_100", 36 | "HaxA7YrQdEC_1_room0_103", 37 | "HaxA7YrQdEC_1_room0_106", 38 | "HaxA7YrQdEC_1_room0_109", 39 | "HaxA7YrQdEC_1_room0_112", 40 | "HaxA7YrQdEC_1_room0_115", 41 | "HaxA7YrQdEC_1_room0_118", 42 | "HaxA7YrQdEC_1_room0_121", 43 | "HaxA7YrQdEC_1_room0_124", 44 | "HaxA7YrQdEC_1_room0_127", 45 | "HaxA7YrQdEC_1_room0_130", 46 | "HaxA7YrQdEC_1_room0_133", 47 | "HaxA7YrQdEC_1_room0_136", 48 | "HaxA7YrQdEC_1_room0_139", 49 | "HaxA7YrQdEC_1_room0_142", 50 | "HaxA7YrQdEC_1_room0_145", 51 | "HaxA7YrQdEC_1_room0_148", 52 | "HaxA7YrQdEC_1_room0_151", 53 | "HaxA7YrQdEC_1_room0_154", 54 | "HaxA7YrQdEC_1_room0_157" 55 | ], 56 | "h1zeeAwLh9Z_3_room0": [ 57 | "h1zeeAwLh9Z_3_room0_0", 58 | "h1zeeAwLh9Z_3_room0_2", 59 | "h1zeeAwLh9Z_3_room0_4", 60 | "h1zeeAwLh9Z_3_room0_6", 61 | "h1zeeAwLh9Z_3_room0_8", 62 | "h1zeeAwLh9Z_3_room0_10", 63 | "h1zeeAwLh9Z_3_room0_12", 64 | "h1zeeAwLh9Z_3_room0_14", 65 | "h1zeeAwLh9Z_3_room0_16", 66 | "h1zeeAwLh9Z_3_room0_18", 67 | "h1zeeAwLh9Z_3_room0_20", 68 | "h1zeeAwLh9Z_3_room0_22", 69 | "h1zeeAwLh9Z_3_room0_24", 70 | "h1zeeAwLh9Z_3_room0_26", 71 | "h1zeeAwLh9Z_3_room0_28", 72 | "h1zeeAwLh9Z_3_room0_30", 73 | "h1zeeAwLh9Z_3_room0_32", 74 | "h1zeeAwLh9Z_3_room0_34", 75 | "h1zeeAwLh9Z_3_room0_36", 76 | "h1zeeAwLh9Z_3_room0_38", 77 | "h1zeeAwLh9Z_3_room0_40", 78 | "h1zeeAwLh9Z_3_room0_42", 79 | "h1zeeAwLh9Z_3_room0_44", 80 | "h1zeeAwLh9Z_3_room0_46", 81 | "h1zeeAwLh9Z_3_room0_48", 82 | "h1zeeAwLh9Z_3_room0_50", 83 | "h1zeeAwLh9Z_3_room0_52", 84 | "h1zeeAwLh9Z_3_room0_54", 85 | "h1zeeAwLh9Z_3_room0_56", 86 | "h1zeeAwLh9Z_3_room0_58", 87 | "h1zeeAwLh9Z_3_room0_60", 88 | "h1zeeAwLh9Z_3_room0_62", 89 | "h1zeeAwLh9Z_3_room0_64", 90 | "h1zeeAwLh9Z_3_room0_66", 91 | "h1zeeAwLh9Z_3_room0_68", 92 | "h1zeeAwLh9Z_3_room0_70", 93 | "h1zeeAwLh9Z_3_room0_72", 94 | "h1zeeAwLh9Z_3_room0_74", 95 | "h1zeeAwLh9Z_3_room0_76", 96 | "h1zeeAwLh9Z_3_room0_78", 97 | "h1zeeAwLh9Z_3_room0_80", 98 | "h1zeeAwLh9Z_3_room0_82", 99 | "h1zeeAwLh9Z_3_room0_84", 100 | "h1zeeAwLh9Z_3_room0_86", 101 | "h1zeeAwLh9Z_3_room0_88", 102 | "h1zeeAwLh9Z_3_room0_90", 103 | "h1zeeAwLh9Z_3_room0_92" 104 | ], 105 | "q3hn1WQ12rz_6_room0": [ 106 | "q3hn1WQ12rz_6_room0_0", 107 | "q3hn1WQ12rz_6_room0_5", 108 | "q3hn1WQ12rz_6_room0_10", 109 | "q3hn1WQ12rz_6_room0_15", 110 | "q3hn1WQ12rz_6_room0_20", 111 | "q3hn1WQ12rz_6_room0_25", 112 | "q3hn1WQ12rz_6_room0_30", 113 | "q3hn1WQ12rz_6_room0_35", 114 | "q3hn1WQ12rz_6_room0_40", 115 | "q3hn1WQ12rz_6_room0_45", 116 | "q3hn1WQ12rz_6_room0_50", 117 | "q3hn1WQ12rz_6_room0_55", 118 | "q3hn1WQ12rz_6_room0_60", 119 | "q3hn1WQ12rz_6_room0_65", 120 | "q3hn1WQ12rz_6_room0_70", 121 | "q3hn1WQ12rz_6_room0_75", 122 | "q3hn1WQ12rz_6_room0_80", 123 | "q3hn1WQ12rz_6_room0_85", 124 | "q3hn1WQ12rz_6_room0_90", 125 | "q3hn1WQ12rz_6_room0_95", 126 | "q3hn1WQ12rz_6_room0_100", 127 | "q3hn1WQ12rz_6_room0_105", 128 | "q3hn1WQ12rz_6_room0_110", 129 | "q3hn1WQ12rz_6_room0_115", 130 | "q3hn1WQ12rz_6_room0_120", 131 | "q3hn1WQ12rz_6_room0_125", 132 | "q3hn1WQ12rz_6_room0_130", 133 | "q3hn1WQ12rz_6_room0_135", 134 | "q3hn1WQ12rz_6_room0_140", 135 | "q3hn1WQ12rz_6_room0_145", 136 | "q3hn1WQ12rz_6_room0_150", 137 | "q3hn1WQ12rz_6_room0_155", 138 | "q3hn1WQ12rz_6_room0_160", 139 | "q3hn1WQ12rz_6_room0_165", 140 | "q3hn1WQ12rz_6_room0_170", 141 | "q3hn1WQ12rz_6_room0_175", 142 | "q3hn1WQ12rz_6_room0_180", 143 | "q3hn1WQ12rz_6_room0_185", 144 | "q3hn1WQ12rz_6_room0_190", 145 | "q3hn1WQ12rz_6_room0_195", 146 | "q3hn1WQ12rz_6_room0_200", 147 | "q3hn1WQ12rz_6_room0_205", 148 | "q3hn1WQ12rz_6_room0_210", 149 | "q3hn1WQ12rz_6_room0_215", 150 | "q3hn1WQ12rz_6_room0_220", 151 | "q3hn1WQ12rz_6_room0_225", 152 | "q3hn1WQ12rz_6_room0_230", 153 | "q3hn1WQ12rz_6_room0_235", 154 | "q3hn1WQ12rz_6_room0_240", 155 | "q3hn1WQ12rz_6_room0_245", 156 | "q3hn1WQ12rz_6_room0_250" 157 | ], 158 | "k1cupFYWXJ6_0_room0": [ 159 | "k1cupFYWXJ6_0_room0_0", 160 | "k1cupFYWXJ6_0_room0_3", 161 | "k1cupFYWXJ6_0_room0_6", 162 | "k1cupFYWXJ6_0_room0_9", 163 | "k1cupFYWXJ6_0_room0_12", 164 | "k1cupFYWXJ6_0_room0_15", 165 | "k1cupFYWXJ6_0_room0_18", 166 | "k1cupFYWXJ6_0_room0_21", 167 | "k1cupFYWXJ6_0_room0_24", 168 | "k1cupFYWXJ6_0_room0_27", 169 | "k1cupFYWXJ6_0_room0_30", 170 | "k1cupFYWXJ6_0_room0_33", 171 | "k1cupFYWXJ6_0_room0_36", 172 | "k1cupFYWXJ6_0_room0_39", 173 | "k1cupFYWXJ6_0_room0_42", 174 | "k1cupFYWXJ6_0_room0_45", 175 | "k1cupFYWXJ6_0_room0_48", 176 | "k1cupFYWXJ6_0_room0_51", 177 | "k1cupFYWXJ6_0_room0_54", 178 | "k1cupFYWXJ6_0_room0_57", 179 | "k1cupFYWXJ6_0_room0_60", 180 | "k1cupFYWXJ6_0_room0_63", 181 | "k1cupFYWXJ6_0_room0_66", 182 | "k1cupFYWXJ6_0_room0_69", 183 | "k1cupFYWXJ6_0_room0_72", 184 | "k1cupFYWXJ6_0_room0_75", 185 | "k1cupFYWXJ6_0_room0_78", 186 | "k1cupFYWXJ6_0_room0_81", 187 | "k1cupFYWXJ6_0_room0_84", 188 | "k1cupFYWXJ6_0_room0_87", 189 | "k1cupFYWXJ6_0_room0_90", 190 | "k1cupFYWXJ6_0_room0_93", 191 | "k1cupFYWXJ6_0_room0_96", 192 | "k1cupFYWXJ6_0_room0_99", 193 | "k1cupFYWXJ6_0_room0_102", 194 | "k1cupFYWXJ6_0_room0_105", 195 | "k1cupFYWXJ6_0_room0_108", 196 | "k1cupFYWXJ6_0_room0_111", 197 | "k1cupFYWXJ6_0_room0_114", 198 | "k1cupFYWXJ6_0_room0_117", 199 | "k1cupFYWXJ6_0_room0_120", 200 | "k1cupFYWXJ6_0_room0_123", 201 | "k1cupFYWXJ6_0_room0_126", 202 | "k1cupFYWXJ6_0_room0_129", 203 | "k1cupFYWXJ6_0_room0_132", 204 | "k1cupFYWXJ6_0_room0_135", 205 | "k1cupFYWXJ6_0_room0_138", 206 | "k1cupFYWXJ6_0_room0_141", 207 | "k1cupFYWXJ6_0_room0_144", 208 | "k1cupFYWXJ6_0_room0_147", 209 | "k1cupFYWXJ6_0_room0_150", 210 | "k1cupFYWXJ6_0_room0_153", 211 | "k1cupFYWXJ6_0_room0_156" 212 | ], 213 | "wcojb4TFT35_0_room0": [ 214 | "wcojb4TFT35_0_room0_0", 215 | "wcojb4TFT35_0_room0_2", 216 | "wcojb4TFT35_0_room0_4", 217 | "wcojb4TFT35_0_room0_6", 218 | "wcojb4TFT35_0_room0_8", 219 | "wcojb4TFT35_0_room0_10", 220 | "wcojb4TFT35_0_room0_12", 221 | "wcojb4TFT35_0_room0_14", 222 | "wcojb4TFT35_0_room0_16", 223 | "wcojb4TFT35_0_room0_18", 224 | "wcojb4TFT35_0_room0_20", 225 | "wcojb4TFT35_0_room0_22", 226 | "wcojb4TFT35_0_room0_24", 227 | "wcojb4TFT35_0_room0_26", 228 | "wcojb4TFT35_0_room0_28", 229 | "wcojb4TFT35_0_room0_30", 230 | "wcojb4TFT35_0_room0_32", 231 | "wcojb4TFT35_0_room0_34", 232 | "wcojb4TFT35_0_room0_36", 233 | "wcojb4TFT35_0_room0_38", 234 | "wcojb4TFT35_0_room0_40", 235 | "wcojb4TFT35_0_room0_42", 236 | "wcojb4TFT35_0_room0_44", 237 | "wcojb4TFT35_0_room0_46", 238 | "wcojb4TFT35_0_room0_48", 239 | "wcojb4TFT35_0_room0_50", 240 | "wcojb4TFT35_0_room0_52", 241 | "wcojb4TFT35_0_room0_54", 242 | "wcojb4TFT35_0_room0_56", 243 | "wcojb4TFT35_0_room0_58", 244 | "wcojb4TFT35_0_room0_60", 245 | "wcojb4TFT35_0_room0_62", 246 | "wcojb4TFT35_0_room0_64", 247 | "wcojb4TFT35_0_room0_66", 248 | "wcojb4TFT35_0_room0_68", 249 | "wcojb4TFT35_0_room0_70", 250 | "wcojb4TFT35_0_room0_72", 251 | "wcojb4TFT35_0_room0_74", 252 | "wcojb4TFT35_0_room0_76", 253 | "wcojb4TFT35_0_room0_78", 254 | "wcojb4TFT35_0_room0_80", 255 | "wcojb4TFT35_0_room0_82", 256 | "wcojb4TFT35_0_room0_84", 257 | "wcojb4TFT35_0_room0_86", 258 | "wcojb4TFT35_0_room0_88" 259 | ] 260 | } --------------------------------------------------------------------------------