├── project ├── datasets │ ├── __init__.py │ ├── waymo │ │ ├── __init__.py │ │ └── waymo_download.py │ ├── base │ │ ├── __init__.py │ │ ├── split_wrapper.py │ │ ├── utils.py │ │ └── scene_dataset.py │ └── tools │ │ ├── multiprocess_utils.py │ │ └── humanpose_process.py ├── models │ ├── __init__.py │ ├── nodes │ │ └── __init__.py │ ├── trainers │ │ └── __init__.py │ └── gaussians │ │ ├── __init__.py │ │ └── scaffold.py ├── tools │ └── __init__.py ├── utils │ ├── __init__.py │ ├── backup.py │ ├── chamfer_distance.py │ ├── geometry.py │ └── misc.py ├── third_party │ └── smplx │ │ ├── optional-requirements.txt │ │ ├── requirements.txt │ │ ├── transfer_model │ │ ├── docs │ │ │ └── images │ │ │ │ └── smpl_smplx_correspondence.png │ │ ├── requirements.txt │ │ ├── __init__.py │ │ ├── losses │ │ │ ├── __init__.py │ │ │ ├── utils.py │ │ │ └── losses.py │ │ ├── config │ │ │ ├── __init__.py │ │ │ ├── utils_cfg.py │ │ │ ├── dataset_defaults.py │ │ │ ├── loss_defaults.py │ │ │ ├── optim_defaults.py │ │ │ ├── cmd_parser.py │ │ │ ├── defaults.py │ │ │ └── body_model_defaults.py │ │ ├── optimizers │ │ │ ├── __init__.py │ │ │ ├── optim_factory.py │ │ │ └── minimize.py │ │ ├── utils │ │ │ ├── torch_utils.py │ │ │ ├── typing.py │ │ │ ├── metrics.py │ │ │ ├── __init__.py │ │ │ ├── np_utils.py │ │ │ ├── o3d_utils.py │ │ │ ├── timer.py │ │ │ ├── mesh_utils.py │ │ │ ├── def_transfer.py │ │ │ └── pose_utils.py │ │ ├── merge_output.py │ │ ├── __main__.py │ │ └── view_pkl.py │ │ ├── config_files │ │ ├── smplh2smpl.yaml │ │ ├── smplx2smpl.yaml │ │ ├── smplh2smplx_as.yaml │ │ ├── smpl2smplx.yaml │ │ ├── smpl2smplh.yaml │ │ ├── smplh2smplx.yaml │ │ ├── smplh2smplx_onepose.yaml │ │ └── smplx2smplh.yaml │ │ ├── smplx │ │ ├── __init__.py │ │ ├── vertex_ids.py │ │ ├── vertex_joint_selector.py │ │ └── utils.py │ │ ├── .gitignore │ │ ├── setup.py │ │ └── examples │ │ ├── vis_flame_vertices.py │ │ └── vis_mano_vertices.py ├── data │ ├── ego_masks │ │ ├── nuplan │ │ │ ├── 1.png │ │ │ ├── 2.png │ │ │ ├── 5.png │ │ │ └── 6.png │ │ ├── nuscenes │ │ │ └── 5.png │ │ ├── pandaset │ │ │ └── 5.png │ │ └── argoverse │ │ │ ├── 0.png │ │ │ ├── 5.png │ │ │ └── 6.png │ ├── kitti_example_scenes.txt │ ├── waymo_example_scenes.txt │ ├── nuplan_example_scenes.txt │ ├── pandaset_example_scenes.txt │ └── argoverse_example_scenes.txt ├── generate_lidar │ ├── generate_radar_detection.py │ └── depth2lidar.py └── configs │ ├── datasets │ ├── kitti │ │ ├── 1cams.yaml │ │ └── 2cams.yaml │ ├── nuscenes │ │ ├── 1cams.yaml │ │ ├── 3cams.yaml │ │ └── 6cams.yaml │ ├── argoverse │ │ ├── 1cams.yaml │ │ ├── 3cams.yaml │ │ ├── 5cams.yaml │ │ └── 7cams.yaml │ ├── pandaset │ │ ├── 1cams.yaml │ │ ├── 3cams.yaml │ │ └── 6cams.yaml │ ├── nuplan │ │ ├── 1cams.yaml │ │ ├── 3cams.yaml │ │ ├── 5cams.yaml │ │ └── 8cams.yaml │ └── waymo │ │ ├── 1cams.yaml │ │ ├── 3cams.yaml │ │ └── 5cams.yaml │ ├── paper_legacy │ └── streetgs.yaml │ └── deformablegs.yaml ├── docs └── media │ ├── paradigm.png │ └── badge-website.svg ├── .gitmodules ├── scripts ├── eval.sh └── train.sh ├── requirements.txt ├── LICENSE └── .gitignore /project/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/datasets/waymo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/third_party/smplx/optional-requirements.txt: -------------------------------------------------------------------------------- 1 | pyrender>=0.1.23 2 | shapely 3 | trimesh>=2.37.6 4 | -------------------------------------------------------------------------------- /project/third_party/smplx/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.16.2 2 | torch>=1.0.1.post2 3 | dataclasses>=0.6 4 | -------------------------------------------------------------------------------- /docs/media/paradigm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/docs/media/paradigm.png -------------------------------------------------------------------------------- /project/data/ego_masks/nuplan/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/nuplan/1.png -------------------------------------------------------------------------------- /project/data/ego_masks/nuplan/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/nuplan/2.png -------------------------------------------------------------------------------- /project/data/ego_masks/nuplan/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/nuplan/5.png -------------------------------------------------------------------------------- /project/data/ego_masks/nuplan/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/nuplan/6.png -------------------------------------------------------------------------------- /project/data/ego_masks/nuscenes/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/nuscenes/5.png -------------------------------------------------------------------------------- /project/data/ego_masks/pandaset/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/pandaset/5.png -------------------------------------------------------------------------------- /project/models/nodes/__init__.py: -------------------------------------------------------------------------------- 1 | from .rigid import RigidNodes 2 | from .deformable import DeformableNodes 3 | from .smpl import SMPLNodes -------------------------------------------------------------------------------- /project/data/ego_masks/argoverse/0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/argoverse/0.png -------------------------------------------------------------------------------- /project/data/ego_masks/argoverse/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/argoverse/5.png -------------------------------------------------------------------------------- /project/data/ego_masks/argoverse/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/data/ego_masks/argoverse/6.png -------------------------------------------------------------------------------- /project/models/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BasicTrainer 2 | from .single import SingleTrainer 3 | from .scene_graph import MultiTrainer -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "project/third_party/Humans4D"] 2 | path = project/third_party/Humans4D 3 | url = https://github.com/shubham-goel/4D-Humans.git 4 | -------------------------------------------------------------------------------- /project/datasets/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .lidar_source import SceneLidarSource 2 | from .pixel_source import ScenePixelSource 3 | from .scene_dataset import SceneDataset 4 | from .split_wrapper import SplitWrapper -------------------------------------------------------------------------------- /project/models/gaussians/__init__.py: -------------------------------------------------------------------------------- 1 | from .vanilla import VanillaGaussians 2 | from .deformgs import DeformableGaussians 3 | from .pvg import PeriodicVibrationGaussians 4 | from .scaffold import ScaffoldGaussians -------------------------------------------------------------------------------- /project/data/kitti_example_scenes.txt: -------------------------------------------------------------------------------- 1 | # scene_id, start_timestep, end_timestep 2 | 2011_09_26_drive_0001_sync,0,-1 3 | 2011_09_26_drive_0018_sync,0,200 4 | 2011_09_26_drive_0091_sync,0,200 5 | 2011_09_26_drive_0059_sync,0,200 -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/docs/images/smpl_smplx_correspondence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigCiLeng/bilateral-driving/HEAD/project/third_party/smplx/transfer_model/docs/images/smpl_smplx_correspondence.png -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.16.2 2 | torch>=1.0.1.post2 3 | dataclasses>=0.6 4 | pyrender>=0.1.23 5 | shapely 6 | trimesh>=2.37.6 7 | open3d 8 | smplx 9 | omegaconf 10 | loguru 11 | -------------------------------------------------------------------------------- /project/data/waymo_example_scenes.txt: -------------------------------------------------------------------------------- 1 | # scene_id, seg_name, start_timestep, end_timestep 2 | 23,seg104554,0,150 3 | 114,seg125050,0,150 4 | 327,seg169514,0,150 5 | 621,seg584622,0,140 6 | 703,seg776165,0,170 7 | 172,seg138251,30,180 8 | 552,seg448767,0,140 9 | 788,seg965324,130,-1 -------------------------------------------------------------------------------- /project/data/nuplan_example_scenes.txt: -------------------------------------------------------------------------------- 1 | # scene_id, start_timestep, end_timestep 2 | 2021.05.12.22.00.38_veh-35_01008_01518,0,-1 3 | 2021.05.12.22.28.35_veh-35_00620_01164,0,-1 4 | 2021.05.12.23.36.44_veh-35_00152_00504,0,-1 5 | 2021.05.12.23.36.44_veh-35_01133_01535,0,-1 6 | 2021.05.12.23.36.44_veh-35_02035_02387,0,-1 7 | 2021.05.25.14.16.10_veh-35_01690_02183,0,-1 -------------------------------------------------------------------------------- /scripts/eval.sh: -------------------------------------------------------------------------------- 1 | ## need edit 2 | scene_idx_all=(152 164 171 200 209 359 529 916) 3 | output_root=../ckpts/nuscenes_pretrained_checkpoints 4 | 5 | project_root=project 6 | 7 | cd $project_root 8 | export PYTHONPATH=$(pwd) 9 | 10 | for scene_idx in "${scene_idx_all[@]}"; do 11 | python tools/eval_metrics.py --resume_from $output_root/$scene_idx/checkpoint_final.pth 12 | done 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | timm==0.9.5 2 | pytorch_msssim==1.0.0 3 | 4 | omegaconf==2.3.0 5 | torchmetrics==0.10.3 6 | 7 | tensorboard==2.11.0 8 | wandb==0.15.8 9 | matplotlib==3.5.3 10 | plotly==5.13.1 11 | viser==0.2.1 12 | 13 | imageio 14 | imageio-ffmpeg 15 | scikit-image==0.20.0 16 | opencv-python 17 | 18 | open3d==0.16.0 19 | pyquaternion==0.9.9 20 | chumpy 21 | numpy==1.23.1 22 | kornia==0.7.2 23 | 24 | tqdm 25 | gdown 26 | 27 | nerfview==0.0.3 28 | lpips==0.1.4 29 | tensorly -------------------------------------------------------------------------------- /project/data/pandaset_example_scenes.txt: -------------------------------------------------------------------------------- 1 | # scene_id, start_timestep, end_timestep 2 | 1,0,-1 3 | 2,0,-1 4 | 3,0,-1 5 | 4,0,-1 6 | 5,0,-1 7 | 6,0,-1 8 | 8,0,-1 9 | 11,0,-1 10 | 12,0,-1 11 | 13,0,-1 12 | 14,0,-1 13 | 15,0,-1 14 | 16,0,-1 15 | 17,0,-1 16 | 18,0,-1 17 | 19,0,-1 18 | 20,0,-1 19 | 21,0,-1 20 | 23,0,-1 21 | 24,0,-1 22 | 27,0,-1 23 | 28,0,-1 24 | 29,0,-1 25 | 30,0,-1 26 | 32,0,-1 27 | 33,0,-1 28 | 34,0,-1 29 | 35,0,-1 30 | 37,0,-1 31 | 38,0,-1 32 | 39,0,-1 33 | 40,0,-1 34 | 41,0,-1 35 | 42,0,-1 36 | 43,0,-1 37 | 44,0,-1 38 | 45,0,-1 39 | 46,0,-1 40 | 47,0,-1 -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smplh2smpl.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'transfer_data/meshes/smplh' 4 | deformation_transfer_path: 'transfer_data/smplh2smpl_def_transfer.pkl' 5 | mask_ids_fname: '' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'trust-ncg' 13 | maxiters: 100 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smpl" 18 | gender: "neutral" 19 | folder: "transfer_data/body_models" 20 | use_compressed: False 21 | use_face_contour: True 22 | smpl: 23 | betas: 24 | num: 10 25 | -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smplx2smpl.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'meshes/smplx' 4 | deformation_transfer_path: 'transfer_data/smplx2smpl_deftrafo_setup.pkl' 5 | mask_ids_fname: '' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'lbfgs' 13 | maxiters: 200 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smpl" 18 | gender: "neutral" 19 | ext: 'pkl' 20 | folder: "transfer_data/body_models" 21 | use_compressed: False 22 | use_face_contour: True 23 | smpl: 24 | betas: 25 | num: 10 26 | -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smplh2smplx_as.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'transfer_data/meshes/amass_sample' 4 | deformation_transfer_path: 'transfer_data/smplh2smplx_deftrafo_setup.pkl' 5 | mask_ids_fname: 'smplx_mask_ids.npy' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'trust-ncg' 13 | maxiters: 100 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smplx" 18 | gender: "neutral" 19 | folder: "models" 20 | use_compressed: False 21 | use_face_contour: True 22 | smplx: 23 | betas: 24 | num: 10 25 | expression: 26 | num: 10 27 | -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smpl2smplx.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'transfer_data/meshes/smpl' 4 | deformation_transfer_path: 'transfer_data/smpl2smplx_deftrafo_setup.pkl' 5 | mask_ids_fname: 'smplx_mask_ids.npy' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'trust-ncg' 13 | maxiters: 100 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smplx" 18 | gender: "neutral" 19 | folder: "transfer_data/body_models" 20 | use_compressed: False 21 | use_face_contour: True 22 | smplx: 23 | betas: 24 | num: 10 25 | expression: 26 | num: 10 27 | -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smpl2smplh.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'transfer_data/meshes/smpl' 4 | deformation_transfer_path: 'transfer_data/smpl2smplh_def_transfer.pkl' 5 | mask_ids_fname: '' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'trust-ncg' 13 | maxiters: 100 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smplh" 18 | # SMPL+H has no neutral model, so we have to manually select the gender 19 | gender: "female" 20 | # gender: "male" 21 | folder: "transfer_data/body_models" 22 | use_compressed: False 23 | smplh: 24 | betas: 25 | num: 10 26 | -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smplh2smplx.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'transfer_data/meshes/smplh' 4 | deformation_transfer_path: 'transfer_data/smplh2smplx_deftrafo_setup.pkl' 5 | mask_ids_fname: 'smplx_mask_ids.npy' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'trust-ncg' 13 | maxiters: 100 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smplx" 18 | gender: "neutral" 19 | folder: "transfer_data/body_models" 20 | use_compressed: False 21 | use_face_contour: True 22 | smplx: 23 | betas: 24 | num: 10 25 | expression: 26 | num: 10 27 | -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smplh2smplx_onepose.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'transfer_data/meshes/amass_onepose' 4 | deformation_transfer_path: 'transfer_data/smplh2smplx_deftrafo_setup.pkl' 5 | mask_ids_fname: 'smplx_mask_ids.npy' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'adam' 13 | lr: 0.1 14 | maxiters: 10000 15 | gtol: 1e-06 16 | 17 | body_model: 18 | model_type: "smplx" 19 | gender: "neutral" 20 | folder: "models" 21 | use_compressed: False 22 | use_face_contour: True 23 | smplx: 24 | betas: 25 | num: 10 26 | expression: 27 | num: 10 28 | -------------------------------------------------------------------------------- /project/models/gaussians/scaffold.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List 2 | from omegaconf import OmegaConf 3 | import logging 4 | 5 | import torch 6 | import torch.nn as nn 7 | from torch.nn import Parameter 8 | 9 | from models.gaussians.basics import * 10 | 11 | logger = logging.getLogger() 12 | 13 | class ScaffoldGaussians(nn.Module): 14 | def __init__(self, 15 | ctrl_cfg: OmegaConf, 16 | reg_cfg: OmegaConf, 17 | scene_scale: float = 1.0, 18 | device: torch.device = torch.device("cuda")): 19 | super().__init__() 20 | self.ctrl_cfg = ctrl_cfg 21 | self.reg_cfg = reg_cfg 22 | self.scene_scale = scene_scale 23 | self.device = device -------------------------------------------------------------------------------- /project/third_party/smplx/config_files/smplx2smplh.yaml: -------------------------------------------------------------------------------- 1 | datasets: 2 | mesh_folder: 3 | data_folder: 'meshes/smplx' 4 | deformation_transfer_path: 'transfer_data/smplx2smplh_deftrafo_setup.pkl' 5 | mask_ids_fname: '' 6 | summary_steps: 100 7 | 8 | edge_fitting: 9 | per_part: False 10 | 11 | optim: 12 | type: 'lbfgs' 13 | maxiters: 200 14 | gtol: 1e-06 15 | 16 | body_model: 17 | model_type: "smplh" 18 | # SMPL+H has no neutral model, so we have to manually select the gender 19 | gender: "female" 20 | # gender: "male" 21 | ext: 'pkl' 22 | folder: "transfer_data/body_models" 23 | use_compressed: False 24 | use_face_contour: True 25 | smplh: 26 | betas: 27 | num: 10 28 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/losses/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from .losses import * 18 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from .cmd_parser import parse_args 18 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from .optim_factory import build_optimizer 18 | from .minimize import minimize 19 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/torch_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import numpy as np 18 | import torch 19 | 20 | 21 | def from_torch(x, dtype=np.float32): 22 | if torch.is_tensor(x): 23 | x = x.detach().cpu().numpy() 24 | return x.astype(dtype) 25 | -------------------------------------------------------------------------------- /project/third_party/smplx/smplx/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: ps-license@tuebingen.mpg.de 16 | 17 | from .body_models import ( 18 | create, 19 | SMPL, 20 | SMPLH, 21 | SMPLX, 22 | MANO, 23 | FLAME, 24 | build_layer, 25 | SMPLLayer, 26 | SMPLHLayer, 27 | SMPLXLayer, 28 | MANOLayer, 29 | FLAMELayer, 30 | ) 31 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/typing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from typing import NewType, List, Union 18 | import numpy as np 19 | import torch 20 | 21 | __all__ = [ 22 | 'Tensor', 23 | 'Array', 24 | ] 25 | 26 | Tensor = NewType('Tensor', torch.Tensor) 27 | Array = NewType('Array', np.ndarray) 28 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/metrics.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import numpy as np 18 | import torch 19 | 20 | 21 | def v2v(x, y): 22 | if torch.is_tensor(x): 23 | return (x - y).pow(2).sum(dim=-1).sqrt().mean() 24 | else: 25 | return np.sqrt(np.power(x - y, 2)).sum(axis=-1).mean() 26 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/utils_cfg.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from typing import Tuple 18 | from dataclasses import dataclass 19 | 20 | 21 | @dataclass 22 | class Variable: 23 | create: bool = True 24 | requires_grad: bool = True 25 | 26 | 27 | @dataclass 28 | class Pose(Variable): 29 | type: str = 'aa' 30 | -------------------------------------------------------------------------------- /scripts/train.sh: -------------------------------------------------------------------------------- 1 | ## gpu 2 | CUDA_IDX=0 3 | 4 | ## project path 5 | output_root=../output 6 | project_root=project 7 | data_root=../data/nuscenes/processed_10Hz/trainval 8 | 9 | ## dataset config 10 | scene_idx_all=(152 164 171 200 209 359 529 916) 11 | dataset=nuscenes/6cams 12 | config_file=configs/omnire_ms_bilateral_extended.yaml 13 | 14 | start_timestep=0 15 | end_timestep=-1 16 | test_image_stride=10 17 | load_smpl=False 18 | 19 | ## train 20 | cd $project_root 21 | export PYTHONPATH=$(pwd) 22 | for scene_idx in "${scene_idx_all[@]}"; do 23 | CUDA_VISIBLE_DEVICES=$CUDA_IDX python tools/train.py \ 24 | --config_file $config_file \ 25 | --output_root $output_root \ 26 | --project ./ \ 27 | --run_name $scene_idx \ 28 | dataset=$dataset \ 29 | data.data_root=$data_root \ 30 | data.scene_idx=$scene_idx \ 31 | data.start_timestep=$start_timestep \ 32 | data.end_timestep=$end_timestep \ 33 | data.pixel_source.test_image_stride=$test_image_stride \ 34 | data.pixel_source.load_smpl=$load_smpl 35 | done 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Nan Wang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /project/data/argoverse_example_scenes.txt: -------------------------------------------------------------------------------- 1 | # scene_id, seg_name, start_timestep, end_timestep 2 | 0,00a6ffc1,0,-1 3 | 1,01bb304d,0,-1 4 | 2,022af476,0,-1 5 | 3,0322b098,0,-1 6 | 4,03b2cf2d,0,-1 7 | 5,03fba633,0,-1 8 | 6,04973bcf,0,-1 9 | 7,0526e68e,0,-1 10 | 8,05853f69,0,-1 11 | 9,05fb81ab,0,-1 12 | 10,067b1c50,0,-1 13 | 11,06852209,0,-1 14 | 12,069cc46d,0,-1 15 | 13,06e5ac08,0,-1 16 | 14,072c8e90,0,-1 17 | 15,0749e9e0,0,-1 18 | 16,074d2237,0,-1 19 | 17,07e4fccb,0,-1 20 | 18,080b1ce2,0,-1 21 | 19,08734a1b,0,-1 22 | 20,087695bd,0,-1 23 | 21,087fec73,0,-1 24 | 22,08bcc06e,0,-1 25 | 23,094c4119,0,-1 26 | 24,098fe60e,0,-1 27 | 25,0a132537,0,-1 28 | 26,0a524e66,0,-1 29 | 27,0a8a4cfa,0,-1 30 | 28,0ab21841,0,-1 31 | 29,0b1b993a,0,-1 32 | 30,0b324587,0,-1 33 | 31,0b9321c1,0,-1 34 | 32,0b97f5dd,0,-1 35 | 33,0c143226,0,-1 36 | 34,0c61aea3,0,-1 37 | 35,0d37aee4,0,-1 38 | 36,0d8aab9f,0,-1 39 | 37,0d9e4cff,0,-1 40 | 38,0f257dcc,0,-1 41 | 39,0fc6aa34,0,-1 42 | 40,105f47eb,0,-1 43 | 41,106d962b,0,-1 44 | 42,108d2060,0,-1 45 | 43,11420316,0,-1 46 | 44,118a1e87,0,-1 47 | 45,11995cbe,0,-1 48 | 46,12071817,0,-1 49 | 47,120d7ac7,0,-1 50 | 48,121007f3,0,-1 51 | 49,12c3c14b,0,-1 -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/losses/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import torch 18 | 19 | def get_reduction_method(reduction='mean'): 20 | if reduction == 'mean': 21 | return torch.mean 22 | elif reduction == 'sum': 23 | return torch.sum 24 | elif reduction == 'none': 25 | return lambda x: x 26 | else: 27 | raise ValueError('Unknown reduction method: {}'.format(reduction)) 28 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/dataset_defaults.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from omegaconf import OmegaConf 18 | from dataclasses import dataclass 19 | 20 | 21 | @dataclass 22 | class MeshFolder: 23 | data_folder: str = 'data/meshes' 24 | 25 | 26 | @dataclass 27 | class DatasetConfig: 28 | num_workers: int = 0 29 | name: str = 'mesh-folder' 30 | mesh_folder: MeshFolder = MeshFolder() 31 | 32 | 33 | conf = OmegaConf.structured(DatasetConfig) 34 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from .np_utils import to_np, rel_change 18 | from .torch_utils import from_torch 19 | from .timer import Timer, timer_decorator 20 | from .typing import * 21 | from .pose_utils import batch_rodrigues, batch_rot2aa 22 | from .metrics import v2v 23 | from .def_transfer import read_deformation_transfer, apply_deformation_transfer 24 | from .mesh_utils import get_vertices_per_edge 25 | from .o3d_utils import np_mesh_to_o3d 26 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/np_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | from __future__ import absolute_import 17 | from __future__ import print_function 18 | from __future__ import division 19 | 20 | import numpy as np 21 | 22 | 23 | def rel_change(prev_val, curr_val): 24 | return (prev_val - curr_val) / max([np.abs(prev_val), np.abs(curr_val), 1]) 25 | 26 | 27 | def max_grad_change(grad_arr): 28 | return grad_arr.abs().max() 29 | 30 | 31 | def to_np(array, dtype=np.float32): 32 | if hasattr(array, 'todense'): 33 | array = array.todense() 34 | return np.array(array, dtype=dtype) 35 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/o3d_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import open3d as o3d 18 | import torch 19 | 20 | Vector3d = o3d.utility.Vector3dVector 21 | Vector3i = o3d.utility.Vector3iVector 22 | 23 | Mesh = o3d.geometry.TriangleMesh 24 | 25 | 26 | def np_mesh_to_o3d(vertices, faces): 27 | if torch.is_tensor(vertices): 28 | vertices = vertices.detach().cpu().numpy() 29 | if torch.is_tensor(faces): 30 | faces = faces.detach().cpu().numpy() 31 | mesh = Mesh() 32 | mesh.vertices = Vector3d(vertices) 33 | mesh.triangles = Vector3i(faces) 34 | return mesh 35 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/loss_defaults.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | # from yacs.config import CfgNode as CN 17 | 18 | from typing import List, Tuple, Union 19 | from omegaconf import OmegaConf 20 | from loguru import logger 21 | from dataclasses import dataclass, make_dataclass 22 | 23 | 24 | @dataclass 25 | class LossTemplate: 26 | type: str = 'l2' 27 | active: bool = False 28 | weight: Tuple[float] = (0.0,) 29 | requires_grad: bool = True 30 | enable: int = 0 31 | 32 | 33 | @dataclass 34 | class LossConfig: 35 | type: str = 'smplify-x' 36 | 37 | 38 | conf = OmegaConf.structured(LossConfig) 39 | -------------------------------------------------------------------------------- /project/third_party/smplx/.gitignore: -------------------------------------------------------------------------------- 1 | #### joe made this: http://goel.io/joe 2 | 3 | #####=== Python ===##### 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | 110 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/optim_defaults.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from typing import Tuple 18 | from omegaconf import OmegaConf 19 | from dataclasses import dataclass 20 | 21 | 22 | @dataclass 23 | class LBFGS: 24 | line_search_fn: str = 'strong_wolfe' 25 | max_iter: int = 50 26 | 27 | 28 | @dataclass 29 | class SGD: 30 | momentum: float = 0.9 31 | nesterov: bool = True 32 | 33 | 34 | @dataclass 35 | class ADAM: 36 | betas: Tuple[float, float] = (0.9, 0.999) 37 | eps: float = 1e-08 38 | amsgrad: bool = False 39 | 40 | 41 | @dataclass 42 | class RMSProp: 43 | alpha: float = 0.99 44 | 45 | 46 | @dataclass 47 | class TrustRegionNewtonCG: 48 | max_trust_radius: float = 1000 49 | initial_trust_radius: float = 0.05 50 | eta: float = 0.15 51 | gtol: float = 1e-05 52 | 53 | 54 | @dataclass 55 | class OptimConfig: 56 | type: str = 'trust-ncg' 57 | lr: float = 1.0 58 | gtol: float = 1e-8 59 | ftol: float = -1.0 60 | maxiters: int = 100 61 | 62 | lbfgs: LBFGS = LBFGS() 63 | sgd: SGD = SGD() 64 | adam: ADAM = ADAM() 65 | trust_ncg: TrustRegionNewtonCG = TrustRegionNewtonCG() 66 | 67 | 68 | conf = OmegaConf.structured(OptimConfig) 69 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/cmd_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | 20 | import sys 21 | import os 22 | 23 | import argparse 24 | from loguru import logger 25 | 26 | from omegaconf import OmegaConf 27 | from .defaults import conf as default_conf 28 | 29 | 30 | def parse_args(argv=None) -> OmegaConf: 31 | arg_formatter = argparse.ArgumentDefaultsHelpFormatter 32 | 33 | description = 'Model transfer script' 34 | parser = argparse.ArgumentParser(formatter_class=arg_formatter, 35 | description=description) 36 | 37 | parser.add_argument('--exp-cfg', type=str, dest='exp_cfg', 38 | help='The configuration of the experiment') 39 | parser.add_argument('--exp-opts', default=[], dest='exp_opts', 40 | nargs='*', 41 | help='Command line arguments') 42 | 43 | cmd_args = parser.parse_args() 44 | 45 | cfg = default_conf.copy() 46 | if cmd_args.exp_cfg: 47 | cfg.merge_with(OmegaConf.load(cmd_args.exp_cfg)) 48 | if cmd_args.exp_opts: 49 | cfg.merge_with(OmegaConf.from_cli(cmd_args.exp_opts)) 50 | 51 | return cfg 52 | -------------------------------------------------------------------------------- /project/datasets/base/split_wrapper.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple 2 | import torch 3 | 4 | from .pixel_source import ScenePixelSource 5 | 6 | class SplitWrapper(torch.utils.data.Dataset): 7 | 8 | # a sufficiently large number to make sure we don't run out of data 9 | _num_iters = 1000000 10 | 11 | def __init__( 12 | self, 13 | datasource: ScenePixelSource, 14 | split_indices: List[int] = None, 15 | split: str = "train", 16 | ): 17 | super().__init__() 18 | self.datasource = datasource 19 | self.split_indices = split_indices 20 | self.split = split 21 | 22 | def get_image(self, idx, camera_downscale) -> dict: 23 | downscale_factor = 1 / camera_downscale * self.datasource.downscale_factor 24 | self.datasource.update_downscale_factor(downscale_factor) 25 | image_infos, cam_infos = self.datasource.get_image(self.split_indices[idx]) 26 | self.datasource.reset_downscale_factor() 27 | return image_infos, cam_infos 28 | 29 | def next(self, camera_downscale) -> Tuple[dict, dict]: 30 | assert self.split == "train", "Only train split supports next()" 31 | 32 | img_idx = self.datasource.propose_training_image( 33 | candidate_indices=self.split_indices 34 | ) 35 | 36 | downscale_factor = 1 / camera_downscale * self.datasource.downscale_factor 37 | self.datasource.update_downscale_factor(downscale_factor) 38 | image_infos, cam_infos = self.datasource.get_image(img_idx) 39 | self.datasource.reset_downscale_factor() 40 | 41 | return image_infos, cam_infos 42 | 43 | def __getitem__(self, idx) -> dict: 44 | return self.get_image(idx, camera_downscale=1.0) 45 | 46 | def __len__(self) -> int: 47 | return len(self.split_indices) 48 | 49 | @property 50 | def num_iters(self) -> int: 51 | return self._num_iters 52 | 53 | def set_num_iters(self, num_iters) -> None: 54 | self._num_iters = num_iters 55 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/timer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import time 18 | import numpy as np 19 | import torch 20 | 21 | from loguru import logger 22 | 23 | 24 | class Timer(object): 25 | def __init__(self, name='', sync=False): 26 | super(Timer, self).__init__() 27 | self.elapsed = [] 28 | self.name = name 29 | self.sync = sync 30 | 31 | def __enter__(self): 32 | if self.sync: 33 | torch.cuda.synchronize() 34 | self.start = time.perf_counter() 35 | 36 | def __exit__(self, type, value, traceback): 37 | if self.sync: 38 | torch.cuda.synchronize() 39 | elapsed = time.perf_counter() - self.start 40 | self.elapsed.append(elapsed) 41 | logger.info(f'[{self.name}]: {np.mean(self.elapsed):.3f}') 42 | 43 | 44 | def timer_decorator(sync=False, name=''): 45 | def wrapper(method): 46 | elapsed = [] 47 | 48 | def timed(*args, **kw): 49 | if sync: 50 | torch.cuda.synchronize() 51 | ts = time.perf_counter() 52 | result = method(*args, **kw) 53 | if sync: 54 | torch.cuda.synchronize() 55 | te = time.perf_counter() 56 | elapsed.append(te - ts) 57 | logger.info(f'[{name}]: {np.mean(elapsed):.3f}') 58 | return result 59 | return timed 60 | return wrapper 61 | -------------------------------------------------------------------------------- /project/generate_lidar/generate_radar_detection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from generate_lidar.visual_bbox_lidar import get_corners 3 | import torch 4 | import json 5 | import os 6 | 7 | def generate_cornrs(frame_idx, instances_info, frame_instances, lidar_to_world, output_path, remove_obj_id_list=None): 8 | """ 9 | Generate corners for the given image and frame index. 10 | 11 | Args: 12 | image (np.ndarray): The input image. 13 | frame_idx (int): The index of the frame. 14 | instances_info (dict): Dictionary containing instance information. 15 | frame_instances (dict): Dictionary containing frame instances. 16 | 17 | Returns: 18 | np.ndarray: The image with corners drawn on it. 19 | """ 20 | # Get the objects in the current frame 21 | bbox = {} 22 | objects = frame_instances[str(frame_idx)] 23 | if len(objects) == 0: 24 | return 25 | for obj_id in objects: 26 | obj_id = str(obj_id) 27 | if remove_obj_id_list is not None: 28 | if obj_id in remove_obj_id_list: 29 | continue 30 | idx_in_obj = instances_info[obj_id]['frame_annotations']['frame_idx'].index(frame_idx) 31 | o2w = np.array( 32 | instances_info[obj_id]['frame_annotations']['obj_to_world'][idx_in_obj] 33 | ) 34 | length, width, height = instances_info[obj_id]['frame_annotations']['box_size'][idx_in_obj] 35 | corners = get_corners(length, width, height) 36 | corners_world = (o2w[:3, :3] @ corners + o2w[:3, 3:4]).T 37 | 38 | world_to_lidar = lidar_to_world.inverse().numpy() 39 | 40 | corners_lidar = ( 41 | world_to_lidar[:3, :3] @ corners_world.T + world_to_lidar[:3, 3:4] 42 | ).T 43 | # bbox.append(corners_lidar) 44 | bbox[obj_id] = { 45 | "class_name": instances_info[obj_id]['class_name'], 46 | "bbox": corners_lidar.tolist() 47 | } 48 | # bbox = np.stack(bbox, axis=0) 49 | # print(bbox.shape) 50 | # np.save(os.path.join(output_path, f"{frame_idx:0>3d}.npy"), bbox) 51 | with open(os.path.join(output_path, f"{frame_idx:0>3d}.json"), 'w') as f: 52 | json.dump(bbox, f, indent=4) 53 | -------------------------------------------------------------------------------- /project/utils/backup.py: -------------------------------------------------------------------------------- 1 | # Modified from: https://github.com/pjlab-ADG/nr3d_lib/ 2 | import os 3 | import shutil 4 | import logging 5 | 6 | logger = logging.getLogger() 7 | 8 | def backup_folder( 9 | backup_dir: str, 10 | source_dir: str="./", 11 | filetypes_to_copy=[".py", ".h", ".cpp", ".cuh", ".cu", ".sh"] 12 | ): 13 | filetypes_to_copy = tuple(filetypes_to_copy) 14 | os.makedirs(backup_dir, exist_ok=True) 15 | for file in os.listdir(source_dir): 16 | if not file.endswith(filetypes_to_copy): 17 | continue 18 | source_file_path = os.path.join(source_dir, file) 19 | target_file_path = os.path.join(backup_dir, file) 20 | shutil.copy(source_file_path, target_file_path) 21 | 22 | def backup_folder_recursive( 23 | backup_dir: str, 24 | source_dir: str="./", 25 | filetypes_to_copy=[".py", ".h", ".cpp", ".cuh", ".cu", ".sh"] 26 | ): 27 | filetypes_to_copy = tuple(filetypes_to_copy) 28 | for root, _, files in os.walk(source_dir): 29 | for file in files: 30 | if not file.endswith(filetypes_to_copy): 31 | continue 32 | source_file_path = os.path.join(root, file) 33 | # Keeps original directory structure 34 | target_file_path = os.path.join(backup_dir, os.path.relpath(source_file_path, source_dir)) 35 | target_dir_path = os.path.dirname(target_file_path) 36 | os.makedirs(target_dir_path, exist_ok=True) 37 | shutil.copy(source_file_path, target_file_path) 38 | 39 | def backup_project( 40 | backup_dir: str, 41 | source_dir: str="./", 42 | subdirs_to_copy=["app", "code_multi", "code_single", "dataio", "nr3d_lib"], 43 | filetypes_to_copy=[".py", ".h", ".cpp", ".cuh", ".cu", ".sh"], 44 | ): 45 | filetypes_to_copy = tuple(filetypes_to_copy) 46 | # Automatic backup codes 47 | logger.info(f"=> Backing up from {source_dir} to {backup_dir}...") 48 | # Backup project root dir, depth = 1 49 | backup_folder(backup_dir, source_dir, filetypes_to_copy) 50 | # Backup cared subdirs, depth = inf 51 | for subdir in subdirs_to_copy: 52 | sub_source_dir = os.path.join(source_dir, subdir) 53 | sub_backup_dir = os.path.join(backup_dir, subdir) 54 | backup_folder_recursive(sub_backup_dir, sub_source_dir, filetypes_to_copy) 55 | logger.info("done.") -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/mesh_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Code from Chumpy and OpenDR. Placed here to avoid chumpy dependency 16 | # The original code can be found in https://github.com/MPI-IS/mesh 17 | import numpy as np 18 | import scipy.sparse as sp 19 | 20 | 21 | def row(A): 22 | return A.reshape((1, -1)) 23 | 24 | 25 | def col(A): 26 | return A.reshape((-1, 1)) 27 | 28 | 29 | def get_vert_connectivity(mesh_v, mesh_f): 30 | """Returns a sparse matrix (of size #verts x #verts) where each nonzero 31 | element indicates a neighborhood relation. For example, if there is a 32 | nonzero element in position (15,12), that means vertex 15 is connected 33 | by an edge to vertex 12.""" 34 | 35 | vpv = sp.csc_matrix((len(mesh_v), len(mesh_v))) 36 | 37 | # for each column in the faces... 38 | for i in range(3): 39 | IS = mesh_f[:, i] 40 | JS = mesh_f[:, (i + 1) % 3] 41 | data = np.ones(len(IS)) 42 | ij = np.vstack((row(IS.flatten()), row(JS.flatten()))) 43 | mtx = sp.csc_matrix((data, ij), shape=vpv.shape) 44 | vpv = vpv + mtx + mtx.T 45 | 46 | return vpv 47 | 48 | 49 | def get_vertices_per_edge(mesh_v, mesh_f): 50 | """Returns an Ex2 array of adjacencies between vertices, where 51 | each element in the array is a vertex index. Each edge is included 52 | only once. If output of get_faces_per_edge is provided, this is used to 53 | avoid call to get_vert_connectivity()""" 54 | 55 | vc = sp.coo_matrix(get_vert_connectivity(mesh_v, mesh_f)) 56 | result = np.hstack((col(vc.row), col(vc.col))) 57 | result = result[result[:, 0] < result[:, 1]] # for uniqueness 58 | 59 | return result 60 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/defaults.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from typing import Tuple, Optional 18 | from copy import deepcopy 19 | # from yacs.config import CfgNode as CN 20 | from dataclasses import dataclass 21 | from omegaconf import OmegaConf 22 | 23 | from .loss_defaults import conf as loss_cfg, LossConfig 24 | from .dataset_defaults import conf as dataset_cfg, DatasetConfig 25 | from .optim_defaults import conf as optim_cfg, OptimConfig 26 | from .body_model_defaults import conf as body_model_cfg, BodyModelConfig 27 | 28 | 29 | @dataclass 30 | class EdgeFitting: 31 | per_part: bool = False 32 | reduction: str = 'mean' 33 | 34 | 35 | @dataclass 36 | class VertexFitting: 37 | per_part: bool = False 38 | reduction: str = 'mean' 39 | type: str = 'l2' 40 | 41 | 42 | @dataclass 43 | class Config: 44 | use_cuda: bool = True 45 | log_file: str = '/tmp/logs' 46 | output_folder: str = 'output' 47 | save_verts: bool = True 48 | save_joints: bool = True 49 | save_mesh: bool = False 50 | save_img_summaries: bool = True 51 | summary_steps: int = 5 52 | degrees: Tuple[float] = (90,) 53 | float_type: str = 'float' 54 | logger_level: str = 'INFO' 55 | interactive: bool = True 56 | batch_size: Optional[int] = 1 57 | color_path: str = 'data/smpl_with_colors.ply' 58 | 59 | optim: OptimConfig = optim_cfg 60 | datasets: DatasetConfig = dataset_cfg 61 | losses: LossConfig = loss_cfg 62 | body_model: BodyModelConfig = body_model_cfg 63 | 64 | deformation_transfer_path: str = '' 65 | mask_ids_fname: str = '' 66 | 67 | per_part: bool = True 68 | edge_fitting: EdgeFitting = EdgeFitting() 69 | 70 | 71 | conf = OmegaConf.structured(Config) 72 | -------------------------------------------------------------------------------- /project/configs/datasets/kitti/1cams.yaml: -------------------------------------------------------------------------------- 1 | # KITTI dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 CAM_LEFT (375, 1242) 5 | # 1 CAM_RIGHT (375, 1242) 6 | 7 | data: 8 | data_root: data/kitti/processed # data root for the dataset 9 | dataset: kitti # dataset type 10 | scene_idx: 2011_09_26_drive_0091_sync # which scene to use, specified by date and drive number 11 | start_timestep: 0 # which timestep to start from 12 | end_timestep: 200 # which timestep to end at, -1 means the last timestep 13 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 14 | pixel_source: # image source and object annotations 15 | type: datasets.kitti.kitti_sourceloader.KITTIPixelSource 16 | cameras: [0] # which cameras to use 17 | downscale_when_loading: [1] # the size of the images to load 18 | downscale: 1 # downscale factor wrt to the downscale_when_loading 19 | undistort: False # whether to undistort the images 20 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 21 | load_sky_mask: True # whether to load sky mask 22 | load_dynamic_mask: True # whether to load dynamic mask 23 | load_objects: True # whether to load object bounding boxes 24 | load_smpl: True # whether to load SMPL template for pedestrians (not available for KITTI) 25 | sampler: # error based image sampler 26 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 27 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 28 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 29 | lidar_source: # everything related to "lidar" --- from lidar points 30 | type: datasets.kitti.kitti_sourceloader.KITTILiDARSource 31 | load_lidar: True # whether to load lidar 32 | # ---- compute aabb from lidar ---- # 33 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 34 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 35 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 36 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 37 | lidar_percentile: 0.02 # percentile to compute aabb from lidar 38 | 39 | # dataset specific configurations 40 | model: 41 | RigidNodes: 42 | ctrl: 43 | stop_split_at: 30000 -------------------------------------------------------------------------------- /project/configs/datasets/kitti/2cams.yaml: -------------------------------------------------------------------------------- 1 | # KITTI dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 CAM_LEFT (375, 1242) 5 | # 1 CAM_RIGHT (375, 1242) 6 | 7 | data: 8 | data_root: data/kitti/processed # data root for the dataset 9 | dataset: kitti # dataset type 10 | scene_idx: 2011_09_26_drive_0091_sync # which scene to use, specified by date and drive number 11 | start_timestep: 0 # which timestep to start from 12 | end_timestep: 200 # which timestep to end at, -1 means the last timestep 13 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 14 | pixel_source: # image source and object annotations 15 | type: datasets.kitti.kitti_sourceloader.KITTIPixelSource 16 | cameras: [0, 1] # which cameras to use 17 | downscale_when_loading: [2, 2] # the size of the images to load 18 | downscale: 1 # downscale factor wrt to the downscale_when_loading 19 | undistort: False # whether to undistort the images 20 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 21 | load_sky_mask: True # whether to load sky mask 22 | load_dynamic_mask: True # whether to load dynamic mask 23 | load_objects: True # whether to load object bounding boxes 24 | load_smpl: True # whether to load SMPL template for pedestrians (not available for KITTI) 25 | sampler: # error based image sampler 26 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 27 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 28 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 29 | lidar_source: # everything related to "lidar" --- from lidar points 30 | type: datasets.kitti.kitti_sourceloader.KITTILiDARSource 31 | load_lidar: True # whether to load lidar 32 | # ---- compute aabb from lidar ---- # 33 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 34 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 35 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 36 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 37 | lidar_percentile: 0.02 # percentile to compute aabb from lidar 38 | 39 | # dataset specific configurations 40 | model: 41 | RigidNodes: 42 | ctrl: 43 | stop_split_at: 30000 -------------------------------------------------------------------------------- /project/third_party/smplx/smplx/vertex_ids.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: ps-license@tuebingen.mpg.de 16 | 17 | from __future__ import print_function 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | 21 | # Joint name to vertex mapping. SMPL/SMPL-H/SMPL-X vertices that correspond to 22 | # MSCOCO and OpenPose joints 23 | vertex_ids = { 24 | 'smplh': { 25 | 'nose': 332, 26 | 'reye': 6260, 27 | 'leye': 2800, 28 | 'rear': 4071, 29 | 'lear': 583, 30 | 'rthumb': 6191, 31 | 'rindex': 5782, 32 | 'rmiddle': 5905, 33 | 'rring': 6016, 34 | 'rpinky': 6133, 35 | 'lthumb': 2746, 36 | 'lindex': 2319, 37 | 'lmiddle': 2445, 38 | 'lring': 2556, 39 | 'lpinky': 2673, 40 | 'LBigToe': 3216, 41 | 'LSmallToe': 3226, 42 | 'LHeel': 3387, 43 | 'RBigToe': 6617, 44 | 'RSmallToe': 6624, 45 | 'RHeel': 6787 46 | }, 47 | 'smplx': { 48 | 'nose': 9120, 49 | 'reye': 9929, 50 | 'leye': 9448, 51 | 'rear': 616, 52 | 'lear': 6, 53 | 'rthumb': 8079, 54 | 'rindex': 7669, 55 | 'rmiddle': 7794, 56 | 'rring': 7905, 57 | 'rpinky': 8022, 58 | 'lthumb': 5361, 59 | 'lindex': 4933, 60 | 'lmiddle': 5058, 61 | 'lring': 5169, 62 | 'lpinky': 5286, 63 | 'LBigToe': 5770, 64 | 'LSmallToe': 5780, 65 | 'LHeel': 8846, 66 | 'RBigToe': 8463, 67 | 'RSmallToe': 8474, 68 | 'RHeel': 8635 69 | }, 70 | 'mano': { 71 | 'thumb': 744, 72 | 'index': 320, 73 | 'middle': 443, 74 | 'ring': 554, 75 | 'pinky': 671, 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /project/configs/datasets/nuscenes/1cams.yaml: -------------------------------------------------------------------------------- 1 | # NuScenes dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 CAM_FRONT (900, 1600) 5 | # 1 CAM_FRONT_LEFT (900, 1600) 6 | # 2 CAM_FRONT_RIGHT (900, 1600) 7 | # 3 CAM_BACK_LEFT (900, 1600) 8 | # 4 CAM_BACK_RIGHT (900, 1600) 9 | # 5 CAM_BACK (900, 1600) 10 | 11 | data: 12 | data_root: data/nuscenes/processed_10Hz/mini # data root for the dataset 13 | dataset: nuscenes # dataset type 14 | scene_idx: 0 # which scene to use, [0, 849] for nuscenes's train/val sets, inclusive 15 | start_timestep: 0 # which timestep to start from 16 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 17 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 18 | pixel_source: # image source and object annotations 19 | type: datasets.nuscenes.nuscenes_sourceloader.NuScenesPixelSource 20 | cameras: [0] # which cameras to use 21 | downscale_when_loading: [2] # the size of the images to load 22 | downscale: 1 # downscale factor wrt to the downscale_when_loading 23 | undistort: False # whether to undistort the images 24 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 25 | load_sky_mask: True # whether to load sky mask 26 | load_dynamic_mask: True # whether to load dynamic mask 27 | load_objects: True # whether to load object bounding boxes 28 | load_smpl: True # whether to load SMPL template for pedestrians 29 | sampler: # error based image sampler 30 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 31 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 32 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 33 | lidar_source: # everything related to "lidar" --- from lidar points 34 | type: datasets.nuscenes.nuscenes_sourceloader.NuScenesLiDARSource 35 | load_lidar: True # whether to load lidar 36 | # ---- compute aabb from lidar ---- # 37 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 38 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 39 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 40 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 41 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/nuscenes/3cams.yaml: -------------------------------------------------------------------------------- 1 | # NuScenes dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 CAM_FRONT (900, 1600) 5 | # 1 CAM_FRONT_LEFT (900, 1600) 6 | # 2 CAM_FRONT_RIGHT (900, 1600) 7 | # 3 CAM_BACK_LEFT (900, 1600) 8 | # 4 CAM_BACK_RIGHT (900, 1600) 9 | # 5 CAM_BACK (900, 1600) 10 | 11 | data: 12 | data_root: data/nuscenes/processed_10Hz/mini # data root for the dataset 13 | dataset: nuscenes # dataset type 14 | scene_idx: 0 # which scene to use, [0, 849] for nuscenes's train/val sets, inclusive 15 | start_timestep: 0 # which timestep to start from 16 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 17 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 18 | pixel_source: # image source and object annotations 19 | type: datasets.nuscenes.nuscenes_sourceloader.NuScenesPixelSource 20 | cameras: [0, 1, 2] # which cameras to use 21 | downscale_when_loading: [2, 2, 2] # the size of the images to load 22 | downscale: 1 # downscale factor wrt to the downscale_when_loading 23 | undistort: False # whether to undistort the images 24 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 25 | load_sky_mask: True # whether to load sky mask 26 | load_dynamic_mask: True # whether to load dynamic mask 27 | load_objects: True # whether to load object bounding boxes 28 | load_smpl: True # whether to load SMPL template for pedestrians 29 | sampler: # error based image sampler 30 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 31 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 32 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 33 | lidar_source: # everything related to "lidar" --- from lidar points 34 | type: datasets.nuscenes.nuscenes_sourceloader.NuScenesLiDARSource 35 | load_lidar: True # whether to load lidar 36 | # ---- compute aabb from lidar ---- # 37 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 38 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 39 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 40 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 41 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/argoverse/1cams.yaml: -------------------------------------------------------------------------------- 1 | # ArgoVerse2 dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 ring_front_center (1550, 2048) 5 | # 1 ring_front_left (2048, 1550) 6 | # 2 ring_front_right (2048, 1550) 7 | # 3 ring_side_left (2048, 1550) 8 | # 4 ring_side_right (2048, 1550) 9 | # 5 ring_rear_left (2048, 1550) 10 | # 6 ring_rear_right (2048, 1550) 11 | 12 | data: 13 | data_root: data/argoverse/processed/training # data root for the dataset 14 | dataset: argoverse # dataset type 15 | scene_idx: 0 # which scene to use 16 | start_timestep: 0 # which timestep to start from 17 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 18 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 19 | pixel_source: # image source and object annotations 20 | type: datasets.argoverse.argoverse_sourceloader.ArgoVersePixelSource 21 | cameras: [0] # which cameras to use 22 | downscale_when_loading: [2] # the size of the images to load 23 | downscale: 1 # downscale factor wrt to the downscale_when_loading 24 | undistort: False # whether to undistort the images 25 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 26 | load_sky_mask: True # whether to load sky mask 27 | load_dynamic_mask: True # whether to load dynamic mask 28 | load_objects: True # whether to load object bounding boxes 29 | load_smpl: True # whether to load SMPL template for pedestrians 30 | sampler: # error based image sampler 31 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 32 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 33 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 34 | lidar_source: # everything related to "lidar" --- from lidar points 35 | type: datasets.argoverse.argoverse_sourceloader.ArgoVerseLiDARSource 36 | load_lidar: True # whether to load lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/pandaset/1cams.yaml: -------------------------------------------------------------------------------- 1 | # Pandaset dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 front_camera (1920, 1080) 5 | # 1 front_left_camera (1920, 1080) 6 | # 2 front_right_camera (1920, 1080) 7 | # 3 left_camera (1920, 1080) 8 | # 4 right_camera (1920, 1080) 9 | # 5 back_camera (1920, 1080) 10 | 11 | data: 12 | data_root: data/pandaset/processed # data root for the dataset 13 | dataset: pandaset # dataset type 14 | scene_idx: 0 # which scene to use 15 | start_timestep: 0 # which timestep to start from 16 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 17 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 18 | pixel_source: # image source and object annotations 19 | type: datasets.pandaset.pandaset_sourceloader.PandaPixelSource 20 | cameras: [0] # which cameras to use 21 | downscale_when_loading: [1] # the size of the images to load 22 | downscale: 1 # downscale factor wrt to the downscale_when_loading 23 | undistort: False # whether to undistort the images 24 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 25 | load_sky_mask: True # whether to load sky mask 26 | load_dynamic_mask: True # whether to load dynamic mask 27 | load_objects: True # whether to load object bounding boxes 28 | load_smpl: True # whether to load SMPL template for pedestrians 29 | sampler: # error based image sampler 30 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 31 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 32 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 33 | lidar_source: # everything related to "lidar" --- from lidar points 34 | type: datasets.pandaset.pandaset_sourceloader.PandaLiDARSource 35 | load_lidar: True # whether to load lidar 36 | only_use_360_lidar: False # whether to only use the 360-degree lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/nuscenes/6cams.yaml: -------------------------------------------------------------------------------- 1 | # NuScenes dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 CAM_FRONT (900, 1600) 5 | # 1 CAM_FRONT_LEFT (900, 1600) 6 | # 2 CAM_FRONT_RIGHT (900, 1600) 7 | # 3 CAM_BACK_LEFT (900, 1600) 8 | # 4 CAM_BACK_RIGHT (900, 1600) 9 | # 5 CAM_BACK (900, 1600) 10 | 11 | data: 12 | data_root: data/nuscenes/processed_10Hz/mini # data root for the dataset 13 | dataset: nuscenes # dataset type 14 | scene_idx: 0 # which scene to use, [0, 849] for nuscenes's train/val sets, inclusive 15 | start_timestep: 0 # which timestep to start from 16 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 17 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device 18 | pixel_source: # image source and object annotations 19 | type: datasets.nuscenes.nuscenes_sourceloader.NuScenesPixelSource 20 | cameras: [0, 1, 2, 3, 4, 5] # which cameras to use 21 | downscale_when_loading: [3, 3, 3, 3, 3, 3] # the size of the images to load 22 | downscale: 1 # downscale factor wrt to the downscale_when_loading 23 | undistort: False # whether to undistort the images 24 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 25 | load_sky_mask: True # whether to load sky mask 26 | load_dynamic_mask: True # whether to load dynamic mask 27 | load_objects: True # whether to load object bounding boxes 28 | load_smpl: True # whether to load SMPL template for pedestrians 29 | sampler: # error based image sampler 30 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 31 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 32 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 33 | lidar_source: # everything related to "lidar" --- from lidar points 34 | type: datasets.nuscenes.nuscenes_sourceloader.NuScenesLiDARSource 35 | load_lidar: True # whether to load lidar 36 | # ---- compute aabb from lidar ---- # 37 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 38 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 39 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 40 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 41 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/argoverse/3cams.yaml: -------------------------------------------------------------------------------- 1 | # ArgoVerse2 dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 ring_front_center (1550, 2048) 5 | # 1 ring_front_left (2048, 1550) 6 | # 2 ring_front_right (2048, 1550) 7 | # 3 ring_side_left (2048, 1550) 8 | # 4 ring_side_right (2048, 1550) 9 | # 5 ring_rear_left (2048, 1550) 10 | # 6 ring_rear_right (2048, 1550) 11 | 12 | data: 13 | data_root: data/argoverse/processed/training # data root for the dataset 14 | dataset: argoverse # dataset type 15 | scene_idx: 0 # which scene to use 16 | start_timestep: 0 # which timestep to start from 17 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 18 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 19 | pixel_source: # image source and object annotations 20 | type: datasets.argoverse.argoverse_sourceloader.ArgoVersePixelSource 21 | cameras: [0, 1, 2] # which cameras to use 22 | downscale_when_loading: [2, 2, 2] # the size of the images to load 23 | downscale: 1 # downscale factor wrt to the downscale_when_loading 24 | undistort: False # whether to undistort the images 25 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 26 | load_sky_mask: True # whether to load sky mask 27 | load_dynamic_mask: True # whether to load dynamic mask 28 | load_objects: True # whether to load object bounding boxes 29 | load_smpl: True # whether to load SMPL template for pedestrians 30 | sampler: # error based image sampler 31 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 32 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 33 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 34 | lidar_source: # everything related to "lidar" --- from lidar points 35 | type: datasets.argoverse.argoverse_sourceloader.ArgoVerseLiDARSource 36 | load_lidar: True # whether to load lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/pandaset/3cams.yaml: -------------------------------------------------------------------------------- 1 | # Pandaset dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 front_camera (1920, 1080) 5 | # 1 front_left_camera (1920, 1080) 6 | # 2 front_right_camera (1920, 1080) 7 | # 3 left_camera (1920, 1080) 8 | # 4 right_camera (1920, 1080) 9 | # 5 back_camera (1920, 1080) 10 | 11 | data: 12 | data_root: data/pandaset/processed # data root for the dataset 13 | dataset: pandaset # dataset type 14 | scene_idx: 0 # which scene to use 15 | start_timestep: 0 # which timestep to start from 16 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 17 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 18 | pixel_source: # image source and object annotations 19 | type: datasets.pandaset.pandaset_sourceloader.PandaPixelSource 20 | cameras: [0, 1, 2] # which cameras to use 21 | downscale_when_loading: [2, 2, 2] # the size of the images to load 22 | downscale: 1 # downscale factor wrt to the downscale_when_loading 23 | undistort: False # whether to undistort the images 24 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 25 | load_sky_mask: True # whether to load sky mask 26 | load_dynamic_mask: True # whether to load dynamic mask 27 | load_objects: True # whether to load object bounding boxes 28 | load_smpl: True # whether to load SMPL template for pedestrians 29 | sampler: # error based image sampler 30 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 31 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 32 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 33 | lidar_source: # everything related to "lidar" --- from lidar points 34 | type: datasets.pandaset.pandaset_sourceloader.PandaLiDARSource 35 | load_lidar: True # whether to load lidar 36 | only_use_360_lidar: False # whether to only use the 360-degree lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/argoverse/5cams.yaml: -------------------------------------------------------------------------------- 1 | # ArgoVerse2 dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 ring_front_center (1550, 2048) 5 | # 1 ring_front_left (2048, 1550) 6 | # 2 ring_front_right (2048, 1550) 7 | # 3 ring_side_left (2048, 1550) 8 | # 4 ring_side_right (2048, 1550) 9 | # 5 ring_rear_left (2048, 1550) 10 | # 6 ring_rear_right (2048, 1550) 11 | 12 | data: 13 | data_root: data/argoverse/processed/training # data root for the dataset 14 | dataset: argoverse # dataset type 15 | scene_idx: 0 # which scene to use 16 | start_timestep: 0 # which timestep to start from 17 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 18 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device 19 | pixel_source: # image source and object annotations 20 | type: datasets.argoverse.argoverse_sourceloader.ArgoVersePixelSource 21 | cameras: [0, 1, 2, 3, 4] # which cameras to use 22 | downscale_when_loading: [3, 3, 3, 3, 3] # the size of the images to load 23 | downscale: 1 # downscale factor wrt to the downscale_when_loading 24 | undistort: False # whether to undistort the images 25 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 26 | load_sky_mask: True # whether to load sky mask 27 | load_dynamic_mask: True # whether to load dynamic mask 28 | load_objects: True # whether to load object bounding boxes 29 | load_smpl: True # whether to load SMPL template for pedestrians 30 | sampler: # error based image sampler 31 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 32 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 33 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 34 | lidar_source: # everything related to "lidar" --- from lidar points 35 | type: datasets.argoverse.argoverse_sourceloader.ArgoVerseLiDARSource 36 | load_lidar: True # whether to load lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/argoverse/7cams.yaml: -------------------------------------------------------------------------------- 1 | # ArgoVerse2 dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 ring_front_center (1550, 2048) 5 | # 1 ring_front_left (2048, 1550) 6 | # 2 ring_front_right (2048, 1550) 7 | # 3 ring_side_left (2048, 1550) 8 | # 4 ring_side_right (2048, 1550) 9 | # 5 ring_rear_left (2048, 1550) 10 | # 6 ring_rear_right (2048, 1550) 11 | 12 | data: 13 | data_root: data/argoverse/processed/training # data root for the dataset 14 | dataset: argoverse # dataset type 15 | scene_idx: 0 # which scene to use 16 | start_timestep: 0 # which timestep to start from 17 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 18 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device 19 | pixel_source: # image source and object annotations 20 | type: datasets.argoverse.argoverse_sourceloader.ArgoVersePixelSource 21 | cameras: [0, 1, 2, 3, 4, 5, 6] # which cameras to use 22 | downscale_when_loading: [3, 3, 3, 3, 3, 3, 3] # the size of the images to load 23 | downscale: 1 # downscale factor wrt to the downscale_when_loading 24 | undistort: False # whether to undistort the images 25 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 26 | load_sky_mask: True # whether to load sky mask 27 | load_dynamic_mask: True # whether to load dynamic mask 28 | load_objects: True # whether to load object bounding boxes 29 | load_smpl: True # whether to load SMPL template for pedestrians 30 | sampler: # error based image sampler 31 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 32 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 33 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 34 | lidar_source: # everything related to "lidar" --- from lidar points 35 | type: datasets.argoverse.argoverse_sourceloader.ArgoVerseLiDARSource 36 | load_lidar: True # whether to load lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/pandaset/6cams.yaml: -------------------------------------------------------------------------------- 1 | # Pandaset dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 front_camera (1920, 1080) 5 | # 1 front_left_camera (1920, 1080) 6 | # 2 front_right_camera (1920, 1080) 7 | # 3 left_camera (1920, 1080) 8 | # 4 right_camera (1920, 1080) 9 | # 5 back_camera (1920, 1080) 10 | 11 | data: 12 | data_root: data/pandaset/processed # data root for the dataset 13 | dataset: pandaset # dataset type 14 | scene_idx: 0 # which scene to use 15 | start_timestep: 0 # which timestep to start from 16 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 17 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device 18 | pixel_source: # image source and object annotations 19 | type: datasets.pandaset.pandaset_sourceloader.PandaPixelSource 20 | cameras: [0, 1, 2, 3, 4, 5] # which cameras to use 21 | downscale_when_loading: [2, 2, 2, 2, 2, 2] # the size of the images to load 22 | downscale: 1 # downscale factor wrt to the downscale_when_loading 23 | undistort: False # whether to undistort the images 24 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 25 | load_sky_mask: True # whether to load sky mask 26 | load_dynamic_mask: True # whether to load dynamic mask 27 | load_objects: True # whether to load object bounding boxes 28 | load_smpl: True # whether to load SMPL template for pedestrians 29 | sampler: # error based image sampler 30 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 31 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 32 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 33 | lidar_source: # everything related to "lidar" --- from lidar points 34 | type: datasets.pandaset.pandaset_sourceloader.PandaLiDARSource 35 | load_lidar: True # whether to load lidar 36 | only_use_360_lidar: False # whether to only use the 360-degree lidar 37 | # ---- compute aabb from lidar ---- # 38 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 39 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 40 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 41 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 42 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/nuplan/1cams.yaml: -------------------------------------------------------------------------------- 1 | # NuPlan dataset configuration 2 | # Processed cameras: 3 | # idx camera original size egocar_visible 4 | # 0 CAM_F0 (1080, 1920) False 5 | # 1 CAM_L0 (1080, 1920) True 6 | # 2 CAM_R0 (1080, 1920) True 7 | # 3 CAM_L1 (1080, 1920) False 8 | # 4 CAM_R1 (1080, 1920) False 9 | # 5 CAM_L2 (1080, 1920) True 10 | # 6 CAM_R2 (1080, 1920) True 11 | # 7 CAM_B0 (1080, 1920) False 12 | 13 | data: 14 | data_root: data/nuplan/processed/mini # data root for the dataset 15 | dataset: nuplan # dataset type 16 | scene_idx: 2021.05.12.22.00.38_veh-35_01008_01518 # which scene to use, specific to NuPlan's naming convention 17 | start_timestep: 0 # which timestep to start from 18 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 19 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 20 | pixel_source: # image source and object annotations 21 | type: datasets.nuplan.nuplan_sourceloader.NuPlanPixelSource 22 | cameras: [0] # which cameras to use 23 | downscale_when_loading: [2] # the size of the images to load 24 | downscale: 1 # downscale factor wrt to the downscale_when_loading 25 | undistort: False # whether to undistort the images 26 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 27 | load_sky_mask: True # whether to load sky mask 28 | load_dynamic_mask: True # whether to load dynamic mask 29 | load_objects: True # whether to load object bounding boxes 30 | load_smpl: True # whether to load SMPL template for pedestrians 31 | sampler: # error based image sampler 32 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 33 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 34 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 35 | lidar_source: # everything related to "lidar" --- from lidar points 36 | type: datasets.nuplan.nuplan_sourceloader.NuPlanLiDARSource 37 | load_lidar: True # whether to load lidar 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/nuplan/3cams.yaml: -------------------------------------------------------------------------------- 1 | # NuPlan dataset configuration 2 | # Processed cameras: 3 | # idx camera original size egocar_visible 4 | # 0 CAM_F0 (1080, 1920) False 5 | # 1 CAM_L0 (1080, 1920) True 6 | # 2 CAM_R0 (1080, 1920) True 7 | # 3 CAM_L1 (1080, 1920) False 8 | # 4 CAM_R1 (1080, 1920) False 9 | # 5 CAM_L2 (1080, 1920) True 10 | # 6 CAM_R2 (1080, 1920) True 11 | # 7 CAM_B0 (1080, 1920) False 12 | 13 | data: 14 | data_root: data/nuplan/processed/mini # data root for the dataset 15 | dataset: nuplan # dataset type 16 | scene_idx: 2021.05.12.22.00.38_veh-35_01008_01518 # which scene to use, specific to NuPlan's naming convention 17 | start_timestep: 0 # which timestep to start from 18 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 19 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device 20 | pixel_source: # image source and object annotations 21 | type: datasets.nuplan.nuplan_sourceloader.NuPlanPixelSource 22 | cameras: [0, 1, 2] # which cameras to use 23 | downscale_when_loading: [2, 2, 2] # the size of the images to load 24 | downscale: 1 # downscale factor wrt to the downscale_when_loading 25 | undistort: False # whether to undistort the images 26 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 27 | load_sky_mask: True # whether to load sky mask 28 | load_dynamic_mask: True # whether to load dynamic mask 29 | load_objects: True # whether to load object bounding boxes 30 | load_smpl: True # whether to load SMPL template for pedestrians 31 | sampler: # error based image sampler 32 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 33 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 34 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 35 | lidar_source: # everything related to "lidar" --- from lidar points 36 | type: datasets.nuplan.nuplan_sourceloader.NuPlanLiDARSource 37 | load_lidar: True # whether to load lidar 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/nuplan/5cams.yaml: -------------------------------------------------------------------------------- 1 | # NuPlan dataset configuration 2 | # Processed cameras: 3 | # idx camera original size egocar_visible 4 | # 0 CAM_F0 (1080, 1920) False 5 | # 1 CAM_L0 (1080, 1920) True 6 | # 2 CAM_R0 (1080, 1920) True 7 | # 3 CAM_L1 (1080, 1920) False 8 | # 4 CAM_R1 (1080, 1920) False 9 | # 5 CAM_L2 (1080, 1920) True 10 | # 6 CAM_R2 (1080, 1920) True 11 | # 7 CAM_B0 (1080, 1920) False 12 | 13 | data: 14 | data_root: data/nuplan/processed/mini # data root for the dataset 15 | dataset: nuplan # dataset type 16 | scene_idx: 2021.05.12.22.00.38_veh-35_01008_01518 # which scene to use, specific to NuPlan's naming convention 17 | start_timestep: 0 # which timestep to start from 18 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 19 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device 20 | pixel_source: # image source and object annotations 21 | type: datasets.nuplan.nuplan_sourceloader.NuPlanPixelSource 22 | cameras: [0, 1, 2, 3, 4] # which cameras to use 23 | downscale_when_loading: [3, 3, 3, 3, 3] # the size of the images to load 24 | downscale: 1 # downscale factor wrt to the downscale_when_loading 25 | undistort: False # whether to undistort the images 26 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 27 | load_sky_mask: True # whether to load sky mask 28 | load_dynamic_mask: True # whether to load dynamic mask 29 | load_objects: True # whether to load object bounding boxes 30 | load_smpl: True # whether to load SMPL template for pedestrians 31 | sampler: # error based image sampler 32 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 33 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 34 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 35 | lidar_source: # everything related to "lidar" --- from lidar points 36 | type: datasets.nuplan.nuplan_sourceloader.NuPlanLiDARSource 37 | load_lidar: True # whether to load lidar 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/nuplan/8cams.yaml: -------------------------------------------------------------------------------- 1 | # NuPlan dataset configuration 2 | # Processed cameras: 3 | # idx camera original size egocar_visible 4 | # 0 CAM_F0 (1080, 1920) False 5 | # 1 CAM_L0 (1080, 1920) True 6 | # 2 CAM_R0 (1080, 1920) True 7 | # 3 CAM_L1 (1080, 1920) False 8 | # 4 CAM_R1 (1080, 1920) False 9 | # 5 CAM_L2 (1080, 1920) True 10 | # 6 CAM_R2 (1080, 1920) True 11 | # 7 CAM_B0 (1080, 1920) False 12 | 13 | data: 14 | data_root: data/nuplan/processed/mini # data root for the dataset 15 | dataset: nuplan # dataset type 16 | scene_idx: 2021.05.12.22.00.38_veh-35_01008_01518 # which scene to use, specific to NuPlan's naming convention 17 | start_timestep: 0 # which timestep to start from 18 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 19 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device 20 | pixel_source: # image source and object annotations 21 | type: datasets.nuplan.nuplan_sourceloader.NuPlanPixelSource 22 | cameras: [0, 1, 2, 3, 4, 5, 6, 7] # which cameras to use 23 | downscale_when_loading: [3, 3, 3, 3, 3, 3, 3, 3] # the size of the images to load 24 | downscale: 1 # downscale factor wrt to the downscale_when_loading 25 | undistort: False # whether to undistort the images 26 | test_image_stride: 0 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 27 | load_sky_mask: True # whether to load sky mask 28 | load_dynamic_mask: True # whether to load dynamic mask 29 | load_objects: True # whether to load object bounding boxes 30 | load_smpl: True # whether to load SMPL template for pedestrians 31 | sampler: # error based image sampler 32 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 33 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 34 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 35 | lidar_source: # everything related to "lidar" --- from lidar points 36 | type: datasets.nuplan.nuplan_sourceloader.NuPlanLiDARSource 37 | load_lidar: True # whether to load lidar 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/third_party/smplx/setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems and the Max Planck Institute for Biological 14 | # Cybernetics. All rights reserved. 15 | # 16 | # Contact: ps-license@tuebingen.mpg.de 17 | 18 | import io 19 | import os 20 | 21 | from setuptools import setup 22 | 23 | # Package meta-data. 24 | NAME = 'smplx' 25 | DESCRIPTION = 'PyTorch module for loading the SMPLX body model' 26 | URL = 'http://smpl-x.is.tuebingen.mpg.de' 27 | EMAIL = 'vassilis.choutas@tuebingen.mpg.de' 28 | AUTHOR = 'Vassilis Choutas' 29 | REQUIRES_PYTHON = '>=3.6.0' 30 | VERSION = '0.1.28' 31 | 32 | here = os.path.abspath(os.path.dirname(__file__)) 33 | 34 | try: 35 | FileNotFoundError 36 | except NameError: 37 | FileNotFoundError = IOError 38 | 39 | # Import the README and use it as the long-description. 40 | # Note: this will only work if 'README.md' is present in your MANIFEST.in file! 41 | try: 42 | with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: 43 | long_description = '\n' + f.read() 44 | except FileNotFoundError: 45 | long_description = DESCRIPTION 46 | 47 | # Load the package's __version__.py module as a dictionary. 48 | about = {} 49 | if not VERSION: 50 | with open(os.path.join(here, NAME, '__version__.py')) as f: 51 | exec(f.read(), about) 52 | else: 53 | about['__version__'] = VERSION 54 | 55 | pyrender_reqs = ['pyrender>=0.1.23', 'trimesh>=2.37.6', 'shapely'] 56 | matplotlib_reqs = ['matplotlib'] 57 | open3d_reqs = ['open3d-python'] 58 | 59 | setup(name=NAME, 60 | version=about['__version__'], 61 | description=DESCRIPTION, 62 | long_description=long_description, 63 | long_description_content_type='text/markdown', 64 | author=AUTHOR, 65 | author_email=EMAIL, 66 | python_requires=REQUIRES_PYTHON, 67 | url=URL, 68 | install_requires=[ 69 | 'numpy>=1.16.2', 70 | 'torch>=1.0.1.post2', 71 | ], 72 | extras_require={ 73 | 'pyrender': pyrender_reqs, 74 | 'open3d': open3d_reqs, 75 | 'matplotlib': matplotlib_reqs, 76 | 'all': pyrender_reqs + matplotlib_reqs + open3d_reqs 77 | }, 78 | packages=['smplx']) 79 | -------------------------------------------------------------------------------- /project/configs/datasets/waymo/1cams.yaml: -------------------------------------------------------------------------------- 1 | # Waymo dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 front_camera (1920, 1280) 5 | # 1 front_left_camera (1920, 1280) 6 | # 2 front_right_camera (1920, 1280) 7 | # 3 left_camera (1920, 866) 8 | # 4 right_camera (1920, 866) 9 | 10 | data: 11 | data_root: /data/Tsinghua/wangn/waymo/processed/training # data root for the dataset 12 | dataset: waymo 13 | scene_idx: 0 # which scene to use, [0, 798] for waymo's training set and [0, 849] for nuscenes's train/val sets, inclusive 14 | start_timestep: 0 # which timestep to start from 15 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 16 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device. 17 | pixel_source: # image source and object annotations 18 | type: datasets.waymo.waymo_sourceloader.WaymoPixelSource 19 | cameras: [0] # which cameras to use 20 | downscale_when_loading: [1] # the size of the images to load 21 | downscale: 1 # downscale factor wrt to the downscale_when_loading 22 | undistort: True # whether to undistort the images 23 | test_image_stride: 10 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 24 | load_sky_mask: True # whether to load sky mask 25 | load_dynamic_mask: True # whether to load dynamic mask 26 | load_objects: True # whether to load object bounding boxes 27 | load_smpl: False # whether to load SMPL template for pedestrians 28 | sampler: # error based image sampler 29 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 30 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 31 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 32 | lidar_source: # everything related to "lidar" --- from lidar points 33 | type: datasets.waymo.waymo_sourceloader.WaymoLiDARSource 34 | load_lidar: True # whether to load lidar 35 | only_use_top_lidar: False # whether to only use the top lidar 36 | truncated_max_range: 80 # max range for truncated lidar in a ego-centric coordinate system 37 | truncated_min_range: -2 # min range for truncated lidar in a ego-centric coordinate system. 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/configs/datasets/waymo/3cams.yaml: -------------------------------------------------------------------------------- 1 | # Waymo dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 front_camera (1920, 1280) 5 | # 1 front_left_camera (1920, 1280) 6 | # 2 front_right_camera (1920, 1280) 7 | # 3 left_camera (1920, 866) 8 | # 4 right_camera (1920, 866) 9 | 10 | data: 11 | data_root: /data/Tsinghua/wangn/waymo/processed/training # data root for the dataset 12 | dataset: waymo 13 | scene_idx: 0 # which scene to use, [0, 798] for waymo's training set and [0, 849] for nuscenes's train/val sets, inclusive 14 | start_timestep: 0 # which timestep to start from 15 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 16 | preload_device: cuda # choose from ["cpu", "cuda"], cache the data on this device. 17 | pixel_source: # image source and object annotations 18 | type: datasets.waymo.waymo_sourceloader.WaymoPixelSource 19 | cameras: [0, 1, 2] # which cameras to use 20 | downscale_when_loading: [2, 2, 2] # the size of the images to load 21 | downscale: 1 # downscale factor wrt to the downscale_when_loading 22 | undistort: True # whether to undistort the images 23 | test_image_stride: 10 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 24 | load_sky_mask: True # whether to load sky mask 25 | load_dynamic_mask: True # whether to load dynamic mask 26 | load_objects: True # whether to load object bounding boxes 27 | load_smpl: False # whether to load SMPL template for pedestrians 28 | sampler: # error based image sampler 29 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 30 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 31 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 32 | lidar_source: # everything related to "lidar" --- from lidar points 33 | type: datasets.waymo.waymo_sourceloader.WaymoLiDARSource 34 | load_lidar: True # whether to load lidar 35 | only_use_top_lidar: False # whether to only use the top lidar 36 | truncated_max_range: 80 # max range for truncated lidar in a ego-centric coordinate system 37 | truncated_min_range: -2 # min range for truncated lidar in a ego-centric coordinate system. 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/def_transfer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import os 18 | import os.path as osp 19 | import pickle 20 | 21 | import numpy as np 22 | import torch 23 | from loguru import logger 24 | 25 | from .typing import Tensor 26 | 27 | 28 | def read_deformation_transfer( 29 | deformation_transfer_path: str, 30 | device=None, 31 | use_normal: bool = False, 32 | ) -> Tensor: 33 | ''' Reads a deformation transfer 34 | ''' 35 | if device is None: 36 | device = torch.device('cpu') 37 | assert osp.exists(deformation_transfer_path), ( 38 | 'Deformation transfer path does not exist:' 39 | f' {deformation_transfer_path}') 40 | logger.info( 41 | f'Loading deformation transfer from: {deformation_transfer_path}') 42 | # Read the deformation transfer matrix 43 | with open(deformation_transfer_path, 'rb') as f: 44 | def_transfer_setup = pickle.load(f, encoding='latin1') 45 | if 'mtx' in def_transfer_setup: 46 | def_matrix = def_transfer_setup['mtx'] 47 | if hasattr(def_matrix, 'todense'): 48 | def_matrix = def_matrix.todense() 49 | def_matrix = np.array(def_matrix, dtype=np.float32) 50 | if not use_normal: 51 | num_verts = def_matrix.shape[1] // 2 52 | def_matrix = def_matrix[:, :num_verts] 53 | elif 'matrix' in def_transfer_setup: 54 | def_matrix = def_transfer_setup['matrix'] 55 | else: 56 | valid_keys = ['mtx', 'matrix'] 57 | raise KeyError(f'Deformation transfer setup must contain {valid_keys}') 58 | 59 | def_matrix = torch.tensor(def_matrix, device=device, dtype=torch.float32) 60 | return def_matrix 61 | 62 | 63 | def apply_deformation_transfer( 64 | def_matrix: Tensor, 65 | vertices: Tensor, 66 | faces: Tensor, 67 | use_normals=False 68 | ) -> Tensor: 69 | ''' Applies the deformation transfer on the given meshes 70 | ''' 71 | if use_normals: 72 | raise NotImplementedError 73 | else: 74 | def_vertices = torch.einsum('mn,bni->bmi', [def_matrix, vertices]) 75 | return def_vertices 76 | -------------------------------------------------------------------------------- /project/configs/datasets/waymo/5cams.yaml: -------------------------------------------------------------------------------- 1 | # Waymo dataset configuration 2 | # Processed cameras: 3 | # idx camera original size 4 | # 0 front_camera (1920, 1280) 5 | # 1 front_left_camera (1920, 1280) 6 | # 2 front_right_camera (1920, 1280) 7 | # 3 left_camera (1920, 866) 8 | # 4 right_camera (1920, 866) 9 | 10 | data: 11 | data_root: /data/Tsinghua/wangn/waymo/processed/training # data root for the dataset 12 | dataset: waymo 13 | scene_idx: 0 # which scene to use, [0, 798] for waymo's training set and [0, 849] for nuscenes's train/val sets, inclusive 14 | start_timestep: 0 # which timestep to start from 15 | end_timestep: -1 # which timestep to end at, -1 means the last timestep 16 | preload_device: cpu # choose from ["cpu", "cuda"], cache the data on this device. 17 | pixel_source: # image source and object annotations 18 | type: datasets.waymo.waymo_sourceloader.WaymoPixelSource 19 | cameras: [0, 1, 2, 3, 4] # which cameras to use 20 | downscale_when_loading: [2, 2, 2, 2, 2] # the size of the images to load 21 | downscale: 1 # downscale factor wrt to the downscale_when_loading 22 | undistort: True # whether to undistort the images 23 | test_image_stride: 10 # use every Nth timestep for the test set. if 0, use all images for training and none for testing 24 | load_sky_mask: True # whether to load sky mask 25 | load_dynamic_mask: True # whether to load dynamic mask 26 | load_objects: True # whether to load object bounding boxes 27 | load_smpl: False # whether to load SMPL template for pedestrians 28 | sampler: # error based image sampler 29 | buffer_downscale: 8 # downscale factor for the buffer wrt load_size 30 | buffer_ratio: 0.5 # the percentage of images sampled according to the error buffer 31 | start_enhance_weight: 3 # give more chance to sample starting frames, which usually have more errors 32 | lidar_source: # everything related to "lidar" --- from lidar points 33 | type: datasets.waymo.waymo_sourceloader.WaymoLiDARSource 34 | load_lidar: True # whether to load lidar 35 | only_use_top_lidar: False # whether to only use the top lidar 36 | truncated_max_range: 80 # max range for truncated lidar in a ego-centric coordinate system 37 | truncated_min_range: -2 # min range for truncated lidar in a ego-centric coordinate system. 38 | # ---- compute aabb from lidar ---- # 39 | # if load_lidar is True, we compute aabb from lidar, otherwise we compute aabb from cameras 40 | # 1) downsample lidar by random sampling to 1/lidar_downsample_factor number of points 41 | # 2) compute aabb from the downsampled lidar points by using the percentile of lidar_percentiles 42 | lidar_downsample_factor: 4 # downsample lidar by this factor to compute percentile 43 | lidar_percentile: 0.02 # percentile to compute aabb from lidar -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/optimizers/optim_factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import sys 18 | 19 | from typing import NewType, List, Dict 20 | 21 | import torch 22 | import torch.optim as optim 23 | from loguru import logger 24 | from torchtrustncg import TrustRegion 25 | 26 | Tensor = NewType('Tensor', torch.Tensor) 27 | 28 | 29 | def build_optimizer(parameters: List[Tensor], 30 | optim_cfg: Dict 31 | ) -> Dict: 32 | ''' Creates the optimizer 33 | ''' 34 | optim_type = optim_cfg.get('type', 'sgd') 35 | logger.info(f'Building: {optim_type.title()}') 36 | 37 | num_params = len(parameters) 38 | parameters = list(filter(lambda x: x.requires_grad, parameters)) 39 | if num_params != len(parameters): 40 | logger.info(f'Some parameters have requires_grad off') 41 | 42 | if optim_type == 'adam': 43 | optimizer = optim.Adam(parameters, **optim_cfg.get('adam', {})) 44 | create_graph = False 45 | elif optim_type == 'lbfgs' or optim_type == 'lbfgsls': 46 | optimizer = optim.LBFGS(parameters, **optim_cfg.get('lbfgs', {})) 47 | create_graph = False 48 | elif optim_type == 'trust_ncg' or optim_type == 'trust-ncg': 49 | optimizer = TrustRegion( 50 | parameters, **optim_cfg.get('trust_ncg', {})) 51 | create_graph = True 52 | elif optim_type == 'rmsprop': 53 | optimizer = optim.RMSprop(parameters, **optim_cfg.get('rmsprop', {})) 54 | create_graph = False 55 | elif optim_type == 'sgd': 56 | optimizer = optim.SGD(parameters, **optim_cfg.get('sgd', {})) 57 | create_graph = False 58 | else: 59 | raise ValueError(f'Optimizer {optim_type} not supported!') 60 | return {'optimizer': optimizer, 'create_graph': create_graph} 61 | 62 | 63 | def build_scheduler(optimizer, sched_type='exp', 64 | lr_lambda=0.1, **kwargs): 65 | if lr_lambda <= 0.0: 66 | return None 67 | 68 | if sched_type == 'exp': 69 | return optim.lr_scheduler.ExponentialLR(optimizer, lr_lambda) 70 | else: 71 | raise ValueError('Unknown learning rate' + 72 | ' scheduler: '.format(sched_type)) 73 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/merge_output.py: -------------------------------------------------------------------------------- 1 | # merges the output of the main transfer_model script 2 | 3 | import torch 4 | from pathlib import Path 5 | import pickle 6 | from scipy.spatial.transform import Rotation as R 7 | 8 | KEYS = [ 9 | "transl", 10 | "global_orient", 11 | "body_pose", 12 | "betas", 13 | "left_hand_pose", 14 | "right_hand_pose", 15 | "jaw_pose", 16 | "leye_pose", 17 | "reye_pose", 18 | "expression", 19 | "vertices", 20 | "joints", 21 | "full_pose", 22 | "v_shaped", 23 | "faces" 24 | ] 25 | 26 | IGNORED_KEYS = [ 27 | "vertices", 28 | "faces", 29 | "v_shaped" 30 | ] 31 | 32 | def aggregate_rotmats(x): 33 | x = torch.cat(x, dim=0).detach().numpy() 34 | s = x.shape[:-2] 35 | x = R.from_matrix(x.reshape(-1, 3, 3)).as_rotvec() 36 | x = x.reshape(s[0], -1) 37 | return x 38 | 39 | aggregate_function = {k: lambda x: torch.cat(x, 0).detach().numpy() for k in KEYS} 40 | aggregate_function["betas"] = lambda x: torch.cat(x, 0).mean(0).detach().numpy() 41 | 42 | for k in ["global_orient", "body_pose", "left_hand_pose", "right_hand_pose", "jaw_pose", "full_pose"]: 43 | aggregate_function[k] = aggregate_rotmats 44 | 45 | def merge(output_dir, gender): 46 | output_dir = Path(output_dir) 47 | assert output_dir.exists() 48 | assert output_dir.is_dir() 49 | 50 | # get list of all pkl files in output_dir with fixed length numeral names 51 | pkl_files = [f for f in output_dir.glob("*.pkl") if f.stem != "merged"] 52 | pkl_files = [f for f in sorted(pkl_files, key=lambda x: int(x.stem))] 53 | assert "merged.pkl" not in [f.name for f in pkl_files] 54 | 55 | merged = {} 56 | # iterate over keys and put all values in lists 57 | keys = set(KEYS) - set(IGNORED_KEYS) 58 | for k in keys: 59 | merged[k] = [] 60 | for pkl_file in pkl_files: 61 | with open(pkl_file, "rb") as f: 62 | data = pickle.load(f) 63 | for k in keys: 64 | if k in data: 65 | merged[k].append(data[k]) 66 | b = torch.cat(merged["betas"], 0) 67 | print("betas:") 68 | for mu, sigma in zip(b.mean(0), b.std(0)): 69 | print(" {:.3f} +/- {:.3f}".format(mu, sigma)) 70 | 71 | # aggregate all values 72 | for k in keys: 73 | merged[k] = aggregate_function[k](merged[k]) 74 | 75 | # add gender 76 | merged["gender"] = gender 77 | 78 | # save merged data to same output_dir 79 | with open(output_dir / "merged.pkl", "wb") as f: 80 | pickle.dump(merged, f) 81 | 82 | if __name__ == '__main__': 83 | import argparse 84 | parser = argparse.ArgumentParser(description='Merge output of transfer_model script') 85 | parser.add_argument('output_dir', type=str, help='output directory of transfer_model script') 86 | parser.add_argument('--gender', type=str, choices=['male', 'female', 'neutral'], help='gender of actor in motion sequence') 87 | args = parser.parse_args() 88 | merge(args.output_dir, args.gender) 89 | -------------------------------------------------------------------------------- /project/utils/chamfer_distance.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import cv2 4 | import numpy as np 5 | from typing import Literal, Tuple, Union 6 | 7 | import torch 8 | import torch.nn.functional as F 9 | 10 | # def chamfer_distance(x: torch.Tensor, y: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: 11 | # """ 12 | # NOTE: Not memory efficient, OOM when x,y are large. 13 | # """ 14 | # assert (x.dim() == y.dim()) and (x.dim() >= 2) and x.shape[-1] == y.shape[-1] 15 | # x_i = x.unsqueeze(-2) # [..., N1, 1, D] 16 | # y_j = x.unsqueeze(-3) # [..., 1, N2, D] 17 | # D_ij = ((x_i - y_j)**2).sum(dim=-1) # [..., N1, N2] 18 | # cham_x = D_ij.min(dim=-1).values 19 | # cham_y = D_ij.min(dim=-2).values 20 | # return cham_x, cham_y 21 | def vis_cd(x, cham_x, name="test.ply"): 22 | import open3d as o3d 23 | import matplotlib.pyplot as plt 24 | points = x[cham_x<5].cpu().numpy() 25 | errors = cham_x[cham_x<5].cpu().numpy() 26 | errors_normalized = (errors - errors.min()) / (errors.max() - errors.min()) 27 | point_cloud = o3d.geometry.PointCloud() 28 | point_cloud.points = o3d.utility.Vector3dVector(points) 29 | colormap = plt.cm.viridis 30 | colors = colormap(errors_normalized)[:, :3] 31 | point_cloud.colors = o3d.utility.Vector3dVector(colors) 32 | o3d.io.write_point_cloud(name, point_cloud) 33 | 34 | def chamfer_distance(x: torch.Tensor, y: torch.Tensor, norm: int = 2) -> Tuple[torch.Tensor, torch.Tensor]: 35 | from pytorch3d.ops.knn import knn_points 36 | if not ((norm == 1) or (norm == 2)): 37 | raise ValueError("Support for 1 or 2 norm.") 38 | 39 | _x = x[None] if len(x.shape) == 2 else x 40 | _y = y[None] if len(y.shape) == 2 else y 41 | 42 | if _y.shape[0] != _x.shape[0] or _y.shape[2] != _x.shape[2]: 43 | raise ValueError("y does not have the correct shape.") 44 | 45 | x_nn = knn_points(_x, _y, norm=norm, K=1) 46 | y_nn = knn_points(_y, _x, norm=norm, K=1) 47 | 48 | cham_x = x_nn.dists[..., 0] # (N, P1) 49 | cham_y = y_nn.dists[..., 0] # (N, P2) 50 | cham_x = cham_x[0] if len(x.shape) == 2 else cham_x 51 | cham_y = cham_y[0] if len(y.shape) == 2 else cham_y 52 | return cham_x, cham_y 53 | 54 | def depth_map_to_point_cloud(depth_map, K, c2w, valid_mask=None): 55 | if valid_mask is None: 56 | valid_mask = torch.ones_like(depth_map, dtype=torch.bool) 57 | v_coords, u_coords = torch.where(valid_mask) 58 | z = depth_map[v_coords, u_coords] 59 | 60 | fx = K[0, 0] 61 | fy = K[1, 1] 62 | cx = K[0, 2] 63 | cy = K[1, 2] 64 | x_cam = (u_coords.float() - cx) * z / fx 65 | y_cam = (v_coords.float() - cy) * z / fy 66 | z_cam = z 67 | points_cam = torch.stack((x_cam, y_cam, z_cam), dim=1) 68 | 69 | ones = torch.ones((points_cam.shape[0], 1), dtype=points_cam.dtype, device=points_cam.device) 70 | points_cam_hom = torch.cat((points_cam, ones), dim=1) 71 | 72 | points_world_hom = (c2w @ points_cam_hom.T).T 73 | points_world = points_world_hom[:, :3] 74 | 75 | return points_world -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/config/body_model_defaults.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from omegaconf import OmegaConf 18 | from loguru import logger 19 | from dataclasses import dataclass 20 | from .utils_cfg import Variable, Pose 21 | 22 | 23 | @dataclass 24 | class PCA: 25 | num_comps: int = 12 26 | flat_hand_mean: bool = False 27 | 28 | 29 | @dataclass 30 | class PoseWithPCA(Pose): 31 | pca: PCA = PCA() 32 | 33 | 34 | @dataclass 35 | class Shape(Variable): 36 | num: int = 10 37 | 38 | 39 | @dataclass 40 | class Expression(Variable): 41 | num: int = 10 42 | 43 | 44 | @dataclass 45 | class SMPL: 46 | betas: Shape = Shape() 47 | global_rot: Pose = Pose() 48 | body_pose: Pose = Pose() 49 | translation: Variable = Variable() 50 | 51 | 52 | @dataclass 53 | class SMPLH(SMPL): 54 | left_hand_pose: PoseWithPCA = PoseWithPCA() 55 | right_hand_pose: PoseWithPCA = PoseWithPCA() 56 | 57 | 58 | @dataclass 59 | class SMPLX(SMPLH): 60 | expression: Expression = Expression() 61 | jaw_pose: Pose = Pose() 62 | leye_pose: Pose = Pose() 63 | reye_pose: Pose = Pose() 64 | 65 | 66 | @dataclass 67 | class MANO: 68 | betas: Shape = Shape() 69 | wrist_pose: Pose = Pose() 70 | hand_pose: PoseWithPCA = PoseWithPCA() 71 | translation: Variable = Variable() 72 | 73 | 74 | @dataclass 75 | class FLAME: 76 | betas: Shape = Shape() 77 | expression: Expression = Expression() 78 | global_rot: Pose = Pose() 79 | neck_pose: Pose = Pose() 80 | jaw_pose: Pose = Pose() 81 | leye_pose: Pose = Pose() 82 | reye_pose: Pose = Pose() 83 | 84 | 85 | @dataclass 86 | class BodyModelConfig: 87 | model_type: str = 'smplx' 88 | use_compressed: bool = True 89 | folder: str = 'models' 90 | gender: str = 'neutral' 91 | extra_joint_path: str = '' 92 | ext: str = 'npz' 93 | 94 | num_expression_coeffs: int = 10 95 | 96 | use_face_contour: bool = True 97 | joint_regressor_path: str = '' 98 | 99 | smpl: SMPL = SMPL() 100 | star: SMPL = SMPL() 101 | smplh: SMPLH = SMPLH() 102 | smplx: SMPLX = SMPLX() 103 | mano: MANO = MANO() 104 | flame: FLAME = FLAME() 105 | 106 | 107 | conf = OmegaConf.structured(BodyModelConfig) 108 | -------------------------------------------------------------------------------- /project/third_party/smplx/smplx/vertex_joint_selector.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: ps-license@tuebingen.mpg.de 16 | 17 | from __future__ import absolute_import 18 | from __future__ import print_function 19 | from __future__ import division 20 | 21 | import numpy as np 22 | 23 | import torch 24 | import torch.nn as nn 25 | 26 | from .utils import to_tensor 27 | 28 | 29 | class VertexJointSelector(nn.Module): 30 | 31 | def __init__(self, vertex_ids=None, 32 | use_hands=True, 33 | use_feet_keypoints=True, **kwargs): 34 | super(VertexJointSelector, self).__init__() 35 | 36 | extra_joints_idxs = [] 37 | 38 | face_keyp_idxs = np.array([ 39 | vertex_ids['nose'], 40 | vertex_ids['reye'], 41 | vertex_ids['leye'], 42 | vertex_ids['rear'], 43 | vertex_ids['lear']], dtype=np.int64) 44 | 45 | extra_joints_idxs = np.concatenate([extra_joints_idxs, 46 | face_keyp_idxs]) 47 | 48 | if use_feet_keypoints: 49 | feet_keyp_idxs = np.array([vertex_ids['LBigToe'], 50 | vertex_ids['LSmallToe'], 51 | vertex_ids['LHeel'], 52 | vertex_ids['RBigToe'], 53 | vertex_ids['RSmallToe'], 54 | vertex_ids['RHeel']], dtype=np.int32) 55 | 56 | extra_joints_idxs = np.concatenate( 57 | [extra_joints_idxs, feet_keyp_idxs]) 58 | 59 | if use_hands: 60 | self.tip_names = ['thumb', 'index', 'middle', 'ring', 'pinky'] 61 | 62 | tips_idxs = [] 63 | for hand_id in ['l', 'r']: 64 | for tip_name in self.tip_names: 65 | tips_idxs.append(vertex_ids[hand_id + tip_name]) 66 | 67 | extra_joints_idxs = np.concatenate( 68 | [extra_joints_idxs, tips_idxs]) 69 | 70 | self.register_buffer('extra_joints_idxs', 71 | to_tensor(extra_joints_idxs, dtype=torch.long)) 72 | 73 | def forward(self, vertices, joints): 74 | extra_joints = torch.index_select(vertices, 1, self.extra_joints_idxs.to(torch.long)) #The '.to(torch.long)'. 75 | # added to make the trace work in c++, 76 | # otherwise you get a runtime error in c++: 77 | # 'index_select(): Expected dtype int32 or int64 for index' 78 | joints = torch.cat([joints, extra_joints], dim=1) 79 | 80 | return joints 81 | -------------------------------------------------------------------------------- /project/datasets/waymo/waymo_download.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import subprocess 4 | from concurrent.futures import ThreadPoolExecutor 5 | from typing import List 6 | 7 | 8 | def download_file(filename, target_dir, source): 9 | result = subprocess.run( 10 | [ 11 | "gsutil", 12 | "cp", 13 | "-n", 14 | f"{source}/{filename}.tfrecord", 15 | target_dir, 16 | ], 17 | capture_output=True, # To capture stderr and stdout for detailed error information 18 | text=True, 19 | ) 20 | 21 | # Check the return code of the gsutil command 22 | if result.returncode != 0: 23 | raise Exception( 24 | result.stderr 25 | ) # Raise an exception with the error message from the gsutil command 26 | 27 | 28 | def download_files( 29 | file_names: List[str], 30 | target_dir: str, 31 | source: str = "gs://waymo_open_dataset_scene_flow/train", 32 | ) -> None: 33 | """ 34 | Downloads a list of files from a given source to a target directory using multiple threads. 35 | 36 | Args: 37 | file_names (List[str]): A list of file names to download. 38 | target_dir (str): The target directory to save the downloaded files. 39 | source (str, optional): The source directory to download the files from. Defaults to "gs://waymo_open_dataset_scene_flow/train". 40 | """ 41 | # Get the total number of file_names 42 | total_files = len(file_names) 43 | 44 | # Use ThreadPoolExecutor to manage concurrent downloads 45 | with ThreadPoolExecutor(max_workers=10) as executor: 46 | futures = [ 47 | executor.submit(download_file, filename, target_dir, source) 48 | for filename in file_names 49 | ] 50 | 51 | for counter, future in enumerate(futures, start=1): 52 | # Wait for the download to complete and handle any exceptions 53 | try: 54 | # inspects the result of the future and raises an exception if one occurred during execution 55 | future.result() 56 | print(f"[{counter}/{total_files}] Downloaded successfully!") 57 | except Exception as e: 58 | print(f"[{counter}/{total_files}] Failed to download. Error: {e}") 59 | 60 | 61 | if __name__ == "__main__": 62 | print("note: `gcloud auth login` is required before running this script") 63 | print("Downloading Waymo dataset from Google Cloud Storage...") 64 | parser = argparse.ArgumentParser() 65 | parser.add_argument( 66 | "--target_dir", 67 | type=str, 68 | default="data/waymo/raw", 69 | help="Path to the target directory", 70 | ) 71 | parser.add_argument( 72 | "--scene_ids", type=int, nargs="+", help="scene ids to download" 73 | ) 74 | parser.add_argument( 75 | "--split_file", type=str, default=None, help="split file in data/waymo_splits" 76 | ) 77 | args = parser.parse_args() 78 | os.makedirs(args.target_dir, exist_ok=True) 79 | total_list = open("data/waymo_train_list.txt", "r").readlines() 80 | if args.split_file is None: 81 | file_names = [total_list[i].strip() for i in args.scene_ids] 82 | else: 83 | # parse the split file 84 | split_file = open(args.split_file, "r").readlines()[1:] 85 | scene_ids = [int(line.strip().split(",")[0]) for line in split_file] 86 | file_names = [total_list[i].strip() for i in scene_ids] 87 | download_files(file_names, args.target_dir) 88 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/optimizers/minimize.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | from typing import List, Union, Callable, Optional, Dict 18 | import torch 19 | from loguru import logger 20 | from tqdm import tqdm 21 | 22 | from transfer_model.utils import ( 23 | from_torch, Tensor, Array, rel_change) 24 | 25 | 26 | def minimize( 27 | optimizer: torch.optim, 28 | closure, 29 | params: List[Tensor], 30 | summary_closure: Optional[Callable[[], Dict[str, float]]] = None, 31 | maxiters=100, 32 | ftol=-1.0, 33 | gtol=1e-9, 34 | interactive=True, 35 | summary_steps=10, 36 | **kwargs 37 | ): 38 | ''' Helper function for running an optimization process 39 | Args: 40 | - optimizer: The PyTorch optimizer object 41 | - closure: The function used to calculate the gradients 42 | - params: a list containing the parameters that will be optimized 43 | Keyword arguments: 44 | - maxiters (100): The maximum number of iterations for the 45 | optimizer 46 | - ftol: The tolerance for the relative change in the loss 47 | function. 48 | If it is lower than this value, then the process stops 49 | - gtol: The tolerance for the maximum change in the gradient. 50 | If the maximum absolute values of the all gradient tensors 51 | are less than this, then the process will stop. 52 | ''' 53 | prev_loss = None 54 | for n in tqdm(range(maxiters), desc='Fitting iterations'): 55 | loss = optimizer.step(closure) 56 | 57 | if n > 0 and prev_loss is not None and ftol > 0: 58 | loss_rel_change = rel_change(prev_loss, loss.item()) 59 | 60 | if loss_rel_change <= ftol: 61 | prev_loss = loss.item() 62 | break 63 | 64 | if (all([var.grad.view(-1).abs().max().item() < gtol 65 | for var in params if var.grad is not None]) and gtol > 0): 66 | prev_loss = loss.item() 67 | break 68 | 69 | if interactive and n % summary_steps == 0: 70 | logger.info(f'[{n:05d}] Loss: {loss.item():.4f}') 71 | if summary_closure is not None: 72 | summaries = summary_closure() 73 | for key, val in summaries.items(): 74 | logger.info(f'[{n:05d}] {key}: {val:.4f}') 75 | 76 | prev_loss = loss.item() 77 | 78 | # Save the final step 79 | if interactive: 80 | logger.info(f'[{n + 1:05d}] Loss: {loss.item():.4f}') 81 | if summary_closure is not None: 82 | summaries = summary_closure() 83 | for key, val in summaries.items(): 84 | logger.info(f'[{n + 1:05d}] {key}: {val:.4f}') 85 | 86 | return prev_loss 87 | -------------------------------------------------------------------------------- /project/datasets/base/utils.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple, Union 2 | import numpy as np 3 | import imageio 4 | import numbers 5 | 6 | import torch 7 | import skimage 8 | from skimage.transform import resize as cpu_resize 9 | from torchvision.transforms.functional import resize as gpu_resize 10 | 11 | def load_rgb(path: str, downscale: numbers.Number = 1) -> np.ndarray: 12 | """ Load image 13 | 14 | Args: 15 | path (str): Given image file path 16 | downscale (numbers.Number, optional): Optional downscale ratio. Defaults to 1. 17 | 18 | Returns: 19 | np.ndarray: [H, W, 3], in range [0,1] 20 | """ 21 | img = imageio.imread(path) 22 | img = skimage.img_as_float32(img) 23 | if downscale != 1: 24 | H, W, _ = img.shape 25 | img = cpu_resize(img, (int(H // downscale), int(W // downscale)), anti_aliasing=False) 26 | # [H, W, 3] 27 | return img 28 | 29 | def img_to_torch_and_downscale( 30 | x: Union[np.ndarray, torch.Tensor], hw: Tuple[int, int], 31 | use_cpu_downscale=False, antialias=False, 32 | dtype=None, device=None): 33 | """ Check, convert and apply downscale to input image `x` 34 | 35 | Args: 36 | x (Union[np.ndarray, torch.Tensor]): [H, W, (...)] Input image 37 | downscale (float, optional): Downscaling ratio. Defaults to 1. 38 | use_cpu_downscale (bool, optional): Whether use CPU downscaling algo (T), or use GPU (F). Defaults to False. 39 | antialias (bool, optional): Whether use anti-aliasing. Defaults to False. 40 | dtype (torch.dtype, optional): Output torch.dtype. Defaults to None. 41 | device (torch.device, optional): Output torch.device. Defaults to None. 42 | 43 | Returns: 44 | torch.Tensor: [new_H, new_W, (...)] Converted and downscaled torch.Tensor image 45 | """ 46 | H_, W_ = hw 47 | if use_cpu_downscale: 48 | x_np = x if isinstance(x, np.ndarray) else x.data.cpu().numpy() 49 | x = torch.tensor(cpu_resize(x_np, (H_, W_), anti_aliasing=antialias), 50 | dtype=dtype, device=device) 51 | else: 52 | x = check_to_torch(x, dtype=dtype, device=device) 53 | x = x.cuda() if not x.is_cuda else x 54 | if x.dim() == 2: 55 | x = gpu_resize(x.unsqueeze(0), (H_, W_), antialias=antialias).squeeze(0) 56 | else: 57 | x = gpu_resize(x.movedim(-1, 0), (H_, W_), antialias=antialias).movedim(0, -1) 58 | assert [H_, W_] == [*x.shape[:2]] 59 | return check_to_torch(x, dtype=dtype, device=device) 60 | 61 | def check_to_torch( 62 | x: Union[np.ndarray, torch.Tensor, List, Tuple], 63 | ref: torch.Tensor=None, dtype: torch.dtype=None, device: torch.device=None) -> torch.Tensor: 64 | """ Check and convert input `x` to torch.Tensor 65 | 66 | Args: 67 | x (Union[np.ndarray, torch.Tensor, List, Tuple]): Input 68 | ref (torch.Tensor, optional): Reference tensor for dtype and device. Defaults to None. 69 | dtype (torch.dtype, optional): Target torch.dtype. Defaults to None. 70 | device (torch.device, optional): Target torch.device. Defaults to None. 71 | 72 | Returns: 73 | torch.Tensor: Converted torch.Tensor 74 | """ 75 | if ref is not None: 76 | if dtype is None: 77 | dtype = ref.dtype 78 | if device is None: 79 | device = ref.device 80 | if x is None: 81 | return x 82 | elif isinstance(x, torch.Tensor): 83 | return x.to(dtype=dtype or x.dtype, device=device or x.device) 84 | else: 85 | return torch.tensor(x, dtype=dtype, device=device) -------------------------------------------------------------------------------- /project/third_party/smplx/examples/vis_flame_vertices.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: ps-license@tuebingen.mpg.de 16 | 17 | import os.path as osp 18 | import argparse 19 | import pickle 20 | 21 | import numpy as np 22 | import torch 23 | import open3d as o3d 24 | 25 | import smplx 26 | 27 | 28 | def main(model_folder, corr_fname, ext='npz', 29 | head_color=(0.3, 0.3, 0.6), 30 | gender='neutral'): 31 | 32 | head_idxs = np.load(corr_fname) 33 | 34 | model = smplx.create(model_folder, model_type='smplx', 35 | gender=gender, 36 | ext=ext) 37 | betas = torch.zeros([1, 10], dtype=torch.float32) 38 | expression = torch.zeros([1, 10], dtype=torch.float32) 39 | 40 | output = model(betas=betas, expression=expression, 41 | return_verts=True) 42 | vertices = output.vertices.detach().cpu().numpy().squeeze() 43 | joints = output.joints.detach().cpu().numpy().squeeze() 44 | 45 | print('Vertices shape =', vertices.shape) 46 | print('Joints shape =', joints.shape) 47 | 48 | mesh = o3d.geometry.TriangleMesh() 49 | mesh.vertices = o3d.utility.Vector3dVector(vertices) 50 | mesh.triangles = o3d.utility.Vector3iVector(model.faces) 51 | mesh.compute_vertex_normals() 52 | 53 | colors = np.ones_like(vertices) * [0.3, 0.3, 0.3] 54 | colors[head_idxs] = head_color 55 | 56 | mesh.vertex_colors = o3d.utility.Vector3dVector(colors) 57 | 58 | o3d.visualization.draw_geometries([mesh]) 59 | 60 | 61 | if __name__ == '__main__': 62 | parser = argparse.ArgumentParser(description='SMPL-X Demo') 63 | 64 | parser.add_argument('--model-folder', required=True, type=str, 65 | help='The path to the model folder') 66 | parser.add_argument('--corr-fname', required=True, type=str, 67 | dest='corr_fname', 68 | help='Filename with the head correspondences') 69 | parser.add_argument('--gender', type=str, default='neutral', 70 | help='The gender of the model') 71 | parser.add_argument('--ext', type=str, default='npz', 72 | help='Which extension to use for loading') 73 | parser.add_argument('--head', default='right', 74 | choices=['right', 'left'], 75 | type=str, help='Which head to plot') 76 | parser.add_argument('--head-color', type=float, nargs=3, dest='head_color', 77 | default=(0.3, 0.3, 0.6), 78 | help='Color for the head vertices') 79 | 80 | args = parser.parse_args() 81 | 82 | model_folder = osp.expanduser(osp.expandvars(args.model_folder)) 83 | corr_fname = args.corr_fname 84 | gender = args.gender 85 | ext = args.ext 86 | head = args.head 87 | head_color = args.head_color 88 | 89 | main(model_folder, corr_fname, ext=ext, 90 | head_color=head_color, 91 | gender=gender 92 | ) 93 | -------------------------------------------------------------------------------- /project/datasets/base/scene_dataset.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | from enum import IntEnum 4 | from typing import List, Callable 5 | 6 | import torch 7 | from omegaconf import OmegaConf 8 | from torch import Tensor 9 | 10 | from .lidar_source import SceneLidarSource 11 | from .pixel_source import ScenePixelSource 12 | from .split_wrapper import SplitWrapper 13 | 14 | logger = logging.getLogger() 15 | 16 | class ModelType(IntEnum): 17 | RigidNodes = 0 18 | SMPLNodes = 1 19 | DeformableNodes = 2 20 | 21 | class SceneDataset(abc.ABC): 22 | """ 23 | Base class for scene dataset. 24 | """ 25 | 26 | data_cfg: OmegaConf = None 27 | pixel_source: ScenePixelSource = None 28 | lidar_source: SceneLidarSource = None 29 | # training and testing indices are indices into the full dataset 30 | # train_indices are img indices, so the length is num_cams * num_timesteps 31 | train_indices: List[int] = None 32 | test_indices: List[int] = None 33 | # train_timesteps are timesteps, so the length is num_timesteps (len(unique_timesteps)) 34 | train_timesteps: Tensor = None 35 | test_timesteps: Tensor = None 36 | 37 | # dataset layout for multi-camera visualization 38 | layout: Callable = None 39 | 40 | # dataset wrappers 41 | # full: includes all data 42 | full_image_set: SplitWrapper = None 43 | full_lidar_set: SplitWrapper = None 44 | # train: includes only training data 45 | train_image_set: SplitWrapper = None 46 | train_lidar_set: SplitWrapper = None 47 | # test: includes only testing data 48 | test_image_set: SplitWrapper = None 49 | test_lidar_set: SplitWrapper = None 50 | 51 | def __init__( 52 | self, 53 | data_config: OmegaConf, 54 | ): 55 | super().__init__() 56 | self.data_cfg = data_config 57 | 58 | @abc.abstractmethod 59 | def build_data_source(self): 60 | """ 61 | Create the data source for the dataset. 62 | """ 63 | raise NotImplementedError 64 | 65 | @abc.abstractmethod 66 | def build_split_wrapper(self): 67 | """ 68 | Makes each data source as a Pytorch Dataset. 69 | """ 70 | raise NotImplementedError 71 | 72 | @abc.abstractmethod 73 | def split_train_test(self): 74 | raise NotImplementedError 75 | 76 | def get_aabb(self) -> Tensor: 77 | if self.lidar_source is not None: 78 | aabb = self.lidar_source.get_aabb() 79 | else: 80 | aabb = self.pixel_source.get_aabb() 81 | return aabb 82 | 83 | def get_init_pcd(self) -> Tensor: 84 | raise NotImplementedError 85 | 86 | @property 87 | def num_cams(self) -> int: 88 | return self.pixel_source.num_cams 89 | 90 | @property 91 | def scene_idx(self) -> int: 92 | return self.data_cfg.scene_idx 93 | 94 | @property 95 | def num_img_timesteps(self) -> int: 96 | return self.pixel_source.num_timesteps 97 | 98 | @property 99 | def num_lidar_timesteps(self) -> int: 100 | if self.lidar_source is None: 101 | logger.warning("No lidar source, returning num_img_timesteps") 102 | return self.num_img_timesteps 103 | return self.lidar_source.num_timesteps 104 | 105 | @property 106 | def num_train_timesteps(self) -> int: 107 | return len(self.train_timesteps) 108 | 109 | @property 110 | def num_test_timesteps(self) -> int: 111 | return len(self.test_timesteps) 112 | 113 | @property 114 | def unique_normalized_training_timestamps(self) -> Tensor: 115 | return self.pixel_source.unique_normalized_timestamps[self.train_timesteps] 116 | 117 | @property 118 | def device(self): 119 | return self.data_cfg.preload_device 120 | -------------------------------------------------------------------------------- /project/third_party/smplx/examples/vis_mano_vertices.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: ps-license@tuebingen.mpg.de 16 | 17 | import os.path as osp 18 | import argparse 19 | import pickle 20 | 21 | import numpy as np 22 | import torch 23 | import open3d as o3d 24 | 25 | import smplx 26 | 27 | 28 | def main(model_folder, corr_fname, ext='npz', 29 | hand_color=(0.3, 0.3, 0.6), 30 | gender='neutral', hand='right'): 31 | 32 | with open(corr_fname, 'rb') as f: 33 | idxs_data = pickle.load(f) 34 | if hand == 'both': 35 | hand_idxs = np.concatenate( 36 | [idxs_data['left_hand'], idxs_data['right_hand']] 37 | ) 38 | else: 39 | hand_idxs = idxs_data[f'{hand}_hand'] 40 | 41 | model = smplx.create(model_folder, model_type='smplx', 42 | gender=gender, 43 | ext=ext) 44 | betas = torch.zeros([1, 10], dtype=torch.float32) 45 | expression = torch.zeros([1, 10], dtype=torch.float32) 46 | 47 | output = model(betas=betas, expression=expression, 48 | return_verts=True) 49 | vertices = output.vertices.detach().cpu().numpy().squeeze() 50 | joints = output.joints.detach().cpu().numpy().squeeze() 51 | 52 | print('Vertices shape =', vertices.shape) 53 | print('Joints shape =', joints.shape) 54 | 55 | mesh = o3d.geometry.TriangleMesh() 56 | mesh.vertices = o3d.utility.Vector3dVector(vertices) 57 | mesh.triangles = o3d.utility.Vector3iVector(model.faces) 58 | mesh.compute_vertex_normals() 59 | 60 | colors = np.ones_like(vertices) * [0.3, 0.3, 0.3] 61 | colors[hand_idxs] = hand_color 62 | 63 | mesh.vertex_colors = o3d.utility.Vector3dVector(colors) 64 | 65 | o3d.visualization.draw_geometries([mesh]) 66 | 67 | 68 | if __name__ == '__main__': 69 | parser = argparse.ArgumentParser(description='SMPL-X Demo') 70 | 71 | parser.add_argument('--model-folder', required=True, type=str, 72 | help='The path to the model folder') 73 | parser.add_argument('--corr-fname', required=True, type=str, 74 | dest='corr_fname', 75 | help='Filename with the hand correspondences') 76 | parser.add_argument('--gender', type=str, default='neutral', 77 | help='The gender of the model') 78 | parser.add_argument('--ext', type=str, default='npz', 79 | help='Which extension to use for loading') 80 | parser.add_argument('--hand', default='right', 81 | choices=['right', 'left', 'both'], 82 | type=str, help='Which hand to plot') 83 | parser.add_argument('--hand-color', type=float, nargs=3, dest='hand_color', 84 | default=(0.3, 0.3, 0.6), 85 | help='Color for the hand vertices') 86 | 87 | args = parser.parse_args() 88 | 89 | model_folder = osp.expanduser(osp.expandvars(args.model_folder)) 90 | corr_fname = args.corr_fname 91 | gender = args.gender 92 | ext = args.ext 93 | hand = args.hand 94 | hand_color = args.hand_color 95 | 96 | main(model_folder, corr_fname, ext=ext, 97 | hand_color=hand_color, 98 | gender=gender, hand=hand 99 | ) 100 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import os 18 | import os.path as osp 19 | import sys 20 | import pickle 21 | 22 | import numpy as np 23 | import open3d as o3d 24 | import torch 25 | from loguru import logger 26 | from tqdm import tqdm 27 | 28 | from smplx import build_layer 29 | 30 | from .config import parse_args 31 | from .data import build_dataloader 32 | from .transfer_model import run_fitting 33 | from .utils import read_deformation_transfer, np_mesh_to_o3d 34 | 35 | 36 | def main() -> None: 37 | exp_cfg = parse_args() 38 | 39 | if torch.cuda.is_available() and exp_cfg["use_cuda"]: 40 | device = torch.device('cuda') 41 | else: 42 | device = torch.device('cpu') 43 | if exp_cfg["use_cuda"]: 44 | if input("use_cuda=True and GPU is not available, using CPU instead," 45 | " would you like to continue? (y/n)") != "y": 46 | sys.exit(3) 47 | 48 | logger.remove() 49 | logger.add( 50 | lambda x: tqdm.write(x, end=''), level=exp_cfg.logger_level.upper(), 51 | colorize=True) 52 | 53 | output_folder = osp.expanduser(osp.expandvars(exp_cfg.output_folder)) 54 | logger.info(f'Saving output to: {output_folder}') 55 | os.makedirs(output_folder, exist_ok=True) 56 | 57 | model_path = exp_cfg.body_model.folder 58 | body_model = build_layer(model_path, **exp_cfg.body_model) 59 | logger.info(body_model) 60 | body_model = body_model.to(device=device) 61 | 62 | deformation_transfer_path = exp_cfg.get('deformation_transfer_path', '') 63 | def_matrix = read_deformation_transfer( 64 | deformation_transfer_path, device=device) 65 | 66 | # Read mask for valid vertex ids 67 | mask_ids_fname = osp.expandvars(exp_cfg.mask_ids_fname) 68 | mask_ids = None 69 | if osp.exists(mask_ids_fname): 70 | logger.info(f'Loading mask ids from: {mask_ids_fname}') 71 | mask_ids = np.load(mask_ids_fname) 72 | mask_ids = torch.from_numpy(mask_ids).to(device=device) 73 | else: 74 | logger.warning(f'Mask ids fname not found: {mask_ids_fname}') 75 | 76 | data_obj_dict = build_dataloader(exp_cfg) 77 | 78 | dataloader = data_obj_dict['dataloader'] 79 | 80 | for ii, batch in enumerate(tqdm(dataloader)): 81 | for key in batch: 82 | if torch.is_tensor(batch[key]): 83 | batch[key] = batch[key].to(device=device) 84 | var_dict = run_fitting( 85 | exp_cfg, batch, body_model, def_matrix, mask_ids) 86 | paths = batch['paths'] 87 | 88 | for ii, path in enumerate(paths): 89 | _, fname = osp.split(path) 90 | 91 | output_path = osp.join( 92 | output_folder, f'{osp.splitext(fname)[0]}.pkl') 93 | with open(output_path, 'wb') as f: 94 | pickle.dump(var_dict, f) 95 | 96 | output_path = osp.join( 97 | output_folder, f'{osp.splitext(fname)[0]}.obj') 98 | mesh = np_mesh_to_o3d( 99 | var_dict['vertices'][ii], var_dict['faces']) 100 | o3d.io.write_triangle_mesh(output_path, mesh) 101 | 102 | 103 | if __name__ == '__main__': 104 | main() 105 | -------------------------------------------------------------------------------- /project/utils/geometry.py: -------------------------------------------------------------------------------- 1 | # Utility functions for geometric transformations and projections. 2 | import numpy as np 3 | import torch 4 | from torch import Tensor 5 | import torch.nn.functional as F 6 | 7 | def transform_points(points, transform_matrix): 8 | """ 9 | Apply a 4x4 transformation matrix to 3D points. 10 | 11 | Args: 12 | points: (N, 3) tensor of 3D points 13 | transform_matrix: (4, 4) transformation matrix 14 | 15 | Returns: 16 | (N, 3) tensor of transformed 3D points 17 | """ 18 | ones = torch.ones((points.shape[0], 1), dtype=points.dtype, device=points.device) 19 | homo_points = torch.cat([points, ones], dim=1) # N x 4 20 | transformed_points = torch.matmul(homo_points, transform_matrix.T) 21 | return transformed_points[:, :3] 22 | 23 | def get_corners(l: float, w: float, h: float): 24 | """ 25 | Get 8 corners of a 3D bounding box centered at origin. 26 | 27 | Args: 28 | l, w, h: length, width, height of the box 29 | 30 | Returns: 31 | (3, 8) array of corner coordinates 32 | """ 33 | return np.array([ 34 | [-l/2, -l/2, l/2, l/2, -l/2, -l/2, l/2, l/2], 35 | [w/2, -w/2, -w/2, w/2, w/2, -w/2, -w/2, w/2], 36 | [h/2, h/2, h/2, h/2, -h/2, -h/2, -h/2, -h/2], 37 | ]) 38 | 39 | def project_camera_points_to_image(points_cam, cam_intrinsics): 40 | """ 41 | Project 3D points from camera space to 2D image space. 42 | 43 | Args: 44 | points_cam (np.ndarray): Shape (N, 3), points in camera space. 45 | cam_intrinsics (np.ndarray): Shape (3, 3), intrinsic matrix of the camera. 46 | 47 | Returns: 48 | tuple: (projected_points, depths) 49 | - projected_points (np.ndarray): Shape (N, 2), projected 2D points in image space. 50 | - depths (np.ndarray): Shape (N,), depth values of the projected points. 51 | """ 52 | points_img = cam_intrinsics @ points_cam.T 53 | depths = points_img[2, :] 54 | projected_points = (points_img[:2, :] / (depths + 1e-6)).T 55 | 56 | return projected_points, depths 57 | 58 | def cube_root(x): 59 | return torch.sign(x) * torch.abs(x) ** (1. / 3) 60 | 61 | def spherical_to_cartesian(r, theta, phi): 62 | x = r * torch.sin(theta) * torch.cos(phi) 63 | y = r * torch.sin(theta) * torch.sin(phi) 64 | z = r * torch.cos(theta) 65 | return torch.stack([x, y, z], dim=1) 66 | 67 | def uniform_sample_sphere(num_samples, device, inverse=False): 68 | """ 69 | refer to https://stackoverflow.com/questions/5408276/sampling-uniformly-distributed-random-points-inside-a-spherical-volume 70 | sample points uniformly inside a sphere 71 | """ 72 | if not inverse: 73 | dist = torch.rand((num_samples,)).to(device) 74 | dist = cube_root(dist) 75 | else: 76 | dist = torch.rand((num_samples,)).to(device) 77 | dist = 1 / dist.clamp_min(0.02) 78 | thetas = torch.arccos(2 * torch.rand((num_samples,)) - 1).to(device) 79 | phis = 2 * torch.pi * torch.rand((num_samples,)).to(device) 80 | pts = spherical_to_cartesian(dist, thetas, phis) 81 | return pts 82 | 83 | def rotation_6d_to_matrix(d6: Tensor) -> Tensor: 84 | """ 85 | Converts 6D rotation representation by Zhou et al. [1] to rotation matrix 86 | using Gram--Schmidt orthogonalization per Section B of [1]. Adapted from pytorch3d. 87 | Args: 88 | d6: 6D rotation representation, of size (*, 6) 89 | 90 | Returns: 91 | batch of rotation matrices of size (*, 3, 3) 92 | 93 | [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. 94 | On the Continuity of Rotation Representations in Neural Networks. 95 | IEEE Conference on Computer Vision and Pattern Recognition, 2019. 96 | Retrieved from http://arxiv.org/abs/1812.07035 97 | """ 98 | 99 | a1, a2 = d6[..., :3], d6[..., 3:] 100 | b1 = F.normalize(a1, dim=-1) 101 | b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1 102 | b2 = F.normalize(b2, dim=-1) 103 | b3 = torch.cross(b1, b2, dim=-1) 104 | return torch.stack((b1, b2, b3), dim=-2) -------------------------------------------------------------------------------- /project/third_party/smplx/smplx/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2019 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: ps-license@tuebingen.mpg.de 16 | 17 | from typing import NewType, Union, Optional 18 | from dataclasses import dataclass, asdict, fields 19 | import numpy as np 20 | import torch 21 | 22 | Tensor = NewType('Tensor', torch.Tensor) 23 | Array = NewType('Array', np.ndarray) 24 | 25 | 26 | @dataclass 27 | class ModelOutput: 28 | vertices: Optional[Tensor] = None 29 | joints: Optional[Tensor] = None 30 | full_pose: Optional[Tensor] = None 31 | global_orient: Optional[Tensor] = None 32 | transl: Optional[Tensor] = None 33 | v_posed: Optional[Tensor] = None 34 | 35 | def __getitem__(self, key): 36 | return getattr(self, key) 37 | 38 | def get(self, key, default=None): 39 | return getattr(self, key, default) 40 | 41 | def __iter__(self): 42 | return self.keys() 43 | 44 | def keys(self): 45 | keys = [t.name for t in fields(self)] 46 | return iter(keys) 47 | 48 | def values(self): 49 | values = [getattr(self, t.name) for t in fields(self)] 50 | return iter(values) 51 | 52 | def items(self): 53 | data = [(t.name, getattr(self, t.name)) for t in fields(self)] 54 | return iter(data) 55 | 56 | 57 | @dataclass 58 | class SMPLOutput(ModelOutput): 59 | betas: Optional[Tensor] = None 60 | body_pose: Optional[Tensor] = None 61 | T: Optional[Tensor] = None 62 | A: Optional[Tensor] = None 63 | J: Optional[Tensor] = None 64 | 65 | 66 | @dataclass 67 | class SMPLHOutput(SMPLOutput): 68 | left_hand_pose: Optional[Tensor] = None 69 | right_hand_pose: Optional[Tensor] = None 70 | transl: Optional[Tensor] = None 71 | 72 | 73 | @dataclass 74 | class SMPLXOutput(SMPLHOutput): 75 | expression: Optional[Tensor] = None 76 | jaw_pose: Optional[Tensor] = None 77 | 78 | 79 | @dataclass 80 | class MANOOutput(ModelOutput): 81 | betas: Optional[Tensor] = None 82 | hand_pose: Optional[Tensor] = None 83 | 84 | 85 | @dataclass 86 | class FLAMEOutput(ModelOutput): 87 | betas: Optional[Tensor] = None 88 | expression: Optional[Tensor] = None 89 | jaw_pose: Optional[Tensor] = None 90 | neck_pose: Optional[Tensor] = None 91 | 92 | 93 | def find_joint_kin_chain(joint_id, kinematic_tree): 94 | kin_chain = [] 95 | curr_idx = joint_id 96 | while curr_idx != -1: 97 | kin_chain.append(curr_idx) 98 | curr_idx = kinematic_tree[curr_idx] 99 | return kin_chain 100 | 101 | 102 | def to_tensor( 103 | array: Union[Array, Tensor], dtype=torch.float32 104 | ) -> Tensor: 105 | if torch.is_tensor(array): 106 | return array 107 | else: 108 | return torch.tensor(array, dtype=dtype) 109 | 110 | 111 | class Struct(object): 112 | def __init__(self, **kwargs): 113 | for key, val in kwargs.items(): 114 | setattr(self, key, val) 115 | 116 | 117 | def to_np(array, dtype=np.float32): 118 | if 'scipy.sparse' in str(type(array)): 119 | array = array.todense() 120 | return np.array(array, dtype=dtype) 121 | 122 | 123 | def rot_mat_to_euler(rot_mats): 124 | # Calculates rotation matrix to euler angles 125 | # Careful for extreme cases of eular angles like [0.0, pi, 0.0] 126 | 127 | sy = torch.sqrt(rot_mats[:, 0, 0] * rot_mats[:, 0, 0] + 128 | rot_mats[:, 1, 0] * rot_mats[:, 1, 0]) 129 | return torch.atan2(-rot_mats[:, 2, 0], sy) 130 | -------------------------------------------------------------------------------- /project/generate_lidar/depth2lidar.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | import tqdm 6 | 7 | 8 | def pto_rec_map(velo_points, H=64, W=512, D=800): 9 | # depth, width, height 10 | valid_inds = (velo_points[:, 0] < 80) & \ 11 | (velo_points[:, 0] >= 0) & \ 12 | (velo_points[:, 1] < 50) & \ 13 | (velo_points[:, 1] >= -50) & \ 14 | (velo_points[:, 2] < 1) & \ 15 | (velo_points[:, 2] >= -2.5) 16 | velo_points = velo_points[valid_inds] 17 | 18 | x, y, z, i = velo_points[:, 0], velo_points[:, 1], velo_points[:, 2], velo_points[:, 3] 19 | x_grid = (x * D / 80.).astype(int) 20 | x_grid[x_grid < 0] = 0 21 | x_grid[x_grid >= D] = D - 1 22 | 23 | y_grid = ((y + 50) * W / 100.).astype(int) 24 | y_grid[y_grid < 0] = 0 25 | y_grid[y_grid >= W] = W - 1 26 | 27 | z_grid = ((z + 2.5) * H / 3.5).astype(int) 28 | z_grid[z_grid < 0] = 0 29 | z_grid[z_grid >= H] = H - 1 30 | 31 | depth_map = - np.ones((D, W, H, 4)) 32 | depth_map[x_grid, y_grid, z_grid, 0] = x 33 | depth_map[x_grid, y_grid, z_grid, 1] = y 34 | depth_map[x_grid, y_grid, z_grid, 2] = z 35 | depth_map[x_grid, y_grid, z_grid, 3] = i 36 | depth_map = depth_map.reshape((-1, 4)) 37 | depth_map = depth_map[depth_map[:, 0] != -1.0] 38 | return depth_map 39 | 40 | 41 | def pto_ang_map(velo_points, H=64, W=512, slice=1): 42 | """ 43 | :param H: the row num of depth map, could be 64(default), 32, 16 44 | :param W: the col num of depth map 45 | :param slice: output every slice lines 46 | """ 47 | 48 | dtheta = np.radians(0.4 * 64.0 / H) 49 | dphi = np.radians(90.0 / W) 50 | 51 | x, y, z, i = velo_points[:, 0], velo_points[:, 1], velo_points[:, 2], velo_points[:, 3] 52 | 53 | d = np.sqrt(x ** 2 + y ** 2 + z ** 2) 54 | r = np.sqrt(x ** 2 + y ** 2) 55 | d[d == 0] = 0.000001 56 | r[r == 0] = 0.000001 57 | phi = np.radians(45.) - np.arcsin(y / r) 58 | phi_ = (phi / dphi).astype(int) 59 | phi_[phi_ < 0] = 0 60 | phi_[phi_ >= W] = W - 1 61 | 62 | theta = np.radians(2.) - np.arcsin(z / d) 63 | theta_ = (theta / dtheta).astype(int) 64 | theta_[theta_ < 0] = 0 65 | theta_[theta_ >= H] = H - 1 66 | 67 | depth_map = - np.ones((H, W, 4)) 68 | depth_map[theta_, phi_, 0] = x 69 | depth_map[theta_, phi_, 1] = y 70 | depth_map[theta_, phi_, 2] = z 71 | depth_map[theta_, phi_, 3] = i 72 | depth_map = depth_map[0::slice, :, :] 73 | depth_map = depth_map.reshape((-1, 4)) 74 | depth_map = depth_map[depth_map[:, 0] != -1.0] 75 | return depth_map 76 | 77 | 78 | def gen_sparse_points(pl_data_path, args): 79 | pc_velo = np.fromfile(pl_data_path, dtype=np.float32).reshape((-1, 4)) 80 | 81 | # depth, width, height 82 | valid_inds = (pc_velo[:, 0] < 120) & \ 83 | (pc_velo[:, 0] >= 0) & \ 84 | (pc_velo[:, 1] < 50) & \ 85 | (pc_velo[:, 1] >= -50) & \ 86 | (pc_velo[:, 2] < 1.5) & \ 87 | (pc_velo[:, 2] >= -2.5) 88 | pc_velo = pc_velo[valid_inds] 89 | 90 | return pto_ang_map(pc_velo, H=args.H, W=args.W, slice=args.slice) 91 | 92 | 93 | def gen_sparse_points_all(args): 94 | outputfolder = args.sparse_pl_path 95 | os.makedirs(outputfolder, exist_ok=True) 96 | data_idx_list = sorted([x.strip() for x in os.listdir(args.pl_path) if x[-3:] == 'bin']) 97 | 98 | for data_idx in tqdm.tqdm(data_idx_list): 99 | sparse_points = gen_sparse_points(os.path.join(args.pl_path, data_idx), args) 100 | sparse_points = sparse_points.astype(np.float32) 101 | sparse_points.tofile(f'{outputfolder}/{data_idx}') 102 | 103 | 104 | if __name__ == '__main__': 105 | parser = argparse.ArgumentParser("Generate sparse pseudo-LiDAR points") 106 | parser.add_argument('--pl_path', default='/scratch/datasets', help='pseudo-lidar path') 107 | parser.add_argument('--sparse_pl_path', default='/scratch/datasets', help='sparsed pseudo lidar path') 108 | parser.add_argument('--slice', default=1, type=int) 109 | parser.add_argument('--H', default=64, type=int) 110 | parser.add_argument('--W', default=512, type=int) 111 | parser.add_argument('--D', default=700, type=int) 112 | args = parser.parse_args() 113 | 114 | gen_sparse_points_all(args) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | google-drive* 2 | rclone* 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | scripts/ 6 | *.py[cod] 7 | *$py.class 8 | smpl_models/ 9 | *.zip 10 | *.html 11 | # self added 12 | fig 13 | output 14 | data/waymo 15 | data/pandaset 16 | data/argoverse 17 | data/nuscenes 18 | data/kitti 19 | data/nuplan 20 | vscode_debugs 21 | debug/ 22 | .vscode 23 | log*/ 24 | wandb/ 25 | nr3d_lib/ 26 | # smplx/ 27 | ckpts/ 28 | run_scripts/ 29 | downloads/ 30 | method_compare/ 31 | simulated_results/ 32 | sky_masks/ 33 | fine_dynamic_masks_example/ 34 | # *.sh 35 | pandaset-devkit/ 36 | 37 | # C extensions 38 | *.so 39 | 40 | # Distribution / packaging 41 | .Python 42 | build/ 43 | develop-eggs/ 44 | dist/ 45 | downloads/ 46 | eggs/ 47 | .eggs/ 48 | lib/ 49 | lib64/ 50 | parts/ 51 | sdist/ 52 | var/ 53 | wheels/ 54 | share/python-wheels/ 55 | *.egg-info/ 56 | .installed.cfg 57 | *.egg 58 | MANIFEST 59 | 60 | # PyInstaller 61 | # Usually these files are written by a python script from a template 62 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 63 | *.manifest 64 | *.spec 65 | 66 | # Installer logs 67 | pip-log.txt 68 | pip-delete-this-directory.txt 69 | 70 | # Unit test / coverage reports 71 | htmlcov/ 72 | .tox/ 73 | .nox/ 74 | .coverage 75 | .coverage.* 76 | .cache 77 | nosetests.xml 78 | coverage.xml 79 | *.cover 80 | *.py,cover 81 | .hypothesis/ 82 | .pytest_cache/ 83 | cover/ 84 | 85 | # Translations 86 | *.mo 87 | *.pot 88 | 89 | # Django stuff: 90 | *.log 91 | local_settings.py 92 | db.sqlite3 93 | db.sqlite3-journal 94 | 95 | # Flask stuff: 96 | instance/ 97 | .webassets-cache 98 | 99 | # Scrapy stuff: 100 | .scrapy 101 | 102 | # Sphinx documentation 103 | docs/_build/ 104 | 105 | # PyBuilder 106 | .pybuilder/ 107 | target/ 108 | 109 | # Jupyter Notebook 110 | .ipynb_checkpoints 111 | 112 | # IPython 113 | profile_default/ 114 | ipython_config.py 115 | 116 | # pyenv 117 | # For a library or package, you might want to ignore these files since the code is 118 | # intended to run in multiple environments; otherwise, check them in: 119 | # .python-version 120 | 121 | # pipenv 122 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 123 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 124 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 125 | # install all needed dependencies. 126 | #Pipfile.lock 127 | 128 | # poetry 129 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 130 | # This is especially recommended for binary packages to ensure reproducibility, and is more 131 | # commonly ignored for libraries. 132 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 133 | #poetry.lock 134 | 135 | # pdm 136 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 137 | #pdm.lock 138 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 139 | # in version control. 140 | # https://pdm.fming.dev/#use-with-ide 141 | .pdm.toml 142 | 143 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 144 | __pypackages__/ 145 | 146 | # Celery stuff 147 | celerybeat-schedule 148 | celerybeat.pid 149 | 150 | # SageMath parsed files 151 | *.sage.py 152 | 153 | # Environments 154 | .env 155 | .venv 156 | env/ 157 | venv/ 158 | ENV/ 159 | env.bak/ 160 | venv.bak/ 161 | 162 | # Spyder project settings 163 | .spyderproject 164 | .spyproject 165 | 166 | # Rope project settings 167 | .ropeproject 168 | 169 | # mkdocs documentation 170 | /site 171 | 172 | # mypy 173 | .mypy_cache/ 174 | .dmypy.json 175 | dmypy.json 176 | 177 | # Pyre type checker 178 | .pyre/ 179 | 180 | # pytype static type analyzer 181 | .pytype/ 182 | 183 | # Cython debug symbols 184 | cython_debug/ 185 | 186 | # PyCharm 187 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 188 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 189 | # and can be added to the global gitignore or merged into this file. For a more nuclear 190 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 191 | #.idea/ 192 | local_scriptsSegFormer/ 193 | -------------------------------------------------------------------------------- /project/utils/misc.py: -------------------------------------------------------------------------------- 1 | # Miscellaneous utility functions for exporting point clouds. 2 | import importlib 3 | import logging 4 | import os 5 | 6 | import numpy as np 7 | import open3d as o3d 8 | import torch 9 | import torch.distributed as dist 10 | 11 | logger = logging.getLogger() 12 | 13 | def import_str(string: str): 14 | """ Import a python module given string paths 15 | 16 | Args: 17 | string (str): The given paths 18 | 19 | Returns: 20 | Any: Imported python module / object 21 | """ 22 | # From https://github.com/CompVis/taming-transformers 23 | module, cls = string.rsplit(".", 1) 24 | return getattr(importlib.import_module(module, package=None), cls) 25 | 26 | def export_points_to_ply( 27 | positions: torch.tensor, 28 | colors: torch.tensor, 29 | save_path: str, 30 | normalize: bool = False, 31 | ): 32 | # normalize points 33 | if normalize: 34 | aabb_min = positions.min(0)[0] 35 | aabb_max = positions.max(0)[0] 36 | positions = (positions - aabb_min) / (aabb_max - aabb_min) 37 | if isinstance(colors, torch.Tensor): 38 | positions = positions.cpu().numpy() 39 | if isinstance(colors, torch.Tensor): 40 | colors = colors.cpu().numpy() 41 | 42 | # clamp colors 43 | colors = np.clip(colors, a_min=0., a_max=1.) 44 | 45 | pcd = o3d.geometry.PointCloud() 46 | pcd.points = o3d.utility.Vector3dVector(positions) 47 | pcd.colors = o3d.utility.Vector3dVector(colors) 48 | o3d.io.write_point_cloud(save_path, pcd) 49 | 50 | def export_gaussians_to_ply(model, path, name='point_cloud.ply', aabb=None): 51 | model.eval() 52 | filename = os.path.join(path, name) 53 | map_to_tensors = {} 54 | 55 | with torch.no_grad(): 56 | positions = model.means 57 | if aabb is not None: 58 | aabb = aabb.to(positions.device) 59 | aabb_min, aabb_max = aabb[:3], aabb[3:] 60 | aabb_center = (aabb_min + aabb_max) / 2 61 | aabb_sacle_max = (aabb_max - aabb_min).max() / 2 * 1.1 62 | vis_mask = torch.logical_and(positions >= aabb_min, positions < aabb_max).all(-1) 63 | else: 64 | aabb_center = positions.mean(0) 65 | aabb_sacle_max = (positions - aabb_center).abs().max() * 1.1 66 | vis_mask = torch.ones_like(positions[:, 0], dtype=torch.bool) 67 | 68 | positions = ((positions[vis_mask] - aabb_center) / aabb_sacle_max).cpu().numpy() 69 | map_to_tensors["positions"] = o3d.core.Tensor(positions, o3d.core.float32) 70 | map_to_tensors["normals"] = o3d.core.Tensor(np.zeros_like(positions), o3d.core.float32) 71 | 72 | colors = model.colors[vis_mask].data.cpu().numpy() 73 | map_to_tensors["colors"] = (colors * 255).astype(np.uint8) 74 | for i in range(colors.shape[1]): 75 | map_to_tensors[f"f_dc_{i}"] = colors[:, i : i + 1] 76 | 77 | shs = model.shs_rest[vis_mask].data.cpu().numpy() 78 | if model.config.sh_degree > 0: 79 | shs = shs.reshape((colors.shape[0], -1, 1)) 80 | for i in range(shs.shape[-1]): 81 | map_to_tensors[f"f_rest_{i}"] = shs[:, i] 82 | 83 | map_to_tensors["opacity"] = model.opacities[vis_mask].data.cpu().numpy() 84 | 85 | scales = model.scales[vis_mask].data.cpu().unsqueeze(-1).numpy() 86 | for i in range(3): 87 | map_to_tensors[f"scale_{i}"] = scales[:, i] 88 | 89 | quats = model.quats[vis_mask].data.cpu().unsqueeze(-1).numpy() 90 | 91 | for i in range(4): 92 | map_to_tensors[f"rot_{i}"] = quats[:, i] 93 | 94 | pcd = o3d.t.geometry.PointCloud(map_to_tensors) 95 | o3d.t.io.write_point_cloud(str(filename), pcd) 96 | 97 | logger.info(f"Exported point cloud to {filename}, containing {vis_mask.sum().item()} points.") 98 | 99 | def is_enabled() -> bool: 100 | """ 101 | Returns: 102 | True if distributed training is enabled 103 | """ 104 | return dist.is_available() and dist.is_initialized() 105 | 106 | 107 | def get_global_rank() -> int: 108 | """ 109 | Returns: 110 | The rank of the current process within the global process group. 111 | """ 112 | return dist.get_rank() if is_enabled() else 0 113 | 114 | 115 | def get_world_size(): 116 | return dist.get_world_size() if is_enabled() else 1 117 | 118 | 119 | def is_main_process() -> bool: 120 | """ 121 | Returns: 122 | True if the current process is the main one. 123 | """ 124 | return get_global_rank() == 0 -------------------------------------------------------------------------------- /docs/media/badge-website.svg: -------------------------------------------------------------------------------- 1 | 2 | 15 | 17 | 35 | project: website 37 | 38 | 42 | 47 | 51 | 52 | 54 | 60 | 61 | 64 | 69 | 75 | 80 | 81 | 88 | 98 | Project 105 | 106 | 116 | Website 123 | 124 | 125 | 129 | 130 | -------------------------------------------------------------------------------- /project/configs/paper_legacy/streetgs.yaml: -------------------------------------------------------------------------------- 1 | seed: 0 2 | dataset: waymo/3cams 3 | 4 | # ------------- Trainer ------------ # 5 | trainer: 6 | type: models.trainers.MultiTrainer 7 | optim: 8 | num_iters: 30000 9 | use_grad_scaler: false 10 | cache_buffer_freq: -1 11 | render: 12 | near_plane: 0.1 13 | far_plane: 10000000000.0 14 | antialiased: false 15 | packed: false 16 | absgrad: true 17 | sparse_grad: false 18 | batch_size: 1 19 | losses: 20 | rgb: 21 | w: 0.8 22 | ssim: 23 | w: 0.2 24 | mask: 25 | w: 0.05 26 | opacity_loss_type: bce # choose from [bce, safe_bce] 27 | depth: 28 | w: 0.1 # weight of depth loss 29 | inverse_depth: True # whether to use inverse depth, NOTE that when set to True, must normalize=True 30 | normalize: False # whether to normalize depth loss 31 | loss_type: l1 # choose from ["l1", "l2"] 32 | opacity_entropy: 33 | w: 0.05 34 | inverse_depth_smoothness: 35 | w: 0.001 36 | dynamic_region: 37 | factor: 5 38 | start_from: 20000 39 | res_schedule: 40 | double_steps: 250 # training starts at 1/d resolution, every n steps this is doubled 41 | downscale_times: 2 # at the beginning, resolution is 1/2^d, where d is this number 42 | gaussian_optim_general_cfg: 43 | xyz: 44 | lr: 1.6e-04 45 | lr_final: 1.6e-06 46 | scale_factor: scene_radius # str or float, if "scene_scale", scale the learning rate by the scene scale 47 | sh_dc: 48 | lr: 0.0025 49 | sh_rest: 50 | lr: 0.000125 51 | opacity: 52 | lr: 0.05 53 | scaling: 54 | lr: 0.005 55 | rotation: 56 | lr: 0.001 57 | gaussian_ctrl_general_cfg: 58 | warmup_steps: 500 # warmup steps for alpha 59 | reset_alpha_interval: 3000 # reset alpha every n steps 60 | refine_interval: 100 # refine gaussians every n steps 61 | sh_degree_interval: 1000 # every n intervals turn on another sh degree 62 | n_split_samples: 2 # number of samples to split gaussians into 63 | # may differ in different models 64 | reset_alpha_value: 0.01 # reset alpha to this value 65 | densify_grad_thresh: 0.0003 # above this grad, gaussians are densified 66 | densify_size_thresh: 0.003 # below this size, gaussians are *duplicated*, otherwise split 67 | cull_alpha_thresh: 0.005 # threshold of opacity for culling gaussians 68 | cull_scale_thresh: 0.5 # threshold of scale for culling gaussians 69 | cull_screen_size: 0.15 # if a gaussian is more than this percent of screen space, cull it 70 | split_screen_size: 0.05 # if a gaussian is more than this percent of screen space, split it 71 | stop_screen_size_at: 4000 # stop culling/splitting at this step WRT screen size of gaussians 72 | stop_split_at: 15000 # stop splitting at this step 73 | sh_degree: 3 # sh degree for gaussians 74 | 75 | # ------------- Model ------------ # 76 | model: 77 | Background: 78 | type: models.gaussians.VanillaGaussians 79 | init: 80 | from_lidar: 81 | num_samples: 600_000 82 | return_color: True 83 | near_randoms: 200_000 84 | far_randoms: 200_000 85 | reg: 86 | sharp_shape_reg: 87 | w: 1. 88 | step_interval: 10 89 | max_gauss_ratio: 10. # threshold of ratio of gaussian max to min scale before applying regularization loss from the PhysGaussian paper 90 | RigidNodes: 91 | type: models.nodes.RigidNodes 92 | init: 93 | instance_max_pts: 5000 94 | only_moving: true 95 | traj_length_thres: 1.0 96 | ctrl: 97 | cull_scale_thresh: 0.1 98 | stop_screen_size_at: 30000 99 | stop_split_at: 20000 100 | cull_out_of_bound: true 101 | reg: 102 | sharp_shape_reg: 103 | w: 1. 104 | step_interval: 10 105 | max_gauss_ratio: 10. 106 | temporal_smooth_reg: 107 | trans: 108 | w: 0.01 # no ablation, maybe some other values will be better 109 | smooth_range: 5 # no ablation 110 | optim: 111 | ins_rotation: 112 | lr: 0.00001 113 | lr_final: 0.000005 114 | ins_translation: 115 | lr: 0.0005 116 | lr_final: 0.0001 117 | Sky: 118 | type: models.modules.EnvLight 119 | params: 120 | resolution: 1024 121 | optim: 122 | all: 123 | lr: 0.01 124 | 125 | # ------------- render ------------ # 126 | render: 127 | fps: 16 # frames per second for the main rendered output 128 | render_full: True # whether to render full resolution videos 129 | render_test: True # whether to render test set 130 | vis_lidar: False # whether to visualize lidar points on ground truth images 131 | vis_sky: False # whether to include "rgb_sky" and "rgb_sky_blend" in rendered keys 132 | vis_error: False # whether to include "rgb_error_map" in rendered keys 133 | 134 | # ------------- logging ------------ # 135 | logging: 136 | vis_freq: 2000 # how often to visualize training stats 137 | print_freq: 500 # how often to print training stats 138 | saveckpt_freq: 15000 # how often to save checkpoints 139 | save_seperate_video: True # whether to save seperate videos for each scene -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/utils/pose_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | 17 | import sys 18 | from typing import NewType, List, Dict, Optional 19 | import os 20 | import os.path as osp 21 | 22 | import pickle 23 | 24 | import torch 25 | import torch.nn as nn 26 | import torch.nn.functional as F 27 | 28 | from omegaconf import OmegaConf 29 | from loguru import logger 30 | 31 | from .typing import Tensor 32 | 33 | 34 | def rotation_matrix_to_cont_repr(x: Tensor) -> Tensor: 35 | assert len(x.shape) == 3, ( 36 | f'Expects an array of size Bx3x3, but received {x.shape}') 37 | return x[:, :3, :2] 38 | 39 | 40 | def cont_repr_to_rotation_matrix( 41 | x: Tensor 42 | ) -> Tensor: 43 | ''' Converts tensor in continous representation to rotation matrices 44 | ''' 45 | batch_size = x.shape[0] 46 | reshaped_input = x.view(-1, 3, 2) 47 | 48 | # Normalize the first vector 49 | b1 = F.normalize(reshaped_input[:, :, 0].clone(), dim=1) 50 | 51 | dot_prod = torch.sum( 52 | b1 * reshaped_input[:, :, 1].clone(), dim=1, keepdim=True) 53 | # Compute the second vector by finding the orthogonal complement to it 54 | b2 = F.normalize(reshaped_input[:, :, 1] - dot_prod * b1, dim=1) 55 | # Finish building the basis by taking the cross product 56 | b3 = torch.cross(b1, b2, dim=1) 57 | rot_mats = torch.stack([b1, b2, b3], dim=-1) 58 | 59 | return rot_mats.view(batch_size, -1, 3, 3) 60 | 61 | 62 | def batch_rodrigues( 63 | rot_vecs: Tensor, 64 | epsilon: float = 1e-8 65 | ) -> Tensor: 66 | ''' Calculates the rotation matrices for a batch of rotation vectors 67 | Parameters 68 | ---------- 69 | rot_vecs: torch.tensor Nx3 70 | array of N axis-angle vectors 71 | Returns 72 | ------- 73 | R: torch.tensor Nx3x3 74 | The rotation matrices for the given axis-angle parameters 75 | ''' 76 | assert len(rot_vecs.shape) == 2, ( 77 | f'Expects an array of size Bx3, but received {rot_vecs.shape}') 78 | 79 | batch_size = rot_vecs.shape[0] 80 | device = rot_vecs.device 81 | dtype = rot_vecs.dtype 82 | 83 | angle = torch.norm(rot_vecs + epsilon, dim=1, keepdim=True, p=2) 84 | rot_dir = rot_vecs / angle 85 | 86 | cos = torch.unsqueeze(torch.cos(angle), dim=1) 87 | sin = torch.unsqueeze(torch.sin(angle), dim=1) 88 | 89 | # Bx1 arrays 90 | rx, ry, rz = torch.split(rot_dir, 1, dim=1) 91 | K = torch.zeros((batch_size, 3, 3), dtype=dtype, device=device) 92 | 93 | zeros = torch.zeros((batch_size, 1), dtype=dtype, device=device) 94 | K = torch.cat([zeros, -rz, ry, rz, zeros, -rx, -ry, rx, zeros], dim=1) \ 95 | .view((batch_size, 3, 3)) 96 | 97 | ident = torch.eye(3, dtype=dtype, device=device).unsqueeze(dim=0) 98 | rot_mat = ident + sin * K + (1 - cos) * torch.bmm(K, K) 99 | return rot_mat 100 | 101 | 102 | def batch_rot2aa( 103 | Rs: Tensor, epsilon: float = 1e-7 104 | ) -> Tensor: 105 | """ 106 | Rs is B x 3 x 3 107 | void cMathUtil::RotMatToAxisAngle(const tMatrix& mat, tVector& out_axis, 108 | double& out_theta) 109 | { 110 | double c = 0.5 * (mat(0, 0) + mat(1, 1) + mat(2, 2) - 1); 111 | c = cMathUtil::Clamp(c, -1.0, 1.0); 112 | 113 | out_theta = std::acos(c); 114 | 115 | if (std::abs(out_theta) < 0.00001) 116 | { 117 | out_axis = tVector(0, 0, 1, 0); 118 | } 119 | else 120 | { 121 | double m21 = mat(2, 1) - mat(1, 2); 122 | double m02 = mat(0, 2) - mat(2, 0); 123 | double m10 = mat(1, 0) - mat(0, 1); 124 | double denom = std::sqrt(m21 * m21 + m02 * m02 + m10 * m10); 125 | out_axis[0] = m21 / denom; 126 | out_axis[1] = m02 / denom; 127 | out_axis[2] = m10 / denom; 128 | out_axis[3] = 0; 129 | } 130 | } 131 | """ 132 | 133 | cos = 0.5 * (torch.einsum('bii->b', [Rs]) - 1) 134 | cos = torch.clamp(cos, -1 + epsilon, 1 - epsilon) 135 | 136 | theta = torch.acos(cos) 137 | 138 | m21 = Rs[:, 2, 1] - Rs[:, 1, 2] 139 | m02 = Rs[:, 0, 2] - Rs[:, 2, 0] 140 | m10 = Rs[:, 1, 0] - Rs[:, 0, 1] 141 | denom = torch.sqrt(m21 * m21 + m02 * m02 + m10 * m10 + epsilon) 142 | 143 | axis0 = torch.where(torch.abs(theta) < 0.00001, m21, m21 / denom) 144 | axis1 = torch.where(torch.abs(theta) < 0.00001, m02, m02 / denom) 145 | axis2 = torch.where(torch.abs(theta) < 0.00001, m10, m10 / denom) 146 | 147 | return theta.unsqueeze(1) * torch.stack([axis0, axis1, axis2], 1) 148 | -------------------------------------------------------------------------------- /project/datasets/tools/multiprocess_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | @brief Parallel processing utilities 3 | @note Borrowed from: https://github.com/NVlabs/EmerNeRF 4 | """ 5 | import sys 6 | import time 7 | from collections.abc import Iterable 8 | from multiprocessing import Pool 9 | from shutil import get_terminal_size 10 | 11 | class ProgressBar: 12 | """A progress bar which can print the progress.""" 13 | 14 | def __init__(self, task_num=0, bar_width=50, start=True, file=sys.stdout): 15 | self.task_num = task_num 16 | self.bar_width = bar_width 17 | self.completed = 0 18 | self.file = file 19 | if start: 20 | self.start() 21 | 22 | @property 23 | def terminal_width(self): 24 | width, _ = get_terminal_size() 25 | return width 26 | 27 | def start(self): 28 | if self.task_num > 0: 29 | self.file.write( 30 | f'[{" " * self.bar_width}] 0/{self.task_num}, ' "elapsed: 0s, ETA:" 31 | ) 32 | else: 33 | self.file.write("completed: 0, elapsed: 0s") 34 | self.file.flush() 35 | self.start_time = time.time() 36 | 37 | def update(self, num_tasks=1): 38 | assert num_tasks > 0 39 | self.completed += num_tasks 40 | elapsed = time.time() - self.start_time 41 | if elapsed > 0: 42 | fps = self.completed / elapsed 43 | else: 44 | fps = float("inf") 45 | if self.task_num > 0: 46 | percentage = self.completed / float(self.task_num) 47 | eta = int(elapsed * (1 - percentage) / percentage + 0.5) 48 | msg = ( 49 | f"\r[{{}}] {self.completed}/{self.task_num}, " 50 | f"{fps:.1f} task/s, elapsed: {int(elapsed + 0.5)}s, " 51 | f"ETA: {eta:5}s" 52 | ) 53 | 54 | bar_width = min( 55 | self.bar_width, 56 | int(self.terminal_width - len(msg)) + 2, 57 | int(self.terminal_width * 0.6), 58 | ) 59 | bar_width = max(2, bar_width) 60 | mark_width = int(bar_width * percentage) 61 | bar_chars = ">" * mark_width + " " * (bar_width - mark_width) 62 | self.file.write(msg.format(bar_chars)) 63 | else: 64 | self.file.write( 65 | f"completed: {self.completed}, elapsed: {int(elapsed + 0.5)}s," 66 | f" {fps:.1f} tasks/s" 67 | ) 68 | self.file.flush() 69 | 70 | 71 | def init_pool(process_num, initializer=None, initargs=None): 72 | if initializer is None: 73 | return Pool(process_num) 74 | elif initargs is None: 75 | return Pool(process_num, initializer) 76 | else: 77 | if not isinstance(initargs, tuple): 78 | raise TypeError('"initargs" must be a tuple') 79 | return Pool(process_num, initializer, initargs) 80 | 81 | 82 | def track_parallel_progress( 83 | func, 84 | tasks, 85 | nproc, 86 | initializer=None, 87 | initargs=None, 88 | bar_width=50, 89 | chunksize=1, 90 | skip_first=False, 91 | keep_order=True, 92 | file=sys.stdout, 93 | ): 94 | """Track the progress of parallel task execution with a progress bar. 95 | 96 | The built-in :mod:`multiprocessing` module is used for process pools and 97 | tasks are done with :func:`Pool.map` or :func:`Pool.imap_unordered`. 98 | 99 | Args: 100 | func (callable): The function to be applied to each task. 101 | tasks (list or tuple[Iterable, int]): A list of tasks or 102 | (tasks, total num). 103 | nproc (int): Process (worker) number. 104 | initializer (None or callable): Refer to :class:`multiprocessing.Pool` 105 | for details. 106 | initargs (None or tuple): Refer to :class:`multiprocessing.Pool` for 107 | details. 108 | chunksize (int): Refer to :class:`multiprocessing.Pool` for details. 109 | bar_width (int): Width of progress bar. 110 | skip_first (bool): Whether to skip the first sample for each worker 111 | when estimating fps, since the initialization step may takes 112 | longer. 113 | keep_order (bool): If True, :func:`Pool.imap` is used, otherwise 114 | :func:`Pool.imap_unordered` is used. 115 | 116 | Returns: 117 | list: The task results. 118 | """ 119 | if isinstance(tasks, tuple): 120 | assert len(tasks) == 2 121 | assert isinstance(tasks[0], Iterable) 122 | assert isinstance(tasks[1], int) 123 | task_num = tasks[1] 124 | tasks = tasks[0] 125 | elif isinstance(tasks, Iterable): 126 | task_num = len(tasks) 127 | else: 128 | raise TypeError('"tasks" must be an iterable object or a (iterator, int) tuple') 129 | pool = init_pool(nproc, initializer, initargs) 130 | start = not skip_first 131 | task_num -= nproc * chunksize * int(skip_first) 132 | prog_bar = ProgressBar(task_num, bar_width, start, file=file) 133 | results = [] 134 | if keep_order: 135 | gen = pool.imap(func, tasks, chunksize) 136 | else: 137 | gen = pool.imap_unordered(func, tasks, chunksize) 138 | for result in gen: 139 | results.append(result) 140 | if skip_first: 141 | if len(results) < nproc * chunksize: 142 | continue 143 | elif len(results) == nproc * chunksize: 144 | prog_bar.start() 145 | continue 146 | prog_bar.update() 147 | prog_bar.file.write("\n") 148 | pool.close() 149 | pool.join() 150 | return results 151 | -------------------------------------------------------------------------------- /project/datasets/tools/humanpose_process.py: -------------------------------------------------------------------------------- 1 | from typing import List, Callable 2 | import os 3 | import joblib 4 | import logging 5 | import argparse 6 | import numpy as np 7 | 8 | from datasets.tools.extract_smpl import run_4DHumans 9 | from datasets.tools.postprocess import match_and_postprocess 10 | 11 | logger = logging.getLogger() 12 | 13 | def extract_humanpose( 14 | scene_dir, 15 | projection_fn: Callable, 16 | camera_list: List[str], 17 | save_temp: bool=True, 18 | verbose: bool=False, 19 | fps: int=12 20 | ): 21 | """Extract human pose from the waymo dataset 22 | 23 | Args: 24 | scene_dir: str, path to the scene directory 25 | save_temp: bool, whether to save the intermediate results 26 | verbose: bool, whether to visualize debug images 27 | fps: int, FPS for the visualization video 28 | """ 29 | # project human boxes to 2D image space 30 | GTTracks_meta = projection_fn( 31 | scene_dir, camera_list=camera_list, 32 | save_temp=save_temp, verbose=verbose, 33 | narrow_width_ratio=0.2, fps=fps 34 | ) 35 | 36 | # run 4DHuman to get predicted human tracks with SMPL parameters 37 | PredTracks_meta = run_4DHumans( 38 | scene_dir, camera_list=camera_list, 39 | save_temp=save_temp, verbose=verbose, fps=fps 40 | ) 41 | 42 | # match the predicted tracks with the ground truth tracks 43 | smpl_meta = match_and_postprocess( 44 | scene_dir, camera_list=camera_list, 45 | GTTracksDict=GTTracks_meta, PredTracksDict=PredTracks_meta, 46 | save_temp=save_temp, verbose=verbose, fps=fps 47 | ) 48 | 49 | joblib.dump( 50 | smpl_meta, 51 | os.path.join(scene_dir, "humanpose", "smpl.pkl") 52 | ) 53 | 54 | if __name__ == "__main__": 55 | parser = argparse.ArgumentParser(description="Data converter arg parser") 56 | parser.add_argument("--data_root", type=str, required=True, help="root path of waymo dataset") 57 | parser.add_argument("--dataset", type=str, default="waymo", help="dataset name") 58 | parser.add_argument( 59 | "--scene_ids", 60 | default=None, 61 | type=int, 62 | nargs="+", 63 | help="scene ids to be processed, a list of integers separated by space. Range: [0, 798] for training, [0, 202] for validation", 64 | ) 65 | parser.add_argument( 66 | "--split_file", type=str, default=None, help="Split file in data/waymo_splits" 67 | ) 68 | parser.add_argument( 69 | "--start_idx", 70 | type=int, 71 | default=0, 72 | help="If no scene id or split_file is given, use start_idx and num_scenes to generate scene_ids_list", 73 | ) 74 | parser.add_argument( 75 | "--num_scenes", 76 | type=int, 77 | default=200, 78 | help="number of scenes to be processed", 79 | ) 80 | parser.add_argument( 81 | "--save_temp", 82 | action="store_true", 83 | help="Whether to save the intermediate results", 84 | ) 85 | parser.add_argument( 86 | "--verbose", 87 | action="store_true", 88 | help="Whether to visualize the intermediate results", 89 | ) 90 | parser.add_argument( 91 | "--fps", 92 | type=int, 93 | default=12, 94 | help="FPS for the visualization video if verbose is True", 95 | ) 96 | args = parser.parse_args() 97 | 98 | if args.dataset == "waymo": 99 | from datasets.waymo.waymo_human_utils import project_human_boxes, CAMERA_LIST 100 | elif args.dataset == "pandaset": 101 | from datasets.pandaset.pandaset_human_utils import project_human_boxes, CAMERA_LIST 102 | elif args.dataset == "argoverse": 103 | from datasets.argoverse.argoverse_human_utils import project_human_boxes, CAMERA_LIST 104 | elif args.dataset == "nuscenes": 105 | from datasets.nuscenes.nuscenes_human_utils import project_human_boxes, CAMERA_LIST 106 | elif args.dataset == "kitti": 107 | from datasets.kitti.kitti_human_utils import project_human_boxes, CAMERA_LIST 108 | elif args.dataset == "nuplan": 109 | from datasets.nuplan.nuplan_human_utils import project_human_boxes, CAMERA_LIST 110 | else: 111 | raise ValueError(f"Unknown dataset {args.dataset}, please choose from waymo, pandaset, argoverse, nuscenes, kitti, nuplan") 112 | 113 | if args.scene_ids is not None: 114 | scene_ids_list = args.scene_ids 115 | elif args.split_file is not None: 116 | # parse the split file 117 | split_file = open(args.split_file, "r").readlines()[1:] 118 | try: 119 | # Waymo, Pandaset, Argoverse, NuScenes 120 | scene_ids_list = [int(line.strip().split(",")[0]) for line in split_file] 121 | except: 122 | # KITTI 123 | scene_ids_list = [line.strip().split(" ")[0] for line in split_file] 124 | else: 125 | scene_ids_list = np.arange(args.start_idx, args.start_idx + args.num_scenes) 126 | 127 | for scene_id in scene_ids_list: 128 | try: 129 | scene_dir = f'{args.data_root}/{str(scene_id).zfill(3)}' 130 | extract_humanpose( 131 | scene_dir=scene_dir, 132 | projection_fn=project_human_boxes, 133 | camera_list=CAMERA_LIST, 134 | save_temp=args.save_temp, 135 | verbose=args.verbose, 136 | fps=args.fps 137 | ) 138 | logger.info(f"Finished processing scene {scene_id}") 139 | except Exception as e: 140 | logger.error(f"Error processing scene {scene_id}: {e}") 141 | continue -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/losses/losses.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is 4 | # holder of all proprietary rights on this computer program. 5 | # You can only use this computer program if you have closed 6 | # a license agreement with MPG or you get the right to use the computer 7 | # program from someone who is authorized to grant you that right. 8 | # Any use of the computer program without a valid license is prohibited and 9 | # liable to prosecution. 10 | # 11 | # Copyright©2020 Max-Planck-Gesellschaft zur Förderung 12 | # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute 13 | # for Intelligent Systems. All rights reserved. 14 | # 15 | # Contact: Vassilis Choutas, vassilis.choutas@tuebingen.mpg.de 16 | from __future__ import print_function 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | 20 | import sys 21 | import time 22 | from typing import Callable, Iterator, Union, Optional, List 23 | 24 | import os.path as osp 25 | import yaml 26 | from loguru import logger 27 | 28 | import pickle 29 | 30 | import numpy as np 31 | 32 | import torch 33 | import torch.autograd as autograd 34 | import torch.nn as nn 35 | import torch.nn.functional as F 36 | 37 | from .utils import get_reduction_method 38 | 39 | __all__ = [ 40 | 'VertexEdgeLoss', 41 | 'build_loss', 42 | ] 43 | 44 | 45 | def build_loss(type='l2', reduction='mean', **kwargs) -> nn.Module: 46 | logger.debug(f'Building loss: {type}') 47 | if type == 'l2': 48 | return WeightedMSELoss(reduction=reduction, **kwargs) 49 | elif type == 'vertex-edge': 50 | return VertexEdgeLoss(reduction=reduction, **kwargs) 51 | elif type == 'l1': 52 | return nn.L1Loss() 53 | else: 54 | raise ValueError(f'Unknown loss type: {type}') 55 | 56 | 57 | class WeightedMSELoss(nn.Module): 58 | def __init__(self, reduction='mean', **kwargs): 59 | super(WeightedMSELoss, self).__init__() 60 | self.reduce_str = reduction 61 | self.reduce = get_reduction_method(reduction) 62 | 63 | def forward(self, input, target, weights=None): 64 | diff = input - target 65 | if weights is None: 66 | return diff.pow(2).sum() / diff.shape[0] 67 | else: 68 | return ( 69 | weights.unsqueeze(dim=-1) * diff.pow(2)).sum() / diff.shape[0] 70 | 71 | 72 | class VertexEdgeLoss(nn.Module): 73 | def __init__(self, norm_type='l2', 74 | gt_edges=None, 75 | gt_edge_path='', 76 | est_edges=None, 77 | est_edge_path='', 78 | robustifier=None, 79 | edge_thresh=0.0, epsilon=1e-8, 80 | reduction='sum', 81 | **kwargs): 82 | super(VertexEdgeLoss, self).__init__() 83 | 84 | assert norm_type in ['l1', 'l2'], 'Norm type must be [l1, l2]' 85 | self.norm_type = norm_type 86 | self.epsilon = epsilon 87 | self.reduction = reduction 88 | assert self.reduction in ['sum', 'mean'] 89 | logger.info(f'Building edge loss with' 90 | f' norm_type={norm_type},' 91 | f' reduction={reduction},' 92 | ) 93 | 94 | gt_edge_path = osp.expandvars(gt_edge_path) 95 | est_edge_path = osp.expandvars(est_edge_path) 96 | assert osp.exists(gt_edge_path) or gt_edges is not None, ( 97 | 'gt_edges must not be None or gt_edge_path must exist' 98 | ) 99 | assert osp.exists(est_edge_path) or est_edges is not None, ( 100 | 'est_edges must not be None or est_edge_path must exist' 101 | ) 102 | if osp.exists(gt_edge_path) and gt_edges is None: 103 | gt_edges = np.load(gt_edge_path) 104 | if osp.exists(est_edge_path) and est_edges is None: 105 | est_edges = np.load(est_edge_path) 106 | 107 | self.register_buffer( 108 | 'gt_connections', torch.tensor(gt_edges, dtype=torch.long)) 109 | self.register_buffer( 110 | 'est_connections', torch.tensor(est_edges, dtype=torch.long)) 111 | 112 | def extra_repr(self): 113 | msg = [ 114 | f'Norm type: {self.norm_type}', 115 | ] 116 | if self.has_connections: 117 | msg.append( 118 | f'GT Connections shape: {self.gt_connections.shape}' 119 | ) 120 | msg.append( 121 | f'Est Connections shape: {self.est_connections.shape}' 122 | ) 123 | return '\n'.join(msg) 124 | 125 | def compute_edges(self, points, connections): 126 | edge_points = torch.index_select( 127 | points, 1, connections.view(-1)).reshape(points.shape[0], -1, 2, 3) 128 | return edge_points[:, :, 1] - edge_points[:, :, 0] 129 | 130 | def forward(self, gt_vertices, est_vertices, weights=None): 131 | gt_edges = self.compute_edges( 132 | gt_vertices, connections=self.gt_connections) 133 | est_edges = self.compute_edges( 134 | est_vertices, connections=self.est_connections) 135 | 136 | raw_edge_diff = (gt_edges - est_edges) 137 | 138 | batch_size = gt_vertices.shape[0] 139 | if self.norm_type == 'l2': 140 | edge_diff = raw_edge_diff.pow(2) 141 | elif self.norm_type == 'l1': 142 | edge_diff = raw_edge_diff.abs() 143 | else: 144 | raise NotImplementedError( 145 | f'Loss type not implemented: {self.loss_type}') 146 | if self.reduction == 'sum': 147 | return edge_diff.sum() 148 | elif self.reduction == 'mean': 149 | return edge_diff.sum() / batch_size 150 | -------------------------------------------------------------------------------- /project/third_party/smplx/transfer_model/view_pkl.py: -------------------------------------------------------------------------------- 1 | import os.path as osp 2 | import argparse 3 | 4 | import numpy as np 5 | import torch 6 | 7 | import pyrender 8 | import trimesh 9 | 10 | import smplx 11 | 12 | from tqdm.auto import tqdm, trange 13 | 14 | from pathlib import Path 15 | 16 | def main(model_folder, 17 | motion_file, 18 | model_type='smplx', 19 | ext='npz', 20 | gender='neutral', 21 | plot_joints=False, 22 | num_betas=10, 23 | sample_expression=True, 24 | num_expression_coeffs=10, 25 | use_face_contour=False): 26 | 27 | # open motion file 28 | motion = np.load(motion_file, allow_pickle=True) 29 | _motion = {} 30 | for k,v in motion.items(): 31 | if isinstance(v, np.ndarray): 32 | print(k, motion[k].shape, motion[k].dtype) 33 | if motion[k].dtype in ("