├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md └── workflows │ ├── build.yml │ ├── lint.yml │ └── publish-to-pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── .readthedocs.yml ├── CITATION.cff ├── LICENSE ├── README.md ├── configs ├── _base_ │ ├── datasets │ │ └── pw3d.py │ ├── default_runtime.py │ ├── models │ │ └── resnet50.py │ ├── post_processing │ │ ├── README.md │ │ ├── deciwatch_interval10_q1.py │ │ ├── deciwatch_interval10_q2.py │ │ ├── deciwatch_interval10_q3.py │ │ ├── deciwatch_interval10_q4.py │ │ ├── deciwatch_interval10_q5.py │ │ ├── deciwatch_interval5_q1.py │ │ ├── deciwatch_interval5_q2.py │ │ ├── deciwatch_interval5_q3.py │ │ ├── deciwatch_interval5_q4.py │ │ ├── deciwatch_interval5_q5.py │ │ ├── guas1d.py │ │ ├── oneeuro.py │ │ ├── savgol.py │ │ ├── smoothnet_windowsize16.py │ │ ├── smoothnet_windowsize32.py │ │ ├── smoothnet_windowsize64.py │ │ └── smoothnet_windowsize8.py │ └── schedulers │ │ └── hmr_bs256.py ├── algorithms │ └── README.md ├── augmentation │ ├── README.md │ ├── resnet50_hmr_coarse_dropout_eftcoco.py │ ├── resnet50_hmr_coarse_dropout_h36m.py │ ├── resnet50_hmr_eftcoco.py │ ├── resnet50_hmr_grid_dropout_eftcoco.py │ ├── resnet50_hmr_grid_dropout_h36m.py │ ├── resnet50_hmr_h36m.py │ ├── resnet50_hmr_photometric_distortion_eftcoco.py │ ├── resnet50_hmr_photometric_distortion_h36m.py │ ├── resnet50_hmr_rand_crop_eftcoco.py │ ├── resnet50_hmr_rand_crop_h36m.py │ ├── resnet50_hmr_rand_occ_eftcoco.py │ ├── resnet50_hmr_rand_occ_h36m.py │ ├── resnet50_hmr_self_mix_eftcoco.py │ ├── resnet50_hmr_self_mix_h36m.py │ ├── resnet50_hmr_soft_erase_eftcoco.py │ ├── resnet50_hmr_soft_erase_h36m.py │ ├── resnet50_hmr_syn_occ_eftcoco.py │ ├── resnet50_hmr_syn_occ_h36m.py │ ├── resnet50_hmr_syn_occkp_eftcoco.py │ └── resnet50_hmr_syn_occkp_h36m.py ├── backbones │ ├── README.md │ ├── efficientnet_hmr_pw3d.py │ ├── hrnet_hmr_pw3d.py │ ├── metafile.yml │ ├── resnet101_hmr_pw3d.py │ ├── resnet152_hmr_pw3d.py │ ├── resnet50_hmr_pw3d.py │ ├── resnext101_hmr_pw3d.py │ ├── swin_hmr_pw3d.py │ ├── twins_pcpvt_hmr_pw3d.py │ ├── twins_svt_hmr_pw3d.py │ └── vit_hmr_pw3d.py ├── combine │ ├── README.md │ ├── hrnet_hmr_mix4_coco_l1.py │ ├── hrnet_hmr_mix4_coco_l1_aug.py │ ├── resnet50_hmr_mix1_coco_l1.py │ ├── resnet50_hmr_mix1_coco_l1_aug.py │ ├── twins_svt_hmr_mix1_coco_l1.py │ ├── twins_svt_hmr_mix1_coco_l1_aug.py │ ├── twins_svt_hmr_mix1_coco_mmpose_l1.py │ ├── twins_svt_hmr_mix1_coco_mmpose_l1_aug.py │ ├── twins_svt_hmr_mix2_coco_l1.py │ ├── twins_svt_hmr_mix2_coco_mmpose_l1.py │ ├── twins_svt_hmr_mix4_coco_l1.py │ ├── twins_svt_hmr_mix4_coco_l1_aug.py │ ├── twins_svt_hmr_mix4_coco_mmpose_l1.py │ ├── twins_svt_hmr_mix4_coco_mmpose_l1_aug.py │ ├── twins_svt_hmr_mix4_coco_mmpose_l1_cache.py │ ├── twins_svt_hmr_mix5_coco_l1.py │ ├── twins_svt_hmr_mix5_coco_mmpose_l1.py │ ├── twins_svt_hmr_mix6_coco_mmpose_l1_aug.py │ └── twins_svt_hmr_mix7_coco_mmpose_l1_aug.py ├── datasets │ ├── README.md │ ├── resnet50_hmr_agora.py │ ├── resnet50_hmr_aic.py │ ├── resnet50_hmr_coco.py │ ├── resnet50_hmr_coco_wholebody.py │ ├── resnet50_hmr_crowdpose.py │ ├── resnet50_hmr_eft_coco.py │ ├── resnet50_hmr_eft_coco_part.py │ ├── resnet50_hmr_eft_lspet.py │ ├── resnet50_hmr_eft_mpii.py │ ├── resnet50_hmr_eft_ochuman.py │ ├── resnet50_hmr_eft_posetrack.py │ ├── resnet50_hmr_h36m.py │ ├── resnet50_hmr_h36m_hp.py │ ├── resnet50_hmr_instavariety.py │ ├── resnet50_hmr_lip.py │ ├── resnet50_hmr_lsp.py │ ├── resnet50_hmr_lspet.py │ ├── resnet50_hmr_mhp.py │ ├── resnet50_hmr_mpi_inf_3dhp.py │ ├── resnet50_hmr_mpi_inf_3dhp_orig.py │ ├── resnet50_hmr_mpi_inf_3dhp_spin.py │ ├── resnet50_hmr_mpi_inf_3dhp_spin_orig.py │ ├── resnet50_hmr_mpii.py │ ├── resnet50_hmr_mtp.py │ ├── resnet50_hmr_muco.py │ ├── resnet50_hmr_muco_aug.py │ ├── resnet50_hmr_muco_noaug.py │ ├── resnet50_hmr_mupots3d.py │ ├── resnet50_hmr_ochuman.py │ ├── resnet50_hmr_oh50k3d.py │ ├── resnet50_hmr_panoptic.py │ ├── resnet50_hmr_penn_action.py │ ├── resnet50_hmr_people3d.py │ ├── resnet50_hmr_posetrack.py │ ├── resnet50_hmr_prox.py │ ├── resnet50_hmr_pw3d.py │ ├── resnet50_hmr_pw3d_no3dkp.py │ ├── resnet50_hmr_surreal.py │ ├── resnet50_hmr_totalcapture.py │ ├── resnet50_hmr_up3d.py │ └── resnet50_hmr_vlog.py ├── gta_human │ ├── README.md │ ├── resnet50_hmr_gta_bt.py │ └── resnet50_hmr_gta_ft.py ├── hmr │ ├── README.md │ ├── metafile.yml │ ├── resnet50_hmr_pw3d.py │ └── resnet50_hmr_pw3d_e50_cache.py ├── hybrik │ ├── README.md │ ├── metafile.yml │ └── resnet34_hybrik_mixed.py ├── mixed-l1 │ ├── README.md │ ├── resnet50_hmr_mix11_l1.py │ ├── resnet50_hmr_mix1_l1.py │ ├── resnet50_hmr_mix2_l1.py │ ├── resnet50_hmr_mix2_mse.py │ ├── resnet50_hmr_mix5_l1.py │ ├── resnet50_hmr_mix5_mse.py │ ├── resnet50_hmr_mix6_l1.py │ ├── resnet50_hmr_mix7_l1.py │ ├── resnet50_hmr_mix8_l1.py │ └── resnet50_hmr_mix9_l1.py ├── mixed │ ├── README.md │ ├── resnet50_hmr_mix1.py │ ├── resnet50_hmr_mix10.py │ ├── resnet50_hmr_mix11.py │ ├── resnet50_hmr_mix2.py │ ├── resnet50_hmr_mix3.py │ ├── resnet50_hmr_mix4.py │ ├── resnet50_hmr_mix5.py │ ├── resnet50_hmr_mix6.py │ ├── resnet50_hmr_mix7.py │ ├── resnet50_hmr_mix8.py │ └── resnet50_hmr_mix9.py ├── pare │ ├── README.md │ ├── hrnet_w32_conv_pare_coco.py │ ├── hrnet_w32_conv_pare_coco_cache.py │ ├── hrnet_w32_conv_pare_mix.py │ ├── hrnet_w32_conv_pare_mix_cache.py │ ├── hrnet_w32_conv_pare_mix_no_mosh.py │ └── metafile.yml ├── pretrained-backbones │ ├── README.md │ ├── hrnet_hmr_coco.py │ ├── hrnet_hmr_imagenet.py │ ├── hrnet_hmr_mpii.py │ ├── resnet50_hmr_coco.py │ ├── resnet50_hmr_imagenet.py │ ├── resnet50_hmr_mpii.py │ ├── twins_svt_hmr_coco.py │ ├── twins_svt_hmr_imagenet.py │ └── twins_svt_hmr_mpii.py ├── render │ └── smpl.py ├── smplify │ ├── smplify.py │ ├── smplify3d.py │ └── smplifyx.py ├── spin │ ├── README.md │ ├── metafile.yml │ └── resnet50_spin_pw3d.py └── vibe │ ├── README.md │ ├── metafile.yml │ └── resnet50_vibe_pw3d.py ├── demo ├── estimate_smpl.py ├── mmdetection_cfg │ └── faster_rcnn_r50_fpn_coco.py ├── mmtracking_cfg │ └── deepsort_faster-rcnn_fpn_4e_mot17-private-half.py └── resources │ ├── S1_Directions_1.54138969_000001.jpg │ ├── multi_person_demo.mp4 │ └── single_person_demo.mp4 ├── docs ├── Makefile ├── README_CN.md ├── README_EN.md ├── _static │ ├── css │ │ └── readthedocs.css │ └── image │ │ └── mmhuman3d-logo.png ├── additional_licenses.md ├── api.rst ├── cameras.md ├── conf.py ├── contribute_dataset.md ├── customize_keypoints_convention.md ├── getting_started.md ├── human_data.md ├── index.rst ├── install.md ├── keypoints_convention.md ├── make.bat ├── model_zoo.md ├── preprocess_dataset.md ├── render.md ├── visualize_keypoints.md └── visualize_smpl.md ├── docs_zh-CN ├── cameras.md ├── contribute_dataset.md ├── customize_keypoints_convention.md ├── getting_started.md ├── human_data.md ├── imgs │ ├── mmhuman3d_qq_qrcode.jpg │ ├── wechat_assistant_qrcode.jpg │ └── zhihu_qrcode.jpg ├── install.md ├── keypoints_convention.md ├── model_zoo.md ├── preprocess_dataset.md ├── render.md ├── visualize_keypoints.md └── visualize_smpl.md ├── mmhuman3d ├── __init__.py ├── apis │ ├── __init__.py │ ├── inference.py │ ├── test.py │ └── train.py ├── core │ ├── __init__.py │ ├── cameras │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── camera_parameters.py │ │ └── cameras.py │ ├── conventions │ │ ├── __init__.py │ │ ├── cameras │ │ │ ├── __init__.py │ │ │ ├── convert_convention.py │ │ │ └── convert_projection.py │ │ ├── joints_mapping │ │ │ └── standard_joint_angles.py │ │ ├── keypoints_mapping │ │ │ ├── __init__.py │ │ │ ├── agora.py │ │ │ ├── coco.py │ │ │ ├── coco_wholebody.py │ │ │ ├── crowdpose.py │ │ │ ├── gta.py │ │ │ ├── h36m.py │ │ │ ├── human_data.py │ │ │ ├── hybrik.py │ │ │ ├── instavariety.py │ │ │ ├── lsp.py │ │ │ ├── mpi_inf_3dhp.py │ │ │ ├── mpii.py │ │ │ ├── openpose.py │ │ │ ├── penn_action.py │ │ │ ├── posetrack.py │ │ │ ├── pw3d.py │ │ │ ├── smpl.py │ │ │ └── smplx.py │ │ └── segmentation │ │ │ ├── __init__.py │ │ │ ├── smpl.py │ │ │ └── smplx.py │ ├── distributed_wrapper.py │ ├── evaluation │ │ ├── __init__.py │ │ ├── eval_hooks.py │ │ ├── eval_utils.py │ │ └── mesh_eval.py │ ├── optimizer │ │ ├── __init__.py │ │ └── builder.py │ ├── post_processing │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── smooth │ │ │ ├── gaus1d_filter.py │ │ │ ├── oneeuro_filter.py │ │ │ ├── savgol_filter.py │ │ │ └── smoothnet.py │ │ └── speed_up │ │ │ └── deciwatch.py │ └── visualization │ │ ├── __init__.py │ │ ├── renderer │ │ ├── __init__.py │ │ ├── matplotlib3d_renderer.py │ │ ├── torch3d_renderer │ │ │ ├── __init__.py │ │ │ ├── base_renderer.py │ │ │ ├── builder.py │ │ │ ├── depth_renderer.py │ │ │ ├── lights.py │ │ │ ├── mesh_renderer.py │ │ │ ├── meshes.py │ │ │ ├── normal_renderer.py │ │ │ ├── pointcloud_renderer.py │ │ │ ├── render_runner.py │ │ │ ├── segmentation_renderer.py │ │ │ ├── shader.py │ │ │ ├── silhouette_renderer.py │ │ │ ├── smpl_renderer.py │ │ │ ├── textures.py │ │ │ ├── utils.py │ │ │ └── uv_renderer.py │ │ └── vedo_render.py │ │ ├── visualize_cameras.py │ │ ├── visualize_keypoints2d.py │ │ ├── visualize_keypoints3d.py │ │ └── visualize_smpl.py ├── data │ ├── data_converters │ │ ├── __init__.py │ │ ├── amass.py │ │ ├── base_converter.py │ │ ├── builder.py │ │ ├── coco.py │ │ ├── coco_hybrik.py │ │ ├── coco_wholebody.py │ │ ├── crowdpose.py │ │ ├── gta_human.py │ │ ├── h36m.py │ │ ├── h36m_hybrik.py │ │ ├── humman.py │ │ ├── insta_vibe.py │ │ ├── lsp.py │ │ ├── lsp_extended.py │ │ ├── mpi_inf_3dhp.py │ │ ├── mpi_inf_3dhp_hybrik.py │ │ ├── mpii.py │ │ ├── muco.py │ │ ├── penn_action.py │ │ ├── posetrack.py │ │ ├── pw3d.py │ │ ├── pw3d_hybrik.py │ │ ├── spin.py │ │ ├── surreal.py │ │ ├── up3d.py │ │ └── vibe.py │ ├── data_structures │ │ ├── __init__.py │ │ ├── human_data.py │ │ ├── human_data_cache.py │ │ └── smc_reader.py │ └── datasets │ │ ├── __init__.py │ │ ├── adversarial_dataset.py │ │ ├── base_dataset.py │ │ ├── builder.py │ │ ├── dataset_wrappers.py │ │ ├── human_hybrik_dataset.py │ │ ├── human_image_dataset.py │ │ ├── human_video_dataset.py │ │ ├── mesh_dataset.py │ │ ├── mixed_dataset.py │ │ ├── pipelines │ │ ├── __init__.py │ │ ├── compose.py │ │ ├── formatting.py │ │ ├── hybrik_transforms.py │ │ ├── loading.py │ │ ├── mix.py │ │ ├── synthetic_occlusion_augmentation.py │ │ └── transforms.py │ │ └── samplers │ │ ├── __init__.py │ │ └── distributed_sampler.py ├── models │ ├── __init__.py │ ├── architectures │ │ ├── __init__.py │ │ ├── base_architecture.py │ │ ├── builder.py │ │ ├── hybrik.py │ │ └── mesh_estimator.py │ ├── backbones │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── efficientnet.py │ │ ├── hourglass.py │ │ ├── hrnet.py │ │ ├── resnet.py │ │ ├── resnext.py │ │ ├── swin.py │ │ ├── twins.py │ │ └── vit.py │ ├── body_models │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── smpl.py │ │ ├── smplx.py │ │ └── utils.py │ ├── discriminators │ │ ├── __init__.py │ │ ├── builder.py │ │ └── pose_discriminator.py │ ├── heads │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── hmr_head.py │ │ ├── hmr_hrnet_head.py │ │ ├── hybrik_head.py │ │ └── pare_head.py │ ├── losses │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── cross_entropy_loss.py │ │ ├── gan_loss.py │ │ ├── mse_loss.py │ │ ├── prior_loss.py │ │ ├── smooth_l1_loss.py │ │ └── utils.py │ ├── necks │ │ ├── __init__.py │ │ ├── builder.py │ │ └── temporal_encoder.py │ ├── registrants │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── smplify.py │ │ └── smplifyx.py │ └── utils │ │ ├── __init__.py │ │ ├── fits_dict.py │ │ ├── inverse_kinematics.py │ │ └── res_layer.py ├── utils │ ├── __init__.py │ ├── camera_utils.py │ ├── collect_env.py │ ├── demo_utils.py │ ├── dist_utils.py │ ├── ffmpeg_utils.py │ ├── geometry.py │ ├── keypoint_utils.py │ ├── logger.py │ ├── mesh_utils.py │ ├── misc.py │ ├── path_utils.py │ └── transforms.py └── version.py ├── requirements.txt ├── requirements ├── docs.txt ├── readthedocs.txt ├── runtime.txt └── tests.txt ├── resources ├── dance.gif ├── dance001.gif ├── dance3.gif └── mmhuman3d-logo.png ├── setup.cfg ├── setup.py ├── tests ├── test_apis │ └── test_inference.py ├── test_cameras │ ├── test_camera_converter.py │ ├── test_camera_parameter.py │ └── test_cameras.py ├── test_convention.py ├── test_data_converters.py ├── test_datasets │ ├── __init__.py │ ├── test_human_image_dataset.py │ └── test_pipelines.py ├── test_eval_hook.py ├── test_evaluation │ └── test_eval_utils.py ├── test_human_data.py ├── test_human_data_cache.py ├── test_hybrik_pipeline.py ├── test_models │ ├── test_architectures │ │ ├── __init__.py │ │ └── test_mesh_estimator.py │ ├── test_backbones │ │ ├── __init__.py │ │ ├── test_hrnet.py │ │ ├── test_resnet.py │ │ └── utils.py │ ├── test_body_models │ │ ├── test_smpl.py │ │ └── test_utils.py │ ├── test_heads │ │ ├── test_hmr_head.py │ │ ├── test_hybrik_forward.py │ │ └── test_pare_head.py │ ├── test_losses │ │ ├── test_loss.py │ │ ├── test_mse_loss.py │ │ ├── test_prior_losses.py │ │ └── test_smooth_l1_loss.py │ └── test_necks │ │ └── test_temporal_encoder.py ├── test_post_processing.py ├── test_smc_reader.py ├── test_smplify.py ├── test_transforms.py ├── test_utils │ ├── test_mesh_utils.py │ └── test_path_utils.py └── test_vis │ ├── test_ffmpeg.py │ ├── test_render.py │ ├── test_uv.py │ ├── test_vis_cameras.py │ ├── test_vis_kp2d.py │ ├── test_vis_kp3d.py │ └── test_vis_smpl.py └── tools ├── convert_datasets.py ├── dist_test.sh ├── dist_train.sh ├── get_flops.py ├── misc └── publish_model.py ├── slurm_smplify.sh ├── slurm_test.sh ├── slurm_train.sh ├── smplify.py ├── test.py ├── test_model_builders.py └── train.py /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to MMHuman3D 2 | 3 | All kinds of contributions are welcome, including but not limited to the following. 4 | 5 | - Fixes (typo, bugs) 6 | - New features and components 7 | 8 | ## Workflow 9 | 10 | 1. Fork and pull the latest mmhuman3d 11 | 1. Checkout a new branch with a meaningful name (do not use master branch for PRs) 12 | 1. Commit your changes 13 | 1. Create a PR 14 | 15 | ```{note} 16 | - If you plan to add some new features that involve large changes, it is encouraged to open an issue for discussion first. 17 | - If you are the author of some papers and would like to include your method to mmhuman3d, please contact us. We will much appreciate your contribution. 18 | ``` 19 | 20 | ## Code style 21 | 22 | ### Python 23 | 24 | We adopt [PEP8](https://www.python.org/dev/peps/pep-0008/) as the preferred code style. 25 | 26 | We use the following tools for linting and formatting: 27 | 28 | - [flake8](http://flake8.pycqa.org/en/latest/): linter 29 | - [yapf](https://github.com/google/yapf): formatter 30 | - [isort](https://github.com/timothycrosley/isort): sort imports 31 | 32 | Style configurations of yapf and isort can be found in [setup.cfg](../setup.cfg). 33 | 34 | We use [pre-commit hook](https://pre-commit.com/) that checks and formats for `flake8`, `yapf`, `isort`, `trailing whitespaces`, 35 | fixes `end-of-files`, sorts `requirments.txt` automatically on every commit. 36 | The config for a pre-commit hook is stored in [.pre-commit-config](../.pre-commit-config.yaml). 37 | 38 | After you clone the repository, you will need to install initialize pre-commit hook. 39 | 40 | ``` 41 | pip install -U pre-commit 42 | ``` 43 | 44 | From the repository folder 45 | 46 | ``` 47 | pre-commit install 48 | ``` 49 | 50 | If you are facing an issue when installing markdown lint, you may install ruby for markdown lint by 51 | referring to [this repo](https://github.com/innerlee/setup) by following the usage and taking [`zzruby.sh`](https://github.com/innerlee/setup/blob/master/zzruby.sh) 52 | 53 | or by the following steps 54 | 55 | ```shell 56 | # install rvm 57 | curl -L https://get.rvm.io | bash -s -- --autolibs=read-fail 58 | rvm autolibs disable 59 | # install ruby 60 | rvm install 2.7.1 61 | ``` 62 | 63 | After this on every commit check code linters and formatter will be enforced. 64 | 65 | > Before you create a PR, make sure that your code lints and is formatted by yapf. 66 | 67 | ### C++ and CUDA 68 | 69 | We follow the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). 70 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: build 5 | 6 | on: 7 | push: 8 | branches: 9 | - main 10 | paths-ignore: 11 | - 'README.md' 12 | - 'README_CN.md' 13 | - 'docs/**' 14 | - 'docs_zh-CN/**' 15 | 16 | pull_request: 17 | paths-ignore: 18 | - 'README.md' 19 | - 'README_CN.md' 20 | - 'docs/**' 21 | - 'docs_zh-CN/**' 22 | 23 | concurrency: 24 | group: ${{ github.workflow }}-${{ github.ref }} 25 | cancel-in-progress: true 26 | 27 | jobs: 28 | build_cuda101: 29 | runs-on: ubuntu-18.04 30 | strategy: 31 | matrix: 32 | python-version: [3.7] 33 | torch: [1.7.0, 1.8.0] 34 | include: 35 | - torch: 1.7.0 36 | torchvision: 0.8.1 37 | - torch: 1.8.0 38 | torchvision: 0.9.0 39 | defaults: 40 | run: 41 | shell: bash -l {0} 42 | 43 | steps: 44 | - uses: actions/checkout@v2 45 | - uses: conda-incubator/setup-miniconda@v2 46 | with: 47 | activate-environment: mmhuman3d_env 48 | python-version: ${{matrix.python-version}} 49 | auto-activate-base: false 50 | - name: Prepare test data 51 | run: | 52 | mkdir mmhuman3d_download 53 | cd mmhuman3d_download 54 | wget -O mmhuman3d.7z -q https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/mmhuman3d.7z 55 | 7za x mmhuman3d.7z 56 | ls -l 57 | cd .. 58 | cp -r mmhuman3d_download/mmhuman3d/* ./ 59 | rm -rf mmhuman3d_download 60 | - name: Upgrade pip 61 | run: pip install pip --upgrade 62 | - name: Install ffmpeg 63 | run: | 64 | conda install ffmpeg 65 | ffmpeg -version 66 | - name: Install PyTorch 67 | run: | 68 | conda install pytorch==${{matrix.torch}} torchvision==${{matrix.torchvision}} cudatoolkit=10.1 -c pytorch 69 | - name: Install pytorch3d 70 | run: | 71 | conda install -c fvcore -c iopath -c conda-forge fvcore iopath -y 72 | conda install pytorch3d -c pytorch3d 73 | - name: Install MMCV 74 | run: | 75 | pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html 76 | python -c 'import mmcv; print(mmcv.__version__)' 77 | - name: Install other dependencies 78 | run: pip install -r requirements.txt 79 | - name: Build and install 80 | run: rm -rf .eggs && pip install -e . 81 | - name: Run unittests and generate coverage report 82 | run: | 83 | coverage run --source mmhuman3d -m pytest tests/ 84 | coverage xml 85 | coverage report -m 86 | - name: Upload coverage to Codecov 87 | uses: codecov/codecov-action@v2 88 | with: 89 | files: ./coverage.xml 90 | flags: unittests 91 | env_vars: OS,PYTHON 92 | name: codecov-umbrella 93 | fail_ci_if_error: false 94 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [push, pull_request] 4 | 5 | concurrency: 6 | group: ${{ github.workflow }}-${{ github.ref }} 7 | cancel-in-progress: true 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-18.04 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 3.7 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.7 18 | - name: Install pre-commit hook 19 | run: | 20 | sudo apt-add-repository ppa:brightbox/ruby-ng -y 21 | sudo apt-get update 22 | sudo apt-get install -y ruby2.7 23 | pip install pre-commit 24 | pre-commit install 25 | - name: Linting 26 | run: pre-commit run --all-files 27 | - name: Check docstring coverage 28 | run: | 29 | pip install interrogate 30 | interrogate -vinmMI --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" -f 80 mmhuman3d/ 31 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: push 4 | 5 | jobs: 6 | build-n-publish: 7 | runs-on: ubuntu-latest 8 | if: startsWith(github.event.ref, 'refs/tags') 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Set up Python 3.7 12 | uses: actions/setup-python@v1 13 | with: 14 | python-version: 3.7 15 | - name: Build MMHUMAN3D 16 | run: | 17 | pip install wheel 18 | python setup.py sdist bdist_wheel 19 | - name: Publish distribution to PyPI 20 | run: | 21 | pip install twine 22 | twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }} 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | **/*.pyc 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | # custom 108 | data 109 | !mmhuman3d/data 110 | # data for pytest moved to http server 111 | # !tests/data 112 | .vscode 113 | .idea 114 | *.pkl 115 | *.pkl.json 116 | *.log.json 117 | work_dirs/ 118 | logs/ 119 | 120 | # Pytorch 121 | *.pth 122 | *.pt 123 | 124 | 125 | # Visualization 126 | *.mp4 127 | *.png 128 | *.gif 129 | *.jpg 130 | *.obj 131 | *.ply 132 | !demo/resources/* 133 | 134 | # Resources as exception 135 | !resources/* 136 | 137 | # Loaded/Saved data files 138 | *.npz 139 | *.npy 140 | *.pickle 141 | 142 | # MacOS 143 | *DS_Store* 144 | # git 145 | *.orig 146 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: .*/tests/data/ 2 | repos: 3 | - repo: https://gitlab.com/pycqa/flake8.git 4 | rev: 3.8.3 5 | hooks: 6 | - id: flake8 7 | - repo: https://github.com/asottile/seed-isort-config.git 8 | rev: v2.2.0 9 | hooks: 10 | - id: seed-isort-config 11 | - repo: https://github.com/timothycrosley/isort.git 12 | rev: 4.3.21 13 | hooks: 14 | - id: isort 15 | - repo: https://github.com/pre-commit/mirrors-yapf.git 16 | rev: v0.30.0 17 | hooks: 18 | - id: yapf 19 | - repo: https://github.com/pre-commit/pre-commit-hooks.git 20 | rev: v3.1.0 21 | hooks: 22 | - id: trailing-whitespace 23 | args: [--markdown-linebreak-ext=md] 24 | exclude: .*/tests/data/ 25 | - id: check-yaml 26 | - id: end-of-file-fixer 27 | - id: requirements-txt-fixer 28 | - id: double-quote-string-fixer 29 | - id: check-merge-conflict 30 | - id: fix-encoding-pragma 31 | args: ["--remove"] 32 | - id: mixed-line-ending 33 | args: ["--fix=lf"] 34 | - repo: https://github.com/markdownlint/markdownlint 35 | rev: v0.11.0 36 | hooks: 37 | - id: markdownlint 38 | args: ["-r", "~MD002,~MD013,~MD029,~MD033,~MD034", 39 | "-t", "allow_different_nesting"] 40 | - repo: https://github.com/codespell-project/codespell 41 | rev: v2.1.0 42 | hooks: 43 | - id: codespell 44 | - repo: https://github.com/myint/docformatter.git 45 | rev: v1.3.1 46 | hooks: 47 | - id: docformatter 48 | args: ["--in-place", "--wrap-descriptions", "79"] 49 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | formats: all 4 | 5 | python: 6 | version: 3.7 7 | install: 8 | - requirements: requirements/docs.txt 9 | - requirements: requirements/readthedocs.txt 10 | - requirements: requirements/runtime.txt 11 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - name: "MMHuman3D Contributors" 5 | title: "MMHuman3D: OpenMMLab 3D Human Parametric Model Toolbox and Benchmark" 6 | date-released: 2021-12-01 7 | url: "https://github.com/open-mmlab/mmhuman3d" 8 | license: Apache-2.0 9 | -------------------------------------------------------------------------------- /configs/_base_/datasets/pw3d.py: -------------------------------------------------------------------------------- 1 | # dataset settings 2 | dataset_type = 'PW3D' 3 | img_norm_cfg = dict( 4 | mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) 5 | data_keys = [ 6 | 'has_smpl', 'smpl_body_pose', 'smpl_global_orient', 'smpl_betas', 7 | 'smpl_transl' 8 | ] 9 | train_pipeline = [ 10 | dict(type='LoadImageFromFile'), 11 | dict(type='RandomResizedCrop', size=224), 12 | dict(type='RandomFlip', flip_prob=0.5, direction='horizontal'), 13 | dict(type='Normalize', **img_norm_cfg), 14 | dict(type='ImageToTensor', keys=['img']), 15 | dict(type='ToTensor', keys=data_keys), 16 | dict(type='Collect', keys=['img', *data_keys]) 17 | ] 18 | data = dict( 19 | samples_per_gpu=32, 20 | workers_per_gpu=2, 21 | train=dict(type=dataset_type, data_prefix='data', pipeline=train_pipeline)) 22 | -------------------------------------------------------------------------------- /configs/_base_/default_runtime.py: -------------------------------------------------------------------------------- 1 | # checkpoint saving 2 | checkpoint_config = dict(interval=1) 3 | # yapf:disable 4 | log_config = dict( 5 | interval=100, 6 | hooks=[ 7 | dict(type='TextLoggerHook'), 8 | # dict(type='TensorboardLoggerHook') 9 | ]) 10 | # yapf:enable 11 | 12 | dist_params = dict(backend='nccl') 13 | log_level = 'INFO' 14 | load_from = None 15 | resume_from = None 16 | workflow = [('train', 1)] 17 | -------------------------------------------------------------------------------- /configs/_base_/models/resnet50.py: -------------------------------------------------------------------------------- 1 | # model settings 2 | model = dict( 3 | type='ImageClassifier', 4 | backbone=dict( 5 | type='ResNet', 6 | depth=50, 7 | num_stages=4, 8 | out_indices=(3, ), 9 | style='pytorch'), 10 | neck=dict(type='GlobalAveragePooling'), 11 | head=dict( 12 | type='LinearClsHead', 13 | num_classes=1000, 14 | in_channels=2048, 15 | loss=dict(type='CrossEntropyLoss', loss_weight=1.0), 16 | topk=(1, 5), 17 | )) 18 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval10_q1.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 10, 2 | # window size of 1 + 10*1(where q=1). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=10, 7 | slide_window_q=1, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval10_q1.pth.tar?versionId=' 10 | 'CAEQOhiBgMChhsS9gxgiIDM5OGUwZGY0MTc4NTQ2M2NhZDEwMzU5MWUzMWNmZjY1') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval10_q2.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 10, 2 | # window size of 1 + 10*2(where q=2). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=10, 7 | slide_window_q=2, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval10_q2.pth.tar?versionId=' 10 | 'CAEQOhiBgICau8O9gxgiIDk1Y2Y0MzUxMmY0MDQzZThiYzhkMGJlMjc3ZDQ2NTQ2') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval10_q3.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 10, 2 | # window size of 1 + 10*3(where q=3). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=10, 7 | slide_window_q=3, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval10_q3.pth.tar?versionId=' 10 | 'CAEQOhiBgICIq8O9gxgiIDZiMjEzMjY3ODA4MTQwNGY5NTU3OWNkZjRjZjI2ZDFi') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval10_q4.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 10, 2 | # window size of 1 + 10*4(where q=4). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=10, 7 | slide_window_q=4, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval10_q4.pth.tar?versionId=' 10 | 'CAEQOhiBgICUq8O9gxgiIDJkZjUwYWJmNTRkNjQxMDE4YmUyNWMwNTcwNGQ4M2Ix') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval10_q5.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 10, 2 | # window size of 1 + 10*5(where q=5). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=10, 7 | slide_window_q=5, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval10_q5.pth.tar?versionId=' 10 | 'CAEQOhiBgMCN7MS9gxgiIDUwNGFhM2Y0MGI3MjRiYWQ5NzZjODMwMDk3ZjU1OTk3') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval5_q1.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 5, 2 | # window size of 1 + 5*1(where q=1). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=5, 7 | slide_window_q=1, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval5_q1.pth.tar?versionId=' 10 | 'CAEQOhiBgIDfocS9gxgiIDkxN2Y3OWQzZmJiMTQyMTM5NWZhZTYxYmI0MDlmMDBh') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval5_q2.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 5, 2 | # window size of 1 + 5*2(where q=2). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=5, 7 | slide_window_q=2, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval5_q2.pth.tar?versionId=' 10 | 'CAEQOhiBgIDgu8O9gxgiIDNjMDEyOWQ3NjRkODQ2YTI5MjUxYWU4NzhjOTc1YTRj') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval5_q3.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 5, 2 | # window size of 1 + 5*3(where q=3). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=5, 7 | slide_window_q=3, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval5_q3.pth.tar?versionId=' 10 | 'CAEQOhiBgIDJs8O9gxgiIDk1MDExMjI5Y2U1MDRmZjViMDBjOGU5YzY3OTRlNmE5') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval5_q4.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 5, 2 | # window size of 1 + 5*4(where q=4). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=5, 7 | slide_window_q=4, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval5_q4.pth.tar?versionId=' 10 | 'CAEQOhiBgMC.t8O9gxgiIGZjZWY3OTdhNGRjZjQyNjY5MGU5YzkxZTZjMWU1MTA2') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/deciwatch_interval5_q5.py: -------------------------------------------------------------------------------- 1 | # Config for DeciWatch trained on PW3D dataset with an interval of 5, 2 | # window size of 1 + 5*5(where q=5). 3 | # The model is trained only on SMPL pose parameters. 4 | speed_up_cfg = dict( 5 | type='deciwatch', 6 | interval=5, 7 | slide_window_q=5, 8 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 9 | 'mmhuman3d/models/deciwatch/deciwatch_interval5_q5.pth.tar?versionId=' 10 | 'CAEQOhiBgMCyq8O9gxgiIDRjMzViMjllNWRiNjRlMzA5ZjczYWIxOGU2OGFkYjdl') 11 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/guas1d.py: -------------------------------------------------------------------------------- 1 | smooth_cfg = dict(type='guas1d', window_size=11, sigma=4) 2 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/oneeuro.py: -------------------------------------------------------------------------------- 1 | smooth_cfg = dict(type='oneeuro', min_cutoff=0.004, beta=0.7) 2 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/savgol.py: -------------------------------------------------------------------------------- 1 | smooth_cfg = dict(type='savgol', window_size=11, polyorder=2) 2 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/smoothnet_windowsize16.py: -------------------------------------------------------------------------------- 1 | # Config for SmoothNet filter trained on SPIN data with a window size of 16. 2 | smooth_cfg = dict( 3 | type='smoothnet', 4 | window_size=16, 5 | output_size=16, 6 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 7 | 'mmhuman3d/models/smoothnet/smoothnet_windowsize16.pth.tar?versionId' 8 | '=CAEQPhiBgMC.s87shhgiIGM3ZTI1ZGY1Y2NhNDQ2YzRiNmEyOGZhY2VjYWFiN2Zi') 9 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/smoothnet_windowsize32.py: -------------------------------------------------------------------------------- 1 | # Config for SmoothNet filter trained on SPIN data with a window size of 32. 2 | smooth_cfg = dict( 3 | type='smoothnet', 4 | window_size=32, 5 | output_size=32, 6 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 7 | 'mmhuman3d/models/smoothnet/smoothnet_windowsize32.pth.tar?versionId' 8 | '=CAEQPhiBgIDf0s7shhgiIDhmYmM3YWQ0ZGI3NjRmZTc4NTk2NDE1MTA2MTUyMGRm') 9 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/smoothnet_windowsize64.py: -------------------------------------------------------------------------------- 1 | # Config for SmoothNet filter trained on SPIN data with a window size of 64. 2 | smooth_cfg = dict( 3 | type='smoothnet', 4 | window_size=64, 5 | output_size=64, 6 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 7 | 'mmhuman3d/models/smoothnet/smoothnet_windowsize64.pth.tar?versionId' 8 | '=CAEQPhiBgMCyw87shhgiIGEwODI4ZjdiYmFkYTQ0NzZiNDVkODk3MDBlYzE1Y2Rh') 9 | -------------------------------------------------------------------------------- /configs/_base_/post_processing/smoothnet_windowsize8.py: -------------------------------------------------------------------------------- 1 | # Config for SmoothNet filter trained on SPIN data with a window size of 8. 2 | smooth_cfg = dict( 3 | type='smoothnet', 4 | window_size=8, 5 | output_size=8, 6 | checkpoint='https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/' 7 | 'mmhuman3d/models/smoothnet/smoothnet_windowsize8.pth.tar?versionId' 8 | '=CAEQPhiBgMDo0s7shhgiIDgzNTRmNWM2ZWEzYTQyYzRhNzUwYTkzZWZkMmU5MWEw') 9 | -------------------------------------------------------------------------------- /configs/_base_/schedulers/hmr_bs256.py: -------------------------------------------------------------------------------- 1 | # optimizer 2 | optimizer = dict(type='Adam', lr=0.0002, weight_decay=0) 3 | optimizer_config = dict(grad_clip=None) 4 | # learning policy 5 | lr_config = dict(policy='step', step=[]) 6 | runner = dict(type='EpochBasedRunner', max_epochs=50) 7 | -------------------------------------------------------------------------------- /configs/augmentation/README.md: -------------------------------------------------------------------------------- 1 | # Augmentations 2 | 3 | ## Introduction 4 | 5 | We provide the config files for training on different augmentations on H36M and EFT-COCO: 6 | 1. Coarse dropout 7 | 2. Grid dropout 8 | 3. Photometric distortion 9 | 4. Random crop 10 | 5. Hard erasing 11 | 6. Soft erasing 12 | 7. Self-mixing 13 | 8. Synthetic occlusion 14 | 9. Synthetic occlusion over keypoints 15 | 16 | 17 | 18 | ## Results and Models 19 | 20 | We evaluate trained models on 3DPW. Values are MPJPE/PA-MPJPE. 21 | 22 | On H36M 23 | 24 | | Config | 3DPW | 25 | |:------:|:-------:| 26 | | [No augmentation](resnet50_hmr_h36m.py) | 105.42 / 60.11 | 27 | | [Coarse dropout](resnet50_hmr_coarse_dropout_h36m.py) | 112.46 / 64.55 | 28 | | [Grid dropout](resnet50_hmr_grid_dropout_h36m.py) | 112.67 / 63.36 | 29 | | [Photometric distortion](resnet50_hmr_photometric_distortion_h36m.py) | 107.13 / 62.13 | 30 | | [Random crop](resnet50_hmr_rand_crop_h36m.py) | 114.43 / 64.95 | 31 | | [Hard erasing](resnet50_hmr_rand_occ_h36m.py) | 112.34 / 67.53 | 32 | | [Soft erasing](resnet50_hmr_soft_erase_h36m.py) | 118.15 / 65.16 | 33 | | [Self-mixing](resnet50_hmr_self_mix_h36m.py) | 111.46 / 62.81 | 34 | | [Synthetic occlusion](resnet50_hmr_syn_occ_h36m.py) | 110.42 / 62.78 | 35 | | [Synthetic occlusion (kp)](resnet50_hmr_syn_occkp_h36m.py) | 100.75 / 59.13 | 36 | 37 | 38 | On EFT-COCO 39 | 40 | | Config | 3DPW | 41 | |:------:|:-------:| 42 | | [No augmentation](resnet50_hmr_eftcoco.py) | 105.42 / 60.11 | 43 | | [Coarse dropout](resnet50_hmr_coarse_dropout_eftcoco.py) | 112.46 / 64.55 | 44 | | [Grid dropout](resnet50_hmr_grid_dropout_eftcoco.py) | 112.67 / 63.36 | 45 | | [Photometric distortion](resnet50_hmr_photometric_distortion_eftcoco.py) | 107.13 / 62.13 | 46 | | [Random crop](resnet50_hmr_rand_crop_eftcoco.py) | 114.43 / 64.95 | 47 | | [Hard erasing](resnet50_hmr_rand_occ_eftcoco.py) | 112.34 / 67.53 | 48 | | [Soft erasing](resnet50_hmr_soft_erase_eftcoco.py) | 118.15 / 65.16 | 49 | | [Self-mixing](resnet50_hmr_self_mix_eftcoco.py) | 111.46 / 62.81 | 50 | | [Synthetic occlusion](resnet50_hmr_syn_occ_eftcoco.py) | 110.42 / 62.78 | 51 | | [Synthetic occlusion (kp)](resnet50_hmr_syn_occkp_eftcoco.py) | 100.75 / 59.13 | 52 | -------------------------------------------------------------------------------- /configs/backbones/metafile.yml: -------------------------------------------------------------------------------- 1 | Collections: 2 | - Name: HMR 3 | Metadata: 4 | Training Data: 5 | - COCO 6 | - Human3.6M 7 | - LSP-Extended 8 | - LSP 9 | - MPI-INF-3DHP 10 | - MPII 11 | - 3DPW 12 | Architecture: 13 | - ResNet 14 | - HMRHead 15 | Paper: 16 | URL: https://arxiv.org/pdf/1712.06584.pdf 17 | Title: "End-to-End Recovery of Human Shape and Pose" 18 | README: configs/hmr/README.md 19 | 20 | Models: 21 | - Name: resnet50_hmr_pw3d 22 | In Collection: HMR 23 | Config: configs/hmr/resnet50_hmr_pw3d.py 24 | Metadata: 25 | Epochs: 100 26 | Results: 27 | - Task: Human Pose and Shape Estimation 28 | Dataset: 3DPW 29 | Metrics: 30 | MPJPE: 112.34 31 | PA-MPJPE: 67.53 32 | Weights: https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hmr/resnet50_hmr_pw3d-04f40f58_20211201.pth?versionId=CAEQHhiBgMD6zJfR6xciIDE0ODQ3OGM2OWJjMTRlNmQ5Y2ZjMWZhMzRkOTFiZDFm 33 | -------------------------------------------------------------------------------- /configs/combine/README.md: -------------------------------------------------------------------------------- 1 | # Combine 2 | 3 | ## Introduction 4 | 5 | We provide the config files for training and pretrained models for inference on the optimal configurations. 6 | 7 | ## Notes 8 | 9 | Download the pretrained backbones from here: 10 | 1. [resnet50_coco_pose.pth](https://drive.google.com/file/d/1K1K1AUxL20Grq8rsyLJ6OdZE0oXY_CNY/view?usp=sharing) 11 | 2. [hrnet_coco_pose.pth](https://drive.google.com/file/d/1Dt1eRN_YnltaDBBe0JU8f6oSfhB2pxeh/view?usp=sharing) 12 | 3. [twins_svt_coco_pose.pth](https://drive.google.com/file/d/1Fcq_4G3ccM-xpmBK4M--Lu3xCXFrQ_ui/view?usp=sharing) 13 | 14 | 15 | 16 | Download the above resources and arrange them in the following file structure: 17 | 18 | ```text 19 | mmhuman3d 20 | ├── mmhuman3d 21 | ├── docs 22 | ├── tests 23 | ├── tools 24 | ├── configs 25 | └── data 26 | └── checkpoints 27 | ├── resnet50_coco_pose.pth 28 | ├── hrnet_coco_pose.pth 29 | └── twins_svt_coco_pose.pth 30 | 31 | ``` 32 | 33 | ## Results and Models 34 | 35 | We evaluate HMR on 3DPW. Values are PA-MPJPE. 36 | 37 | | Config | Dataset | Backbone | 3DPW | Download | 38 | |:------:|:-------:|:------:|:-------:|:------:| 39 | | [resnet50_hmr_mix1_coco_l1.py](resnet50_hmr_mix1_coco_l1.py) | H36M, MI, COCO, LSP, LSPET, MPII | ResNet-50 | 51.66 | [model](https://drive.google.com/file/d/1ifPYeQY8w-uJzl6yFejaTy_O86OmrjNH/view?usp=sharing) | 40 | | [hrnet_hmr_mix1_coco_l1.py](hrnet_hmr_mix1_coco_l1.py) | H36M, MI, COCO, LSP, LSPET, MPII | HRNet-W32 | 49.18 | [model](https://drive.google.com/file/d/1GV7T8ub5CCw_Tt0e-6SYlI_vimEl_ETy/view?usp=sharing) | 41 | | [twins_svt_hmr_mix1_coco_l1.py](twins_svt_hmr_mix1_coco_l1.py) | H36M, MI, COCO, LSP, LSPET, MPII | Twins-SVT | 48.77 | [model](https://drive.google.com/file/d/1UOLovoUUCvwXE14yoaJO9o-vpaeSvMPA/view?usp=sharing) | 42 | | [twins_svt_hmr_mix1_coco_l1_aug.py](twins_svt_hmr_mix1_coco_l1_aug.py) | H36M, MI, COCO, LSP, LSPET, MPII | Twins-SVT | 47.70 | [model](https://drive.google.com/file/d/1zk2JanLjkJ1W0TIAPhUaSZtVB-uayWFi/view?usp=sharing) | 43 | | [hrnet_hmr_mix4_coco_l1_aug.py](hrnet_hmr_mix4_coco_l1_aug.py) | EFT-[COCO, LSPET, MPII], H36M, SPIN-MI | HRNet-W32 | 47.68 | [model](https://drive.google.com/file/d/1NkijOkAKeNaDUx5XsF8nhL-MiboIcLRu/view?usp=sharing) | 44 | | [twins_svt_hmr_mix4_coco_l1.py](twins_svt_hmr_mix4_coco_l1.py) | EFT-[COCO, LSPET, MPII], H36M, SPIN-MI | Twins-SVT | 47.31 | [model](https://drive.google.com/file/d/1ostUnbf8MIVerlLo0AAP7As4V_gs-41k/view?usp=share_link) | 45 | | [hrnet_hmr_mix2_coco_l1_aug.py](hrnet_hmr_mix2_coco_l1_aug.py) | H36M, MI, EFT-COCO | HRNet-W32 | 48.08 | [model](https://drive.google.com/file/d/19poA9gmmuOlMbcREBGRF70EqyM00bDxi/view?usp=sharing) | 46 | | [twins_svt_hmr_mix2_coco_l1.py](twins_svt_hmr_mix2_coco_l1.py) | H36M, MI, EFT-COCO | Twins-SVT | 48.27 | [model](https://drive.google.com/file/d/1hnk8cMQ2QbA1jrZyHaRqqAN1jXdBo7ed/view?usp=sharing) | 47 | | [twins_svt_hmr_mix6_coco_l1.py](twins_svt_hmr_mix6_coco_l1.py) | H36M, MuCo, EFT-COCO | Twins-SVT | 47.92 | [model](https://drive.google.com/file/d/1ZQG0LCArhM3k-C1IQ-ZEc3cKAUbH8xFf/view?usp=share_link) | 48 | -------------------------------------------------------------------------------- /configs/hmr/README.md: -------------------------------------------------------------------------------- 1 | # HMR 2 | 3 | ## Introduction 4 | 5 | We provide the config files for HMR: [End-to-End Recovery of Human Shape and Pose](https://arxiv.org/pdf/1712.06584.pdf). 6 | 7 | ```BibTeX 8 | @inproceedings{HMR, 9 | author = {Angjoo Kanazawa and 10 | Michael J. Black and 11 | David W. Jacobs and 12 | Jitendra Malik}, 13 | title = {End-to-End Recovery of Human Shape and Pose}, 14 | booktitle = {CVPR}, 15 | year = {2018} 16 | } 17 | ``` 18 | 19 | ## Notes 20 | 21 | - [SMPL](https://smpl.is.tue.mpg.de/) v1.0 is used in our experiments. 22 | - [J_regressor_extra.npy](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/J_regressor_extra.npy?versionId=CAEQHhiBgIDD6c3V6xciIGIwZDEzYWI5NTBlOTRkODU4OTE1M2Y4YTI0NTVlZGM1) 23 | - [J_regressor_h36m.npy](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/J_regressor_h36m.npy?versionId=CAEQHhiBgIDE6c3V6xciIDdjYzE3MzQ4MmU4MzQyNmRiZDA5YTg2YTI5YWFkNjRi) 24 | - [smpl_mean_params.npz](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/smpl_mean_params.npz?versionId=CAEQHhiBgICN6M3V6xciIDU1MzUzNjZjZGNiOTQ3OWJiZTJmNThiZmY4NmMxMTM4) 25 | 26 | Download the above resources and arrange them in the following file structure: 27 | 28 | ```text 29 | mmhuman3d 30 | ├── mmhuman3d 31 | ├── docs 32 | ├── tests 33 | ├── tools 34 | ├── configs 35 | └── data 36 | ├── body_models 37 | │ ├── J_regressor_extra.npy 38 | │ ├── J_regressor_h36m.npy 39 | │ ├── smpl_mean_params.npz 40 | │ └── smpl 41 | │ ├── SMPL_FEMALE.pkl 42 | │ ├── SMPL_MALE.pkl 43 | │ └── SMPL_NEUTRAL.pkl 44 | ├── preprocessed_datasets 45 | │ ├── cmu_mosh.npz 46 | │ ├── coco_2014_train.npz 47 | │ ├── h36m_mosh_train.npz 48 | │ ├── lspet_train.npz 49 | │ ├── lsp_train.npz 50 | │ ├── mpi_inf_3dhp_train.npz 51 | │ ├── mpii_train.npz 52 | │ └── pw3d_test.npz 53 | └── datasets 54 | ├── coco 55 | ├── h36m 56 | ├── lspet 57 | ├── lsp 58 | ├── mpi_inf_3dhp 59 | ├── mpii 60 | └── pw3d 61 | 62 | ``` 63 | 64 | ## Results and Models 65 | 66 | We evaluate HMR on 3DPW. Values are MPJPE/PA-MPJPE. 67 | 68 | | Config | 3DPW | Download | 69 | |:------:|:-------:|:------:| 70 | | [resnet50_hmr_pw3d.py](resnet50_hmr_pw3d.py) | 112.34 / 67.53 | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hmr/resnet50_hmr_pw3d-04f40f58_20211201.pth?versionId=CAEQHhiBgMD6zJfR6xciIDE0ODQ3OGM2OWJjMTRlNmQ5Y2ZjMWZhMzRkOTFiZDFm) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hmr/20211128_053633.log?versionId=CAEQHhiBgMDbzZfR6xciIGZkZjM2NWEwN2ExYzQ1NGViNzg2ODA0YTAxMmU4M2Vi) | 71 | -------------------------------------------------------------------------------- /configs/hmr/metafile.yml: -------------------------------------------------------------------------------- 1 | Collections: 2 | - Name: HMR 3 | Metadata: 4 | Training Data: 5 | - COCO 6 | - Human3.6M 7 | - LSP-Extended 8 | - LSP 9 | - MPI-INF-3DHP 10 | - MPII 11 | - 3DPW 12 | Architecture: 13 | - ResNet 14 | - HMRHead 15 | Paper: 16 | URL: https://arxiv.org/pdf/1712.06584.pdf 17 | Title: "End-to-End Recovery of Human Shape and Pose" 18 | README: configs/hmr/README.md 19 | 20 | Models: 21 | - Name: resnet50_hmr_pw3d 22 | In Collection: HMR 23 | Config: configs/hmr/resnet50_hmr_pw3d.py 24 | Metadata: 25 | Epochs: 100 26 | Results: 27 | - Task: Human Pose and Shape Estimation 28 | Dataset: 3DPW 29 | Metrics: 30 | MPJPE: 112.34 31 | PA-MPJPE: 67.53 32 | Weights: https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hmr/resnet50_hmr_pw3d-04f40f58_20211201.pth?versionId=CAEQHhiBgMD6zJfR6xciIDE0ODQ3OGM2OWJjMTRlNmQ5Y2ZjMWZhMzRkOTFiZDFm 33 | -------------------------------------------------------------------------------- /configs/hybrik/README.md: -------------------------------------------------------------------------------- 1 | # HybrIK 2 | 3 | ## Introduction 4 | 5 | We provide the config files for HybrIK: [HybrIK: A Hybrid Analytical-Neural Inverse Kinematics Solution for 3D Human Pose and Shape Estimation](https://arxiv.org/pdf/2011.14672.pdf). 6 | 7 | ```BibTeX 8 | @inproceedings{HybrIK, 9 | author = {Jiefeng Li and 10 | Chao Xu and 11 | Zhicun Chen and 12 | Siyuan Bian and 13 | Lixin Yang and 14 | Cewu Lu}, 15 | title = {{HybrIK}: A Hybrid Analytical-Neural Inverse Kinematics Solution for 3D Human Pose and Shape Estimation}, 16 | booktitle = {CVPR}, 17 | year = {2021} 18 | } 19 | ``` 20 | 21 | ## Notes 22 | 23 | - [SMPL](https://smpl.is.tue.mpg.de/) v1.0 is used in our experiments. 24 | - [J_regressor_h36m.npy](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/J_regressor_h36m.npy?versionId=CAEQHhiBgIDE6c3V6xciIDdjYzE3MzQ4MmU4MzQyNmRiZDA5YTg2YTI5YWFkNjRi) 25 | - [smpl_mean_beta.npz](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hybrik/h36m_mean_beta.npy?versionId=CAEQHhiBgMDnt_DV6xciIGM5MzM0MGI1NzBmYjRkNDU5MzUxMjdkM2Y1ZWRiZWM2) 26 | - [basicModel_neutral_lbs_10_207_0_v1.0.0.pkl](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hybrik/basicModel_neutral_lbs_10_207_0_v1.0.0.pkl?versionId=CAEQHhiBgIC_v.zV6xciIDkwMDE4M2NjZTRkMjRmMWRiNTY3MWQ5YjQ0YzllNDYz) 27 | 28 | Download the above resources and arrange them in the following file structure: 29 | 30 | ```text 31 | mmhuman3d 32 | ├── mmhuman3d 33 | ├── docs 34 | ├── tests 35 | ├── tools 36 | ├── configs 37 | └── data 38 | ├── body_models 39 | │ ├── basicModel_neutral_lbs_10_207_0_v1.0.0.pkl 40 | │ ├── h36m_mean_beta.npy 41 | │ ├── J_regressor_h36m.npy 42 | │ └── smpl 43 | │ ├── SMPL_FEMALE.pkl 44 | │ ├── SMPL_MALE.pkl 45 | │ └── SMPL_NEUTRAL.pkl 46 | ├── preprocessed_datasets 47 | │ ├── hybrik_coco_2017_train.npz 48 | │ ├── hybrik_h36m_train.npz 49 | │ └── hybrik_mpi_inf_3dhp_train.npz 50 | └── datasets 51 | ├── coco 52 | ├── h36m 53 | └── mpi_inf_3dhp 54 | ``` 55 | 56 | 57 | ## Results and Models 58 | 59 | We evaluate HybrIK on 3DPW. Values are MPJPE/PA-MPJPE. 60 | 61 | | Config | 3DPW | Download | 62 | |:------:|:-------:|:------:| 63 | | [resnet34_hybrik_mixed.py](resnet34_hybrik_mixed.py) | 81.08 / 49.02 | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hybrik/resnet34_hybrik_mixed-a61b3c9c_20220211.pth?versionId=CAEQKhiBgMDx0.Kd9xciIDA2NWFlMGVmNjNkMDQyYzE4NTFmMGJiYjczZWZmM2Rk) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hybrik/20220121_170847.log?versionId=CAEQKhiBgICMgOyb9xciIDZkMTMyODYzMDc4NjRhZTliOThiM2JlMDE1MzhhYTBm) | 64 | -------------------------------------------------------------------------------- /configs/hybrik/metafile.yml: -------------------------------------------------------------------------------- 1 | Collections: 2 | - Name: HybrIK 3 | Metadata: 4 | Training Data: 5 | - COCO 6 | - Human3.6M 7 | - MPI-INF-3DHP 8 | Architecture: 9 | - ResNet 10 | - HybrIKHead 11 | Paper: 12 | URL: https://arxiv.org/pdf/2011.14672.pdf 13 | Title: "HybrIK: A Hybrid Analytical-Neural Inverse Kinematics Solution for 3D Human Pose and Shape Estimation" 14 | README: configs/hybrik/README.md 15 | 16 | Models: 17 | - Name: resnet34_hybrik_mixed 18 | In Collection: HybrIK 19 | Config: configs/hybrik/resnet34_hybrik_mixed.py 20 | Metadata: 21 | Epochs: 400 22 | Results: 23 | - Task: Human Pose and Shape Estimation 24 | Dataset: 3DPW 25 | Metrics: 26 | MPJPE: 81.08 27 | PA-MPJPE: 49.02 28 | Weights: https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/hybrik/resnet34_hybrik_mixed-a61b3c9c_20220211.pth?versionId=CAEQKhiBgMDx0.Kd9xciIDA2NWFlMGVmNjNkMDQyYzE4NTFmMGJiYjczZWZmM2Rk 29 | -------------------------------------------------------------------------------- /configs/mixed-l1/README.md: -------------------------------------------------------------------------------- 1 | # Mixed datasets 2 | 3 | ## Introduction 4 | 5 | We provide the config files for training on different losses for different dataset combination: 6 | 1. Mix 1: H36M, MI, COCO 7 | 2. Mix 2: H36M, MI, EFT-COCO 8 | 3. Mix 5: H36M, MI, COCO, LSP, LSPET, MPII 9 | 4. Mix 6: EFT-[COCO, MPII, LSPET], SPIN-MI, H36M 10 | 5. Mix 7: EFT-[COCO, MPII, LSPET], MuCo, H36M, PROX 11 | 6. Mix 8: EFT-[COCO, PT, LSPET], MI, H36M 12 | 7. Mix 11: EFT-[COCO, MPII, LSPET], MuCo, H36M 13 | 14 | 15 | ## Results and Models 16 | 17 | We evaluate trained models on 3DPW. Values are MPJPE/PA-MPJPE. 18 | 19 | | Mixes | Datasets Config | 3DPW (w/ L1) | 3DPW (w/o L1) | 20 | |:------:|:------:|:-------:|:-------:|:-------:| 21 | | Mix 1 | H36M, MI, COCO | [resnet50_hmr_mix1.py](resnet50_hmr_mix1.py) | 57.01 | 66.14 | 22 | | Mix 2 | H36M, MI, EFT-COCO |[resnet50_hmr_mix2.py](resnet50_hmr_mix2.py) | 55.25 | 55.98 | 23 | | Mix 5 | H36M, MI, COCO, LSP, LSPET, MPII |[resnet50_hmr_mix5.py](resnet50_hmr_mix5.py) | 58.20 | 64.55 | 24 | | Mix 6 | EFT-[COCO, MPII, LSPET], SPIN-MI, H36M |[resnet50_hmr_mix6.py](resnet50_hmr_mix6.py) | 53.62 | 55.47 | 25 | | Mix 7 | EFT-[COCO, MPII, LSPET], MuCo, H36M, PROX |[resnet50_hmr_mix7.py](resnet50_hmr_mix7.py) | 52.93 | 53.44 | 26 | | Mix 8 | EFT-[COCO, PT, LSPET], MI, H36M |[resnet50_hmr_mix8.py](resnet50_hmr_mix8.py) | 53.43 | 55.97 | 27 | | Mix 11 | EFT-[COCO, MPII, LSPET], MuCo, H36M |[resnet50_hmr_mix11.py](resnet50_hmr_mix11.py) | 53.17 | 52.54 | 28 | -------------------------------------------------------------------------------- /configs/mixed/README.md: -------------------------------------------------------------------------------- 1 | # Mixed datasets 2 | 3 | ## Introduction 4 | 5 | We provide the config files for training on different dataset combination: 6 | 1. Mix 1: H36M, MI, COCO 7 | 2. Mix 2: H36M, MI, EFT-COCO 8 | 3. Mix 3: H36M, MI, EFT-COCO, MPII 9 | 4. Mix 4: H36M, MuCo, EFT-COCO 10 | 5. Mix 5: H36M, MI, COCO, LSP, LSPET, MPII 11 | 6. Mix 6: EFT-[COCO, MPII, LSPET], SPIN-MI, H36M 12 | 7. Mix 7: EFT-[COCO, MPII, LSPET], MuCo, H36M, PROX 13 | 8. Mix 8: EFT-[COCO, PT, LSPET], MI, H36M 14 | 9. Mix 9: EFT-[COCO, PT, LSPET, OCH], MI, H36M 15 | 10. Mix 10: PROX, MuCo, EFT-[COCO, PT, LSPET, OCH], UP-3D, MTP, Crowdpose 16 | 11. Mix 11: EFT-[COCO, MPII, LSPET], MuCo, H36M 17 | 18 | 19 | 20 | ## Results and Models 21 | 22 | We evaluate trained models on 3DPW. Values are MPJPE/PA-MPJPE. 23 | 24 | | Mixes | Datasets Config | 3DPW | 25 | |:------:|:------:|:-------:|:-------:| 26 | | Mix 1 | H36M, MI, COCO | [resnet50_hmr_mix1.py](resnet50_hmr_mix1.py) | 66.14 | 27 | | Mix 2 | H36M, MI, EFT-COCO |[resnet50_hmr_mix2.py](resnet50_hmr_mix2.py) | 55.98 | 28 | | Mix 3 | H36M, MI, EFT-COCO, MPII |[resnet50_hmr_mix3.py](resnet50_hmr_mix3.py) | 56.12 | 29 | | Mix 4 | H36M, MuCo, EFT-COCO |[resnet50_hmr_mix4.py](resnet50_hmr_mix4.py) | 53.90 | 30 | | Mix 5 | H36M, MI, COCO, LSP, LSPET, MPII |[resnet50_hmr_mix5.py](resnet50_hmr_mix5.py) | 64.55 | 31 | | Mix 6 | EFT-[COCO, MPII, LSPET], SPIN-MI, H36M |[resnet50_hmr_mix6.py](resnet50_hmr_mix6.py) | 55.47 | 32 | | Mix 7 | EFT-[COCO, MPII, LSPET], MuCo, H36M, PROX |[resnet50_hmr_mix7.py](resnet50_hmr_mix7.py) | 53.44 | 33 | | Mix 8 | EFT-[COCO, PT, LSPET], MI, H36M |[resnet50_hmr_mix8.py](resnet50_hmr_mix8.py) | 55.97 | 34 | | Mix 9 | EFT-[COCO, PT, LSPET, OCH], MI, H36M |[resnet50_hmr_mix9.py](resnet50_hmr_mix9.py) | 55.59 | 35 | | Mix 10 | PROX, MuCo, EFT-[COCO, PT, LSPET, OCH], UP-3D, MTP, Crowdpose |[resnet50_hmr_mix10.py](resnet50_hmr_mix10.py) | 57.84 | 36 | | Mix 11 | EFT-[COCO, MPII, LSPET], MuCo, H36M |[resnet50_hmr_mix11.py](resnet50_hmr_mix11.py) | 52.54 | 37 | -------------------------------------------------------------------------------- /configs/pare/metafile.yml: -------------------------------------------------------------------------------- 1 | Collections: 2 | - Name: SPIN 3 | Metadata: 4 | Training Data: 5 | - COCO 6 | - Human3.6M 7 | - LSP-Extended 8 | - LSP 9 | - MPI-INF-3DHP 10 | - MPII 11 | - 3DPW 12 | Architecture: 13 | - PoseHighResolutionNet 14 | - PareHead 15 | Paper: 16 | URL: https://arxiv.org/abs/2104.08527 17 | Title: "PARE: Part Attention Regressor for 3D Human Body Estimation" 18 | README: configs/pare/README.md 19 | 20 | Models: 21 | - Name: hrnet_w32_conv_pare_mix 22 | In Collection: PARE 23 | Config: configs/spin/hrnet_w32_conv_pare_mix.py 24 | Metadata: 25 | Epochs: 50 26 | Results: 27 | - Task: Human Pose and Shape Estimation 28 | Dataset: 3DPW 29 | Metrics: 30 | MPJPE: 81.74 31 | PA-MPJPE: 48.69 32 | Weights: https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/pare/with_mosh/hrnet_w32_conv_pare_mosh.pth?versionId=CAEQOhiBgIDooeHSgxgiIDkwYzViMTUyNjM1MjQ3ZDNiNzNjMjJlOGFlNjgxYjlh 33 | -------------------------------------------------------------------------------- /configs/pretrained-backbones/README.md: -------------------------------------------------------------------------------- 1 | # Pretrained backbones 2 | 3 | ## Introduction 4 | 5 | Download the pretrained backbones from here: 6 | 1. [resnet50_mpii_pose.pth](https://drive.google.com/file/d/1XEyYR88S9LnAh_bztybtWhBix6vQgg5k/view?usp=sharing) 7 | 2. [resnet50_coco_pose.pth](https://drive.google.com/file/d/1K1K1AUxL20Grq8rsyLJ6OdZE0oXY_CNY/view?usp=sharing) 8 | 3. [hrnet_imagenet.pth](https://drive.google.com/file/d/1snrLDyHgpTXximcJX6EqX7M8okQHVjH7/view?usp=sharing) 9 | 4. [hrnet_mpii_pose.pth](https://drive.google.com/file/d/1JaKYRbP-hKKZCwAqlvDQ5hIhueBYGf0i/view?usp=sharing) 10 | 5. [hrnet_coco_pose.pth](https://drive.google.com/file/d/1Dt1eRN_YnltaDBBe0JU8f6oSfhB2pxeh/view?usp=sharing) 11 | 6. [twins_svt_imagenet.pth](https://drive.google.com/file/d/155neTTkGZ_jtNbRS-OYwJWwsbx59jtzK/view?usp=sharing) 12 | 7. [twins_svt_mpii_pose.pth](https://drive.google.com/file/d/1RItsH4dDQmsk6Xc9wyaKyW3wYWFKpN6p/view?usp=sharing) 13 | 8. [twins_svt_coco_pose.pth](https://drive.google.com/file/d/1Fcq_4G3ccM-xpmBK4M--Lu3xCXFrQ_ui/view?usp=sharing) 14 | 15 | 16 | 17 | Download the above resources and arrange them in the following file structure: 18 | 19 | ```text 20 | mmhuman3d 21 | ├── mmhuman3d 22 | ├── docs 23 | ├── tests 24 | ├── tools 25 | ├── configs 26 | └── data 27 | └── checkpoints 28 | ├── resnet50_mpii_pose.pth 29 | ├── resnet50_coco_pose.pth 30 | ├── hrnet_imagenet.pth 31 | ├── hrnet_mpii_pose.pth 32 | ├── hrnet_coco_pose.pth 33 | ├── twins_svt_imagenet.pth 34 | ├── twins_svt_mpii_pose.pth 35 | └── twins_svt_coco_pose.pth 36 | 37 | ``` 38 | 39 | ## Results and Models 40 | 41 | We evaluate trained models on 3DPW. Values are MPJPE/PA-MPJPE. 42 | 43 | | Backbones | Weights | Config | 3DPW | 44 | |:------:|:------:|:-------:|:-------:| 45 | |ResNet-50| ImageNet | [resnet50_hmr_imagenet.py](resnet50_hmr_imagenet.py) | 64.55 | 46 | |ResNet-50| MPII | [resnet50_hmr_mpii.py](resnet50_hmr_pw3d_mpii.py) | 60.60 | 47 | |ResNet-50| COCO | [resnet50_hmr_coco.py](resnet50_hmr_coco.py) | 57.26 | 48 | |HRNet-W32| ImageNet | [hrnet_hmr_imagenet.py](hrnet_hmr_imagenet.py) | 64.27 | 49 | |HRNet-W32| MPII | [hrnet_hmr_mpii.py](hrnet_hmr_mpii.py) | 55.93 | 50 | |HRNet-W32| COCO | [hrnet_hmr_coco.py](hrnet_hmr_coco.py) | 54.47 | 51 | |Twins-SVT| ImageNet | [twins_svt_hmr_imagenet.py](twins_svt_hmr_imagenet.py) | 60.11 | 52 | |Twins-SVT| MPII | [twins_svt_hmr_mpii.py](twins_svt_hmr_mpii.py) | 56.80 | 53 | |Twins-SVT| COCO | [twins_svt_hmr_coco.py](twins_svt_hmr_coco.py) | 52.61 | 54 | -------------------------------------------------------------------------------- /configs/smplify/smplify.py: -------------------------------------------------------------------------------- 1 | type = 'SMPLify' 2 | 3 | body_model = dict( 4 | type='SMPL', 5 | gender='neutral', 6 | num_betas=10, 7 | keypoint_src='smpl_45', 8 | keypoint_dst='smpl_45', 9 | model_path='data/body_models/smpl', 10 | batch_size=1) 11 | 12 | stages = [ 13 | # stage 1 14 | dict( 15 | num_iter=20, 16 | fit_global_orient=True, 17 | fit_transl=True, 18 | fit_body_pose=False, 19 | fit_betas=False, 20 | joint_weights=dict( 21 | body_weight=5.0, 22 | use_shoulder_hip_only=True, 23 | )), 24 | # stage 2 25 | dict( 26 | num_iter=10, 27 | fit_global_orient=True, 28 | fit_transl=True, 29 | fit_body_pose=True, 30 | fit_betas=True, 31 | joint_weights=dict(body_weight=5.0, use_shoulder_hip_only=False)) 32 | ] 33 | 34 | optimizer = dict( 35 | type='LBFGS', max_iter=20, lr=1e-2, line_search_fn='strong_wolfe') 36 | 37 | keypoints2d_loss = dict( 38 | type='KeypointMSELoss', loss_weight=1.0, reduction='sum', sigma=100) 39 | 40 | keypoints3d_loss = dict( 41 | type='KeypointMSELoss', loss_weight=10, reduction='sum', sigma=100) 42 | 43 | shape_prior_loss = dict(type='ShapePriorLoss', loss_weight=1, reduction='sum') 44 | 45 | joint_prior_loss = dict( 46 | type='JointPriorLoss', 47 | loss_weight=20, 48 | reduction='sum', 49 | smooth_spine=True, 50 | smooth_spine_loss_weight=20, 51 | use_full_body=True) 52 | 53 | smooth_loss = dict(type='SmoothJointLoss', loss_weight=0, reduction='sum') 54 | 55 | pose_prior_loss = dict( 56 | type='MaxMixturePrior', 57 | prior_folder='data', 58 | num_gaussians=8, 59 | loss_weight=4.78**2, 60 | reduction='sum') 61 | 62 | ignore_keypoints = [ 63 | 'neck_openpose', 'right_hip_openpose', 'left_hip_openpose', 64 | 'right_hip_extra', 'left_hip_extra' 65 | ] 66 | 67 | camera = dict( 68 | type='PerspectiveCameras', 69 | convention='opencv', 70 | in_ndc=False, 71 | focal_length=5000, 72 | image_size=(224, 224), 73 | principal_point=(112, 112)) 74 | -------------------------------------------------------------------------------- /configs/smplify/smplify3d.py: -------------------------------------------------------------------------------- 1 | type = 'SMPLify' 2 | verbose = True 3 | 4 | body_model = dict( 5 | type='SMPL', 6 | gender='neutral', 7 | num_betas=10, 8 | keypoint_src='smpl_45', 9 | # keypoint_dst='smpl_45', 10 | keypoint_dst='smpl', 11 | model_path='data/body_models/smpl', 12 | batch_size=1) 13 | 14 | stages = [ 15 | # stage 0: optimize `betas` 16 | dict( 17 | num_iter=10, 18 | ftol=1e-4, 19 | fit_global_orient=False, 20 | fit_transl=False, 21 | fit_body_pose=False, 22 | fit_betas=True, 23 | keypoints3d_weight=0.0, 24 | pose_reg_weight=0.0, 25 | smooth_loss_weight=0.0, 26 | limb_length_weight=1.0, 27 | shape_prior_weight=5e-3, 28 | ), 29 | # stage 1: optimize `global_orient` and `transl` 30 | dict( 31 | num_iter=50, 32 | ftol=1e-4, 33 | fit_global_orient=True, 34 | fit_transl=True, 35 | fit_body_pose=False, 36 | fit_betas=False, 37 | keypoints3d_weight=1.0, 38 | pose_reg_weight=0.0, 39 | smooth_loss_weight=0.0, 40 | limb_length_weight=0.0, 41 | shape_prior_weight=0.0, 42 | joint_weights=dict( 43 | body_weight=5.0, 44 | use_shoulder_hip_only=True, 45 | )), 46 | # stage 2: optimize `global_orient`, `transl` and `body_pose` 47 | dict( 48 | num_iter=120, 49 | ftol=1e-4, 50 | fit_global_orient=True, 51 | fit_transl=True, 52 | fit_body_pose=True, 53 | fit_betas=False, 54 | keypoints3d_weight=10.0, 55 | pose_reg_weight=0.001, 56 | smooth_loss_weight=1.0, 57 | limb_length_weight=0.0, 58 | shape_prior_weight=0.0, 59 | joint_weights=dict(body_weight=1.0, use_shoulder_hip_only=False)) 60 | # joint_weights=dict(body_weight=5.0, use_shoulder_hip_only=False)) 61 | ] 62 | 63 | optimizer = dict( 64 | type='LBFGS', max_iter=20, lr=1.0, line_search_fn='strong_wolfe') 65 | 66 | keypoints3d_loss = dict( 67 | type='KeypointMSELoss', loss_weight=10, reduction='sum', sigma=100) 68 | 69 | shape_prior_loss = dict( 70 | type='ShapePriorLoss', loss_weight=5e-3, reduction='mean') 71 | 72 | limb_length_loss = dict( 73 | type='LimbLengthLoss', convention='smpl', loss_weight=1., reduction='mean') 74 | 75 | pose_reg_loss = dict(type='PoseRegLoss', loss_weight=0.001, reduction='mean') 76 | 77 | smooth_loss = dict( 78 | type='SmoothJointLoss', loss_weight=1.0, reduction='mean', loss_func='L2') 79 | 80 | ignore_keypoints = [ 81 | 'neck_openpose', 'right_hip_openpose', 'left_hip_openpose', 82 | 'right_hip_extra', 'left_hip_extra' 83 | ] 84 | -------------------------------------------------------------------------------- /configs/smplify/smplifyx.py: -------------------------------------------------------------------------------- 1 | type = 'SMPLifyX' 2 | 3 | body_model = dict( 4 | type='SMPLX', 5 | gender='neutral', 6 | num_betas=10, 7 | use_face_contour=True, 8 | keypoint_src='smplx', 9 | keypoint_dst='smplx', 10 | model_path='data/body_models/smplx', 11 | batch_size=1) 12 | 13 | stages = [ 14 | # stage 1 15 | dict( 16 | num_iter=10, 17 | fit_global_orient=True, 18 | fit_transl=True, 19 | fit_body_pose=False, 20 | fit_betas=False, 21 | fit_left_hand_pose=False, 22 | fit_right_hand_pose=False, 23 | fit_expression=False, 24 | fit_jaw_pose=False, 25 | fit_leye_pose=False, 26 | fit_reye_pose=False, 27 | joint_weights=dict( 28 | body_weight=5.0, 29 | use_shoulder_hip_only=True, 30 | hand_weight=0.0, 31 | face_weight=0.0)), 32 | # stage 2 33 | dict( 34 | num_iter=5, 35 | fit_global_orient=True, 36 | fit_transl=True, 37 | fit_body_pose=True, 38 | fit_betas=True, 39 | fit_left_hand_pose=False, 40 | fit_right_hand_pose=False, 41 | fit_expression=False, 42 | fit_jaw_pose=False, 43 | fit_leye_pose=False, 44 | fit_reye_pose=False, 45 | joint_weights=dict( 46 | body_weight=5.0, 47 | use_shoulder_hip_only=False, 48 | hand_weight=0.0, 49 | face_weight=0.0)), 50 | # stage 3 51 | dict( 52 | num_iter=3, 53 | fit_global_orient=True, 54 | fit_transl=True, 55 | fit_body_pose=True, 56 | fit_betas=True, 57 | fit_left_hand_pose=True, 58 | fit_right_hand_pose=True, 59 | fit_expression=False, 60 | fit_jaw_pose=False, 61 | fit_leye_pose=False, 62 | fit_reye_pose=False, 63 | joint_weights=dict( 64 | body_weight=10.0, 65 | use_shoulder_hip_only=False, 66 | hand_weight=1.0, 67 | face_weight=1.0)) 68 | ] 69 | 70 | optimizer = dict( 71 | type='LBFGS', max_iter=20, lr=1e-2, line_search_fn='strong_wolfe') 72 | 73 | keypoints2d_loss = dict( 74 | type='KeypointMSELoss', loss_weight=1.0, reduction='sum', sigma=100) 75 | 76 | keypoints3d_loss = dict( 77 | type='KeypointMSELoss', loss_weight=10, reduction='sum', sigma=100) 78 | 79 | shape_prior_loss = dict(type='ShapePriorLoss', loss_weight=1, reduction='sum') 80 | 81 | joint_prior_loss = dict( 82 | type='JointPriorLoss', 83 | loss_weight=20, 84 | reduction='sum', 85 | smooth_spine=True, 86 | smooth_spine_loss_weight=20, 87 | use_full_body=True) 88 | 89 | smooth_loss = dict(type='SmoothJointLoss', loss_weight=0, reduction='sum') 90 | 91 | camera = dict( 92 | type='PerspectiveCameras', 93 | convention='opencv', 94 | in_ndc=False, 95 | focal_length=5000, 96 | image_size=(224, 224), 97 | principal_point=(112, 112)) 98 | -------------------------------------------------------------------------------- /configs/spin/metafile.yml: -------------------------------------------------------------------------------- 1 | Collections: 2 | - Name: SPIN 3 | Metadata: 4 | Training Data: 5 | - COCO 6 | - Human3.6M 7 | - LSP-Extended 8 | - LSP 9 | - MPI-INF-3DHP 10 | - MPII 11 | - 3DPW 12 | Architecture: 13 | - ResNet 14 | - HMRHead 15 | Paper: 16 | URL: https://arxiv.org/pdf/1909.12828.pdf 17 | Title: "Learning to Reconstruct 3D Human Pose and Shape via Model-fitting in the Loop" 18 | README: configs/spin/README.md 19 | 20 | Models: 21 | - Name: resnet50_spin_pw3d 22 | In Collection: SPIN 23 | Config: configs/spin/resnet50_spin_pw3d.py 24 | Metadata: 25 | Epochs: 10 26 | Results: 27 | - Task: Human Pose and Shape Estimation 28 | Dataset: 3DPW 29 | Metrics: 30 | MPJPE: 96.06 31 | PA-MPJPE: 59.06 32 | Weights: https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/spin/resnet50_spin_pw3d-e1857270_20211201.pth?versionId=CAEQHhiBgMDyvYnS6xciIDZhNTg4NmM4OGE4MTQ0ODRhY2JlY2JmZDI4ZWQ0ZmU3 33 | -------------------------------------------------------------------------------- /configs/vibe/README.md: -------------------------------------------------------------------------------- 1 | # VIBE 2 | 3 | ## Introduction 4 | 5 | We provide the config files for VIBE: [VIBE: Video Inference for Human Body Pose and Shape Estimation](https://arxiv.org/pdf/1912.05656.pdf). 6 | 7 | ```BibTeX 8 | @inproceedings{VIBE, 9 | author = {Muhammed Kocabas and 10 | Nikos Athanasiou and 11 | Michael J. Black}, 12 | title = {{VIBE}: Video Inference for Human Body Pose and Shape Estimation}, 13 | booktitle = {CVPR}, 14 | year = {2020} 15 | } 16 | ``` 17 | 18 | ## Notes 19 | 20 | - [SMPL](https://smpl.is.tue.mpg.de/) v1.0 is used in our experiments. 21 | - [J_regressor_extra.npy](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/J_regressor_extra.npy?versionId=CAEQHhiBgIDD6c3V6xciIGIwZDEzYWI5NTBlOTRkODU4OTE1M2Y4YTI0NTVlZGM1) 22 | - [J_regressor_h36m.npy](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/J_regressor_h36m.npy?versionId=CAEQHhiBgIDE6c3V6xciIDdjYzE3MzQ4MmU4MzQyNmRiZDA5YTg2YTI5YWFkNjRi) 23 | - [smpl_mean_params.npz](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/smpl_mean_params.npz?versionId=CAEQHhiBgICN6M3V6xciIDU1MzUzNjZjZGNiOTQ3OWJiZTJmNThiZmY4NmMxMTM4) 24 | - The pretrained frame feature extractor [spin.pth](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/vibe/spin.pth?versionId=CAEQKhiBgMD9hdub9xciIDIxMGI4NmMxMGIyNzQxOGM5YzYxZjMyYTVmMjllOTAy) 25 | 26 | Download the above resources and arrange them in the following file structure: 27 | 28 | ```text 29 | mmhuman3d 30 | ├── mmhuman3d 31 | ├── docs 32 | ├── tests 33 | ├── tools 34 | ├── configs 35 | └── data 36 | ├── body_models 37 | │ ├── J_regressor_extra.npy 38 | │ ├── J_regressor_h36m.npy 39 | │ ├── smpl_mean_params.npz 40 | │ └── smpl 41 | │ ├── SMPL_FEMALE.pkl 42 | │ ├── SMPL_MALE.pkl 43 | │ └── SMPL_NEUTRAL.pkl 44 | ├── pretrained 45 | │ └── spin.pth 46 | ├── preprocessed_datasets 47 | │ ├── vibe_mpi_inf_3dhp_train.npz 48 | │ └── vibe_insta_variety.npz 49 | └── datasets 50 | └── mpi_inf_3dhp 51 | ``` 52 | 53 | ## Results and Models 54 | 55 | We evaluate VIBE on 3DPW. Values are MPJPE/PA-MPJPE. 56 | 57 | | Config | 3DPW | Download | 58 | |:------:|:-------:|:------:| 59 | | [resnet50_vibe_pw3d.py]() | 94.89 / 57.08 | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/vibe/resnet50_vibe_pw3d-2e05a122_20211201.pth?versionId=CAEQHhiBgMCNvonS6xciIGEyOGM1M2M0ZTdiMDQ4NTc4NDI1MjBmYzgyMjUwMWI2) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/vibe/20211201_112201.log?versionId=CAEQHhiBgMDdvInS6xciIDI0Yzg1NzVhZTNjZDQ1Nzg4MDAyZDE5NTllYTM5ZmU2) | 60 | -------------------------------------------------------------------------------- /configs/vibe/metafile.yml: -------------------------------------------------------------------------------- 1 | Collections: 2 | - Name: VIBE 3 | Metadata: 4 | Training Data: 5 | - MPI-INF-3DHP 6 | - InstaVariety 7 | Architecture: 8 | - ResNet 9 | - HMRHead 10 | Paper: 11 | URL: https://arxiv.org/pdf/1912.05656.pdf 12 | Title: "VIBE: Video Inference for Human Body Pose and Shape Estimation" 13 | README: configs/vibe/README.md 14 | 15 | Models: 16 | - Name: resnet50_vibe_pw3d 17 | In Collection: VIBE 18 | Config: configs/vibe/resnet50_vibe_pw3d.py 19 | Metadata: 20 | Epochs: 100 21 | Results: 22 | - Task: Human Pose and Shape Estimation 23 | Dataset: 3DPW 24 | Metrics: 25 | MPJPE: 94.89 26 | PA-MPJPE: 57.08 27 | Weights: https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/vibe/resnet50_vibe_pw3d-2e05a122_20211201.pth?versionId=CAEQHhiBgMCNvonS6xciIGEyOGM1M2M0ZTdiMDQ4NTc4NDI1MjBmYzgyMjUwMWI2 28 | -------------------------------------------------------------------------------- /demo/resources/S1_Directions_1.54138969_000001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/demo/resources/S1_Directions_1.54138969_000001.jpg -------------------------------------------------------------------------------- /demo/resources/multi_person_demo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/demo/resources/multi_person_demo.mp4 -------------------------------------------------------------------------------- /demo/resources/single_person_demo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/demo/resources/single_person_demo.mp4 -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/css/readthedocs.css: -------------------------------------------------------------------------------- 1 | .header-logo { 2 | background-image: url("../image/mmhuman3d-logo.png"); 3 | background-size: 156px 40px; 4 | height: 40px; 5 | width: 156px; 6 | } 7 | -------------------------------------------------------------------------------- /docs/_static/image/mmhuman3d-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/docs/_static/image/mmhuman3d-logo.png -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | mmhuman3d.apis 2 | ---------------- 3 | .. automodule:: mmhuman3d.apis 4 | :members: 5 | 6 | mmhuman3d.core 7 | ---------------- 8 | 9 | cameras 10 | ^^^^^^^^^^ 11 | .. automodule:: mmhuman3d.core.cameras 12 | :members: 13 | 14 | conventions 15 | ^^^^^^^^^^^^^ 16 | .. automodule:: mmhuman3d.core.conventions 17 | :members: 18 | 19 | evaluation 20 | ^^^^^^^^^^ 21 | .. automodule:: mmhuman3d.core.evaluation 22 | :members: 23 | 24 | filter 25 | ^^^^^^^^^^ 26 | .. automodule:: mmhuman3d.core.filter 27 | :members: 28 | 29 | optimizer 30 | ^^^^^^^^^^ 31 | .. automodule:: mmhuman3d.core.optimizer 32 | :members: 33 | 34 | parametric_model 35 | ^^^^^^^^^^^^^^^^ 36 | .. automodule:: mmhuman3d.core.parametric_model 37 | :members: 38 | 39 | visualization 40 | ^^^^^^^^^^^^^ 41 | .. automodule:: mmhuman3d.core.visualization 42 | :members: 43 | 44 | mmhuman3d.models 45 | ---------------- 46 | 47 | models 48 | ^^^^^^ 49 | .. automodule:: mmhuman3d.models 50 | :members: 51 | 52 | architectures 53 | ^^^^^^^^^^^^^^ 54 | .. automodule:: mmhuman3d.models.architectures 55 | :members: 56 | 57 | backbones 58 | ^^^^^^^^^^ 59 | .. automodule:: mmhuman3d.models.backbones 60 | :members: 61 | 62 | discriminators 63 | ^^^^^^^^^^^^^^ 64 | .. automodule:: mmhuman3d.models.discriminators 65 | :members: 66 | 67 | necks 68 | ^^^^^^ 69 | .. automodule:: mmhuman3d.models.necks 70 | :members: 71 | 72 | heads 73 | ^^^^^^ 74 | .. automodule:: mmhuman3d.models.heads 75 | :members: 76 | 77 | losses 78 | ^^^^^^ 79 | .. automodule:: mmhuman3d.models.losses 80 | :members: 81 | 82 | utils 83 | ^^^^^^ 84 | .. automodule:: mmhuman3d.models.utils 85 | :members: 86 | 87 | mmhuman3d.data 88 | ----------------- 89 | 90 | data 91 | ^^^^^^^^ 92 | .. automodule:: mmhuman3d.data 93 | :members: 94 | 95 | datasets 96 | ^^^^^^^^^ 97 | .. automodule:: mmhuman3d.data.datasets 98 | :members: 99 | 100 | data_converters 101 | ^^^^^^^^^^^^^^^ 102 | .. automodule:: mmhuman3d.data.data_converters 103 | :members: 104 | 105 | data_structures 106 | ^^^^^^^^^^^^^^^ 107 | .. automodule:: mmhuman3d.data.data_structures 108 | :members: 109 | 110 | mmhuman3d.utils 111 | --------------- 112 | .. automodule:: mmhuman3d.utils 113 | :members: 114 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to MMHuman3D's documentation! 2 | ======================================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Get Started 7 | 8 | install.md 9 | getting_started.md 10 | model_zoo.md 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | :caption: Datasets 15 | 16 | human_data.md 17 | preprocess_dataset.md 18 | 19 | .. toctree:: 20 | :maxdepth: 2 21 | :caption: Keypoints convention 22 | 23 | keypoints_convention.md 24 | customize_keypoints_convention.md 25 | 26 | .. toctree:: 27 | :maxdepth: 2 28 | :caption: Visualization 29 | 30 | cameras.md 31 | visualize_keypoints.md 32 | visualize_smpl.md 33 | 34 | .. toctree:: 35 | :maxdepth: 2 36 | :caption: License 37 | 38 | additional_licenses.md 39 | 40 | .. toctree:: 41 | :maxdepth: 1 42 | :caption: API Reference 43 | 44 | api.rst 45 | 46 | Indices and tables 47 | ================== 48 | 49 | * :ref:`genindex` 50 | * :ref:`search` 51 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/model_zoo.md: -------------------------------------------------------------------------------- 1 | # Benchmark and Model Zoo 2 | 3 | We provide configuration files, log files and pretrained models for all supported methods. 4 | Moreover, all pretrain models are evaluated on three common benchmarks: 3DPW, Human3.6M, and 5 | MPI-INF-3DHP. 6 | 7 | ## Baselines 8 | 9 | ### HMR 10 | 11 | Please refer to [HMR](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/hmr/) for details. 12 | 13 | ### SPIN 14 | 15 | Please refer to [SPIN](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/spin/) for details. 16 | 17 | ### VIBE 18 | 19 | Please refer to [VIBE](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/vibe/) for details. 20 | 21 | ### HybrIK 22 | 23 | Please refer to [HybrIK](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/hybrik/) for details. 24 | 25 | ### PARE 26 | 27 | Please refer to [PARE](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/pare/) for details. 28 | -------------------------------------------------------------------------------- /docs_zh-CN/customize_keypoints_convention.md: -------------------------------------------------------------------------------- 1 | ## 客制化关键点类型 2 | 3 | ### 总览 4 | 5 | 如果使用的数据集在MMHuman3D中不受支持，您可以根据这个文档，添加一个新的关键点类型。 6 | 7 | 下列是MMHuman3D中支持的关键点类型: 8 | - agora 9 | - coco 10 | - coco_wholebody 11 | - crowdpose 12 | - h36m 13 | - human_data 14 | - hybrik 15 | - lsp 16 | - mpi_inf_3dhp 17 | - mpii 18 | - openpose 19 | - penn_action 20 | - posetrack 21 | - pw3d 22 | - smpl 23 | - smplx 24 | 25 | 26 | **1. 创建新的关键点类型** 27 | 28 | 请根据 29 | [`mmhuman3d/core/conventions/keypoints_mapping/human_data.py`](https://github.com/open-mmlab/mmhuman3d/tree/main/mmhuman3d/core/conventions/keypoints_mapping/human_data.py) , 创建一个名为 `NEW_CONVENTION.py` 的新文件。 30 | 该文件中, `NEW_KEYPOINTS` 是一个包含关键点名字和具体顺序的列表。 31 | 32 | 例如, 想要为`AGORA`数据集创建一个新的关键点类型, `agora.py` 文件应该包含以下项目: 33 | ``` 34 | AGORA_KEYPOINTS = [ 35 | 'pelvis', 36 | 'left_hip', 37 | 'right_hip' 38 | ... 39 | ] 40 | ``` 41 | 42 | **2. 寻找`human_data`中的关键点名字** 43 | 44 | 在MMHuman3D中，不同数据集中具有相同名字的关键点应该对应相同的人体部件。`human_data`类型已经整合了支持的数据集中不同名字的关键点和对应关系。 45 | 46 | 对于`NEW_KEYPOINTS`中的每一个关键点, 需要进行如下检查: 47 | 48 | (1)关健点名称是否存在于[`mmhuman3d/core/conventions/keypoints_mapping/human_data.py`](https://github.com/open-mmlab/mmhuman3d/tree/main/mmhuman3d/core/conventions/keypoints_mapping/human_data.py); 49 | 50 | (2)关键点是否有相对于`human_data`中相同位置关键点的映射关系。 51 | 52 | 如果两个条件都满足, 请在`NEW_CONVENTION.py`中保留关键点的名字。 53 | 54 | 55 | **3. 寻找`human_data`中关键点的对应关系** 56 | 57 | 如果`NEW_KEYPOINTS`中的关键点与`human_data`中不同名字的关键点有相同的对应关系, 例如`NEW_KEYPOINTS`中的`head`对应于`human_data`中的`head_extra`, 将`NEW_KEYPOINTS`中的关键点按照`human_data`的规范重命名, 例如`head` -> `head_extra`。 58 | 59 | **4. 在`human_data`中添加新的关键点** 60 | 61 | 如果新的关键点与`human_data`中的关键点没有对应关系，就需要将其罗列出来，并且在原始名称上加上前缀以作区分，例如`spine_3dhp`。 62 | 63 | 如有需要的话，我们会对`human_data`进行拓展以匹配新的关键点，但这必须在检查新的关键点没有对应关系，且没有命名冲突的情况下完成。 64 | 65 | **5. 初始化新的关键点类型集合** 66 | 67 | 在[`mmhuman3d/core/conventions/keypoints_mapping/__init__.py`](https://github.com/open-mmlab/mmhuman3d/tree/main/mmhuman3d/core/conventions/keypoints_mapping/__init__.py#L8-25)中添加`NEW_CONVENTION.py`的`import`, 并在[KEYPOINTS_FACTORY](https://github.com/open-mmlab/mmhuman3d/tree/main/mmhuman3d/core/conventions/keypoints_mapping/__init__.py#L27-52)的字典中添加标识符。 68 | 69 | 例如, 我们所添加的新的关键点类型为`agora`: 70 | ``` 71 | # 添加 import 72 | from mmhuman3d.core.conventions.keypoints_mapping import ( 73 | agora, 74 | ... 75 | ) 76 | 77 | # 添加到 factory 78 | KEYPOINTS_FACTORY = { 79 | 'agora': agora.AGORA_KEYPOINTS, 80 | ... 81 | } 82 | ``` 83 | 84 | **6. 将新的关键点类型用于关键点转换** 85 | 86 | 想要将现有的关键点类型转换为新的类型，可以使用[`mmhuman3d/core/conventions/keypoints_mapping/__init__.py`](https://github.com/open-mmlab/mmhuman3d/tree/main/mmhuman3d/core/conventions/keypoints_mapping/__init__.py)中的`convert_kps`函数。该函数会产生一个由0,1构成的mask，指示是否应该滤除或者保留。 87 | 88 | 将`coco`关键点转换为新的关键点: 89 | ``` 90 | new_kps, mask = convert_kps(smplx_keypoints, src='coco', dst='NEW_CONVENTION') 91 | ``` 92 | 93 | 将新的关键点转换为`human_data`: 94 | ``` 95 | new_kps, mask = convert_kps(smplx_keypoints, src='NEW_CONVENTION', dst='human_data') 96 | ``` 97 | -------------------------------------------------------------------------------- /docs_zh-CN/imgs/mmhuman3d_qq_qrcode.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/docs_zh-CN/imgs/mmhuman3d_qq_qrcode.jpg -------------------------------------------------------------------------------- /docs_zh-CN/imgs/wechat_assistant_qrcode.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/docs_zh-CN/imgs/wechat_assistant_qrcode.jpg -------------------------------------------------------------------------------- /docs_zh-CN/imgs/zhihu_qrcode.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/docs_zh-CN/imgs/zhihu_qrcode.jpg -------------------------------------------------------------------------------- /docs_zh-CN/model_zoo.md: -------------------------------------------------------------------------------- 1 | # 模型库 2 | 3 | 我们为所有支持的算法提供配置文件，log文件和预训练模型。 4 | 所有的预训练模型在三个benchmark上进行评估：`3DPW`, `Human3.6M`, 和 `MPI-INF-3DHP`。 5 | 6 | ## 基准算法 7 | 8 | ### HMR 9 | 10 | 详情请参考[HMR](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/hmr/)。 11 | 12 | ### SPIN 13 | 14 | 详情请参考[SPIN](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/spin/)。 15 | 16 | ### VIBE 17 | 18 | 详情请参考[VIBE](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/vibe/)。 19 | 20 | ### HybrIK 21 | 22 | 详情请参考[HybrIK](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/hybrik/)。 23 | 24 | ### PARE 25 | 26 | 详情请参考[PARE](https://github.com/open-mmlab/mmhuman3d/tree/main/configs/pare/)。 27 | -------------------------------------------------------------------------------- /mmhuman3d/__init__.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | 3 | from .version import __version__ 4 | 5 | 6 | def digit_version(version_str): 7 | digit_version = [] 8 | for x in version_str.split('.'): 9 | if x.isdigit(): 10 | digit_version.append(int(x)) 11 | elif x.find('rc') != -1: 12 | patch_version = x.split('rc') 13 | digit_version.append(int(patch_version[0]) - 1) 14 | digit_version.append(int(patch_version[1])) 15 | return digit_version 16 | 17 | 18 | mmcv_minimum_version = '1.3.17' 19 | mmcv_maximum_version = '1.6.0' 20 | mmcv_version = digit_version(mmcv.__version__) 21 | 22 | 23 | assert (mmcv_version >= digit_version(mmcv_minimum_version) 24 | and mmcv_version < digit_version(mmcv_maximum_version)), \ 25 | f'MMCV=={mmcv.__version__} is used but incompatible. ' \ 26 | f'Please install mmcv>={mmcv_minimum_version}, <{mmcv_maximum_version}.' 27 | 28 | __all__ = ['__version__'] 29 | -------------------------------------------------------------------------------- /mmhuman3d/apis/__init__.py: -------------------------------------------------------------------------------- 1 | from mmhuman3d.apis import inference, test, train 2 | from mmhuman3d.apis.inference import ( 3 | feature_extract, 4 | inference_image_based_model, 5 | inference_video_based_model, 6 | init_model, 7 | ) 8 | from mmhuman3d.apis.test import ( 9 | collect_results_cpu, 10 | collect_results_gpu, 11 | multi_gpu_test, 12 | single_gpu_test, 13 | ) 14 | from mmhuman3d.apis.train import set_random_seed, train_model 15 | 16 | __all__ = [ 17 | 'LoadImage', 'collect_results_cpu', 'collect_results_gpu', 'inference', 18 | 'feature_extract', 'inference_image_based_model', 19 | 'inference_video_based_model', 'init_model', 'multi_gpu_test', 20 | 'set_random_seed', 'single_gpu_test', 'test', 'train', 'train_model' 21 | ] 22 | -------------------------------------------------------------------------------- /mmhuman3d/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/core/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/core/cameras/__init__.py: -------------------------------------------------------------------------------- 1 | from mmhuman3d.core.cameras import builder, camera_parameters, cameras 2 | from mmhuman3d.core.cameras.builder import CAMERAS, build_cameras 3 | from mmhuman3d.core.cameras.cameras import ( 4 | FoVOrthographicCameras, 5 | FoVPerspectiveCameras, 6 | MMCamerasBase, 7 | OrthographicCameras, 8 | PerspectiveCameras, 9 | WeakPerspectiveCameras, 10 | compute_direction_cameras, 11 | compute_orbit_cameras, 12 | ) 13 | 14 | __all__ = [ 15 | 'CAMERAS', 'FoVOrthographicCameras', 'FoVPerspectiveCameras', 16 | 'MMCamerasBase', 'OrthographicCameras', 'PerspectiveCameras', 17 | 'WeakPerspectiveCameras', 'build_cameras', 'builder', 'camera_parameters', 18 | 'cameras', 'compute_orbit_cameras', 'compute_direction_cameras' 19 | ] 20 | -------------------------------------------------------------------------------- /mmhuman3d/core/cameras/builder.py: -------------------------------------------------------------------------------- 1 | from mmcv.utils import Registry 2 | 3 | CAMERAS = Registry('cameras') 4 | 5 | 6 | def build_cameras(cfg): 7 | """Build cameras.""" 8 | return CAMERAS.build(cfg) 9 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/core/conventions/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/cameras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/core/conventions/cameras/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/joints_mapping/standard_joint_angles.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | TRANSFORMATION_AA_TO_SJA = torch.Tensor([ 4 | [[1, 0, 0], [0, 0, 1], [0, -1, 0]], # 00, 'left_hip', 5 | [[1, 0, 0], [0, 0, 1], [0, -1, 0]], # 01, 'right_hip', 6 | [[1, 0, 0], [0, 0, -1], [0, 1, 0]], # 02, 'spine1', 7 | [[1, 0, 0], [0, 0, 1], [0, -1, 0]], # 03, 'left_knee', 8 | [[1, 0, 0], [0, 0, 1], [0, -1, 0]], # 04, 'right_knee', 9 | [[1, 0, 0], [0, 0, -1], [0, 1, 0]], # 05, 'spine2', 10 | [[1, 0, 0], [0, 1, 0], [0, 0, 1]], # 06, 'left_ankle', 11 | [[1, 0, 0], [0, 1, 0], [0, 0, 1]], # 07, 'right_ankle', 12 | [[1, 0, 0], [0, 0, -1], [0, 1, 0]], # 08, 'spine3', 13 | [[1, 0, 0], [0, 1, 0], [0, 0, 1]], # 09, 'left_foot', 14 | [[1, 0, 0], [0, 1, 0], [0, 0, 1]], # 10, 'right_foot', 15 | [[1, 0, 0], [0, 0, -1], [0, 1, 0]], # 11, 'neck', 16 | [[0, 0, -1], [0, 1, 0], [1, 0, 0]], # 12, 'left_collar', 17 | [[0, 0, 1], [0, 1, 0], [-1, 0, 0]], # 13, 'right_collar', 18 | [[1, 0, 0], [0, 0, -1], [0, 1, 0]], # 14, 'head', 19 | [[0, 0, -1], [0, 1, 0], [1, 0, 0]], # 15, 'left_shoulder', 20 | [[0, 0, 1], [0, 1, 0], [-1, 0, 0]], # 16, 'right_shoulder', 21 | [[0, 0, -1], [0, 1, 0], [1, 0, 0]], # 17, 'left_elbow', 22 | [[0, 0, 1], [0, 1, 0], [-1, 0, 0]], # 18, 'right_elbow', 23 | [[0, 0, -1], [0, 1, 0], [1, 0, 0]], # 19, 'left_wrist', 24 | [[0, 0, 1], [0, 1, 0], [-1, 0, 0]], # 20, 'right_wrist', 25 | ]) 26 | 27 | TRANSFORMATION_SJA_TO_AA = \ 28 | torch.inverse(TRANSFORMATION_AA_TO_SJA) 29 | 30 | # TODO: spines and shoulders may need further adjustment 31 | STANDARD_JOINT_ANGLE_LIMITS = torch.deg2rad( 32 | torch.Tensor([ 33 | [[-45, 155], [-88, 17], [-105, 85]], # 00, 'left_hip', 34 | [[-45, 155], [-17, 88], [-85, 105]], # 01, 'right_hip', 35 | [[-25, 15], [-20, 20], [-30, 30]], # 02, 'spine1', 36 | [[0, 150], [0, 0], [0, 0]], # 03, 'left_knee', 37 | [[0, 150], [0, 0], [0, 0]], # 04, 'right_knee', 38 | [[-25, 15], [-15, 15], [-25, 25]], # 05, 'spine2', 39 | [[-31, 63], [-26, 26], [-74, 15]], # 06, 'left_ankle', 40 | [[-31, 63], [-26, 26], [-15, 74]], # 07, 'right_ankle', 41 | [[-25, 15], [-15, 15], [-25, 25]], # 08, 'spine3', 42 | [[-60, 45], [0, 0], [-45, 45]], # 09, 'left_foot', 43 | [[-60, 45], [0, 0], [-45, 45]], # 10, 'right_foot', 44 | [[-37, 22], [-30, 30], [-45, 45]], # 11, 'neck', 45 | [[-30, 30], [-30, 10], [0, 0]], # 12, 'left_collar', 46 | [[-30, 30], [-10, 30], [0, 0]], # 13, 'right_collar', 47 | [[-37, 22], [-30, 30], [-45, 45]], # 14, 'head', 48 | [[-90, 135], [-97, 91], [-90, 135]], # 15, 'left_shoulder', 49 | [[-135, 90], [-91, 97], [-135, 90]], # 16, 'right_shoulder', 50 | [[0, 0], [-150, 0], [0, 0]], # 17, 'left_elbow', 51 | [[0, 0], [0, 150], [0, 0]], # 18, 'right_elbow', 52 | [[-90, 90], [-45, 45], [-180, 60]], # 19, 'left_wrist', 53 | [[-90, 90], [-45, 45], [-60, 180]], # 20, 'right_wrist', 54 | ])) 55 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/agora.py: -------------------------------------------------------------------------------- 1 | AGORA_KEYPOINTS = [ 2 | 'pelvis', 'left_hip', 'right_hip', 'spine_1', 'left_knee', 'right_knee', 3 | 'spine_2', 'left_ankle', 'right_ankle', 'spine_3', 'left_foot', 4 | 'right_foot', 'neck', 'left_collar', 'right_collar', 'head', 5 | 'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow', 6 | 'left_wrist', 'right_wrist', 'jaw', 'left_eyeball', 'right_eyeball', 7 | 'left_index_1', 'left_index_2', 'left_index_3', 'left_middle_1', 8 | 'left_middle_2', 'left_middle_3', 'left_pinky_1', 'left_pinky_2', 9 | 'left_pinky_3', 'left_ring_1', 'left_ring_2', 'left_ring_3', 10 | 'left_thumb_1', 'left_thumb_2', 'left_thumb_3', 'right_index_1', 11 | 'right_index_2', 'right_index_3', 'right_middle_1', 'right_middle_2', 12 | 'right_middle_3', 'right_pinky_1', 'right_pinky_2', 'right_pinky_3', 13 | 'right_ring_1', 'right_ring_2', 'right_ring_3', 'right_thumb_1', 14 | 'right_thumb_2', 'right_thumb_3', 'nose', 'right_eye', 'left_eye', 15 | 'right_ear', 'left_ear', 'left_bigtoe', 'left_smalltoe', 'left_heel', 16 | 'right_bigtoe', 'right_smalltoe', 'right_heel', 'left_thumb', 'left_index', 17 | 'left_middle', 'left_ring', 'left_pinky', 'right_thumb', 'right_index', 18 | 'right_middle', 'right_ring', 'right_pinky', 'right_eyebrow_1', 19 | 'right_eyebrow_2', 'right_eyebrow_3', 'right_eyebrow_4', 'right_eyebrow_5', 20 | 'left_eyebrow_5', 'left_eyebrow_4', 'left_eyebrow_3', 'left_eyebrow_2', 21 | 'left_eyebrow_1', 'nosebridge_1', 'nosebridge_2', 'nosebridge_3', 22 | 'nosebridge_4', 'nose_1', 'nose_2', 'nose_3', 'nose_4', 'nose_5', 23 | 'right_eye_1', 'right_eye_2', 'right_eye_3', 'right_eye_4', 'right_eye_5', 24 | 'right_eye_6', 'left_eye_4', 'left_eye_3', 'left_eye_2', 'left_eye_1', 25 | 'left_eye_6', 'left_eye_5', 'mouth_1', 'mouth_2', 'mouth_3', 'mouth_4', 26 | 'mouth_5', 'mouth_6', 'mouth_7', 'mouth_8', 'mouth_9', 'mouth_10', 27 | 'mouth_11', 'mouth_12', 'lip_1', 'lip_2', 'lip_3', 'lip_4', 'lip_5', 28 | 'lip_6', 'lip_7', 'lip_8' 29 | ] 30 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/coco.py: -------------------------------------------------------------------------------- 1 | COCO_KEYPOINTS = [ 2 | 'nose', 3 | 'left_eye', 4 | 'right_eye', 5 | 'left_ear', 6 | 'right_ear', 7 | 'left_shoulder', 8 | 'right_shoulder', 9 | 'left_elbow', 10 | 'right_elbow', 11 | 'left_wrist', 12 | 'right_wrist', 13 | 'left_hip_extra', 14 | 'right_hip_extra', 15 | 'left_knee', 16 | 'right_knee', 17 | 'left_ankle', 18 | 'right_ankle', 19 | ] 20 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/coco_wholebody.py: -------------------------------------------------------------------------------- 1 | COCO_WHOLEBODY_KEYPOINTS = [ 2 | 'nose', 3 | 'left_eye', 4 | 'right_eye', 5 | 'left_ear', 6 | 'right_ear', 7 | 'left_shoulder', 8 | 'right_shoulder', 9 | 'left_elbow', 10 | 'right_elbow', 11 | 'left_wrist', 12 | 'right_wrist', 13 | 'left_hip', 14 | 'right_hip', 15 | 'left_knee', 16 | 'right_knee', 17 | 'left_ankle', 18 | 'right_ankle', 19 | 'left_bigtoe', 20 | 'left_smalltoe', 21 | 'left_heel', 22 | 'right_bigtoe', 23 | 'right_smalltoe', 24 | 'right_heel', 25 | 'face_contour_1', 26 | 'face_contour_2', 27 | 'face_contour_3', 28 | 'face_contour_4', 29 | 'face_contour_5', 30 | 'face_contour_6', 31 | 'face_contour_7', 32 | 'face_contour_8', 33 | 'face_contour_9', 34 | 'face_contour_10', 35 | 'face_contour_11', 36 | 'face_contour_12', 37 | 'face_contour_13', 38 | 'face_contour_14', 39 | 'face_contour_15', 40 | 'face_contour_16', 41 | 'face_contour_17', 42 | 'right_eyebrow_1', 43 | 'right_eyebrow_2', 44 | 'right_eyebrow_3', 45 | 'right_eyebrow_4', 46 | 'right_eyebrow_5', 47 | 'left_eyebrow_5', 48 | 'left_eyebrow_4', 49 | 'left_eyebrow_3', 50 | 'left_eyebrow_2', 51 | 'left_eyebrow_1', 52 | 'nosebridge_1', 53 | 'nosebridge_2', 54 | 'nosebridge_3', 55 | 'nosebridge_4', 56 | 'nose_1', 57 | 'nose_2', 58 | 'nose_3', 59 | 'nose_4', 60 | 'nose_5', 61 | 'right_eye_1', 62 | 'right_eye_2', 63 | 'right_eye_3', 64 | 'right_eye_4', 65 | 'right_eye_5', 66 | 'right_eye_6', 67 | 'left_eye_4', 68 | 'left_eye_3', 69 | 'left_eye_2', 70 | 'left_eye_1', 71 | 'left_eye_6', 72 | 'left_eye_5', 73 | 'mouth_1', 74 | 'mouth_2', 75 | 'mouth_3', 76 | 'mouth_4', 77 | 'mouth_5', 78 | 'mouth_6', 79 | 'mouth_7', 80 | 'mouth_8', 81 | 'mouth_9', 82 | 'mouth_10', 83 | 'mouth_11', 84 | 'mouth_12', 85 | 'lip_1', 86 | 'lip_2', 87 | 'lip_3', 88 | 'lip_4', 89 | 'lip_5', 90 | 'lip_6', 91 | 'lip_7', 92 | 'lip_8', 93 | 'left_hand_root', 94 | 'left_thumb_1', 95 | 'left_thumb_2', 96 | 'left_thumb_3', 97 | 'left_thumb', 98 | 'left_index_1', 99 | 'left_index_2', 100 | 'left_index_3', 101 | 'left_index', 102 | 'left_middle_1', 103 | 'left_middle_2', 104 | 'left_middle_3', 105 | 'left_middle', 106 | 'left_ring_1', 107 | 'left_ring_2', 108 | 'left_ring_3', 109 | 'left_ring', 110 | 'left_pinky_1', 111 | 'left_pinky_2', 112 | 'left_pinky_3', 113 | 'left_pinky', 114 | 'right_hand_root', 115 | 'right_thumb_1', 116 | 'right_thumb_2', 117 | 'right_thumb_3', 118 | 'right_thumb', 119 | 'right_index_1', 120 | 'right_index_2', 121 | 'right_index_3', 122 | 'right_index', 123 | 'right_middle_1', 124 | 'right_middle_2', 125 | 'right_middle_3', 126 | 'right_middle', 127 | 'right_ring_1', 128 | 'right_ring_2', 129 | 'right_ring_3', 130 | 'right_ring', 131 | 'right_pinky_1', 132 | 'right_pinky_2', 133 | 'right_pinky_3', 134 | 'right_pinky', 135 | ] 136 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/crowdpose.py: -------------------------------------------------------------------------------- 1 | CROWDPOSE_KEYPOINTS = [ 2 | 'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow', 3 | 'left_wrist', 'right_wrist', 'left_hip', 'right_hip', 'left_knee', 4 | 'right_knee', 'left_ankle', 'right_ankle', 'head', 'neck' 5 | ] 6 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/h36m.py: -------------------------------------------------------------------------------- 1 | H36M_KEYPOINTS = [ 2 | 'pelvis_extra', 3 | 'left_hip_extra', 4 | 'left_knee', 5 | 'left_ankle', 6 | 'right_hip_extra', 7 | 'right_knee', 8 | 'right_ankle', 9 | 'spine_extra', 10 | 'neck_extra', 11 | 'head_extra', 12 | 'headtop', 13 | 'left_shoulder', 14 | 'left_elbow', 15 | 'left_wrist', 16 | 'right_shoulder', 17 | 'right_elbow', 18 | 'right_wrist', 19 | ] 20 | 21 | H36M_KEYPOINTS_MMPOSE = [ 22 | 'pelvis_extra', 23 | 'right_hip_extra', 24 | 'right_knee', 25 | 'right_ankle', 26 | 'left_hip_extra', 27 | 'left_knee', 28 | 'left_ankle', 29 | 'spine_extra', 30 | 'neck_extra', 31 | 'head_extra', 32 | 'headtop', 33 | 'left_shoulder', 34 | 'left_elbow', 35 | 'left_wrist', 36 | 'right_shoulder', 37 | 'right_elbow', 38 | 'right_wrist', 39 | ] 40 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/hybrik.py: -------------------------------------------------------------------------------- 1 | HYBRIK_29_KEYPOINTS = [ 2 | 'pelvis', 3 | 'left_hip', 4 | 'right_hip', # 2 5 | 'spine_1', 6 | 'left_knee', 7 | 'right_knee', # 5 8 | 'spine_2', 9 | 'left_ankle', 10 | 'right_ankle', # 8 11 | 'spine_3', 12 | 'left_foot', 13 | 'right_foot', # 11 14 | 'neck', 15 | 'left_collar', 16 | 'right_collar', # 14 17 | 'jaw', # 15 18 | 'left_shoulder', 19 | 'right_shoulder', # 17 20 | 'left_elbow', 21 | 'right_elbow', # 19 22 | 'left_wrist', 23 | 'right_wrist', # 21 24 | 'left_thumb', 25 | 'right_thumb', # 23 26 | 'head', 27 | 'left_middle', 28 | 'right_middle', # 26 29 | 'left_bigtoe', 30 | 'right_bigtoe' # 28 31 | ] 32 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/instavariety.py: -------------------------------------------------------------------------------- 1 | INSTAVARIETY_KEYPOINTS = [ 2 | 'right_heel_openpose', 3 | 'right_knee_openpose', 4 | 'right_hip_openpose', 5 | 'left_hip_openpose', 6 | 'left_knee_openpose', 7 | 'left_heel_openpose', 8 | 'right_wrist_openpose', 9 | 'right_elbow_openpose', 10 | 'right_shoulder_openpose', 11 | 'left_shoulder_openpose', 12 | 'left_elbow_openpose', 13 | 'left_wrist_openpose', 14 | 'neck_openpose', 15 | 'headtop', 16 | 'nose_openpose', 17 | 'left_eye_openpose', 18 | 'right_eye_openpose', 19 | 'left_ear_openpose', 20 | 'right_ear_openpose', 21 | 'left_bigtoe_openpose', 22 | 'right_bigtoe_openpose', 23 | 'left_smalltoe_openpose', 24 | 'right_smalltoe_openpose', 25 | 'left_ankle_openpose', 26 | 'right_ankle_openpose', 27 | ] 28 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/lsp.py: -------------------------------------------------------------------------------- 1 | LSP_KEYPOINTS = [ 2 | 'right_ankle', 3 | 'right_knee', 4 | 'right_hip_extra', 5 | 'left_hip_extra', 6 | 'left_knee', 7 | 'left_ankle', 8 | 'right_wrist', 9 | 'right_elbow', 10 | 'right_shoulder', 11 | 'left_shoulder', 12 | 'left_elbow', 13 | 'left_wrist', 14 | 'neck_extra', 15 | 'headtop', 16 | ] 17 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/mpi_inf_3dhp.py: -------------------------------------------------------------------------------- 1 | MPI_INF_3DHP_KEYPOINTS = [ 2 | 'spine_3', 3 | 'spine_4_3dhp', 4 | 'spine_2', 5 | 'spine_extra', # close to spine2 6 | 'pelvis_extra', 7 | 'neck_extra', # throat 8 | 'head_extra', 9 | 'headtop', 10 | 'left_clavicle_3dhp', 11 | 'left_shoulder', 12 | 'left_elbow', 13 | 'left_wrist', 14 | 'left_hand_3dhp', 15 | 'right_clavicle_3dhp', 16 | 'right_shoulder', 17 | 'right_elbow', 18 | 'right_wrist', 19 | 'right_hand_3dhp', 20 | 'left_hip_extra', 21 | 'left_knee', 22 | 'left_ankle', 23 | 'left_foot', 24 | 'left_toe_3dhp', 25 | 'right_hip_extra', 26 | 'right_knee', 27 | 'right_ankle', 28 | 'right_foot', 29 | 'right_toe_3dhp' 30 | ] 31 | 32 | MPI_INF_3DHP_TEST_KEYPOINTS = [ 33 | 'headtop', 34 | 'neck_extra', 35 | 'right_shoulder', 36 | 'right_elbow', 37 | 'right_wrist', 38 | 'left_shoulder', 39 | 'left_elbow', 40 | 'left_wrist', 41 | 'right_hip_extra', 42 | 'right_knee', 43 | 'right_ankle', 44 | 'left_hip_extra', 45 | 'left_knee', 46 | 'left_ankle', 47 | 'pelvis_extra', 48 | 'spine_extra', # close to spine2 49 | 'head_extra' 50 | ] 51 | 52 | HYBRIK_MPI_INF_3DHP_KEYPOINTS = [ 53 | 'spine_3', 54 | 'spine_4_3dhp', 55 | 'spine_2', 56 | 'spine_extra', # close to spine2 57 | 'pelvis', 58 | 'neck', # throat 59 | 'head_extra', 60 | 'headtop', 61 | 'left_clavicle_3dhp', 62 | 'left_shoulder', 63 | 'left_elbow', 64 | 'left_wrist', 65 | 'left_hand_3dhp', 66 | 'right_clavicle_3dhp', 67 | 'right_shoulder', 68 | 'right_elbow', 69 | 'right_wrist', 70 | 'right_hand_3dhp', 71 | 'left_hip', 72 | 'left_knee', 73 | 'left_ankle', 74 | 'left_foot', 75 | 'left_toe_3dhp', 76 | 'right_hip', 77 | 'right_knee', 78 | 'right_ankle', 79 | 'right_foot', 80 | 'right_toe_3dhp' 81 | ] 82 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/mpii.py: -------------------------------------------------------------------------------- 1 | MPII_KEYPOINTS = [ 2 | 'right_ankle', 3 | 'right_knee', 4 | 'right_hip_extra', 5 | 'left_hip_extra', 6 | 'left_knee', 7 | 'left_ankle', 8 | 'pelvis_extra', 9 | 'thorax_extra', 10 | 'neck_extra', 11 | 'headtop', 12 | 'right_wrist', 13 | 'right_elbow', 14 | 'right_shoulder', 15 | 'left_shoulder', 16 | 'left_elbow', 17 | 'left_wrist', 18 | ] 19 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/penn_action.py: -------------------------------------------------------------------------------- 1 | PENN_ACTION_KEYPOINTS = [ 2 | 'head', 3 | 'left_shoulder', 4 | 'right_shoulder', 5 | 'left_elbow', 6 | 'right_elbow', 7 | 'left_wrist', 8 | 'right_wrist', 9 | 'left_hip', 10 | 'right_hip', 11 | 'left_knee', 12 | 'right_knee', 13 | 'left_ankle', 14 | 'right_ankle', 15 | ] 16 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/posetrack.py: -------------------------------------------------------------------------------- 1 | POSETRACK_KEYPOINTS = [ 2 | 'nose', 'head_bottom_pt', 'headtop', 'left_ear', 'right_ear', 3 | 'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow', 4 | 'left_wrist', 'right_wrist', 'left_hip', 'right_hip', 'left_knee', 5 | 'right_knee', 'left_ankle', 'right_ankle' 6 | ] 7 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/pw3d.py: -------------------------------------------------------------------------------- 1 | PW3D_KEYPOINTS = [ 2 | 'nose', 3 | 'neck_extra', 4 | 'right_shoulder', 5 | 'right_elbow', 6 | 'right_wrist', 7 | 'left_shoulder', 8 | 'left_elbow', 9 | 'left_wrist', 10 | 'right_hip_extra', 11 | 'right_knee', 12 | 'right_ankle', 13 | 'left_hip_extra', 14 | 'left_knee', 15 | 'left_ankle', 16 | 'right_eye', 17 | 'left_eye', 18 | 'right_ear', 19 | 'left_ear', 20 | ] 21 | -------------------------------------------------------------------------------- /mmhuman3d/core/conventions/keypoints_mapping/smpl.py: -------------------------------------------------------------------------------- 1 | # the keypoints defined in the SMPL paper 2 | SMPL_KEYPOINTS = [ 3 | 'pelvis', 4 | 'left_hip', 5 | 'right_hip', 6 | 'spine_1', 7 | 'left_knee', 8 | 'right_knee', 9 | 'spine_2', 10 | 'left_ankle', 11 | 'right_ankle', 12 | 'spine_3', 13 | 'left_foot', 14 | 'right_foot', 15 | 'neck', 16 | 'left_collar', 17 | 'right_collar', 18 | 'head', 19 | 'left_shoulder', 20 | 'right_shoulder', 21 | 'left_elbow', 22 | 'right_elbow', 23 | 'left_wrist', 24 | 'right_wrist', 25 | 'left_hand', 26 | 'right_hand', 27 | ] 28 | 29 | # the full keypoints produced by the default SMPL J_regressor 30 | SMPL_45_KEYPOINTS = SMPL_KEYPOINTS + [ 31 | 'nose', 32 | 'right_eye', 33 | 'left_eye', 34 | 'right_ear', 35 | 'left_ear', 36 | 'left_bigtoe', 37 | 'left_smalltoe', 38 | 'left_heel', 39 | 'right_bigtoe', 40 | 'right_smalltoe', 41 | 'right_heel', 42 | 'left_thumb', 43 | 'left_index', 44 | 'left_middle', 45 | 'left_ring', 46 | 'left_pinky', 47 | 'right_thumb', 48 | 'right_index', 49 | 'right_middle', 50 | 'right_ring', 51 | 'right_pinky', 52 | ] 53 | 54 | # the full keypoints produced by the default SMPL J_regressor and 55 | # extra_J_regressor (provided by SPIN) 56 | SMPL_54_KEYPOINTS = SMPL_45_KEYPOINTS + [ 57 | 'right_hip_extra', # LSP 58 | 'left_hip_extra', # LSP 59 | 'neck_extra', # LSP 60 | 'headtop', # LSP 61 | 'pelvis_extra', # MPII 62 | 'thorax_extra', # MPII 63 | 'spine_extra', # H36M 64 | 'jaw_extra', # H36M 65 | 'head_extra', # H36M 66 | ] 67 | 68 | # SMPL keypoint convention used by SPIN, EFT and so on 69 | SMPL_49_KEYPOINTS = [ 70 | # OpenPose 71 | 'nose_openpose', 72 | 'neck_openpose', # 'upper_neck' 73 | 'right_shoulder_openpose', 74 | 'right_elbow_openpose', 75 | 'right_wrist_openpose', 76 | 'left_shoulder_openpose', 77 | 'left_elbow_openpose', 78 | 'left_wrist_openpose', 79 | 'pelvis_openpose', 80 | 'right_hip_openpose', 81 | 'right_knee_openpose', 82 | 'right_ankle_openpose', 83 | 'left_hip_openpose', 84 | 'left_knee_openpose', 85 | 'left_ankle_openpose', 86 | 'right_eye_openpose', 87 | 'left_eye_openpose', 88 | 'right_ear_openpose', 89 | 'left_ear_openpose', 90 | 'left_bigtoe_openpose', 91 | 'left_smalltoe_openpose', 92 | 'left_heel_openpose', 93 | 'right_bigtoe_openpose', 94 | 'right_smalltoe_openpose', 95 | 'right_heel_openpose', 96 | # 24 Keypoints 97 | 'right_ankle', 98 | 'right_knee', 99 | 'right_hip_extra', # LSP 100 | 'left_hip_extra', # LSP 101 | 'left_knee', 102 | 'left_ankle', 103 | 'right_wrist', 104 | 'right_elbow', 105 | 'right_shoulder', 106 | 'left_shoulder', 107 | 'left_elbow', 108 | 'left_wrist', 109 | 'neck_extra', # LSP 110 | 'headtop', # LSP mpii peen_action mpi_inf_3dhp 111 | 'pelvis_extra', # MPII 112 | 'thorax_extra', # MPII 113 | 'spine_extra', # H36M 114 | 'jaw_extra', # H36M 115 | 'head_extra', # H36M 116 | 'nose', 117 | 'left_eye', 118 | 'right_eye', 119 | 'left_ear', 120 | 'right_ear' 121 | ] 122 | 123 | SMPL_24_KEYPOINTS = SMPL_49_KEYPOINTS[-24:] 124 | -------------------------------------------------------------------------------- /mmhuman3d/core/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from mmhuman3d.core.evaluation import mesh_eval 2 | from mmhuman3d.core.evaluation.eval_hooks import DistEvalHook, EvalHook 3 | from mmhuman3d.core.evaluation.eval_utils import ( 4 | keypoint_3d_auc, 5 | keypoint_3d_pck, 6 | keypoint_accel_error, 7 | keypoint_mpjpe, 8 | vertice_pve, 9 | ) 10 | from mmhuman3d.core.evaluation.mesh_eval import compute_similarity_transform 11 | 12 | __all__ = [ 13 | 'compute_similarity_transform', 'keypoint_mpjpe', 'mesh_eval', 14 | 'DistEvalHook', 'EvalHook', 'vertice_pve', 'keypoint_3d_pck', 15 | 'keypoint_3d_auc', 'keypoint_accel_error' 16 | ] 17 | -------------------------------------------------------------------------------- /mmhuman3d/core/evaluation/mesh_eval.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Adapted from https://github.com/akanazawa/hmr 3 | # Original licence: Copyright (c) 2018 akanazawa, under the MIT License. 4 | # ------------------------------------------------------------------------------ 5 | 6 | import numpy as np 7 | 8 | 9 | def compute_similarity_transform(source_points, target_points): 10 | """Computes a similarity transform (sR, t) that takes a set of 3D points 11 | source_points (N x 3) closest to a set of 3D points target_points, where R 12 | is an 3x3 rotation matrix, t 3x1 translation, s scale. 13 | 14 | And return the 15 | transformed 3D points source_points_hat (N x 3). i.e. solves the orthogonal 16 | Procrutes problem. 17 | Notes: 18 | Points number: N 19 | Args: 20 | source_points (np.ndarray([N, 3])): Source point set. 21 | target_points (np.ndarray([N, 3])): Target point set. 22 | Returns: 23 | source_points_hat (np.ndarray([N, 3])): Transformed source point set. 24 | """ 25 | 26 | assert target_points.shape[0] == source_points.shape[0] 27 | assert target_points.shape[1] == 3 and source_points.shape[1] == 3 28 | 29 | source_points = source_points.T 30 | target_points = target_points.T 31 | 32 | # 1. Remove mean. 33 | mu1 = source_points.mean(axis=1, keepdims=True) 34 | mu2 = target_points.mean(axis=1, keepdims=True) 35 | X1 = source_points - mu1 36 | X2 = target_points - mu2 37 | 38 | # 2. Compute variance of X1 used for scale. 39 | var1 = np.sum(X1**2) 40 | 41 | # 3. The outer product of X1 and X2. 42 | K = X1.dot(X2.T) 43 | 44 | # 4. Solution that Maximizes trace(R'K) is R=U*V', where U, V are 45 | # singular vectors of K. 46 | U, _, Vh = np.linalg.svd(K) 47 | V = Vh.T 48 | # Construct Z that fixes the orientation of R to get det(R)=1. 49 | Z = np.eye(U.shape[0]) 50 | Z[-1, -1] *= np.sign(np.linalg.det(U.dot(V.T))) 51 | # Construct R. 52 | R = V.dot(Z.dot(U.T)) 53 | 54 | # 5. Recover scale. 55 | scale = np.trace(R.dot(K)) / var1 56 | 57 | # 6. Recover translation. 58 | t = mu2 - scale * (R.dot(mu1)) 59 | 60 | # 7. Transform the source points: 61 | source_points_hat = scale * R.dot(source_points) + t 62 | 63 | source_points_hat = source_points_hat.T 64 | 65 | return source_points_hat 66 | -------------------------------------------------------------------------------- /mmhuman3d/core/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .builder import OPTIMIZERS, build_optimizers 3 | 4 | __all__ = ['build_optimizers', 'OPTIMIZERS'] 5 | -------------------------------------------------------------------------------- /mmhuman3d/core/optimizer/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from mmcv.runner import build_optimizer 3 | from mmcv.utils import Registry 4 | 5 | OPTIMIZERS = Registry('optimizers') 6 | 7 | 8 | def build_optimizers(model, cfgs): 9 | """Build multiple optimizers from configs. If `cfgs` contains several dicts 10 | for optimizers, then a dict for each constructed optimizers will be 11 | returned. If `cfgs` only contains one optimizer config, the constructed 12 | optimizer itself will be returned. For example, 13 | 14 | 1) Multiple optimizer configs: 15 | 16 | .. code-block:: python 17 | 18 | optimizer_cfg = dict( 19 | model1=dict(type='SGD', lr=lr), 20 | model2=dict(type='SGD', lr=lr)) 21 | 22 | The return dict is 23 | ``dict('model1': torch.optim.Optimizer, 'model2': torch.optim.Optimizer)`` 24 | 25 | 2) Single optimizer config: 26 | 27 | .. code-block:: python 28 | 29 | optimizer_cfg = dict(type='SGD', lr=lr) 30 | 31 | The return is ``torch.optim.Optimizer``. 32 | 33 | Args: 34 | model (:obj:`nn.Module`): The model with parameters to be optimized. 35 | cfgs (dict): The config dict of the optimizer. 36 | 37 | Returns: 38 | dict[:obj:`torch.optim.Optimizer`] | :obj:`torch.optim.Optimizer`: 39 | The initialized optimizers. 40 | """ 41 | optimizers = {} 42 | if hasattr(model, 'module'): 43 | model = model.module 44 | # determine whether 'cfgs' has several dicts for optimizers 45 | if all(isinstance(v, dict) for v in cfgs.values()): 46 | for key, cfg in cfgs.items(): 47 | cfg_ = cfg.copy() 48 | module = getattr(model, key) 49 | optimizers[key] = build_optimizer(module, cfg_) 50 | return optimizers 51 | 52 | return build_optimizer(model, cfgs) 53 | -------------------------------------------------------------------------------- /mmhuman3d/core/post_processing/__init__.py: -------------------------------------------------------------------------------- 1 | from .builder import build_post_processing 2 | from .smooth.gaus1d_filter import Gaus1dFilter 3 | from .smooth.oneeuro_filter import OneEuroFilter 4 | from .smooth.savgol_filter import SGFilter 5 | from .smooth.smoothnet import SmoothNetFilter 6 | from .speed_up.deciwatch import DeciWatchPostProcessing 7 | 8 | __all__ = [ 9 | 'build_post_processing', 10 | 'OneEuroFilter', 11 | 'SGFilter', 12 | 'Gaus1dFilter', 13 | 'SmoothNetFilter', 14 | 'DeciWatchPostProcessing', 15 | ] 16 | -------------------------------------------------------------------------------- /mmhuman3d/core/post_processing/builder.py: -------------------------------------------------------------------------------- 1 | from mmcv.utils import Registry 2 | 3 | POST_PROCESSING = Registry('post_processing') 4 | 5 | 6 | def build_post_processing(cfg): 7 | """Build post processing function.""" 8 | return POST_PROCESSING.build(cfg) 9 | -------------------------------------------------------------------------------- /mmhuman3d/core/post_processing/smooth/gaus1d_filter.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import numpy as np 4 | import scipy.signal as signal 5 | import torch 6 | from scipy.ndimage.filters import gaussian_filter1d 7 | 8 | from ..builder import POST_PROCESSING 9 | 10 | 11 | @POST_PROCESSING.register_module(name=['Gaus1dFilter', 'gaus1d']) 12 | class Gaus1dFilter: 13 | """Applies median filter and then gaussian filter. code from: 14 | https://github.com/akanazawa/human_dynamics/blob/mas 15 | ter/src/util/smooth_bbox.py. 16 | 17 | Args: 18 | x (np.ndarray): input pose 19 | window_size (int, optional): for median filters (must be odd). 20 | sigma (float, optional): Sigma for gaussian smoothing. 21 | 22 | Returns: 23 | np.ndarray: Smoothed poses 24 | """ 25 | 26 | def __init__(self, window_size=11, sigma=4): 27 | super(Gaus1dFilter, self).__init__() 28 | 29 | self.window_size = window_size 30 | self.sigma = sigma 31 | 32 | def __call__(self, x=None): 33 | if self.window_size % 2 == 0: 34 | window_size = self.window_size - 1 35 | else: 36 | window_size = self.window_size 37 | if window_size > x.shape[0]: 38 | window_size = x.shape[0] 39 | if len(x.shape) != 3: 40 | warnings.warn('x should be a tensor or numpy of [T*M,K,C]') 41 | assert len(x.shape) == 3 42 | x_type = x 43 | if isinstance(x, torch.Tensor): 44 | if x.is_cuda: 45 | x = x.cpu().numpy() 46 | else: 47 | x = x.numpy() 48 | 49 | smoothed = np.array( 50 | [signal.medfilt(param, window_size) for param in x.T]).T 51 | smooth_poses = np.array( 52 | [gaussian_filter1d(traj, self.sigma) for traj in smoothed.T]).T 53 | 54 | if isinstance(x_type, torch.Tensor): 55 | # we also return tensor by default 56 | if x_type.is_cuda: 57 | smooth_poses = torch.from_numpy(smooth_poses).cuda() 58 | else: 59 | smooth_poses = torch.from_numpy(smooth_poses) 60 | 61 | return smooth_poses 62 | -------------------------------------------------------------------------------- /mmhuman3d/core/post_processing/smooth/savgol_filter.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import numpy as np 4 | import scipy.signal as signal 5 | import torch 6 | 7 | from ..builder import POST_PROCESSING 8 | 9 | 10 | @POST_PROCESSING.register_module(name=['SGFilter', 'savgol']) 11 | class SGFilter: 12 | """savgol_filter lib is from: 13 | https://docs.scipy.org/doc/scipy/reference/generated/ 14 | scipy.signal.savgol_filter.html. 15 | 16 | Args: 17 | window_size (float): 18 | The length of the filter window 19 | (i.e., the number of coefficients). 20 | window_length must be a positive odd integer. 21 | polyorder (int): 22 | The order of the polynomial used to fit the samples. 23 | polyorder must be less than window_length. 24 | 25 | Returns: 26 | smoothed poses (np.ndarray, torch.tensor) 27 | """ 28 | 29 | def __init__(self, window_size=11, polyorder=2): 30 | super(SGFilter, self).__init__() 31 | 32 | # 1-D Savitzky-Golay filter 33 | self.window_size = window_size 34 | self.polyorder = polyorder 35 | 36 | def __call__(self, x=None): 37 | # x.shape: [t,k,c] 38 | if self.window_size % 2 == 0: 39 | window_size = self.window_size - 1 40 | else: 41 | window_size = self.window_size 42 | if window_size > x.shape[0]: 43 | window_size = x.shape[0] 44 | if window_size <= self.polyorder: 45 | polyorder = window_size - 1 46 | else: 47 | polyorder = self.polyorder 48 | assert polyorder > 0 49 | assert window_size > polyorder 50 | if len(x.shape) != 3: 51 | warnings.warn('x should be a tensor or numpy of [T*M,K,C]') 52 | assert len(x.shape) == 3 53 | x_type = x 54 | if isinstance(x, torch.Tensor): 55 | if x.is_cuda: 56 | x = x.cpu().numpy() 57 | else: 58 | x = x.numpy() 59 | smooth_poses = np.zeros_like(x) 60 | # smooth at different axis 61 | C = x.shape[-1] 62 | for i in range(C): 63 | smooth_poses[..., i] = signal.savgol_filter( 64 | x[..., i], window_size, polyorder, axis=0) 65 | 66 | if isinstance(x_type, torch.Tensor): 67 | # we also return tensor by default 68 | if x_type.is_cuda: 69 | smooth_poses = torch.from_numpy(smooth_poses).cuda() 70 | else: 71 | smooth_poses = torch.from_numpy(smooth_poses) 72 | return smooth_poses 73 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/__init__.py: -------------------------------------------------------------------------------- 1 | from .renderer.torch3d_renderer import render_runner 2 | from .renderer.torch3d_renderer.builder import ( 3 | build_lights, 4 | build_renderer, 5 | build_shader, 6 | build_textures, 7 | ) 8 | from .visualize_keypoints2d import visualize_kp2d 9 | from .visualize_keypoints3d import visualize_kp3d 10 | from .visualize_smpl import ( 11 | render_smpl, 12 | visualize_smpl_calibration, 13 | visualize_smpl_hmr, 14 | visualize_smpl_pose, 15 | visualize_smpl_vibe, 16 | visualize_T_pose, 17 | ) 18 | 19 | __all__ = [ 20 | 'visualize_kp2d', 'visualize_kp3d', 'visualize_smpl_pose', 21 | 'visualize_T_pose', 'render_smpl', 'visualize_smpl_vibe', 22 | 'visualize_smpl_calibration', 'visualize_smpl_hmr', 'render_runner', 23 | 'build_lights', 'build_renderer', 'build_shader', 'build_textures' 24 | ] 25 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/__init__.py: -------------------------------------------------------------------------------- 1 | from .matplotlib3d_renderer import Axes3dJointsRenderer 2 | from .torch3d_renderer import render_runner 3 | from .torch3d_renderer.base_renderer import BaseRenderer 4 | from .torch3d_renderer.builder import ( 5 | build_lights, 6 | build_renderer, 7 | build_shader, 8 | build_textures, 9 | ) 10 | from .torch3d_renderer.depth_renderer import DepthRenderer 11 | from .torch3d_renderer.mesh_renderer import MeshRenderer 12 | from .torch3d_renderer.normal_renderer import NormalRenderer 13 | from .torch3d_renderer.pointcloud_renderer import PointCloudRenderer 14 | from .torch3d_renderer.segmentation_renderer import SegmentationRenderer 15 | from .torch3d_renderer.shader import ( 16 | DepthShader, 17 | NoLightShader, 18 | NormalShader, 19 | SegmentationShader, 20 | ) 21 | from .torch3d_renderer.silhouette_renderer import SilhouetteRenderer 22 | from .torch3d_renderer.smpl_renderer import SMPLRenderer 23 | from .torch3d_renderer.textures import TexturesNearest 24 | from .torch3d_renderer.uv_renderer import UVRenderer 25 | from .vedo_render import VedoRenderer 26 | 27 | __all__ = [ 28 | 'NoLightShader', 'BaseRenderer', 'TexturesNearest', 'SMPLRenderer', 29 | 'SilhouetteRenderer', 'Axes3dJointsRenderer', 'VedoRenderer', 30 | 'MeshRenderer', 'DepthRenderer', 'NormalRenderer', 'SegmentationRenderer', 31 | 'PointCloudRenderer', 'UVRenderer', 'build_renderer', 'build_shader', 32 | 'build_textures', 'build_lights', 'DepthShader', 'SegmentationShader', 33 | 'NormalShader', 'NoLightShader', 'render_runner' 34 | ] 35 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/torch3d_renderer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/core/visualization/renderer/torch3d_renderer/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/torch3d_renderer/builder.py: -------------------------------------------------------------------------------- 1 | from mmcv.utils import Registry 2 | from pytorch3d.renderer import ( 3 | HardFlatShader, 4 | HardGouraudShader, 5 | HardPhongShader, 6 | SoftGouraudShader, 7 | SoftPhongShader, 8 | TexturesAtlas, 9 | TexturesUV, 10 | TexturesVertex, 11 | ) 12 | 13 | from .lights import AmbientLights, DirectionalLights, PointLights 14 | from .shader import ( 15 | DepthShader, 16 | NoLightShader, 17 | NormalShader, 18 | SegmentationShader, 19 | SilhouetteShader, 20 | ) 21 | from .textures import TexturesNearest 22 | 23 | RENDERER = Registry('renderer') 24 | 25 | LIGHTS = Registry('lights') 26 | LIGHTS.register_module( 27 | name=['directional', 'directional_lights', 'DirectionalLights'], 28 | module=DirectionalLights) 29 | LIGHTS.register_module( 30 | name=['point', 'point_lights', 'PointLights'], module=PointLights) 31 | LIGHTS.register_module( 32 | name=['ambient', 'ambient_lights', 'AmbientLights'], module=AmbientLights) 33 | 34 | SHADER = Registry('shader') 35 | SHADER.register_module( 36 | name=[ 37 | 'flat', 'hard_flat_shader', 'hard_flat', 'HardFlat', 'HardFlatShader' 38 | ], 39 | module=HardFlatShader) 40 | SHADER.register_module( 41 | name=['hard_phong', 'HardPhong', 'HardPhongShader'], 42 | module=HardPhongShader) 43 | SHADER.register_module( 44 | name=['hard_gouraud', 'HardGouraud', 'HardGouraudShader'], 45 | module=HardGouraudShader) 46 | SHADER.register_module( 47 | name=['soft_gouraud', 'SoftGouraud', 'SoftGouraudShader'], 48 | module=SoftGouraudShader) 49 | SHADER.register_module( 50 | name=['soft_phong', 'SoftPhong', 'SoftPhongShader'], 51 | module=SoftPhongShader) 52 | SHADER.register_module( 53 | name=['silhouette', 'Silhouette', 'SilhouetteShader'], 54 | module=SilhouetteShader) 55 | SHADER.register_module( 56 | name=['nolight', 'nolight_shader', 'NoLight', 'NoLightShader'], 57 | module=NoLightShader) 58 | SHADER.register_module( 59 | name=['normal', 'normal_shader', 'Normal', 'NormalShader'], 60 | module=NormalShader) 61 | SHADER.register_module( 62 | name=['depth', 'depth_shader', 'Depth', 'DepthShader'], module=DepthShader) 63 | SHADER.register_module( 64 | name=[ 65 | 'segmentation', 'segmentation_shader', 'Segmentation', 66 | 'SegmentationShader' 67 | ], 68 | module=SegmentationShader) 69 | 70 | TEXTURES = Registry('textures') 71 | TEXTURES.register_module( 72 | name=['TexturesAtlas', 'textures_atlas', 'atlas', 'Atlas'], 73 | module=TexturesAtlas) 74 | TEXTURES.register_module( 75 | name=['TexturesNearest', 'textures_nearest', 'nearest', 'Nearest'], 76 | module=TexturesNearest) 77 | TEXTURES.register_module( 78 | name=['TexturesUV', 'textures_uv', 'uv'], module=TexturesUV) 79 | TEXTURES.register_module( 80 | name=['TexturesVertex', 'textures_vertex', 'vertex', 'vc'], 81 | module=TexturesVertex) 82 | 83 | 84 | def build_textures(cfg): 85 | """Build textures.""" 86 | return TEXTURES.build(cfg) 87 | 88 | 89 | def build_shader(cfg): 90 | """Build shader.""" 91 | return SHADER.build(cfg) 92 | 93 | 94 | def build_lights(cfg): 95 | """Build lights.""" 96 | return LIGHTS.build(cfg) 97 | 98 | 99 | def build_renderer(cfg): 100 | """Build renderers.""" 101 | return RENDERER.build(cfg) 102 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/torch3d_renderer/lights.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | import torch 4 | from pytorch3d.renderer.lighting import AmbientLights as _AmbientLights 5 | from pytorch3d.renderer.lighting import DirectionalLights as _DirectionalLights 6 | from pytorch3d.renderer.lighting import PointLights as _PointLights 7 | from pytorch3d.renderer.utils import TensorProperties 8 | 9 | MMLIGHT_ATTR = [ 10 | 'ambient_color', 'diffuse_color', 'specular_color', 'location', 'direction' 11 | ] 12 | 13 | 14 | class MMLights(TensorProperties): 15 | 16 | def __init__(self, **kwargs) -> None: 17 | super().__init__(**kwargs) 18 | _N = 1 19 | self.mmlight_attr_list = [] 20 | for attr_name in MMLIGHT_ATTR: 21 | if hasattr(self, attr_name): 22 | self.mmlight_attr_list.append(attr_name) 23 | for k in self.mmlight_attr_list: 24 | v = getattr(self, k) 25 | if not isinstance(v, torch.Tensor): 26 | v = torch.Tensor(v) 27 | v = v.view(-1, 3) 28 | setattr(self, k, v) 29 | 30 | if getattr(self, k).shape[0] > _N: 31 | _N = getattr(self, k).shape[0] 32 | for k in self.mmlight_attr_list: 33 | if getattr(self, k).shape[0] == 1: 34 | setattr(self, k, getattr(self, k).repeat(_N, 1)) 35 | self._N = _N 36 | 37 | def __len__(self, ): 38 | return self._N 39 | 40 | def __getitem__(self, index: Union[int, slice]): 41 | if isinstance(index, int): 42 | index = [index] 43 | kwargs = {} 44 | for k in self.mmlight_attr_list: 45 | kwargs[k] = getattr(self, k)[index] 46 | 47 | return self.__class__(device=self.device, **kwargs) 48 | 49 | def extend(self, N): 50 | kwargs = {} 51 | for k in self.mmlight_attr_list: 52 | kwargs[k] = getattr(self, k).repeat(N, 1) 53 | return self.__class__(device=self.device, **kwargs) 54 | 55 | def extend_(self, N): 56 | for k in self.mmlight_attr_list: 57 | setattr(self, k, getattr(self, k).repeat(N, 1)) 58 | self._N = N 59 | 60 | 61 | class AmbientLights(_AmbientLights, MMLights): 62 | 63 | def __init__(self, ambient_color=None, device='cpu', **kwargs) -> None: 64 | if ambient_color is None: 65 | ambient_color = ((1.0, 1.0, 1.0), ) 66 | diffuse_color = ((0.0, 0.0, 0.0), ) 67 | super(_AmbientLights, self).__init__( 68 | ambient_color=ambient_color, 69 | diffuse_color=diffuse_color, 70 | device=device) 71 | 72 | def __getitem__(self, index: Union[int, slice]): 73 | return super(_AmbientLights, self).__getitem__(index) 74 | 75 | 76 | class PointLights(_PointLights, MMLights): 77 | 78 | def __getitem__(self, index: Union[int, slice]): 79 | return super(_PointLights, self).__getitem__(index) 80 | 81 | 82 | class DirectionalLights(_DirectionalLights, MMLights): 83 | 84 | def __getitem__(self, index: Union[int, slice]): 85 | return super(_DirectionalLights, self).__getitem__(index) 86 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/torch3d_renderer/mesh_renderer.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Optional, Tuple, Union 2 | 3 | import torch 4 | from pytorch3d.structures import Meshes 5 | 6 | from mmhuman3d.core.cameras import MMCamerasBase 7 | from mmhuman3d.core.visualization.renderer.torch3d_renderer.lights import \ 8 | MMLights # noqa: E501 9 | from .base_renderer import BaseRenderer 10 | from .builder import RENDERER 11 | 12 | 13 | @RENDERER.register_module( 14 | name=['Mesh', 'mesh', 'mesh_renderer', 'MeshRenderer']) 15 | class MeshRenderer(BaseRenderer): 16 | """Render RGBA image with the help of camera system.""" 17 | shader_type = 'SoftPhongShader' 18 | 19 | def __init__( 20 | self, 21 | resolution: Tuple[int, int] = None, 22 | device: Union[torch.device, str] = 'cpu', 23 | output_path: Optional[str] = None, 24 | out_img_format: str = '%06d.png', 25 | **kwargs, 26 | ) -> None: 27 | """Renderer for RGBA image of meshes. 28 | 29 | Args: 30 | resolution (Iterable[int]): 31 | (width, height) of the rendered images resolution. 32 | device (Union[torch.device, str], optional): 33 | You can pass a str or torch.device for cpu or gpu render. 34 | Defaults to 'cpu'. 35 | output_path (Optional[str], optional): 36 | Output path of the video or images to be saved. 37 | Defaults to None. 38 | out_img_format (str, optional): The image format string for 39 | saving the images. 40 | Defaults to '%06d.png'. 41 | """ 42 | super().__init__( 43 | resolution=resolution, 44 | device=device, 45 | output_path=output_path, 46 | out_img_format=out_img_format, 47 | **kwargs) 48 | 49 | def forward(self, 50 | meshes: Meshes, 51 | cameras: Optional[MMCamerasBase] = None, 52 | lights: Optional[MMLights] = None, 53 | indexes: Optional[Iterable[int]] = None, 54 | backgrounds: Optional[torch.Tensor] = None, 55 | **kwargs) -> Union[torch.Tensor, None]: 56 | """Render Meshes. 57 | 58 | Args: 59 | meshes (Meshes): meshes to be rendered. 60 | cameras (Optional[MMCamerasBase], optional): cameras for render. 61 | Defaults to None. 62 | lights (Optional[MMLights], optional): lights for render. 63 | Defaults to None. 64 | indexes (Optional[Iterable[int]], optional): indexes for images. 65 | Defaults to None. 66 | backgrounds (Optional[torch.Tensor], optional): background images. 67 | Defaults to None. 68 | 69 | Returns: 70 | Union[torch.Tensor, None]: return tensor or None. 71 | """ 72 | 73 | meshes = meshes.to(self.device) 74 | self._update_resolution(cameras, **kwargs) 75 | fragments = self.rasterizer(meshes_world=meshes, cameras=cameras) 76 | 77 | rendered_images = self.shader( 78 | fragments=fragments, 79 | meshes=meshes, 80 | cameras=cameras, 81 | lights=self.lights if lights is None else lights) 82 | 83 | if self.output_path is not None: 84 | rgba = self.tensor2rgba(rendered_images) 85 | self._write_images(rgba, backgrounds, indexes) 86 | return rendered_images 87 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/torch3d_renderer/normal_renderer.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Optional, Union 2 | 3 | import torch 4 | from pytorch3d.structures import Meshes 5 | 6 | from mmhuman3d.core.cameras import MMCamerasBase 7 | from .base_renderer import BaseRenderer 8 | from .builder import RENDERER 9 | from .utils import normalize 10 | 11 | 12 | @RENDERER.register_module( 13 | name=['Normal', 'normal', 'normal_renderer', 'NormalRenderer']) 14 | class NormalRenderer(BaseRenderer): 15 | """Render normal map with the help of camera system.""" 16 | shader_type = 'NormalShader' 17 | 18 | def __init__( 19 | self, 20 | resolution: Iterable[int] = None, 21 | device: Union[torch.device, str] = 'cpu', 22 | output_path: Optional[str] = None, 23 | out_img_format: str = '%06d.png', 24 | **kwargs, 25 | ) -> None: 26 | """Renderer for normal map of meshes. 27 | 28 | Args: 29 | resolution (Iterable[int]): 30 | (width, height) of the rendered images resolution. 31 | device (Union[torch.device, str], optional): 32 | You can pass a str or torch.device for cpu or gpu render. 33 | Defaults to 'cpu'. 34 | output_path (Optional[str], optional): 35 | Output path of the video or images to be saved. 36 | Defaults to None. 37 | out_img_format (str, optional): The image format string for 38 | saving the images. 39 | Defaults to '%06d.png'. 40 | 41 | Returns: 42 | None 43 | """ 44 | super().__init__( 45 | resolution=resolution, 46 | device=device, 47 | output_path=output_path, 48 | obj_path=None, 49 | out_img_format=out_img_format, 50 | **kwargs) 51 | 52 | def forward(self, 53 | meshes: Optional[Meshes] = None, 54 | cameras: Optional[MMCamerasBase] = None, 55 | indexes: Optional[Iterable[int]] = None, 56 | backgrounds: Optional[torch.Tensor] = None, 57 | **kwargs): 58 | """Render Meshes. 59 | 60 | Args: 61 | meshes (Optional[Meshes], optional): meshes to be rendered. 62 | Defaults to None. 63 | cameras (Optional[MMCamerasBase], optional): cameras for render. 64 | Defaults to None. 65 | indexes (Optional[Iterable[int]], optional): indexes for the 66 | images. 67 | Defaults to None. 68 | backgrounds (Optional[torch.Tensor], optional): background images. 69 | Defaults to None. 70 | 71 | Returns: 72 | Union[torch.Tensor, None]: return tensor or None. 73 | """ 74 | 75 | meshes = meshes.to(self.device) 76 | self._update_resolution(cameras, **kwargs) 77 | fragments = self.rasterizer(meshes_world=meshes, cameras=cameras) 78 | normal_map = self.shader( 79 | fragments=fragments, meshes=meshes, cameras=cameras) 80 | 81 | if self.output_path is not None: 82 | rgba = self.tensor2rgba(normal_map) 83 | self._write_images(rgba, backgrounds, indexes) 84 | 85 | return normal_map 86 | 87 | def tensor2rgba(self, tensor: torch.Tensor): 88 | rgbs, valid_masks = tensor[..., :3], (tensor[..., 3:] > 0) * 1.0 89 | rgbs = normalize( 90 | rgbs, origin_value_range=(-1, 1), out_value_range=(0, 1)) 91 | return torch.cat([rgbs, valid_masks], -1) 92 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/renderer/torch3d_renderer/textures.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch3d.ops import interpolate_face_attributes 3 | from pytorch3d.renderer import TexturesVertex 4 | 5 | 6 | class TexturesNearest(TexturesVertex): 7 | """Textures for nearest interpolation.""" 8 | 9 | def sample_textures(self, fragments, faces_packed=None) -> torch.Tensor: 10 | """Rewrite sample_textures to use the nearest interpolation. 11 | 12 | This function will only be called in render forwarding. 13 | """ 14 | verts_features_packed = self.verts_features_packed() 15 | faces_verts_features = verts_features_packed[faces_packed] 16 | bary_coords = fragments.bary_coords 17 | _, idx = torch.max(bary_coords, -1) 18 | mask = torch.arange(bary_coords.size(-1)).reshape(1, 1, -1).to( 19 | self.device) == idx.unsqueeze(-1) 20 | bary_coords *= 0 21 | bary_coords[mask] = 1 22 | texels = interpolate_face_attributes(fragments.pix_to_face, 23 | bary_coords, faces_verts_features) 24 | return texels 25 | -------------------------------------------------------------------------------- /mmhuman3d/core/visualization/visualize_cameras.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from mmhuman3d.core.cameras.camera_parameters import CameraParameter 5 | from mmhuman3d.utils.path_utils import check_path_suffix 6 | from .renderer.vedo_render import VedoRenderer 7 | 8 | 9 | def visualize_chessboard_kinects_rgb(chessboard_path: str, 10 | interactive: bool = True, 11 | show: bool = True): 12 | """Visualize all the RGB cameras in a chessboard file. 13 | 14 | Args: 15 | chessboard_path (str): 16 | Path to the chessboard file. 17 | interactive (bool, optional): 18 | Pause and interact with window (True) or 19 | continue execution (False). 20 | Defaults to True. 21 | show (bool, optional): 22 | Whether to show in a window. 23 | Defaults to True. 24 | """ 25 | # Load camera parameter from a json file 26 | camera_para_json_dict = json.load(open(chessboard_path)) 27 | camera_para_dict = {} 28 | for camera_id in camera_para_json_dict.keys(): 29 | try: 30 | camera_id_int = int(camera_id) 31 | # if camera_id is an instance of int 32 | # and it can be divided by 2, it's an rgb camera 33 | if camera_id_int % 2 == 0: 34 | pass 35 | else: 36 | continue 37 | except ValueError: 38 | continue 39 | temp_camera_parameter = CameraParameter(name=camera_id) 40 | temp_camera_parameter.load_from_chessboard( 41 | camera_para_json_dict[camera_id], camera_id) 42 | camera_para_dict[camera_id] = temp_camera_parameter 43 | camera_vedo_renderer = VedoRenderer() 44 | camera_vedo_renderer.set_y_reverse() 45 | for camera_id in camera_para_dict.keys(): 46 | camera_vedo_renderer.add_camera(camera_para_dict[camera_id]) 47 | if show: 48 | camera_vedo_renderer.show(with_axis=False, interactive=interactive) 49 | 50 | 51 | def visualize_dumped_camera_parameter(dumped_dir: str, 52 | interactive: bool = True, 53 | show: bool = True): 54 | """Visualize all cameras dumped in a directory. 55 | 56 | Args: 57 | dumped_dir (str): 58 | Path to the directory. 59 | interactive (bool, optional): 60 | Pause and interact with window (True) or 61 | continue execution (False). 62 | Defaults to True. 63 | show (bool, optional): 64 | Whether to show in a window. 65 | Defaults to True. 66 | """ 67 | file_list = os.listdir(dumped_dir) 68 | camera_para_list = [] 69 | for file_name in file_list: 70 | file_path = os.path.join(dumped_dir, file_name) 71 | if not check_path_suffix(file_path, ['.json']): 72 | continue 73 | else: 74 | cam_para = CameraParameter() 75 | cam_para.load(file_path) 76 | camera_para_list.append(cam_para) 77 | camera_vedo_renderer = VedoRenderer() 78 | camera_vedo_renderer.set_y_reverse() 79 | for camera_para in camera_para_list: 80 | camera_vedo_renderer.add_camera(camera_para) 81 | if show: 82 | camera_vedo_renderer.show(with_axis=False, interactive=interactive) 83 | -------------------------------------------------------------------------------- /mmhuman3d/data/data_converters/__init__.py: -------------------------------------------------------------------------------- 1 | from .agora import AgoraConverter 2 | from .amass import AmassConverter 3 | from .builder import build_data_converter 4 | from .coco import CocoConverter 5 | from .coco_hybrik import CocoHybrIKConverter 6 | from .coco_wholebody import CocoWholebodyConverter 7 | from .crowdpose import CrowdposeConverter 8 | from .eft import EftConverter 9 | from .gta_human import GTAHumanConverter 10 | from .h36m import H36mConverter 11 | from .h36m_hybrik import H36mHybrIKConverter 12 | from .humman import HuMManConverter 13 | from .insta_vibe import InstaVibeConverter 14 | from .lsp import LspConverter 15 | from .lsp_extended import LspExtendedConverter 16 | from .mpi_inf_3dhp import MpiInf3dhpConverter 17 | from .mpi_inf_3dhp_hybrik import MpiInf3dhpHybrIKConverter 18 | from .mpii import MpiiConverter 19 | from .muco import Muco3dhpConverter 20 | from .penn_action import PennActionConverter 21 | from .posetrack import PosetrackConverter 22 | from .pw3d import Pw3dConverter 23 | from .pw3d_hybrik import Pw3dHybrIKConverter 24 | from .spin import SpinConverter 25 | from .surreal import SurrealConverter 26 | from .up3d import Up3dConverter 27 | from .vibe import VibeConverter 28 | 29 | __all__ = [ 30 | 'build_data_converter', 'AgoraConverter', 'MpiiConverter', 'H36mConverter', 31 | 'AmassConverter', 'CocoConverter', 'CocoWholebodyConverter', 32 | 'H36mConverter', 'LspExtendedConverter', 'LspConverter', 33 | 'MpiInf3dhpConverter', 'PennActionConverter', 'PosetrackConverter', 34 | 'Pw3dConverter', 'Up3dConverter', 'CrowdposeConverter', 'EftConverter', 35 | 'GTAHumanConverter', 'CocoHybrIKConverter', 'H36mHybrIKConverter', 36 | 'MpiInf3dhpHybrIKConverter', 'Pw3dHybrIKConverter', 'SurrealConverter', 37 | 'InstaVibeConverter', 'SpinConverter', 'VibeConverter', 'HuMManConverter', 38 | 'Muco3dhpConverter' 39 | ] 40 | -------------------------------------------------------------------------------- /mmhuman3d/data/data_converters/base_converter.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from typing import List 3 | 4 | 5 | class BaseConverter(metaclass=ABCMeta): 6 | """Base dataset. 7 | 8 | Args: 9 | prefix (str): the prefix of data path 10 | modes (list): the modes of data for converter 11 | """ 12 | 13 | ACCEPTED_MODES = None 14 | 15 | def __init__(self, modes=[]): 16 | self.modes = modes 17 | 18 | for mode in self.modes: 19 | if mode not in self.ACCEPTED_MODES: 20 | raise ValueError(f'Input mode not in {self.ACCEPTED_MODES}') 21 | 22 | @abstractmethod 23 | def convert(self): 24 | pass 25 | 26 | @staticmethod 27 | def _bbox_expand(bbox_xyxy: List[float], 28 | scale_factor: float) -> List[float]: 29 | """Expand bbox in xyxy format by scale factor 30 | Args: 31 | bbox_xyxy (List[float]): Bounding box in xyxy format 32 | scale_factor (float): Scale factor to expand bbox 33 | 34 | Returns: 35 | bbox_xyxy (List[float]): Expanded bounding box in xyxy format 36 | """ 37 | center = [(bbox_xyxy[0] + bbox_xyxy[2]) / 2, 38 | (bbox_xyxy[1] + bbox_xyxy[3]) / 2] 39 | x1 = scale_factor * (bbox_xyxy[0] - center[0]) + center[0] 40 | y1 = scale_factor * (bbox_xyxy[1] - center[1]) + center[1] 41 | x2 = scale_factor * (bbox_xyxy[2] - center[0]) + center[0] 42 | y2 = scale_factor * (bbox_xyxy[3] - center[1]) + center[1] 43 | return [x1, y1, x2, y2] 44 | 45 | @staticmethod 46 | def _xyxy2xywh(bbox_xyxy: List[float]) -> List[float]: 47 | """Obtain bbox in xywh format given bbox in xyxy format 48 | Args: 49 | bbox_xyxy (List[float]): Bounding box in xyxy format 50 | 51 | Returns: 52 | bbox_xywh (List[float]): Bounding box in xywh format 53 | """ 54 | x1, y1, x2, y2 = bbox_xyxy 55 | return [x1, y1, x2 - x1, y2 - y1] 56 | 57 | 58 | class BaseModeConverter(BaseConverter): 59 | """Convert datasets by mode. 60 | 61 | Args: 62 | prefix (str): the prefix of data path 63 | modes (list): the modes of data for converter 64 | """ 65 | 66 | def convert(self, dataset_path: str, out_path: str): 67 | for mode in self.modes: 68 | self.convert_by_mode(dataset_path, out_path, mode) 69 | 70 | @abstractmethod 71 | def convert_by_mode(self): 72 | pass 73 | -------------------------------------------------------------------------------- /mmhuman3d/data/data_converters/builder.py: -------------------------------------------------------------------------------- 1 | from mmcv.utils import Registry 2 | 3 | DATA_CONVERTERS = Registry('data_converters') 4 | 5 | 6 | def build_data_converter(cfg): 7 | """Build data converter.""" 8 | return DATA_CONVERTERS.build(cfg) 9 | -------------------------------------------------------------------------------- /mmhuman3d/data/data_converters/coco.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import numpy as np 5 | from tqdm import tqdm 6 | 7 | from mmhuman3d.core.conventions.keypoints_mapping import convert_kps 8 | from mmhuman3d.data.data_structures.human_data import HumanData 9 | from .base_converter import BaseConverter 10 | from .builder import DATA_CONVERTERS 11 | 12 | 13 | @DATA_CONVERTERS.register_module() 14 | class CocoConverter(BaseConverter): 15 | """CocoDataset dataset `Microsoft COCO: Common Objects in Context' 16 | ECCV'2014 More details can be found in the `paper. 17 | 18 | `__ . 19 | """ 20 | 21 | def convert(self, dataset_path: str, out_path: str) -> dict: 22 | """ 23 | Args: 24 | dataset_path (str): Path to directory where raw images and 25 | annotations are stored. 26 | out_path (str): Path to directory to save preprocessed npz file 27 | 28 | Returns: 29 | dict: 30 | A dict containing keys image_path, bbox_xywh, keypoints2d, 31 | keypoints2d_mask stored in HumanData() format 32 | """ 33 | # use HumanData to store all data 34 | human_data = HumanData() 35 | 36 | # structs we need 37 | image_path_, keypoints2d_, bbox_xywh_ = [], [], [] 38 | 39 | # json annotation file 40 | json_path = os.path.join(dataset_path, 'annotations', 41 | 'person_keypoints_train2014.json') 42 | 43 | json_data = json.load(open(json_path, 'r')) 44 | 45 | imgs = {} 46 | for img in json_data['images']: 47 | imgs[img['id']] = img 48 | 49 | for annot in tqdm(json_data['annotations']): 50 | 51 | # keypoints processing 52 | keypoints2d = annot['keypoints'] 53 | keypoints2d = np.reshape(keypoints2d, (17, 3)) 54 | keypoints2d[keypoints2d[:, 2] > 0, 2] = 1 55 | # check if all major body joints are annotated 56 | if sum(keypoints2d[5:, 2] > 0) < 12: 57 | continue 58 | 59 | # image name 60 | image_id = annot['image_id'] 61 | img_path = str(imgs[image_id]['file_name']) 62 | img_path = os.path.join('train2014', img_path) 63 | 64 | # scale and center 65 | bbox_xywh = annot['bbox'] 66 | 67 | # store data 68 | image_path_.append(img_path) 69 | keypoints2d_.append(keypoints2d) 70 | bbox_xywh_.append(bbox_xywh) 71 | 72 | # convert keypoints 73 | bbox_xywh_ = np.array(bbox_xywh_).reshape((-1, 4)) 74 | bbox_xywh_ = np.hstack([bbox_xywh_, np.ones([bbox_xywh_.shape[0], 1])]) 75 | keypoints2d_ = np.array(keypoints2d_).reshape((-1, 17, 3)) 76 | keypoints2d_, mask = convert_kps(keypoints2d_, 'coco', 'human_data') 77 | 78 | human_data['image_path'] = image_path_ 79 | human_data['bbox_xywh'] = bbox_xywh_ 80 | human_data['keypoints2d_mask'] = mask 81 | human_data['keypoints2d'] = keypoints2d_ 82 | human_data['config'] = 'coco' 83 | human_data.compress_keypoints_by_mask() 84 | 85 | # store the data struct 86 | if not os.path.isdir(out_path): 87 | os.makedirs(out_path) 88 | out_file = os.path.join(out_path, 'coco_2014_train.npz') 89 | human_data.dump(out_file) 90 | -------------------------------------------------------------------------------- /mmhuman3d/data/data_converters/mpii.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import List 3 | 4 | import h5py 5 | import numpy as np 6 | from tqdm import tqdm 7 | 8 | from mmhuman3d.core.conventions.keypoints_mapping import convert_kps 9 | from mmhuman3d.data.data_structures.human_data import HumanData 10 | from .base_converter import BaseConverter 11 | from .builder import DATA_CONVERTERS 12 | 13 | 14 | @DATA_CONVERTERS.register_module() 15 | class MpiiConverter(BaseConverter): 16 | """MPII Dataset `2D Human Pose Estimation: New Benchmark and State of the 17 | Art Analysis' CVPR'2014. More details can be found in the `paper. 18 | 19 | `__ . 20 | """ 21 | 22 | @staticmethod 23 | def center_scale_to_bbox(center: float, scale: float) -> List[float]: 24 | """Obtain bbox given center and scale.""" 25 | w, h = scale * 200, scale * 200 26 | x, y = center[0] - w / 2, center[1] - h / 2 27 | return [x, y, w, h] 28 | 29 | def convert(self, dataset_path: str, out_path: str) -> dict: 30 | """ 31 | Args: 32 | dataset_path (str): Path to directory where raw images and 33 | annotations are stored. 34 | out_path (str): Path to directory to save preprocessed npz file 35 | 36 | Returns: 37 | dict: 38 | A dict containing keys image_path, bbox_xywh, keypoints2d, 39 | keypoints2d_mask stored in HumanData() format 40 | """ 41 | # use HumanData to store all data 42 | human_data = HumanData() 43 | 44 | # structs we use 45 | image_path_, bbox_xywh_, keypoints2d_ = [], [], [] 46 | 47 | # annotation files 48 | annot_file = os.path.join(dataset_path, 'train.h5') 49 | 50 | # read annotations 51 | f = h5py.File(annot_file, 'r') 52 | centers, image_path, keypoints2d, scales = \ 53 | f['center'], f['imgname'], f['part'], f['scale'] 54 | 55 | # go over all annotated examples 56 | for center, imgname, keypoints2d16, scale in tqdm( 57 | zip(centers, image_path, keypoints2d, scales)): 58 | imgname = imgname.decode('utf-8') 59 | # check if all major body joints are annotated 60 | if (keypoints2d16 > 0).sum() < 2 * 16: 61 | continue 62 | 63 | # keypoints 64 | keypoints2d16 = np.hstack([keypoints2d16, np.ones([16, 1])]) 65 | 66 | # bbox 67 | bbox_xywh = self.center_scale_to_bbox(center, scale) 68 | 69 | # store data 70 | image_path_.append(os.path.join('images', imgname)) 71 | bbox_xywh_.append(bbox_xywh) 72 | keypoints2d_.append(keypoints2d16) 73 | 74 | bbox_xywh_ = np.array(bbox_xywh_).reshape((-1, 4)) 75 | bbox_xywh_ = np.hstack([bbox_xywh_, np.ones([bbox_xywh_.shape[0], 1])]) 76 | keypoints2d_ = np.array(keypoints2d_).reshape((-1, 16, 3)) 77 | keypoints2d_, mask = convert_kps(keypoints2d_, 'mpii', 'human_data') 78 | 79 | human_data['image_path'] = image_path_ 80 | human_data['bbox_xywh'] = bbox_xywh_ 81 | human_data['keypoints2d_mask'] = mask 82 | human_data['keypoints2d'] = keypoints2d_ 83 | human_data['config'] = 'mpii' 84 | human_data.compress_keypoints_by_mask() 85 | 86 | # store the data struct 87 | if not os.path.isdir(out_path): 88 | os.makedirs(out_path) 89 | 90 | out_file = os.path.join(out_path, 'mpii_train.npz') 91 | human_data.dump(out_file) 92 | -------------------------------------------------------------------------------- /mmhuman3d/data/data_structures/__init__.py: -------------------------------------------------------------------------------- 1 | from mmhuman3d.data.data_structures import human_data, smc_reader 2 | from mmhuman3d.data.data_structures.human_data import HumanData 3 | from mmhuman3d.data.data_structures.smc_reader import SMCReader 4 | 5 | __all__ = ['HumanData', 'human_data', 'SMCReader', 'smc_reader'] 6 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .adversarial_dataset import AdversarialDataset 2 | from .base_dataset import BaseDataset 3 | from .builder import DATASETS, PIPELINES, build_dataloader, build_dataset 4 | from .dataset_wrappers import ConcatDataset, RepeatDataset 5 | from .human_hybrik_dataset import HybrIKHumanImageDataset 6 | from .human_image_dataset import HumanImageDataset 7 | from .human_video_dataset import HumanVideoDataset 8 | from .mesh_dataset import MeshDataset 9 | from .mixed_dataset import MixedDataset 10 | from .pipelines import Compose 11 | from .samplers import DistributedSampler 12 | 13 | __all__ = [ 14 | 'BaseDataset', 15 | 'HumanImageDataset', 16 | 'build_dataloader', 17 | 'build_dataset', 18 | 'Compose', 19 | 'DistributedSampler', 20 | 'ConcatDataset', 21 | 'RepeatDataset', 22 | 'DATASETS', 23 | 'PIPELINES', 24 | 'MixedDataset', 25 | 'AdversarialDataset', 26 | 'MeshDataset', 27 | 'HumanVideoDataset', 28 | 'HybrIKHumanImageDataset', 29 | ] 30 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/adversarial_dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torch.utils.data import Dataset 3 | 4 | from .builder import DATASETS, build_dataset 5 | 6 | 7 | @DATASETS.register_module() 8 | class AdversarialDataset(Dataset): 9 | """Mix Dataset for the adversarial training in 3D human mesh estimation 10 | task. 11 | 12 | The dataset combines data from two datasets and 13 | return a dict containing data from two datasets. 14 | Args: 15 | train_dataset (:obj:`Dataset`): Dataset for 3D human mesh estimation. 16 | adv_dataset (:obj:`Dataset`): Dataset for adversarial learning. 17 | """ 18 | 19 | def __init__(self, train_dataset: Dataset, adv_dataset: Dataset): 20 | super().__init__() 21 | self.train_dataset = build_dataset(train_dataset) 22 | self.adv_dataset = build_dataset(adv_dataset) 23 | self.num_train_data = len(self.train_dataset) 24 | self.num_adv_data = len(self.adv_dataset) 25 | 26 | def __len__(self): 27 | """Get the size of the dataset.""" 28 | return self.num_train_data 29 | 30 | def __getitem__(self, idx: int): 31 | """Given index, get the data from train dataset and randomly sample an 32 | item from adversarial dataset. 33 | 34 | Return a dict containing data from train and adversarial dataset. 35 | """ 36 | data = self.train_dataset[idx] 37 | adv_idx = np.random.randint(low=0, high=self.num_adv_data, dtype=int) 38 | adv_data = self.adv_dataset[adv_idx] 39 | for k, v in adv_data.items(): 40 | data['adv_' + k] = v 41 | return data 42 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/base_dataset.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from abc import ABCMeta, abstractmethod 3 | from typing import Optional, Union 4 | 5 | from torch.utils.data import Dataset 6 | 7 | from .pipelines import Compose 8 | 9 | 10 | class BaseDataset(Dataset, metaclass=ABCMeta): 11 | """Base dataset. 12 | 13 | Args: 14 | data_prefix (str): the prefix of data path. 15 | pipeline (list): a list of dict, where each element represents 16 | a operation defined in `mmhuman3d.datasets.pipelines`. 17 | ann_file (str | None, optional): the annotation file. When ann_file is 18 | str, the subclass is expected to read from the ann_file. When 19 | ann_file is None, the subclass is expected to read according 20 | to data_prefix. 21 | test_mode (bool): in train mode or test mode. Default: None. 22 | dataset_name (str | None, optional): the name of dataset. It is used 23 | to identify the type of evaluation metric. Default: None. 24 | """ 25 | 26 | def __init__(self, 27 | data_prefix: str, 28 | pipeline: list, 29 | ann_file: Optional[Union[str, None]] = None, 30 | test_mode: Optional[bool] = False, 31 | dataset_name: Optional[Union[str, None]] = None): 32 | super(BaseDataset, self).__init__() 33 | 34 | self.ann_file = ann_file 35 | self.data_prefix = data_prefix 36 | self.test_mode = test_mode 37 | self.pipeline = Compose(pipeline) 38 | if dataset_name is not None: 39 | self.dataset_name = dataset_name 40 | 41 | self.load_annotations() 42 | 43 | @abstractmethod 44 | def load_annotations(self): 45 | """Load annotations from ``ann_file``""" 46 | pass 47 | 48 | def prepare_data(self, idx: int): 49 | """"Prepare raw data for the f'{idx'}-th data.""" 50 | results = copy.deepcopy(self.data_infos[idx]) 51 | results['dataset_name'] = self.dataset_name 52 | results['sample_idx'] = idx 53 | return self.pipeline(results) 54 | 55 | def __len__(self): 56 | """Return the length of current dataset.""" 57 | return self.num_data 58 | 59 | def __getitem__(self, idx: int): 60 | """Prepare data for the ``idx``-th data. 61 | 62 | As for video dataset, we can first parse raw data for each frame. Then 63 | we combine annotations from all frames. This interface is used to 64 | simplify the logic of video dataset and other special datasets. 65 | """ 66 | return self.prepare_data(idx) 67 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/dataset_wrappers.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data.dataset import ConcatDataset as _ConcatDataset 2 | from torch.utils.data.dataset import Dataset 3 | 4 | from .builder import DATASETS 5 | 6 | 7 | @DATASETS.register_module() 8 | class ConcatDataset(_ConcatDataset): 9 | """A wrapper of concatenated dataset. 10 | 11 | Same as :obj:`torch.utils.data.dataset.ConcatDataset`, but 12 | add `get_cat_ids` function. 13 | 14 | Args: 15 | datasets (list[:obj:`Dataset`]): A list of datasets. 16 | """ 17 | 18 | def __init__(self, datasets: list): 19 | super(ConcatDataset, self).__init__(datasets) 20 | 21 | 22 | @DATASETS.register_module() 23 | class RepeatDataset(object): 24 | """A wrapper of repeated dataset. 25 | 26 | The length of repeated dataset will be `times` larger than the original 27 | dataset. This is useful when the data loading time is long but the dataset 28 | is small. Using RepeatDataset can reduce the data loading time between 29 | epochs. 30 | 31 | Args: 32 | dataset (:obj:`Dataset`): The dataset to be repeated. 33 | times (int): Repeat times. 34 | """ 35 | 36 | def __init__(self, dataset: Dataset, times: int): 37 | self.dataset = dataset 38 | self.times = times 39 | self.CLASSES = dataset.CLASSES 40 | 41 | self._ori_len = len(self.dataset) 42 | 43 | def __getitem__(self, idx: int): 44 | return self.dataset[idx % self._ori_len] 45 | 46 | def __len__(self): 47 | return self.times * self._ori_len 48 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/mesh_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | from abc import ABCMeta 3 | from typing import Optional, Union 4 | 5 | import numpy as np 6 | 7 | from .base_dataset import BaseDataset 8 | from .builder import DATASETS 9 | 10 | 11 | @DATASETS.register_module() 12 | class MeshDataset(BaseDataset, metaclass=ABCMeta): 13 | """Mesh Dataset. This dataset only contains smpl data. 14 | 15 | Args: 16 | data_prefix (str): the prefix of data path. 17 | pipeline (list): a list of dict, where each element represents 18 | a operation defined in `mmhuman3d.datasets.pipelines`. 19 | dataset_name (str | None): the name of dataset. It is used to 20 | identify the type of evaluation metric. Default: None. 21 | ann_file (str | None, optional): the annotation file. When ann_file 22 | is str, the subclass is expected to read from the ann_file. When 23 | ann_file is None, the subclass is expected to read according 24 | to data_prefix. 25 | test_mode (bool, optional): in train mode or test mode. Default: False. 26 | """ 27 | 28 | def __init__(self, 29 | data_prefix: str, 30 | pipeline: list, 31 | dataset_name: str, 32 | ann_file: Optional[Union[str, None]] = None, 33 | test_mode: Optional[bool] = False): 34 | self.dataset_name = dataset_name 35 | super(MeshDataset, self).__init__( 36 | data_prefix=data_prefix, 37 | pipeline=pipeline, 38 | ann_file=ann_file, 39 | test_mode=test_mode) 40 | 41 | def get_annotation_file(self): 42 | ann_prefix = os.path.join(self.data_prefix, 'preprocessed_datasets') 43 | self.ann_file = os.path.join(ann_prefix, self.ann_file) 44 | 45 | def load_annotations(self): 46 | 47 | self.get_annotation_file() 48 | data = np.load(self.ann_file, allow_pickle=True) 49 | 50 | self.smpl = data['smpl'].item() 51 | num_data = self.smpl['global_orient'].shape[0] 52 | if 'transl' not in self.smpl: 53 | self.smpl['transl'] = np.zeros((num_data, 3)) 54 | self.has_smpl = np.ones((num_data)) 55 | 56 | data_infos = [] 57 | 58 | for idx in range(num_data): 59 | info = {} 60 | for k, v in self.smpl.items(): 61 | info['smpl_' + k] = v[idx] 62 | 63 | data_infos.append(info) 64 | self.num_data = len(data_infos) 65 | self.data_infos = data_infos 66 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/mixed_dataset.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | 3 | import numpy as np 4 | from torch.utils.data import ConcatDataset, Dataset, WeightedRandomSampler 5 | 6 | from .builder import DATASETS, build_dataset 7 | 8 | 9 | @DATASETS.register_module() 10 | class MixedDataset(Dataset): 11 | """Mixed Dataset. 12 | 13 | Args: 14 | config (list): the list of different datasets. 15 | partition (list): the ratio of datasets in each batch. 16 | num_data (int | None, optional): if num_data is not None, the number 17 | of iterations is set to this fixed value. Otherwise, the number of 18 | iterations is set to the maximum size of each single dataset. 19 | Default: None. 20 | """ 21 | 22 | def __init__(self, 23 | configs: list, 24 | partition: list, 25 | num_data: Optional[Union[int, None]] = None): 26 | """Load data from multiple datasets.""" 27 | assert min(partition) >= 0 28 | datasets = [build_dataset(cfg) for cfg in configs] 29 | self.dataset = ConcatDataset(datasets) 30 | if num_data is not None: 31 | self.length = num_data 32 | else: 33 | self.length = max(len(ds) for ds in datasets) 34 | weights = [ 35 | np.ones(len(ds)) * p / len(ds) 36 | for (p, ds) in zip(partition, datasets) 37 | ] 38 | weights = np.concatenate(weights, axis=0) 39 | self.sampler = WeightedRandomSampler(weights, 1) 40 | 41 | def __len__(self): 42 | """Get the size of the dataset.""" 43 | return self.length 44 | 45 | def __getitem__(self, idx): 46 | """Given index, sample the data from multiple datasets with the given 47 | proportion.""" 48 | idx_new = list(self.sampler)[0] 49 | return self.dataset[idx_new] 50 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | from .compose import Compose 2 | from .formatting import ( 3 | Collect, 4 | ImageToTensor, 5 | ToNumpy, 6 | ToPIL, 7 | ToTensor, 8 | Transpose, 9 | to_tensor, 10 | ) 11 | from .hybrik_transforms import ( 12 | GenerateHybrIKTarget, 13 | HybrIKAffine, 14 | HybrIKRandomFlip, 15 | NewKeypointsSelection, 16 | RandomDPG, 17 | RandomOcclusion, 18 | ) 19 | from .loading import LoadImageFromFile 20 | from .synthetic_occlusion_augmentation import SyntheticOcclusion 21 | from .transforms import ( 22 | CenterCrop, 23 | ColorJitter, 24 | GetRandomScaleRotation, 25 | Lighting, 26 | MeshAffine, 27 | Normalize, 28 | RandomChannelNoise, 29 | RandomHorizontalFlip, 30 | ) 31 | 32 | __all__ = [ 33 | 'Compose', 34 | 'to_tensor', 35 | 'ToTensor', 36 | 'ImageToTensor', 37 | 'ToPIL', 38 | 'ToNumpy', 39 | 'Transpose', 40 | 'Collect', 41 | 'LoadImageFromFile', 42 | 'CenterCrop', 43 | 'RandomHorizontalFlip', 44 | 'ColorJitter', 45 | 'Lighting', 46 | 'RandomChannelNoise', 47 | 'GetRandomScaleRotation', 48 | 'MeshAffine', 49 | 'HybrIKRandomFlip', 50 | 'HybrIKAffine', 51 | 'GenerateHybrIKTarget', 52 | 'RandomDPG', 53 | 'RandomOcclusion', 54 | 'NewKeypointsSelection', 55 | 'Normalize', 56 | 'SyntheticOcclusion', 57 | ] 58 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/pipelines/compose.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Sequence 2 | 3 | from mmcv.utils import build_from_cfg 4 | 5 | from ..builder import PIPELINES 6 | 7 | 8 | @PIPELINES.register_module() 9 | class Compose(object): 10 | """Compose a data pipeline with a sequence of transforms. 11 | 12 | Args: 13 | transforms (list[dict | callable]): 14 | Either config dicts of transforms or transform objects. 15 | """ 16 | 17 | def __init__(self, transforms): 18 | assert isinstance(transforms, Sequence) 19 | self.transforms = [] 20 | for transform in transforms: 21 | if isinstance(transform, dict): 22 | transform = build_from_cfg(transform, PIPELINES) 23 | self.transforms.append(transform) 24 | elif callable(transform): 25 | self.transforms.append(transform) 26 | else: 27 | raise TypeError('transform must be callable or a dict, but got' 28 | f' {type(transform)}') 29 | 30 | def __call__(self, data): 31 | for t in self.transforms: 32 | data = t(data) 33 | if data is None: 34 | return None 35 | return data 36 | 37 | def __repr__(self): 38 | format_string = self.__class__.__name__ + '(' 39 | for t in self.transforms: 40 | format_string += f'\n {t}' 41 | format_string += '\n)' 42 | return format_string 43 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/pipelines/loading.py: -------------------------------------------------------------------------------- 1 | import os.path as osp 2 | 3 | import cv2 4 | import mmcv 5 | import numpy as np 6 | 7 | import mmhuman3d.data.data_structures as data_structures 8 | from ..builder import PIPELINES 9 | 10 | 11 | @PIPELINES.register_module() 12 | class LoadImageFromFile(object): 13 | """Load an image from file. 14 | 15 | Required keys are "img_prefix" and "img_info" (a dict that must contain the 16 | key "filename"). Added or updated keys are "filename", "img", "img_shape", 17 | "ori_shape" (same as `img_shape`) and "img_norm_cfg" (means=0 and stds=1). 18 | Both "img_shape" and "ori_shape" use (height, width) convention. 19 | 20 | Args: 21 | to_float32 (bool): Whether to convert the loaded image to a float32 22 | numpy array. If set to False, the loaded image is an uint8 array. 23 | Defaults to False. 24 | color_type (str): The flag argument for :func:`mmcv.imfrombytes()`. 25 | Defaults to 'color'. 26 | file_client_args (dict): Arguments to instantiate a FileClient. 27 | See :class:`mmcv.fileio.FileClient` for details. 28 | Defaults to ``dict(backend='disk')``. 29 | """ 30 | 31 | def __init__(self, 32 | to_float32=False, 33 | color_type='color', 34 | file_client_args=dict(backend='disk')): 35 | self.to_float32 = to_float32 36 | self.color_type = color_type 37 | self.file_client_args = file_client_args.copy() 38 | self.file_client = None 39 | 40 | def __call__(self, results): 41 | if self.file_client is None: 42 | self.file_client = mmcv.FileClient(**self.file_client_args) 43 | 44 | if results['img_prefix'] is not None: 45 | filename = osp.join(results['img_prefix'], results['image_path']) 46 | else: 47 | filename = results['image_path'] 48 | 49 | if filename.endswith('smc'): 50 | assert 'image_id' in results, 'Load image from .smc, ' \ 51 | 'but image_id is not provided.' 52 | device, device_id, frame_id = results['image_id'] 53 | smc_reader = data_structures.SMCReader(filename) 54 | img = smc_reader.get_color( 55 | device, device_id, frame_id, disable_tqdm=True) 56 | img = img.squeeze() # (1, H, W, 3) -> (H, W, 3) 57 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # BGR is used 58 | del smc_reader 59 | else: 60 | img_bytes = self.file_client.get(filename) 61 | img = mmcv.imfrombytes(img_bytes, flag=self.color_type) 62 | 63 | if self.to_float32: 64 | img = img.astype(np.float32) 65 | 66 | results['filename'] = filename 67 | results['ori_filename'] = results['image_path'] 68 | results['img'] = img 69 | results['img_shape'] = img.shape[:2] 70 | results['ori_shape'] = img.shape[:2] 71 | num_channels = 1 if len(img.shape) < 3 else img.shape[2] 72 | results['img_norm_cfg'] = dict( 73 | mean=np.zeros(num_channels, dtype=np.float32), 74 | std=np.ones(num_channels, dtype=np.float32), 75 | to_rgb=False) 76 | return results 77 | 78 | def __repr__(self): 79 | repr_str = (f'{self.__class__.__name__}(' 80 | f'to_float32={self.to_float32}, ' 81 | f"color_type='{self.color_type}', " 82 | f'file_client_args={self.file_client_args})') 83 | return repr_str 84 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/samplers/__init__.py: -------------------------------------------------------------------------------- 1 | from .distributed_sampler import DistributedSampler 2 | 3 | __all__ = ['DistributedSampler'] 4 | -------------------------------------------------------------------------------- /mmhuman3d/data/datasets/samplers/distributed_sampler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import DistributedSampler as _DistributedSampler 3 | 4 | 5 | class DistributedSampler(_DistributedSampler): 6 | 7 | def __init__(self, 8 | dataset, 9 | num_replicas=None, 10 | rank=None, 11 | shuffle=True, 12 | round_up=True): 13 | super().__init__(dataset, num_replicas=num_replicas, rank=rank) 14 | self.shuffle = shuffle 15 | self.round_up = round_up 16 | if self.round_up: 17 | self.total_size = self.num_samples * self.num_replicas 18 | else: 19 | self.total_size = len(self.dataset) 20 | 21 | def __iter__(self): 22 | # deterministically shuffle based on epoch 23 | if self.shuffle: 24 | g = torch.Generator() 25 | g.manual_seed(self.epoch) 26 | indices = torch.randperm(len(self.dataset), generator=g).tolist() 27 | else: 28 | indices = torch.arange(len(self.dataset)).tolist() 29 | 30 | # add extra samples to make it evenly divisible 31 | if self.round_up: 32 | indices = ( 33 | indices * 34 | int(self.total_size / len(indices) + 1))[:self.total_size] 35 | assert len(indices) == self.total_size 36 | 37 | # subsample 38 | indices = indices[self.rank:self.total_size:self.num_replicas] 39 | if self.round_up: 40 | assert len(indices) == self.num_samples 41 | 42 | return iter(indices) 43 | -------------------------------------------------------------------------------- /mmhuman3d/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/architectures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/architectures/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/architectures/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.cnn import MODELS as MMCV_MODELS 4 | from mmcv.utils import Registry 5 | 6 | from .hybrik import HybrIK_trainer 7 | from .mesh_estimator import ImageBodyModelEstimator, VideoBodyModelEstimator 8 | 9 | 10 | def build_from_cfg(cfg, registry, default_args=None): 11 | if cfg is None: 12 | return None 13 | return MMCV_MODELS.build_func(cfg, registry, default_args) 14 | 15 | 16 | ARCHITECTURES = Registry( 17 | 'architectures', parent=MMCV_MODELS, build_func=build_from_cfg) 18 | 19 | ARCHITECTURES.register_module(name='HybrIK_trainer', module=HybrIK_trainer) 20 | ARCHITECTURES.register_module( 21 | name='ImageBodyModelEstimator', module=ImageBodyModelEstimator) 22 | ARCHITECTURES.register_module( 23 | name='VideoBodyModelEstimator', module=VideoBodyModelEstimator) 24 | 25 | 26 | def build_architecture(cfg): 27 | """Build framework.""" 28 | return ARCHITECTURES.build(cfg) 29 | -------------------------------------------------------------------------------- /mmhuman3d/models/backbones/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/backbones/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/backbones/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .efficientnet import EfficientNet 6 | from .hourglass import HourglassNet 7 | from .hrnet import PoseHighResolutionNet 8 | from .resnet import ResNet, ResNetV1d 9 | from .resnext import ResNeXt 10 | from .swin import SwinTransformer 11 | from .twins import PCPVT, SVT 12 | from .vit import VisionTransformer 13 | 14 | BACKBONES = Registry('backbones') 15 | 16 | BACKBONES.register_module(name='ResNet', module=ResNet) 17 | BACKBONES.register_module(name='ResNetV1d', module=ResNetV1d) 18 | BACKBONES.register_module( 19 | name='PoseHighResolutionNet', module=PoseHighResolutionNet) 20 | BACKBONES.register_module(name='EfficientNet', module=EfficientNet) 21 | BACKBONES.register_module(name='HourglassNet', module=HourglassNet) 22 | BACKBONES.register_module(name='ResNeXt', module=ResNeXt) 23 | BACKBONES.register_module(name='SwinTransformer', module=SwinTransformer) 24 | BACKBONES.register_module(name='SVT', module=SVT) 25 | BACKBONES.register_module(name='PCPVT', module=PCPVT) 26 | BACKBONES.register_module(name='VisionTransformer', module=VisionTransformer) 27 | 28 | 29 | def build_backbone(cfg): 30 | """Build backbone.""" 31 | if cfg is None: 32 | return None 33 | return BACKBONES.build(cfg) 34 | -------------------------------------------------------------------------------- /mmhuman3d/models/body_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/body_models/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/body_models/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .smpl import SMPL, GenderedSMPL, HybrIKSMPL 6 | from .smplx import SMPLX 7 | 8 | BODY_MODELS = Registry('body_models') 9 | 10 | BODY_MODELS.register_module(name=['SMPL', 'smpl'], module=SMPL) 11 | BODY_MODELS.register_module(name='GenderedSMPL', module=GenderedSMPL) 12 | BODY_MODELS.register_module( 13 | name=['HybrIKSMPL', 'HybrIKsmpl', 'hybriksmpl', 'hybrik', 'hybrIK'], 14 | module=HybrIKSMPL) 15 | BODY_MODELS.register_module(name=['SMPLX', 'smplx'], module=SMPLX) 16 | 17 | 18 | def build_body_model(cfg): 19 | """Build body_models.""" 20 | if cfg is None: 21 | return None 22 | return BODY_MODELS.build(cfg) 23 | -------------------------------------------------------------------------------- /mmhuman3d/models/discriminators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/discriminators/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/discriminators/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .pose_discriminator import ( 6 | FullPoseDiscriminator, 7 | PoseDiscriminator, 8 | ShapeDiscriminator, 9 | SMPLDiscriminator, 10 | ) 11 | 12 | DISCRIMINATORS = Registry('discriminators') 13 | 14 | DISCRIMINATORS.register_module( 15 | name='ShapeDiscriminator', module=ShapeDiscriminator) 16 | DISCRIMINATORS.register_module( 17 | name='PoseDiscriminator', module=PoseDiscriminator) 18 | DISCRIMINATORS.register_module( 19 | name='FullPoseDiscriminator', module=FullPoseDiscriminator) 20 | DISCRIMINATORS.register_module( 21 | name='SMPLDiscriminator', module=SMPLDiscriminator) 22 | 23 | 24 | def build_discriminator(cfg): 25 | """Build discriminator.""" 26 | if cfg is None: 27 | return None 28 | return DISCRIMINATORS.build(cfg) 29 | -------------------------------------------------------------------------------- /mmhuman3d/models/heads/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/heads/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/heads/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .hmr_head import HMRHead 6 | from .hmr_hrnet_head import HMRHrNetHead 7 | from .hybrik_head import HybrIKHead 8 | from .pare_head import PareHead 9 | 10 | HEADS = Registry('heads') 11 | 12 | HEADS.register_module(name='HybrIKHead', module=HybrIKHead) 13 | HEADS.register_module(name='HMRHead', module=HMRHead) 14 | HEADS.register_module(name='PareHead', module=PareHead) 15 | HEADS.register_module(name='HMRHrNetHead', module=HMRHrNetHead) 16 | 17 | 18 | def build_head(cfg): 19 | """Build head.""" 20 | if cfg is None: 21 | return None 22 | return HEADS.build(cfg) 23 | -------------------------------------------------------------------------------- /mmhuman3d/models/losses/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/losses/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/losses/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .cross_entropy_loss import CrossEntropyLoss 6 | from .gan_loss import GANLoss 7 | from .mse_loss import KeypointMSELoss, MSELoss 8 | from .prior_loss import ( 9 | CameraPriorLoss, 10 | JointPriorLoss, 11 | LimbLengthLoss, 12 | MaxMixturePrior, 13 | PoseRegLoss, 14 | ShapePriorLoss, 15 | SmoothJointLoss, 16 | SmoothPelvisLoss, 17 | SmoothTranslationLoss, 18 | ) 19 | from .smooth_l1_loss import L1Loss, SmoothL1Loss 20 | 21 | LOSSES = Registry('losses') 22 | 23 | LOSSES.register_module(name='GANLoss', module=GANLoss) 24 | LOSSES.register_module(name='MSELoss', module=MSELoss) 25 | LOSSES.register_module(name='KeypointMSELoss', module=KeypointMSELoss) 26 | LOSSES.register_module(name='ShapePriorLoss', module=ShapePriorLoss) 27 | LOSSES.register_module(name='PoseRegLoss', module=PoseRegLoss) 28 | LOSSES.register_module(name='LimbLengthLoss', module=LimbLengthLoss) 29 | LOSSES.register_module(name='JointPriorLoss', module=JointPriorLoss) 30 | LOSSES.register_module(name='SmoothJointLoss', module=SmoothJointLoss) 31 | LOSSES.register_module(name='SmoothPelvisLoss', module=SmoothPelvisLoss) 32 | LOSSES.register_module( 33 | name='SmoothTranslationLoss', module=SmoothTranslationLoss) 34 | LOSSES.register_module(name='CameraPriorLoss', module=CameraPriorLoss) 35 | LOSSES.register_module(name='MaxMixturePrior', module=MaxMixturePrior) 36 | LOSSES.register_module(name='L1Loss', module=L1Loss) 37 | LOSSES.register_module(name='SmoothL1Loss', module=SmoothL1Loss) 38 | LOSSES.register_module(name='CrossEntropyLoss', module=CrossEntropyLoss) 39 | 40 | 41 | def build_loss(cfg): 42 | """Build loss.""" 43 | if cfg is None: 44 | return None 45 | return LOSSES.build(cfg) 46 | -------------------------------------------------------------------------------- /mmhuman3d/models/losses/gan_loss.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch.nn as nn 3 | 4 | 5 | class GANLoss(nn.Module): 6 | """Define GAN loss. 7 | 8 | Args: 9 | gan_type (str): Support 'vanilla', 'lsgan', 'wgan', 'hinge'. 10 | real_label_val (float): The value for real label. Default: 1.0. 11 | fake_label_val (float): The value for fake label. Default: 0.0. 12 | loss_weight (float): Loss weight. Default: 1.0. 13 | Note that loss_weight is only for generators; and it is always 1.0 14 | for discriminators. 15 | """ 16 | 17 | def __init__(self, 18 | gan_type, 19 | real_label_val=1.0, 20 | fake_label_val=0.0, 21 | loss_weight=1.0): 22 | super().__init__() 23 | self.gan_type = gan_type 24 | self.loss_weight = loss_weight 25 | self.real_label_val = real_label_val 26 | self.fake_label_val = fake_label_val 27 | 28 | if self.gan_type == 'vanilla': 29 | self.loss = nn.BCEWithLogitsLoss() 30 | elif self.gan_type == 'lsgan': 31 | self.loss = nn.MSELoss() 32 | elif self.gan_type == 'wgan': 33 | self.loss = self._wgan_loss 34 | elif self.gan_type == 'hinge': 35 | self.loss = nn.ReLU() 36 | else: 37 | raise NotImplementedError( 38 | f'GAN type {self.gan_type} is not implemented.') 39 | 40 | @staticmethod 41 | def _wgan_loss(input, target): 42 | """wgan loss. 43 | 44 | Args: 45 | input (Tensor): Input tensor. 46 | target (bool): Target label. 47 | Returns: 48 | Tensor: wgan loss. 49 | """ 50 | return -input.mean() if target else input.mean() 51 | 52 | def get_target_label(self, input, target_is_real): 53 | """Get target label. 54 | 55 | Args: 56 | input (Tensor): Input tensor. 57 | target_is_real (bool): Whether the target is real or fake. 58 | Returns: 59 | (bool | Tensor): Target tensor. Return bool for wgan, otherwise, 60 | return Tensor. 61 | """ 62 | 63 | if self.gan_type == 'wgan': 64 | return target_is_real 65 | target_val = ( 66 | self.real_label_val if target_is_real else self.fake_label_val) 67 | return input.new_ones(input.size()) * target_val 68 | 69 | def forward(self, input, target_is_real, is_disc=False): 70 | """ 71 | Args: 72 | input (Tensor): The input for the loss module, i.e., the network 73 | prediction. 74 | target_is_real (bool): Whether the targe is real or fake. 75 | is_disc (bool): Whether the loss for discriminators or not. 76 | Default: False. 77 | Returns: 78 | Tensor: GAN loss value. 79 | """ 80 | target_label = self.get_target_label(input, target_is_real) 81 | if self.gan_type == 'hinge': 82 | if is_disc: # for discriminators in hinge-gan 83 | input = -input if target_is_real else input 84 | loss = self.loss(1 + input).mean() 85 | else: # for generators in hinge-gan 86 | loss = -input.mean() 87 | else: # other gan types 88 | loss = self.loss(input, target_label) 89 | 90 | # loss_weight is always 1.0 for discriminators 91 | return loss if is_disc else loss * self.loss_weight 92 | -------------------------------------------------------------------------------- /mmhuman3d/models/necks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/necks/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/necks/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .temporal_encoder import TemporalGRUEncoder 6 | 7 | NECKS = Registry('necks') 8 | 9 | NECKS.register_module(name='TemporalGRUEncoder', module=TemporalGRUEncoder) 10 | 11 | 12 | def build_neck(cfg): 13 | """Build neck.""" 14 | if cfg is None: 15 | return None 16 | return NECKS.build(cfg) 17 | -------------------------------------------------------------------------------- /mmhuman3d/models/necks/temporal_encoder.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | 3 | import torch.nn as nn 4 | from mmcv.runner.base_module import BaseModule 5 | 6 | 7 | class TemporalGRUEncoder(BaseModule): 8 | """TemporalEncoder used for VIBE. Adapted from 9 | https://github.com/mkocabas/VIBE. 10 | 11 | Args: 12 | input_size (int, optional): dimension of input feature. Default: 2048. 13 | num_layer (int, optional): number of layers for GRU. Default: 1. 14 | hidden_size (int, optional): hidden size for GRU. Default: 2048. 15 | init_cfg (dict or list[dict], optional): Initialization config dict. 16 | Default: None. 17 | """ 18 | 19 | def __init__(self, 20 | input_size: Optional[int] = 2048, 21 | num_layers: Optional[int] = 1, 22 | hidden_size: Optional[int] = 2048, 23 | init_cfg: Optional[Union[list, dict, None]] = None): 24 | super(TemporalGRUEncoder, self).__init__(init_cfg) 25 | 26 | self.input_size = input_size 27 | self.hidden_size = hidden_size 28 | self.gru = nn.GRU( 29 | input_size=input_size, 30 | hidden_size=hidden_size, 31 | bidirectional=False, 32 | num_layers=num_layers) 33 | self.relu = nn.ReLU() 34 | self.linear = self.linear = nn.Linear(hidden_size, input_size) 35 | 36 | def forward(self, x): 37 | N, T = x.shape[:2] 38 | x = x.permute(1, 0, 2) 39 | y, _ = self.gru(x) 40 | y = self.linear(self.relu(y).view(-1, self.hidden_size)) 41 | y = y.view(T, N, self.input_size) + x 42 | y = y.permute(1, 0, 2).contiguous() 43 | return y 44 | -------------------------------------------------------------------------------- /mmhuman3d/models/registrants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/mmhuman3d/models/registrants/__init__.py -------------------------------------------------------------------------------- /mmhuman3d/models/registrants/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | from mmcv.utils import Registry 4 | 5 | from .smplify import SMPLify 6 | from .smplifyx import SMPLifyX 7 | 8 | REGISTRANTS = Registry('registrants') 9 | 10 | REGISTRANTS.register_module(name='SMPLify', module=SMPLify) 11 | REGISTRANTS.register_module(name='SMPLifyX', module=SMPLifyX) 12 | 13 | 14 | def build_registrant(cfg): 15 | """Build registrant.""" 16 | if cfg is None: 17 | return None 18 | return REGISTRANTS.build(cfg) 19 | -------------------------------------------------------------------------------- /mmhuman3d/models/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .fits_dict import FitsDict 2 | from .inverse_kinematics import batch_inverse_kinematics_transform 3 | from .res_layer import ResLayer, SimplifiedBasicBlock 4 | 5 | __all__ = [ 6 | 'FitsDict', 'ResLayer', 'SimplifiedBasicBlock', 7 | 'batch_inverse_kinematics_transform' 8 | ] 9 | -------------------------------------------------------------------------------- /mmhuman3d/utils/collect_env.py: -------------------------------------------------------------------------------- 1 | from mmcv.utils import collect_env as collect_base_env 2 | from mmcv.utils import get_git_hash 3 | 4 | import mmhuman3d 5 | 6 | 7 | def collect_env(): 8 | """Collect the information of the running environments.""" 9 | env_info = collect_base_env() 10 | env_info['MMHuman3d'] = mmhuman3d.__version__ + '+' + get_git_hash()[:7] 11 | return env_info 12 | 13 | 14 | if __name__ == '__main__': 15 | for name, val in collect_env().items(): 16 | print(f'{name}: {val}') 17 | -------------------------------------------------------------------------------- /mmhuman3d/utils/dist_utils.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import torch.distributed as dist 4 | from mmcv.runner import OptimizerHook 5 | from torch._utils import ( 6 | _flatten_dense_tensors, 7 | _take_tensors, 8 | _unflatten_dense_tensors, 9 | ) 10 | 11 | 12 | def _allreduce_coalesced(tensors, world_size, bucket_size_mb=-1): 13 | if bucket_size_mb > 0: 14 | bucket_size_bytes = bucket_size_mb * 1024 * 1024 15 | buckets = _take_tensors(tensors, bucket_size_bytes) 16 | else: 17 | buckets = OrderedDict() 18 | for tensor in tensors: 19 | tp = tensor.type() 20 | if tp not in buckets: 21 | buckets[tp] = [] 22 | buckets[tp].append(tensor) 23 | buckets = buckets.values() 24 | 25 | for bucket in buckets: 26 | flat_tensors = _flatten_dense_tensors(bucket) 27 | dist.all_reduce(flat_tensors) 28 | flat_tensors.div_(world_size) 29 | for tensor, synced in zip( 30 | bucket, _unflatten_dense_tensors(flat_tensors, bucket)): 31 | tensor.copy_(synced) 32 | 33 | 34 | def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): 35 | grads = [ 36 | param.grad.data for param in params 37 | if param.requires_grad and param.grad is not None 38 | ] 39 | world_size = dist.get_world_size() 40 | if coalesce: 41 | _allreduce_coalesced(grads, world_size, bucket_size_mb) 42 | else: 43 | for tensor in grads: 44 | dist.all_reduce(tensor.div_(world_size)) 45 | 46 | 47 | class DistOptimizerHook(OptimizerHook): 48 | 49 | def __init__(self, grad_clip=None, coalesce=True, bucket_size_mb=-1): 50 | self.grad_clip = grad_clip 51 | self.coalesce = coalesce 52 | self.bucket_size_mb = bucket_size_mb 53 | 54 | def after_train_iter(self, runner): 55 | runner.optimizer.zero_grad() 56 | runner.outputs['loss'].backward() 57 | if self.grad_clip is not None: 58 | self.clip_grads(runner.model.parameters()) 59 | runner.optimizer.step() 60 | -------------------------------------------------------------------------------- /mmhuman3d/utils/keypoint_utils.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple, Union 2 | 3 | import numpy as np 4 | 5 | from mmhuman3d.core.conventions.keypoints_mapping import KEYPOINTS_FACTORY 6 | from mmhuman3d.core.conventions.keypoints_mapping.human_data import ( 7 | HUMAN_DATA_LIMBS_INDEX, 8 | HUMAN_DATA_PALETTE, 9 | ) 10 | 11 | 12 | def search_limbs( 13 | data_source: str, 14 | mask: Optional[Union[np.ndarray, tuple, list]] = None, 15 | keypoints_factory: dict = KEYPOINTS_FACTORY) -> Tuple[dict, dict]: 16 | """Search the corresponding limbs following the basis human_data limbs. The 17 | mask could mask out the incorrect keypoints. 18 | 19 | Args: 20 | data_source (str): data source type. 21 | mask (Optional[Union[np.ndarray, tuple, list]], optional): 22 | refer to keypoints_mapping. Defaults to None. 23 | keypoints_factory (dict, optional): Dict of all the conventions. 24 | Defaults to KEYPOINTS_FACTORY. 25 | Returns: 26 | Tuple[dict, dict]: (limbs_target, limbs_palette). 27 | """ 28 | limbs_source = HUMAN_DATA_LIMBS_INDEX 29 | limbs_palette = HUMAN_DATA_PALETTE 30 | keypoints_source = keypoints_factory['human_data'] 31 | keypoints_target = keypoints_factory[data_source] 32 | limbs_target = {} 33 | for k, part_limbs in limbs_source.items(): 34 | limbs_target[k] = [] 35 | for limb in part_limbs: 36 | flag = False 37 | if (keypoints_source[limb[0]] 38 | in keypoints_target) and (keypoints_source[limb[1]] 39 | in keypoints_target): 40 | if mask is not None: 41 | if mask[keypoints_target.index(keypoints_source[ 42 | limb[0]])] != 0 and mask[keypoints_target.index( 43 | keypoints_source[limb[1]])] != 0: 44 | flag = True 45 | else: 46 | flag = True 47 | if flag: 48 | limbs_target.setdefault(k, []).append([ 49 | keypoints_target.index(keypoints_source[limb[0]]), 50 | keypoints_target.index(keypoints_source[limb[1]]) 51 | ]) 52 | if k in limbs_target: 53 | if k == 'body': 54 | np.random.seed(0) 55 | limbs_palette[k] = np.random.randint( 56 | 0, high=255, size=(len(limbs_target[k]), 3)) 57 | else: 58 | limbs_palette[k] = np.array(limbs_palette[k]) 59 | return limbs_target, limbs_palette 60 | -------------------------------------------------------------------------------- /mmhuman3d/utils/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from mmcv.utils import get_logger 4 | 5 | 6 | def get_root_logger(log_file=None, log_level=logging.INFO): 7 | return get_logger('mmhuman3d', log_file, log_level) 8 | -------------------------------------------------------------------------------- /mmhuman3d/utils/misc.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import torch 4 | 5 | 6 | def multi_apply(func, *args, **kwargs): 7 | pfunc = partial(func, **kwargs) if kwargs else func 8 | map_results = map(pfunc, *args) 9 | return tuple(map(list, zip(*map_results))) 10 | 11 | 12 | def torch_to_numpy(x): 13 | assert isinstance(x, torch.Tensor) 14 | return x.detach().cpu().numpy() 15 | -------------------------------------------------------------------------------- /mmhuman3d/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Open-MMLab. All rights reserved. 2 | 3 | __version__ = '0.8.0' 4 | 5 | 6 | def parse_version_info(version_str): 7 | """Parse a version string into a tuple. 8 | 9 | Args: 10 | version_str (str): The version string. 11 | Returns: 12 | tuple[int | str]: The version info, e.g., "1.3.0" is parsed into 13 | (1, 3, 0), and "2.0.0rc1" is parsed into (2, 0, 0, 'rc1'). 14 | """ 15 | version_info = [] 16 | for x in version_str.split('.'): 17 | if x.isdigit(): 18 | version_info.append(int(x)) 19 | elif x.find('rc') != -1: 20 | patch_version = x.split('rc') 21 | version_info.append(int(patch_version[0])) 22 | version_info.append(f'rc{patch_version[1]}') 23 | return tuple(version_info) 24 | 25 | 26 | version_info = parse_version_info(__version__) 27 | 28 | __all__ = ['__version__', 'version_info', 'parse_version_info'] 29 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements/runtime.txt 2 | -r requirements/tests.txt 3 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | docutils==0.16.0 2 | myst-parser 3 | -e git+https://github.com/open-mmlab/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme 4 | sphinx==4.0.2 5 | sphinx-copybutton 6 | sphinx_markdown_tables 7 | sphinx_rtd_theme==0.5.2 8 | -------------------------------------------------------------------------------- /requirements/readthedocs.txt: -------------------------------------------------------------------------------- 1 | mmcv 2 | torch 3 | torchvision 4 | -------------------------------------------------------------------------------- /requirements/runtime.txt: -------------------------------------------------------------------------------- 1 | astropy 2 | cdflib<0.4.0 3 | chumpy 4 | colormap 5 | easydev 6 | h5py 7 | matplotlib 8 | numpy 9 | opencv-python 10 | pandas 11 | pickle5 12 | scikit-image 13 | scipy 14 | smplx 15 | tqdm 16 | vedo 17 | -------------------------------------------------------------------------------- /requirements/tests.txt: -------------------------------------------------------------------------------- 1 | codecov 2 | flake8 3 | interrogate 4 | isort==4.3.21 5 | pytest 6 | xdoctest >= 0.10.0 7 | yapf 8 | -------------------------------------------------------------------------------- /resources/dance.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/resources/dance.gif -------------------------------------------------------------------------------- /resources/dance001.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/resources/dance001.gif -------------------------------------------------------------------------------- /resources/dance3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/resources/dance3.gif -------------------------------------------------------------------------------- /resources/mmhuman3d-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/resources/mmhuman3d-logo.png -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | 4 | [aliases] 5 | test=pytest 6 | 7 | [yapf] 8 | based_on_style = pep8 9 | blank_line_before_nested_class_or_def = true 10 | split_before_expression_after_opening_paren = true 11 | 12 | [isort] 13 | line_length = 79 14 | multi_line_output = 3 15 | include_trailing_comma = true 16 | known_standard_library = pkg_resources,setuptools 17 | known_first_party = mmhuman3d 18 | known_third_party =PIL,albumentations,cdflib,colormap,cv2,h5py,matplotlib,mmcv,mpl_toolkits,numpy,pytest,pytorch3d,pytorch_sphinx_theme,scipy,skimage,smplx,torch,tqdm,vedo 19 | no_lines_before = STDLIB,LOCALFOLDER 20 | default_section = THIRDPARTY 21 | -------------------------------------------------------------------------------- /tests/test_datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/tests/test_datasets/__init__.py -------------------------------------------------------------------------------- /tests/test_datasets/test_pipelines.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | from mmhuman3d.data.datasets.pipelines import ( 5 | LoadImageFromFile, 6 | SyntheticOcclusion, 7 | ) 8 | 9 | test_image_path = 'tests/data/dataset_sample/3DPW/imageFiles/' \ 10 | 'courtyard_arguing_00/image_00000.jpg' 11 | test_smc_path = 'tests/data/dataset_sample/humman/p000003_a000014_tiny.smc' 12 | 13 | 14 | def test_load_image_from_file(): 15 | results = {'img_prefix': None, 'image_path': test_image_path} 16 | 17 | pipeline = LoadImageFromFile() 18 | results = pipeline(results) 19 | 20 | assert results['filename'] == results['ori_filename'] == test_image_path 21 | assert isinstance(results['img'], np.ndarray) 22 | assert results['img_shape'] == results['ori_shape'] == (1920, 1080) 23 | assert isinstance(results['img_norm_cfg'], dict) 24 | 25 | 26 | def test_load_image_from_file_smc(): 27 | results = {'img_prefix': None, 'image_path': test_smc_path} 28 | 29 | pipeline = LoadImageFromFile() 30 | 31 | with pytest.raises(AssertionError): 32 | results = pipeline(results) 33 | 34 | results['image_id'] = ('Kinect', 0, 0) 35 | results = pipeline(results) 36 | 37 | assert results['filename'] == results['ori_filename'] == test_smc_path 38 | assert isinstance(results['img'], np.ndarray) 39 | assert results['img_shape'] == results['ori_shape'] == (1080, 1920) 40 | assert isinstance(results['img_norm_cfg'], dict) 41 | 42 | results['image_id'] = ('iPhone', 0, 0) 43 | results = pipeline(results) 44 | 45 | assert results['filename'] == results['ori_filename'] == test_smc_path 46 | assert isinstance(results['img'], np.ndarray) 47 | assert results['img_shape'] == results['ori_shape'] == (1920, 1440) 48 | assert isinstance(results['img_norm_cfg'], dict) 49 | 50 | 51 | def test_synthetic_occlusion(): 52 | results = {'img': None} 53 | results['img'] = np.ones((224, 224, 3)) 54 | occluders = [np.zeros((18, 18, 4))] 55 | occluders[0][2:5, 2:5, 3] = 255 56 | pipeline = SyntheticOcclusion(occluders=occluders) 57 | 58 | results = pipeline(results) 59 | assert results['img'].shape == (224, 224, 3) 60 | -------------------------------------------------------------------------------- /tests/test_human_data_cache.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | from mmhuman3d.core.conventions.keypoints_mapping import get_keypoint_num 5 | from mmhuman3d.data.data_structures.human_data import HumanData 6 | from mmhuman3d.data.data_structures.human_data_cache import ( 7 | HumanDataCacheReader, 8 | HumanDataCacheWriter, 9 | ) 10 | 11 | human_data_cache_path = 'tests/data/human_data/human_data_cache.npz' 12 | 13 | 14 | def test_write(): 15 | human_data = HumanData.new(key_strict=False) 16 | keypoint_num_hd = get_keypoint_num(convention='human_data') 17 | human_data['keypoints2d'] = np.ones(shape=(100, keypoint_num_hd, 3)) 18 | human_data['keypoints2d_mask'] = np.ones(shape=(keypoint_num_hd, )) 19 | human_data['keypoints2d_convention'] = 'human_data' 20 | human_data['keypoints2d'][50, 50, :] *= 3 21 | keypoint_num_smplx = get_keypoint_num(convention='smplx') 22 | human_data['keypoints4d'] = np.ones(shape=(100, keypoint_num_smplx, 3)) 23 | human_data['keypoints4d_mask'] = np.ones(shape=(keypoint_num_smplx, )) 24 | human_data['keypoints4d_mask'][0:10] *= 0 25 | human_data['keypoints4d_convention'] = 'smplx' 26 | human_data['config'] = 'config/example' 27 | human_data['image_path'] = [str(x) for x in range(100)] 28 | human_data['smpl'] = { 29 | 'global_orient': np.ones(shape=(100, 3)), 30 | 'betas': np.ones(shape=(100, 10)), 31 | 'body_pose': np.ones(shape=(100, 21, 3)), 32 | 'expression': [ 33 | 0, 34 | ], 35 | } 36 | human_data.compress_keypoints_by_mask() 37 | 38 | writer_kwargs, sliced_data = human_data.get_sliced_cache() 39 | writer = HumanDataCacheWriter(**writer_kwargs) 40 | writer.update_sliced_dict(sliced_data) 41 | writer.dump(human_data_cache_path, overwrite=True) 42 | with pytest.raises(FileExistsError): 43 | writer.dump(human_data_cache_path, overwrite=False) 44 | 45 | 46 | def test_read(): 47 | reader = HumanDataCacheReader(npz_path=human_data_cache_path) 48 | config = reader.get_non_sliced_data('config') 49 | assert config == 'config/example' 50 | slice_with_0 = reader.get_item(0) 51 | assert isinstance(slice_with_0, HumanData) 52 | slice_with_50 = reader.get_item(50) 53 | assert slice_with_50['image_path'][0] == '50' 54 | assert slice_with_50['keypoints2d'][0, 50, 1] == 3 55 | assert 'config' not in slice_with_50.keys() 56 | slice_with_config = reader.get_item(51, required_keys=['config']) 57 | assert 'config' in slice_with_config.keys() 58 | slice_compressed = reader.get_item(52) 59 | assert slice_compressed.get_raw_value('keypoints4d').shape[2] < 144 60 | slice_with_smpl = reader.get_item(52) 61 | assert slice_with_smpl['smpl']['body_pose'].shape[1:] == (21, 3) 62 | assert 'expression' not in slice_with_smpl['smpl'] 63 | slice_with_smpl = reader.get_item(53, ['smpl']) 64 | assert slice_with_smpl['smpl']['global_orient'].shape[1] == 3 65 | assert slice_with_smpl['smpl']['expression'][0] == 0 66 | -------------------------------------------------------------------------------- /tests/test_models/test_architectures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SMPLCap/hmr-benchmarks/47d959c7bc44e8d8c7348f4fb9cc96e6e35a5b97/tests/test_models/test_architectures/__init__.py -------------------------------------------------------------------------------- /tests/test_models/test_backbones/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import check_norm_state, is_block, is_norm 2 | 3 | __all__ = ['is_block', 'is_norm', 'check_norm_state'] 4 | -------------------------------------------------------------------------------- /tests/test_models/test_backbones/utils.py: -------------------------------------------------------------------------------- 1 | from torch.nn.modules import GroupNorm 2 | from torch.nn.modules.batchnorm import _BatchNorm 3 | 4 | from mmhuman3d.models.backbones.resnet import BasicBlock, Bottleneck 5 | from mmhuman3d.models.utils import SimplifiedBasicBlock 6 | 7 | 8 | def is_block(modules): 9 | """Check if is ResNet building block.""" 10 | if isinstance(modules, (BasicBlock, Bottleneck, SimplifiedBasicBlock)): 11 | return True 12 | return False 13 | 14 | 15 | def is_norm(modules): 16 | """Check if is one of the norms.""" 17 | if isinstance(modules, (GroupNorm, _BatchNorm)): 18 | return True 19 | return False 20 | 21 | 22 | def check_norm_state(modules, train_state): 23 | """Check if norm layer is in correct train state.""" 24 | for mod in modules: 25 | if isinstance(mod, _BatchNorm): 26 | if mod.training != train_state: 27 | return False 28 | return True 29 | -------------------------------------------------------------------------------- /tests/test_models/test_heads/test_hmr_head.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from mmhuman3d.models.heads.builder import HMRHead 5 | 6 | 7 | def test_hmr_head(): 8 | model = HMRHead(2048) 9 | # image feature from backbone 10 | x0_shape = (32, 2048, 7, 7) 11 | x0 = _demo_inputs(x0_shape) 12 | y0 = model(x0) 13 | assert y0['pred_pose'].shape == (32, 24, 3, 3) 14 | assert y0['pred_shape'].shape == (32, 10) 15 | assert y0['pred_cam'].shape == (32, 3) 16 | 17 | # image feature from multi-layer backbone 18 | x1_1_shape = (32, 1024, 14, 14) 19 | x1_2_shape = (32, 2048, 7, 7) 20 | x1 = [_demo_inputs(x1_1_shape), _demo_inputs(x1_2_shape)] 21 | y1 = model(x1) 22 | assert y1['pred_pose'].shape == (32, 24, 3, 3) 23 | assert y1['pred_shape'].shape == (32, 10) 24 | assert y1['pred_cam'].shape == (32, 3) 25 | 26 | # image feature from dataset 27 | x2_shape = (32, 2048) 28 | x2 = _demo_inputs(x2_shape) 29 | y2 = model(x2) 30 | assert y2['pred_pose'].shape == (32, 24, 3, 3) 31 | assert y2['pred_shape'].shape == (32, 10) 32 | assert y2['pred_cam'].shape == (32, 3) 33 | 34 | # video feature from dataset 35 | x3_shape = (32, 32, 2048) 36 | x3 = _demo_inputs(x3_shape) 37 | y3 = model(x3) 38 | assert y3['pred_pose'].shape == (32, 32, 24, 3, 3) 39 | assert y3['pred_shape'].shape == (32, 32, 10) 40 | assert y3['pred_cam'].shape == (32, 32, 3) 41 | 42 | 43 | def _demo_inputs(input_shape=(1, 3, 64, 64)): 44 | """Create a superset of inputs needed to run models. 45 | 46 | Args: 47 | input_shape (tuple): input batch dimensions. 48 | Default: (1, 3, 64, 64). 49 | """ 50 | feat = np.random.random(input_shape) 51 | feat = torch.FloatTensor(feat) 52 | 53 | return feat 54 | -------------------------------------------------------------------------------- /tests/test_models/test_heads/test_pare_head.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | import torch 4 | 5 | from mmhuman3d.models.heads.builder import PareHead 6 | 7 | 8 | @pytest.mark.parametrize('deconv_with_bias', [True, False]) 9 | def test_pare_head(deconv_with_bias): 10 | 11 | # generate weight file for SMPL model. 12 | 13 | # initialize models 14 | head = PareHead( 15 | backbone='hrnet_w32-conv', 16 | use_keypoint_attention=True, 17 | smpl_mean_params='data/body_models/smpl_mean_params.npz', 18 | deconv_with_bias=deconv_with_bias) 19 | 20 | # mock inputs 21 | batch_size = 4 22 | input_shape = (batch_size, 480, 64, 64) 23 | features = _demo_head_inputs(input_shape) 24 | features = torch.tensor(features).float() 25 | 26 | predictions = head(features) 27 | pred_keys = ['pred_pose', 'pred_cam', 'pred_shape'] 28 | 29 | for k in pred_keys: 30 | assert k in predictions 31 | assert predictions[k].shape[0] == batch_size 32 | 33 | 34 | def test_pare_head_no_attention(): 35 | 36 | # generate weight file for SMPL model. 37 | 38 | # initialize models 39 | head = PareHead( 40 | backbone='hrnet_w32-conv', 41 | use_keypoint_attention=False, 42 | use_heatmaps='', 43 | smpl_mean_params='data/body_models/smpl_mean_params.npz', 44 | ) 45 | 46 | # mock inputs 47 | batch_size = 4 48 | input_shape = (batch_size, 480, 64, 64) 49 | features = _demo_head_inputs(input_shape) 50 | features = torch.tensor(features).float() 51 | 52 | predictions = head(features) 53 | pred_keys = ['pred_pose', 'pred_cam', 'pred_shape'] 54 | 55 | for k in pred_keys: 56 | assert k in predictions 57 | assert predictions[k].shape[0] == batch_size 58 | 59 | 60 | def _demo_head_inputs(input_shape=(1, 480, 56, 56)): 61 | """Create a superset of inputs needed to run test or train batches. 62 | 63 | Args: 64 | input_shape (tuple): 65 | input batch dimensions 66 | """ 67 | (N, C, H, W) = input_shape 68 | 69 | rng = np.random.RandomState(0) 70 | 71 | features = rng.rand(*input_shape) 72 | 73 | return features 74 | -------------------------------------------------------------------------------- /tests/test_models/test_losses/test_loss.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import torch 3 | 4 | from mmhuman3d.models.losses.builder import CrossEntropyLoss, L1Loss, MSELoss 5 | 6 | 7 | @pytest.mark.parametrize('loss_class', [MSELoss, L1Loss, CrossEntropyLoss]) 8 | def test_loss_with_reduction_override(loss_class): 9 | pred = torch.rand((10, 3)) 10 | target = torch.rand((10, 3)), 11 | weight = None 12 | 13 | with pytest.raises(AssertionError): 14 | # only reduction_override from [None, 'none', 'mean', 'sum'] 15 | # is not allowed 16 | reduction_override = True 17 | loss_class()( 18 | pred, target, weight, reduction_override=reduction_override) 19 | 20 | 21 | @pytest.mark.parametrize('loss_class', [MSELoss, L1Loss]) 22 | def test_regression_losses(loss_class): 23 | pred = torch.rand((10, 3)) 24 | target = torch.rand((10, 3)) 25 | weight = torch.rand((10, 3)) 26 | 27 | # Test loss forward 28 | loss = loss_class()(pred, target) 29 | assert isinstance(loss, torch.Tensor) 30 | 31 | # Test loss forward with weight 32 | loss = loss_class()(pred, target, weight) 33 | assert isinstance(loss, torch.Tensor) 34 | 35 | # Test loss forward with reduction_override 36 | loss = loss_class()(pred, target, reduction_override='mean') 37 | assert isinstance(loss, torch.Tensor) 38 | 39 | # Test loss forward with avg_factor 40 | loss = loss_class()(pred, target, avg_factor=10) 41 | assert isinstance(loss, torch.Tensor) 42 | 43 | with pytest.raises(ValueError): 44 | # loss can evaluate with avg_factor only if 45 | # reduction is None, 'none' or 'mean'. 46 | reduction_override = 'sum' 47 | loss_class()( 48 | pred, target, avg_factor=10, reduction_override=reduction_override) 49 | 50 | # Test loss forward with avg_factor and reduction 51 | for reduction_override in [None, 'none', 'mean']: 52 | loss_class()( 53 | pred, target, avg_factor=10, reduction_override=reduction_override) 54 | assert isinstance(loss, torch.Tensor) 55 | 56 | 57 | @pytest.mark.parametrize('use_sigmoid', [True, False]) 58 | @pytest.mark.parametrize('reduction', ['sum', 'mean', None]) 59 | def test_loss_with_ignore_index(use_sigmoid, reduction): 60 | # Test cross_entropy loss 61 | 62 | loss_class = CrossEntropyLoss( 63 | use_sigmoid=use_sigmoid, 64 | use_mask=False, 65 | ignore_index=255, 66 | ) 67 | pred = torch.rand((10, 5)) 68 | target = torch.randint(0, 5, (10, )) 69 | 70 | ignored_indices = torch.randint(0, 10, (2, ), dtype=torch.long) 71 | target[ignored_indices] = 255 72 | 73 | # Test loss forward with default ignore 74 | loss_with_ignore = loss_class(pred, target, reduction_override=reduction) 75 | assert isinstance(loss_with_ignore, torch.Tensor) 76 | 77 | # Test loss forward with forward ignore 78 | target[ignored_indices] = 255 79 | loss_with_forward_ignore = loss_class( 80 | pred, target, ignore_index=255, reduction_override=reduction) 81 | assert isinstance(loss_with_forward_ignore, torch.Tensor) 82 | 83 | # Verify correctness 84 | 85 | loss = loss_class(pred, target, reduction_override=reduction) 86 | 87 | assert torch.allclose(loss, loss_with_ignore) 88 | assert torch.allclose(loss, loss_with_forward_ignore) 89 | 90 | # test ignore all target 91 | pred = torch.rand((10, 5)) 92 | target = torch.ones((10, ), dtype=torch.long) * 255 93 | loss = loss_class(pred, target, reduction_override=reduction) 94 | assert loss == 0 95 | -------------------------------------------------------------------------------- /tests/test_models/test_losses/test_mse_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from mmhuman3d.models.losses.builder import build_loss 4 | 5 | 6 | def test_keypoint_mse_loss(): 7 | loss_cfg = dict(type='KeypointMSELoss') 8 | loss = build_loss(loss_cfg) 9 | pred = torch.zeros(1, 3, 2) 10 | target = torch.zeros(1, 3, 2) 11 | assert torch.allclose(loss(pred, target), torch.tensor(0.)) 12 | 13 | pred = torch.ones(1, 3, 2) 14 | target = torch.zeros(1, 3, 2) 15 | assert torch.allclose(loss(pred, target), torch.tensor(.5)) 16 | 17 | # test sum reduction 18 | loss_cfg = dict(type='KeypointMSELoss', reduction='sum') 19 | loss = build_loss(loss_cfg) 20 | pred = torch.zeros(1, 3, 2) 21 | target = torch.zeros(1, 3, 2) 22 | assert torch.allclose(loss(pred, target), torch.tensor(0.)) 23 | 24 | pred = torch.ones(1, 3, 2) 25 | target = torch.zeros(1, 3, 2) 26 | assert torch.allclose(loss(pred, target), torch.tensor(3.)) 27 | 28 | # test None reduction 29 | loss_cfg = dict(type='KeypointMSELoss', reduction=None) 30 | loss = build_loss(loss_cfg) 31 | pred = torch.zeros(1, 3, 2) 32 | target = torch.zeros(1, 3, 2) 33 | assert torch.allclose(loss(pred, target), pred) 34 | 35 | pred = torch.ones(1, 3, 2) 36 | target = torch.zeros(1, 3, 2) 37 | result = torch.ones(1, 3, 2) * 0.5 38 | assert torch.allclose(loss(pred, target), result) 39 | 40 | # test None reduction 41 | loss_cfg = dict(type='KeypointMSELoss', reduction='none') 42 | loss = build_loss(loss_cfg) 43 | pred = torch.zeros(1, 3, 2) 44 | target = torch.zeros(1, 3, 2) 45 | assert torch.allclose(loss(pred, target), pred) 46 | 47 | pred = torch.ones(1, 3, 2) 48 | target = torch.zeros(1, 3, 2) 49 | result = torch.ones(1, 3, 2) * 0.5 50 | assert torch.allclose(loss(pred, target), result) 51 | 52 | # test loss weight 53 | loss_cfg = dict(type='KeypointMSELoss', loss_weight=2.) 54 | loss = build_loss(loss_cfg) 55 | pred = torch.zeros(1, 3, 2) 56 | target = torch.zeros(1, 3, 2) 57 | assert torch.allclose(loss(pred, target), torch.tensor(0.)) 58 | 59 | pred = torch.ones(1, 3, 2) 60 | target = torch.zeros(1, 3, 2) 61 | assert torch.allclose(loss(pred, target), torch.tensor(1.)) 62 | -------------------------------------------------------------------------------- /tests/test_models/test_losses/test_smooth_l1_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from mmhuman3d.models.losses.builder import build_loss 4 | 5 | 6 | def test_smooth_l1_loss(): 7 | loss_cfg = dict(type='SmoothL1Loss') 8 | loss = build_loss(loss_cfg) 9 | 10 | fake_pred = torch.zeros(1, 3, 2) 11 | fake_target = torch.zeros(1, 3, 2) 12 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(0.)) 13 | 14 | fake_pred = torch.ones(1, 3, 2) 15 | fake_target = torch.zeros(1, 3, 2) 16 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(.5)) 17 | 18 | # test beta 19 | loss_cfg = dict(type='SmoothL1Loss', beta=2.) 20 | loss = build_loss(loss_cfg) 21 | 22 | fake_pred = torch.zeros(1, 3, 2) 23 | fake_target = torch.zeros(1, 3, 2) 24 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(0.)) 25 | 26 | fake_pred = torch.ones(1, 3, 2) 27 | fake_target = torch.zeros(1, 3, 2) 28 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(.25)) 29 | 30 | # test reduction 31 | loss_cfg = dict(type='SmoothL1Loss', reduction='sum') 32 | loss = build_loss(loss_cfg) 33 | 34 | fake_pred = torch.zeros(1, 3, 2) 35 | fake_target = torch.zeros(1, 3, 2) 36 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(0.)) 37 | 38 | fake_pred = torch.ones(1, 3, 2) 39 | fake_target = torch.zeros(1, 3, 2) 40 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(3.)) 41 | 42 | # test loss weight 43 | loss_cfg = dict(type='SmoothL1Loss', loss_weight=2.) 44 | loss = build_loss(loss_cfg) 45 | 46 | fake_pred = torch.zeros(1, 3, 2) 47 | fake_target = torch.zeros(1, 3, 2) 48 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(0.)) 49 | 50 | fake_pred = torch.ones(1, 3, 2) 51 | fake_target = torch.zeros(1, 3, 2) 52 | assert torch.allclose(loss(fake_pred, fake_target), torch.tensor(1.)) 53 | -------------------------------------------------------------------------------- /tests/test_models/test_necks/test_temporal_encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from mmhuman3d.models.necks.temporal_encoder import TemporalGRUEncoder 4 | 5 | 6 | def test_hmr_head(): 7 | model = TemporalGRUEncoder(2048) 8 | x = torch.rand(32, 32, 2048) 9 | y = model(x) 10 | assert y.shape == (32, 32, 2048) 11 | -------------------------------------------------------------------------------- /tests/test_utils/test_path_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from mmhuman3d.utils.path_utils import ( 6 | Existence, 7 | check_input_path, 8 | check_path_existence, 9 | prepare_output_path, 10 | ) 11 | 12 | 13 | def test_existence(): 14 | empty_dir = 'test/data/path_utils/' 15 | os.makedirs(empty_dir, exist_ok=False) 16 | check_path_existence(empty_dir, 'file') == Existence.DirectoryExistEmpty 17 | os.removedirs(empty_dir) 18 | check_path_existence('mmhuman3d/core', 19 | 'file') == Existence.DirectoryExistNotEmpty 20 | check_path_existence('mmhuman3d/not_exist/', 21 | 'file') == Existence.DirectoryNotExist 22 | check_path_existence('mmhuman3d/not_exist.txt', 23 | 'auto') == Existence.FileNotExist 24 | 25 | 26 | def test_prepare_output_path(): 27 | prepare_output_path('tests/data/', overwrite=True) 28 | 29 | 30 | def test_check_input_path(): 31 | with pytest.raises(FileNotFoundError): 32 | check_input_path( 33 | 'tests/data/human_data/human_data_00.npz', 34 | allowed_suffix=[ 35 | '.pkl', 36 | ]) 37 | -------------------------------------------------------------------------------- /tests/test_vis/test_render.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch3d.renderer.mesh.textures import TexturesVertex 3 | from pytorch3d.utils import ico_sphere 4 | 5 | from mmhuman3d.core.cameras import compute_orbit_cameras 6 | from mmhuman3d.core.cameras.builder import build_cameras 7 | from mmhuman3d.core.visualization import render_runner 8 | from mmhuman3d.core.visualization.renderer import build_renderer 9 | 10 | 11 | def test_render_runner(): 12 | if torch.cuda.is_available(): 13 | device_name = 'cuda:0' 14 | else: 15 | device_name = 'cpu' 16 | 17 | device = torch.device(device_name) 18 | meshes = ico_sphere(3, device) 19 | 20 | meshes.textures = TexturesVertex( 21 | verts_features=torch.ones_like(meshes.verts_padded()).to(device)) 22 | K, R, T = compute_orbit_cameras(orbit_speed=1.0, batch_size=2) 23 | resolution = 128 24 | cameras = build_cameras( 25 | dict(type='fovperspective', K=K, R=R, T=T, resolution=resolution)) 26 | renderer = build_renderer( 27 | dict( 28 | type='mesh', 29 | resolution=resolution, 30 | shader=dict(type='soft_phong'), 31 | lights=dict(type='ambient'))) 32 | tensor = render_runner.render( 33 | meshes=meshes.extend(2), 34 | cameras=cameras, 35 | renderer=renderer, 36 | device=device, 37 | return_tensor=True, 38 | batch_size=2, 39 | output_path='/tmp/demo.mp4') 40 | assert tensor.shape == (2, 128, 128, 4) 41 | -------------------------------------------------------------------------------- /tests/test_vis/test_uv.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch3d.structures import Meshes 3 | 4 | from mmhuman3d.core.visualization.renderer import UVRenderer 5 | from mmhuman3d.models.body_models.builder import build_body_model 6 | 7 | if torch.cuda.is_available(): 8 | device_name = 'cuda:0' 9 | else: 10 | device_name = 'cpu' 11 | 12 | device = torch.device(device_name) 13 | uv_param_path = 'data/body_models/smpl/smpl_uv.npz' 14 | uv_renderer = UVRenderer( 15 | resolution=(512, 512), 16 | model_type='smpl', 17 | uv_param_path=uv_param_path, 18 | device=device) 19 | 20 | model_path = 'data/body_models/smpl' 21 | body_model = build_body_model(dict(type='smpl', 22 | model_path=model_path)).to(device) 23 | 24 | 25 | def test_uv_resample(): 26 | pose_dict = body_model.tensor2dict( 27 | torch.zeros(1, (body_model.NUM_BODY_JOINTS + 1) * 3).to(device)) 28 | smpl_output = body_model(**pose_dict) 29 | verts = smpl_output['vertices'].view(1, body_model.NUM_VERTS, 3) 30 | mesh = Meshes( 31 | verts=verts, 32 | faces=body_model.faces_tensor.to(device).view(1, body_model.NUM_FACES, 33 | 3)) 34 | 35 | displacement_map = torch.ones(1, 600, 600, 3).to(device) 36 | normal_map = torch.ones(1, 600, 600, 3).to(device) 37 | texture_map = torch.ones(1, 600, 600, 3).to(device) 38 | mesh1 = uv_renderer.wrap_displacement( 39 | mesh, displacement_map=displacement_map) 40 | assert torch.isclose( 41 | mesh.verts_padded(), mesh1.verts_padded() - 1, atol=1e-3).all() 42 | mesh2 = uv_renderer.wrap_normal(mesh, normal_map=normal_map) 43 | assert (mesh2.verts_normals_padded() == 1).all() 44 | mesh3 = mesh2.clone() 45 | mesh3.textures = uv_renderer.wrap_texture(texture_map=texture_map) 46 | assert mesh3.textures.maps_padded().shape == (1, 600, 600, 3) 47 | 48 | normal_map_small = torch.ones(1, 200, 200, 3).to(device) 49 | mesh4 = uv_renderer.wrap_normal(mesh, normal_map=normal_map_small) 50 | assert (mesh4.verts_normals_padded() == 1).all() 51 | 52 | 53 | def test_uv_forward(): 54 | verts_attr = torch.zeros(2, 6890, 3) 55 | attr_map = uv_renderer(verts_attr, resolution=(600, 600)) 56 | assert attr_map.shape == (2, 600, 600, 3) 57 | 58 | pose_dict = body_model.tensor2dict( 59 | torch.zeros(1, (body_model.NUM_BODY_JOINTS + 1) * 3).to(device)) 60 | smpl_output = body_model(**pose_dict) 61 | verts = smpl_output['vertices'].view(1, body_model.NUM_VERTS, 3) 62 | normal_map = uv_renderer.forward_normal_map( 63 | vertices=verts, resolution=(512, 512)) 64 | assert normal_map.shape == (1, 512, 512, 3) 65 | 66 | uvd_map = uv_renderer.forward_uvd_map( 67 | vertices=verts, resolution=(512, 512)) 68 | assert uvd_map.shape == (1, 512, 512, 3) 69 | -------------------------------------------------------------------------------- /tests/test_vis/test_vis_cameras.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from mmhuman3d.core.visualization.visualize_cameras import ( 4 | visualize_chessboard_kinects_rgb, 5 | visualize_dumped_camera_parameter, 6 | ) 7 | 8 | 9 | def test_visualize_chessboard_kinects_rgb(): 10 | chessboard_test_file_path = \ 11 | 'tests/data/camera/calibration_chessboard_05_28_18_05_19.json' 12 | # interactive and show is set to False for pytest, 13 | # use GUI with interactive=True and show=True 14 | visualize_chessboard_kinects_rgb( 15 | chessboard_path=chessboard_test_file_path, 16 | interactive=False, 17 | show=False) 18 | 19 | 20 | def test_visualize_dumped_camera_parameter(): 21 | dumped_dir = \ 22 | 'tests/data/camera/dumped' 23 | non_json_file_path = os.path.join(dumped_dir, 'non_json_file.txt') 24 | with open(non_json_file_path, 'w') as f_write: 25 | f_write.write('test string\n') 26 | visualize_dumped_camera_parameter( 27 | dumped_dir, interactive=False, show=False) 28 | os.remove(non_json_file_path) 29 | -------------------------------------------------------------------------------- /tools/dist_test.sh: -------------------------------------------------------------------------------- 1 | CONFIG=$1 2 | WORK_DIR=$2 3 | CHECKPOINT=$3 4 | GPUS=$4 5 | PORT=${PORT:-29500} 6 | 7 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 8 | python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ 9 | $(dirname "$0")/test.py $CONFIG --work-dir=${WORK_DIR} $CHECKPOINT --launcher pytorch ${@:5} 10 | -------------------------------------------------------------------------------- /tools/dist_train.sh: -------------------------------------------------------------------------------- 1 | CONFIG=$1 2 | WORK_DIR=$2 3 | GPUS=$3 4 | PORT=${PORT:-29500} 5 | 6 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 7 | python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ 8 | $(dirname "$0")/train.py $CONFIG --work-dir=${WORK_DIR} --launcher pytorch ${@:4} 9 | -------------------------------------------------------------------------------- /tools/get_flops.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | 4 | import mmcv 5 | import numpy as np 6 | import torch 7 | from mmcv import DictAction 8 | 9 | from mmhuman3d.models import build_architecture 10 | 11 | try: 12 | from mmcv.cnn import get_model_complexity_info 13 | except ImportError: 14 | raise ImportError('Please upgrade mmcv to >0.6.2') 15 | 16 | 17 | def parse_args(): 18 | parser = argparse.ArgumentParser(description='Train a detector') 19 | parser.add_argument('config', help='train config file path') 20 | parser.add_argument( 21 | '--shape', 22 | type=int, 23 | nargs='+', 24 | default=[224, 224], 25 | help='input image size') 26 | parser.add_argument( 27 | '--cfg-options', 28 | nargs='+', 29 | action=DictAction, 30 | help='override some settings in the used config, the key-value pair ' 31 | 'in xxx=yyy format will be merged into config file. If the value to ' 32 | 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 33 | 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 34 | 'Note that the quotation marks are necessary and that no white space ' 35 | 'is allowed.') 36 | parser.add_argument( 37 | '--size-divisor', 38 | type=int, 39 | default=32, 40 | help='Pad the input image, the minimum size that is divisible ' 41 | 'by size_divisor, -1 means do not pad the image.') 42 | args = parser.parse_args() 43 | return args 44 | 45 | 46 | def main(): 47 | 48 | args = parse_args() 49 | 50 | if len(args.shape) == 1: 51 | h = w = args.shape[0] 52 | elif len(args.shape) == 2: 53 | h, w = args.shape 54 | else: 55 | raise ValueError('invalid input shape') 56 | ori_shape = (3, h, w) 57 | divisor = args.size_divisor 58 | if divisor > 0: 59 | h = int(np.ceil(h / divisor)) * divisor 60 | w = int(np.ceil(w / divisor)) * divisor 61 | 62 | input_shape = (3, h, w) 63 | 64 | cfg = mmcv.Config.fromfile(args.config) 65 | if args.cfg_options is not None: 66 | cfg.merge_from_dict(args.cfg_options) 67 | 68 | model = build_architecture(cfg.model) 69 | if torch.cuda.is_available(): 70 | model.cuda() 71 | model.eval() 72 | 73 | if hasattr(model, 'forward_dummy'): 74 | model.forward = model.forward_dummy 75 | else: 76 | raise NotImplementedError( 77 | 'FLOPs counter is currently not currently supported with {}'. 78 | format(model.__class__.__name__)) 79 | 80 | flops, params = get_model_complexity_info(model, input_shape) 81 | split_line = '=' * 30 82 | 83 | if divisor > 0 and \ 84 | input_shape != ori_shape: 85 | print(f'{split_line}\nUse size divisor set input shape ' 86 | f'from {ori_shape} to {input_shape}\n') 87 | print(f'{split_line}\nInput shape: {input_shape}\n' 88 | f'Flops: {flops}\nParams: {params}\n{split_line}') 89 | print('!!!Please be cautious if you use the results in papers. ' 90 | 'You may need to check if all ops are supported and verify that the ' 91 | 'flops computation is correct.') 92 | 93 | 94 | if __name__ == '__main__': 95 | main() 96 | -------------------------------------------------------------------------------- /tools/misc/publish_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import subprocess 4 | from datetime import date 5 | 6 | import torch 7 | 8 | 9 | def parse_args(): 10 | parser = argparse.ArgumentParser( 11 | description='Process a checkpoint to be published') 12 | parser.add_argument('in_file', help='input checkpoint filename') 13 | parser.add_argument('out_file', help='output checkpoint filename') 14 | args = parser.parse_args() 15 | return args 16 | 17 | 18 | def process_checkpoint(in_file, out_file): 19 | checkpoint = torch.load(in_file, map_location='cpu') 20 | # remove optimizer for smaller file size 21 | if 'optimizer' in checkpoint: 22 | del checkpoint['optimizer'] 23 | # if it is necessary to remove some sensitive data in checkpoint['meta'], 24 | # add the code here. 25 | torch.save(checkpoint, out_file) 26 | sha = subprocess.check_output(['sha256sum', out_file]).decode() 27 | if out_file.endswith('.pth'): 28 | out_file_name = out_file[:-4] 29 | else: 30 | out_file_name = out_file 31 | 32 | date_now = date.today().strftime('%Y%m%d') 33 | final_file = out_file_name + f'-{sha[:8]}_{date_now}.pth' 34 | subprocess.Popen(['mv', out_file, final_file]) 35 | 36 | 37 | def main(): 38 | args = parse_args() 39 | process_checkpoint(args.in_file, args.out_file) 40 | 41 | 42 | if __name__ == '__main__': 43 | main() 44 | -------------------------------------------------------------------------------- /tools/slurm_smplify.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -x 4 | 5 | PARTITION=$1 6 | JOB_NAME=$2 7 | INPUT=$3 8 | INPUT_TYPE=$4 9 | CONFIG=$5 10 | BODY_MODEL_DIR=$6 11 | OUTPUT=$7 12 | SHOW_PATH=$8 13 | GPUS=1 # Only support single GPU currently 14 | GPUS_PER_NODE=1 15 | CPUS_PER_TASK=1 16 | SRUN_ARGS=${SRUN_ARGS:-""} 17 | PY_ARGS=${@:9} 18 | 19 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 20 | srun -p ${PARTITION} \ 21 | --job-name=${JOB_NAME} \ 22 | --gres=gpu:${GPUS_PER_NODE} \ 23 | --ntasks=${GPUS} \ 24 | --ntasks-per-node=${GPUS_PER_NODE} \ 25 | --cpus-per-task=${CPUS_PER_TASK} \ 26 | --kill-on-bad-exit=1 \ 27 | ${SRUN_ARGS} \ 28 | python -u tools/smplify.py \ 29 | --use_one_betas_per_video \ 30 | --input=${INPUT} \ 31 | --input_type=${INPUT_TYPE} \ 32 | --config=${CONFIG} \ 33 | --body_model_dir=${BODY_MODEL_DIR} \ 34 | --output=${OUTPUT} \ 35 | --show_path=${SHOW_PATH} ${PY_ARGS} 36 | -------------------------------------------------------------------------------- /tools/slurm_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | 4 | set -x 5 | 6 | PARTITION=$1 7 | JOB_NAME=$2 8 | CONFIG=$3 9 | WORK_DIR=$4 10 | CHECKPOINT=$5 11 | GPUS=$6 12 | GPUS_PER_NODE=$((${GPUS}<8?${GPUS}:8)) 13 | CPUS_PER_TASK=${CPUS_PER_TASK:-2} 14 | SRUN_ARGS=${SRUN_ARGS:-""} 15 | PY_ARGS=${@:7} 16 | 17 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 18 | srun -p ${PARTITION} \ 19 | --job-name=${JOB_NAME} \ 20 | --gres=gpu:${GPUS_PER_NODE} \ 21 | --ntasks=${GPUS} \ 22 | --ntasks-per-node=${GPUS_PER_NODE} \ 23 | --cpus-per-task=${CPUS_PER_TASK} \ 24 | --kill-on-bad-exit=1 \ 25 | ${SRUN_ARGS} \ 26 | python -u tools/test.py ${CONFIG} --work-dir=${WORK_DIR} ${CHECKPOINT} --launcher="slurm" ${PY_ARGS} 27 | -------------------------------------------------------------------------------- /tools/slurm_train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | 4 | set -x 5 | 6 | PARTITION=$1 7 | JOB_NAME=$2 8 | CONFIG=$3 9 | WORK_DIR=$4 10 | GPUS=$5 11 | GPUS_PER_NODE=$((${GPUS}<8?${GPUS}:8)) 12 | CPUS_PER_TASK=${CPUS_PER_TASK:-2} 13 | SRUN_ARGS=${SRUN_ARGS:-""} 14 | PY_ARGS=${@:6} 15 | 16 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 17 | srun -p ${PARTITION} \ 18 | --job-name=${JOB_NAME} \ 19 | --gres=gpu:${GPUS_PER_NODE} \ 20 | --ntasks=${GPUS} \ 21 | --ntasks-per-node=${GPUS_PER_NODE} \ 22 | --cpus-per-task=${CPUS_PER_TASK} \ 23 | --kill-on-bad-exit=1 \ 24 | ${SRUN_ARGS} \ 25 | python -u tools/train.py ${CONFIG} --work-dir=${WORK_DIR} --launcher="slurm" ${PY_ARGS} 26 | -------------------------------------------------------------------------------- /tools/test_model_builders.py: -------------------------------------------------------------------------------- 1 | from mmhuman3d.models.architectures.builder import build_architecture 2 | 3 | arch_cfg = dict( 4 | type='PareImportTestor', 5 | backbone=dict( 6 | type='ResNet', 7 | depth=34, 8 | out_indices=[3], 9 | norm_eval=False, 10 | init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet34')), 11 | head=dict( 12 | type='HybrIKHead', 13 | smpl_mean_params='data/body_models/h36m_mean_beta.npy'), 14 | body_model=dict( 15 | type='HybrIKSMPL', 16 | model_path= # noqa: E251 17 | 'data/body_models/smpl', 18 | extra_joints_regressor='data/body_models/J_regressor_h36m.npy'), 19 | loss_beta=dict(type='MSELoss', loss_weight=1), 20 | loss_theta=dict(type='MSELoss', loss_weight=0.01), 21 | loss_twist=dict(type='MSELoss', loss_weight=0.2), 22 | loss_uvd=dict(type='L1Loss', loss_weight=1), 23 | ) 24 | arch = build_architecture(arch_cfg) 25 | --------------------------------------------------------------------------------