├── .gitignore ├── LICENSE ├── README.md ├── assets └── GLAMR │ ├── LICENSE.txt │ └── P9_80_seed1.pkl ├── configuration.py ├── dataset.md ├── evaluate.py ├── evaluation_engine.py ├── evaluation_loaders.py ├── evaluation_metrics.py ├── pyproject.toml ├── requirements.txt ├── visualize.py └── visualize_GLAMR.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | .idea/ 163 | imgui.ini -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 AIT 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EMDB: The Electromagnetic Database of Global 3D Human Pose and Shape in the Wild 2 | 3 | _Official Repository for the ICCV 2023 paper [EMDB: The Electromagnetic Database of Global 3D Human Pose and Shape in the Wild]()_. 4 | 5 | ## [Project Page](https://ait.ethz.ch/emdb) | [Paper](https://files.ait.ethz.ch/projects/emdb/main.pdf) | [Supplementary](https://files.ait.ethz.ch/projects/emdb/supp.pdf) | [Video](https://youtu.be/H66-YE4GUHI?feature=shared) | [Data](https://emdb.ait.ethz.ch) 6 | 7 | 8 | 9 | ## Dataset 10 | To receive access to the data, please fill out the [application form](https://emdb.ait.ethz.ch). You will receive an e-mail with more information after your application has been approved. 11 | 12 | For an overview of how EMDB is structured, please refer to the [dataset overview](dataset.md). 13 | 14 | ## Visualization 15 | We use [aitviewer](https://github.com/eth-ait/aitviewer) to visualize the data. The code was tested with Python 3.8 on Windows 10. 16 | 17 | ### Installation 18 | ```bash 19 | conda create -n emdb python=3.8 20 | pip install aitviewer tabulate 21 | ``` 22 | This does not automatically install a GPU-version of PyTorch. If your environment already contains it, you should be good to go, otherwise you may wish to install it manually, e.g. on Windows 23 | 24 | ```python 25 | pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 26 | ``` 27 | 28 | Please also download the SMPL model by following the instructions on the [SMPL-X GitHub page](https://github.com/vchoutas/smplx#downloading-the-model). 29 | 30 | ### Setup 31 | 1. Change the `SMPLX_MODELS` variable in `configuration.py` to where you downloaded the SMPL model. This folder should contain a subfolder `smpl` with the respective model files. 32 | 2. Change the `EMDB_ROOT` variable in `configuration.py` to where you extracted the EMDB dataset. This folder should contain subfolders `P0`, `P1`, etc. 33 | 34 | ### Visualize EMDB Data 35 | Run the following command to visualize a sequence. `SUBJECT_ID` refers to the ID Of the participant, i.e. `P0-P9` and `SEQUENCE_ID` is the 2-digit identifier that is prepended to each sequence's name: 36 | 37 | ```python 38 | python visualize.py --subject {SUBJECT_ID} --sequence {SEQUENCE_ID} 39 | ``` 40 | 41 | By default, this opens the viewer in the 3D view. You can choose to show the reprojected poses instead by specifying `--view_from_camera`. If you specify `--draw_2d` the 2D keypoints and bounding boxes will be drawn on top of the image. If you pass `--draw_trajectories` the SMPL root and camera trajectories will be drawn in addition. 42 | 43 | ### Visualize GLAMR 44 | We provide a script to visualize [GLAMR](https://github.com/NVlabs/GLAMR) results. An example result that reproduces Figure 9 of the main paper is provided in `assets/GLAMR`. To visualize it, run the following command: 45 | 46 | ```python 47 | python visualize_GLAMR.py 48 | ``` 49 | 50 | ## Evaluation 51 | 52 | ### Example using HybrIK 53 | We provide code to load and evaluate HybrIK results on the EMDB test set, which in the paper is referred to as `EMDB 1`. Based on this evaluation code, it should be straight-forward to extend the evaluation to other methods (see below). 54 | 55 | To run the evaluation, use the following command: 56 | ```python 57 | python evaluate.py {RESULT_ROOT} 58 | ``` 59 | 60 | The `RESULT_ROOT` is a folder that is expected to have the same general folder structure as EMDB, i.e.: 61 | ``` 62 | RESULT_ROOT 63 | ├── PX 64 | ├── sequence1 65 | ├── hybrIK-out 66 | ├── 000000.pkl 67 | ├── 000001.pkl 68 | ├── ... 69 | ├── sequence2 70 | ├── ... 71 | ├── sequenceN 72 | ``` 73 | 74 | I.e., `EMDB_ROOT` can function as a `RESULT_ROOT`, if the corresponding results are stored in a subfolder `hybrIK-out` for each sequence. The evaluation code computes the MPJPE, MPJAE, MVE, and jitter metrics as reported in the paper. It reports both the pelvis-aligned and Procrustes-aligned versions (*-PA), as well as standard deviations. Further, it prints the metrics for each sequence individually, as well as the average over all sequences. 75 | 76 | ### How to evaluate your own method 77 | In order to run the evaluations with your own results, follow these steps: 78 | 1. In [`evaluation_loaders.py`](evaluation_loaders.py) define a function to load your result. Follow the signature and return values of the existing `load_hybrik` function. 79 | 2. In [`evaluation_engine.py`](evaluation_engine.py) register your method by giving it a name in a global variable, e.g. `MYMETHOD = 'My Method'`. Then using that name as a key, extend the following two `dict`s (follow the existing example with HybrIK for reference): 80 | - `METHOD_TO_RESULT_FOLDER`: This maps to the subfolder in `{RESULT_ROOT}/{SUBJECT_ID}/{SEQUENCE_ID}` where your methods result will be stored. 81 | - `METHOD_TO_LOAD_FUNCTION`: This maps to the loading function you defined in step 1. 82 | 3. In the function [`EvaluationEngine.get_gender_for_baseline`](evaluation_engine.py) select the appropriate SMPL gender for your method. 83 | 4. Finally, in [`evaluate.py`](evaluate.py) import `MYMETHOD` and add it to the list of methods that the evaluation engine should evaluate. 84 | 85 | ## Citation 86 | If you use this code or data, please cite the following paper: 87 | ```bibtex 88 | @inproceedings{kaufmann2023emdb, 89 | author = {Kaufmann, Manuel and Song, Jie and Guo, Chen and Shen, Kaiyue and Jiang, Tianjian and Tang, Chengcheng and Z{\'a}rate, Juan Jos{\'e} and Hilliges, Otmar}, 90 | title = {{EMDB}: The {E}lectromagnetic {D}atabase of {G}lobal 3{D} {H}uman {P}ose and {S}hape in the {W}ild}, 91 | booktitle = {International Conference on Computer Vision (ICCV)}, 92 | year = {2023} 93 | } 94 | ``` 95 | 96 | ## Contact 97 | For any questions or problems, please open an issue or contact [Manuel Kaufmann](mailto:manuel.kaufmann@inf.ethz.ch). 98 | -------------------------------------------------------------------------------- /assets/GLAMR/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021-22, NVIDIA Corporation & affiliates. All rights reserved. 2 | 3 | 4 | NVIDIA Source Code License for GLAMR: Global Occlusion-Aware Human Mesh Recovery with Dynamic Cameras 5 | 6 | 7 | ======================================================================= 8 | 9 | 1. Definitions 10 | 11 | “Licensor” means any person or entity that distributes its Work. 12 | 13 | “Software” means the original work of authorship made available under 14 | this License. 15 | 16 | “Work” means the Software and any additions to or derivative works of 17 | the Software that are made available under this License. 18 | 19 | The terms “reproduce,” “reproduction,” “derivative works,” and 20 | “distribution” have the meaning as provided under U.S. copyright law; 21 | provided, however, that for the purposes of this License, derivative 22 | works shall not include works that remain separable from, or merely 23 | link (or bind by name) to the interfaces of, the Work. 24 | 25 | Works, including the Software, are “made available” under this License 26 | by including in or with the Work either (a) a copyright notice 27 | referencing the applicability of this License to the Work, or (b) a 28 | copy of this License. 29 | 30 | 2. License Grants 31 | 32 | 2.1 Copyright Grant. Subject to the terms and conditions of this 33 | License, each Licensor grants to you a perpetual, worldwide, 34 | non-exclusive, royalty-free, copyright license to reproduce, 35 | prepare derivative works of, publicly display, publicly perform, 36 | sublicense and distribute its Work and any resulting derivative 37 | works in any form. 38 | 39 | 3. Limitations 40 | 41 | 3.1 Redistribution. You may reproduce or distribute the Work only 42 | if (a) you do so under this License, (b) you include a complete 43 | copy of this License with your distribution, and (c) you retain 44 | without modification any copyright, patent, trademark, or 45 | attribution notices that are present in the Work. 46 | 47 | 3.2 Derivative Works. You may specify that additional or different 48 | terms apply to the use, reproduction, and distribution of your 49 | derivative works of the Work ("Your Terms") only if (a) Your Terms 50 | provide that the use limitation in Section 3.3 applies to your 51 | derivative works, and (b) you identify the specific derivative 52 | works that are subject to Your Terms. Notwithstanding Your Terms, 53 | this License (including the redistribution requirements in Section 54 | 3.1) will continue to apply to the Work itself. 55 | 56 | 3.3 Use Limitation. The Work and any derivative works thereof only 57 | may be used or intended for use non-commercially. Notwithstanding 58 | the foregoing, NVIDIA and its affiliates may use the Work and any 59 | derivative works commercially. As used herein, "non-commercially" 60 | means for research or evaluation purposes only. 61 | 62 | 3.4 Patent Claims. If you bring or threaten to bring a patent claim 63 | against any Licensor (including any claim, cross-claim or 64 | counterclaim in a lawsuit) to enforce any patents that you allege 65 | are infringed by any Work, then your rights under this License from 66 | such Licensor (including the grant in Section 2.1) will terminate 67 | immediately. 68 | 69 | 3.5 Trademarks. This License does not grant any rights to use any 70 | Licensor’s or its affiliates’ names, logos, or trademarks, except 71 | as necessary to reproduce the notices described in this License. 72 | 73 | 3.6 Termination. If you violate any term of this License, then your 74 | rights under this License (including the grant in Section 2.1) will 75 | terminate immediately. 76 | 77 | 4. Disclaimer of Warranty. 78 | 79 | THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY 80 | KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF 81 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR 82 | NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER 83 | THIS LICENSE. 84 | 85 | 5. Limitation of Liability. 86 | 87 | EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL 88 | THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE 89 | SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, 90 | INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF 91 | OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK 92 | (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, 93 | LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER 94 | COMMERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF 95 | THE POSSIBILITY OF SUCH DAMAGES. 96 | 97 | ======================================================================= -------------------------------------------------------------------------------- /assets/GLAMR/P9_80_seed1.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eth-ait/emdb/9a4eab677181a3789bda7ba5c36ab8cff797380c/assets/GLAMR/P9_80_seed1.pkl -------------------------------------------------------------------------------- /configuration.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | 4 | Define a few constants that we need throughout. 5 | """ 6 | import numpy as np 7 | 8 | # TODO update paths to match your setup 9 | EMDB_ROOT = "V:/emdb" 10 | SMPLX_MODELS = "D:/data/smplx_models" 11 | 12 | SMPL_SKELETON = np.array( 13 | [ 14 | [0, 1], 15 | [0, 2], 16 | [0, 3], 17 | [2, 5], 18 | [5, 8], 19 | [8, 11], 20 | [1, 4], 21 | [4, 7], 22 | [7, 10], 23 | [3, 6], 24 | [6, 9], 25 | [9, 14], 26 | [9, 13], 27 | [9, 12], 28 | [12, 15], 29 | [14, 17], 30 | [17, 19], 31 | [19, 21], 32 | [21, 23], 33 | [13, 16], 34 | [16, 18], 35 | [18, 20], 36 | [20, 22], 37 | ] 38 | ) 39 | 40 | # fmt: off 41 | # 0 center, 1 right, 2 left 42 | SMPL_SIDE_INDEX = [0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1, 2, 1, 2, 1, 2, 1] 43 | # fmt: on 44 | 45 | SMPL_SIDE_COLOR = [ 46 | (255, 0, 255), 47 | (0, 0, 255), 48 | (255, 0, 0), 49 | ] 50 | -------------------------------------------------------------------------------- /dataset.md: -------------------------------------------------------------------------------- 1 | # EMDB Dataset Structure 2 | 3 | After unzipping the downloaded files, please arrange their contents as follows: 4 | 5 | ```python 6 | EMDB_ROOT 7 | ├── P0 8 | ├── P1 9 | ├── ... 10 | ├── P9 11 | ``` 12 | 13 | Each participant's folder looks like this: 14 | 15 | ``` 16 | EMDB_ROOT 17 | ├── PX 18 | ├── sequence1 19 | ├── images 20 | ├── {PX}_{sequence1}_data.pkl 21 | ├── {PX}_{sequence1}_video.mp4 22 | ├── sequence2 23 | ├── ... 24 | ├── sequenceN 25 | ``` 26 | 27 | The `images` subfolder contains the raw images as single files in named in the format `{%5d}.jpg` from 0. The video file shows a side-by-side view of the original RGB images and the EMDB reference overlaid on top. The contents of the pickle file are described in the following. 28 | 29 | 30 | ## `*_data.pkl` 31 | 32 | The pickle file contains a dictionary with the following keys: 33 | 34 | | Key | Type | Description | 35 | | --- | --- | --- | 36 | | `gender` | `string` | Either `"female"`, `"male"`, or `"neutral"`. | 37 | | `name` | `string` | The name of this sequence in the format `{subject_id}_{sequence_id}`. | 38 | | `emdb1` | `bool` | Whether this sequence is part of EMDB 1 as mentioned in the paper. We evaluated state-of-the-art RGB-based baselines on EMDB 1. | 39 | | `emdb2` | `bool` | Whether this sequence is part of EMDB 2 as mentioned in the paper. We evaluated GLAMR's global trajectories on EMDB 2. | 40 | | `n_frames` | `int` | Length of this sequence. | 41 | | `good_frames_mask` | `np.ndarray`| Of shape `(n_frames, )` indicating which frames are considered valid (`True`) and which aren't. The invalid frames are hand-selected instances where the person is out-of-view or occluded entirely. | 42 | | `camera` | `dict` | Camera information (see below). | 43 | | `smpl` | `dict` | SMPL parameters (see below). | 44 | | `kp2d` | `np.ndarray` | Of shape `(n_frames, 24, 2)` containing the SMPL joints projected into the camera. | 45 | | `bboxes` | `dict` | Bounding box data (see below). | 46 | 47 | ### `camera` 48 | 49 | | Key | Type | Description | 50 | | --- | --- | --- | 51 | | `intrinsics` | `np.ndarray` | Of shape `(3, 3)` containing the camera intrinsics. | 52 | | `extrinsics` | `np.ndarray` | Of shape `(n_frames, 4, 4)` containing the camera extrinsics. | 53 | | `width` | `int` | The width of the image. | 54 | | `height` | `int` | The height of the image. | 55 | 56 | ### `smpl` 57 | 58 | | Key | Type | Description | 59 | | --- | --- | --- | 60 | | `poses_root` | `np.ndarray` | Of shape `(n_frames, 3)` containing the SMPL root orientation. | 61 | | `poses_body` | `np.ndarray` | Of shape `(n_frames, 69)` containing the SMPL pose parameters. | 62 | | `trans` | `np.ndarray` | Of shape `(n_frames, 3)` containing the SMPL root translation. | 63 | | `betas` | `np.ndarray` | Of shape `(10, )` containing the SMPL shape parameters. | 64 | 65 | 66 | ### `bboxes` 67 | 68 | | Key | Type | Description | 69 | | --- | --- | --- | 70 | | `bboxes` | `np.ndarray` | Of shape `(n_frames, 4)` containing the 2D bounding boxes in the format `(x_min, y_min, x_max, y_max)`. | 71 | | `invalid_idxs` | `np.ndarray` | Indexes of invalid bounding boxes. A bounding box at frame `i` is invalid if `good_frames_mask[i] == False` or if `x_max - x_min <= 0 or y_max - y_min <= 0`. | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | """ 4 | import argparse 5 | import glob 6 | import os 7 | import pickle as pkl 8 | 9 | from configuration import EMDB_ROOT 10 | from evaluation_engine import HYBRIK, EvaluationEngine 11 | from evaluation_metrics import compute_metrics 12 | 13 | if __name__ == "__main__": 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument("result_root", help="Where the baseline results are stored.") 16 | args = parser.parse_args() 17 | 18 | def is_emdb1(emdb_pkl_file): 19 | with open(emdb_pkl_file, "rb") as f: 20 | data = pkl.load(f) 21 | return data["emdb1"] 22 | 23 | # Search for all the test sequences on which we evaluated the baselines in the paper. 24 | all_emdb_pkl_files = glob.glob(os.path.join(EMDB_ROOT, "*/*/*_data.pkl")) 25 | emdb1_sequence_roots = [] 26 | for emdb_pkl_file in all_emdb_pkl_files: 27 | if is_emdb1(emdb_pkl_file): 28 | emdb1_sequence_roots.append(os.path.dirname(emdb_pkl_file)) 29 | 30 | # Select the baselines we want to evaluate. 31 | baselines_to_evaluate = [HYBRIK] 32 | 33 | # Run the evaluation. 34 | evaluator_public = EvaluationEngine(compute_metrics) 35 | evaluator_public.run(emdb1_sequence_roots, args.result_root, baselines_to_evaluate) 36 | -------------------------------------------------------------------------------- /evaluation_engine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | """ 4 | 5 | import collections 6 | import functools 7 | import os 8 | import pickle as pkl 9 | 10 | import numpy as np 11 | from tabulate import tabulate 12 | 13 | from evaluation_loaders import load_hybrik 14 | 15 | HYBRIK = "HybrIK" 16 | 17 | METHOD_TO_RESULT_FOLDER = { 18 | HYBRIK: "hybrIK-out", 19 | } 20 | 21 | METHOD_TO_LOAD_FUNCTION = { 22 | HYBRIK: load_hybrik, 23 | } 24 | 25 | 26 | class EvaluationEngine(object): 27 | def __init__(self, metrics_compute_func, force_load=False, ignore_smpl_trans=True): 28 | # Function to be used to compute the metrics. 29 | self.compute_metrics = metrics_compute_func 30 | # If true, it will invalidate all caches and reload the baseline results. 31 | self.force_load = force_load 32 | # If set, the SMPL translation of the predictions will be set to 0. This only affects the jitter metric because 33 | # we always align either by the pelvis or via Procrustes for the other metrics. 34 | self.ignore_smpl_trans = ignore_smpl_trans 35 | 36 | def get_ids_from_sequence_root(self, sequence_root): 37 | res = sequence_root.split(os.path.sep) 38 | subject_id = res[-2] 39 | seq_id = res[-1] 40 | return subject_id, seq_id 41 | 42 | @functools.lru_cache() 43 | def _get_emdb_data(self, sequence_root): 44 | subject_id, seq_id = self.get_ids_from_sequence_root(sequence_root) 45 | data_file = os.path.join(sequence_root, f"{subject_id}_{seq_id}_data.pkl") 46 | with open(os.path.join(sequence_root, data_file), "rb") as f: 47 | data = pkl.load(f) 48 | return data 49 | 50 | def load_emdb_gt(self, sequence_root): 51 | """ 52 | Load EMDB SMPL pose parameters. 53 | :param sequence_root: Where the .pkl file is stored. 54 | :return: 55 | poses_gt: a np array of shape (N, 72) 56 | betas_gt: a np array of shape (N, 10) 57 | trans_gt: a np array of shape (N, 3) 58 | """ 59 | data = self._get_emdb_data(sequence_root) 60 | 61 | poses_body = data["smpl"]["poses_body"] 62 | poses_root = data["smpl"]["poses_root"] 63 | betas = data["smpl"]["betas"] 64 | trans = data["smpl"]["trans"] 65 | 66 | poses_gt = np.concatenate([poses_root, poses_body], axis=-1) 67 | betas_gt = np.repeat(betas.reshape((1, -1)), repeats=data["n_frames"], axis=0) 68 | trans_gt = trans 69 | 70 | return poses_gt, betas_gt, trans_gt 71 | 72 | def load_good_frames_mask(self, sequence_root): 73 | """Return the mask that says which frames are good and whic are not (because the human is too occluded).""" 74 | data = self._get_emdb_data(sequence_root) 75 | return data["good_frames_mask"] 76 | 77 | def get_gender_for_baseline(self, method): 78 | """Which gender to use for the baseline method.""" 79 | if method in [HYBRIK]: 80 | return "neutral" 81 | else: 82 | # This will select whatever gender the ground-truth specifies. 83 | return None 84 | 85 | def compare2method(self, poses_gt, betas_gt, trans_gt, sequence_root, result_root, method): 86 | """Load this method's results and compute the metrics on them.""" 87 | 88 | # Load the baseline results 89 | subject_id, seq_id = self.get_ids_from_sequence_root(sequence_root) 90 | method_result_dir = os.path.join(result_root, subject_id, seq_id, METHOD_TO_RESULT_FOLDER[method]) 91 | poses_cmp, betas_cmp, trans_cmp = METHOD_TO_LOAD_FUNCTION[method](method_result_dir, self.force_load) 92 | 93 | if self.ignore_smpl_trans: 94 | trans_cmp = np.zeros_like(trans_cmp) 95 | 96 | # Load camera parameters. 97 | data = self._get_emdb_data(sequence_root) 98 | world2cam = data["camera"]["extrinsics"] 99 | 100 | gender_gt = data["gender"] 101 | gender_hat = self.get_gender_for_baseline(method) 102 | 103 | # For some frames there is too much occlusion, we ignore these. 104 | good_frames_mask = self.load_good_frames_mask(sequence_root) 105 | 106 | metrics, metrics_extra = self.compute_metrics( 107 | poses_gt[good_frames_mask], 108 | betas_gt[good_frames_mask], 109 | trans_gt[good_frames_mask], 110 | poses_cmp[good_frames_mask], 111 | betas_cmp[good_frames_mask], 112 | trans_cmp[good_frames_mask], 113 | gender_gt, 114 | gender_hat, 115 | world2cam[good_frames_mask], 116 | ) 117 | 118 | return metrics, metrics_extra, method 119 | 120 | def evaluate_single_sequence(self, sequence_root, result_root, methods): 121 | """Evaluate a single sequence for all methods.""" 122 | ms, ms_extra, ms_names = [], [], [] 123 | 124 | poses_gt, betas_gt, trans_gt = self.load_emdb_gt(sequence_root) 125 | 126 | for method in methods: 127 | m, m_extra, ms_name = self.compare2method(poses_gt, betas_gt, trans_gt, sequence_root, result_root, method) 128 | 129 | ms.append(m) 130 | ms_extra.append(m_extra) 131 | ms_names.append(ms_name) 132 | 133 | return ms, ms_extra, ms_names 134 | 135 | def to_pretty_string(self, metrics, model_names): 136 | """Print the metrics onto the console, but pretty.""" 137 | if not isinstance(metrics, list): 138 | metrics = [metrics] 139 | model_names = [model_names] 140 | assert len(metrics) == len(model_names) 141 | headers, rows = [], [] 142 | for i in range(len(metrics)): 143 | values = [] 144 | for k in metrics[i]: 145 | if i == 0: 146 | headers.append(k) 147 | values.append(metrics[i][k]) 148 | rows.append([model_names[i]] + values) 149 | return tabulate(rows, headers=["Model"] + headers) 150 | 151 | def run(self, sequence_roots, result_root, methods): 152 | """Run the evaluation on all sequences and all methods.""" 153 | if not isinstance(sequence_roots, list): 154 | sequence_roots = [sequence_roots] 155 | 156 | # For every baseline, accumulate the metrics of all frames so that we can later compute statistics on them. 157 | ms_all = None 158 | ms_names = None 159 | n_frames = 0 160 | for sequence_root in sequence_roots: 161 | ms, ms_extra, ms_names = self.evaluate_single_sequence(sequence_root, result_root, methods) 162 | 163 | print("Metrics for sequence {}".format(sequence_root)) 164 | print(self.to_pretty_string(ms, ms_names)) 165 | 166 | n_frames += ms_extra[0]["mpjpe_all"].shape[0] 167 | 168 | if ms_all is None: 169 | ms_all = [collections.defaultdict(list) for _ in ms] 170 | 171 | for i in range(len(ms)): 172 | ms_all[i]["mpjpe_all"].append(ms_extra[i]["mpjpe_all"]) 173 | ms_all[i]["mpjpe_pa_all"].append(ms_extra[i]["mpjpe_pa_all"]) 174 | ms_all[i]["mpjae_all"].append(np.mean(ms_extra[i]["mpjae_all"], axis=-1)) # Mean over joints. 175 | ms_all[i]["mpjae_pa_all"].append(np.mean(ms_extra[i]["mpjae_pa_all"], axis=-1)) # Mean over joints. 176 | ms_all[i]["jitter_pd"].append(ms_extra[i]["jitter_pd"]) 177 | if "mve_all" in ms_extra[i]: 178 | ms_all[i]["mve_all"].append(ms_extra[i]["mve_all"]) 179 | if "mve_pa_all" in ms_extra[i]: 180 | ms_all[i]["mve_pa_all"].append(ms_extra[i]["mve_pa_all"]) 181 | 182 | # Compute the mean and std over all sequences. 183 | ms_all_agg = [] 184 | for i in range(len(ms_all)): 185 | mpjpe_all = np.concatenate(ms_all[i]["mpjpe_all"], axis=0) 186 | mpjpe_pa_all = np.concatenate(ms_all[i]["mpjpe_pa_all"], axis=0) 187 | mpjae_all = np.concatenate(ms_all[i]["mpjae_all"], axis=0) 188 | mpjae_pa_all = np.concatenate(ms_all[i]["mpjae_pa_all"], axis=0) 189 | jitter_all = np.array(ms_all[i]["jitter_pd"]) 190 | metrics = { 191 | "MPJPE [mm]": np.mean(mpjpe_all), 192 | "MPJPE std": np.std(mpjpe_all), 193 | "MPJPE_PA [mm]": np.mean(mpjpe_pa_all), 194 | "MPJPE_PA std": np.std(mpjpe_pa_all), 195 | "MPJAE [deg]": np.mean(mpjae_all), 196 | "MPJAE std": np.std(mpjae_all), 197 | "MPJAE_PA [deg]": np.mean(mpjae_pa_all), 198 | "MPJAE_PA std": np.std(mpjae_pa_all), 199 | "Jitter [10m/s^3]": np.mean(jitter_all), 200 | "Jitter std": np.std(jitter_all), 201 | } 202 | if "mve_all" in ms_all[i]: 203 | mve_all = np.concatenate(ms_all[i]["mve_all"], axis=0) 204 | metrics["MVE [mm]"] = np.mean(mve_all) 205 | metrics["MVE std"] = np.std(mve_all) 206 | if "mve_pa_all" in ms_all[i]: 207 | mve_pa_all = np.concatenate(ms_all[i]["mve_pa_all"], axis=0) 208 | metrics["MVE_PA [mm]"] = np.mean(mve_pa_all) 209 | metrics["MVE_PA std"] = np.std(mve_pa_all) 210 | ms_all_agg.append(metrics) 211 | 212 | print("Metrics for all sequences") 213 | print(self.to_pretty_string(ms_all_agg, ms_names)) 214 | print(" ") 215 | 216 | print("Total Number of Frames:", n_frames) 217 | -------------------------------------------------------------------------------- /evaluation_loaders.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | """ 4 | import glob 5 | import os 6 | import pickle as pkl 7 | 8 | import numpy as np 9 | from scipy.spatial.transform import Rotation as R 10 | 11 | 12 | def load_hybrik(result_root, force_load): 13 | """Load HybrIK results.""" 14 | hybrik_dir = result_root 15 | hybrik_cache_dir = os.path.join(hybrik_dir, "cache") 16 | hybrik_cache_file = os.path.join(hybrik_cache_dir, "romp-out.npz") 17 | 18 | if not os.path.exists(hybrik_cache_file) or force_load: 19 | hybrik_betas, hybrik_poses_rot, hybrik_trans = [], [], [] 20 | for pkl_file in sorted(glob.glob(os.path.join(hybrik_dir, "*.pkl"))): 21 | with open(pkl_file, "rb") as f: 22 | hybrik_data = pkl.load(f) 23 | 24 | hybrik_poses_rot.append(hybrik_data["pred_theta_mats"].reshape((1, -1, 3, 3))) 25 | hybrik_betas.append(hybrik_data["pred_shape"]) 26 | 27 | # NOTE This is not the SMPL translation, it's a translation added to the outputs of SMPL 28 | # but this does not matter because we align to the root, except for the jitter metric. 29 | hybrik_trans.append(hybrik_data["transl"]) 30 | 31 | hybrik_poses_rot = np.concatenate(hybrik_poses_rot, axis=0) 32 | hybrik_poses = R.as_rotvec(R.from_matrix(hybrik_poses_rot.reshape((-1, 3, 3)))).reshape( 33 | hybrik_poses_rot.shape[0], -1 34 | ) 35 | hybrik_betas = np.concatenate(hybrik_betas, axis=0) 36 | hybrik_trans = np.concatenate(hybrik_trans, axis=0) 37 | 38 | os.makedirs(hybrik_cache_dir, exist_ok=True) 39 | np.savez_compressed( 40 | hybrik_cache_file, 41 | hybrik_poses=hybrik_poses, 42 | hybrik_betas=hybrik_betas, 43 | hybrik_trans=hybrik_trans, 44 | ) 45 | else: 46 | hybrik_results = np.load(hybrik_cache_file) 47 | hybrik_poses = hybrik_results["hybrik_poses"] 48 | hybrik_betas = hybrik_results["hybrik_betas"] 49 | hybrik_trans = hybrik_results["hybrik_trans"] 50 | 51 | return hybrik_poses, hybrik_betas, hybrik_trans 52 | -------------------------------------------------------------------------------- /evaluation_metrics.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | """ 4 | import cv2 5 | import numpy as np 6 | import torch 7 | from aitviewer.models.smpl import SMPLLayer 8 | from aitviewer.renderables.smpl import SMPLSequence 9 | from aitviewer.utils import local_to_global 10 | 11 | SMPL_OR_JOINTS = np.array([0, 1, 2, 4, 5, 16, 17, 18, 19]) 12 | 13 | 14 | def get_data( 15 | pose_gt, 16 | shape_gt, 17 | trans_gt, 18 | pose_hat, 19 | shape_hat, 20 | trans_hat, 21 | gender_gt, 22 | gender_hat=None, 23 | ): 24 | """ 25 | Return SMPL joint positions, vertices, and global joint orientations for both ground truth and predictions. 26 | :param pose_gt: np array, ground-truth SMPL pose parameters including the root, shape (N, 72) 27 | :param shape_gt: np array, ground-truth SMPL shape parameters, shape (N, 10) or (1, 10). 28 | :param trans_gt: np array, ground-truth SMPL translation parameters, shape (N, 3). 29 | :param pose_hat: np array, predicted SMPL pose parameters including the root, shape (N, 72). 30 | :param shape_hat: np array, predicted SMPL shape parameters, shape (N, 10) or (1, 10). 31 | :param trans_hat: np array, predicted SMPL translation parameters, shape (N, 3). 32 | :param gender_gt: "male", "female", or "neutral" 33 | :param gender_hat: "male", "female", "neutral", or None (in which case it defaults ot `gender_gt`) 34 | :return: the predicted joints as a np array of shape (N, 24, 3) 35 | the ground-truth joints as a np array of shape (N, 24, 3) 36 | the predicted global joint orientations as a np array of shape (N, 9, 3, 3), i.e. only the major joints 37 | the ground-truth global joint orientations as a np array of shape (N, 24, 3, 3), i.e. all SMPL joints 38 | the predicted vertices as a np array of shape (N, 6890, 3) 39 | the ground-truth vertices as a np array of shape (N, 6890, 3) 40 | """ 41 | # We use the SMPL layer and model from aitviewer for convenience. 42 | smpl_layer = SMPLLayer(model_type="smpl", gender=gender_gt) 43 | 44 | # Create a SMPLSequence to perform the forward pass. 45 | smpl_seq = SMPLSequence( 46 | pose_gt[:, 3:], 47 | smpl_layer=smpl_layer, 48 | poses_root=pose_gt[:, :3], 49 | betas=shape_gt, 50 | trans=trans_gt, 51 | ) 52 | 53 | verts_gt, jp_gt = smpl_seq.vertices, smpl_seq.joints 54 | 55 | # Compute the global joint orientations. 56 | global_oris = local_to_global( 57 | torch.cat([smpl_seq.poses_root, smpl_seq.poses_body], dim=-1), 58 | smpl_seq.skeleton[:, 0], 59 | output_format="rotmat", 60 | ) 61 | 62 | n_frames = pose_gt.shape[0] 63 | glb_rot_mats_gt = global_oris.reshape((n_frames, -1, 3, 3)).detach().cpu().numpy() 64 | 65 | if gender_hat is None: 66 | gender_hat = gender_gt 67 | 68 | if gender_hat != gender_gt: 69 | smpl_layer_hat = SMPLLayer(model_type="smpl", gender=gender_hat) 70 | else: 71 | smpl_layer_hat = smpl_layer 72 | 73 | # Repeat the same for the predictions. 74 | smpl_seq_hat = SMPLSequence( 75 | pose_hat[:, 3:], 76 | smpl_layer=smpl_layer_hat, 77 | poses_root=pose_hat[:, :3], 78 | betas=shape_hat, 79 | trans=trans_hat, 80 | ) 81 | verts_pred, jp_pred = smpl_seq_hat.vertices, smpl_seq_hat.joints 82 | global_oris_hat = local_to_global( 83 | torch.cat([smpl_seq_hat.poses_root, smpl_seq_hat.poses_body], dim=-1), 84 | smpl_seq_hat.skeleton[:, 0], 85 | output_format="rotmat", 86 | ) 87 | 88 | glb_rot_mats_pred = global_oris_hat.reshape((n_frames, -1, 3, 3)).detach().cpu().numpy() 89 | glb_rot_mats_pred = glb_rot_mats_pred[:, SMPL_OR_JOINTS] 90 | 91 | return jp_pred, jp_gt, glb_rot_mats_pred, glb_rot_mats_gt, verts_pred, verts_gt 92 | 93 | 94 | def align_by_pelvis(joints, verts=None): 95 | """ "Align the SMPL joints and vertices by the pelvis.""" 96 | left_id = 1 97 | right_id = 2 98 | 99 | pelvis = (joints[left_id, :] + joints[right_id, :]) / 2.0 100 | if verts is not None: 101 | return verts - np.expand_dims(pelvis, axis=0) 102 | else: 103 | return joints - np.expand_dims(pelvis, axis=0) 104 | 105 | 106 | def joint_angle_error(pred_mat, gt_mat): 107 | """ 108 | Compute the geodesic distance between the two input matrices. Borrowed from 109 | https://github.com/aymenmir1/3dpw-eval/blob/master/evaluate.py 110 | :param pred_mat: np array, predicted rotation matrices, shape (N, 9, 3, 3). 111 | :param gt_mat: np array, ground truth rotation matrices, shape (N, 24, 3, 3). 112 | :return: Mean geodesic distance between input matrices. 113 | """ 114 | n_frames = pred_mat.shape[0] 115 | gt_mat = gt_mat[:, SMPL_OR_JOINTS, :, :] 116 | 117 | # Reshape the matrices into B x 3 x 3 arrays 118 | r1 = np.reshape(pred_mat, [-1, 3, 3]) 119 | r2 = np.reshape(gt_mat, [-1, 3, 3]) 120 | 121 | # Transpose gt matrices 122 | r2t = np.transpose(r2, [0, 2, 1]) 123 | 124 | # compute R1 * R2.T, if prediction and target match, this will be the identity matrix 125 | r = np.matmul(r1, r2t) 126 | 127 | angles = [] 128 | # Convert rotation matrix to axis angle representation and find the angle 129 | for i in range(r1.shape[0]): 130 | aa, _ = cv2.Rodrigues(r[i]) 131 | angles.append(np.linalg.norm(aa)) 132 | 133 | angles_all = np.degrees(np.array(angles).reshape((n_frames, -1))) 134 | return np.mean(angles_all), angles_all 135 | 136 | 137 | def compute_jitter(preds3d, gt3ds, ignored_joints_idxs=None, fps=30): 138 | """ 139 | Calculate the jitter as defined in PIP paper. https://arxiv.org/pdf/2203.08528.pdf 140 | Code Reference: https://github.com/Xinyu-Yi/PIP/blob/main/evaluate.py 141 | :param preds3d: np array, ground truth joints in global space, shape (N, 24, 3). 142 | :param gt3ds: np array, predicted joints in global space, shape (N, 24, 3). 143 | :param ignored_joints_idxs: np array, SMPL joint indices to ignore, if any. 144 | :param fps: int, frame rate of the sequence. 145 | :return: mean and std. of jerk (time derivative of acceleration) of all body joints in the global space in km/s^3 146 | """ 147 | if ignored_joints_idxs is None: 148 | ignored_joints_idxs = [0, 7, 8, 10, 11, 20, 21, 22, 23] 149 | 150 | if ignored_joints_idxs is not None: 151 | preds3d[:, ignored_joints_idxs] = 0 152 | gt3ds[:, ignored_joints_idxs] = 0 153 | 154 | jkp = np.linalg.norm( 155 | (preds3d[3:] - 3 * preds3d[2:-1] + 3 * preds3d[1:-2] - preds3d[:-3]) * (fps**3), 156 | axis=2, 157 | ) 158 | jkt = np.linalg.norm( 159 | (gt3ds[3:] - 3 * gt3ds[2:-1] + 3 * gt3ds[1:-2] - gt3ds[:-3]) * (fps**3), 160 | axis=2, 161 | ) 162 | return ( 163 | jkp.mean() / 10, 164 | jkp.std(axis=0).mean() / 10, 165 | jkt.mean() / 10, 166 | jkt.std(axis=0).mean() / 10, 167 | ) 168 | 169 | 170 | def apply_camera_transforms(joints, rotations, world2camera): 171 | """ 172 | Applies camera transformations to joint locations and rotations matrices. Based on 173 | https://github.com/aymenmir1/3dpw-eval/blob/master/evaluate.py 174 | :param joints: np array, joint positions, shape (N, 24, 3). 175 | :param rotations: np array, joint orientations, shape (N, 24, 3, 3). 176 | :param world2camera: np array, the world to camera transformation, shape (N, 4, 4). 177 | :return: the joints after applying the camera transformation, shape (N, 24, 3) 178 | the orientations after applying the camera transformation, shape (N, 24, 3, 3) 179 | """ 180 | joints_h = np.concatenate([joints, np.ones(joints.shape[:-1] + (1,))], axis=-1)[..., None] 181 | joints_c = np.matmul(world2camera[:, None], joints_h)[..., :3, 0] 182 | 183 | rotations_c = np.matmul(world2camera[:, None, :3, :3], rotations) 184 | 185 | return joints_c, rotations_c 186 | 187 | 188 | def compute_similarity_transform(S1, S2, num_joints, verts=None): 189 | """ 190 | Computes a similarity transform (sR, t) that takes a set of 3D points S1 (3 x N) closest to a set of 3D points S2, 191 | where R is an 3x3 rotation matrix, t 3x1 translation, s scale. I.e., solves the orthogonal Procrutes problem. 192 | Borrowed from https://github.com/aymenmir1/3dpw-eval/blob/master/evaluate.py 193 | """ 194 | transposed = False 195 | if S1.shape[0] != 3 and S1.shape[0] != 2: 196 | S1 = S1.T 197 | S2 = S2.T 198 | if verts is not None: 199 | verts = verts.T 200 | transposed = True 201 | assert S2.shape[1] == S1.shape[1] 202 | 203 | # Use only body joints for procrustes 204 | S1_p = S1[:, :num_joints] 205 | S2_p = S2[:, :num_joints] 206 | # 1. Remove mean. 207 | mu1 = S1_p.mean(axis=1, keepdims=True) 208 | mu2 = S2_p.mean(axis=1, keepdims=True) 209 | X1 = S1_p - mu1 210 | X2 = S2_p - mu2 211 | 212 | # 2. Compute variance of X1 used for scale. 213 | var1 = np.sum(X1**2) 214 | 215 | # 3. The outer product of X1 and X2. 216 | K = X1.dot(X2.T) 217 | 218 | # 4. Solution that Maximizes trace(R'K) is R=U*V', where U, V are 219 | # singular vectors of K. 220 | U, s, Vh = np.linalg.svd(K) 221 | V = Vh.T 222 | # Construct Z that fixes the orientation of R to get det(R)=1. 223 | Z = np.eye(U.shape[0]) 224 | Z[-1, -1] *= np.sign(np.linalg.det(U.dot(V.T))) 225 | # Construct R. 226 | R = V.dot(Z.dot(U.T)) 227 | 228 | # 5. Recover scale. 229 | scale = np.trace(R.dot(K)) / var1 230 | 231 | # 6. Recover translation. 232 | t = mu2 - scale * (R.dot(mu1)) 233 | 234 | # 7. Error: 235 | S1_hat = scale * R.dot(S1) + t 236 | 237 | verts_hat = None 238 | if verts is not None: 239 | verts_hat = scale * R.dot(verts) + t 240 | if transposed: 241 | verts_hat = verts_hat.T 242 | 243 | if transposed: 244 | S1_hat = S1_hat.T 245 | 246 | procrustes_params = {"scale": scale, "R": R, "trans": t} 247 | 248 | if verts_hat is not None: 249 | return S1_hat, verts_hat, procrustes_params 250 | else: 251 | return S1_hat, procrustes_params 252 | 253 | 254 | def compute_positional_errors(pred_joints, gt_joints, pred_verts, gt_verts, do_pelvis_alignment=True): 255 | """ 256 | Computes the MPJPE and PVE errors between the predicted and ground truth joints and vertices. 257 | :param pred_joints: np array, predicted joints, shape (N, 24, 3). 258 | :param gt_joints: np array, ground truth joints, shape (N, 24, 3). 259 | :param pred_verts: np array, predicted vertices, shape (N, 6890, 3). 260 | :param gt_verts: np array, ground truth vertices, shape (N, 6890, 3). 261 | :param do_pelvis_alignment: bool, whether to align the predictions to the ground truth pelvis. 262 | :return: A dictionary with the MPJPE and MVE errors. We return the errors both with and without a PA alignment. 263 | Further, the dictionary contains the mean errors for this sequence, as well as the errors for each frame (_pf). 264 | """ 265 | num_joints = gt_joints[0].shape[0] 266 | errors_jps, errors_pa_jps = [], [] 267 | errors_verts, errors_pa_verts = [], [] 268 | proc_rot = [] 269 | 270 | for i, (gt3d_jps, pd3d_jps) in enumerate(zip(gt_joints, pred_joints)): 271 | # Get corresponding ground truth and predicted 3d joints and verts 272 | gt3d_jps = gt3d_jps.reshape(-1, 3) 273 | gt3d_verts = gt_verts[i].reshape(-1, 3) 274 | pd3d_verts = pred_verts[i].reshape(-1, 3) 275 | 276 | # Root align. 277 | if do_pelvis_alignment: 278 | gt3d_verts = align_by_pelvis(gt3d_jps, gt3d_verts) 279 | pd3d_verts = align_by_pelvis(pd3d_jps, pd3d_verts) 280 | gt3d_jps = align_by_pelvis(gt3d_jps) 281 | pd3d_jps = align_by_pelvis(pd3d_jps) 282 | 283 | # Calculate joints and verts pelvis align error 284 | joint_error = np.sqrt(np.sum((gt3d_jps - pd3d_jps) ** 2, axis=1)) 285 | verts_error = np.sqrt(np.sum((gt3d_verts - pd3d_verts) ** 2, axis=1)) 286 | errors_jps.append(np.mean(joint_error)) 287 | errors_verts.append(np.mean(verts_error)) 288 | 289 | # Get procrustes align error. 290 | pd3d_jps_sym, pd3d_verts_sym, procrustesParam = compute_similarity_transform( 291 | pd3d_jps, gt3d_jps, num_joints, pd3d_verts 292 | ) 293 | proc_rot.append(procrustesParam["R"]) 294 | 295 | pa_jps_error = np.sqrt(np.sum((gt3d_jps - pd3d_jps_sym) ** 2, axis=1)) 296 | pa_verts_error = np.sqrt(np.sum((gt3d_verts - pd3d_verts_sym) ** 2, axis=1)) 297 | 298 | errors_pa_jps.append(np.mean(pa_jps_error)) 299 | errors_pa_verts.append(np.mean(pa_verts_error)) 300 | 301 | result_dict = { 302 | "mpjpe": np.mean(errors_jps), 303 | "mpjpe_pa": np.mean(errors_pa_jps), 304 | "mve": np.mean(errors_verts), 305 | "mve_pa": np.mean(errors_pa_verts), 306 | "mat_procs": np.stack(proc_rot, 0), 307 | "mpjpe_pf": np.stack(errors_jps, 0), 308 | "mpjpe_pf_pa": np.stack(errors_pa_jps, 0), 309 | "mve_pf": np.stack(errors_verts, 0), 310 | "mve_pf_pa": np.stack(errors_pa_verts, 0), 311 | } 312 | 313 | return result_dict 314 | 315 | 316 | def compute_metrics( 317 | pose_gt, 318 | shape_gt, 319 | trans_gt, 320 | pose_hat, 321 | shape_hat, 322 | trans_hat, 323 | gender_gt, 324 | gender_hat, 325 | camera_pose_gt=None, 326 | ): 327 | """ 328 | Computes all the metrics we want to report. 329 | :param pose_gt: np array, ground-truth SMPL pose parameters including the root, shape (N, 72) 330 | :param shape_gt: np array, ground-truth SMPL shape parameters, shape (N, 10) or (1, 10). 331 | :param trans_gt: np array, ground-truth SMPL translation parameters, shape (N, 3). 332 | :param pose_hat: np array, predicted SMPL pose parameters including the root, shape (N, 72). 333 | :param shape_hat: np array, predicted SMPL shape parameters, shape (N, 10) or (1, 10). 334 | :param trans_hat: np array, predicted SMPL translation parameters, shape (N, 3). 335 | :param gender_gt: "male", "female", or "neutral" 336 | :param gender_hat: "male", "female", "neutral", or None (in which case it defaults ot `gender_gt`) 337 | :param camera_pose_gt: np array, the world to camera transformation, shape (N, 4, 4). 338 | :return: The function returns two dictionarys: one with the average metrics for the whole sequence and one with the 339 | metrics for the whole sequence put per frame. 340 | """ 341 | # Get the 3D keypoints and joint angles for both ground-truth and prediction. 342 | pred_joints, gt_joints, pred_mats, gt_mats, pred_verts, gt_verts = get_data( 343 | pose_gt, 344 | shape_gt, 345 | trans_gt, 346 | pose_hat, 347 | shape_hat, 348 | trans_hat, 349 | gender_gt, 350 | gender_hat, 351 | ) 352 | 353 | # Rotate the ground-truth joints and rotation matrices into the camera's view. 354 | # I.e. results of the baselines/methods should be given in camera-relative coordinates. 355 | if camera_pose_gt is not None: 356 | gt_joints, gt_mats = apply_camera_transforms(gt_joints, gt_mats, camera_pose_gt) 357 | gt_verts, _ = apply_camera_transforms(gt_verts, gt_mats, camera_pose_gt) 358 | 359 | pos_errors = compute_positional_errors( 360 | pred_joints * 1000.0, gt_joints * 1000.0, pred_verts * 1000.0, gt_verts * 1000.0 361 | ) 362 | 363 | # Apply Procrustes rotation to the global rotation matrices. 364 | mats_procs_exp = np.expand_dims(pos_errors["mat_procs"], 1) 365 | mats_procs_exp = np.tile(mats_procs_exp, (1, len(SMPL_OR_JOINTS), 1, 1)) 366 | mats_pred_prc = np.matmul(mats_procs_exp, pred_mats) 367 | 368 | # Compute differences between the predicted matrices after procrustes and GT matrices. 369 | mpjae_pa_final, all_angles_pa = joint_angle_error(mats_pred_prc, gt_mats) 370 | 371 | # Compute MPJAE without Procrustes. 372 | mpjae_final, all_angles = joint_angle_error(pred_mats, gt_mats) 373 | 374 | # Compute Jitter Error. 375 | jkp_mean, jkp_std, jkt_mean, jkt_std = compute_jitter(pred_joints, gt_joints) 376 | 377 | # These are all scalars. Choose nice names for pretty printing later. 378 | metrics = { 379 | "MPJPE [mm]": pos_errors["mpjpe"], 380 | "MPJPE_PA [mm]": pos_errors["mpjpe_pa"], 381 | "MPJAE [deg]": mpjae_final, 382 | "MPJAE_PA [deg]": mpjae_pa_final, 383 | "MVE [mm]": pos_errors["mve"], 384 | "MVE_PA [mm]": pos_errors["mve_pa"], 385 | "Jitter [km/s^3]": jkp_mean, 386 | } 387 | 388 | metrics_extra = { 389 | "mpjpe_all": pos_errors["mpjpe_pf"], # (N,) 390 | "mpjpe_pa_all": pos_errors["mpjpe_pf_pa"], # (N,) 391 | "mpjae_all": all_angles, # (N, 9) 392 | "mpjae_pa_all": all_angles_pa, # (N, 9) 393 | "mve_all": pos_errors["mve_pf"], # (N,) 394 | "mve_pa_all": pos_errors["mve_pf_pa"], # (N,) 395 | "jitter_pd": jkp_mean, # Scalar 396 | "jitter_pd_std": jkp_std, # Scalar 397 | "jitter_gt_mean": jkt_mean, # Scalar 398 | "jitter_gt_std": jkt_std, # Scalar 399 | } 400 | 401 | return metrics, metrics_extra 402 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.isort] 2 | profile = "black" 3 | 4 | [tool.black] 5 | line-length = 120 6 | target-version = ['py37'] -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aitviewer>=1.11.0 2 | tabulate -------------------------------------------------------------------------------- /visualize.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | 4 | Script to visualize an EMDB sequence. Make sure to set the path of `EMDB_ROOT` and `SMPLX_MODELS` below. 5 | 6 | Usage: 7 | python visualize.py P8 68_outdoor_handstand 8 | """ 9 | import argparse 10 | import glob 11 | import os 12 | import pickle as pkl 13 | 14 | import cv2 15 | import numpy as np 16 | from aitviewer.configuration import CONFIG as C 17 | from aitviewer.models.smpl import SMPLLayer 18 | from aitviewer.renderables.billboard import Billboard 19 | from aitviewer.renderables.lines import LinesTrail 20 | from aitviewer.renderables.smpl import SMPLSequence 21 | from aitviewer.scene.camera import OpenCVCamera 22 | from aitviewer.viewer import Viewer 23 | 24 | from configuration import ( 25 | EMDB_ROOT, 26 | SMPL_SIDE_COLOR, 27 | SMPL_SIDE_INDEX, 28 | SMPL_SKELETON, 29 | SMPLX_MODELS, 30 | ) 31 | 32 | 33 | def draw_kp2d(kp2d, bboxes=None): 34 | """Draw 2D keypoints and bounding boxes on the image with OpenCV.""" 35 | 36 | def _draw_kp2d(img, current_frame_id): 37 | current_kp2d = kp2d[current_frame_id].copy() 38 | scale = img.shape[0] / 1000 39 | 40 | # Draw lines. 41 | for index in range(SMPL_SKELETON.shape[0]): 42 | i, j = SMPL_SKELETON[index] 43 | # color = SIDE_COLOR[max(SIDE_INDEX[i], SIDE_INDEX[j])] 44 | cv2.line( 45 | img, 46 | tuple(current_kp2d[i, :2].astype(np.int32)), 47 | tuple(current_kp2d[j, :2].astype(np.int32)), 48 | (0, 0, 0), 49 | int(scale * 3), 50 | ) 51 | 52 | # Draw points. 53 | for jth in range(0, kp2d.shape[1]): 54 | color = SMPL_SIDE_COLOR[SMPL_SIDE_INDEX[jth]] 55 | radius = scale * 5 56 | 57 | out_color = (0, 0, 0) 58 | in_color = color 59 | 60 | img = cv2.circle( 61 | img, 62 | tuple(current_kp2d[jth, :2].astype(np.int32)), 63 | int(radius * 1.4), 64 | out_color, 65 | -1, 66 | ) 67 | img = cv2.circle( 68 | img, 69 | tuple(current_kp2d[jth, :2].astype(np.int32)), 70 | int(radius), 71 | in_color, 72 | -1, 73 | ) 74 | 75 | # Draw bounding box if available. 76 | if bboxes is not None: 77 | bbox = bboxes[current_frame_id] 78 | x_min, y_min, x_max, y_max = ( 79 | int(bbox[0]), 80 | int(bbox[1]), 81 | int(bbox[2]), 82 | int(bbox[3]), 83 | ) 84 | cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (255, 0, 0), 2) 85 | return img 86 | 87 | return _draw_kp2d 88 | 89 | 90 | def draw_nothing(kp2d, bboxes=None): 91 | """Dummy function.""" 92 | 93 | def _draw_nothing(img, current_frame_id): 94 | return img 95 | 96 | return _draw_nothing 97 | 98 | 99 | def get_camera_position(Rt): 100 | """Get the orientation and position of the camera in world space.""" 101 | pos = -np.transpose(Rt[:, :3, :3], axes=(0, 2, 1)) @ Rt[:, :3, 3:] 102 | return pos.squeeze(-1) 103 | 104 | 105 | def get_sequence_root(args): 106 | """Parse the path of the sequence to be visualized.""" 107 | sequence_id = "{:0>2d}".format(int(args.sequence)) 108 | candidates = glob.glob(os.path.join(EMDB_ROOT, args.subject, sequence_id + "*")) 109 | if len(candidates) == 0: 110 | raise ValueError(f"Could not find sequence {args.sequence} for subject {args.subject}.") 111 | elif len(candidates) > 1: 112 | raise ValueError(f"Sequence ID {args.sequence}* for subject {args.subject} is ambiguous.") 113 | return candidates[0] 114 | 115 | 116 | def main(args): 117 | # Access EMDB data. 118 | sequence_root = get_sequence_root(args) 119 | data_file = glob.glob(os.path.join(sequence_root, "*_data.pkl"))[0] 120 | with open(data_file, "rb") as f: 121 | data = pkl.load(f) 122 | 123 | # Set up SMPL layer. 124 | gender = data["gender"] 125 | smpl_layer = SMPLLayer(model_type="smpl", gender=gender) 126 | 127 | # Create SMPL sequence. 128 | smpl_seq = SMPLSequence( 129 | data["smpl"]["poses_body"], 130 | smpl_layer=smpl_layer, 131 | poses_root=data["smpl"]["poses_root"], 132 | betas=data["smpl"]["betas"].reshape((1, -1)), 133 | trans=data["smpl"]["trans"], 134 | name="EMDB Fit", 135 | ) 136 | 137 | # Load 2D information. 138 | kp2d = data["kp2d"] 139 | bboxes = data["bboxes"]["bboxes"] 140 | drawing_function = draw_kp2d if args.draw_2d else draw_nothing 141 | 142 | # Load images. 143 | image_dir = os.path.join(sequence_root, "images") 144 | image_files = sorted(glob.glob(os.path.join(image_dir, "*.jpg"))) 145 | 146 | # Load camera information. 147 | intrinsics = data["camera"]["intrinsics"] 148 | extrinsics = data["camera"]["extrinsics"] 149 | cols, rows = data["camera"]["width"], data["camera"]["height"] 150 | 151 | # Create the viewer. 152 | viewer_size = None 153 | if args.view_from_camera: 154 | target_height = 1080 155 | width = int(target_height * cols / rows) 156 | viewer_size = (width, target_height) 157 | 158 | # If we view it from the camera drawing the 3D trajectories might be disturbing, suppress it. 159 | args.draw_trajectories = False 160 | 161 | viewer = Viewer(size=viewer_size) 162 | 163 | # Prepare the camera. 164 | intrinsics = np.repeat(intrinsics[np.newaxis, :, :], len(extrinsics), axis=0) 165 | cameras = OpenCVCamera(intrinsics, extrinsics[:, :3], cols, rows, viewer=viewer, name="Camera") 166 | 167 | # Display the images on a billboard. 168 | raw_images_bb = Billboard.from_camera_and_distance( 169 | cameras, 170 | 10.0, 171 | cols, 172 | rows, 173 | image_files, 174 | image_process_fn=drawing_function(kp2d, bboxes), 175 | name="Image", 176 | ) 177 | 178 | # Add everything to the scene. 179 | viewer.scene.add(smpl_seq, cameras, raw_images_bb) 180 | 181 | if args.draw_trajectories: 182 | # Add a path trail for the SMPL root trajectory. 183 | smpl_path = LinesTrail( 184 | smpl_seq.joints[:, 0], 185 | r_base=0.003, 186 | color=(0, 0, 1, 1), 187 | cast_shadow=False, 188 | name="SMPL Trajectory", 189 | ) 190 | 191 | # Add a path trail for the camera trajectory. 192 | # A fixed path (i.e. not a trail), could also be enabled in the GUI on the camera node by clicking "Show path". 193 | cam_pos = get_camera_position(extrinsics) 194 | camera_path = LinesTrail( 195 | cam_pos, 196 | r_base=0.003, 197 | color=(0.5, 0.5, 0.5, 1), 198 | cast_shadow=False, 199 | name="Camera Trajectory", 200 | ) 201 | 202 | viewer.scene.add(smpl_path, camera_path) 203 | 204 | # Remaining viewer setup. 205 | if args.view_from_camera: 206 | # We view the scene through the camera. 207 | viewer.set_temp_camera(cameras) 208 | 209 | # Hide all the GUI controls, they can be re-enabled by pressing `ESC`. 210 | viewer.render_gui = False 211 | else: 212 | # We center the scene on the first frame of the SMPL sequence. 213 | viewer.center_view_on_node(smpl_seq) 214 | 215 | viewer.scene.origin.enabled = False 216 | viewer.scene.floor.enabled = False 217 | viewer.playback_fps = 30.0 218 | 219 | viewer.run() 220 | 221 | 222 | if __name__ == "__main__": 223 | parser = argparse.ArgumentParser() 224 | 225 | parser.add_argument("subject", type=str, help="The subject ID, P0 - P9.") 226 | parser.add_argument( 227 | "sequence", 228 | type=str, 229 | help="The sequence ID. This can be any unambiguous prefix of the sequence's name, i.e. for the " 230 | "sequence '66_outdoor_rom' it could be '66' or any longer prefix including the full name.", 231 | ) 232 | parser.add_argument( 233 | "--view_from_camera", 234 | action="store_true", 235 | help="View it from the camera's perspective.", 236 | ) 237 | parser.add_argument( 238 | "--draw_2d", 239 | action="store_true", 240 | help="Draw 2D keypoints and bounding boxes on the image.", 241 | ) 242 | parser.add_argument( 243 | "--draw_trajectories", 244 | action="store_true", 245 | help="Render SMPL and camera trajectories.", 246 | ) 247 | 248 | args = parser.parse_args() 249 | 250 | C.update_conf({"smplx_models": SMPLX_MODELS}) 251 | 252 | main(args) 253 | -------------------------------------------------------------------------------- /visualize_GLAMR.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 ETH Zurich, Manuel Kaufmann 3 | 4 | Script to visualize an example GLAMR result. This reproduces Fig. 9 of the paper. GLAMR results were obtained using 5 | the code from https://github.com/NVlabs/GLAMR. 6 | 7 | Usage: 8 | python visualize_GLAMR.py 9 | """ 10 | import glob 11 | import os 12 | import pickle 13 | 14 | import numpy as np 15 | from aitviewer.configuration import CONFIG as C 16 | from aitviewer.models.smpl import SMPLLayer 17 | from aitviewer.renderables.lines import Lines 18 | from aitviewer.renderables.meshes import Meshes 19 | from aitviewer.renderables.smpl import SMPLSequence 20 | from aitviewer.scene.camera import OpenCVCamera 21 | from aitviewer.viewer import Viewer 22 | 23 | from configuration import EMDB_ROOT 24 | 25 | 26 | def cam2world(Rt): 27 | new_Rt = np.eye(4)[np.newaxis].repeat(repeats=Rt.shape[0], axis=0) 28 | pos = -np.transpose(Rt[:, :3, :3], axes=(0, 2, 1)) @ Rt[:, :3, 3:] 29 | rot = np.copy(np.transpose(Rt[:, :3, :3], axes=(0, 2, 1))) 30 | rot[:, :3, 1:3] *= -1.0 31 | new_Rt[:, :3, :3] = rot 32 | new_Rt[:, :3, 3:] = pos 33 | return new_Rt 34 | 35 | 36 | def world2cam(Rt): 37 | new_Rt = np.eye(4)[np.newaxis].repeat(repeats=Rt.shape[0], axis=0) 38 | new_Rt[:, :3, :3] = Rt[:, :3, :3] 39 | new_Rt[:, :3, 1:3] *= -1.0 40 | new_Rt[:, :3, :3] = np.transpose(new_Rt[:, :3, :3], axes=(0, 2, 1)) 41 | new_Rt[:, :3, 3:] = -new_Rt[:, :3, :3] @ Rt[:, :3, 3:] 42 | return new_Rt 43 | 44 | 45 | if __name__ == "__main__": 46 | # Load GLAMR output. 47 | glamr_file = "assets/GLAMR/P9_80_seed1.pkl" 48 | with open(glamr_file, "rb") as f: 49 | glamr_data = pickle.load(f) 50 | 51 | person = glamr_data["person_data"][0] 52 | poses_glamr = person["smpl_pose"] 53 | betas_glamr = person["smpl_beta"] 54 | trans_glamr = person["root_trans_world"] 55 | ori_glamr = person["smpl_orient_world"] 56 | Rt_pd = glamr_data["cam_pose"] 57 | K_pd = person["cam_K"] 58 | 59 | # Load EMDB ground-truth data. 60 | sequence_root = os.path.join(os.path.join(EMDB_ROOT, "P9", "80_outdoor_walk_big_circle")) 61 | gt_data_file = glob.glob(os.path.join(sequence_root, "*_data.pkl"))[0] 62 | with open(gt_data_file, "rb") as f: 63 | gt_data = pickle.load(f) 64 | 65 | poses_gt = gt_data["smpl"]["poses_body"] 66 | ori_gt = gt_data["smpl"]["poses_root"] 67 | betas_gt = gt_data["smpl"]["betas"] 68 | betas_gt = np.repeat(betas_gt.reshape((1, -1)), repeats=gt_data["n_frames"], axis=0) 69 | trans_gt = gt_data["smpl"]["trans"] 70 | cols, rows = gt_data["camera"]["width"], gt_data["camera"]["height"] 71 | Rt_gt = gt_data["camera"]["extrinsics"] 72 | K_gt = gt_data["camera"]["intrinsics"][np.newaxis].repeat(Rt_gt.shape[0], axis=0) 73 | 74 | # Align the two camera trajectories at the first frame. 75 | Rt_pd_w = cam2world(Rt_pd) 76 | Rt_gt_w = cam2world(Rt_gt) 77 | assert Rt_pd_w.shape == Rt_gt_w.shape 78 | 79 | # Compute the relative transformation between the two cameras. 80 | R_pd_f0 = Rt_pd_w[0, :3, :3] 81 | T_pd_f0 = Rt_pd_w[0, :3, 3:] 82 | R_gt_f0 = Rt_gt_w[0, :3, :3] 83 | T_gt_f0 = Rt_gt_w[0, :3, 3:] 84 | 85 | R_rel = R_gt_f0 @ R_pd_f0.T 86 | T_rel = T_gt_f0 - R_gt_f0 @ R_pd_f0.T @ T_pd_f0 87 | Rt_rel = np.eye(4) 88 | Rt_rel[:3, :3] = R_rel 89 | Rt_rel[:3, 3:] = T_rel 90 | Rt_rel = Rt_rel[np.newaxis].repeat(repeats=Rt_pd_w.shape[0], axis=0) 91 | 92 | # Apply the relative transformation to the predicted camera trajectory. 93 | Rt_pd_aligned_w = Rt_rel @ Rt_pd_w 94 | Rt_pd_aligned = world2cam(Rt_pd_aligned_w) 95 | 96 | # Define a helper function so that we can treat the root joint position the same way GLAMR does it and apply the 97 | # alignment that we found above. 98 | def post_fk_glamr(vertices, joints, align=False): 99 | # Subtract the position of the root joint from all vertices and joint positions and add the root translation. 100 | t = trans_glamr[:] 101 | cur_root_trans = joints[:, [0], :] 102 | vertices = vertices - cur_root_trans + t[:, None, :] 103 | joints = joints - cur_root_trans + t[:, None, :] 104 | 105 | def _to_h(x): 106 | return np.concatenate([x, np.ones(shape=(x.shape[:-1]) + (1,))], axis=-1) 107 | 108 | def _apply_transform(x, R): 109 | x_h = _to_h(x) 110 | return np.matmul(R[:, None], x_h[..., None]).squeeze(-1)[..., :3] 111 | 112 | if align: 113 | vertices_r = _apply_transform(vertices, Rt_rel) 114 | joints_r = _apply_transform(joints, Rt_rel) 115 | return vertices_r, joints_r 116 | 117 | return vertices, joints 118 | 119 | # Instantiate an SMPL sequence for the GLAMR data so that we can perform a forward pass through the SMPL model. 120 | # We set z_up=True because GLAMR data is using z_up coordinates. 121 | smpl_layer = SMPLLayer(model_type="smpl", gender="neutral", device=C.device) 122 | smpl_sequence_glamr = SMPLSequence( 123 | poses_body=poses_glamr, 124 | poses_root=ori_glamr, 125 | betas=betas_glamr, 126 | is_rigged=False, 127 | smpl_layer=smpl_layer, 128 | color=(149 / 255, 149 / 255, 149 / 255, 0.8), 129 | z_up=True, 130 | name="GLAMR", 131 | ) 132 | 133 | # Align the GLAMR result with the transformation that we found above. 134 | vs, js = smpl_sequence_glamr.vertices, smpl_sequence_glamr.joints 135 | vs_ori, js_ori = post_fk_glamr(vs, js) 136 | 137 | glamr_color = (68 / 255, 115 / 255, 23 / 255, 1.0) 138 | vs_aligned, js_aligned = post_fk_glamr(vs, js, align=True) 139 | 140 | # Prepare renderables to be displayed in the viewer. 141 | glamr_meshes_aligned = Meshes( 142 | vs_aligned, 143 | smpl_sequence_glamr.faces, 144 | name="GLAMR SMPL Prediction", 145 | color=glamr_color, 146 | ) 147 | 148 | # We also render the trajectories explicitly. We use thick lines because the trajectories are long and they 149 | # wouldn't otherwise be visible very well. 150 | # If you want the trajectories to be built up progressively, use a `LinesTrail` instead. 151 | glamr_root_aligned = Lines( 152 | js_aligned[:, 0], 153 | r_base=0.06, 154 | mode="line_strip", 155 | color=(1, 0, 0, 1), 156 | cast_shadow=False, 157 | name="GLAMR SMPL Root Trajectory", 158 | ) 159 | glamr_root_aligned.n_frames = js_aligned[:, 0].shape[0] 160 | 161 | glamr_cam_aligned = Lines( 162 | Rt_pd_aligned_w[:, :3, 3], 163 | r_base=0.04, 164 | mode="line_strip", 165 | color=(180 / 255, 180 / 255, 180 / 255, 1), 166 | cast_shadow=False, 167 | name="GLAMR Camera Trajectory", 168 | ) 169 | glamr_cam_aligned.n_frames = Rt_pd_aligned_w[:, :3, 3].shape[0] 170 | 171 | # Instantiate a SMPL sequence for the EMDB ground-truth. 172 | smpl_sequence_gt = SMPLSequence( 173 | poses_body=poses_gt, 174 | poses_root=ori_gt, 175 | betas=betas_gt, 176 | trans=trans_gt, 177 | is_rigged=False, 178 | smpl_layer=smpl_layer, 179 | color=(160 / 255, 160 / 255, 160 / 255, 1.0), 180 | z_up=False, 181 | name="EMDB SMPL Ground Truth", 182 | ) 183 | 184 | gt_root = Lines( 185 | smpl_sequence_gt.joints[:, 0], 186 | r_base=0.06, 187 | mode="line_strip", 188 | color=(0, 0, 1, 1), 189 | cast_shadow=False, 190 | name="EMDB SMPL Root Trajectory", 191 | ) 192 | gt_root.n_frames = smpl_sequence_gt.joints[:, 0].shape[0] 193 | 194 | gt_cam = Lines( 195 | Rt_gt_w[:, :3, 3], 196 | r_base=0.04, 197 | mode="line_strip", 198 | color=(0, 0, 0, 1), 199 | cast_shadow=False, 200 | name="EMDB Camera Trajectory", 201 | ) 202 | gt_cam.n_frames = Rt_gt_w[:, :3, 3].shape[0] 203 | 204 | # Create the viewer. 205 | viewer = Viewer() 206 | 207 | # We instantiate the GLAMR and EMDB cameras as actual OpenCVCameras so that the scene can be viewed from the 208 | # camera's perspective if desired. The thick trajectory lines obstruct the view from the camera, but they 209 | # can be disabled in the GUI. 210 | gt_opencv_cam = OpenCVCamera(K_gt, Rt_gt[:, :3, :], cols, rows, viewer=viewer, name="EMDB Camera") 211 | glamr_opencv_cam = OpenCVCamera(K_pd, Rt_pd_aligned[:, :3, :], cols, rows, viewer=viewer, name="GLAMR Camera") 212 | 213 | # Add everything to the scene 214 | viewer.scene.add(glamr_meshes_aligned, smpl_sequence_gt) 215 | viewer.scene.add(glamr_root_aligned, gt_root) 216 | viewer.scene.add(glamr_cam_aligned, gt_cam) 217 | viewer.scene.add(glamr_opencv_cam, gt_opencv_cam) 218 | 219 | # Set initial viewer camera. 220 | viewer.center_view_on_node(glamr_meshes_aligned) 221 | 222 | # Other viewer settings 223 | viewer.scene.floor.enabled = False 224 | viewer.scene.origin.enabled = False 225 | viewer.playback_fps = 30.0 226 | viewer.shadows_enabled = False 227 | viewer.auto_set_camera_target = False 228 | 229 | viewer.run() 230 | viewer.close() 231 | --------------------------------------------------------------------------------