├── .gitignore ├── README.md ├── compare_manual_anno.py ├── eval.py ├── joint_order.png ├── logo.png ├── pred.py ├── setup_mano.py ├── teaser.png ├── utils ├── __init__.py ├── eval_util.py ├── fh_utils.py ├── hand_r.txt └── vis_utils.py ├── vis_H2O3D.py ├── vis_HO3D.py └── vis_pcl_all_cameras.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .idea 132 | pred_for_reference.py 133 | vis_HO3D_test.py 134 | vis_HO3D_v3.py 135 | compute_visibility_ho3d.py 136 | pred_for_gt_checking.py 137 | tmp 138 | *.json 139 | mano 140 | 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HO-3D/H2O-3D - scripts 2 | 3 |
4 | 5 | **HO-3D v2 download link**: [https://files.icg.tugraz.at/d/5224b9b16422474892d7/?](https://files.icg.tugraz.at/d/5224b9b16422474892d7/?)\ 6 | **HO-3D v3 download link**: [https://1drv.ms/f/s!AsG9HA3ULXQRlFy5tCZXahAe3bEV?e=BevrKO](https://1drv.ms/f/s!AsG9HA3ULXQRlFy5tCZXahAe3bEV?e=BevrKO)\ 7 | **H2O-3D v1 download link**: [https://onedrive.live.com/?authkey=%21APhjjldozGGRZpE&id=11742DD40D1CBDC1%2134858&cid=11742DD40D1CBDC1](https://onedrive.live.com/?authkey=%21APhjjldozGGRZpE&id=11742DD40D1CBDC1%2134858&cid=11742DD40D1CBDC1) 8 | 9 | - Update - Nov 3rd, 2024 10 | 11 | We have now released the ground-truth annotations for HO-3D v2 and v3 datasets. 12 | 13 | | Dataset | GT download link | Command for running eval locally | 14 | | -------- | :-------: |------| 15 | | HO-3D v2 | [link](https://1drv.ms/f/c/11742dd40d1cbdc1/EjSh5wMqNilPoQKA0nQF2NMBl0rfyg1gZQCo0k3iXv8vig) | `python3 eval.py --version v2` | 16 | | HO-3D v3 | [link](https://1drv.ms/f/c/11742dd40d1cbdc1/EqzdBm7UDWVCmdxCyP373eQBhia924vXa4i85WqvWNLHYg?e=k6kshD) | `python3 eval.py --version v3` | 17 | 18 | - Update - Mar 29th, 2022 19 | 20 | We have released version 1 of H2O-3D dataset which contains 2 hands and an object interaction. 21 | The annotations are obtained using the [HOnnotate](https://github.com/shreyashampali/HOnnotate) method similar to 22 | the HO-3D dataset and hence contains the annotations in the same format. The dataset can be downloaded 23 | from [here](https://1drv.ms/u/s!AsG9HA3ULXQRgpAq-GOOV2jMYZFmkQ?e=AEEg13). This repo now also contains a script for visualizing H2O-3D dataset, `vis_H2O3D.py` 24 | - Update - Juy 1st, 2021 25 | 26 | We have now released **version 3** of the HO-3D dataset (HO-3D_v3) with **more accurate hand-object poses**. See this [report](https://arxiv.org/abs/2107.00887) and official 27 | [website](https://www.tugraz.at/institute/icg/research/team-lepetit/research-projects/hand-object-3d-pose-annotation/) for details and links. 28 | The annotation format and folder structure follows almost the same format as the previous version (HO-3D_v2) and hence replacing the old dataset with 29 | the new one should work just fine. The only change being all the rgb images are now in 'jpg' format instead of 'png' format due to storage constraints. 30 | 31 | A new codalab challenge for **version 3** has been created [here](https://codalab.lisn.upsaclay.fr/competitions/4393). Submission to this new challenge follows the exact same format as for 32 | [version 2](https://codalab.lisn.upsaclay.fr/competitions/4318). 33 |
34 | 35 | 36 | # About this Repo 37 | HO-3D and H2O-3D are datasets with 3D pose annotations for hands and object under severe occlusions from each other. 38 | The HO-3D dataset contains sequences of right hand interacting with an object and was released as part of the 39 | CVPR'20 paper, [HOnnotate](https://arxiv.org/pdf/1907.01481). The H2O-3D dataset contains sequences of 40 | two hands interacting with an object and was released as part of the CVPR'22 paper, [Keypoint Transformer](). 41 | 42 | The sequences in the dataset contain different persons manipulating different objects, which are taken from [YCB dataset](https://rse-lab.cs.washington.edu/projects/posecnn/). Details about the 43 | proposed annotation method can be found in our [paper](https://arxiv.org/pdf/1907.01481). The HO-3D (version 3) dataset contains 103,462 annotated images and their 44 | corresponding depth maps. The H2O-3D (version 1) dataset contains 76,340 annotated images and their 45 | corresponding depth maps. 46 | 47 | For more details about the dataset and the corresponding work, visit our [project page](https://www.tugraz.at/index.php?id=40231) 48 | 49 | An online codalab challenge which provides a platform to evaluate different hand pose estimation methods on our dataset with standard metrics is launched 50 | [here](https://codalab.lisn.upsaclay.fr/competitions/4318) (for version 2) and [here](https://codalab.lisn.upsaclay.fr/competitions/4393) (for version 3) 51 | 52 | 53 | 54 | This repository contains a collection of scripts for: 55 | * Visualization of HO-3D dataset 56 | * Visualization of H2O-3D dataset 57 | * Evaluation scripts used in the challenges 58 | 59 | # Comparison with SOTA methods on HO-3D dataset 60 | ![image](https://user-images.githubusercontent.com/19649112/165757439-1edc31db-2ca7-4b1b-a27a-cec70bcfc6f5.png) 61 | ![image](https://user-images.githubusercontent.com/19649112/165757566-7cd96ff2-d2d9-44fe-8b29-2f4ec0cb022e.png) 62 | 63 | 64 | # Basic setup 65 | 66 | 1. Install basic requirements: 67 | ``` 68 | conda create -n python2.7 python=2.7 69 | source activate python2.7 70 | pip install numpy matplotlib scikit-image transforms3d tqdm opencv-python cython open3d 71 | ``` 72 | 2. Download Models&code from the MANO website 73 | ``` 74 | http://mano.is.tue.mpg.de 75 | ``` 76 | 3. Assuming ${MANO_PATH} contains the path to where you unpacked the downloaded archive, use the provided script to setup the MANO folder as required. 77 | ``` 78 | python setup_mano.py ${MANO_PATH} 79 | ``` 80 | 81 | 4. Download the YCB object models by clicking on `The YCB-Video 3D Models` in [https://rse-lab.cs.washington.edu/projects/posecnn/]. Assume ${YCB_PATH} 82 | is the path where you unpacked the object models into (path to where _models_ folder branches off) 83 | 84 | 5. Download the HO-3D dataset. See project page for instructions. 85 | 86 | 6. Assuming ${DB_PATH} is the path to where you unpacked the dataset (path to where _./train/_ and _./evaluation/_ folder branch off), 87 | This should enable you to run the following to show some dataset samples. 88 | ``` 89 | python vis_HO3D.py ${DB_PATH} ${YCB_PATH} 90 | python vis_HO3D.py ${DB_PATH} ${YCB_PATH} -split 'evaluation' 91 | python vis_HO3D.py ${DB_PATH} ${YCB_PATH} -visType 'open3d' 92 | 93 | python vis_H2O3D.py ${DB_PATH} ${YCB_PATH} 94 | python vis_H2O3D.py ${DB_PATH} ${YCB_PATH} -split 'evaluation' 95 | python vis_H2O3D.py ${DB_PATH} ${YCB_PATH} -visType 'open3d' 96 | ``` 97 | 98 | The script provides parameters to visualize the annotations in 3D using open3d or in 2D in matplotlib window. Use `-visType` to set the visualization type. 99 | The script also provides parameters to visualize samples in the training and evaluation split using the parameters `-split`. 100 | 101 | 102 | # Evaluate on the dataset 103 | 104 | In order to have consistent evaluation of the hand pose estimation algorithms on HO-3D dataset, evaluation is handled through CodaLab competition. 105 | 106 | 1. Make predictions for the evaluation dataset. The code provided here predicts zeros for all joints and vertices. ${ver} specifies the version 107 | of the dataset ('v2' or 'v3') 108 | ``` 109 | python pred.py ${DB_PATH} --version ${ver} 110 | ``` 111 | 112 | 2. Zip the `pred.json` file 113 | ``` 114 | zip -j pred.zip pred.json 115 | ``` 116 | 117 | 3. Upload `pred.zip` to our Codalab competition ([version 2](https://codalab.lisn.upsaclay.fr/competitions/4318) 118 | or [version3](https://codalab.lisn.upsaclay.fr/competitions/4393))website (Participate -> Submit) 119 | 120 | 4. Wait for the evaluation server to report back your results and publish your results to the leaderboard. The zero predictor will give you the following results 121 | ``` 122 | Mean joint error 56.87cm 123 | Mean joint error (procrustes alignment) 5.19cm 124 | Mean joint error (scale and trans alignment) NaN 125 | Mesh error 57.12cm 126 | Mesh error (procrustes alignment) 5.47cm 127 | F@5mm=0.0, F@15mm=0.0 128 | F_aliged@5mm= 0.000, F_aligned@15mm=0.017 129 | ``` 130 | 131 | 5. Modify `pred.py` to use your method for making hand pose estimation and see how well it performs compared to the baselines. Note that the pose 132 | estimates need to be in **OpenGL** coordinate system (hand is along negative z-axis in a right-handed coordinate system with origin at camera optic center) 133 | during the submission. 134 | 135 | 6. The calculation of the evaluation metrics can be found in `eval.py` 136 | 137 | # Visualize Point Cloud from All the Cameras (only in HO-3D version 3) 138 | 139 | We provide the extrinsic camera parameters in 'calibration' folder of the dataset. The RGB-D data from all the cameras 140 | for multi-camera sequences can be combined to visualize the point-cloud using the below script: 141 | ```python 142 | python vis_pcl_all_cameras.py ${DB_PATH} --seq SEQ --fid FID 143 | ``` 144 | `SEQ` and `FID` are the sequence name and file name. Try `-h` for list of accepted sequence names. 145 | 146 | # Compare with Manual Annotations (only in HO-3D version 3) 147 | 148 | We manually annotated 5 finger tip locations in 53 frames using the point-cloud from all the cameras. The manually annotated 149 | finger tip locations are provided in 'manual_annotations' folder of the dataset. We measure the accuracy of our automatic 150 | annotations by comparing with the manual annotations using the below script: 151 | ```python 152 | python compare_manual_anno.py ${DB_PATH} 153 | ``` 154 | 155 | 156 | # Terms of use 157 | 158 | The download and use of the dataset is for academic research only and it is free to researchers from educational or research institutes 159 | for non-commercial purposes. When downloading the dataset you agree to (unless with expressed permission of the authors): 160 | not redistribute, modificate, or commercial usage of this dataset in any way or form, either partially or entirely. 161 | If using one of these dataset, please cite the corresponding paper. 162 | 163 | 164 | @INPROCEEDINGS{hampali2020honnotate, 165 | title={HOnnotate: A method for 3D Annotation of Hand and Object Poses}, 166 | author={Shreyas Hampali and Mahdi Rad and Markus Oberweger and Vincent Lepetit}, 167 | booktitle = {CVPR}, 168 | year = {2020} 169 | } 170 | 171 | @INPROCEEDINGS{hampali2022keypointtransformer, 172 | title={Keypoint Transformer: Solving Joint Identification in Challenging Hands and Object Interactions for Accurate 3D Pose Estimation}, 173 | author={Shreyas Hampali and Sayan Deb Sarkar and Mahdi Rad and Vincent Lepetit}, 174 | booktitle = {CVPR}, 175 | year = {2022} 176 | } 177 | 178 | # Acknowledgments 179 | 180 | 1. The evaluation scripts used in the HO-3D challenge have been mostly re-purposed from [Freihand challenge](https://github.com/lmb-freiburg/freihand). We 181 | thank the authors for making their code public. 182 | 183 | 2. This work was supported by the Christian Doppler Laboratory for Semantic 3D Computer Vision, funded in part 184 | by Qualcomm Inc 185 | -------------------------------------------------------------------------------- /compare_manual_anno.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import argparse 4 | from os.path import join 5 | from utils.vis_utils import load_pickle_data 6 | 7 | # Paths and Parameters 8 | 9 | 10 | fingerTipIds = [20, 19, 18, 17, 16] 11 | 12 | 13 | def calculateMSEPerFrame(manualAnno, annotFile): 14 | seq = annotFile.split('_')[0] 15 | fID = annotFile.split('_')[1] 16 | 17 | optDir = (os.path.join(args.base_path, 'train', seq+'1', 18 | 'meta', 19 | fID + '.pkl')) 20 | 21 | 22 | if not os.path.exists(optDir): 23 | print('[INFO] Skipping sequence %s file ID %s as it is part of test set'%(seq,fID)) 24 | return np.nan 25 | 26 | optPickData = load_pickle_data(optDir) 27 | 28 | rightHandJointLocs = optPickData['handJoints3D'][fingerTipIds] 29 | 30 | mse = np.mean(np.linalg.norm(manualAnno - rightHandJointLocs, axis=1)) 31 | 32 | return mse 33 | 34 | 35 | if __name__ == '__main__': 36 | parser = argparse.ArgumentParser(description='Compare with manual annotations') 37 | parser.add_argument('base_path', type=str, 38 | help='Path to where the HO3D dataset is located.') 39 | 40 | args = parser.parse_args() 41 | 42 | annotSaveDir = join(args.base_path, 'manual_annotations') 43 | annotFiles = sorted(os.listdir(annotSaveDir)) 44 | 45 | mseSum = [] 46 | for annotFile in annotFiles: 47 | annotPickData = np.load(os.path.join(annotSaveDir, annotFile)) 48 | 49 | mseSum.append(calculateMSEPerFrame(annotPickData, annotFile[:-4])) 50 | 51 | mseSum = np.array(mseSum, dtype=np.float32) 52 | 53 | print('Number of samples = %d'%(mseSum.shape[0] - np.sum(np.isnan(mseSum)))) 54 | print('Average MSE - {}mm, Standard Deviation - {}mm'.format(np.nanmean(mseSum)*1000, 55 | np.std(mseSum[np.logical_not(np.isnan(mseSum))])*1000)) 56 | -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | import matplotlib 3 | matplotlib.use('Agg') 4 | import matplotlib.pyplot as plt 5 | 6 | import pip 7 | import argparse 8 | import base64 9 | import json 10 | 11 | def install(package): 12 | if hasattr(pip, 'main'): 13 | pip.main(['install', package]) 14 | else: 15 | from pip._internal.main import main as pipmain 16 | pipmain(['install', package]) 17 | 18 | try: 19 | from scipy.linalg import orthogonal_procrustes 20 | except: 21 | install('scipy') 22 | from scipy.linalg import orthogonal_procrustes 23 | 24 | 25 | from utils.fh_utils import * 26 | from utils.eval_util import EvalUtil 27 | 28 | 29 | def calculate_fscore(gt, pr, th=0.01): 30 | d1 = np.min(np.linalg.norm(np.expand_dims(gt, axis=-2) - np.expand_dims(pr, axis=-3),axis=-1),axis=-1) 31 | d2 = np.min(np.linalg.norm(np.expand_dims(pr, axis=-2) - np.expand_dims(gt, axis=-3),axis=-1),axis=-1) 32 | if len(d1) and len(d2): 33 | recall = float(sum(d < th for d in d2)) / float(len(d2)) # how many of our predicted points lie close to a gt point? 34 | precision = float(sum(d < th for d in d1)) / float(len(d1)) # how many of gt points are matched? 35 | 36 | if recall+precision > 0: 37 | fscore = 2 * recall * precision / (recall + precision) 38 | else: 39 | fscore = 0 40 | else: 41 | fscore = 0 42 | precision = 0 43 | recall = 0 44 | return fscore, precision, recall 45 | 46 | def align_sc_tr(mtx1, mtx2): 47 | """ Align the 3D joint location with the ground truth by scaling and translation """ 48 | 49 | predCurr = mtx2.copy() 50 | # normalize the predictions 51 | s = np.sqrt(np.sum(np.square(predCurr[4] - predCurr[0]))) 52 | if s>0: 53 | predCurr = predCurr / s 54 | 55 | # get the scale of the ground truth 56 | sGT = np.sqrt(np.sum(np.square(mtx1[4] - mtx1[0]))) 57 | 58 | # make predictions scale same as ground truth scale 59 | predCurr = predCurr * sGT 60 | 61 | # make preditions translation of the wrist joint same as ground truth 62 | predCurrRel = predCurr - predCurr[0:1, :] 63 | preds_sc_tr_al = predCurrRel + mtx1[0:1, :] 64 | 65 | return preds_sc_tr_al 66 | 67 | 68 | 69 | 70 | def align_w_scale(mtx1, mtx2, return_trafo=False): 71 | """ Align the predicted entity in some optimality sense with the ground truth. """ 72 | # center 73 | t1 = mtx1.mean(0) 74 | t2 = mtx2.mean(0) 75 | mtx1_t = mtx1 - t1 76 | mtx2_t = mtx2 - t2 77 | 78 | # scale 79 | s1 = np.linalg.norm(mtx1_t) + 1e-8 80 | mtx1_t /= s1 81 | s2 = np.linalg.norm(mtx2_t) + 1e-8 82 | mtx2_t /= s2 83 | 84 | # orth alignment 85 | R, s = orthogonal_procrustes(mtx1_t, mtx2_t) 86 | 87 | # apply trafos to the second matrix 88 | mtx2_t = np.dot(mtx2_t, R.T) * s 89 | mtx2_t = mtx2_t * s1 + t1 90 | if return_trafo: 91 | return R, s, s1, t1 - t2 92 | else: 93 | return mtx2_t 94 | 95 | 96 | def align_by_trafo(mtx, trafo): 97 | t2 = mtx.mean(0) 98 | mtx_t = mtx - t2 99 | R, s, s1, t1 = trafo 100 | return np.dot(mtx_t, R.T) * s * s1 + t1 + t2 101 | 102 | 103 | class curve: 104 | def __init__(self, x_data, y_data, x_label, y_label, text): 105 | self.x_data = x_data 106 | self.y_data = y_data 107 | self.x_label = x_label 108 | self.y_label = y_label 109 | self.text = text 110 | 111 | 112 | def createHTML(outputDir, curve_list): 113 | curve_data_list = list() 114 | for item in curve_list: 115 | fig1 = plt.figure() 116 | ax = fig1.add_subplot(111) 117 | ax.plot(item.x_data, item.y_data) 118 | ax.set_xlabel(item.x_label) 119 | ax.set_ylabel(item.y_label) 120 | img_path = os.path.join(outputDir, "img_path_path.png") 121 | plt.savefig(img_path, bbox_inches=0, dpi=300) 122 | 123 | # write image and create html embedding 124 | data_uri1 = base64.b64encode(open(img_path, 'rb').read()).replace(b'\n', b'') 125 | img_tag1 = 'src="data:image/png;base64,{0}"'.format(data_uri1) 126 | curve_data_list.append((item.text, img_tag1)) 127 | 128 | os.remove(img_path) 129 | 130 | htmlString = ''' 131 | 132 | 133 |

Detailed results:

''' 134 | 135 | for i, (text, img_embed) in enumerate(curve_data_list): 136 | htmlString += ''' 137 |

%s

138 |

139 | FROC 140 |

141 |

Raw curve data:

142 | 143 |

x_axis: %s

144 |

y_axis: %s

145 | 146 | ''' % (text, img_embed, curve_list[i].x_data, curve_list[i].y_data) 147 | 148 | htmlString += ''' 149 | 150 | ''' 151 | 152 | htmlfile = open(os.path.join(outputDir, "scores.html"), "w") 153 | htmlfile.write(htmlString) 154 | htmlfile.close() 155 | 156 | 157 | def _search_pred_file(pred_path, pred_file_name): 158 | """ Tries to select the prediction file. Useful, in case people deviate from the canonical prediction file name. """ 159 | pred_file = os.path.join(pred_path, pred_file_name) 160 | if os.path.exists(pred_file): 161 | # if the given prediction file exists we are happy 162 | return pred_file 163 | 164 | print('Predition file "%s" was NOT found' % pred_file_name) 165 | 166 | # search for a file to use 167 | print('Trying to locate the prediction file automatically ...') 168 | files = [os.path.join(pred_path, x) for x in os.listdir(pred_path) if x.endswith('.json')] 169 | if len(files) == 1: 170 | pred_file_name = files[0] 171 | print('Found file "%s"' % pred_file_name) 172 | return pred_file_name 173 | else: 174 | print('Found %d candidate files for evaluation' % len(files)) 175 | raise Exception('Giving up, because its not clear which file to evaluate.') 176 | 177 | 178 | def main(gt_path, pred_path, output_dir, version, pred_file_name=None, set_name=None): 179 | if pred_file_name is None: 180 | pred_file_name = 'pred.json' 181 | if set_name is None: 182 | set_name = 'evaluation' 183 | 184 | # load eval annotations 185 | xyz_list, verts_list = json_load(os.path.join(gt_path, '%s_xyz.json' % set_name)), json_load(os.path.join(gt_path, '%s_verts.json' % set_name)) 186 | 187 | 188 | # load predicted values 189 | pred_file = _search_pred_file(pred_path, pred_file_name) 190 | print('Loading predictions from %s' % pred_file) 191 | with open(pred_file, 'r') as fi: 192 | pred = json.load(fi) 193 | 194 | assert len(pred) == 2, 'Expected format mismatch.' 195 | assert len(pred[0]) == len(xyz_list), 'Expected format mismatch.' 196 | assert len(pred[1]) == len(xyz_list), 'Expected format mismatch.' 197 | assert len(pred[0]) == db_size(set_name, version=version) 198 | 199 | 200 | # init eval utils 201 | eval_xyz, eval_xyz_procrustes_aligned, eval_xyz_sc_tr_aligned = EvalUtil(), EvalUtil(), EvalUtil() 202 | eval_mesh_err, eval_mesh_err_aligned = EvalUtil(num_kp=778), EvalUtil(num_kp=778) 203 | f_score, f_score_aligned = list(), list() 204 | f_threshs = [0.005, 0.015] 205 | 206 | shape_is_mano = None 207 | 208 | try: 209 | from tqdm import tqdm 210 | rng = tqdm(range(db_size(set_name, version))) 211 | except: 212 | rng = range(db_size(set_name,version)) 213 | 214 | # iterate over the dataset once 215 | for idx in rng: 216 | if idx >= db_size(set_name,version): 217 | break 218 | 219 | xyz, verts = xyz_list[idx], verts_list[idx] 220 | xyz, verts = [np.array(x) for x in [xyz, verts]] 221 | 222 | xyz_pred, verts_pred = pred[0][idx], pred[1][idx] 223 | xyz_pred, verts_pred = [np.array(x) for x in [xyz_pred, verts_pred]] 224 | 225 | # Not aligned errors 226 | eval_xyz.feed( 227 | xyz, 228 | np.ones_like(xyz[:, 0]), 229 | xyz_pred 230 | ) 231 | 232 | if shape_is_mano is None: 233 | if verts_pred.shape[0] == verts.shape[0]: 234 | shape_is_mano = True 235 | else: 236 | shape_is_mano = False 237 | 238 | if shape_is_mano: 239 | eval_mesh_err.feed( 240 | verts, 241 | np.ones_like(verts[:, 0]), 242 | verts_pred 243 | ) 244 | 245 | # scale and translation aligned predictions for xyz 246 | xyz_pred_sc_tr_aligned = align_sc_tr(xyz, xyz_pred) 247 | eval_xyz_sc_tr_aligned.feed( 248 | xyz, 249 | np.ones_like(xyz[:, 0]), 250 | xyz_pred_sc_tr_aligned 251 | ) 252 | 253 | # align predictions 254 | xyz_pred_aligned = align_w_scale(xyz, xyz_pred) 255 | if shape_is_mano: 256 | verts_pred_aligned = align_w_scale(verts, verts_pred) 257 | else: 258 | # use trafo estimated from keypoints 259 | trafo = align_w_scale(xyz, xyz_pred, return_trafo=True) 260 | verts_pred_aligned = align_by_trafo(verts_pred, trafo) 261 | 262 | # Aligned errors 263 | eval_xyz_procrustes_aligned.feed( 264 | xyz, 265 | np.ones_like(xyz[:, 0]), 266 | xyz_pred_aligned 267 | ) 268 | 269 | if shape_is_mano: 270 | eval_mesh_err_aligned.feed( 271 | verts, 272 | np.ones_like(verts[:, 0]), 273 | verts_pred_aligned 274 | ) 275 | 276 | # F-scores 277 | l, la = list(), list() 278 | for t in f_threshs: 279 | # for each threshold calculate the f score and the f score of the aligned vertices 280 | f, _, _ = calculate_fscore(verts, verts_pred, t) 281 | # f = 0. 282 | l.append(f) 283 | f, _, _ = calculate_fscore(verts, verts_pred_aligned, t) 284 | # f = 0. 285 | la.append(f) 286 | f_score.append(l) 287 | f_score_aligned.append(la) 288 | 289 | # Calculate results 290 | xyz_mean3d, _, xyz_auc3d, pck_xyz, thresh_xyz = eval_xyz.get_measures(0.0, 0.05, 100) 291 | print('Evaluation 3D KP results:') 292 | print('auc=%.3f, mean_kp3d_avg=%.2f cm' % (xyz_auc3d, xyz_mean3d * 100.0)) 293 | 294 | xyz_procrustes_al_mean3d, _, xyz_procrustes_al_auc3d, pck_xyz_procrustes_al, thresh_xyz_procrustes_al = eval_xyz_procrustes_aligned.get_measures(0.0, 0.05, 100) 295 | print('Evaluation 3D KP PROCRUSTES ALIGNED results:') 296 | print('auc=%.3f, mean_kp3d_avg=%.2f cm' % (xyz_procrustes_al_auc3d, xyz_procrustes_al_mean3d * 100.0)) 297 | 298 | xyz_sc_tr_al_mean3d, _, xyz_sc_tr_al_auc3d, pck_xyz_sc_tr_al, thresh_xyz_sc_tr_al = eval_xyz_sc_tr_aligned.get_measures(0.0, 0.05, 100) 299 | print('Evaluation 3D KP SCALE-TRANSLATION ALIGNED results:') 300 | print('auc=%.3f, mean_kp3d_avg=%.2f cm\n' % (xyz_sc_tr_al_auc3d, xyz_sc_tr_al_mean3d * 100.0)) 301 | 302 | 303 | if shape_is_mano: 304 | mesh_mean3d, _, mesh_auc3d, pck_mesh, thresh_mesh = eval_mesh_err.get_measures(0.0, 0.05, 100) 305 | print('Evaluation 3D MESH results:') 306 | print('auc=%.3f, mean_kp3d_avg=%.2f cm' % (mesh_auc3d, mesh_mean3d * 100.0)) 307 | 308 | mesh_al_mean3d, _, mesh_al_auc3d, pck_mesh_al, thresh_mesh_al = eval_mesh_err_aligned.get_measures(0.0, 0.05, 100) 309 | print('Evaluation 3D MESH ALIGNED results:') 310 | print('auc=%.3f, mean_kp3d_avg=%.2f cm\n' % (mesh_al_auc3d, mesh_al_mean3d * 100.0)) 311 | else: 312 | mesh_mean3d, mesh_auc3d, mesh_al_mean3d, mesh_al_auc3d = -1.0, -1.0, -1.0, -1.0 313 | 314 | pck_mesh, thresh_mesh = np.array([-1.0, -1.0]), np.array([0.0, 1.0]) 315 | pck_mesh_al, thresh_mesh_al = np.array([-1.0, -1.0]), np.array([0.0, 1.0]) 316 | 317 | print('F-scores') 318 | f_out = list() 319 | f_score, f_score_aligned = np.array(f_score).T, np.array(f_score_aligned).T 320 | for f, fa, t in zip(f_score, f_score_aligned, f_threshs): 321 | print('F@%.1fmm = %.3f' % (t*1000, f.mean()), '\tF_aligned@%.1fmm = %.3f' % (t*1000, fa.mean())) 322 | f_out.append('f_score_%d: %f' % (round(t*1000), f.mean())) 323 | f_out.append('f_al_score_%d: %f' % (round(t*1000), fa.mean())) 324 | 325 | # Dump results 326 | score_path = os.path.join(output_dir, 'scores.txt') 327 | with open(score_path, 'w') as fo: 328 | xyz_mean3d *= 100 329 | xyz_procrustes_al_mean3d *= 100 330 | xyz_sc_tr_al_mean3d *= 100 331 | fo.write('xyz_mean3d: %f\n' % xyz_mean3d) 332 | fo.write('xyz_auc3d: %f\n' % xyz_auc3d) 333 | fo.write('xyz_procrustes_al_mean3d: %f\n' % xyz_procrustes_al_mean3d) 334 | fo.write('xyz_procrustes_al_auc3d: %f\n' % xyz_procrustes_al_auc3d) 335 | fo.write('xyz_scale_trans_al_mean3d: %f\n' % xyz_sc_tr_al_mean3d) 336 | fo.write('xyz_scale_trans_al_auc3d: %f\n' % xyz_sc_tr_al_auc3d) 337 | 338 | mesh_mean3d *= 100 339 | mesh_al_mean3d *= 100 340 | fo.write('mesh_mean3d: %f\n' % mesh_mean3d) 341 | fo.write('mesh_auc3d: %f\n' % mesh_auc3d) 342 | fo.write('mesh_al_mean3d: %f\n' % mesh_al_mean3d) 343 | fo.write('mesh_al_auc3d: %f\n' % mesh_al_auc3d) 344 | for t in f_out: 345 | fo.write('%s\n' % t) 346 | print('Scores written to: %s' % score_path) 347 | 348 | createHTML( 349 | output_dir, 350 | [ 351 | curve(thresh_xyz*100, pck_xyz, 'Distance in cm', 'Percentage of correct keypoints', 'PCK curve for aligned keypoint error'), 352 | curve(thresh_xyz_procrustes_al*100, pck_xyz_procrustes_al, 'Distance in cm', 'Percentage of correct keypoints', 'PCK curve for procrustes aligned keypoint error'), 353 | curve(thresh_xyz_sc_tr_al * 100, pck_xyz_sc_tr_al, 'Distance in cm', 354 | 'Percentage of correct keypoints', 'PCK curve for scale-translation aligned keypoint error'), 355 | curve(thresh_mesh*100, pck_mesh, 'Distance in cm', 'Percentage of correct vertices', 'PCV curve for mesh error'), 356 | curve(thresh_mesh_al*100, pck_mesh_al, 'Distance in cm', 'Percentage of correct vertices', 'PCV curve for aligned mesh error') 357 | ] 358 | ) 359 | 360 | print('Evaluation complete.') 361 | 362 | 363 | if __name__ == '__main__': 364 | parser = argparse.ArgumentParser(description='Show some samples from the dataset.') 365 | parser.add_argument('input_dir', type=str, 366 | help='Path to where prediction the submited result and the ground truth is.') 367 | parser.add_argument('output_dir', type=str, 368 | help='Path to where the eval result should be.') 369 | parser.add_argument('--pred_file_name', type=str, default='pred.json', 370 | help='Name of the eval file.') 371 | parser.add_argument('--version', type=str, choices=['v2', 'v3'], 372 | help='HO3D version', default='v2') 373 | args = parser.parse_args() 374 | 375 | # call eval 376 | main( 377 | os.path.join(args.input_dir), 378 | os.path.join(args.input_dir), 379 | args.output_dir, 380 | pred_file_name=args.pred_file_name, 381 | version=args.version, 382 | set_name='evaluation', 383 | ) 384 | -------------------------------------------------------------------------------- /joint_order.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shreyashampali/ho3d/c1c8923f2f90fc2ec7c502f491b3851e2c58c388/joint_order.png -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shreyashampali/ho3d/c1c8923f2f90fc2ec7c502f491b3851e2c58c388/logo.png -------------------------------------------------------------------------------- /pred.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | import argparse 3 | from tqdm import tqdm 4 | 5 | from utils.vis_utils import * 6 | 7 | 8 | def main(base_path, pred_out_path, pred_func, version, set_name=None): 9 | """ 10 | Main eval loop: Iterates over all evaluation samples and saves the corresponding predictions. 11 | """ 12 | # default value 13 | if set_name is None: 14 | set_name = 'evaluation' 15 | 16 | # init output containers 17 | xyz_pred_list, verts_pred_list = list(), list() 18 | 19 | # read list of evaluation files 20 | with open(os.path.join(base_path, set_name+'.txt')) as f: 21 | file_list = f.readlines() 22 | file_list = [f.strip() for f in file_list] 23 | 24 | assert len(file_list) == db_size(set_name, version), '%s.txt is not accurate. Aborting'%set_name 25 | 26 | # iterate over the dataset once 27 | for idx in tqdm(range(db_size(set_name, version))): 28 | if idx >= db_size(set_name, version): 29 | break 30 | 31 | seq_name = file_list[idx].split('/')[0] 32 | file_id = file_list[idx].split('/')[1] 33 | 34 | # load input image 35 | img = read_RGB_img(base_path, seq_name, file_id, set_name) 36 | aux_info = read_annotation(base_path, seq_name, file_id, set_name) 37 | 38 | # use some algorithm for prediction 39 | xyz, verts = pred_func( 40 | img, 41 | aux_info 42 | ) 43 | 44 | # simple check if xyz and verts are in opengl coordinate system 45 | if np.all(xyz[:,2]>0) or np.all(verts[:,2]>0): 46 | raise Exception('It appears the pose estimates are not in OpenGL coordinate system. Please read README.txt in dataset folder. Aborting!') 47 | 48 | xyz_pred_list.append(xyz) 49 | verts_pred_list.append(verts) 50 | 51 | # dump results 52 | dump(pred_out_path, xyz_pred_list, verts_pred_list) 53 | 54 | 55 | def dump(pred_out_path, xyz_pred_list, verts_pred_list): 56 | """ Save predictions into a json file. """ 57 | # make sure its only lists 58 | xyz_pred_list = [x.tolist() for x in xyz_pred_list] 59 | verts_pred_list = [x.tolist() for x in verts_pred_list] 60 | 61 | # save to a json 62 | with open(pred_out_path, 'w') as fo: 63 | json.dump( 64 | [ 65 | xyz_pred_list, 66 | verts_pred_list 67 | ], fo) 68 | print('Dumped %d joints and %d verts predictions to %s' % (len(xyz_pred_list), len(verts_pred_list), pred_out_path)) 69 | 70 | 71 | def pred_template(img, aux_info): 72 | """ Predict joints and vertices from a given sample. 73 | img: (640, 480, 3) RGB image. 74 | aux_info: dictionary containing hand bounding box, camera matrix and root joint 3D location 75 | """ 76 | # TODO: Put your algorithm here, which computes (metric) 3D joint coordinates and 3D vertex positions 77 | xyz = np.zeros((21, 3)) # 3D coordinates of the 21 joints 78 | verts = np.zeros((778, 3)) # 3D coordinates of the shape vertices 79 | return xyz, verts 80 | 81 | 82 | if __name__ == '__main__': 83 | parser = argparse.ArgumentParser(description='Show some samples from the dataset.') 84 | parser.add_argument('base_path', type=str, 85 | help='Path to where the HO3D dataset is located.') 86 | parser.add_argument('--out', type=str, default='pred.json', 87 | help='File to save the predictions.') 88 | parser.add_argument('--version', type=str, choices=['v2', 'v3'], 89 | help='version number') 90 | args = parser.parse_args() 91 | 92 | # call with a predictor function 93 | main( 94 | args.base_path, 95 | args.out, 96 | pred_func=pred_template, 97 | set_name='evaluation', 98 | version=args.version 99 | ) 100 | 101 | -------------------------------------------------------------------------------- /setup_mano.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | import argparse 3 | import os 4 | from tempfile import mkstemp 5 | from shutil import move 6 | from os import fdopen, remove 7 | import hashlib 8 | import shutil 9 | import pickle 10 | 11 | def replace(file_path, line_ids, new_lines): 12 | """ Replace a line in a given file with a new given line. """ 13 | line_ids = [i - 1 for i in line_ids] 14 | # Create temp file 15 | fh, abs_path = mkstemp() 16 | with fdopen(fh,'w') as new_file: 17 | with open(file_path) as old_file: 18 | for i, line in enumerate(old_file): 19 | if i in line_ids: 20 | new_file.write(new_lines[line_ids.index(i)] + '\n') 21 | else: 22 | new_file.write(line) 23 | 24 | #Remove original file 25 | remove(file_path) 26 | 27 | #Move new file 28 | move(abs_path, file_path) 29 | 30 | def md5(fname): 31 | hash_md5 = hashlib.md5() 32 | with open(fname, "rb") as f: 33 | for chunk in iter(lambda: f.read(4096), b""): 34 | hash_md5.update(chunk) 35 | return hash_md5.hexdigest() 36 | 37 | def _patch_mano_loader(): 38 | file = 'mano/webuser/smpl_handpca_wrapper_HAND_only.py' 39 | 40 | replace(file, *zip(* 41 | [ 42 | (23, ' import pickle'), 43 | (26, ' from mano.webuser.posemapper import posemap'), 44 | (66, ' from mano.webuser.verts import verts_core'), 45 | (92, ' smpl_data[\'fullpose\'] = ch.array(smpl_data[\'fullpose\'].r)'), 46 | (74, ' smpl_data = pickle.load(open(fname_or_dict, \'rb\'))') 47 | ] 48 | )) 49 | 50 | file = 'mano/webuser/verts.py' 51 | 52 | replace(file, *zip(* 53 | [ 54 | (29, 'import mano.webuser.lbs as lbs'), 55 | (30, 'from mano.webuser.posemapper import posemap'), 56 | ] 57 | )) 58 | 59 | file = 'mano/webuser/lbs.py' 60 | 61 | replace(file, *zip(* 62 | [ 63 | (27, 'from mano.webuser.posemapper import posemap'), 64 | (38, ' from mano.webuser.posemapper import Rodrigues'), 65 | (77, ' v = v[:,:3]'), 66 | (78, ' for tp in [744, 333, 444, 555, 672]: # THUMB, INDEX, MIDDLE, RING, PINKY'), 67 | (79, ' A_global.append(xp.vstack((xp.hstack((np.zeros((3, 3)), v[tp, :3].reshape((3, 1)))), xp.array([[0.0, 0.0, 0.0, 1.0]]))))') 68 | ] 69 | )) 70 | 71 | 72 | def patch_files(): 73 | _patch_mano_loader() 74 | 75 | 76 | if __name__ == '__main__': 77 | parser = argparse.ArgumentParser(description='Import needed files from MANO repository.') 78 | parser.add_argument('mano_path', type=str, help='Path to where the original MANO repository is located.') 79 | parser.add_argument('--clear', action='store_true', help='Util call for me to remove mano files before committing.') 80 | args = parser.parse_args() 81 | 82 | # files we attempt to copy from the original mano repository 83 | files_needed = [ 84 | 'models/MANO_RIGHT.pkl', 85 | 'models/MANO_LEFT.pkl', 86 | 'webuser/verts.py', 87 | 'webuser/posemapper.py', 88 | 'webuser/lbs.py', 89 | 'webuser/smpl_handpca_wrapper_HAND_only.py', 90 | '__init__.py', 91 | 'webuser/__init__.py' 92 | ] 93 | 94 | 95 | if args.clear: 96 | if os.path.exists('./mano'): 97 | shutil.rmtree('./mano') 98 | print('Repository cleaned.') 99 | exit() 100 | 101 | # check input files 102 | files_copy_to = [os.path.join('mano', f) for f in files_needed] 103 | files_needed = [os.path.join(args.mano_path, f) for f in files_needed] 104 | assert all([os.path.exists(f) for f in files_needed]), 'Could not find one of the needed MANO files in the directory you provided.' 105 | 106 | # coursely check content 107 | hash_ground_truth = [ 108 | 'fd5a9d35f914987cf1cc04ffe338caa1', 109 | '11aeaca8631aa20db964d4eba491e885', 110 | '998c30fd83c473da6178aa2cb23b6a5d', 111 | 'c5e9eacc535ec7d03060e0c8d6f80f45', 112 | 'd11c767d5db8d4a55b4ece1c46a4e4ac', 113 | '5afc7a3eb1a6ce0c2dac1483338a5f58', 114 | 'fd4025c7ee315dc1ec29ac94e4105825' 115 | ] 116 | # print([md5(f) for f, gt in zip(files_needed, hash_ground_truth)]) 117 | assert all([md5(f) == gt for f, gt in zip(files_needed, hash_ground_truth)]), 'Hash sum of provided files differs from what was expected.' 118 | 119 | # copy files 120 | if not os.path.exists('mano'): 121 | os.mkdir('mano') 122 | if not os.path.exists('mano/models'): 123 | os.mkdir('mano/models') 124 | if not os.path.exists('mano/webuser'): 125 | os.mkdir('mano/webuser') 126 | for a, b in zip(files_needed, files_copy_to): 127 | shutil.copy2(a, b) 128 | if 'MANO_LEFT' in b: 129 | smpl_data = pickle.load(open(b, 'rb')) 130 | smpl_data['shapedirs'] *= -1 131 | pickle.dump(smpl_data, open(b, 'wb'), protocol=2) 132 | 133 | 134 | # some files need to be modified 135 | patch_files() 136 | 137 | -------------------------------------------------------------------------------- /teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shreyashampali/ho3d/c1c8923f2f90fc2ec7c502f491b3851e2c58c388/teaser.png -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shreyashampali/ho3d/c1c8923f2f90fc2ec7c502f491b3851e2c58c388/utils/__init__.py -------------------------------------------------------------------------------- /utils/eval_util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class EvalUtil: 5 | """ Util class for evaluation networks. 6 | """ 7 | def __init__(self, num_kp=21): 8 | # init empty data storage 9 | self.data = list() 10 | self.num_kp = num_kp 11 | for _ in range(num_kp): 12 | self.data.append(list()) 13 | 14 | def feed(self, keypoint_gt, keypoint_vis, keypoint_pred, skip_check=False): 15 | """ Used to feed data to the class. Stores the euclidean distance between gt and pred, when it is visible. """ 16 | if not skip_check: 17 | keypoint_gt = np.squeeze(keypoint_gt) 18 | keypoint_pred = np.squeeze(keypoint_pred) 19 | keypoint_vis = np.squeeze(keypoint_vis).astype('bool') 20 | 21 | assert len(keypoint_gt.shape) == 2 22 | assert len(keypoint_pred.shape) == 2 23 | assert len(keypoint_vis.shape) == 1 24 | 25 | # calc euclidean distance 26 | diff = keypoint_gt - keypoint_pred 27 | euclidean_dist = np.sqrt(np.sum(np.square(diff), axis=1)) 28 | 29 | num_kp = keypoint_gt.shape[0] 30 | for i in range(num_kp): 31 | if keypoint_vis[i]: 32 | self.data[i].append(euclidean_dist[i]) 33 | 34 | def _get_pck(self, kp_id, threshold): 35 | """ Returns pck for one keypoint for the given threshold. """ 36 | if len(self.data[kp_id]) == 0: 37 | return None 38 | 39 | data = np.array(self.data[kp_id]) 40 | pck = np.mean((data <= threshold).astype('float')) 41 | return pck 42 | 43 | def _get_epe(self, kp_id): 44 | """ Returns end point error for one keypoint. """ 45 | if len(self.data[kp_id]) == 0: 46 | return None, None 47 | 48 | data = np.array(self.data[kp_id]) 49 | epe_mean = np.mean(data) 50 | epe_median = np.median(data) 51 | return epe_mean, epe_median 52 | 53 | def get_measures(self, val_min, val_max, steps): 54 | """ Outputs the average mean and median error as well as the pck score. """ 55 | thresholds = np.linspace(val_min, val_max, steps) 56 | thresholds = np.array(thresholds) 57 | norm_factor = np.trapz(np.ones_like(thresholds), thresholds) 58 | 59 | # init mean measures 60 | epe_mean_all = list() 61 | epe_median_all = list() 62 | auc_all = list() 63 | pck_curve_all = list() 64 | 65 | # Create one plot for each part 66 | for part_id in range(self.num_kp): 67 | # mean/median error 68 | mean, median = self._get_epe(part_id) 69 | 70 | if mean is None: 71 | # there was no valid measurement for this keypoint 72 | continue 73 | 74 | epe_mean_all.append(mean) 75 | epe_median_all.append(median) 76 | 77 | # pck/auc 78 | pck_curve = list() 79 | for t in thresholds: 80 | pck = self._get_pck(part_id, t) 81 | pck_curve.append(pck) 82 | 83 | pck_curve = np.array(pck_curve) 84 | pck_curve_all.append(pck_curve) 85 | auc = np.trapz(pck_curve, thresholds) 86 | auc /= norm_factor 87 | auc_all.append(auc) 88 | 89 | epe_mean_all = np.mean(np.array(epe_mean_all)) 90 | epe_median_all = np.mean(np.array(epe_median_all)) 91 | auc_all = np.mean(np.array(auc_all)) 92 | pck_curve_all = np.mean(np.array(pck_curve_all), 0) # mean only over keypoints 93 | 94 | return epe_mean_all, epe_median_all, auc_all, pck_curve_all, thresholds 95 | -------------------------------------------------------------------------------- /utils/fh_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | import numpy as np 3 | import json 4 | import os 5 | import time 6 | import skimage.io as io 7 | import pickle 8 | import math 9 | import sys 10 | import matplotlib.pyplot as plt 11 | 12 | """ General util functions. """ 13 | def _assert_exist(p): 14 | msg = 'File does not exists: %s' % p 15 | assert os.path.exists(p), msg 16 | 17 | 18 | def json_load(p): 19 | _assert_exist(p) 20 | with open(p, 'r') as fi: 21 | d = json.load(fi) 22 | return d 23 | 24 | def projectPoints(xyz, K): 25 | """ Project 3D coordinates into image space. """ 26 | xyz = np.array(xyz) 27 | K = np.array(K) 28 | uv = np.matmul(K, xyz.T).T 29 | return uv[:, :2] / uv[:, -1:] 30 | 31 | def loadPickleData(fName): 32 | with open(fName, 'rb') as f: 33 | try: 34 | pickData = pickle.load(f, encoding='latin1') 35 | except: 36 | pickData = pickle.load(f) 37 | 38 | return pickData 39 | 40 | def showHandJoints(imgInOrg, gtIn, filename=None): 41 | ''' 42 | Utility function for displaying hand annotations 43 | :param imgIn: image on which annotation is shown 44 | :param gtIn: ground truth annotation 45 | :param filename: dump image name 46 | :return: 47 | ''' 48 | import cv2 49 | 50 | imgIn = np.copy(imgInOrg) 51 | 52 | # Set color for each finger 53 | joint_color_code = [[139, 53, 255], 54 | [0, 56, 255], 55 | [43, 140, 237], 56 | [37, 168, 36], 57 | [147, 147, 0], 58 | [70, 17, 145]] 59 | 60 | limbs = [[0, 1], 61 | [1, 2], 62 | [2, 3], 63 | [3, 4], 64 | [0, 5], 65 | [5, 6], 66 | [6, 7], 67 | [7, 8], 68 | [0, 9], 69 | [9, 10], 70 | [10, 11], 71 | [11, 12], 72 | [0, 13], 73 | [13, 14], 74 | [14, 15], 75 | [15, 16], 76 | [0, 17], 77 | [17, 18], 78 | [18, 19], 79 | [19, 20] 80 | ] 81 | 82 | PYTHON_VERSION = sys.version_info[0] 83 | 84 | gtIn = np.round(gtIn).astype(np.int) 85 | 86 | for joint_num in range(gtIn.shape[0]): 87 | 88 | color_code_num = (joint_num // 4) 89 | if joint_num in [0, 4, 8, 12, 16]: 90 | if PYTHON_VERSION == 3: 91 | joint_color = list(map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) 92 | else: 93 | joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) 94 | 95 | cv2.circle(imgIn, center=(gtIn[joint_num][0], gtIn[joint_num][1]), radius=3, color=joint_color, thickness=-1) 96 | else: 97 | if PYTHON_VERSION == 3: 98 | joint_color = list(map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) 99 | else: 100 | joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) 101 | 102 | cv2.circle(imgIn, center=(gtIn[joint_num][0], gtIn[joint_num][1]), radius=3, color=joint_color, thickness=-1) 103 | 104 | for limb_num in range(len(limbs)): 105 | 106 | x1 = gtIn[limbs[limb_num][0], 1] 107 | y1 = gtIn[limbs[limb_num][0], 0] 108 | x2 = gtIn[limbs[limb_num][1], 1] 109 | y2 = gtIn[limbs[limb_num][1], 0] 110 | length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 111 | if length < 150 and length > 5: 112 | deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) 113 | polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), 114 | (int(length / 2), 3), 115 | int(deg), 116 | 0, 360, 1) 117 | color_code_num = limb_num // 4 118 | if PYTHON_VERSION == 3: 119 | limb_color = list(map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) 120 | else: 121 | limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) 122 | 123 | cv2.fillConvexPoly(imgIn, polygon, color=limb_color) 124 | 125 | 126 | if filename is not None: 127 | cv2.imwrite(filename, imgIn) 128 | 129 | return imgIn 130 | 131 | def showObjJoints(imgIn, gtIn, estIn=None, filename=None, upscale=1, lineThickness=3): 132 | ''' 133 | Utility function for displaying object annotations 134 | :param imgIn: image on which annotation is shown 135 | :param gtIn: ground truth annotation 136 | :param estIn: estimated keypoints 137 | :param filename: dump image name 138 | :param upscale: scale factor 139 | :param lineThickness: 140 | :return: 141 | ''' 142 | import cv2 143 | jointConns = [[0, 1, 3, 2, 0], [4, 5, 7, 6, 4], [0, 4], [1, 5], [2, 6], [3,7]] 144 | jointColsGt = (255,255,0) 145 | newCol = (jointColsGt[0] + jointColsGt[1] + jointColsGt[2]) / 3 146 | jointColsEst = (newCol, newCol, newCol) 147 | 148 | # draws lines connected using jointConns 149 | img = np.zeros((imgIn.shape[0], imgIn.shape[1], imgIn.shape[2]), dtype=np.uint8) 150 | img[:, :, :] = (imgIn).astype(np.uint8) 151 | 152 | img = cv2.resize(img, (upscale * imgIn.shape[1], upscale * imgIn.shape[0]), interpolation=cv2.INTER_CUBIC) 153 | if gtIn is not None: 154 | gt = gtIn.copy() * upscale 155 | if estIn is not None: 156 | est = estIn.copy() * upscale 157 | 158 | for i in range(len(jointConns)): 159 | for j in range(len(jointConns[i]) - 1): 160 | jntC = jointConns[i][j] 161 | jntN = jointConns[i][j+1] 162 | if gtIn is not None: 163 | cv2.line(img, (int(gt[jntC,0]), int(gt[jntC,1])), (int(gt[jntN,0]), int(gt[jntN,1])), jointColsGt, lineThickness) 164 | if estIn is not None: 165 | cv2.line(img, (int(est[jntC,0]), int(est[jntC,1])), (int(est[jntN,0]), int(est[jntN,1])), jointColsEst, lineThickness) 166 | 167 | if filename is not None: 168 | cv2.imwrite(filename, img) 169 | 170 | return img 171 | 172 | def cam_equal_aspect_3d(ax, verts, flip_x=False): 173 | """ 174 | Centers view on cuboid containing hand and flips y and z axis 175 | and fixes azimuth 176 | """ 177 | extents = np.stack([verts.min(0), verts.max(0)], axis=1) 178 | sz = extents[:, 1] - extents[:, 0] 179 | centers = np.mean(extents, axis=1) 180 | maxsize = max(abs(sz)) 181 | r = maxsize / 2 182 | if flip_x: 183 | ax.set_xlim(centers[0] + r, centers[0] - r) 184 | else: 185 | ax.set_xlim(centers[0] - r, centers[0] + r) 186 | # Invert y and z axis 187 | ax.set_ylim(centers[1] + r, centers[1] - r) 188 | ax.set_zlim(centers[2] + r, centers[2] - r) 189 | 190 | def plot3dVisualize(ax, m, flip_x=False, c="b", alpha=0.1, camPose=np.eye(4, dtype=np.float32), isOpenGLCoords=False): 191 | ''' 192 | Create 3D visualization 193 | :param ax: matplotlib axis 194 | :param m: mesh 195 | :param flip_x: flix x axis? 196 | :param c: mesh color 197 | :param alpha: transperency 198 | :param camPose: camera pose 199 | :param isOpenGLCoords: is mesh in openGL coordinate system? 200 | :return: 201 | ''' 202 | from mpl_toolkits.mplot3d.art3d import Poly3DCollection 203 | if hasattr(m, 'r'): 204 | verts = np.copy(m.r)*1000 205 | elif hasattr(m, 'v'): 206 | verts = np.copy(m.v) * 1000 207 | else: 208 | raise Exception('Unknown Mesh format') 209 | vertsHomo = np.concatenate([verts, np.ones((verts.shape[0],1), dtype=np.float32)], axis=1) 210 | verts = vertsHomo.dot(camPose.T)[:,:3] 211 | 212 | coordChangeMat = np.array([[1., 0., 0.], [0, -1., 0.], [0., 0., -1.]], dtype=np.float32) 213 | if isOpenGLCoords: 214 | verts = verts.dot(coordChangeMat.T) 215 | 216 | faces = np.copy(m.f) 217 | ax.view_init(elev=90, azim=-90) 218 | mesh = Poly3DCollection(verts[faces], alpha=alpha) 219 | if c == "b": 220 | face_color = (141 / 255, 184 / 255, 226 / 255) 221 | face_color = np.tile(np.array([[0., 0., 1., 1.]]), [verts.shape[0], 1]) 222 | edge_color = (0 / 255, 0 / 255, 112 / 255) 223 | elif c == "r": 224 | face_color = (226 / 255, 141 / 255, 141 / 255) 225 | face_color = np.tile(np.array([[1., 0., 0., 1.]]), [verts.shape[0], 1]) 226 | edge_color = (112 / 255, 0 / 255, 0 / 255) 227 | elif c == "viridis": 228 | face_color = plt.cm.viridis(np.linspace(0, 1, faces.shape[0])) 229 | edge_color = None 230 | edge_color = (0 / 255, 0 / 255, 112 / 255) 231 | elif c == "plasma": 232 | face_color = plt.cm.plasma(np.linspace(0, 1, faces.shape[0])) 233 | edge_color = None 234 | # edge_color = (0 / 255, 0 / 255, 112 / 255) 235 | else: 236 | face_color = c 237 | edge_color = c 238 | 239 | mesh.set_edgecolor(edge_color) 240 | mesh.set_facecolor(face_color) 241 | ax.add_collection3d(mesh) 242 | cam_equal_aspect_3d(ax, verts, flip_x=flip_x) 243 | # plt.tight_layout() 244 | 245 | class Minimal(object): 246 | def __init__(self, **kwargs): 247 | self.__dict__ = kwargs 248 | 249 | def open3dVisualize(mList, colorList): 250 | import open3d 251 | o3dMeshList = [] 252 | for i, m in enumerate(mList): 253 | mesh = open3d.geometry.TriangleMesh() 254 | numVert = 0 255 | if hasattr(m, 'r'): 256 | mesh.vertices = open3d.utility.Vector3dVector(np.copy(m.r)) 257 | numVert = m.r.shape[0] 258 | elif hasattr(m, 'v'): 259 | mesh.vertices = open3d.utility.Vector3dVector(np.copy(m.v)) 260 | numVert = m.v.shape[0] 261 | else: 262 | raise Exception('Unknown Mesh format') 263 | mesh.triangles = open3d.utility.Vector3iVector(np.copy(m.f)) 264 | if colorList[i] == 'r': 265 | mesh.vertex_colors = open3d.utility.Vector3dVector(np.tile(np.array([[1., 0., 0.]]), [numVert, 1])) 266 | elif colorList[i] == 'b': 267 | mesh.vertex_colors = open3d.utility.Vector3dVector(np.tile(np.array([[0., 0., 1.]]), [numVert, 1])) 268 | else: 269 | raise Exception('Unknown mesh color') 270 | 271 | o3dMeshList.append(mesh) 272 | open3d.visualization.draw_geometries(o3dMeshList) 273 | 274 | def read_obj(filename): 275 | """ Reads the Obj file. Function reused from Matthew Loper's OpenDR package""" 276 | 277 | lines = open(filename).read().split('\n') 278 | 279 | d = {'v': [], 'vn': [], 'f': [], 'vt': [], 'ft': [], 'fn': []} 280 | 281 | for line in lines: 282 | line = line.split() 283 | if len(line) < 2: 284 | continue 285 | 286 | key = line[0] 287 | values = line[1:] 288 | 289 | if key == 'v': 290 | d['v'].append([np.array([float(v) for v in values[:3]])]) 291 | elif key == 'f': 292 | spl = [l.split('/') for l in values] 293 | d['f'].append([np.array([int(l[0])-1 for l in spl[:3]], dtype=np.uint32)]) 294 | if len(spl[0]) > 1 and spl[1] and 'ft' in d: 295 | d['ft'].append([np.array([int(l[1])-1 for l in spl[:3]])]) 296 | if len(spl[0]) > 2 and spl[2] and 'fn' in d: 297 | d['fn'].append([np.array([int(l[2])-1 for l in spl[:3]])]) 298 | 299 | # TOO: redirect to actual vert normals? 300 | #if len(line[0]) > 2 and line[0][2]: 301 | # d['fn'].append([np.concatenate([l[2] for l in spl[:3]])]) 302 | elif key == 'vn': 303 | d['vn'].append([np.array([float(v) for v in values])]) 304 | elif key == 'vt': 305 | d['vt'].append([np.array([float(v) for v in values])]) 306 | 307 | 308 | for k, v in d.items(): 309 | if k in ['v','vn','f','vt','ft', 'fn']: 310 | if v: 311 | d[k] = np.vstack(v) 312 | else: 313 | del d[k] 314 | else: 315 | d[k] = v 316 | 317 | result = Minimal(**d) 318 | 319 | return result 320 | 321 | """ Dataset related functions. """ 322 | def db_size(set_name, version='v2'): 323 | """ Hardcoded size of the datasets. """ 324 | if set_name == 'training': 325 | if version == 'v2': 326 | return 66034 # number of unique samples (they exists in multiple 'versions') 327 | elif version == 'v3': 328 | return 78297 329 | else: 330 | raise NotImplementedError 331 | elif set_name == 'evaluation': 332 | if version == 'v2': 333 | return 11524 334 | elif version == 'v3': 335 | return 20137 336 | else: 337 | raise NotImplementedError 338 | else: 339 | assert 0, 'Invalid choice.' 340 | 341 | 342 | def load_db_annotation(base_path, set_name=None): 343 | if set_name is None: 344 | # only training set annotations are released so this is a valid default choice 345 | set_name = 'training' 346 | 347 | print('Loading FreiHAND dataset index ...') 348 | t = time.time() 349 | 350 | # assumed paths to data containers 351 | k_path = os.path.join(base_path, '%s_K.json' % set_name) 352 | mano_path = os.path.join(base_path, '%s_mano.json' % set_name) 353 | xyz_path = os.path.join(base_path, '%s_xyz.json' % set_name) 354 | 355 | # load if exist 356 | K_list = json_load(k_path) 357 | mano_list = json_load(mano_path) 358 | xyz_list = json_load(xyz_path) 359 | 360 | # should have all the same length 361 | assert len(K_list) == len(mano_list), 'Size mismatch.' 362 | assert len(K_list) == len(xyz_list), 'Size mismatch.' 363 | 364 | print('Loading of %d samples done in %.2f seconds' % (len(K_list), time.time()-t)) 365 | return zip(K_list, mano_list, xyz_list) 366 | 367 | 368 | class sample_version: 369 | gs = 'gs' # green screen 370 | hom = 'hom' # homogenized 371 | sample = 'sample' # auto colorization with sample points 372 | auto = 'auto' # auto colorization without sample points: automatic color hallucination 373 | 374 | db_size = db_size('training') 375 | 376 | @classmethod 377 | def valid_options(cls): 378 | return [cls.gs, cls.hom, cls.sample, cls.auto] 379 | 380 | 381 | @classmethod 382 | def check_valid(cls, version): 383 | msg = 'Invalid choice: "%s" (must be in %s)' % (version, cls.valid_options()) 384 | assert version in cls.valid_options(), msg 385 | 386 | @classmethod 387 | def map_id(cls, id, version): 388 | cls.check_valid(version) 389 | return id + cls.db_size*cls.valid_options().index(version) 390 | 391 | 392 | def read_img(idx, base_path, set_name, version=None): 393 | if version is None: 394 | version = sample_version.gs 395 | 396 | if set_name == 'evaluation': 397 | assert version == sample_version.gs, 'This the only valid choice for samples from the evaluation split.' 398 | 399 | img_rgb_path = os.path.join(base_path, set_name, 'rgb', 400 | '%08d.jpg' % sample_version.map_id(idx, version)) 401 | _assert_exist(img_rgb_path) 402 | return io.imread(img_rgb_path) 403 | 404 | 405 | def read_msk(idx, base_path): 406 | mask_path = os.path.join(base_path, 'training', 'mask', 407 | '%08d.jpg' % idx) 408 | _assert_exist(mask_path) 409 | return io.imread(mask_path) 410 | -------------------------------------------------------------------------------- /utils/hand_r.txt: -------------------------------------------------------------------------------- 1 | -9.508606023184423828e-02 1.009099456008522716e-02 9.954179087088872446e-01 -1.005017649251837897e+00 2 | 9.952401998543348727e-01 -2.047684506877687355e-02 9.527666771005924296e-02 -1.137066864157082885e-01 3 | 2.134445463088301825e-02 9.997394013665850121e-01 -8.095900919140365873e-03 2.379387091167539658e-01 4 | 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.000000000000000000e+00 5 | -------------------------------------------------------------------------------- /utils/vis_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | import numpy as np 3 | import json 4 | import os 5 | import time 6 | import skimage.io as io 7 | import pickle 8 | import math 9 | import sys 10 | import matplotlib.pyplot as plt 11 | import cv2 12 | 13 | """ General util functions. """ 14 | def _assert_exist(p): 15 | msg = 'File does not exists: %s' % p 16 | assert os.path.exists(p), msg 17 | 18 | 19 | def json_load(p): 20 | _assert_exist(p) 21 | with open(p, 'r') as fi: 22 | d = json.load(fi) 23 | return d 24 | 25 | def projectPoints(xyz, K): 26 | """ Project 3D coordinates into image space. """ 27 | xyz = np.array(xyz) 28 | K = np.array(K) 29 | uv = np.matmul(K, xyz.T).T 30 | return uv[:, :2] / uv[:, -1:] 31 | 32 | def show2DBoundingBox(imgInOrg, bb): 33 | """ Show bounding box on the image""" 34 | imgIn = np.copy(imgInOrg) 35 | imgIn = cv2.rectangle(imgIn, (int(bb[0]), int(bb[1])), 36 | (int(bb[2]), int(bb[3])), (0, 0, 255), thickness=3) 37 | return imgIn 38 | 39 | def showHandJoints(imgInOrg, gtIn, filename=None): 40 | ''' 41 | Utility function for displaying hand annotations 42 | :param imgIn: image on which annotation is shown 43 | :param gtIn: ground truth annotation 44 | :param filename: dump image name 45 | :return: 46 | ''' 47 | import cv2 48 | 49 | imgIn = np.copy(imgInOrg) 50 | 51 | # Set color for each finger 52 | joint_color_code = [[139, 53, 255], 53 | [0, 56, 255], 54 | [43, 140, 237], 55 | [37, 168, 36], 56 | [147, 147, 0], 57 | [70, 17, 145]] 58 | 59 | limbs = [[0, 1], 60 | [1, 2], 61 | [2, 3], 62 | [3, 4], 63 | [0, 5], 64 | [5, 6], 65 | [6, 7], 66 | [7, 8], 67 | [0, 9], 68 | [9, 10], 69 | [10, 11], 70 | [11, 12], 71 | [0, 13], 72 | [13, 14], 73 | [14, 15], 74 | [15, 16], 75 | [0, 17], 76 | [17, 18], 77 | [18, 19], 78 | [19, 20] 79 | ] 80 | 81 | PYTHON_VERSION = sys.version_info[0] 82 | 83 | gtIn = np.round(gtIn).astype(np.int) 84 | 85 | if gtIn.shape[0]==1: 86 | imgIn = cv2.circle(imgIn, center=(gtIn[0][0], gtIn[0][1]), radius=3, color=joint_color_code[0], 87 | thickness=-1) 88 | else: 89 | 90 | for joint_num in range(gtIn.shape[0]): 91 | 92 | color_code_num = (joint_num // 4) 93 | if joint_num in [0, 4, 8, 12, 16]: 94 | if PYTHON_VERSION == 3: 95 | joint_color = list(map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) 96 | else: 97 | joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) 98 | 99 | cv2.circle(imgIn, center=(gtIn[joint_num][0], gtIn[joint_num][1]), radius=3, color=joint_color, thickness=-1) 100 | else: 101 | if PYTHON_VERSION == 3: 102 | joint_color = list(map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) 103 | else: 104 | joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) 105 | 106 | cv2.circle(imgIn, center=(gtIn[joint_num][0], gtIn[joint_num][1]), radius=3, color=joint_color, thickness=-1) 107 | 108 | for limb_num in range(len(limbs)): 109 | 110 | x1 = gtIn[limbs[limb_num][0], 1] 111 | y1 = gtIn[limbs[limb_num][0], 0] 112 | x2 = gtIn[limbs[limb_num][1], 1] 113 | y2 = gtIn[limbs[limb_num][1], 0] 114 | length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 115 | if length < 150 and length > 5: 116 | deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) 117 | polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), 118 | (int(length / 2), 3), 119 | int(deg), 120 | 0, 360, 1) 121 | color_code_num = limb_num // 4 122 | if PYTHON_VERSION == 3: 123 | limb_color = list(map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) 124 | else: 125 | limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) 126 | 127 | cv2.fillConvexPoly(imgIn, polygon, color=limb_color) 128 | 129 | 130 | if filename is not None: 131 | cv2.imwrite(filename, imgIn) 132 | 133 | return imgIn 134 | 135 | def showObjJoints(imgIn, gtIn, estIn=None, filename=None, upscale=1, lineThickness=3): 136 | ''' 137 | Utility function for displaying object annotations 138 | :param imgIn: image on which annotation is shown 139 | :param gtIn: ground truth annotation 140 | :param estIn: estimated keypoints 141 | :param filename: dump image name 142 | :param upscale: scale factor 143 | :param lineThickness: 144 | :return: 145 | ''' 146 | import cv2 147 | jointConns = [[0, 1, 3, 2, 0], [4, 5, 7, 6, 4], [0, 4], [1, 5], [2, 6], [3,7]] 148 | jointColsGt = (255,255,0) 149 | newCol = (jointColsGt[0] + jointColsGt[1] + jointColsGt[2]) / 3 150 | jointColsEst = (newCol, newCol, newCol) 151 | 152 | # draws lines connected using jointConns 153 | img = np.zeros((imgIn.shape[0], imgIn.shape[1], imgIn.shape[2]), dtype=np.uint8) 154 | img[:, :, :] = (imgIn).astype(np.uint8) 155 | 156 | img = cv2.resize(img, (upscale * imgIn.shape[1], upscale * imgIn.shape[0]), interpolation=cv2.INTER_CUBIC) 157 | if gtIn is not None: 158 | gt = gtIn.copy() * upscale 159 | if estIn is not None: 160 | est = estIn.copy() * upscale 161 | 162 | for i in range(len(jointConns)): 163 | for j in range(len(jointConns[i]) - 1): 164 | jntC = jointConns[i][j] 165 | jntN = jointConns[i][j+1] 166 | if gtIn is not None: 167 | cv2.line(img, (int(gt[jntC,0]), int(gt[jntC,1])), (int(gt[jntN,0]), int(gt[jntN,1])), jointColsGt, lineThickness) 168 | if estIn is not None: 169 | cv2.line(img, (int(est[jntC,0]), int(est[jntC,1])), (int(est[jntN,0]), int(est[jntN,1])), jointColsEst, lineThickness) 170 | 171 | if filename is not None: 172 | cv2.imwrite(filename, img) 173 | 174 | return img 175 | 176 | def cam_equal_aspect_3d(ax, verts, flip_x=False): 177 | """ 178 | Centers view on cuboid containing hand and flips y and z axis 179 | and fixes azimuth 180 | """ 181 | extents = np.stack([verts.min(0), verts.max(0)], axis=1) 182 | sz = extents[:, 1] - extents[:, 0] 183 | centers = np.mean(extents, axis=1) 184 | maxsize = max(abs(sz)) 185 | r = maxsize / 2 186 | if flip_x: 187 | ax.set_xlim(centers[0] + r, centers[0] - r) 188 | else: 189 | ax.set_xlim(centers[0] - r, centers[0] + r) 190 | # Invert y and z axis 191 | ax.set_ylim(centers[1] + r, centers[1] - r) 192 | ax.set_zlim(centers[2] + r, centers[2] - r) 193 | 194 | def plot3dVisualize(ax, m, flip_x=False, c="b", alpha=0.1, camPose=np.eye(4, dtype=np.float32), isOpenGLCoords=False): 195 | ''' 196 | Create 3D visualization 197 | :param ax: matplotlib axis 198 | :param m: mesh 199 | :param flip_x: flix x axis? 200 | :param c: mesh color 201 | :param alpha: transperency 202 | :param camPose: camera pose 203 | :param isOpenGLCoords: is mesh in openGL coordinate system? 204 | :return: 205 | ''' 206 | from mpl_toolkits.mplot3d.art3d import Poly3DCollection 207 | if hasattr(m, 'r'): 208 | verts = np.copy(m.r)*1000 209 | elif hasattr(m, 'v'): 210 | verts = np.copy(m.v) * 1000 211 | else: 212 | raise Exception('Unknown Mesh format') 213 | vertsHomo = np.concatenate([verts, np.ones((verts.shape[0],1), dtype=np.float32)], axis=1) 214 | verts = vertsHomo.dot(camPose.T)[:,:3] 215 | 216 | coordChangeMat = np.array([[1., 0., 0.], [0, -1., 0.], [0., 0., -1.]], dtype=np.float32) 217 | if isOpenGLCoords: 218 | verts = verts.dot(coordChangeMat.T) 219 | 220 | faces = np.copy(m.f) 221 | ax.view_init(elev=90, azim=-90) 222 | mesh = Poly3DCollection(verts[faces], alpha=alpha) 223 | if c == "b": 224 | face_color = (141 / 255, 184 / 255, 226 / 255) 225 | face_color = np.tile(np.array([[0., 0., 1., 1.]]), [verts.shape[0], 1]) 226 | edge_color = (0 / 255, 0 / 255, 112 / 255) 227 | elif c == "r": 228 | face_color = (226 / 255, 141 / 255, 141 / 255) 229 | face_color = np.tile(np.array([[1., 0., 0., 1.]]), [verts.shape[0], 1]) 230 | edge_color = (112 / 255, 0 / 255, 0 / 255) 231 | elif c == "viridis": 232 | face_color = plt.cm.viridis(np.linspace(0, 1, faces.shape[0])) 233 | edge_color = None 234 | edge_color = (0 / 255, 0 / 255, 112 / 255) 235 | elif c == "plasma": 236 | face_color = plt.cm.plasma(np.linspace(0, 1, faces.shape[0])) 237 | edge_color = None 238 | # edge_color = (0 / 255, 0 / 255, 112 / 255) 239 | else: 240 | face_color = c 241 | edge_color = c 242 | 243 | mesh.set_edgecolor(edge_color) 244 | mesh.set_facecolor(face_color) 245 | ax.add_collection3d(mesh) 246 | cam_equal_aspect_3d(ax, verts, flip_x=flip_x) 247 | # plt.tight_layout() 248 | 249 | class Minimal(object): 250 | def __init__(self, **kwargs): 251 | self.__dict__ = kwargs 252 | 253 | 254 | 255 | class Open3DWin(): 256 | def __init__(self): 257 | import open3d 258 | self.vis = open3d.visualization.Visualizer() 259 | self.vis.create_window(window_name='Open3D', width=640, height=480, left=0, top=0, 260 | visible=True) # use visible=True to visualize the point cloud 261 | # vis.get_render_option().light_on = False 262 | self.vis.get_render_option().mesh_show_back_face = True 263 | 264 | def capture_view(self, mesh, view_mat_path=None,intrinsics=None): 265 | 266 | if not isinstance(view_mat_path, np.ndarray) and view_mat_path is not None: 267 | assert os.path.exists(view_mat_path) 268 | view_mat = np.loadtxt(view_mat_path) 269 | else: 270 | view_mat = view_mat_path 271 | 272 | camera_param = self.vis.get_view_control().convert_to_pinhole_camera_parameters() 273 | cx = camera_param.intrinsic.intrinsic_matrix[0, 2] 274 | cy = camera_param.intrinsic.intrinsic_matrix[1, 2] 275 | 276 | if intrinsics is not None: 277 | camera_param.intrinsic.set_intrinsics(camera_param.intrinsic.width, camera_param.intrinsic.height, 278 | intrinsics[0, 0], intrinsics[1, 1], cx, cy) 279 | 280 | if view_mat is not None: 281 | camera_param = self.vis.get_view_control().convert_to_pinhole_camera_parameters() 282 | camera_param.extrinsic = view_mat 283 | 284 | ctr = self.vis.get_view_control() 285 | # ctr.set_constant_z_far(20.) 286 | # ctr.set_constant_z_near(-2) 287 | for m in mesh: 288 | self.vis.add_geometry(m) 289 | 290 | ctr.convert_from_pinhole_camera_parameters(camera_param) 291 | 292 | 293 | 294 | # vis.run() 295 | 296 | render = self.vis.capture_screen_float_buffer(do_render=True) 297 | 298 | render = (np.asarray(render)*255).astype(np.uint8) 299 | 300 | for m in mesh: 301 | self.vis.remove_geometry(m) 302 | 303 | return render 304 | 305 | def open3dVisualize(mList, colorList): 306 | import open3d 307 | o3dMeshList = [] 308 | for i, m in enumerate(mList): 309 | mesh = open3d.geometry.TriangleMesh() 310 | numVert = 0 311 | if hasattr(m, 'r'): 312 | mesh.vertices = open3d.utility.Vector3dVector(np.copy(m.r)) 313 | numVert = m.r.shape[0] 314 | elif hasattr(m, 'v'): 315 | mesh.vertices = open3d.utility.Vector3dVector(np.copy(m.v)) 316 | numVert = m.v.shape[0] 317 | else: 318 | raise Exception('Unknown Mesh format') 319 | mesh.triangles = open3d.utility.Vector3iVector(np.copy(m.f)) 320 | if colorList[i] == 'r': 321 | mesh.vertex_colors = open3d.utility.Vector3dVector(np.tile(np.array([[0.6, 0.2, 0.2]]), [numVert, 1])) 322 | elif colorList[i] == 'g': 323 | mesh.vertex_colors = open3d.utility.Vector3dVector(np.tile(np.array([[0.5, 0.5, 0.5]]), [numVert, 1])) 324 | elif isinstance(colorList[i],np.ndarray): 325 | assert colorList[i].shape == np.array(mesh.vertices).shape 326 | mesh.vertex_colors = open3d.utility.Vector3dVector(colorList[i]) 327 | else: 328 | raise Exception('Unknown mesh color') 329 | 330 | o3dMeshList.append(mesh) 331 | open3d.visualization.draw_geometries(o3dMeshList) 332 | 333 | def read_obj(filename): 334 | """ Reads the Obj file. Function reused from Matthew Loper's OpenDR package""" 335 | 336 | lines = open(filename).read().split('\n') 337 | 338 | d = {'v': [], 'vn': [], 'f': [], 'vt': [], 'ft': [], 'fn': []} 339 | 340 | for line in lines: 341 | line = line.split() 342 | if len(line) < 2: 343 | continue 344 | 345 | key = line[0] 346 | values = line[1:] 347 | 348 | if key == 'v': 349 | d['v'].append([np.array([float(v) for v in values[:3]])]) 350 | elif key == 'f': 351 | spl = [l.split('/') for l in values] 352 | d['f'].append([np.array([int(l[0])-1 for l in spl[:3]], dtype=np.uint32)]) 353 | if len(spl[0]) > 1 and spl[1] and 'ft' in d: 354 | d['ft'].append([np.array([int(l[1])-1 for l in spl[:3]])]) 355 | if len(spl[0]) > 2 and spl[2] and 'fn' in d: 356 | d['fn'].append([np.array([int(l[2])-1 for l in spl[:3]])]) 357 | 358 | # TOO: redirect to actual vert normals? 359 | #if len(line[0]) > 2 and line[0][2]: 360 | # d['fn'].append([np.concatenate([l[2] for l in spl[:3]])]) 361 | elif key == 'vn': 362 | d['vn'].append([np.array([float(v) for v in values])]) 363 | elif key == 'vt': 364 | d['vt'].append([np.array([float(v) for v in values])]) 365 | 366 | 367 | for k, v in d.items(): 368 | if k in ['v','vn','f','vt','ft', 'fn']: 369 | if v: 370 | d[k] = np.vstack(v) 371 | else: 372 | del d[k] 373 | else: 374 | d[k] = v 375 | 376 | result = Minimal(**d) 377 | 378 | return result 379 | 380 | 381 | def db_size(set_name, version='v2'): 382 | """ Hardcoded size of the datasets. """ 383 | if set_name == 'training': 384 | if version == 'v2': 385 | return 66034 # number of unique samples (they exists in multiple 'versions') 386 | elif version == 'v3': 387 | return 78297 388 | else: 389 | raise NotImplementedError 390 | elif set_name == 'evaluation': 391 | if version == 'v2': 392 | return 11524 393 | elif version == 'v3': 394 | return 20137 395 | else: 396 | raise NotImplementedError 397 | else: 398 | assert 0, 'Invalid choice.' 399 | 400 | def load_pickle_data(f_name): 401 | """ Loads the pickle data """ 402 | if not os.path.exists(f_name): 403 | raise Exception('Unable to find annotations picle file at %s. Aborting.'%(f_name)) 404 | with open(f_name, 'rb') as f: 405 | try: 406 | pickle_data = pickle.load(f, encoding='latin1') 407 | except: 408 | pickle_data = pickle.load(f) 409 | 410 | return pickle_data 411 | 412 | def project_3D_points(cam_mat, pts3D, is_OpenGL_coords=True): 413 | ''' 414 | Function for projecting 3d points to 2d 415 | :param camMat: camera matrix 416 | :param pts3D: 3D points 417 | :param isOpenGLCoords: If True, hand/object along negative z-axis. If False hand/object along positive z-axis 418 | :return: 419 | ''' 420 | assert pts3D.shape[-1] == 3 421 | assert len(pts3D.shape) == 2 422 | 423 | coord_change_mat = np.array([[1., 0., 0.], [0, -1., 0.], [0., 0., -1.]], dtype=np.float32) 424 | if is_OpenGL_coords: 425 | pts3D = pts3D.dot(coord_change_mat.T) 426 | 427 | proj_pts = pts3D.dot(cam_mat.T) 428 | proj_pts = np.stack([proj_pts[:,0]/proj_pts[:,2], proj_pts[:,1]/proj_pts[:,2]],axis=1) 429 | 430 | assert len(proj_pts.shape) == 2 431 | 432 | return proj_pts 433 | 434 | def read_RGB_img(base_dir, seq_name, file_id, split): 435 | """Read the RGB image in dataset""" 436 | if os.path.exists(os.path.join(base_dir, split, seq_name, 'rgb', file_id + '.png')): 437 | img_filename = os.path.join(base_dir, split, seq_name, 'rgb', file_id + '.png') 438 | else: 439 | img_filename = os.path.join(base_dir, split, seq_name, 'rgb', file_id + '.jpg') 440 | 441 | _assert_exist(img_filename) 442 | 443 | img = cv2.imread(img_filename) 444 | 445 | return img 446 | 447 | def read_seg(base_dir, seq_name, file_id, split): 448 | """Read the RGB image in dataset""" 449 | img_filename = os.path.join(base_dir, split, seq_name, 'segr', file_id + '.png') 450 | 451 | _assert_exist(img_filename) 452 | 453 | img = cv2.imread(img_filename) 454 | 455 | return img 456 | 457 | 458 | def read_depth_img(base_dir, seq_name, file_id, split): 459 | """Read the depth image in dataset and decode it""" 460 | depth_filename = os.path.join(base_dir, split, seq_name, 'depth', file_id + '.png') 461 | 462 | _assert_exist(depth_filename) 463 | 464 | depth_scale = 0.00012498664727900177 465 | depth_img = cv2.imread(depth_filename) 466 | 467 | dpt = depth_img[:, :, 2] + depth_img[:, :, 1] * 256 468 | dpt = dpt * depth_scale 469 | 470 | return dpt 471 | 472 | def read_annotation(base_dir, seq_name, file_id, split): 473 | meta_filename = os.path.join(base_dir, split, seq_name, 'meta', file_id + '.pkl') 474 | 475 | _assert_exist(meta_filename) 476 | 477 | pkl_data = load_pickle_data(meta_filename) 478 | 479 | return pkl_data 480 | 481 | 482 | -------------------------------------------------------------------------------- /vis_H2O3D.py: -------------------------------------------------------------------------------- 1 | """ 2 | Visualize the projections in published HO-3D dataset 3 | """ 4 | from os.path import join 5 | import pip 6 | import argparse 7 | from utils.vis_utils import * 8 | import random 9 | from copy import deepcopy 10 | import open3d 11 | 12 | def install(package): 13 | if hasattr(pip, 'main'): 14 | pip.main(['install', package]) 15 | else: 16 | from pip._internal.main import main as pipmain 17 | pipmain(['install', package]) 18 | 19 | try: 20 | import matplotlib.pyplot as plt 21 | except: 22 | install('matplotlib') 23 | import matplotlib.pyplot as plt 24 | 25 | try: 26 | import chumpy as ch 27 | except: 28 | install('chumpy') 29 | import chumpy as ch 30 | 31 | 32 | try: 33 | import pickle 34 | except: 35 | install('pickle') 36 | import pickle 37 | 38 | import cv2 39 | from mpl_toolkits.mplot3d import Axes3D 40 | 41 | MANO_RIGHT_MODEL_PATH = './mano/models/MANO_RIGHT.pkl' 42 | MANO_LEFT_MODEL_PATH = './mano/models/MANO_LEFT.pkl' 43 | 44 | # mapping of joints from MANO model order to simple order(thumb to pinky finger) 45 | jointsMapManoToSimple = [0, 46 | 13, 14, 15, 16, 47 | 1, 2, 3, 17, 48 | 4, 5, 6, 18, 49 | 10, 11, 12, 19, 50 | 7, 8, 9, 20] 51 | 52 | if not os.path.exists(MANO_RIGHT_MODEL_PATH) or not os.path.exists(MANO_LEFT_MODEL_PATH): 53 | raise Exception('MANO model missing! Please run setup_mano.py to setup mano folder') 54 | else: 55 | from mano.webuser.smpl_handpca_wrapper_HAND_only import load_model 56 | 57 | 58 | def forwardKinematics(fullpose, trans, beta, mano_path): 59 | ''' 60 | MANO parameters --> 3D pts, mesh 61 | :param fullpose: 62 | :param trans: 63 | :param beta: 64 | :return: 3D pts of size (21,3) 65 | ''' 66 | 67 | assert fullpose.shape == (48,) 68 | assert trans.shape == (3,) 69 | assert beta.shape == (10,) 70 | 71 | m = load_model(mano_path, ncomps=6, flat_hand_mean=True) 72 | m.fullpose[:] = fullpose 73 | m.trans[:] = trans 74 | m.betas[:] = beta 75 | 76 | return m.J_transformed.r, m 77 | 78 | 79 | if __name__ == '__main__': 80 | 81 | # parse the input arguments 82 | ap = argparse.ArgumentParser() 83 | ap.add_argument("h2o3d_path", type=str, help="Path to H2O3D dataset") 84 | ap.add_argument("ycbModels_path", type=str, help="Path to ycb models directory") 85 | ap.add_argument("-split", required=False, type=str, 86 | help="split type", choices=['train', 'evaluation'], default='train') 87 | ap.add_argument("-seq", required=False, type=str, 88 | help="sequence name") 89 | ap.add_argument("-id", required=False, type=str, 90 | help="image ID") 91 | ap.add_argument("-visType", required=False, 92 | help="Type of visualization", choices=['open3d', 'matplotlib'], default='matplotlib') 93 | args = vars(ap.parse_args()) 94 | 95 | baseDir = args['h2o3d_path'] 96 | YCBModelsDir = args['ycbModels_path'] 97 | split = args['split'] 98 | 99 | # some checks to decide if visualizing one single image or randomly picked images 100 | if args['seq'] is None: 101 | radomizeSeq = True 102 | args['seq'] = random.choice(os.listdir(join(baseDir, split))) 103 | else: 104 | radomizeSeq = False 105 | 106 | if args['id'] is None: 107 | radomizeID = True 108 | args['id'] = random.choice(os.listdir(join(baseDir, split, args['seq'], 'rgb'))).split('.')[0] 109 | else: 110 | radomizeID = False 111 | 112 | 113 | 114 | while(True): 115 | seqName = args['seq'] 116 | id = args['id'] 117 | if radomizeID or radomizeSeq: 118 | print('Visualizing %s/%s from %s split... (press \'Q\' to visualize another image)' % (seqName, id, split)) 119 | else: 120 | print('Visualizing %s/%s from %s split...'%(seqName, id)) 121 | 122 | # read image, depths maps and annotations 123 | img = read_RGB_img(baseDir, seqName, id, split) 124 | depth = read_depth_img(baseDir, seqName, id, split) 125 | anno = read_annotation(baseDir, seqName, id, split) 126 | seg = read_seg(baseDir, seqName, id, split) 127 | seg = cv2.resize(seg, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST) 128 | 129 | if anno['objRot'] is None: 130 | print('Frame %s in sequence %s does not have annotations'%(args['id'], args['seq'])) 131 | if not radomizeSeq or not radomizeID: 132 | break 133 | else: 134 | args['seq'] = random.choice(os.listdir(join(baseDir, split))) 135 | args['id'] = random.choice(os.listdir(join(baseDir, split, args['seq'], 'rgb'))).split('.')[0] 136 | continue 137 | 138 | # get object 3D corner locations for the current pose 139 | objCorners = anno['objCorners3DRest'] 140 | objCornersTrans = np.matmul(objCorners, cv2.Rodrigues(anno['objRot'])[0].T) + anno['objTrans'] 141 | 142 | # get the hand Mesh from MANO model for the current pose 143 | if split == 'train': 144 | rightHandJoints3D, rightHandMesh = forwardKinematics(anno['rightHandPose'], anno['rightHandTrans'], anno['handBeta'], MANO_RIGHT_MODEL_PATH) 145 | leftHandJoints3D, leftHandMesh = forwardKinematics(anno['leftHandPose'], anno['leftHandTrans'], 146 | anno['handBeta'], MANO_LEFT_MODEL_PATH) 147 | 148 | # project to 2D 149 | if split == 'train': 150 | rightHandKps = project_3D_points(anno['camMat'], rightHandJoints3D, is_OpenGL_coords=True) 151 | leftHandKps = project_3D_points(anno['camMat'], leftHandJoints3D, is_OpenGL_coords=True) 152 | else: 153 | # Only root joint available in evaluation split 154 | rightHandKps = project_3D_points(anno['camMat'], np.expand_dims(anno['rightHandJoints3D'],0), is_OpenGL_coords=True) 155 | leftHandKps = project_3D_points(anno['camMat'], np.expand_dims(anno['leftHandJoints3D'], 0), 156 | is_OpenGL_coords=True) 157 | objKps = project_3D_points(anno['camMat'], objCornersTrans, is_OpenGL_coords=True) 158 | 159 | # Visualize 160 | if args['visType'] == 'open3d': 161 | # open3d visualization 162 | 163 | if not os.path.exists(os.path.join(YCBModelsDir, 'models', anno['objName'], 'textured_simple.obj')): 164 | raise Exception('3D object models not available in %s'%(os.path.join(YCBModelsDir, 'models', anno['objName'], 'textured_simple.obj'))) 165 | 166 | # load object model 167 | objMesh = read_obj(os.path.join(YCBModelsDir, 'models', anno['objName'], 'textured_simple.obj')) 168 | 169 | # apply current pose to the object model 170 | objMesh.v = np.matmul(objMesh.v, cv2.Rodrigues(anno['objRot'])[0].T) + anno['objTrans'] 171 | 172 | # show 173 | if split == 'train': 174 | open3dVisualize([rightHandMesh, leftHandMesh, objMesh], ['r','r','g']) 175 | else: 176 | open3dVisualize([objMesh], ['r', 'g']) 177 | 178 | 179 | 180 | elif args['visType'] == 'matplotlib': 181 | 182 | # draw 2D projections of annotations on RGB image 183 | if split == 'train': 184 | imgAnno = showHandJoints(img, rightHandKps[jointsMapManoToSimple]) 185 | imgAnno = showHandJoints(imgAnno, leftHandKps[jointsMapManoToSimple]) 186 | else: 187 | # show only projection of root joint in evaluation split 188 | imgAnno = showHandJoints(img, rightHandKps) 189 | imgAnno = showHandJoints(imgAnno, leftHandKps) 190 | # show the hand bounding box 191 | imgAnno = show2DBoundingBox(imgAnno, anno['rightHandBoundingBox']) 192 | imgAnno = show2DBoundingBox(imgAnno, anno['leftHandBoundingBox']) 193 | imgAnno = showObjJoints(imgAnno, objKps, lineThickness=2) 194 | 195 | # create matplotlib window 196 | fig = plt.figure(figsize=(2, 2)) 197 | figManager = plt.get_current_fig_manager() 198 | figManager.resize(*figManager.window.maxsize()) 199 | 200 | # show RGB image 201 | ax0 = fig.add_subplot(2, 2, 1) 202 | ax0.imshow(img[:, :, [2, 1, 0]]) 203 | ax0.title.set_text('RGB Image') 204 | 205 | # show depth map 206 | ax1 = fig.add_subplot(2, 2, 2) 207 | im = ax1.imshow(depth) 208 | ax1.title.set_text('Depth Map') 209 | 210 | # show 2D projections of annotations on RGB image 211 | ax3 = fig.add_subplot(2, 2, 3) 212 | ax3.imshow(imgAnno[:, :, [2, 1, 0]]) 213 | ax3.title.set_text('3D Annotations projected to 2D') 214 | 215 | # show Seg 216 | ax0 = fig.add_subplot(2, 2, 4) 217 | ax0.imshow(seg[:, :, [2, 1, 0]]) 218 | ax0.title.set_text('Segmentation') 219 | 220 | plt.show() 221 | else: 222 | raise Exception('Unknown visualization type') 223 | 224 | if radomizeSeq: 225 | args['seq'] = random.choice(os.listdir(join(baseDir, split))) 226 | if radomizeID: 227 | args['id'] = random.choice(os.listdir(join(baseDir, split, args['seq'], 'rgb'))).split('.')[0] 228 | else: 229 | break -------------------------------------------------------------------------------- /vis_HO3D.py: -------------------------------------------------------------------------------- 1 | """ 2 | Visualize the projections in published HO-3D dataset 3 | """ 4 | from os.path import join 5 | import pip 6 | import argparse 7 | from utils.vis_utils import * 8 | import random 9 | from copy import deepcopy 10 | import open3d 11 | 12 | def install(package): 13 | if hasattr(pip, 'main'): 14 | pip.main(['install', package]) 15 | else: 16 | from pip._internal.main import main as pipmain 17 | pipmain(['install', package]) 18 | 19 | try: 20 | import matplotlib.pyplot as plt 21 | except: 22 | install('matplotlib') 23 | import matplotlib.pyplot as plt 24 | 25 | try: 26 | import chumpy as ch 27 | except: 28 | install('chumpy') 29 | import chumpy as ch 30 | 31 | 32 | try: 33 | import pickle 34 | except: 35 | install('pickle') 36 | import pickle 37 | 38 | import cv2 39 | from mpl_toolkits.mplot3d import Axes3D 40 | 41 | MANO_MODEL_PATH = './mano/models/MANO_RIGHT.pkl' 42 | 43 | # mapping of joints from MANO model order to simple order(thumb to pinky finger) 44 | jointsMapManoToSimple = [0, 45 | 13, 14, 15, 16, 46 | 1, 2, 3, 17, 47 | 4, 5, 6, 18, 48 | 10, 11, 12, 19, 49 | 7, 8, 9, 20] 50 | 51 | if not os.path.exists(MANO_MODEL_PATH): 52 | raise Exception('MANO model missing! Please run setup_mano.py to setup mano folder') 53 | else: 54 | from mano.webuser.smpl_handpca_wrapper_HAND_only import load_model 55 | 56 | 57 | def forwardKinematics(fullpose, trans, beta): 58 | ''' 59 | MANO parameters --> 3D pts, mesh 60 | :param fullpose: 61 | :param trans: 62 | :param beta: 63 | :return: 3D pts of size (21,3) 64 | ''' 65 | 66 | assert fullpose.shape == (48,) 67 | assert trans.shape == (3,) 68 | assert beta.shape == (10,) 69 | 70 | m = load_model(MANO_MODEL_PATH, ncomps=6, flat_hand_mean=True) 71 | m.fullpose[:] = fullpose 72 | m.trans[:] = trans 73 | m.betas[:] = beta 74 | 75 | return m.J_transformed.r, m 76 | 77 | 78 | if __name__ == '__main__': 79 | 80 | # parse the input arguments 81 | ap = argparse.ArgumentParser() 82 | ap.add_argument("ho3d_path", type=str, help="Path to HO3D dataset") 83 | ap.add_argument("ycbModels_path", type=str, help="Path to ycb models directory") 84 | ap.add_argument("-split", required=False, type=str, 85 | help="split type", choices=['train', 'evaluation'], default='train') 86 | ap.add_argument("-seq", required=False, type=str, 87 | help="sequence name") 88 | ap.add_argument("-id", required=False, type=str, 89 | help="image ID") 90 | ap.add_argument("-visType", required=False, 91 | help="Type of visualization", choices=['open3d', 'matplotlib'], default='matplotlib') 92 | args = vars(ap.parse_args()) 93 | 94 | baseDir = args['ho3d_path'] 95 | YCBModelsDir = args['ycbModels_path'] 96 | split = args['split'] 97 | 98 | # some checks to decide if visualizing one single image or randomly picked images 99 | if args['seq'] is None: 100 | args['seq'] = random.choice(os.listdir(join(baseDir, split))) 101 | runLoop = True 102 | else: 103 | runLoop = False 104 | 105 | if args['id'] is None: 106 | args['id'] = random.choice(os.listdir(join(baseDir, split, args['seq'], 'rgb'))).split('.')[0] 107 | else: 108 | pass 109 | 110 | if args['visType'] == 'matplotlib': 111 | o3dWin = Open3DWin() 112 | 113 | 114 | while(True): 115 | seqName = args['seq'] 116 | id = args['id'] 117 | 118 | # read image, depths maps and annotations 119 | img = read_RGB_img(baseDir, seqName, id, split) 120 | depth = read_depth_img(baseDir, seqName, id, split) 121 | anno = read_annotation(baseDir, seqName, id, split) 122 | 123 | if anno['objRot'] is None: 124 | print('Frame %s in sequence %s does not have annotations'%(args['id'], args['seq'])) 125 | if not runLoop: 126 | break 127 | else: 128 | args['seq'] = random.choice(os.listdir(join(baseDir, split))) 129 | args['id'] = random.choice(os.listdir(join(baseDir, split, args['seq'], 'rgb'))).split('.')[0] 130 | continue 131 | 132 | # get object 3D corner locations for the current pose 133 | objCorners = anno['objCorners3DRest'] 134 | objCornersTrans = np.matmul(objCorners, cv2.Rodrigues(anno['objRot'])[0].T) + anno['objTrans'] 135 | 136 | # get the hand Mesh from MANO model for the current pose 137 | if split == 'train': 138 | handJoints3D, handMesh = forwardKinematics(anno['handPose'], anno['handTrans'], anno['handBeta']) 139 | 140 | # project to 2D 141 | if split == 'train': 142 | handKps = project_3D_points(anno['camMat'], handJoints3D, is_OpenGL_coords=True) 143 | else: 144 | # Only root joint available in evaluation split 145 | handKps = project_3D_points(anno['camMat'], np.expand_dims(anno['handJoints3D'],0), is_OpenGL_coords=True) 146 | objKps = project_3D_points(anno['camMat'], objCornersTrans, is_OpenGL_coords=True) 147 | 148 | # visualize the hand contact map 149 | if 'handVertContact' in anno.keys() and args['visType'] == 'matplotlib' and split == 'train': 150 | contactMesh = deepcopy(handMesh) 151 | contactMesh.fullpose[:] = contactMesh.fullpose.r * 0 152 | contactMesh.trans[:] = np.array([0., 0., 1.0]) 153 | contactCols = np.zeros((contactMesh.r.shape[0], 3)) 154 | contactCols[:, 2] = anno['handVertContact'] 155 | contactMeshO3d = open3d.geometry.TriangleMesh() 156 | contactMeshO3d.vertices = open3d.utility.Vector3dVector(np.copy(contactMesh.r)) 157 | contactMeshO3d.triangles = open3d.utility.Vector3iVector(contactMesh.f) 158 | contactMeshO3d.vertex_colors = open3d.utility.Vector3dVector(contactCols) 159 | 160 | contactMapHand = o3dWin.capture_view([contactMeshO3d], 'utils/hand_r.txt') 161 | 162 | # Visualize 163 | if args['visType'] == 'open3d': 164 | # open3d visualization 165 | 166 | if not os.path.exists(os.path.join(YCBModelsDir, 'models', anno['objName'], 'textured_simple.obj')): 167 | raise Exception('3D object models not available in %s'%(os.path.join(YCBModelsDir, 'models', anno['objName'], 'textured_simple.obj'))) 168 | 169 | # load object model 170 | objMesh = read_obj(os.path.join(YCBModelsDir, 'models', anno['objName'], 'textured_simple.obj')) 171 | 172 | # apply current pose to the object model 173 | objMesh.v = np.matmul(objMesh.v, cv2.Rodrigues(anno['objRot'])[0].T) + anno['objTrans'] 174 | 175 | # show 176 | if split == 'train': 177 | open3dVisualize([handMesh, objMesh], ['r', 'g']) 178 | else: 179 | open3dVisualize([objMesh], ['r', 'g']) 180 | 181 | 182 | 183 | elif args['visType'] == 'matplotlib': 184 | 185 | # draw 2D projections of annotations on RGB image 186 | if split == 'train': 187 | imgAnno = showHandJoints(img, handKps[jointsMapManoToSimple]) 188 | else: 189 | # show only projection of root joint in evaluation split 190 | imgAnno = showHandJoints(img, handKps) 191 | # show the hand bounding box 192 | imgAnno = show2DBoundingBox(imgAnno, anno['handBoundingBox']) 193 | imgAnno = showObjJoints(imgAnno, objKps, lineThickness=2) 194 | 195 | # create matplotlib window 196 | fig = plt.figure(figsize=(2, 3)) 197 | figManager = plt.get_current_fig_manager() 198 | figManager.resize(*figManager.window.maxsize()) 199 | 200 | # show RGB image 201 | ax0 = fig.add_subplot(2, 3, 1) 202 | ax0.imshow(img[:, :, [2, 1, 0]]) 203 | ax0.title.set_text('RGB Image') 204 | 205 | # show depth map 206 | ax1 = fig.add_subplot(2, 3, 2) 207 | im = ax1.imshow(depth) 208 | ax1.title.set_text('Depth Map') 209 | 210 | # show contact map 211 | if 'handVertContact' in anno.keys(): 212 | ax3 = fig.add_subplot(2, 3, 3) 213 | im = ax3.imshow(contactMapHand) 214 | ax3.title.set_text('Contact Map') 215 | 216 | # show 3D hand mesh 217 | ax3 = fig.add_subplot(2, 3, 4, projection="3d") 218 | if split=='train': 219 | plot3dVisualize(ax3, handMesh, flip_x=False, isOpenGLCoords=True, c="r") 220 | ax3.title.set_text('Hand Mesh') 221 | 222 | # show 2D projections of annotations on RGB image 223 | ax4 = fig.add_subplot(2, 3, 5) 224 | ax4.imshow(imgAnno[:, :, [2, 1, 0]]) 225 | ax4.title.set_text('3D Annotations projected to 2D') 226 | 227 | plt.show() 228 | else: 229 | raise Exception('Unknown visualization type') 230 | 231 | if runLoop: 232 | args['seq'] = random.choice(os.listdir(join(baseDir, split))) 233 | args['id'] = random.choice(os.listdir(join(baseDir, split, args['seq'], 'rgb'))).split('.')[0] 234 | else: 235 | break -------------------------------------------------------------------------------- /vis_pcl_all_cameras.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import open3d as o3d 4 | from os.path import join 5 | import cv2 6 | import argparse 7 | from open3d.open3d.geometry import create_rgbd_image_from_color_and_depth 8 | from open3d.open3d.geometry import create_point_cloud_from_rgbd_image 9 | 10 | # Paths and Params 11 | 12 | # Path to the 'train' directory of HO3D dataset 13 | # sequences_dir = '/media/shreyas/ssd2/Dataset/HO3D_Release_Final/train' 14 | 15 | 16 | height, width = 480, 640 17 | depth_threshold = 800 18 | 19 | multiCamSeqs = [ 20 | 'ABF1', 21 | 'BB1', 22 | 'GPMF1', 23 | 'GSF1', 24 | 'MDF1', 25 | 'SB1', 26 | 'ShSu1', 27 | 'SiBF1', 28 | 'SMu4', 29 | 'MPM1', 30 | 'AP1' 31 | ] 32 | 33 | 34 | def inverse_relative(pose_1_to_2): 35 | pose_2_to_1 = np.zeros((4, 4), dtype='float32') 36 | pose_2_to_1[:3, :3] = np.transpose(pose_1_to_2[:3, :3]) 37 | pose_2_to_1[:3, 3:4] = -np.dot(np.transpose(pose_1_to_2[:3, :3]), pose_1_to_2[:3, 3:4]) 38 | pose_2_to_1[3, 3] = 1 39 | return pose_2_to_1 40 | 41 | 42 | def get_intrinsics(filename): 43 | with open(filename, 'r') as f: 44 | line = f.readline() 45 | line = line.strip() 46 | items = line.split(',') 47 | for item in items: 48 | if 'fx' in item: 49 | fx = float(item.split(':')[1].strip()) 50 | elif 'fy' in item: 51 | fy = float(item.split(':')[1].strip()) 52 | elif 'ppx' in item: 53 | ppx = float(item.split(':')[1].strip()) 54 | elif 'ppy' in item: 55 | ppy = float(item.split(':')[1].strip()) 56 | 57 | camMat = np.array([[fx, 0, ppx], [0, fy, ppy], [0, 0, 1]]) 58 | return camMat 59 | 60 | def read_depth_img(depth_filename): 61 | """Read the depth image in dataset and decode it""" 62 | 63 | depth_scale = 0.00012498664727900177 64 | depth_img = cv2.imread(depth_filename) 65 | 66 | dpt = depth_img[:, :, 2] + depth_img[:, :, 1] * 256 67 | dpt = dpt * depth_scale 68 | 69 | return dpt 70 | 71 | 72 | def load_point_clouds(seq, fID, setDir): 73 | pcds = [] 74 | 75 | seqDir = os.path.join(setDir, seq) 76 | calibDir = os.path.join(setDir, '../', 'calibration', seq, 'calibration') 77 | cams_order = np.loadtxt(join(calibDir, 'cam_orders.txt')).astype('uint8').tolist() 78 | depth_scale = 1. / np.loadtxt(join(calibDir, 'cam_0_depth_scale.txt')) 79 | Ts = [] 80 | for i in range(len(cams_order)): 81 | T_i = np.loadtxt(join(calibDir, 'trans_{}.txt'.format(i))) 82 | Ts.append(T_i) 83 | 84 | for i in range(len(cams_order)): 85 | path_color = join(seqDir+str(cams_order[i]), 'rgb', fID + '.png') 86 | if not os.path.exists(path_color): 87 | path_color = join(seqDir + str(cams_order[i]), 'rgb', fID + '.jpg') 88 | if not os.path.exists(path_color): 89 | continue 90 | 91 | color_raw = o3d.io.read_image(path_color)#o3d.geometry.Image(color_raw.astype(np.float32)) 92 | depth_raw = read_depth_img(path_color.replace('rgb', 'depth').replace('jpg', 'png')) 93 | 94 | depth_raw = o3d.geometry.Image(depth_raw.astype(np.float32)) 95 | K = get_intrinsics(join(calibDir, 'cam_{}_intrinsics.txt'.format(i))).tolist() 96 | 97 | rgbd_image = create_rgbd_image_from_color_and_depth(color_raw, 98 | depth_raw, 99 | depth_scale=1, 100 | convert_rgb_to_intensity=False) 101 | pcd = create_point_cloud_from_rgbd_image( 102 | rgbd_image, o3d.camera.PinholeCameraIntrinsic(width=width, 103 | height=height, 104 | fx=K[0][0], 105 | fy=K[1][1], 106 | cx=K[0][2], 107 | cy=K[1][2])) 108 | 109 | pcd.transform((Ts[i])) 110 | pcds.append(pcd) 111 | 112 | return pcds 113 | 114 | 115 | def combine_point_clouds(pcds): 116 | pcd_combined = o3d.geometry.PointCloud() 117 | for point_id in range(len(pcds)): 118 | pcd_combined += pcds[point_id] 119 | 120 | return pcd_combined 121 | 122 | 123 | 124 | def manual_registration(annoFiles): 125 | if args.seq is not None: 126 | if args.seq not in multiCamSeqs: 127 | print('[ERROR] Sequence should be one of the following: ', multiCamSeqs) 128 | return 129 | annoFiles = [args.seq] 130 | 131 | for idx, annoFile in enumerate(annoFiles): 132 | 133 | seq = annoFile 134 | 135 | setDir = os.path.join(args.base_path, 'train') 136 | if not os.path.exists(os.path.join(setDir, seq+'0','rgb')): 137 | setDir = os.path.join(args.base_path, 'evaluation') 138 | 139 | files = os.listdir(os.path.join(setDir,seq+'0','rgb')) 140 | files = [f[:-4] for f in files] 141 | if args.fid is not None: 142 | files = [args.fid] 143 | for fID in files[:]: 144 | pcds = load_point_clouds(seq, fID, setDir) 145 | pcd = combine_point_clouds(pcds) 146 | 147 | # o3d.visualization.draw_geometries([pcd]) 148 | vis = o3d.visualization.Visualizer() 149 | vis.create_window() 150 | vis.add_geometry(pcd) 151 | vis.run() 152 | vis.destroy_window() 153 | 154 | 155 | 156 | 157 | 158 | if __name__ == "__main__": 159 | parser = argparse.ArgumentParser(description='Show some samples from the dataset.') 160 | parser.add_argument('base_path', type=str, 161 | help='Path to where the HO3D dataset is located.') 162 | parser.add_argument('--seq', type=str, choices=['ABF1','BB1','GPMF1','GSF1','MDF1','SB1','ShSu1', 'SiBF1','SMu4', 'MPM1', 'AP1'], 163 | help='Sequence name.', required=False) 164 | parser.add_argument('--fid', type=str, 165 | help='File ID', required=False) 166 | args = parser.parse_args() 167 | 168 | manual_registration(multiCamSeqs) 169 | # show_manual_annotations() 170 | --------------------------------------------------------------------------------