├── src ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── resnet.cpython-37.pyc │ │ ├── __init__.cpython-35.pyc │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── resnet2d.cpython-35.pyc │ │ ├── resnet2d.cpython-36.pyc │ │ ├── resnet2d.cpython-37.pyc │ │ ├── resnext2d.cpython-37.pyc │ │ ├── resnet2d_int.cpython-35.pyc │ │ ├── resnet2d_int.cpython-37.pyc │ │ ├── resnet2d_lift.cpython-37.pyc │ │ ├── resnet2d_lift1.cpython-35.pyc │ │ ├── resnet2d_lift2.cpython-35.pyc │ │ ├── resnet2d_lift3.cpython-35.pyc │ │ ├── resnet2d_lift4.cpython-35.pyc │ │ ├── resnet2d_lift5.cpython-35.pyc │ │ ├── resnet2d_lift6.cpython-35.pyc │ │ ├── resnet2d_lift7.cpython-35.pyc │ │ ├── resnet2d_lift8.cpython-35.pyc │ │ ├── resnet2d_lift9.cpython-35.pyc │ │ ├── resnext2d_int.cpython-37.pyc │ │ ├── resnet2d_lift02.cpython-35.pyc │ │ ├── resnet2d_lift02.cpython-37.pyc │ │ ├── resnet2d_lift03.cpython-35.pyc │ │ ├── resnet2d_lift03.cpython-37.pyc │ │ ├── resnet2d_lift10.cpython-35.pyc │ │ ├── resnet2d_lift11.cpython-35.pyc │ │ ├── resnet2d_int_clone.cpython-35.pyc │ │ └── resnet2d_int_clone.cpython-37.pyc │ ├── resnet2d_lift.py │ ├── resnet2d_int.py │ ├── resnet2d.py │ └── resnet.py ├── utils │ ├── __init__.py │ ├── __pycache__ │ │ ├── img.cpython-35.pyc │ │ ├── img.cpython-37.pyc │ │ ├── eval.cpython-35.pyc │ │ ├── eval.cpython-37.pyc │ │ ├── utils.cpython-35.pyc │ │ ├── utils.cpython-37.pyc │ │ ├── __init__.cpython-35.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── logger.cpython-35.pyc │ │ └── logger.cpython-37.pyc │ ├── utils.py │ ├── logger.py │ ├── img.py │ └── eval.py ├── datasets_2d │ ├── __init__.py │ ├── __pycache__ │ │ ├── mpii.cpython-35.pyc │ │ ├── mpii.cpython-37.pyc │ │ ├── fusion.cpython-35.pyc │ │ ├── fusion.cpython-37.pyc │ │ ├── h36m17.cpython-35.pyc │ │ ├── h36m17.cpython-37.pyc │ │ ├── h36m17r.cpython-35.pyc │ │ ├── h36m17s.cpython-35.pyc │ │ ├── h36m17s.cpython-37.pyc │ │ ├── mpiinf.cpython-37.pyc │ │ ├── __init__.cpython-35.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── h36m17_mpii.cpython-37.pyc │ │ ├── h36m17s_moon.cpython-37.pyc │ │ ├── h36m17s_mpii.cpython-37.pyc │ │ ├── mpiinf_mpii.cpython-37.pyc │ │ └── mpiinf_small.cpython-37.pyc │ ├── mpiinf_mpii.py │ ├── h36m17_mpii.py │ ├── mpii.py │ ├── mpiinf.py │ └── h36m17.py ├── datasets_lift │ ├── __init__.py │ ├── __pycache__ │ │ ├── fusion.cpython-35.pyc │ │ ├── fusion.cpython-37.pyc │ │ ├── h36m17.cpython-35.pyc │ │ ├── h36m17.cpython-37.pyc │ │ ├── h36m17s.cpython-35.pyc │ │ ├── h36m17s.cpython-37.pyc │ │ ├── mpiinf.cpython-37.pyc │ │ ├── __init__.cpython-35.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── h36m17_aug.cpython-35.pyc │ │ ├── h36m17_aug.cpython-37.pyc │ │ ├── h36m17_scale.cpython-37.pyc │ │ └── h36m17_mpiinf.cpython-37.pyc │ ├── mpiinf.py │ ├── h36m17.py │ └── h36m17_mpiinf.py ├── __pycache__ │ ├── conf.cpython-36.pyc │ ├── conf.cpython-37.pyc │ ├── opts.cpython-35.pyc │ ├── opts.cpython-36.pyc │ ├── opts.cpython-37.pyc │ ├── ref.cpython-35.pyc │ ├── ref.cpython-37.pyc │ ├── train.cpython-35.pyc │ ├── train.cpython-37.pyc │ ├── opts_2d.cpython-37.pyc │ ├── opts_lift.cpython-37.pyc │ ├── train_2d.cpython-37.pyc │ ├── train_lift.cpython-37.pyc │ └── train_moon.cpython-37.pyc ├── matlab │ ├── test_util │ │ ├── mpii_3D_error.m │ │ ├── mpii_get_activity_name.m │ │ ├── camera_calibration │ │ │ ├── ts1-4cameras.calib │ │ │ └── ts5-6cameras.calib │ │ ├── mpii_get_pck_auc_joint_groups.m │ │ ├── mpii_perspective_correction_code.m │ │ ├── test_dataset.m │ │ ├── mpii_test_predictions.m │ │ ├── test.m │ │ ├── mpii_test.m │ │ ├── mpii_get_joints.m │ │ ├── mpii_evaluate_errors.m │ │ └── mpii_compute_3d_pck.m │ ├── evaluate_all.m │ └── evaluate_result.m ├── conf.py ├── analysis_error_simple.py ├── analysis_error_mixture.py ├── opts_lift.py ├── train_lift.py ├── opts_2d.py ├── main_lift.py ├── train_2d.py └── main_2d.py ├── requirements.txt ├── annot └── .gitignore ├── data └── .gitignore ├── exp └── .gitignore ├── script ├── table6_eval.sh ├── table2.sh ├── table3_1.sh ├── table5.sh ├── table3_2.sh └── table6.sh ├── readme.txt ├── README.md └── LICENSE /src/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/datasets_2d/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/datasets_lift/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # python requirements 2 | opencv-python 3 | progress 4 | 5 | -------------------------------------------------------------------------------- /annot/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /exp/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /src/__pycache__/conf.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/conf.cpython-36.pyc -------------------------------------------------------------------------------- /src/__pycache__/conf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/conf.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/opts.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/opts.cpython-35.pyc -------------------------------------------------------------------------------- /src/__pycache__/opts.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/opts.cpython-36.pyc -------------------------------------------------------------------------------- /src/__pycache__/opts.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/opts.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/ref.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/ref.cpython-35.pyc -------------------------------------------------------------------------------- /src/__pycache__/ref.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/ref.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/train.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/train.cpython-35.pyc -------------------------------------------------------------------------------- /src/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/opts_2d.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/opts_2d.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/opts_lift.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/opts_lift.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/train_2d.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/train_2d.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/img.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/img.cpython-35.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/img.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/img.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/train_lift.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/train_lift.cpython-37.pyc -------------------------------------------------------------------------------- /src/__pycache__/train_moon.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/__pycache__/train_moon.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/eval.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/eval.cpython-35.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/eval.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/eval.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/utils.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/utils.cpython-35.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/logger.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/logger.cpython-35.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/logger.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/utils/__pycache__/logger.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/mpii.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/mpii.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/mpii.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/mpii.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d.cpython-36.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnext2d.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnext2d.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/fusion.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/fusion.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/fusion.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/fusion.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17r.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17r.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17s.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17s.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17s.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17s.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/mpiinf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/mpiinf.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_int.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_int.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_int.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_int.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/fusion.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/fusion.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/fusion.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/fusion.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17s.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17s.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17s.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17s.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/mpiinf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/mpiinf.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift1.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift1.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift2.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift2.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift3.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift3.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift4.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift4.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift5.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift5.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift6.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift6.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift7.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift7.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift8.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift8.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift9.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift9.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnext2d_int.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnext2d_int.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17_mpii.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17_mpii.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17s_moon.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17s_moon.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/h36m17s_mpii.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/h36m17s_mpii.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/mpiinf_mpii.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/mpiinf_mpii.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_2d/__pycache__/mpiinf_small.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_2d/__pycache__/mpiinf_small.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17_aug.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17_aug.cpython-35.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17_aug.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17_aug.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift02.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift02.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift02.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift02.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift03.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift03.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift03.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift03.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift10.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift10.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_lift11.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_lift11.cpython-35.pyc -------------------------------------------------------------------------------- /script/table6_eval.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # evaluation script for table 5 4 | matlab -nodisplay -nosplash -nodesktop -r "run('./src/matlab/evaluate_all.m');exit;" 5 | 6 | -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17_scale.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17_scale.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_int_clone.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_int_clone.cpython-35.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet2d_int_clone.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/models/__pycache__/resnet2d_int_clone.cpython-37.pyc -------------------------------------------------------------------------------- /src/datasets_lift/__pycache__/h36m17_mpiinf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juyongchang/PoseLifter/HEAD/src/datasets_lift/__pycache__/h36m17_mpiinf.cpython-37.pyc -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_3D_error.m: -------------------------------------------------------------------------------- 1 | function out_struct = mpii_3D_error(method_name, error_vector) 2 | out_struct = struct('method', method_name, 'error', error_vector); 3 | end -------------------------------------------------------------------------------- /readme.txt: -------------------------------------------------------------------------------- 1 | 2 | # install cuda 10.2 3 | # install anaconda3 4 | 5 | # create conda environment 6 | conda create --name poselifter --clone base 7 | 8 | # activate new environment 9 | conda activate poselifter 10 | 11 | # install pytorch 1.1.0 for cuda 10.2 12 | conda install pytorch==1.1.0 torchvision==0.3.0 cudatoolkit=10.0 -c pytorch 13 | 14 | # install packages 15 | pip install -r requirements.txt 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_get_activity_name.m: -------------------------------------------------------------------------------- 1 | function [activity_names] = mpii_get_activity_name(activity_id) 2 | 3 | activities{1} = 'Standing/Walking'; 4 | activities{2} = 'Exercising'; 5 | activities{3} = 'Sitting'; 6 | activities{4} = 'Reaching/Crouching'; 7 | activities{5} = 'On The Floor'; 8 | activities{6} = 'Sports'; 9 | activities{7} = 'Miscellaneous'; 10 | 11 | activity_names = activities(activity_id); 12 | end -------------------------------------------------------------------------------- /src/matlab/test_util/camera_calibration/ts1-4cameras.calib: -------------------------------------------------------------------------------- 1 | tc camera calibration v0.3 2 | camera 0 cam_8 3 | frame 0 4 | sensorSize 10 10 # in mm 5 | focalLength 7.32506 # in mm 6 | pixelAspect 1.00044 7 | centerOffset -0.0322884 0.0929296 # in mm (positive values move right and down) 8 | distortionModel OpenCV 9 | distortion 0.0 0.0 0.0 0.0 0.0 10 | origin 3427.28 1387.86 309.42 11 | up -0.208215 0.976233 0.06014 12 | right 0.000575281 0.0616098 -0.9981 13 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_get_pck_auc_joint_groups.m: -------------------------------------------------------------------------------- 1 | function [joint_groups] = mpii_get_pck_auc_joint_groups() 2 | 3 | joint_groups = { %'Head', [1,17]; 4 | 'Head', [1]; 5 | 'Neck', [2]; 6 | 'Shou', [3,6]; 7 | 'Elbow', [4,7]; 8 | 'Wrist', [5,8]; 9 | %'spine', [16]; 10 | 'Hip', [9,12]; 11 | 'Knee', [10,13]; 12 | 'Ankle', [11,14]; 13 | }; 14 | end -------------------------------------------------------------------------------- /src/datasets_2d/mpiinf_mpii.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | from datasets_2d.mpiinf import MPIINF 3 | from datasets_2d.mpii import MPII 4 | 5 | class MPIINF_MPII(data.Dataset): 6 | def __init__(self, split): 7 | self.split = split 8 | self.INF = MPIINF(split) 9 | self.MPII = MPII() 10 | self.num_inf = len(self.INF) 11 | self.num_mpii = len(self.MPII) 12 | print('Load %d MPIINF and %d MPII samples' % (self.num_inf, self.num_mpii)) 13 | 14 | def __getitem__(self, index): 15 | if index < self.num_mpii: 16 | return self.INF[index] 17 | else: 18 | return self.MPII[index - self.num_mpii] 19 | 20 | def __len__(self): 21 | return self.num_mpii*2 22 | 23 | -------------------------------------------------------------------------------- /src/datasets_2d/h36m17_mpii.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | from datasets_2d.h36m17 import H36M17 3 | from datasets_2d.mpii import MPII 4 | 5 | class H36M17_MPII(data.Dataset): 6 | def __init__(self, protocol, split): 7 | self.split = split 8 | self.H36M = H36M17(protocol, split) 9 | self.MPII = MPII() 10 | self.num_h36m = len(self.H36M) 11 | self.num_mpii = len(self.MPII) 12 | print('Load %d H36M and %d MPII samples' % (self.num_h36m, self.num_mpii)) 13 | 14 | def __getitem__(self, index): 15 | if index < self.num_mpii: 16 | return self.H36M[index] 17 | else: 18 | return self.MPII[index - self.num_mpii] 19 | 20 | def __len__(self): 21 | return self.num_mpii*2 22 | 23 | -------------------------------------------------------------------------------- /src/matlab/evaluate_all.m: -------------------------------------------------------------------------------- 1 | close all; 2 | clear; 3 | clc; 4 | 5 | % Add path for evaluation 6 | addpath('./test_util'); 7 | 8 | % 3d dataset: H36M 9 | % w/o canonical depth 10 | opt.dataset2d = 'inf_mpii'; 11 | opt.dataset3d = 'h36m'; 12 | opt.canonical = 0; 13 | evaluate_result(opt); 14 | 15 | % 3d dataset: H36M 16 | % w/ canonical depth 17 | opt.dataset2d = 'inf_mpii'; 18 | opt.dataset3d = 'h36m'; 19 | opt.canonical = 1; 20 | evaluate_result(opt); 21 | 22 | % 3d dataset: INF 23 | % w/o canonical depth 24 | opt.dataset2d = 'inf_mpii'; 25 | opt.dataset3d = 'inf'; 26 | opt.canonical = 0; 27 | evaluate_result(opt); 28 | 29 | % 3d dataset: INF 30 | % w/ canonical depth 31 | opt.dataset2d = 'inf_mpii'; 32 | opt.dataset3d = 'inf'; 33 | opt.canonical = 1; 34 | evaluate_result(opt); 35 | 36 | -------------------------------------------------------------------------------- /src/utils/utils.py: -------------------------------------------------------------------------------- 1 | from numpy.random import randn 2 | import conf 3 | 4 | class AverageMeter(object): 5 | def __init__(self): 6 | self.reset() 7 | 8 | def reset(self): 9 | self.val = 0 10 | self.avg = 0 11 | self.sum = 0 12 | self.count = 0 13 | 14 | def update(self, val, n=1): 15 | self.val = val 16 | self.sum += val * n 17 | self.count += n 18 | self.avg = self.sum / self.count 19 | 20 | def rnd(x): 21 | return max(-2 * x, min(2 * x, randn() * x)) 22 | 23 | def flip(img): 24 | return img[:, :, ::-1].copy() 25 | 26 | def shuffle_lr(x): 27 | for e in conf.flip_index: 28 | x[e[0]], x[e[1]] = x[e[1]].copy(), x[e[0]].copy() 29 | return x 30 | 31 | def adjust_learning_rate(optimizer, epoch, drop_lr, lr): 32 | v = lr * (0.1 ** (epoch // drop_lr)) 33 | for param_group in optimizer.param_groups: 34 | param_group['lr'] = v 35 | 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PoseLifter: Absolute 3D human pose lifting network from a single noisy 2D human pose 2 | 3 | ## Introduction 4 | This repo provides the source code to reproduce all the results of our [arXiv paper](https://arxiv.org/abs/1910.12029): 5 | 6 | > Ju Yong Chang, Gyeongsik Moon, Kyoung Mu Lee. PoseLifter: Absolute 3D human pose lifting network from a single noisy 2D human pose. arXiv preprint arXiv:1910.12029, 2019. 7 | 8 | ## Dataset setup 9 | You can download the [annotation data](https://drive.google.com/open?id=1ldmoS7h49Ww-4pdAMTtdFmpoyz9nwSKV) and the images for [Human3.6M](https://drive.google.com/open?id=1A_cRrGaUjFRywECQR0Sn7AMWU6LSeLkc), [MPI-INF-3DHP](https://drive.google.com/open?id=1IxxGvFVgJ52JF1emEfjxOQoIQvBIhlxU), and [MPII Human Pose Dataset](http://human-pose.mpi-inf.mpg.de/). 10 | 11 | ## Reproducing our results 12 | You can reproduce the results of our paper by running the script files in the script directory. For example, to generate the numbers in Table 2, run: 13 | ```bash 14 | ./script/table2.sh 15 | ``` 16 | 17 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_perspective_correction_code.m: -------------------------------------------------------------------------------- 1 | test_subject_id = [1,2,3,4,5,6]; 2 | focalL{1} = (2048/10)*7.320339203; % res/sensorsize*focalLengthMM 3 | focalL{2} = focalL{1}; 4 | focalL{3} = focalL{1}; 5 | focalL{4} = focalL{1}; 6 | focalL{5} = (1920/10)*8.770747185; % res/sensorsize*focalLengthMM 7 | focalL{6} = focalL{5}; 8 | 9 | for ts = 1:6 10 | 11 | %Fancy predictions here: predict_2d and predict_3d. predict_2d is in the uncropped(?) image space 12 | 13 | focalLengthInPX = focalL{ts}; 14 | 15 | resolutionXInPX = image_size{ts}(2); %I can't seem to remember which one is x or why. Try both until something works :) 16 | resolutionYInPX = image_size{ts}(1); 17 | principalPointX = resolutionXInPX/2; 18 | principalPointY = resolutionYInPX/2; 19 | center = predict_2d(15,:) - [principalPointX, principalPointY]; % (pelvis location) 20 | R = mpii_perspective_correction(center(1), 0, focalLengthInPX); 21 | predict_3d = R * predict_3d; 22 | end -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Ju Yong Chang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /script/table2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # script for table 1 4 | 5 | # baseline 6 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 7 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 0 -canonical \ 8 | -network 'resnet' -mode 0 -num_layers 2 -num_features 4096 \ 9 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 10 | 11 | # + location 12 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 13 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 0 -canonical \ 14 | -network 'resnet' -mode 3 -num_layers 2 -num_features 4096 \ 15 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 16 | 17 | # + scale 18 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 19 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 0 -canonical \ 20 | -network 'resnet' -mode 4 -num_layers 2 -num_features 4096 \ 21 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 22 | 23 | # + location & scale 24 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 25 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 0 -canonical \ 26 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 27 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 28 | 29 | -------------------------------------------------------------------------------- /src/conf.py: -------------------------------------------------------------------------------- 1 | 2 | # annot directory 3 | data_dir = './annot' 4 | 5 | # image directory 6 | h36m_img_dir = './data/h36m' 7 | mpii_img_dir = './data/mpii' 8 | inf_img_dir = './data/inf' 9 | 10 | # experiment directory 11 | exp_dir = './exp' 12 | 13 | # number of threads 14 | num_threads = 4 15 | 16 | # number of joints 17 | num_joints = 17 18 | 19 | # root index (hip) 20 | root = 0 21 | 22 | # input/output resolutions 23 | res_in = 256 24 | res_out = 64 25 | 26 | # standard deviation of gaussian function for heatmap generation 27 | std = 1.0 28 | 29 | # parameters for data augmentation 30 | scale = 0.25 31 | rotate = 30 32 | flip_index = [[3, 6], [2, 5], [1, 4], 33 | [16, 13], [15, 12], [14, 11]] 34 | 35 | # joint index mapping from MPII to H36M 36 | inds = [3, 2, 1, 4, 5, 6, 0, 7, 8, 10, 16, 15, 14, 11, 12, 13] 37 | 38 | # sum of bone lengths (mm) 39 | sum_of_bone_length = 4139.15 40 | 41 | # bone structure 42 | bone = [[0, 1], [1, 2], [2, 3], 43 | [0, 4], [4, 5], [5, 6], 44 | [0, 7], [7, 8], [8, 9], [9, 10], 45 | [8, 11], [11, 12], [12, 13], 46 | [8, 14], [14, 15], [15, 16]] 47 | 48 | # number of actions for human3.6m dataset 49 | num_actions = 15 50 | 51 | # original image size 52 | width = 1000 53 | height = 1000 54 | 55 | # for canonical depth 56 | f0 = 1000 57 | 58 | -------------------------------------------------------------------------------- /script/table3_1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # script for table 2 4 | 5 | ########################################################################################################### 6 | # 2D pose estimation for protocol 0 7 | 8 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 9 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 0 -multi_gpu \ 10 | -network 'resnet152' \ 11 | -opt_method 'rmsprop' -lr 1e-4 \ 12 | -num_epochs 50 -batch_size 48 13 | 14 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 15 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 0 -multi_gpu -save_results \ 16 | -network 'resnet152' -integral 1 -weight1 1e0 \ 17 | -model_2d_path 'test_h36m_protocol0/resnet152/train_h36m_mpii/rmsprop_lr1.0e-04_batch48/final_model.pth' \ 18 | -opt_method 'rmsprop' -lr 1e-5 \ 19 | -num_epochs 10 -batch_size 48 20 | 21 | ########################################################################################################### 22 | # Error analysis for protocol 0 23 | 24 | python ./src/analysis_error_mixture.py 25 | 26 | ########################################################################################################### 27 | # 2D pose estimation for protocol 1 28 | 29 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 30 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 -multi_gpu \ 31 | -network 'resnet152' \ 32 | -opt_method 'rmsprop' -lr 1e-4 \ 33 | -num_epochs 50 -batch_size 48 34 | 35 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 36 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 -multi_gpu \ 37 | -network 'resnet152' -integral 1 -weight1 1e0 \ 38 | -model_2d_path 'test_h36m_protocol1/resnet152/train_h36m_mpii/rmsprop_lr1.0e-04_batch48/final_model.pth' \ 39 | -opt_method 'rmsprop' -lr 1e-5 \ 40 | -num_epochs 10 -batch_size 48 41 | 42 | -------------------------------------------------------------------------------- /src/matlab/test_util/test_dataset.m: -------------------------------------------------------------------------------- 1 | close all 2 | clear 3 | clc 4 | 5 | % 1: Site -> 11 6 | % 2: Neck -> 9 7 | % 3: RightArm -> 15 8 | % 4: RightForeArm -> 16 9 | % 5: RightHand -> 17 10 | % 6: LeftArm -> 12 11 | % 7: LeftForeArm -> 13 12 | % 8: LeftHand -> 14 13 | % 9: RightUpLeg -> 2 14 | % 10: RightLeg -> 3 15 | % 11: RightFoot -> 4 16 | % 12: LeftUpLeg -> 5 17 | % 13: LeftLeg -> 6 18 | % 14: LeftFoot -> 7 19 | % 15: Hip -> 1 20 | % 16: Spine -> 8 21 | % 17: Head -> 10 22 | 23 | % Mapping table from H36M to MPI 24 | map = [15, 9, 10, 11, 12, 13, 14, 16, 2, 17, 1, 6, 7, 8, 3, 4, 5]; 25 | 26 | % 27 | pose2d = []; 28 | pose3d = []; 29 | bbox = []; 30 | cam_f = []; 31 | cam_c = []; 32 | 33 | % For each sequence 34 | for k = 1:6 35 | id_sequence = k; 36 | 37 | % Load annot data 38 | annot_name = sprintf('../TS%d/annot_data.mat', id_sequence); 39 | load(annot_name); 40 | 41 | % Select valid frames 42 | idx = find(valid_frame == 1); 43 | num_valid = length(idx); 44 | 45 | % Focal lengths 46 | if k <= 4 47 | f = (2048/10)*7.320339203; 48 | else 49 | f = (1920/10)*8.770747185; 50 | end 51 | 52 | % Load first image 53 | id_img = idx(1); 54 | img_name = sprintf('../TS%d/imageSequence/img_%06d.jpg', id_sequence, id_img); 55 | img = imread(img_name); 56 | 57 | % Image size & principal points 58 | w = size(img, 2); 59 | h = size(img, 1); 60 | px = w / 2; 61 | py = h / 2; 62 | 63 | % 64 | p2d = permute(squeeze(annot2(:,map,1,idx)), [3 2 1]); 65 | p2d(:,:,1) = p2d(:,:,1) / w * 255.0; 66 | p2d(:,:,2) = p2d(:,:,2) / h * 255.0; 67 | pose2d = cat(1, pose2d, p2d); 68 | pose3d = cat(1, pose3d, permute(squeeze(univ_annot3(:,map,1,idx)), [3 2 1])); 69 | bbox = cat(1, bbox, repmat([1 1 w h], num_valid, 1)); 70 | cam_f = cat(1, cam_f, repmat([f f], num_valid, 1)); 71 | cam_c = cat(1, cam_c, repmat([px py], num_valid, 1)); 72 | end 73 | 74 | % Save data 75 | filename = '../inf.mat'; 76 | save(filename, 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c'); 77 | 78 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_test_predictions.m: -------------------------------------------------------------------------------- 1 | close all 2 | clear 3 | clc 4 | 5 | test_subject_id = [1,2,3,4,5,6]; 6 | test_data_path = '/media/juyongchang/2e6123d0-bc51-4f5a-a01b-818048d27c37/MPI_INF_3DHP/test/'; %Change to wherever you put this data. 7 | data_base_path = [test_data_path filesep 'TS']; 8 | 9 | [~,o1,o2,relevant_labels] = mpii_get_joints('relevant'); 10 | 11 | net_base = 'result'; 12 | snapshot_base = net_base; 13 | 14 | % 15 | sequencewise_per_joint_error = cell(6,1); 16 | sequencewise_activity_labels = cell(6,1); 17 | for i = 1:length(test_subject_id) 18 | dat = load([data_base_path int2str(test_subject_id(i)) filesep 'annot_data.mat']); 19 | num_test_points = sum(dat.valid_frame(:)); 20 | per_joint_error = zeros(17,1,num_test_points); 21 | pje_idx = 1; 22 | sequencewise_activity_labels{i} = dat.activity_annotation(dat.valid_frame == 1); 23 | % 24 | for j = 1:length(dat.valid_frame) 25 | if(dat.valid_frame(j)) 26 | fprintf('Image %d of %d for Test ID %d\n',j, length(dat.valid_frame), test_subject_id(i)); 27 | error = zeros(17,1); 28 | 29 | img = imread([data_base_path int2str(test_subject_id(i)) filesep 'imageSequence' filesep sprintf('img_%06d.jpg',j)]); 30 | %The GT has 17 joints, and the order and the annotation of the joints can be observed through the 'relevant_labels' variable 31 | P = dat.univ_annot3(:,:,:,j)-repmat(dat.univ_annot3(:,15,:,j),1,17); 32 | 33 | % 34 | pred_p = P; %Replace with the actual prediction formatted as 3x17; 35 | error_p = (pred_p - P).^2; 36 | error_p = sqrt(sum(error_p, 1)); 37 | error(:,1) = error(:,1) + error_p(:); 38 | 39 | 40 | per_joint_error(:,:,pje_idx) = error; 41 | pje_idx = pje_idx +1; 42 | end 43 | end 44 | sequencewise_per_joint_error{i} = per_joint_error; 45 | 46 | end 47 | 48 | save([net_base filesep 'mpii_3dhp_prediction.mat'], 'sequencewise_per_joint_error', 'sequencewise_activity_labels'); 49 | [seq_table, activity_table] = mpii_evaluate_errors(sequencewise_per_joint_error, sequencewise_activity_labels); 50 | 51 | out_file = [net_base filesep 'mpii_3dhp_evaluation']; 52 | writetable(cell2table(seq_table), [out_file '_sequencewise.csv']); 53 | writetable(cell2table(activity_table), [out_file '_activitywise.csv']); 54 | 55 | -------------------------------------------------------------------------------- /src/models/resnet2d_lift.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn as nn 4 | from models.resnet2d_int import ResNetInt 5 | from models.resnet import ResNet 6 | import pdb 7 | 8 | class ResNetLift(nn.Module): 9 | def __init__(self, network, num_joints, num_layers, num_features, mode, model_2d_path, model_lift_path): 10 | super(ResNetLift, self).__init__() 11 | 12 | # 2d pose estimation module 13 | self.model_2d = ResNetInt(network, num_joints) 14 | if model_2d_path is not None: 15 | if os.path.isfile(model_2d_path): 16 | print('Load pretrained 2D pose estimation model..') 17 | state = torch.load(model_2d_path) 18 | pretrained_dict = state['model'] 19 | model_dict = self.model_2d.state_dict() 20 | new_pretrained_dict = {k[7:]: v for k, v in pretrained_dict.items() if k[7:] in model_dict} 21 | model_dict.update(new_pretrained_dict) 22 | self.model_2d.load_state_dict(model_dict) 23 | else: 24 | raise ValueError('model does not exist: %s' % model_2d_path) 25 | 26 | # 2d-to-3d pose lifting module 27 | self.model_lift = ResNet(mode, num_joints, num_layers, num_features) 28 | if model_lift_path is not None: 29 | if os.path.isfile(model_lift_path): 30 | print('Load pretrained 2D pose estimation model..') 31 | state = torch.load(model_lift_path) 32 | pretrained_dict = state['model'] 33 | model_dict = self.model_lift.state_dict() 34 | new_pretrained_dict = {k[7:]: v for k, v in pretrained_dict.items() if k[7:] in model_dict} 35 | model_dict.update(new_pretrained_dict) 36 | self.model_lift.load_state_dict(model_dict) 37 | else: 38 | raise ValueError('model does not exist: %s' % model_lift_path) 39 | 40 | def forward(self, inp): 41 | [img, bbox, cam_c] = inp 42 | 43 | # 2d prediction 44 | [H, pred2d] = self.model_2d(img) 45 | 46 | # 3d prediction 47 | pose2d = pred2d.clone() 48 | [pose_local, depth_root] = self.model_lift([pose2d, bbox, cam_c]) 49 | 50 | return [H, pred2d, pose_local, depth_root] 51 | 52 | def set_fliptest(self, val): 53 | self.model_2d.set_fliptest(val) 54 | self.model_lift.set_fliptest(val) 55 | 56 | -------------------------------------------------------------------------------- /src/matlab/test_util/test.m: -------------------------------------------------------------------------------- 1 | close all 2 | clear 3 | clc 4 | 5 | % 1: Site -> 11 6 | % 2: Neck -> 9 7 | % 3: RightArm -> 15 8 | % 4: RightForeArm -> 16 9 | % 5: RightHand -> 17 10 | % 6: LeftArm -> 12 11 | % 7: LeftForeArm -> 13 12 | % 8: LeftHand -> 14 13 | % 9: RightUpLeg -> 2 14 | % 10: RightLeg -> 3 15 | % 11: RightFoot -> 4 16 | % 12: LeftUpLeg -> 5 17 | % 13: LeftLeg -> 6 18 | % 14: LeftFoot -> 7 19 | % 15: Hip -> 1 20 | % 16: Spine -> 8 21 | % 17: Head -> 10 22 | 23 | % Mapping table from H36M to MPI 24 | map = [15, 9, 10, 11, 12, 13, 14, 16, 2, 17, 1, 6, 7, 8, 3, 4, 5]; 25 | 26 | % Lines 27 | line_idx = {[11 10 9 8 1], ... 28 | [9 12 13 14], ... 29 | [9 15 16 17], ... 30 | [1 2 3 4], ... 31 | [1 5 6 7]}; 32 | line_color = {[0 1 0], ... 33 | [1 0 0], ... 34 | [0 0 1], ... 35 | [0 0 1], ... 36 | [1 0 0]}; 37 | 38 | id_sequence = 1; 39 | num_sequence = 6; 40 | 41 | annot_name = sprintf('../TS%d/annot_data.mat', id_sequence); 42 | load(annot_name); 43 | 44 | idx = find(valid_frame == 1); 45 | num_valid = length(idx); 46 | 47 | f = (2048/10)*7.320339203; 48 | 49 | id_img = idx(1); 50 | img_name = sprintf('../TS%d/imageSequence/img_%06d.jpg', id_sequence, id_img); 51 | img = imread(img_name); 52 | 53 | w = size(img, 2); 54 | h = size(img, 1); 55 | px = w / 2; 56 | py = h / 2; 57 | 58 | figure; 59 | h1 = imshow(zeros(h, w, 3, 'uint8')); hold on; 60 | h2 = cell(1,5); 61 | for i = 1:5 62 | h2{i} = line(zeros(length(line_idx{i}),1), zeros(length(line_idx{i}),1), ... 63 | 'LineWidth', 4, ... 64 | 'Color', line_color{i}); 65 | end 66 | 67 | for i = 1:num_valid 68 | id_img = idx(i); 69 | img_name = sprintf('../TS%d/imageSequence/img_%06d.jpg', id_sequence, id_img); 70 | img = imread(img_name); 71 | 72 | pose3d = annot3(:,map,1,id_img); 73 | pose2d = annot2(:,map,1,id_img); 74 | 75 | P = [f 0 px; 0 f py; 0 0 1]; 76 | proj = P*pose3d; 77 | proj = proj ./ repmat(proj(3,:),3,1); 78 | 79 | set(h1, 'CData', img); 80 | for j = 1:5 81 | %set(h2{j}, 'XData', squeeze(pose2d(1,line_idx{j})), ... 82 | % 'YData', squeeze(pose2d(2,line_idx{j}))); 83 | set(h2{j}, 'XData', squeeze(proj(1,line_idx{j})), ... 84 | 'YData', squeeze(proj(2,line_idx{j}))); 85 | end 86 | title(sprintf('frame %d/%d', i, num_valid)); 87 | drawnow; 88 | pause(0.03); 89 | end 90 | 91 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_test.m: -------------------------------------------------------------------------------- 1 | close all 2 | clear 3 | clc 4 | 5 | % 1: Site -> 11 6 | % 2: Neck -> 9 7 | % 3: RightArm -> 15 8 | % 4: RightForeArm -> 16 9 | % 5: RightHand -> 17 10 | % 6: LeftArm -> 12 11 | % 7: LeftForeArm -> 13 12 | % 8: LeftHand -> 14 13 | % 9: RightUpLeg -> 2 14 | % 10: RightLeg -> 3 15 | % 11: RightFoot -> 4 16 | % 12: LeftUpLeg -> 5 17 | % 13: LeftLeg -> 6 18 | % 14: LeftFoot -> 7 19 | % 15: Hip -> 1 20 | % 16: Spine -> 8 21 | % 17: Head -> 10 22 | 23 | % Mapping table from H36M to MPI 24 | map = [11, 9, 15, 16, 17, 12, 13, 14, 2, 3, 4, 5, 6, 7, 1, 8, 10]; 25 | 26 | test_subject_id = [1,2,3,4,5,6]; 27 | test_data_path = '/media/juyongchang/2e6123d0-bc51-4f5a-a01b-818048d27c37/MPI_INF_3DHP/test/'; %Change to wherever you put this data. 28 | data_base_path = [test_data_path filesep 'TS']; 29 | 30 | [~,o1,o2,relevant_labels] = mpii_get_joints('relevant'); 31 | 32 | net_base = 'result'; 33 | 34 | % Load results 35 | load('result.mat'); 36 | pred = pred(:,map,:); 37 | pred = pred - pred(:,15,:); 38 | count = 1; 39 | 40 | % 41 | sequencewise_per_joint_error = cell(6,1); 42 | sequencewise_activity_labels = cell(6,1); 43 | for i = 1:length(test_subject_id) 44 | dat = load([data_base_path int2str(test_subject_id(i)) filesep 'annot_data.mat']); 45 | num_test_points = sum(dat.valid_frame(:)); 46 | per_joint_error = zeros(17,1,num_test_points); 47 | pje_idx = 1; 48 | sequencewise_activity_labels{i} = dat.activity_annotation(dat.valid_frame == 1); 49 | 50 | for j = 1:length(dat.valid_frame) 51 | if(dat.valid_frame(j)) 52 | fprintf('Image %d of %d for Test ID %d\n',j, length(dat.valid_frame), test_subject_id(i)); 53 | error = zeros(17,1); 54 | 55 | %img = imread([data_base_path int2str(test_subject_id(i)) filesep 'imageSequence' filesep sprintf('img_%06d.jpg',j)]); 56 | %The GT has 17 joints, and the order and the annotation of the joints can be observed through the 'relevant_labels' variable 57 | P = dat.univ_annot3(:,:,:,j)-repmat(dat.univ_annot3(:,15,:,j),1,17); 58 | 59 | % 60 | pred_p = squeeze(pred(count,:,:))'; %Replace with the actual prediction formatted as 3x17; 61 | count = count + 1; 62 | error_p = (pred_p - P).^2; 63 | error_p = sqrt(sum(error_p, 1)); 64 | error(:,1) = error(:,1) + error_p(:); 65 | 66 | 67 | per_joint_error(:,:,pje_idx) = error; 68 | pje_idx = pje_idx +1; 69 | end 70 | end 71 | sequencewise_per_joint_error{i} = per_joint_error; 72 | 73 | end 74 | 75 | save([net_base filesep 'mpii_3dhp_prediction.mat'], 'sequencewise_per_joint_error', 'sequencewise_activity_labels'); 76 | [seq_table, activity_table] = mpii_evaluate_errors(sequencewise_per_joint_error, sequencewise_activity_labels); 77 | 78 | out_file = [net_base filesep 'mpii_3dhp_evaluation']; 79 | writetable(cell2table(seq_table), [out_file '_sequencewise.csv']); 80 | writetable(cell2table(activity_table), [out_file '_activitywise.csv']); 81 | 82 | -------------------------------------------------------------------------------- /src/models/resnet2d_int.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn as nn 4 | import torchvision.models as models 5 | from models.resnet2d import ResNet 6 | from collections import OrderedDict 7 | import pdb 8 | 9 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 10 | 11 | # modules[0]: conv2d (stride=2, res=128) 12 | # modules[1]: batchnorm 13 | # modules[2]: relu 14 | # modules[3]: maxpool (stride=2, res=64) 15 | # modules[4]: 1st conv block - output: C2 (256 channels) 16 | # modules[5]: 2nd conv block (stride=2, res=32) - output: C3 (512 channels) 17 | # modules[6]: 3rd conv block (stride=2, res=16) - output: C4 (1024 channels) 18 | # modules[7]: 4th conv block (stride=2, res=8) - output: C5 (2048 channels) 19 | # modules[8]: 1st deconv block (res=16) 20 | # modules[9]: 2nd deconv block (res=32) 21 | # modules[10]: 3rd deconv block (res=64) 22 | # modules[11]: regression layer 23 | 24 | class ResNetInt(nn.Module): 25 | def __init__(self, network, num_joints, model_path=None): 26 | super(ResNetInt, self).__init__() 27 | 28 | # heatmap regression module 29 | self.resnet = ResNet(network, num_joints) 30 | 31 | # load pretrained heatmap regression model 32 | if model_path is not None: 33 | if os.path.isfile(model_path): 34 | print('Load pretrained heatmap regression model..') 35 | state = torch.load(model_path) 36 | pretrained_dict = state['model'] 37 | model_dict = self.resnet.state_dict() 38 | new_pretrained_dict = {k[7:]: v for k, v in pretrained_dict.items() if k[7:] in model_dict} 39 | model_dict.update(new_pretrained_dict) 40 | self.resnet.load_state_dict(model_dict) 41 | else: 42 | raise ValueError('model does not exist: %s' % model_path) 43 | 44 | # for integration 45 | self.relu = nn.ReLU(inplace=True) 46 | self.register_buffer('wx', torch.arange(64.0)*4.0+2.0) 47 | self.wx = self.wx.reshape(1,64).repeat(64,1).reshape(64*64,1) 48 | self.register_buffer('wy', torch.arange(64.0)*4.0+2.0) 49 | self.wy = self.wy.reshape(64,1).repeat(1,64).reshape(64*64,1) 50 | 51 | self.network = network 52 | self.num_joints = num_joints 53 | self.fliptest = False 54 | 55 | def forward(self, img): 56 | H = self.resnet(img)[0] 57 | 58 | if self.fliptest == True: 59 | img_flipped = torch.flip(img, [3]) 60 | H_flipped = self.resnet(img_flipped)[0] 61 | H1 = torch.flip(H_flipped, [3]) 62 | index = torch.tensor(flip_index).to(H1.device) 63 | H2 = H1.clone() 64 | H2.index_copy_(1, index, H1) 65 | H = (H + H2) * 0.5 66 | 67 | hmap = H.clone() 68 | num_batch = hmap.shape[0] 69 | hmap = torch.reshape(hmap, (num_batch, self.num_joints, 64*64)) 70 | hmap = self.relu(hmap) # for numerical stability 71 | denom = torch.sum(hmap, 2, keepdim=True).add_(1e-6) # for numerical stability 72 | hmap = hmap / denom 73 | 74 | x = torch.matmul(hmap, self.wx) 75 | y = torch.matmul(hmap, self.wy) 76 | coord = torch.cat((x,y), 2) 77 | 78 | return [H, coord] 79 | 80 | def set_fliptest(self, val): 81 | self.fliptest = val 82 | 83 | -------------------------------------------------------------------------------- /src/utils/logger.py: -------------------------------------------------------------------------------- 1 | # Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514 2 | import os 3 | LOG = True 4 | try: 5 | import tensorflow as tf 6 | except: 7 | LOG = False 8 | import numpy as np 9 | import scipy.misc 10 | try: 11 | from StringIO import StringIO # Python 2.7 12 | except ImportError: 13 | from io import BytesIO # Python 3.x 14 | 15 | 16 | class Logger(object): 17 | 18 | def __init__(self, log_dir, reset=True): 19 | """Create a summary writer logging to log_dir.""" 20 | if LOG: 21 | self.writer = tf.summary.FileWriter(log_dir) 22 | self.f = open(log_dir + '/log.txt', 'w') 23 | else: 24 | if not os.path.exists(log_dir): 25 | os.mkdir(log_dir) 26 | if reset == True: 27 | self.f = open(log_dir + '/log.txt', 'w') 28 | else: 29 | self.f = open(log_dir + '/log.txt', 'a') 30 | 31 | def write(self, txt): 32 | self.f.write(txt) 33 | 34 | def close(self): 35 | self.f.close() 36 | 37 | def scalar_summary(self, tag, value, step): 38 | """Log a scalar variable.""" 39 | if LOG: 40 | summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)]) 41 | self.writer.add_summary(summary, step) 42 | 43 | def image_summary(self, tag, images, step): 44 | """Log a list of images.""" 45 | 46 | img_summaries = [] 47 | for i, img in enumerate(images): 48 | # Write the image to a string 49 | try: 50 | s = StringIO() 51 | except: 52 | s = BytesIO() 53 | scipy.misc.toimage(img).save(s, format="png") 54 | 55 | # Create an Image object 56 | img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), 57 | height=img.shape[0], 58 | width=img.shape[1]) 59 | # Create a Summary value 60 | img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum)) 61 | 62 | # Create and write Summary 63 | summary = tf.Summary(value=img_summaries) 64 | self.writer.add_summary(summary, step) 65 | 66 | def histo_summary(self, tag, values, step, bins=1000): 67 | """Log a histogram of the tensor of values.""" 68 | 69 | # Create a histogram using numpy 70 | counts, bin_edges = np.histogram(values, bins=bins) 71 | 72 | # Fill the fields of the histogram proto 73 | hist = tf.HistogramProto() 74 | hist.min = float(np.min(values)) 75 | hist.max = float(np.max(values)) 76 | hist.num = int(np.prod(values.shape)) 77 | hist.sum = float(np.sum(values)) 78 | hist.sum_squares = float(np.sum(values**2)) 79 | 80 | # Drop the start of the first bin 81 | bin_edges = bin_edges[1:] 82 | 83 | # Add bin edges and counts 84 | for edge in bin_edges: 85 | hist.bucket_limit.append(edge) 86 | for c in counts: 87 | hist.bucket.append(c) 88 | 89 | # Create and write Summary 90 | summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) 91 | self.writer.add_summary(summary, step) 92 | self.writer.flush() 93 | 94 | -------------------------------------------------------------------------------- /src/matlab/test_util/camera_calibration/ts5-6cameras.calib: -------------------------------------------------------------------------------- 1 | tc camera calibration v0.3 2 | camera 0 GOPR0046_intrinsics.MP4 3 | colorCorrection 4 | red 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 5 | green 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 6 | blue 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 7 | frame 0 8 | sensorSize 10.000000000 5.625000000 # in mm 9 | focalLength 8.770747185 # in mm 10 | pixelAspect 0.993236423 # y / x 11 | centerOffset -0.104908645 0.104899704 # in mm (positive values move right and down) 12 | distortionModel OpenCV 13 | distortion -0.276859611 0.131125256 -0.000360494 -0.001149441 -0.049318332 14 | origin -2104.3074 1038.6707 -4596.6367 15 | up 0.025272345 0.995038509 0.096227370 16 | right -0.939647257 -0.009210289 0.342020929 17 | 18 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_get_joints.m: -------------------------------------------------------------------------------- 1 | function [joint_idx, joint_parents_o1, joint_parents_o2, joint_names] = mpii_get_joints(joint_set_name) 2 | 3 | original_joint_idx = [10, 13, 16, 19, 22, 25, 28, 29, 31, 36, 40, 42, 43, 45, 50, 54, 56, 57, 63, 64, 69, 70, 71, 77, 78, 83, 84, 85]; % 4 | 5 | original_joint_names = {'spine3', 'spine4', 'spine2', 'spine1', 'spine', ... %5 6 | 'neck', 'head', 'head_top', 'left_shoulder', 'left_arm', 'left_forearm', ... %11 7 | 'left_hand', 'left_hand_ee', 'right_shoulder', 'right_arm', 'right_forearm', 'right_hand', ... %17 8 | 'right_hand_ee', 'left_leg_up', 'left_leg', 'left_foot', 'left_toe', 'left_ee', ... %23 9 | 'right_leg_up' , 'right_leg', 'right_foot', 'right_toe', 'right_ee'}; 10 | 11 | 12 | all_joint_names = {'spine3', 'spine4', 'spine2', 'spine', 'pelvis', ... %5 13 | 'neck', 'head', 'head_top', 'left_clavicle', 'left_shoulder', 'left_elbow', ... %11 14 | 'left_wrist', 'left_hand', 'right_clavicle', 'right_shoulder', 'right_elbow', 'right_wrist', ... %17 15 | 'right_hand', 'left_hip', 'left_knee', 'left_ankle', 'left_foot', 'left_toe', ... %23 16 | 'right_hip' , 'right_knee', 'right_ankle', 'right_foot', 'right_toe'}; 17 | 18 | 19 | %The O1 and O2 indices are relaive to the joint_idx, regardless of the joint set 20 | 21 | switch joint_set_name 22 | %For internal use only!!! 23 | case 'original' %%These give the original indices from the dumped out mddd file, the remaining joint sets are wrt the 'all' labels 24 | joint_idx = original_joint_idx; % 25 | joint_parents_o1 = [3, 1, 4, 5, 5, 2, 6, 7, 6, 9, 10, 11, 12, 6, 14, 15, 16, 17, 5, 19, 20, 21, 22, 5, 24, 25, 26, 27 ]; 26 | joint_parents_o2 = [4, 3, 5, 5, 5, 1, 2, 6, 2, 6, 9, 10, 11, 2, 6, 14, 15, 16, 4, 5, 19, 20, 21, 4, 5, 24, 25, 26]; 27 | joint_names = original_joint_names; 28 | %Use joint sets from here 29 | case 'all' 30 | joint_idx = 1:28; %These index into the joints extracted in the original set 31 | joint_parents_o1 = [3, 1, 4, 5, 5, 2, 6, 7, 6, 9, 10, 11, 12, 6, 14, 15, 16, 17, 5, 19, 20, 21, 22, 5, 24, 25, 26, 27 ]; 32 | joint_parents_o2 = [4, 3, 5, 5, 5, 1, 2, 6, 2, 6, 9, 10, 11, 2, 6, 14, 15, 16, 4, 5, 19, 20, 21, 4, 5, 24, 25, 26]; 33 | joint_names = all_joint_names; 34 | 35 | case 'cpm' %CPM Joints in CPM Order 36 | joint_idx = [8, 6, 15, 16, 17, 10, 11, 12, 24, 25, 26, 19, 20, 21, 5]; 37 | joint_parents_o1 = [ 2, 15, 2, 3, 4, 2, 6, 7, 15, 9, 10, 15, 12, 13, 15]; 38 | joint_parents_o2 = [15, 15, 15, 2, 3, 15, 2, 6, 2, 15, 9, 2, 15, 12, 15]; 39 | joint_names = all_joint_names(joint_idx); 40 | 41 | case 'relevant' %Human3.6m joints in CPM order 42 | joint_idx = [8, 6, 15, 16, 17, 10, 11, 12, 24, 25, 26, 19, 20, 21, 5, 4, 7]; 43 | joint_parents_o1 = [ 2, 16, 2, 3, 4, 2, 6, 7, 15, 9, 10, 15, 12, 13, 15, 15, 2]; 44 | joint_parents_o2 = [ 16, 15, 16, 2, 3, 16, 2, 6, 16, 15, 9, 16, 15, 12, 15, 15, 16]; 45 | joint_names = all_joint_names(joint_idx); 46 | 47 | otherwise 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_evaluate_errors.m: -------------------------------------------------------------------------------- 1 | function [sequencewise_table, activitywise_table] = mpii_evaluate_errors(sequencewise_error, sequencewise_activity) 2 | 3 | joint_groups = mpii_get_pck_auc_joint_groups(); 4 | [~,~,~,joint_names] = mpii_get_joints('relevant'); 5 | all_errors = []; 6 | all_activities = []; 7 | sequencewise_pck = {}; 8 | sequencewise_auc = {}; 9 | nj = length(joint_names); 10 | sequencewise_mpjpe = cell(length(sequencewise_error)+1,nj+2); 11 | sequencewise_mpjpe(1,2:(nj+1)) = joint_names; 12 | sequencewise_mpjpe{1,(nj+2)} = 'Average'; 13 | %Generate MPJPE and PCK/AUC By sequence first 14 | %error_dat = {}; 15 | %delete('error_dat'); 16 | for i = 1:length(sequencewise_error) 17 | if(isempty(all_errors)) 18 | all_errors = sequencewise_error{i}(:,1,:); 19 | else 20 | all_errors = cat(3,all_errors, sequencewise_error{i}(:,1,:)); 21 | end 22 | all_activities = [all_activities; sequencewise_activity{i}(:)]; 23 | 24 | error_dat(i) = mpii_3D_error(['TestSeq' int2str(i)], sequencewise_error{i}(:,1,:)); 25 | sequencewise_mpjpe{i+1,1}= ['TestSeq' int2str(i)]; 26 | mpjpe = mean(sequencewise_error{i}(:,1,:),3); 27 | sequencewise_mpjpe(i+1,2:(nj+1)) = num2cell(mpjpe'); 28 | sequencewise_mpjpe{i+1,(nj+2)} = mean(mpjpe(:)); 29 | end 30 | [pck, auc] = mpii_compute_3d_pck(error_dat, joint_groups, []); 31 | sequencewise_pck = [sequencewise_pck; pck]; 32 | sequencewise_pck{1,1} = 'PCK'; 33 | sequencewise_auc = [sequencewise_auc; auc]; 34 | sequencewise_auc{1,1} = 'AUC'; 35 | 36 | activitywise_pck = {}; 37 | activitywise_auc = {}; 38 | activitywise_mpjpe = cell(7+2,nj+2); 39 | activitywise_mpjpe(1,2:(nj+1)) = joint_names; 40 | activitywise_mpjpe{1,(nj+2)} = 'Average'; 41 | %Generate MPJPE and PCK/AUC By activity 42 | %error_dat = {}; 43 | clear('error_dat'); 44 | for i = 1:7 45 | error_dat(i) = mpii_3D_error(mpii_get_activity_name(i), all_errors(:,:,all_activities == i)); 46 | activitywise_mpjpe{i+1,1} = mpii_get_activity_name(i); 47 | mpjpe = mean(all_errors(:,:,all_activities == i),3); 48 | activitywise_mpjpe(i+1,2:(nj+1)) = num2cell(mpjpe'); 49 | activitywise_mpjpe{i+1,(nj+2)} = mean(mpjpe(:)); 50 | end 51 | overall_mpjpe = mean(all_errors,3); 52 | activitywise_mpjpe{end,1} = 'All'; 53 | activitywise_mpjpe(end,2:(nj+1)) = num2cell(overall_mpjpe'); 54 | activitywise_mpjpe{end,(nj+2)} = mean(overall_mpjpe(:)); 55 | [pck, auc] = mpii_compute_3d_pck(error_dat, joint_groups, []); 56 | activitywise_pck = [activitywise_pck; pck]; 57 | activitywise_pck{1,1} = 'PCK'; 58 | activitywise_auc = [activitywise_auc; auc]; 59 | activitywise_auc{1,1} = 'AUC'; 60 | clear('error_dat'); 61 | error_dat(1) = mpii_3D_error('All', all_errors); 62 | [pck, auc] = mpii_compute_3d_pck(error_dat, joint_groups, []); 63 | activitywise_pck = [activitywise_pck; pck(2:end,:)]; 64 | activitywise_auc = [activitywise_auc; auc(2:end,:)]; 65 | 66 | sequencewise_table = sequencewise_mpjpe; 67 | sequencewise_table(size(sequencewise_table,1)+1:size(sequencewise_table,1)+size(sequencewise_pck,1),1:size(sequencewise_pck,2)) = sequencewise_pck; 68 | sequencewise_table(size(sequencewise_table,1)+1:size(sequencewise_table,1)+size(sequencewise_auc,1),1:size(sequencewise_auc,2)) = sequencewise_auc; 69 | activitywise_table = activitywise_mpjpe; 70 | activitywise_table(size(activitywise_table,1)+1:size(activitywise_table,1)+size(activitywise_pck,1),1:size(activitywise_pck,2)) = activitywise_pck; 71 | activitywise_table(size(activitywise_table,1)+1:size(activitywise_table,1)+size(activitywise_auc,1),1:size(activitywise_auc,2)) = activitywise_auc; 72 | 73 | 74 | 75 | end -------------------------------------------------------------------------------- /script/table5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # script for table 4 4 | 5 | ########################################################################################################### 6 | # protocol 1 7 | 8 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 9 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 4 -canonical \ 10 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 11 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 12 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 13 | 14 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 15 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 -multi_gpu \ 16 | -network 'resnet152' \ 17 | -opt_method 'rmsprop' -lr 1e-4 \ 18 | -num_epochs 50 -batch_size 48 19 | 20 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 21 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 -multi_gpu \ 22 | -network 'resnet152' -integral 1 -weight1 1e0 \ 23 | -model_2d_path 'test_h36m_protocol1/resnet152/train_h36m_mpii/rmsprop_lr1.0e-04_batch48/final_model.pth' \ 24 | -opt_method 'rmsprop' -lr 1e-5 \ 25 | -num_epochs 10 -batch_size 48 26 | 27 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 28 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 -fliptest \ 29 | -network 'resnet152' \ 30 | -model_2d_path 'test_h36m_protocol1/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 31 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 4 \ 32 | -model_lift_path 'test_h36m_protocol1/resnet3dr-canonical/train_h36m_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 33 | -opt_method 'rmsprop' -lr 1e-5 \ 34 | -num_epochs 0 -batch_size 64 35 | 36 | ########################################################################################################### 37 | # protocol 2 38 | 39 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 40 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 2 -noise 4 -canonical \ 41 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 42 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 43 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 44 | 45 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 46 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 2 -multi_gpu \ 47 | -network 'resnet152' \ 48 | -opt_method 'rmsprop' -lr 1e-4 \ 49 | -num_epochs 50 -batch_size 48 50 | 51 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 52 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 2 -multi_gpu \ 53 | -network 'resnet152' -integral 1 -weight1 1e0 \ 54 | -model_2d_path 'test_h36m_protocol1/resnet152/train_h36m_mpii/rmsprop_lr1.0e-04_batch48/final_model.pth' \ 55 | -opt_method 'rmsprop' -lr 1e-5 \ 56 | -num_epochs 10 -batch_size 48 57 | 58 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 59 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 2 -fliptest \ 60 | -network 'resnet152' \ 61 | -model_2d_path 'test_h36m_protocol2/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 62 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 4 \ 63 | -model_lift_path 'test_h36m_protocol2/resnet3dr-canonical/train_h36m_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 64 | -opt_method 'rmsprop' -lr 1e-5 \ 65 | -num_epochs 0 -batch_size 64 66 | 67 | -------------------------------------------------------------------------------- /src/analysis_error_simple.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from scipy.stats import norm 5 | import matplotlib.pyplot as plt 6 | import pdb 7 | 8 | import conf 9 | 10 | #---------------------------------------------------------------------- 11 | # options 12 | dataset_test = 'h36m' 13 | dataset_train = 'h36m_mpii' 14 | protocol = 0 15 | 16 | # joint names 17 | name_joint = ['Hip', \ 18 | 'RightHip', 'RightKnee', 'RightFoot', \ 19 | 'LeftHip', 'LeftKnee', 'LeftFoot', \ 20 | 'Spine', 'Neck', 'Head', 'Site', \ 21 | 'LeftShoulder', 'LeftElbow', 'LeftHand', \ 22 | 'RightShoulder', 'RightElbow', 'RightHand'] 23 | 24 | # experiment directory 25 | exp_dir = '%s/test_%s_protocol%d/resnet152-int/train_%s/rmsprop_lr1.0e-05_batch48_weight1.0e+00' \ 26 | % (conf.exp_dir, dataset_test, protocol, dataset_train) 27 | 28 | # load result file 29 | filename = '%s/result.pth' % (exp_dir) 30 | result = torch.load(filename) 31 | pred = result['pred2d'].cpu().numpy() 32 | gt = result['gt2d'].cpu().numpy() 33 | 34 | # number of samples 35 | num_sample = pred.shape[0] 36 | 37 | # number of joints 38 | num_joint = pred.shape[1] 39 | 40 | #---------------------------------------------------------------------- 41 | # compute error 42 | error = pred - gt 43 | 44 | # stats 45 | mean = np.zeros((3, num_joint, 2)) 46 | std = np.zeros((3, num_joint, 2)) 47 | 48 | # non-robust estimation (we assume zero mean) 49 | for j in range(num_joint): 50 | for i in range(2): 51 | mean[0, j, i] = np.mean(error[:, j, i]) 52 | std[0, j, i] = np.sqrt(np.mean(error[:, j, i] ** 2.0)) 53 | 54 | # robust estimation of std (we assume zero mean) 55 | std1 = np.zeros((num_joint, 2)) 56 | for j in range(num_joint): 57 | for i in range(2): 58 | mean[1, j, i] = np.median(error[:, j, i]) 59 | e = np.absolute(error[:, j, i] - mean[1, j, i]) 60 | m = np.median(e) 61 | std[1, j, i] = m * 1.4826 62 | 63 | # robust estimation of std (we assume zero mean) 64 | std2 = np.zeros((num_joint, 2)) 65 | for j in range(num_joint): 66 | for i in range(2): 67 | e = error[:, j, i] 68 | q75 = np.percentile(e, 75) 69 | q25 = np.percentile(e, 25) 70 | mean[2, j, i] = np.percentile(e, 50) 71 | std[2, j, i] = 0.7413 * (q75 - q25) 72 | 73 | # save std 74 | print(mean[0]) 75 | print(std[0]) 76 | print(mean[1]) 77 | print(std[1]) 78 | print(mean[2]) 79 | print(std[2]) 80 | mean = torch.from_numpy(mean) 81 | std = torch.from_numpy(std) 82 | stat = {'mean': mean, 'std': std} 83 | filename = '%s/analysis/stat_simple.pth' % (exp_dir) 84 | torch.save(stat, filename) 85 | 86 | #---------------------------------------------------------------------- 87 | # plot error histogram 88 | save_dir = '%s/analysis/stat_simple' % (exp_dir) 89 | if not os.path.exists(save_dir): 90 | os.makedirs(save_dir) 91 | 92 | for j in range(num_joint): 93 | for i in range(2): 94 | e = error[:, j, i] 95 | 96 | plt.clf() 97 | plt.hist(e, bins=100, histtype='stepfilled', range=[-20.0, 20.0], normed=True) 98 | 99 | x_sample = np.linspace(-20, 20, 1000) 100 | plt.plot(x_sample, norm(mean[0, j, i], std[0, j, i]).pdf(x_sample), '-k', lw=1.5, 101 | label='non-robust fit') 102 | plt.plot(x_sample, norm(mean[1, j, i], std[1, j, i]).pdf(x_sample), '--k', lw=1.5, 103 | label='robust fit - mad') 104 | plt.plot(x_sample, norm(mean[2, j, i], std[2, j, i]).pdf(x_sample), ':k', lw=1.5, 105 | label='robust fit - iqr') 106 | plt.legend() 107 | 108 | plt.xlabel('Error (in pixels)') 109 | plt.ylabel('Numbers') 110 | plt.title('Mean(%.2f), Std0(%.2f), Std1(%.2f), Std2(%.2f)' % (e.mean(), std[0, j, i], std[1, j, i], std[2, j, i])) 111 | plt.grid(True) 112 | if i == 0: 113 | plt.savefig('%s/error_%s_x.png' % (save_dir, name_joint[j])) 114 | elif i == 1: 115 | plt.savefig('%s/error_%s_y.png' % (save_dir, name_joint[j])) 116 | 117 | -------------------------------------------------------------------------------- /src/matlab/test_util/mpii_compute_3d_pck.m: -------------------------------------------------------------------------------- 1 | function [pck_table, auc_table] = mpii_compute_3d_pck(error_data, joint_groups, output_base_path) 2 | 3 | %Input 4 | %error_data is a struct array of type mpii_3d_error 5 | %The struct zcarries information about the name of the method as well as an 6 | %nj x 1 x nf matrix with the joint errors. 7 | %joint_groups is an ng x 2 cell, where ng is the number of groups. It 8 | %carries the name of the group as well as the indices of the joints that 9 | %belong to the group. 10 | 11 | %If the error_data array has multiple inputs, there are additional 12 | %comparative AUC plots output per joint in addition to the individual ones. 13 | ng = size(joint_groups,1); 14 | 15 | 16 | pck_curve_array = cell(length(error_data), ng+1); %Contains the PCK results per joint group, per error_data cell 17 | pck_array = cell(length(error_data), ng+1); %Contains the AUC results per joint group 18 | auc_array = cell(length(error_data), ng+1); %Contains the AUC results per joint group 19 | %thresh = 0:5:200; 20 | thresh = 0:5:150; 21 | pck_thresh = 150; 22 | 23 | 24 | for i = 1:length(error_data) 25 | joint_count = 0; 26 | nf = size(error_data(i).error,3); 27 | for j = 1:ng 28 | for ti =1:length(thresh) 29 | t = thresh(ti); 30 | pck_curve_array{i,j} = [pck_curve_array{i,j}, sum(sum(error_data(i).error(joint_groups{j,2},1,:) < t, 3),1) / (length(joint_groups{j,2}) *nf)]; 31 | end 32 | 33 | joint_count = joint_count + length(joint_groups{j,2}); 34 | if(isempty(pck_curve_array{i,ng+1})) 35 | pck_curve_array{i,ng+1} = pck_curve_array{i,j} * length(joint_groups{j,2}); 36 | else 37 | pck_curve_array{i,ng+1} = pck_curve_array{i,ng+1} + pck_curve_array{i,j} * length(joint_groups{j,2}); 38 | end 39 | auc_array{i,j} = 100* sum(pck_curve_array{i,j}(:))/ length(thresh); 40 | pck_array{i,j} = 100* sum(sum(error_data(i).error(joint_groups{j,2},1,:) < pck_thresh, 3),1) / (length(joint_groups{j,2}) *nf); 41 | if(isempty(pck_array{i,ng+1})) 42 | pck_array{i,ng+1} = pck_array{i,j} * length(joint_groups{j,2}); 43 | else 44 | pck_array{i,ng+1} = pck_array{i,ng+1} + pck_array{i,j} * length(joint_groups{j,2}); 45 | end 46 | end 47 | pck_array{i,ng+1} = pck_array{i,ng+1} / joint_count; 48 | pck_curve_array{i,ng+1} = pck_curve_array{i,ng+1} / joint_count; 49 | auc_array{i,ng+1} = 100* sum(pck_curve_array{i,ng+1}(:))/ length(thresh); 50 | end 51 | 52 | pck_table = cell(length(error_data)+1, ng+2); 53 | pck_table{1,ng+2} = 'Total'; 54 | for i = 1:length(error_data) 55 | pck_table{1+i,1} = error_data(i).method; 56 | end 57 | for i = 1:ng 58 | pck_table{1,i+1} = joint_groups{i,1}; 59 | end 60 | auc_table = pck_table; 61 | auc_table(2:end,2:end) = auc_array; 62 | pck_table(2:end,2:end) = pck_array; 63 | 64 | 65 | if(~isempty(output_base_path)) 66 | %Generate and save plots to output_path 67 | %First generate individual plots from each row of the pck_curve_array 68 | colormap default; 69 | 70 | for i = 1:length(error_data) 71 | all_plot = []; 72 | for j = 1:ng+1 73 | figure(1); 74 | cla; 75 | plot(thresh,pck_curve_array{i,j},'LineWidth',2); 76 | all_plot = [all_plot; pck_curve_array{i,j}]; 77 | axis([0 150 0 1]); 78 | title([pck_table{1,j+1} ' PCK150mm']); 79 | output_dir = [output_base_path filesep error_data(i).method]; 80 | if(exist(output_dir,'dir') ~= 7) 81 | mkdir(output_dir); 82 | end 83 | saveas(gcf,[output_dir filesep pck_table{1,j+1}], 'fig'); 84 | saveas(gcf,[output_dir filesep pck_table{1,j+1}], 'svg'); 85 | saveas(gcf,[output_dir filesep pck_table{1,j+1}], 'png'); 86 | 87 | end 88 | figure(2); 89 | cla; 90 | plot(thresh,all_plot,'LineWidth',2); 91 | axis([0 150 0 1]); 92 | hold off; 93 | legend(pck_table(1,2:end)); 94 | saveas(gcf,[output_dir filesep 'All'], 'fig'); 95 | saveas(gcf,[output_dir filesep 'All'], 'svg'); 96 | saveas(gcf,[output_dir filesep 'All'], 'png'); 97 | end 98 | end 99 | 100 | end 101 | %Then group the plots by methods 102 | 103 | -------------------------------------------------------------------------------- /src/models/resnet2d.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.models as models 4 | import pdb 5 | 6 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 7 | 8 | # modules[0]: conv2d (stride=2, res=128) 9 | # modules[1]: batchnorm 10 | # modules[2]: relu 11 | # modules[3]: maxpool (stride=2, res=64) 12 | # modules[4]: 1st conv block - output: C2 (256 channels) 13 | # modules[5]: 2nd conv block (stride=2, res=32) - output: C3 (512 channels) 14 | # modules[6]: 3rd conv block (stride=2, res=16) - output: C4 (1024 channels) 15 | # modules[7]: 4th conv block (stride=2, res=8) - output: C5 (2048 channels) 16 | # modules[8]: 1st deconv block (res=16) 17 | # modules[9]: 2nd deconv block (res=32) 18 | # modules[10]: 3rd deconv block (res=64) 19 | # modules[11]: regression layer 20 | 21 | class ResNet(nn.Module): 22 | def __init__(self, network, num_joints): 23 | super(ResNet, self).__init__() 24 | 25 | # load pretrained resnet model 26 | if network == 'resnet50': 27 | pretrained = models.resnet50(pretrained=True) 28 | elif network == 'resnet101': 29 | pretrained = models.resnet101(pretrained=True) 30 | elif network == 'resnet152': 31 | pretrained = models.resnet152(pretrained=True) 32 | elif network == 'resnext101': 33 | pretrained = models.resnext101_32x8d(pretrained=True) 34 | else: 35 | raise ValueError('unsupported network %s' % network) 36 | 37 | # remove last 2 layers 38 | modules = list(pretrained.children())[:-2] 39 | 40 | # add 1st deconv block (stride = 16) 41 | modules.append(nn.Sequential(nn.ConvTranspose2d(2048, 256, kernel_size=4, stride=2, padding=1, bias=False), 42 | nn.BatchNorm2d(256), 43 | nn.ReLU(inplace=True))) 44 | 45 | # add 2nd deconv block (stride = 32) 46 | modules.append(nn.Sequential(nn.ConvTranspose2d(256, 256, kernel_size=4, stride=2, padding=1, bias=False), 47 | nn.BatchNorm2d(256), 48 | nn.ReLU(inplace=True))) 49 | 50 | # add 3rd deconv block (stride = 64) 51 | modules.append(nn.Sequential(nn.ConvTranspose2d(256, 256, kernel_size=4, stride=2, padding=1, bias=False), 52 | nn.BatchNorm2d(256), 53 | nn.ReLU(inplace=True))) 54 | 55 | # add regression layer 56 | modules.append(nn.Conv2d(256, num_joints, 1)) 57 | 58 | self.module = nn.ModuleList(modules) 59 | 60 | self.network = network 61 | self.num_joints = num_joints 62 | self.fliptest = False 63 | 64 | def forward(self, img): 65 | x = self.module[0](img) 66 | x = self.module[1](x) 67 | x = self.module[2](x) 68 | x = self.module[3](x) 69 | C2 = self.module[4](x) 70 | C3 = self.module[5](C2) 71 | C4 = self.module[6](C3) 72 | C5 = self.module[7](C4) 73 | H = self.module[8](C5) 74 | H = self.module[9](H) 75 | H = self.module[10](H) 76 | H = self.module[11](H) 77 | 78 | if self.fliptest == True: 79 | img_flipped = torch.flip(img, [3]) 80 | x_flipped = self.module[0](img_flipped) 81 | x_flipped = self.module[1](x_flipped) 82 | x_flipped = self.module[2](x_flipped) 83 | x_flipped = self.module[3](x_flipped) 84 | C2_flipped = self.module[4](x_flipped) 85 | C3_flipped = self.module[5](C2_flipped) 86 | C4_flipped = self.module[6](C3_flipped) 87 | C5_flipped = self.module[7](C4_flipped) 88 | H_flipped = self.module[8](C5_flipped) 89 | H_flipped = self.module[9](H_flipped) 90 | H_flipped = self.module[10](H_flipped) 91 | H_flipped = self.module[11](H_flipped) 92 | H1 = torch.flip(H_flipped, [3]) 93 | index = torch.tensor(flip_index).to(H1.device) 94 | H2 = H1.clone() 95 | H2.index_copy_(1, index, H1) 96 | H = (H + H2) * 0.5 97 | 98 | return [H] 99 | 100 | def set_fliptest(self, val): 101 | self.fliptest = val 102 | 103 | -------------------------------------------------------------------------------- /src/matlab/evaluate_result.m: -------------------------------------------------------------------------------- 1 | function [] = evaluate_result(opt) 2 | % Evaluate result using PCK3D and AUC measures 3 | 4 | % Joint for MPI-3DHP-INF 5 | % 1: Site -> 11 6 | % 2: Neck -> 9 7 | % 3: RightArm -> 15 8 | % 4: RightForeArm -> 16 9 | % 5: RightHand -> 17 10 | % 6: LeftArm -> 12 11 | % 7: LeftForeArm -> 13 12 | % 8: LeftHand -> 14 13 | % 9: RightUpLeg -> 2 14 | % 10: RightLeg -> 3 15 | % 11: RightFoot -> 4 16 | % 12: LeftUpLeg -> 5 17 | % 13: LeftLeg -> 6 18 | % 14: LeftFoot -> 7 19 | % 15: Hip -> 1 20 | % 16: Spine -> 8 21 | % 17: Head -> 10 22 | 23 | % Drive name 24 | drivename = '/media/juyongchang/5ea9f10d-ae53-447f-96bc-c7002e535930'; 25 | 26 | % Options 27 | dataset2d = opt.dataset2d; 28 | dataset3d = opt.dataset3d; 29 | canonical = opt.canonical; 30 | mode = 1; 31 | noise = 4; 32 | 33 | % Target directory 34 | target_dir = sprintf('%s/2018_pose/PoseLifter/test_inf/resnet152-lift/train2d_%s_train3d_%s/canonical%d_mode%d_noise%d', ... 35 | drivename, dataset2d, dataset3d, canonical, mode, noise); 36 | 37 | %========================================================================== 38 | % Evaluate code 39 | 40 | % Mapping table from H36M to MPI 41 | map = [11, 9, 15, 16, 17, 12, 13, 14, 2, 3, 4, 5, 6, 7, 1, 8, 10]; 42 | 43 | % Test dataset 44 | test_subject_id = [1,2,3,4,5,6]; 45 | test_data_path = '/media/juyongchang/71098543-c79b-4a69-b5fc-949ebbc749a3/Human_Pose_Estimation/MPI_INF_3DHP/test'; 46 | data_base_path = [test_data_path filesep 'TS']; 47 | 48 | [~,o1,o2,relevant_labels] = mpii_get_joints('relevant'); 49 | 50 | % Load results 51 | target_file = sprintf('%s/result.mat', target_dir); 52 | load(target_file); 53 | pred = pred3d; 54 | pred = pred(:,map,:); 55 | pred = pred - pred(:,15,:); 56 | count = 1; 57 | 58 | % 59 | sequencewise_per_joint_error = cell(6,1); 60 | sequencewise_activity_labels = cell(6,1); 61 | for i = 1:length(test_subject_id) 62 | dat = load([data_base_path int2str(test_subject_id(i)) filesep 'annot_data.mat']); 63 | num_test_points = sum(dat.valid_frame(:)); 64 | per_joint_error = zeros(17,1,num_test_points); 65 | pje_idx = 1; 66 | sequencewise_activity_labels{i} = dat.activity_annotation(dat.valid_frame == 1); 67 | 68 | for j = 1:length(dat.valid_frame) 69 | if(dat.valid_frame(j)) 70 | fprintf('Image %d of %d for Test ID %d\n',j, length(dat.valid_frame), test_subject_id(i)); 71 | error = zeros(17,1); 72 | 73 | % The GT has 17 joints, and the order and the annotation of the joints can be observed through the 'relevant_labels' variable 74 | P = dat.univ_annot3(:,:,:,j)-repmat(dat.univ_annot3(:,15,:,j),1,17); 75 | 76 | % Prediction 77 | pred_p = squeeze(pred(count,:,:))'; 78 | 79 | % Compute error 80 | count = count + 1; 81 | error_p = (pred_p - P).^2; 82 | error_p = sqrt(sum(error_p, 1)); 83 | error(:,1) = error(:,1) + error_p(:); 84 | 85 | % Per joint error 86 | per_joint_error(:,:,pje_idx) = error; 87 | pje_idx = pje_idx +1; 88 | end 89 | end 90 | sequencewise_per_joint_error{i} = per_joint_error; 91 | end 92 | 93 | % Save evaluation results 94 | save([target_dir filesep 'mpii_3dhp_prediction.mat'], 'sequencewise_per_joint_error', 'sequencewise_activity_labels'); 95 | [seq_table, activity_table] = mpii_evaluate_errors(sequencewise_per_joint_error, sequencewise_activity_labels); 96 | 97 | out_file = [target_dir filesep 'mpii_3dhp_evaluation']; 98 | writetable(cell2table(seq_table), [out_file '_sequencewise.csv']); 99 | writetable(cell2table(activity_table), [out_file '_activitywise.csv']); 100 | 101 | % Sequence-wise PCK3D and AUC 102 | PCK3D = zeros(1,6); 103 | AUC = zeros(1,6); 104 | for i = 1:6 105 | PCK3D(i) = seq_table{8+i,10}; 106 | AUC(i) = seq_table{15+i,10}; 107 | end 108 | 109 | % Total PCK3D and AUC 110 | PCK3D_all = activity_table{18,10}; 111 | AUC_all = activity_table{27,10}; 112 | 113 | % Metric for scenes 114 | PCK3D_scene = zeros(1,3); 115 | AUC_scene = zeros(1,3); 116 | PCK3D_scene(1) = (PCK3D(1)*603+PCK3D(2)*540)/(603+540); 117 | PCK3D_scene(2) = (PCK3D(3)*505+PCK3D(4)*553)/(505+553); 118 | PCK3D_scene(3) = (PCK3D(5)*276+PCK3D(6)*452)/(276+452); 119 | AUC_scene(1) = (AUC(1)*603+AUC(2)*540)/(603+540); 120 | AUC_scene(2) = (AUC(3)*505+AUC(4)*553)/(505+553); 121 | AUC_scene(3) = (AUC(5)*276+AUC(6)*452)/(276+452); 122 | 123 | % Print 124 | fid = fopen([target_dir filesep 'mpii_result.txt'], 'w'); 125 | fprintf(fid, 'PCK3D_all: %.2f\n', PCK3D_all); 126 | fprintf(fid, 'PCK3D_scene: %.2f, %.2f, %.2f\n', PCK3D_scene); 127 | fprintf(fid, 'AUC_all: %.2f\n', AUC_all); 128 | fprintf(fid, 'AUC_scene: %.2f, %.2f, %.2f\n', AUC_scene); 129 | fclose(fid); 130 | 131 | -------------------------------------------------------------------------------- /src/datasets_2d/mpii.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | from h5py import File 3 | import conf 4 | import numpy as np 5 | import cv2 6 | from utils.utils import rnd, flip, shuffle_lr 7 | from utils.img import transform, crop, draw_gaussian 8 | import pdb 9 | 10 | class MPII(data.Dataset): 11 | def __init__(self): 12 | print('==> Initializing MPII data') 13 | annot = {} 14 | tags = ['imgname', 'part', 'center', 'scale'] 15 | f1 = File('%s/mpii/annot/%s.h5' % (conf.data_dir, 'train'), 'r') 16 | f2 = File('%s/mpii/annot/%s.h5' % (conf.data_dir, 'val'), 'r') 17 | for tag in tags: 18 | annot[tag] = np.concatenate((np.asarray(f1[tag]).copy(), np.asarray(f2[tag]).copy()), axis=0) 19 | f1.close() 20 | f2.close() 21 | 22 | self.annot = annot 23 | self.num_samples = len(self.annot['scale']) 24 | 25 | print('Load %d MPII samples' % (len(annot['scale']))) 26 | 27 | def load_image(self, index): 28 | imgname = '%s/%s' % (conf.mpii_img_dir, self.annot['imgname'][index].decode('UTF-8')) 29 | img = cv2.imread(imgname) 30 | return img 31 | 32 | def get_part_info(self, index): 33 | pts = self.annot['part'][index].copy() 34 | c = self.annot['center'][index].copy() 35 | s = self.annot['scale'][index] 36 | s = s * 200 37 | return pts, c, s 38 | 39 | def __getitem__(self, index): 40 | # get global constants 41 | num_joints = conf.num_joints 42 | res_in = conf.res_in 43 | res_out = conf.res_out 44 | res_ratio = res_in / res_out 45 | 46 | # get raw data 47 | img = self.load_image(index) 48 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 49 | pts, c, s = self.get_part_info(index) 50 | r = 0 51 | 52 | # data augmentation (scaling and rotation) 53 | s = s * (2 ** rnd(conf.scale)) 54 | r = 0 if np.random.random() < 0.6 else rnd(conf.rotate) 55 | inp = crop(img, c, s, r, res_in) / 255. 56 | 57 | # initialize valid joints 58 | valid2d = np.zeros((num_joints), dtype=np.float32) 59 | 60 | # set output heatmap and 2d pose 61 | pose2d = np.zeros((num_joints, 2), dtype=np.float32) 62 | hmap = np.zeros((int(num_joints), int(res_out), int(res_out)), dtype=np.float32) 63 | for i in range(16): 64 | if (conf.inds[i] != 7 and pts[i][0] > 1): # check whether there is a ground-truth annotation 65 | pt = transform(pts[i], c, s, r, res_in) 66 | pt = pt.astype(np.float32) 67 | if (pt[0] >= 0) and (pt[0] <= res_in-1) and (pt[1] >= 0) and (pt[1] <= res_in-1): 68 | pose2d[conf.inds[i]] = pt 69 | valid2d[conf.inds[i]] = 1.0 70 | hmap[conf.inds[i]] = draw_gaussian(hmap[conf.inds[i]], pt/res_ratio+0.5, conf.std) 71 | 72 | # data augmentation (flipping and jittering) 73 | if np.random.random() < 0.5: 74 | inp = flip(inp) 75 | hmap = shuffle_lr(flip(hmap)) 76 | pose2d = shuffle_lr(pose2d) 77 | for i in range(num_joints): 78 | if pose2d[i][0] > 1: 79 | pose2d[i][0] = res_in - pose2d[i][0] - 1 80 | inp[0] = np.clip(inp[0] * (np.random.random() * .4 + .6), 0, 1) 81 | inp[1] = np.clip(inp[1] * (np.random.random() * .4 + .6), 0, 1) 82 | inp[2] = np.clip(inp[2] * (np.random.random() * .4 + .6), 0, 1) 83 | 84 | # 3d and camera information 85 | valid3d = 0.0 86 | bbox = np.array([0.0, 0.0, 255.0, 255.0], dtype=np.int32) 87 | pose3d = np.zeros((num_joints-1, 3), dtype=np.float32) 88 | cam_f = np.array([1.0, 1.0], dtype=np.float32) 89 | cam_c = np.array([0.0, 0.0], dtype=np.float32) 90 | meta3d = np.zeros((num_joints, 3), dtype=np.float32) 91 | action = 0 92 | coords_root = np.zeros((3), dtype=np.float32) 93 | depth_root = 0. 94 | depth_root_canonical = 0. 95 | 96 | # set data 97 | data = {'inp': inp, 'bbox': bbox, 98 | 'hmap': hmap, 99 | 'pose2d': pose2d, 'valid2d': valid2d, 100 | 'pose3d': pose3d, 'valid3d': valid3d, 101 | 'cam_f': cam_f, 'cam_c': cam_c, 102 | 'meta3d': meta3d, 'action': action, 103 | 'coords_root': coords_root, 104 | 'depth_root': depth_root, 105 | 'depth_root_canonical': depth_root_canonical 106 | } 107 | 108 | # return input image, output heatmap and 2d pose 109 | return index, data 110 | 111 | def __len__(self): 112 | return self.num_samples 113 | 114 | 115 | -------------------------------------------------------------------------------- /script/table3_2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # script for table 2 4 | 5 | ########################################################################################################### 6 | # noise=0: use GT for learning PoseLifter 7 | 8 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 9 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 0 -canonical \ 10 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 11 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 12 | 13 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 14 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 \ 15 | -network 'resnet152' \ 16 | -model_2d_path 'test_h36m_protocol1/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 17 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 0 \ 18 | -model_lift_path 'test_h36m_protocol1/resnet3dr-canonical/train_h36m_noise0/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 19 | -opt_method 'rmsprop' -lr 1e-5 \ 20 | -num_epochs 0 -batch_size 64 21 | 22 | ########################################################################################################### 23 | # noise=2: use 2D estimate for learning PoseLifter 24 | 25 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 26 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 -multi_gpu -save_results_train \ 27 | -network 'resnet152' -integral 1 -weight1 1e0 \ 28 | -model_2d_path 'test_h36m_protocol1/resnet152/train_h36m_mpii/rmsprop_lr1.0e-04_batch48/final_model.pth' \ 29 | -opt_method 'rmsprop' -lr 1e-5 \ 30 | -num_epochs 10 -batch_size 48 31 | 32 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 33 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 2 -canonical \ 34 | -noise_path 'test_h36m_protocol1/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/test_train.pth' \ 35 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 36 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 37 | 38 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 39 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 \ 40 | -network 'resnet152' \ 41 | -model_2d_path 'test_h36m_protocol1/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 42 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 2 \ 43 | -model_lift_path 'test_h36m_protocol1/resnet3dr-canonical/train_h36m_noise2/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 44 | -opt_method 'rmsprop' -lr 1e-5 \ 45 | -num_epochs 0 -batch_size 64 46 | 47 | ########################################################################################################### 48 | # noise=3: use single Gaussian noise for learning PoseLifter 49 | 50 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 51 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 3 -canonical \ 52 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_simple.pth' \ 53 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 54 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 55 | 56 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 57 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 \ 58 | -network 'resnet152' \ 59 | -model_2d_path 'test_h36m_protocol1/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 60 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 3 \ 61 | -model_lift_path 'test_h36m_protocol1/resnet3dr-canonical/train_h36m_noise3/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 62 | -opt_method 'rmsprop' -lr 1e-5 \ 63 | -num_epochs 0 -batch_size 64 64 | 65 | ########################################################################################################### 66 | # noise=4: use proposed mixture noise for learning PoseLifter 67 | 68 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 69 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 4 -canonical \ 70 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 71 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 72 | -opt_method rmsprop -lr 1e-3 -num_epochs 300 -batch_size 64 73 | 74 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 75 | -dataset_test 'h36m' -dataset_train 'h36m_mpii' -protocol 1 \ 76 | -network 'resnet152' \ 77 | -model_2d_path 'test_h36m_protocol1/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 78 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 4 \ 79 | -model_lift_path 'test_h36m_protocol1/resnet3dr-canonical/train_h36m_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 80 | -opt_method 'rmsprop' -lr 1e-5 \ 81 | -num_epochs 0 -batch_size 64 82 | 83 | -------------------------------------------------------------------------------- /src/utils/img.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import cv2 4 | import pdb 5 | 6 | # Transform from original image to (res x res) target image, 7 | # which is defined by center, scale, and rot 8 | def get_transform(center, scale, rot, res): 9 | h = scale 10 | t = np.eye(3) 11 | 12 | t[0, 0] = res / h 13 | t[1, 1] = res / h 14 | t[0, 2] = res * (-center[0] / h + 0.5) 15 | t[1, 2] = res * (-center[1] / h + 0.5) 16 | 17 | if rot != 0: 18 | rot = -rot 19 | r = np.eye(3) 20 | ang = rot * np.math.pi / 180 21 | s = np.math.sin(ang) 22 | c = np.math.cos(ang) 23 | r[0, 0] = c 24 | r[0, 1] = -s 25 | r[1, 0] = s 26 | r[1, 1] = c 27 | t_ = np.eye(3) 28 | t_[0, 2] = -res / 2 29 | t_[1, 2] = -res / 2 30 | t_inv = np.eye(3) 31 | t_inv[0, 2] = res / 2 32 | t_inv[1, 2] = res / 2 33 | t = np.dot(np.dot(np.dot(t_inv, r), t_), t) 34 | 35 | return t 36 | 37 | def transform(pt, center, scale, rot, res, invert = False): 38 | pt_ = np.ones(3) 39 | pt_[0], pt_[1] = pt[0], pt[1] 40 | 41 | t = get_transform(center, scale, rot, res) 42 | if invert: 43 | t = np.linalg.inv(t) 44 | new_point = np.dot(t, pt_)[:2] 45 | new_point = new_point.astype(np.int32) 46 | return new_point 47 | 48 | def crop(img, center, scale, rot, res): 49 | ht, wd = img.shape[0], img.shape[1] 50 | tmpImg, newImg = img.copy(), np.zeros((res, res, 3), dtype = np.uint8) 51 | 52 | scaleFactor = scale / res 53 | if scaleFactor < 2: 54 | scaleFactor = 1 55 | else: 56 | newSize = int(np.math.floor(max(ht, wd) / scaleFactor)) 57 | newSize_ht = int(np.math.floor(ht / scaleFactor)) 58 | newSize_wd = int(np.math.floor(wd / scaleFactor)) 59 | if newSize < 2: 60 | return torch.from_numpy(newImg.transpose(2, 0, 1).astype(np.float32) / 256.) 61 | else: 62 | tmpImg = cv2.resize(tmpImg, (newSize_wd, newSize_ht)) 63 | ht, wd = tmpImg.shape[0], tmpImg.shape[1] 64 | 65 | c, s = 1.0 * center / scaleFactor, scale / scaleFactor 66 | c[0], c[1] = c[1], c[0] 67 | ul = transform((0, 0), c, s, 0, res, invert = True) 68 | br = transform((res, res), c, s, 0, res, invert = True) 69 | 70 | if scaleFactor >= 2: 71 | br = br - (br - ul - res) 72 | 73 | pad = int(np.math.ceil((((ul - br) ** 2).sum() ** 0.5) / 2 - (br[0] - ul[0]) / 2)) 74 | if rot != 0: 75 | ul = ul - pad 76 | br = br + pad 77 | 78 | old_ = [max(0, ul[0]), min(br[0], ht), max(0, ul[1]), min(br[1], wd)] 79 | new_ = [max(0, -ul[0]), min(br[0], ht) - ul[0], max(0, -ul[1]), min(br[1], wd) - ul[1]] 80 | 81 | newImg = np.zeros((br[0] - ul[0], br[1] - ul[1], 3), dtype = np.uint8) 82 | try: 83 | newImg[new_[0]:new_[1], new_[2]:new_[3], :] = tmpImg[old_[0]:old_[1], old_[2]:old_[3], :] 84 | except: 85 | return np.zeros((3, res, res), np.uint8) 86 | if rot != 0: 87 | M = cv2.getRotationMatrix2D((newImg.shape[0] / 2, newImg.shape[1] / 2), rot, 1) 88 | newImg = cv2.warpAffine(newImg, M, (newImg.shape[0], newImg.shape[1])) 89 | newImg = newImg[pad+1:-pad+1, pad+1:-pad+1, :].copy() 90 | 91 | if scaleFactor < 2: 92 | newImg = cv2.resize(newImg, (res, res)) 93 | 94 | return newImg.transpose(2, 0, 1).astype(np.float32) 95 | 96 | def gaussian(width): 97 | if width == 7: 98 | return np.array([0.0529, 0.1197, 0.1954, 0.2301, 0.1954, 0.1197, 0.0529, 99 | 0.1197, 0.2709, 0.4421, 0.5205, 0.4421, 0.2709, 0.1197, 100 | 0.1954, 0.4421, 0.7214, 0.8494, 0.7214, 0.4421, 0.1954, 101 | 0.2301, 0.5205, 0.8494, 1.0000, 0.8494, 0.5205, 0.2301, 102 | 0.1954, 0.4421, 0.7214, 0.8494, 0.7214, 0.4421, 0.1954, 103 | 0.1197, 0.2709, 0.4421, 0.5205, 0.4421, 0.2709, 0.1197, 104 | 0.0529, 0.1197, 0.1954, 0.2301, 0.1954, 0.1197, 0.0529]).reshape(7, 7) 105 | else: 106 | raise Exception('Gaussian with width {} not implemented!!'.format(width)) 107 | 108 | def draw_gaussian(img, pt, sigma): 109 | tmpSize = int(np.math.ceil(3 * sigma)) 110 | ul = [int(np.math.floor(pt[0] - tmpSize)), int(np.math.floor(pt[1] - tmpSize))] 111 | br = [int(np.math.floor(pt[0] + tmpSize)), int(np.math.floor(pt[1] + tmpSize))] 112 | 113 | if ul[0] > img.shape[1]-1 or ul[1] > img.shape[0]-1 or br[0] < 0 or br[1] < 0: 114 | return img 115 | 116 | size = 2 * tmpSize + 1 117 | g = gaussian(size) 118 | 119 | g_x = [max(0, -ul[0]), min(br[0], img.shape[1]-1) - max(0, ul[0]) + max(0, -ul[0]) + 1] 120 | g_y = [max(0, -ul[1]), min(br[1], img.shape[0]-1) - max(0, ul[1]) + max(0, -ul[1]) + 1] 121 | 122 | img_x = [max(0, ul[0]), min(br[0], img.shape[1]-1) + 1] 123 | img_y = [max(0, ul[1]), min(br[1], img.shape[0]-1) + 1] 124 | 125 | img[img_y[0]:img_y[1], img_x[0]:img_x[1]] = g[g_y[0]:g_y[1], g_x[0]:g_x[1]] 126 | 127 | return img 128 | 129 | -------------------------------------------------------------------------------- /script/table6.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # script for table 5 4 | 5 | ########################################################################################################### 6 | # 2D pose estimation 7 | 8 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 9 | -dataset_test 'inf' -dataset_train 'inf_mpii' -multi_gpu \ 10 | -network 'resnet152' \ 11 | -opt_method 'rmsprop' -lr 1e-4 \ 12 | -num_epochs 50 -batch_size 48 13 | 14 | CUDA_VISIBLE_DEVICES=1,2,3 python ./src/main_2d.py \ 15 | -dataset_test 'inf' -dataset_train 'inf_mpii' -multi_gpu \ 16 | -network 'resnet152' -integral 1 -weight1 1e0 \ 17 | -model_2d_path 'test_inf/resnet152/train_inf_mpii/rmsprop_lr1.0e-04_batch48/final_model.pth' \ 18 | -opt_method 'rmsprop' -lr 1e-5 \ 19 | -num_epochs 10 -batch_size 48 20 | 21 | ########################################################################################################### 22 | # 3d dataset: H36M 23 | # w/o canonical depth 24 | 25 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 26 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 4 -scale \ 27 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 28 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 29 | -opt_method 'rmsprop' -lr 1e-3 -num_epochs 300 -batch_size 64 30 | 31 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 32 | -dataset_test 'inf' -dataset_train 'inf_mpii' -fliptest -save_results \ 33 | -network 'resnet152' \ 34 | -model_2d_path 'test_inf/resnet152-int/train_inf_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 35 | -lift3d 1 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 4 \ 36 | -model_lift_path 'test_h36m_protocol1/resnet3dr/train_h36m_scale_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 37 | -opt_method 'rmsprop' -lr 1e-5 \ 38 | -num_epochs 0 -batch_size 64 39 | 40 | ########################################################################################################### 41 | # 3d dataset: H36M 42 | # w/ canonical depth 43 | 44 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 45 | -dataset_test 'h36m' -dataset_train 'h36m' -protocol 1 -noise 4 -canonical -scale \ 46 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 47 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 48 | -opt_method 'rmsprop' -lr 1e-3 -num_epochs 300 -batch_size 64 49 | 50 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 51 | -dataset_test 'inf' -dataset_train 'inf_mpii' -fliptest -save_results \ 52 | -network 'resnet152' \ 53 | -model_2d_path 'test_inf/resnet152-int/train_inf_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 54 | -lift3d 2 -dataset3d 'h36m' -mode 1 -num_features 4096 -noise 4 \ 55 | -model_lift_path 'test_h36m_protocol1/resnet3dr-canonical/train_h36m_scale_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 56 | -opt_method 'rmsprop' -lr 1e-5 \ 57 | -num_epochs 0 -batch_size 64 58 | 59 | ########################################################################################################### 60 | # 3d dataset: INF 61 | # w/o canonical depth 62 | 63 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 64 | -dataset_test 'inf' -dataset_train 'inf' -noise 4 \ 65 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 66 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 67 | -opt_method 'rmsprop' -lr 1e-3 -num_epochs 300 -batch_size 64 68 | 69 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 70 | -dataset_test 'inf' -dataset_train 'inf_mpii' -fliptest -save_results \ 71 | -network 'resnet152' \ 72 | -model_2d_path 'test_inf/resnet152-int/train_inf_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 73 | -lift3d 1 -dataset3d 'inf' -mode 1 -num_features 4096 -noise 4 \ 74 | -model_lift_path 'test_inf/resnet3dr/train_inf_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 75 | -opt_method 'rmsprop' -lr 1e-5 \ 76 | -num_epochs 0 -batch_size 64 77 | 78 | ########################################################################################################### 79 | # 3d dataset: INF 80 | # w/ canonical depth 81 | 82 | CUDA_VISIBLE_DEVICES=2 python ./src/main_lift.py \ 83 | -dataset_test 'inf' -dataset_train 'inf' -noise 4 -canonical \ 84 | -noise_path 'test_h36m_protocol0/resnet152-int/train_h36m_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/analysis/stat_2d_em1.pth' \ 85 | -network 'resnet' -mode 1 -num_layers 2 -num_features 4096 \ 86 | -opt_method 'rmsprop' -lr 1e-3 -num_epochs 300 -batch_size 64 87 | 88 | CUDA_VISIBLE_DEVICES=2 python ./src/main_2d.py \ 89 | -dataset_test 'inf' -dataset_train 'inf_mpii' -fliptest -save_results \ 90 | -network 'resnet152' \ 91 | -model_2d_path 'test_inf/resnet152-int/train_inf_mpii/rmsprop_lr1.0e-05_batch48_weight1.0e+00/final_model.pth' \ 92 | -lift3d 2 -dataset3d 'inf' -mode 1 -num_features 4096 -noise 4 \ 93 | -model_lift_path 'test_inf/resnet3dr-canonical/train_inf_noise4/mode1_nLayer2_nFeat4096_rmsprop_lr1.0e-03_batch64/final_model.pth' \ 94 | -opt_method 'rmsprop' -lr 1e-5 \ 95 | -num_epochs 0 -batch_size 64 96 | 97 | -------------------------------------------------------------------------------- /src/analysis_error_mixture.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | from scipy.stats import norm 5 | from scipy.optimize import minimize 6 | import matplotlib 7 | import matplotlib.pyplot as plt 8 | import pdb 9 | 10 | import conf 11 | 12 | #---------------------------------------------------------------------- 13 | # options 14 | dataset_test = 'h36m' 15 | dataset_train = 'h36m_mpii' 16 | protocol = 0 17 | 18 | # joint names 19 | name_joint = ['Hip', \ 20 | 'RightHip', 'RightKnee', 'RightFoot', \ 21 | 'LeftHip', 'LeftKnee', 'LeftFoot', \ 22 | 'Spine', 'Neck', 'Head', 'Site', \ 23 | 'LeftShoulder', 'LeftElbow', 'LeftHand', \ 24 | 'RightShoulder', 'RightElbow', 'RightHand'] 25 | 26 | # experiment directory 27 | exp_dir = '%s/test_%s_protocol%d/resnet152-int/train_%s/rmsprop_lr1.0e-05_batch48_weight1.0e+00' \ 28 | % (conf.exp_dir, dataset_test, protocol, dataset_train) 29 | 30 | # load result file 31 | filename = '%s/result.pth' % (exp_dir) 32 | result = torch.load(filename) 33 | pred = result['pred2d'].cpu().numpy() 34 | gt = result['gt2d'].cpu().numpy() 35 | 36 | # number of samples 37 | num_sample = pred.shape[0] 38 | 39 | # number of joints 40 | num_joint = pred.shape[1] 41 | 42 | # set font 43 | font = {'family' : 'normal', 44 | 'size' : 14} 45 | 46 | matplotlib.rc('font', **font) 47 | 48 | #---------------------------------------------------------------------- 49 | # compute error 50 | error = pred - gt 51 | 52 | # create save directory 53 | save_dir = '%s/analysis/stat_2d_em1' % (exp_dir) 54 | if not os.path.exists(save_dir): 55 | os.makedirs(save_dir) 56 | 57 | #---------------------------------------------------------------------- 58 | # parameter estimation using EM 59 | mean = np.zeros((num_joint, 2)) 60 | std = np.zeros((num_joint, 2)) 61 | weight = np.zeros((num_joint)) 62 | for j in range(num_joint): 63 | e = error[:, j, :] 64 | 65 | # non-robust estimation of std (we assume zero mean) 66 | m1 = np.mean(e, axis=0) 67 | s1 = np.sqrt(np.mean((e-m1)**2.0, axis=0)) 68 | 69 | # robust estimation of std (we assume zero mean) 70 | q75 = np.percentile(e, 75, axis=0) 71 | q25 = np.percentile(e, 25, axis=0) 72 | m2 = np.percentile(e, 50, axis=0) 73 | s2 = 0.7413 * (q75 - q25) 74 | 75 | # initial estimate 76 | m = m2 77 | s = s2 78 | w = 0.5 79 | print(m, s, w) 80 | 81 | # NLL log 82 | NLL = np.zeros((10)) 83 | 84 | for k in range(10): 85 | # E-step: compute responsibility 86 | p1 = w * np.exp(-np.sum(((e-m)**2.0)/(2*s**2.0), axis=1, keepdims=True))/(2*np.pi*s[0]*s[1]) 87 | p2 = (1.0-w) * 1/10000.0 88 | r1 = p1 / (p1 + p2) 89 | r2 = p2 / (p1 + p2) 90 | 91 | # M-step: re-estimate the parameters 92 | m = np.sum(r1 * e, axis=0) / np.sum(r1) 93 | s = np.sqrt(np.sum(r1*(e-m)**2.0, axis=0)/np.sum(r1)) 94 | w = np.mean(r1) 95 | 96 | # compute NLL 97 | nll = -np.mean(np.log(p1+p2)) 98 | NLL[k] = nll 99 | 100 | # 101 | print(m, s, w, nll) 102 | 103 | # 104 | mean[j] = m 105 | std[j] = s 106 | weight[j] = w 107 | 108 | # 109 | tick_x_val = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 110 | tick_x_lab = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] 111 | plt.clf() 112 | plt.plot(np.linspace(1,10,10), NLL, '-s', lw=1.5, markeredgecolor='k', markerfacecolor='r') 113 | plt.xlabel('Iteration') 114 | plt.ylabel('Negative Log Likelihood') 115 | plt.title('%s' % (name_joint[j])) 116 | plt.xticks(tick_x_val, tick_x_lab) 117 | plt.grid(True) 118 | plt.savefig('%s/em_%s.pdf' % (save_dir, name_joint[j])) 119 | 120 | # 121 | for i in range(2): 122 | pdf = lambda x: np.exp(-((x-m[i])**2.0)/(2*s[i]**2.0))*w/np.sqrt(2*np.pi*s[i]**2.0)+(1-w)/100.0 123 | 124 | tick_x_val = [-20, -10, 0, 10, 20] 125 | tick_x_lab = ['-20', '-10', '0', '10', '20'] 126 | 127 | plt.clf() 128 | plt.hist(e[:, i], bins=100, histtype='stepfilled', range=[-20.0, 20.0], normed=True) 129 | 130 | x_sample = np.linspace(-20, 20, 1000) 131 | plt.plot(x_sample, norm(m1[i], s1[i]).pdf(x_sample), '--g', lw=1.5, 132 | label='Gaussian Fit') 133 | #plt.plot(x_sample, norm(m2[i], s2[i]).pdf(x_sample), ':k', lw=1.5, 134 | # label='robust gaussian fit') 135 | plt.plot(x_sample, pdf(x_sample), '-r', lw=1.5, 136 | label='Mixture Fit') 137 | plt.legend() 138 | 139 | plt.xlabel('Error (in pixels)') 140 | plt.ylabel('Probability') 141 | if i == 0: 142 | plt.title('%s (x)' % (name_joint[j])) 143 | elif i == 1: 144 | plt.title('%s (y)' % (name_joint[j])) 145 | plt.xticks(tick_x_val, tick_x_lab) 146 | plt.grid(True) 147 | if i == 0: 148 | plt.savefig('%s/error_%s_x.pdf' % (save_dir, name_joint[j])) 149 | elif i == 1: 150 | plt.savefig('%s/error_%s_y.pdf' % (save_dir, name_joint[j])) 151 | 152 | # save results 153 | print(mean) 154 | print(std) 155 | print(weight) 156 | mean = torch.from_numpy(mean) 157 | std = torch.from_numpy(std) 158 | weight = torch.from_numpy(weight) 159 | stat = {'mean': mean, 'std': std, 'weight': weight} 160 | filename = '%s/analysis/stat_2d_em1.pth' % (exp_dir) 161 | torch.save(stat, filename) 162 | 163 | # save txt results 164 | file = open('%s/analysis/stat_2d_em1.txt' % (exp_dir), 'w') 165 | for i in range(num_joint): 166 | file.write('============================\n') 167 | file.write('Joint: %s\n' % (name_joint[i])) 168 | file.write('mean = (%.02f, %.02f)\n' % (mean[i,0], mean[i,1])) 169 | file.write('std = (%.02f, %.02f)\n' % (std[i,0], std[i,1])) 170 | file.write('weight = (%.02f)\n' % (weight[i])) 171 | file.close() 172 | 173 | -------------------------------------------------------------------------------- /src/opts_lift.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import conf 4 | import pdb 5 | 6 | class Opts(): 7 | def __init__(self): 8 | self.parser = argparse.ArgumentParser() 9 | 10 | def init(self): 11 | # miscellaneous 12 | self.parser.add_argument('-dataset_test', default='h36m', help='Test dataset') 13 | self.parser.add_argument('-dataset_train', default='h36m', help='Training dataset') 14 | self.parser.add_argument('-protocol', type=int, default=1, help='Experiment protocol for H36M: 0 | 1 | 2') 15 | self.parser.add_argument('-multi_gpu', default=False, action='store_true', help='Use multiple gpus?') 16 | self.parser.add_argument('-noise', type=int, default=0, help='Noise mode') 17 | self.parser.add_argument('-noise_path', default=None, help='Path to noise info') 18 | self.parser.add_argument('-std_train', type=float, default=0.0, help='Std of Gaussian noise for robust training') 19 | self.parser.add_argument('-std_test', type=float, default=0.0, help='Std of Gaussian noise for testing') 20 | self.parser.add_argument('-canonical', default=False, action='store_true', help='Use canonical coordinate for root?') 21 | self.parser.add_argument('-scale', default=False, action='store_true', help='Induce random scaling for data augmentation?') 22 | self.parser.add_argument('-fliptest', default=False, action='store_true', help='Do flip test?') 23 | self.parser.add_argument('-analysis', default=False, action='store_true', help='Analyze results?') 24 | 25 | # network structure 26 | self.parser.add_argument('-network', default='resnet', help='Network to use: resnet') 27 | self.parser.add_argument('-mode', type=int, default='1', help='Use location and scale info?') 28 | self.parser.add_argument('-num_layers', type=int, default=2, help='Number of hidden layers') 29 | self.parser.add_argument('-num_features', type=int, default=4096, help='Number of features') 30 | self.parser.add_argument('-weight_root', type=float, default=1.0, help='Weight for root loss') 31 | 32 | # optimization 33 | self.parser.add_argument('-opt_method', default='rmsprop', help='Optimization method: rmsprop | adam') 34 | self.parser.add_argument('-lr', type=float, default=1e-3, help='Learning rate') 35 | self.parser.add_argument('-alpha', type=float, default=0.99, help='Smoothing constant') 36 | self.parser.add_argument('-epsilon', type=float, default=1e-8, help='For numerical stability') 37 | self.parser.add_argument('-weight_decay', type=float, default=0, help='Weight decay') 38 | self.parser.add_argument('-lr_decay', type=float, default=0, help='Learning rate decay') 39 | self.parser.add_argument('-beta1', type=float, default=0.9, help='First mement coefficient') 40 | self.parser.add_argument('-beta2', type=float, default=0.99, help='Second moment coefficient') 41 | self.parser.add_argument('-momentum', type=float, default=0, help='Momentum') 42 | 43 | # training options 44 | self.parser.add_argument('-num_epochs', type=int, default=200, help='Number of training epochs') 45 | self.parser.add_argument('-batch_size', type=int, default=512, help='Mini-batch size') 46 | self.parser.add_argument('-save_intervals', type=int, default=50, help='Number of iterations for saving model') 47 | 48 | def parse(self): 49 | self.init() 50 | self.opt = self.parser.parse_args() 51 | 52 | # set directories for experiments 53 | if self.opt.dataset_test == 'h36m': 54 | self.opt.save_dir = '%s/test_%s_protocol%d' % (conf.exp_dir, self.opt.dataset_test, self.opt.protocol) 55 | elif self.opt.dataset_test == 'inf': 56 | self.opt.save_dir = '%s/test_%s' % (conf.exp_dir, self.opt.dataset_test) 57 | if not os.path.exists(self.opt.save_dir): 58 | os.makedirs(self.opt.save_dir) 59 | 60 | if self.opt.canonical == False: 61 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s3dr' % self.opt.network) 62 | else: 63 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s3dr-canonical' % self.opt.network) 64 | if not os.path.exists(self.opt.save_dir): 65 | os.makedirs(self.opt.save_dir) 66 | 67 | if self.opt.noise != 1: 68 | if self.opt.scale == False: 69 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train_%s_noise%d' % (self.opt.dataset_train, self.opt.noise)) 70 | else: 71 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train_%s_scale_noise%d' % (self.opt.dataset_train, self.opt.noise)) 72 | else: 73 | if self.opt.scale == False: 74 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train_%s_noise%d_std%.3f' % (self.opt.dataset_train, self.opt.noise, self.opt.std_train)) 75 | else: 76 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train_%s_scale_noise%d_std%.3f' % (self.opt.dataset_train, self.opt.noise, self.opt.std_train)) 77 | if not os.path.exists(self.opt.save_dir): 78 | os.makedirs(self.opt.save_dir) 79 | 80 | self.opt.save_dir = '%s/mode%d_nLayer%d_nFeat%d_%s_lr%1.1e_batch%d' % (self.opt.save_dir, self.opt.mode, self.opt.num_layers, self.opt.num_features, self.opt.opt_method, self.opt.lr, self.opt.batch_size) 81 | if not os.path.exists(self.opt.save_dir): 82 | os.makedirs(self.opt.save_dir) 83 | 84 | # save options 85 | args = dict((name, getattr(self.opt, name)) for name in dir(self.opt) 86 | if not name.startswith('_')) 87 | refs = dict((name, getattr(conf, name)) for name in dir(conf) 88 | if not name.startswith('_')) 89 | file_name = os.path.join(self.opt.save_dir, 'opt.txt') 90 | with open(file_name, 'wt') as opt_file: 91 | opt_file.write('==> Args:\n') 92 | for k, v in sorted(args.items()): 93 | opt_file.write(' %s: %s\n' % (str(k), str(v))) 94 | opt_file.write('==> Args:\n') 95 | for k, v in sorted(refs.items()): 96 | opt_file.write(' %s: %s\n' % (str(k), str(v))) 97 | 98 | return self.opt 99 | 100 | -------------------------------------------------------------------------------- /src/datasets_2d/mpiinf.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data as data 3 | from h5py import File 4 | import conf 5 | import numpy as np 6 | import cv2 7 | from utils.utils import rnd, flip, shuffle_lr 8 | from utils.img import transform, crop, draw_gaussian 9 | import pdb 10 | 11 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 12 | 13 | class MPIINF(data.Dataset): 14 | def __init__(self, split): 15 | print('==> Initializing MPI_INF %s data' % (split)) 16 | 17 | annot = {} 18 | tags = ['idx', 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'sequence', 'video'] 19 | f = File('%s/inf/inf_%s.h5' % (conf.data_dir, split), 'r') 20 | for tag in tags: 21 | annot[tag] = np.asarray(f[tag]).copy() 22 | f.close() 23 | 24 | self.split = split 25 | self.annot = annot 26 | self.num_samples = self.annot['pose2d'].shape[0] 27 | 28 | print('Load %d MPI_INF %s samples' % (self.num_samples, self.split)) 29 | 30 | def load_image(self, index): 31 | if self.split == 'train': 32 | dirname = 'subject_%d_sequence_%d_video_%d' % (self.annot['subject'][index], self.annot['sequence'][index], self.annot['video'][index]) 33 | imgname = '%s/train/%s/%s_%06d.jpg' % (conf.inf_img_dir, dirname, dirname, self.annot['idx'][index]) 34 | elif self.split == 'val': 35 | dirname = 'sequence_%d' % (self.annot['sequence'][index]) 36 | imgname = '%s/valid/%s/%s_%06d.jpg' % (conf.inf_img_dir, dirname, dirname, self.annot['idx'][index]) 37 | img = cv2.imread(imgname) 38 | return img 39 | 40 | def get_part_info(self, index): 41 | pose2d = self.annot['pose2d'][index].copy() 42 | bbox = self.annot['bbox'][index].copy() 43 | pose3d = self.annot['pose3d'][index].copy() 44 | cam_f = self.annot['cam_f'][index].copy() 45 | cam_c = self.annot['cam_c'][index].copy() 46 | return pose2d, bbox, pose3d, cam_f, cam_c 47 | 48 | def __getitem__(self, index): 49 | if self.split == 'train': 50 | index = np.random.randint(self.num_samples) 51 | 52 | # get global constants 53 | num_joints = conf.num_joints 54 | res_in = conf.res_in 55 | res_out = conf.res_out 56 | res_ratio = res_in / res_out 57 | 58 | # get image 59 | img = self.load_image(index) 60 | 61 | # get 2d/3d pose, bounding box, camera information 62 | pts, bbox, meta3d, cam_f, cam_c = self.get_part_info(index) 63 | meta3d = meta3d.astype(np.float32) 64 | cam_f = cam_f.astype(np.float32) 65 | cam_c = cam_c.astype(np.float32) 66 | 67 | # SCALING!! 68 | pts = pts * 0.5 69 | bbox = bbox * 0.5 70 | cam_f = cam_f * 0.5 71 | cam_c = cam_c * 0.5 72 | 73 | # set 2d pose 74 | pts = pts - bbox[0:2] 75 | pts = pts / bbox[2:4] 76 | pts = pts * float(res_in - 1) 77 | 78 | # set 3d pose 79 | pose3d = meta3d.copy() 80 | pose3d = pose3d - pose3d[conf.root] 81 | 82 | # data augmentation (small random translation) 83 | inp = np.zeros_like(img) 84 | if self.split == 'train': 85 | xr = np.random.randint(9)-4 86 | yr = np.random.randint(9)-4 87 | in_x1, in_x2 = max(0,-xr), min(256,256-xr) 88 | in_y1, in_y2 = max(0,-yr), min(256,256-yr) 89 | out_x1, out_x2 = max(0,xr), min(256,256+xr) 90 | out_y1, out_y2 = max(0,yr), min(256,256+yr) 91 | inp[out_y1:out_y2, out_x1:out_x2, :] = img[in_y1:in_y2, in_x1:in_x2, :] 92 | pts[:,0] = pts[:,0] + xr 93 | pts[:,1] = pts[:,1] + yr 94 | else: 95 | inp[:,:,:] = img[:,:,:] 96 | 97 | inp = cv2.cvtColor(inp, cv2.COLOR_BGR2RGB) 98 | inp = inp.transpose(2,0,1).astype(np.float32) 99 | 100 | # normalization 101 | inp = inp / 255.0 102 | 103 | # set valid joints 104 | valid2d = np.ones((num_joints), dtype=np.float32) 105 | valid3d = 1.0 106 | 107 | # set output heatmap and 2d pose 108 | pose2d = np.zeros((num_joints, 2), dtype=np.float32) 109 | hmap = np.zeros((int(num_joints), int(res_out), int(res_out)), dtype=np.float32) 110 | for i in range(num_joints): 111 | pt = pts[i].astype(np.float32) 112 | pose2d[i] = pt 113 | hmap[i] = draw_gaussian(hmap[i], pt/res_ratio+0.5, conf.std) 114 | 115 | # data augmentation (color jittering) 116 | if self.split == 'train': 117 | if np.random.random() < 0.5: 118 | inp = flip(inp) 119 | hmap = flip(hmap) 120 | hmap_flip = hmap.copy() 121 | for i in range(len(flip_index)): 122 | hmap_flip[i] = hmap[flip_index[i]].copy() 123 | hmap = hmap_flip.copy() 124 | pose2d_flip = pose2d.copy() 125 | for i in range(len(flip_index)): 126 | pose2d_flip[i] = pose2d[flip_index[i]].copy() 127 | pose2d = pose2d_flip.copy() 128 | pose2d[:, 0] = conf.res_in - pose2d[:, 0] 129 | pose3d_flip = pose3d.copy() 130 | for i in range(len(flip_index)): 131 | pose3d_flip[i] = pose3d[flip_index[i]].copy() 132 | pose3d = pose3d_flip.copy() 133 | pose3d[:, 0] *= -1 134 | inp[0] = np.clip(inp[0] * (np.random.random() * .4 + .6), 0, 1) 135 | inp[1] = np.clip(inp[1] * (np.random.random() * .4 + .6), 0, 1) 136 | inp[2] = np.clip(inp[2] * (np.random.random() * .4 + .6), 0, 1) 137 | 138 | # set dummy action 139 | action = 0 140 | 141 | # root coordinates 142 | coords_root = meta3d[conf.root].copy() 143 | depth_root = coords_root[2].copy() 144 | depth_root_canonical = coords_root[2].copy() / np.sqrt(np.prod(cam_f)) 145 | 146 | # set 3d pose 147 | pose3d = np.delete(pose3d, (conf.root), axis=0) 148 | 149 | # set data 150 | data = {'inp': inp, 'bbox': bbox.astype(np.int32), 151 | 'hmap': hmap, 152 | 'pose2d': pose2d, 'valid2d': valid2d, 153 | 'pose3d': pose3d, 'valid3d': valid3d, 154 | 'cam_f': cam_f, 'cam_c': cam_c, 155 | 'meta3d': meta3d, 156 | 'action': action, 157 | 'coords_root': coords_root, 158 | 'depth_root': depth_root, 159 | 'depth_root_canonical': depth_root_canonical} 160 | 161 | # return 162 | return index, data 163 | 164 | def __len__(self): 165 | return self.num_samples 166 | 167 | -------------------------------------------------------------------------------- /src/datasets_lift/mpiinf.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data as data 3 | from h5py import File 4 | import conf 5 | import numpy as np 6 | import pdb 7 | 8 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 9 | 10 | class MPIINF(data.Dataset): 11 | def __init__(self, split, noise=0, std_train=0, std_test=0, noise_path=None): 12 | print('==> Initializing MPI_INF %s data' % (split)) 13 | 14 | annot = {} 15 | tags = ['pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'sequence', 'video'] 16 | f = File('%s/inf/inf_%s.h5' % (conf.data_dir, split), 'r') 17 | for tag in tags: 18 | annot[tag] = np.asarray(f[tag]).copy() 19 | f.close() 20 | 21 | self.split = split 22 | self.noise = noise 23 | self.std_train = std_train 24 | self.std_test = std_test 25 | self.noise_path = noise_path 26 | self.annot = annot 27 | self.num_samples = self.annot['pose2d'].shape[0] 28 | 29 | # image size 30 | self.width = 2048 * 0.5 31 | self.height = 2048 * 0.5 32 | 33 | # load error statistics 34 | self.load_error_stat() 35 | 36 | print('Load %d MPI_INF %s samples' % (self.num_samples, self.split)) 37 | 38 | def get_part_info(self, index): 39 | pose2d = self.annot['pose2d'][index].copy() 40 | bbox = self.annot['bbox'][index].copy() 41 | pose3d = self.annot['pose3d'][index].copy() 42 | cam_f = self.annot['cam_f'][index].copy() 43 | cam_c = self.annot['cam_c'][index].copy() 44 | return pose2d, bbox, pose3d, cam_f, cam_c 45 | 46 | def load_error_stat(self): 47 | # load error stat 48 | if self.split == 'train': 49 | if self.noise == 0: # do not use noise 50 | pass 51 | elif self.noise == 1: # use specified gaussian noise 52 | pass 53 | elif self.noise == 2: # use estimated 2d pose 54 | filename = '%s_protocol%d/resnet152-int/fusion/rmsprop_lr1.0e-05_batch48/test_train.pth' % (conf.exp_dir, self.protocol) 55 | result = torch.load(filename) 56 | self.annot['pose2d'] = result['pred'].cpu().numpy() 57 | elif self.noise == 3: # use estimated single gaussian noise 58 | filename = '%s/%s' % (conf.exp_dir, self.noise_path) 59 | result = torch.load(filename) 60 | mean = result['mean'].numpy() / float(conf.res_in - 1) 61 | self.mean = mean[0] 62 | std = result['std'].numpy() / float(conf.res_in - 1) 63 | self.std = std[0] 64 | elif self.noise == 4: # use estimated mixture noise 65 | filename = '%s/%s' % (conf.exp_dir, self.noise_path) 66 | result = torch.load(filename) 67 | self.mean = result['mean'].numpy() / float(conf.res_in - 1) 68 | self.std = result['std'].numpy() / float(conf.res_in - 1) 69 | self.weight = result['weight'].numpy() 70 | else: 71 | raise ValueError('unsupported noise mode %d' % self.noise) 72 | 73 | def __getitem__(self, index): 74 | # get 2d/3d pose, bounding box, camera information 75 | pose2d, bbox, pose3d, cam_f, cam_c = self.get_part_info(index) 76 | cam_f = cam_f.astype(np.float32) 77 | cam_c = cam_c.astype(np.float32) 78 | 79 | # SCALING 80 | pose2d = pose2d * 0.5 81 | bbox = bbox * 0.5 82 | cam_f = cam_f * 0.5 83 | cam_c = cam_c * 0.5 84 | 85 | # data augmentation (flipping) 86 | if self.split == 'train' and np.random.random() < 0.5: 87 | pose2d_flip = pose2d.copy() 88 | for i in range(len(flip_index)): 89 | pose2d_flip[i] = pose2d[flip_index[i]].copy() 90 | pose3d_flip = pose3d.copy() 91 | for i in range(len(flip_index)): 92 | pose3d_flip[i] = pose3d[flip_index[i]].copy() 93 | pose2d = pose2d_flip.copy() 94 | pose3d = pose3d_flip.copy() 95 | pose2d[:, 0] = self.width - pose2d[:, 0] 96 | pose3d[:, 0] *= -1 97 | 98 | #bbox[0] = self.width - bbox[0] - bbox[2] 99 | #cam_c[0] = self.width - cam_c[0] 100 | 101 | # original 2d pose 102 | meta2d = pose2d.copy() 103 | 104 | # set 2d pose 105 | if self.noise == 2: 106 | if not self.split == 'train': 107 | pose2d = pose2d - bbox[0:2] 108 | pose2d = pose2d / bbox[2:4] 109 | pose2d = pose2d * float(conf.res_in - 1) 110 | else: 111 | pose2d = pose2d - bbox[0:2] 112 | pose2d = pose2d / bbox[2:4] 113 | if self.split == 'train': 114 | if self.noise == 1: 115 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std_train 116 | if self.noise == 3: 117 | val = np.random.random((pose2d.shape[0])) 118 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean) * (val < self.weight).reshape((pose2d.shape[0], 1)) 119 | elif self.split == 'test': 120 | if not self.std_test == 0.0: 121 | if (self.std_test > 0.0) and (self.std_test < 1.0): 122 | # gaussian noise 123 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std_test 124 | elif (self.std_test > 1.0) and (self.std_test < 2.0): 125 | # uniform noise 126 | val = np.random.random((pose2d.shape[0])) + 1.0 127 | pose2d += ((np.random.random((pose2d.shape[0], pose2d.shape[1])) * 100.0 - 50.0) / float(conf.res_in - 1)) * (val < self.opt.std_test).reshape((pose2d.shape[0], 1)) 128 | pose2d = pose2d * float(conf.res_in - 1) 129 | 130 | # root coordinates 131 | coords_root = pose3d[conf.root].copy() 132 | depth_root = coords_root[2].copy() 133 | depth_root_canonical = coords_root[2].copy() / np.sqrt(np.prod(cam_f)) * conf.f0 134 | 135 | # set 3d pose 136 | pose3d = pose3d - pose3d[conf.root] 137 | pose3d = np.delete(pose3d, (conf.root), axis=0) 138 | 139 | # set data 140 | data = {'pose2d': pose2d, 'bbox': bbox, 141 | 'pose3d': pose3d, 'coords_root': coords_root, 142 | 'depth_root': depth_root, 143 | 'depth_root_canonical': depth_root_canonical, 144 | 'cam_f': cam_f, 'cam_c': cam_c, 145 | 'meta2d': meta2d} 146 | 147 | return data 148 | 149 | def __len__(self): 150 | return self.num_samples 151 | 152 | -------------------------------------------------------------------------------- /src/train_lift.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import torch 4 | from utils.utils import AverageMeter 5 | from utils.eval import compute_error3d, compute_error3d_pa, compute_error_root 6 | from utils.eval import compute_error3d_x, compute_error3d_y, compute_error3d_z 7 | from progress.bar import Bar 8 | import pdb 9 | import conf 10 | 11 | def weighted_mse_loss(prediction, target, weight=1.0): 12 | return torch.sum(weight*(prediction-target)**2)/prediction.shape[0] 13 | 14 | def weighted_l1_loss(prediction, target, weight=1.0): 15 | return torch.sum(weight*torch.abs(prediction-target))/prediction.shape[0] 16 | 17 | def step(split, epoch, opt, dataLoader, model, optimizer = None): 18 | # training mode 19 | if split == 'train': 20 | model.train() 21 | else: 22 | model.eval() 23 | 24 | # 25 | if opt.analysis == True: 26 | std2d = [] 27 | depth = [] 28 | 29 | # initialize evaluations 30 | cost, error3d1, error3d2 = AverageMeter(), AverageMeter(), AverageMeter() 31 | error3d3 = AverageMeter() 32 | error3dx, error3dy, error3dz = AverageMeter(), AverageMeter(), AverageMeter() 33 | 34 | num_iters = len(dataLoader) 35 | bar = Bar('==>', max=num_iters) 36 | 37 | # for each mini-batch, 38 | for i, (data) in enumerate(dataLoader): 39 | pose2d = data['pose2d'].float().to("cuda") 40 | bbox = data['bbox'].float().to("cuda") 41 | pose3d = data['pose3d'].float().to("cuda") 42 | coords_root = data['coords_root'].float().to("cuda") 43 | depth_root = data['depth_root'].float().to("cuda") 44 | depth_root_canonical = data['depth_root_canonical'].float().to("cuda") 45 | cam_f = data['cam_f'].float().to("cuda") 46 | cam_c = data['cam_c'].float().to("cuda") 47 | meta2d = data['meta2d'].float().to("cuda") 48 | 49 | # forward propagation 50 | if split == 'test': 51 | model.module.fliptest = opt.fliptest 52 | outputs = model([pose2d, bbox, cam_c]) 53 | 54 | # 55 | nb = pose2d.size(0) 56 | nj = conf.num_joints 57 | 58 | # compute cost 59 | loss = 0 60 | loss += weighted_l1_loss(outputs[0], pose3d) 61 | if opt.canonical == False: 62 | loss += weighted_l1_loss(outputs[1], depth_root) * opt.weight_root 63 | else: 64 | loss += weighted_l1_loss(outputs[1], depth_root_canonical) * opt.weight_root 65 | 66 | # update model parameters with backpropagation 67 | if split == 'train': 68 | optimizer.zero_grad() 69 | loss.backward() 70 | optimizer.step() 71 | 72 | # unnormalized 2d pose 73 | pose2d = pose2d.detach() 74 | pose2d = pose2d / 255.0 75 | pose2d = pose2d * torch.reshape(bbox[:, 2:4], (nb, 1, 2)) 76 | pose2d = pose2d + torch.reshape(bbox[:, 0:2], (nb, 1, 2)) 77 | 78 | # get predicted root coordinates 79 | x = pose2d[:, conf.root, 0] 80 | y = pose2d[:, conf.root, 1] 81 | cx = cam_c[:, 0].detach() 82 | cy = cam_c[:, 1].detach() 83 | if opt.canonical == False: 84 | Z = outputs[1].detach() 85 | f = torch.sqrt(torch.prod(cam_f.detach(), 1)) 86 | X = (x-cx)*Z/f 87 | Y = (y-cy)*Z/f 88 | pred_root = torch.cat((X.view(nb,1), Y.view(nb,1), Z.view(nb,1)), 1) 89 | else: 90 | Z = outputs[1].detach() 91 | X = (x-cx)*Z/conf.f0 92 | Y = (y-cy)*Z/conf.f0 93 | f = torch.sqrt(torch.prod(cam_f.detach(), 1))/conf.f0 94 | Z = Z*f 95 | pred_root = torch.cat((X.view(nb,1), Y.view(nb,1), Z.view(nb,1)), 1) 96 | 97 | # update evaluations 98 | cost.update(loss.detach().item(), pose2d.size(0)) 99 | error3d1.update(compute_error3d(outputs[0].detach(), pose3d.detach())) 100 | error3d2.update(compute_error_root(pred_root, coords_root.detach())) 101 | if split == 'test': 102 | error3d3.update(compute_error3d_pa(outputs[0].detach(), pose3d.detach())) 103 | error3dx.update(compute_error3d_x(outputs[0].detach(), pose3d.detach())) 104 | error3dy.update(compute_error3d_y(outputs[0].detach(), pose3d.detach())) 105 | error3dz.update(compute_error3d_z(outputs[0].detach(), pose3d.detach())) 106 | 107 | # 108 | if opt.analysis == True: 109 | std2d.append(outputs[2].detach()) 110 | depth.append(outputs[1].detach()) 111 | 112 | Bar.suffix = '{split} Epoch: [{0}][{1}/{2}]| Total: {total:} | ETA: {eta:} | Cost {cost.avg:.6f} | Error3D1 {error3d1.avg:.6f} | Error3D2 {error3d2.avg:.6f}'.format(epoch, i, num_iters, split=split, total=bar.elapsed_td, eta=bar.eta_td, cost=cost, error3d1=error3d1, error3d2=error3d2) 113 | bar.next() 114 | 115 | bar.finish() 116 | 117 | if split == 'test': 118 | if (opt.std_test > 0.0) and (opt.std_test < 1.0): 119 | file = open(os.path.join(opt.save_dir, 'result_noise_gaussian%.3f.txt' % (opt.std_test)), 'w') 120 | elif (opt.std_test > 1.0) and (opt.std_test < 2.0): 121 | file = open(os.path.join(opt.save_dir, 'result_noise_uniform%.3f.txt' % (opt.std_test)), 'w') 122 | else: 123 | if opt.fliptest == True: 124 | file = open(os.path.join(opt.save_dir, 'result_fliptest.txt'), 'w') 125 | else: 126 | file = open(os.path.join(opt.save_dir, 'result.txt'), 'w') 127 | file.write('L1 loss for test set = {:6f}\n'.format(cost.avg)) 128 | file.write('3D MPJPE error for test set = {:6f}\n'.format(error3d1.avg)) 129 | file.write('3D Root error for test set = {:6f}\n'.format(error3d2.avg)) 130 | file.write('3D PA MPJPE error for test set = {:6f}\n'.format(error3d3.avg)) 131 | file.write('3D error in X axis = {:6f}\n'.format(error3dx.avg)) 132 | file.write('3D error in Y axis = {:6f}\n'.format(error3dy.avg)) 133 | file.write('3D error in Z axis = {:6f}\n'.format(error3dz.avg)) 134 | file.close() 135 | 136 | if opt.analysis == True: 137 | std2d = torch.cat(std2d, dim=0).squeeze().cpu().numpy() * 1000. 138 | depth = torch.cat(depth, dim=0).squeeze().cpu().numpy() / 1000. 139 | 140 | filename = os.path.join(opt.save_dir, 'result.npz') 141 | np.savez(filename, std2d, depth) 142 | 143 | return cost.avg, error3d1.avg, error3d2.avg 144 | 145 | def train(epoch, opt, train_loader, model, optimizer): 146 | return step('train', epoch, opt, train_loader, model, optimizer) 147 | 148 | def val(epoch, opt, val_loader, model): 149 | return step('val', epoch, opt, val_loader, model) 150 | 151 | def test(epoch, opt, test_loader, model): 152 | return step('test', epoch, opt, test_loader, model) 153 | 154 | -------------------------------------------------------------------------------- /src/utils/eval.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import conf 3 | import pdb 4 | 5 | def get_pred(hmap): 6 | """get predicted 2d pose from heat map""" 7 | num_batch = hmap.shape[0] 8 | num_joint = hmap.shape[1] 9 | h = hmap.shape[2] 10 | w = hmap.shape[3] 11 | hmap = hmap.reshape(num_batch, num_joint, h*w) 12 | idx = torch.argmax(hmap, dim=2) 13 | pred = torch.zeros(num_batch, num_joint, 2).to('cuda') 14 | for i in range(num_batch): 15 | for j in range(num_joint): 16 | pred[i, j, 0], pred[i, j, 1] = idx[i, j] % w, idx[i, j] / w 17 | return pred 18 | 19 | def compute_error(output, target, weight): 20 | """compute 2d pose estimation error in pixels""" 21 | num_batch = output.shape[0] 22 | num_joint = output.shape[1] 23 | res_in = conf.res_in 24 | res_out = output.shape[2] 25 | res_ratio = res_in / res_out 26 | pred = get_pred(output) 27 | pred = pred * res_ratio + res_ratio/2 28 | val = torch.sqrt(torch.mul(((pred - target) ** 2).sum(2), weight)) 29 | error = val.sum()/weight.sum().item() 30 | return error 31 | 32 | def compute_error_direct(output, target, weight): 33 | """compute 2d pose estimation error in pixels""" 34 | num_batch = output.shape[0] 35 | num_joint = output.shape[1] 36 | val = torch.sqrt(torch.mul(((output - target) ** 2).sum(2), weight)) 37 | error = val.sum()/weight.sum().item() 38 | return error 39 | 40 | def compute_error3d(output, target, weight=None): 41 | """compute 3d pose estimation error in millimeters (MPJPE)""" 42 | num_batch = output.shape[0] 43 | num_joint = output.shape[1]+1 44 | if weight is None: 45 | val = torch.sqrt(((output - target) ** 2).sum(2)) 46 | error = val.sum()/(num_batch*num_joint) 47 | else: 48 | val = torch.sqrt(torch.mul(((output - target) ** 2).sum(2), weight.reshape(num_batch, 1))) 49 | error = val.sum()/(weight.sum().item()*num_joint) 50 | return error 51 | 52 | def compute_error3d_x(output, target): 53 | """compute 3d pose estimation error in millimeters""" 54 | num_batch = output.shape[0] 55 | num_joint = output.shape[1]+1 56 | val = torch.abs(output[:,:,0] - target[:,:,0]) 57 | error = val.sum()/(num_batch*num_joint) 58 | return error 59 | 60 | def compute_error3d_y(output, target): 61 | """compute 3d pose estimation error in millimeters""" 62 | num_batch = output.shape[0] 63 | num_joint = output.shape[1]+1 64 | val = torch.abs(output[:,:,1] - target[:,:,1]) 65 | error = val.sum()/(num_batch*num_joint) 66 | return error 67 | 68 | def compute_error3d_z(output, target): 69 | """compute 3d pose estimation error in millimeters""" 70 | num_batch = output.shape[0] 71 | num_joint = output.shape[1]+1 72 | val = torch.abs(output[:,:,2] - target[:,:,2]) 73 | error = val.sum()/(num_batch*num_joint) 74 | return error 75 | 76 | def compute_error3d_pa(output, target): 77 | """compute 3d pose estimation error in millimeters (PA-MPJPE)""" 78 | nb = output.shape[0] 79 | nj = output.shape[1]+1 80 | pred = torch.zeros((nb, nj, 3), dtype=output.dtype, device=output.device) 81 | pred[:,1:,:] = output 82 | pose3d = torch.zeros((nb, nj, 3), dtype=target.dtype, device=target.device) 83 | pose3d[:,1:,:] = target 84 | 85 | # Translation 86 | pred_mean = pred.mean(dim=1, keepdim=True) 87 | pred = pred - pred_mean 88 | pose3d_mean = pose3d.mean(dim=1, keepdim=True) 89 | pose3d = pose3d - pose3d_mean 90 | 91 | # Scaling 92 | pred_scale = torch.sqrt((pred**2.0).sum(dim=2, keepdim=True).sum(dim=1, keepdim=True)/nj) 93 | pose3d_scale = torch.sqrt((pose3d**2.0).sum(dim=2, keepdim=True).sum(dim=1, keepdim=True)/nj) 94 | scale_ratio = pose3d_scale / pred_scale 95 | pred = pred * scale_ratio 96 | 97 | # Rotation (solve orthogonal Procrustes problem) 98 | pred_pa = pred.clone() 99 | for i in range(nb): 100 | A = pred[i].t() 101 | B = pose3d[i].t() 102 | M = torch.mm(B, A.t()) 103 | U, S, V = torch.svd(M) 104 | R = torch.mm(U, V.t()) 105 | A = torch.mm(R, A) 106 | pred_pa[i] = A.t() 107 | 108 | # Compute MPJPE error 109 | val = torch.sqrt(((pred_pa - pose3d) ** 2).sum(2)) 110 | error = val.sum()/(nb*nj) 111 | 112 | return error 113 | 114 | def compute_error3d_pa_simil(output, target): 115 | """compute 3d pose estimation error in millimeters (PA-MPJPE)""" 116 | num_batch = output.shape[0] 117 | num_joint = output.shape[1]+1 118 | 119 | # Translation 120 | output_mean = output.mean(dim=1, keepdim=True) 121 | output = output - output_mean 122 | target_mean = target.mean(dim=1, keepdim=True) 123 | target = target - target_mean 124 | 125 | # Scaling 126 | output_scale = torch.sqrt((output**2.0).sum(dim=2, keepdim=True).sum(dim=1, keepdim=True)/num_joint) 127 | target_scale = torch.sqrt((target**2.0).sum(dim=2, keepdim=True).sum(dim=1, keepdim=True)/num_joint) 128 | scale_ratio = target_scale / output_scale 129 | output = output * scale_ratio 130 | 131 | # Rotation (solve orthogonal Procrustes problem) 132 | output_pa = output.clone() 133 | for i in range(num_batch): 134 | A = output[i].t() 135 | B = target[i].t() 136 | M = torch.mm(B, A.t()) 137 | U, S, V = torch.svd(M) 138 | R = torch.mm(U, V.t()) 139 | A = torch.mm(R, A) 140 | output_pa[i] = A.t() 141 | 142 | # Compute MPJPE error 143 | val = torch.sqrt(((output_pa - target) ** 2).sum(2)) 144 | error = val.sum()/(num_batch*num_joint) 145 | 146 | return error 147 | 148 | def compute_error3d_pa_rigid(output, target): 149 | """compute 3d pose estimation error in millimeters (PA-MPJPE)""" 150 | num_batch = output.shape[0] 151 | num_joint = output.shape[1]+1 152 | 153 | # Translation 154 | output_mean = output.mean(dim=1, keepdim=True) 155 | output = output - output_mean 156 | target_mean = target.mean(dim=1, keepdim=True) 157 | target = target - target_mean 158 | 159 | # Rotation (solve orthogonal Procrustes problem) 160 | output_pa = output.clone() 161 | for i in range(num_batch): 162 | A = output[i].t() 163 | B = target[i].t() 164 | M = torch.mm(B, A.t()) 165 | U, S, V = torch.svd(M) 166 | R = torch.mm(U, V.t()) 167 | A = torch.mm(R, A) 168 | output_pa[i] = A.t() 169 | 170 | # Compute MPJPE error 171 | val = torch.sqrt(((output_pa - target) ** 2).sum(2)) 172 | error = val.sum()/(num_batch*num_joint) 173 | 174 | return error 175 | 176 | def compute_error_root(output, target): 177 | """compute root coordinate error in millimeters (MRPE)""" 178 | num_batch = output.shape[0] 179 | val = torch.sqrt(((output - target) ** 2).sum(1)) 180 | error = val.sum()/(num_batch) 181 | return error 182 | 183 | -------------------------------------------------------------------------------- /src/datasets_2d/h36m17.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | from h5py import File 3 | import conf 4 | import numpy as np 5 | import cv2 6 | from utils.utils import rnd, flip, shuffle_lr 7 | from utils.img import transform, crop, draw_gaussian 8 | import pdb 9 | 10 | subject_list = [[[1, 5, 6, 7], [8]], [[1, 5, 6, 7, 8], [9, 11]], [[1, 5, 6, 7, 8, 9], [11]]] 11 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 12 | 13 | class H36M17(data.Dataset): 14 | def __init__(self, protocol, split, dense=False): 15 | print('==> Initializing H36M %s data' % (split)) 16 | annot = {} 17 | tags = ['idx', 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'action', 'subaction', 'camera'] 18 | f = File('%s/h36m/h36m17.h5' % (conf.data_dir), 'r') 19 | for tag in tags: 20 | annot[tag] = np.asarray(f[tag]).copy() 21 | f.close() 22 | 23 | if dense == False: 24 | idxs = np.mod(annot['idx'], 50) == 1 25 | idxs = np.arange(annot['idx'].shape[0])[idxs] 26 | for tag in tags: 27 | annot[tag] = annot[tag][idxs] 28 | 29 | idxs = np.full(annot['idx'].shape[0], False) 30 | subject = subject_list[protocol][1-int(split=='train' or split=='test_train')] 31 | for i in range(len(subject)): 32 | idxs = idxs + (annot['subject']==subject[i]) 33 | idxs = np.arange(annot['idx'].shape[0])[idxs] 34 | for tag in tags: 35 | annot[tag] = annot[tag][idxs] 36 | 37 | self.protocol = protocol 38 | self.split = split 39 | self.dense = dense 40 | self.annot = annot 41 | self.num_samples = len(self.annot['idx']) 42 | 43 | print('Load %d H36M %s samples' % (self.num_samples, self.split)) 44 | 45 | def load_image(self, index): 46 | dirname = 's_%02d_act_%02d_subact_%02d_ca_%02d' % (self.annot['subject'][index], \ 47 | self.annot['action'][index], self.annot['subaction'][index], self.annot['camera'][index]) 48 | imgname = '%s/%s/%s_%06d.jpg' % (conf.h36m_img_dir, dirname, dirname, self.annot['idx'][index]) 49 | img = cv2.imread(imgname) 50 | return img 51 | 52 | def get_part_info(self, index): 53 | pose2d = self.annot['pose2d'][index].copy() 54 | bbox = self.annot['bbox'][index].copy() 55 | pose3d = self.annot['pose3d'][index].copy() 56 | cam_f = self.annot['cam_f'][index].copy() 57 | cam_c = self.annot['cam_c'][index].copy() 58 | action = self.annot['action'][index].copy() 59 | return pose2d, bbox, pose3d, cam_f, cam_c, action 60 | 61 | def __getitem__(self, index): 62 | if self.split == 'train': 63 | index = np.random.randint(self.num_samples) 64 | 65 | # get global constants 66 | num_joints = conf.num_joints 67 | res_in = conf.res_in 68 | res_out = conf.res_out 69 | res_ratio = res_in / res_out 70 | 71 | # get image 72 | img = self.load_image(index) 73 | 74 | # get 2d/3d pose and bounding box 75 | pts, bbox, meta3d, cam_f, cam_c, action = self.get_part_info(index) 76 | cam_f = cam_f.astype(np.float32) 77 | cam_c = cam_c.astype(np.float32) 78 | action = action.item() 79 | 80 | # set 2d pose 81 | pts = pts - bbox[0:2] 82 | pts = pts / bbox[2:4] 83 | pts = pts * float(res_in - 1) 84 | 85 | # set 3d pose 86 | pose3d = meta3d.copy() 87 | pose3d = pose3d - pose3d[conf.root] 88 | 89 | # data augmentation (small random translation) 90 | inp = np.zeros_like(img) 91 | if self.split == 'train': 92 | xr = np.random.randint(9)-4 93 | yr = np.random.randint(9)-4 94 | in_x1, in_x2 = max(0,-xr), min(256,256-xr) 95 | in_y1, in_y2 = max(0,-yr), min(256,256-yr) 96 | out_x1, out_x2 = max(0,xr), min(256,256+xr) 97 | out_y1, out_y2 = max(0,yr), min(256,256+yr) 98 | inp[out_y1:out_y2, out_x1:out_x2, :] = img[in_y1:in_y2, in_x1:in_x2, :] 99 | pts[:,0] = pts[:,0] + xr 100 | pts[:,1] = pts[:,1] + yr 101 | else: 102 | inp[:,:,:] = img[:,:,:] 103 | 104 | inp = cv2.cvtColor(inp, cv2.COLOR_BGR2RGB) 105 | inp = inp.transpose(2,0,1).astype(np.float32) 106 | 107 | # normalization 108 | inp = inp / 255.0 109 | 110 | # set valid joints 111 | valid2d = np.ones((num_joints), dtype=np.float32) 112 | valid3d = 1.0 113 | 114 | # set output heatmap and 2d pose 115 | pose2d = np.zeros((num_joints, 2), dtype=np.float32) 116 | hmap = np.zeros((int(num_joints), int(res_out), int(res_out)), dtype=np.float32) 117 | for i in range(num_joints): 118 | pt = pts[i].astype(np.float32) 119 | pose2d[i] = pt 120 | hmap[i] = draw_gaussian(hmap[i], pt/res_ratio+0.5, conf.std) 121 | 122 | # data augmentation (flipping and color jittering) 123 | if self.split == 'train': 124 | if np.random.random() < 0.5: 125 | inp = flip(inp) 126 | hmap = flip(hmap) 127 | hmap_flip = hmap.copy() 128 | for i in range(len(flip_index)): 129 | hmap_flip[i] = hmap[flip_index[i]].copy() 130 | hmap = hmap_flip.copy() 131 | pose2d_flip = pose2d.copy() 132 | for i in range(len(flip_index)): 133 | pose2d_flip[i] = pose2d[flip_index[i]].copy() 134 | pose2d = pose2d_flip.copy() 135 | pose2d[:, 0] = conf.res_in - pose2d[:, 0] 136 | pose3d_flip = pose3d.copy() 137 | for i in range(len(flip_index)): 138 | pose3d_flip[i] = pose3d[flip_index[i]].copy() 139 | pose3d = pose3d_flip.copy() 140 | pose3d[:, 0] *= -1 141 | inp[0] = np.clip(inp[0] * (np.random.random() * .4 + .6), 0, 1) 142 | inp[1] = np.clip(inp[1] * (np.random.random() * .4 + .6), 0, 1) 143 | inp[2] = np.clip(inp[2] * (np.random.random() * .4 + .6), 0, 1) 144 | 145 | # root coordinates 146 | coords_root = meta3d[conf.root].copy() 147 | depth_root = coords_root[2].copy() 148 | depth_root_canonical = coords_root[2].copy() / np.sqrt(np.prod(cam_f)) 149 | 150 | # set 3d pose 151 | pose3d = np.delete(pose3d, (conf.root), axis=0) 152 | 153 | # set data 154 | data = {'inp': inp, 'bbox': bbox, 155 | 'hmap': hmap, 156 | 'pose2d': pose2d, 'valid2d': valid2d, 157 | 'pose3d': pose3d, 'valid3d': valid3d, 158 | 'cam_f': cam_f, 'cam_c': cam_c, 159 | 'meta3d': meta3d, 'action': action, 160 | 'coords_root': coords_root, 161 | 'depth_root': depth_root, 162 | 'depth_root_canonical': depth_root_canonical 163 | } 164 | 165 | # return 166 | return index, data 167 | 168 | def __len__(self): 169 | return self.num_samples 170 | 171 | -------------------------------------------------------------------------------- /src/models/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import conf 4 | import pdb 5 | 6 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 7 | flip_index_ = [3, 4, 5, 0, 1, 2, 6, 7, 8, 9, 13, 14, 15, 10, 11, 12] 8 | 9 | class ResNetModule(nn.Module): 10 | def __init__(self, num_features): 11 | super(ResNetModule, self).__init__() 12 | 13 | modules = [] 14 | modules.append(nn.BatchNorm1d(num_features)) 15 | modules.append(nn.Dropout(0.5)) 16 | modules.append(nn.ReLU()) 17 | modules.append(nn.Linear(num_features, num_features)) 18 | modules.append(nn.BatchNorm1d(num_features)) 19 | modules.append(nn.Dropout(0.5)) 20 | modules.append(nn.ReLU()) 21 | modules.append(nn.Linear(num_features, num_features)) 22 | 23 | # set weights 24 | for m in modules: 25 | if isinstance(m, nn.Linear): 26 | nn.init.normal_(m.weight, mean=0, std=0.001) 27 | nn.init.constant_(m.bias, 0) 28 | elif isinstance(m, nn.BatchNorm1d): 29 | nn.init.constant_(m.weight, 1) 30 | nn.init.constant_(m.bias, 0) 31 | 32 | self.submod = nn.Sequential(*modules) 33 | 34 | def forward(self, x): 35 | return self.submod(x) + x 36 | 37 | class ResNet(nn.Module): 38 | def __init__(self, mode, num_joints, num_layers, num_features): 39 | super(ResNet, self).__init__() 40 | 41 | self.mode = mode 42 | self.num_joints = num_joints 43 | if mode == 0 or mode == 2: 44 | self.num_in = 2*num_joints 45 | elif mode == 1: 46 | self.num_in = 2*num_joints+3 47 | elif mode == 3: 48 | self.num_in = 2*num_joints+2 49 | elif mode == 4: 50 | self.num_in = 2*num_joints+1 51 | self.num_out = 3*(num_joints-1)+1 52 | self.num_layers = num_layers 53 | self.num_features = num_features 54 | 55 | mod = [] 56 | mod.append(nn.Linear(self.num_in, num_features)) 57 | for i in range(num_layers): 58 | mod.append(ResNetModule(num_features)) 59 | mod.append(nn.Linear(num_features, self.num_out)) 60 | 61 | # set weights 62 | nn.init.normal_(mod[0].weight, mean=0, std=0.001) 63 | nn.init.constant_(mod[0].bias, 0) 64 | nn.init.normal_(mod[-1].weight, mean=0, std=0.001) 65 | nn.init.constant_(mod[-1].bias, 0) 66 | 67 | self.mod = nn.ModuleList(mod) 68 | 69 | self.fliptest = False 70 | 71 | def forward(self, inp): 72 | pose2d = inp[0].clone() 73 | bbox = inp[1].clone() 74 | cam_c = inp[2].clone() 75 | 76 | nb = pose2d.shape[0] 77 | 78 | if self.mode == 0 or self.mode == 1 or self.mode == 3 or self.mode == 4: 79 | pose2d = pose2d / 255.0 80 | pose2d = pose2d * torch.reshape(bbox[:,2:4], (nb, 1, 2)) 81 | pose2d = pose2d + torch.reshape(bbox[:,0:2], (nb, 1, 2)) 82 | 83 | pose2d = pose2d - torch.reshape(cam_c, (nb, 1, 2)) 84 | 85 | mean2d = torch.mean(pose2d, 1, keepdim=True) 86 | dist = torch.sqrt(torch.sum((pose2d-mean2d)**2.0, 2, keepdim=True)) 87 | std2d = torch.std(dist, 1, keepdim=True) 88 | pose2d = (pose2d - mean2d) / std2d 89 | mean2d = mean2d * 0.001 90 | std2d = std2d * 0.001 91 | 92 | if self.mode == 0: # normalized 2d pose 93 | x = pose2d.reshape(nb, -1) 94 | elif self.mode == 1: # normalized 2d pose, mean, std 95 | x = torch.cat((pose2d.reshape(nb, -1), mean2d.reshape(nb, -1), std2d.reshape(nb, -1)), 1) 96 | elif self.mode == 3: # mean 97 | x = torch.cat((pose2d.reshape(nb, -1), mean2d.reshape(nb, -1)), 1) 98 | elif self.mode == 4: # std 99 | x = torch.cat((pose2d.reshape(nb, -1), std2d.reshape(nb, -1)), 1) 100 | elif self.mode == 2: # unnormalized 2d pose 101 | x = pose2d.reshape(nb, -1) 102 | else: 103 | raise ValueError('unsupported poselifter mode') 104 | 105 | x = self.mod[0](x) 106 | for i in range(self.num_layers): 107 | x = self.mod[i+1](x) 108 | x = self.mod[-1](x) 109 | 110 | pose_local = x[:, 0:(self.num_out-1)].view(-1, self.num_joints-1, 3) 111 | depth_root = x[:, self.num_out-1] 112 | 113 | if self.fliptest == True: 114 | [pose_local_flip, depth_root_flip] = self.forward_flip(inp) 115 | pose_local = (pose_local + pose_local_flip) * 0.5 116 | depth_root = (depth_root + depth_root_flip) * 0.5 117 | 118 | return [pose_local, depth_root] 119 | 120 | def forward_flip(self, inp): 121 | pose2d_orig = inp[0].clone() 122 | bbox = inp[1].clone() 123 | cam_c = inp[2].clone() 124 | 125 | nb = pose2d_orig.shape[0] 126 | 127 | index = torch.tensor(flip_index).to(pose2d_orig.device) 128 | pose2d = pose2d_orig.clone() 129 | pose2d.index_copy_(1, index, pose2d_orig) 130 | 131 | if self.mode == 0 or self.mode == 1 or self.mode == 3 or self.mode == 4: 132 | pose2d = pose2d / 255.0 133 | pose2d = pose2d * torch.reshape(bbox[:,2:4], (nb, 1, 2)) 134 | pose2d = pose2d + torch.reshape(bbox[:,0:2], (nb, 1, 2)) 135 | 136 | pose2d[:, :, 0] = conf.width - pose2d[:, :, 0] 137 | cam_c[:, 0] = conf.width - cam_c[:, 0] 138 | 139 | pose2d = pose2d - torch.reshape(cam_c, (nb, 1, 2)) 140 | 141 | mean2d = torch.mean(pose2d, 1, keepdim=True) 142 | dist = torch.sqrt(torch.sum((pose2d-mean2d)**2.0, 2, keepdim=True)) 143 | std2d = torch.std(dist, 1, keepdim=True) 144 | pose2d = (pose2d - mean2d) / std2d 145 | mean2d = mean2d * 0.001 146 | std2d = std2d * 0.001 147 | 148 | if self.mode == 0: 149 | x = pose2d.reshape(nb, -1) 150 | elif self.mode == 1: 151 | x = torch.cat((pose2d.reshape(nb, -1), mean2d.reshape(nb, -1), std2d.reshape(nb, -1)), 1) 152 | elif self.mode == 3: # mean only 153 | x = torch.cat((pose2d.reshape(nb, -1), mean2d.reshape(nb, -1)), 1) 154 | elif self.mode == 4: # std only 155 | x = torch.cat((pose2d.reshape(nb, -1), std2d.reshape(nb, -1)), 1) 156 | elif self.mode == 2: 157 | x = pose2d.reshape(nb, -1) 158 | else: 159 | raise ValueError('unsupported poselifter mode') 160 | 161 | x = self.mod[0](x) 162 | for i in range(self.num_layers): 163 | x = self.mod[i+1](x) 164 | x = self.mod[-1](x) 165 | 166 | pose_local = x[:, 0:(self.num_out-1)].view(-1, self.num_joints-1, 3) 167 | depth_root = x[:, self.num_out-1] 168 | 169 | pose_local[:, :, 0] *= -1 170 | 171 | index_ = torch.tensor(flip_index_).to(pose_local.device) 172 | pose_local_ = pose_local.clone() 173 | pose_local_.index_copy_(1, index_, pose_local) 174 | 175 | return [pose_local_, depth_root] 176 | 177 | def set_fliptest(self, val): 178 | self.fliptest = val 179 | 180 | -------------------------------------------------------------------------------- /src/opts_2d.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import conf 4 | import pdb 5 | 6 | class Opts(): 7 | def __init__(self): 8 | self.parser = argparse.ArgumentParser() 9 | 10 | def init(self): 11 | # miscellaneous 12 | self.parser.add_argument('-dataset_test', default='h36m', help='Dataset to test: h36m | inf') 13 | self.parser.add_argument('-dataset_train', default='h36m', help='Dataset to train: h36m | mpii | h36m_mpii') 14 | self.parser.add_argument('-protocol', type=int, default=1, help='Experiment protocol: 0| 1 | 2') 15 | self.parser.add_argument('-multi_gpu', default=False, action='store_true', help='Use multiple gpus?') 16 | self.parser.add_argument('-fliptest', default=False, action='store_true', help='Do flip test?') 17 | self.parser.add_argument('-save_results', default=False, action='store_true', help='Save prediction results?') 18 | self.parser.add_argument('-save_results_train', default=False, action='store_true', help='Save prediction results?') 19 | 20 | # network structure (2d) 21 | self.parser.add_argument('-network', default='resnet50 | resnet101 | resnet152', help='Network to use') 22 | self.parser.add_argument('-integral', type=int, default=0, help='Use the integral module?') 23 | self.parser.add_argument('-weight1', type=float, default=1.0, help='Weight for coordinate loss') 24 | self.parser.add_argument('-model_2d_path', default=None, help='Path to pretrained model') 25 | 26 | # network structure (3d) 27 | self.parser.add_argument('-lift3d', type=int, default=0, help='Lift to 3D?') 28 | self.parser.add_argument('-dataset3d', default='h36m', help='Dataset for 3D lifting') 29 | self.parser.add_argument('-weight2', type=float, default=1.0, help='Weight for 3d loss') 30 | self.parser.add_argument('-mode', type=int, default=1, help='Option for lifting') 31 | self.parser.add_argument('-num_layers', type=int, default=2, help='Number of hidden layers') 32 | self.parser.add_argument('-num_features', type=int, default=3000, help='Number of features') 33 | self.parser.add_argument('-noise', type=int, default=1, help='Noise mode') 34 | self.parser.add_argument('-std_train', type=float, default=0.005, help='Std of Gaussian noise for robust training') 35 | self.parser.add_argument('-model_lift_path', default=None, help='Path to pretrained model') 36 | 37 | # optimization hyperparameters 38 | self.parser.add_argument('-opt_method', default='rmsprop', help='Optimization method: rmsprop | adam | sgd') 39 | self.parser.add_argument('-lr', type=float, default=1e-4, help='Learning rate') 40 | self.parser.add_argument('-alpha', type=float, default=0.99, help='Smoothing constant') 41 | self.parser.add_argument('-epsilon', type=float, default=1e-8, help='For numerical stability') 42 | self.parser.add_argument('-weight_decay', type=float, default=0, help='Weight decay') 43 | self.parser.add_argument('-lr_decay', type=float, default=0, help='Learning rate decay') 44 | self.parser.add_argument('-beta1', type=float, default=0.9, help='First mement coefficient') 45 | self.parser.add_argument('-beta2', type=float, default=0.99, help='Second moment coefficient') 46 | self.parser.add_argument('-momentum', type=float, default=0, help='Momentum') 47 | 48 | # training options 49 | self.parser.add_argument('-num_epochs', type=int, default=30, help='Number of training epochs') 50 | self.parser.add_argument('-batch_size', type=int, default=8, help='Mini-batch size') 51 | self.parser.add_argument('-save_intervals', type=int, default=10, help='Number of iterations for saving model') 52 | 53 | def parse(self): 54 | self.init() 55 | self.opt = self.parser.parse_args() 56 | 57 | # set directories for experiments 58 | if self.opt.dataset_test == 'h36m': 59 | self.opt.save_dir = '%s/test_%s_protocol%d' % (conf.exp_dir, self.opt.dataset_test, self.opt.protocol) 60 | elif self.opt.dataset_test == 'inf': 61 | self.opt.save_dir = '%s/test_%s' % (conf.exp_dir, self.opt.dataset_test) 62 | if not os.path.exists(self.opt.save_dir): 63 | os.makedirs(self.opt.save_dir) 64 | 65 | if self.opt.lift3d == 0: # no 3d lifting 66 | if self.opt.integral == 0: 67 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s' % self.opt.network) 68 | if not os.path.exists(self.opt.save_dir): 69 | os.makedirs(self.opt.save_dir) 70 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train_%s' % self.opt.dataset_train) 71 | if not os.path.exists(self.opt.save_dir): 72 | os.makedirs(self.opt.save_dir) 73 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s_lr%1.1e_batch%d' % \ 74 | (self.opt.opt_method, self.opt.lr, self.opt.batch_size)) 75 | else: 76 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s-int' % (self.opt.network)) 77 | if not os.path.exists(self.opt.save_dir): 78 | os.makedirs(self.opt.save_dir) 79 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train_%s' % self.opt.dataset_train) 80 | if not os.path.exists(self.opt.save_dir): 81 | os.makedirs(self.opt.save_dir) 82 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s_lr%1.1e_batch%d_weight%1.1e' % \ 83 | (self.opt.opt_method, self.opt.lr, self.opt.batch_size, self.opt.weight1)) 84 | 85 | else: # piecewise lifting (integrated 2d pose -> 3d pose + root depth) 86 | self.opt.save_dir = os.path.join(self.opt.save_dir, '%s-lift' % (self.opt.network)) 87 | if not os.path.exists(self.opt.save_dir): 88 | os.makedirs(self.opt.save_dir) 89 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'train2d_%s_train3d_%s' % (self.opt.dataset_train, self.opt.dataset3d)) 90 | if not os.path.exists(self.opt.save_dir): 91 | os.makedirs(self.opt.save_dir) 92 | if self.opt.noise != 1: 93 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'canonical%d_mode%d_noise%d' % (int(self.opt.lift3d==2), self.opt.mode, self.opt.noise)) 94 | else: 95 | self.opt.save_dir = os.path.join(self.opt.save_dir, 'canonical%d_mode%d_noise%d_std%.3f' % (int(self.opt.lift3d==2), self.opt.mode, self.opt.noise, self.opt.std_train)) 96 | 97 | # save options 98 | args = dict((name, getattr(self.opt, name)) for name in dir(self.opt) 99 | if not name.startswith('_')) 100 | confs = dict((name, getattr(conf, name)) for name in dir(conf) 101 | if not name.startswith('_')) 102 | if not os.path.exists(self.opt.save_dir): 103 | os.makedirs(self.opt.save_dir) 104 | file_name = os.path.join(self.opt.save_dir, 'opt.txt') 105 | if not os.path.exists(file_name): 106 | with open(file_name, 'wt') as opt_file: 107 | opt_file.write('==> Args:\n') 108 | for k, v in sorted(args.items()): 109 | opt_file.write(' %s: %s\n' % (str(k), str(v))) 110 | opt_file.write('==> Args:\n') 111 | for k, v in sorted(confs.items()): 112 | opt_file.write(' %s: %s\n' % (str(k), str(v))) 113 | 114 | return self.opt 115 | 116 | -------------------------------------------------------------------------------- /src/datasets_lift/h36m17.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data as data 3 | from h5py import File 4 | import conf 5 | import numpy as np 6 | import pdb 7 | 8 | from utils.utils import rnd 9 | 10 | subject_list = [[[1, 5, 6, 7], [8]], [[1, 5, 6, 7, 8], [9, 11]], [[1, 5, 6, 7, 8, 9], [11]]] 11 | flip_index = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13] 12 | 13 | class H36M17(data.Dataset): 14 | def __init__(self, protocol, split, dense=False, scale=False, noise=0, std_train=0, std_test=0, noise_path=None): 15 | print('==> Initializing H36M %s data' % (split)) 16 | annot = {} 17 | tags = ['idx', 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'action', 'subaction', 'camera'] 18 | f = File('%s/h36m/h36m17.h5' % (conf.data_dir), 'r') 19 | for tag in tags: 20 | annot[tag] = np.asarray(f[tag]).copy() 21 | f.close() 22 | 23 | if dense == False: 24 | idxs = np.mod(annot['idx'], 50) == 1 25 | idxs = np.arange(annot['idx'].shape[0])[idxs] 26 | for tag in tags: 27 | annot[tag] = annot[tag][idxs] 28 | 29 | idxs = np.full(annot['idx'].shape[0], False) 30 | subject = subject_list[protocol][1-int(split=='train' or split=='test_train')] 31 | for i in range(len(subject)): 32 | idxs = idxs + (annot['subject']==subject[i]) 33 | idxs = np.arange(annot['idx'].shape[0])[idxs] 34 | for tag in tags: 35 | annot[tag] = annot[tag][idxs] 36 | 37 | self.protocol = protocol 38 | self.split = split 39 | self.dense = dense 40 | self.scale = scale 41 | self.noise = noise 42 | self.std_train = std_train 43 | self.std_test = std_test 44 | self.noise_path = noise_path 45 | self.annot = annot 46 | self.num_samples = len(self.annot['idx']) 47 | 48 | # image size 49 | self.width = 1000 50 | self.height = 1000 51 | 52 | # load error statistics 53 | self.load_error_stat() 54 | 55 | print('Load %d H36M %s samples' % (self.num_samples, self.split)) 56 | 57 | def get_part_info(self, index): 58 | pose2d = self.annot['pose2d'][index].copy() 59 | bbox = self.annot['bbox'][index].copy() 60 | pose3d = self.annot['pose3d'][index].copy() 61 | cam_f = self.annot['cam_f'][index].copy() 62 | cam_c = self.annot['cam_c'][index].copy() 63 | return pose2d, bbox, pose3d, cam_f, cam_c 64 | 65 | def load_error_stat(self): 66 | # load error stat 67 | if self.split == 'train': 68 | if self.noise == 0: # do not use noise 69 | pass 70 | elif self.noise == 1: # use specified gaussian noise 71 | pass 72 | elif self.noise == 2: # use estimated 2d pose 73 | filename = '%s/%s' % (conf.exp_dir, self.noise_path) 74 | result = torch.load(filename) 75 | self.annot['pose2d'] = result['pred'].cpu().numpy() 76 | elif self.noise == 3: # use estimated single gaussian noise 77 | filename = '%s/%s' % (conf.exp_dir, self.noise_path) 78 | result = torch.load(filename) 79 | mean = result['mean'].numpy() / float(conf.res_in - 1) 80 | self.mean = mean[0] 81 | std = result['std'].numpy() / float(conf.res_in - 1) 82 | self.std = std[0] 83 | elif self.noise == 4: # use estimated mixture noise 84 | filename = '%s/%s' % (conf.exp_dir, self.noise_path) 85 | result = torch.load(filename) 86 | self.mean = result['mean'].numpy() / float(conf.res_in - 1) 87 | self.std = result['std'].numpy() / float(conf.res_in - 1) 88 | self.weight = result['weight'].numpy() 89 | else: 90 | raise ValueError('unsupported noise mode %d' % self.noise) 91 | 92 | def __getitem__(self, index): 93 | # get 2d/3d pose, bounding box, camera information 94 | pose2d, bbox, pose3d, cam_f, cam_c = self.get_part_info(index) 95 | cam_f = cam_f.astype(np.float32) 96 | cam_c = cam_c.astype(np.float32) 97 | 98 | # induce scale variation? 99 | if self.scale == True: 100 | s = 2 ** rnd(0.25) 101 | pose2d = pose2d * s 102 | bbox = bbox * s 103 | cam_f = cam_f * s 104 | cam_c = cam_c * s 105 | width = self.width * s 106 | else: 107 | width = self.width 108 | 109 | # data augmentation (flipping) 110 | if self.split == 'train' and np.random.random() < 0.5: 111 | pose2d_flip = pose2d.copy() 112 | for i in range(len(flip_index)): 113 | pose2d_flip[i] = pose2d[flip_index[i]].copy() 114 | pose3d_flip = pose3d.copy() 115 | for i in range(len(flip_index)): 116 | pose3d_flip[i] = pose3d[flip_index[i]].copy() 117 | pose2d = pose2d_flip.copy() 118 | pose3d = pose3d_flip.copy() 119 | pose2d[:, 0] = width - pose2d[:, 0] 120 | pose3d[:, 0] *= -1 121 | 122 | if self.scale == True: 123 | bbox[0] = width - bbox[0] - bbox[2] 124 | cam_c[0] = width - cam_c[0] 125 | 126 | # original 2d pose 127 | meta2d = pose2d.copy() 128 | 129 | # set 2d pose 130 | if self.noise == 2: 131 | if not self.split == 'train': 132 | pose2d = pose2d - bbox[0:2] 133 | pose2d = pose2d / bbox[2:4] 134 | pose2d = pose2d * float(conf.res_in - 1) 135 | else: 136 | pose2d = pose2d - bbox[0:2] 137 | pose2d = pose2d / bbox[2:4] 138 | if self.split == 'train': 139 | if self.noise == 1: 140 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std_train 141 | elif self.noise == 3: 142 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean 143 | elif self.noise == 4: 144 | val = np.random.random((pose2d.shape[0])) 145 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean) * (val < self.weight).reshape((pose2d.shape[0], 1)) 146 | elif self.split == 'test': 147 | if not self.std_test == 0.0: 148 | if (self.std_test > 0.0) and (self.std_test < 1.0): 149 | # gaussian noise 150 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std_test 151 | elif (self.std_test > 1.0) and (self.std_test < 2.0): 152 | # uniform noise 153 | val = np.random.random((pose2d.shape[0])) + 1.0 154 | pose2d += ((np.random.random((pose2d.shape[0], pose2d.shape[1])) * 100.0 - 50.0) / float(conf.res_in - 1)) * (val < self.std_test).reshape((pose2d.shape[0], 1)) 155 | pose2d = pose2d * float(conf.res_in - 1) 156 | 157 | # root coordinates 158 | coords_root = pose3d[conf.root].copy() 159 | depth_root = coords_root[2].copy() 160 | depth_root_canonical = coords_root[2].copy() / np.sqrt(np.prod(cam_f)) * conf.f0 161 | 162 | # set 3d pose 163 | pose3d = pose3d - pose3d[conf.root] 164 | pose3d = np.delete(pose3d, (conf.root), axis=0) 165 | 166 | # set data 167 | data = {'pose2d': pose2d, 'bbox': bbox, 168 | 'pose3d': pose3d, 'coords_root': coords_root, 169 | 'depth_root': depth_root, 170 | 'depth_root_canonical': depth_root_canonical, 171 | 'cam_f': cam_f, 'cam_c': cam_c, 172 | 'meta2d': meta2d} 173 | 174 | return data 175 | 176 | def __len__(self): 177 | return self.num_samples 178 | 179 | 180 | -------------------------------------------------------------------------------- /src/main_lift.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import torch 6 | import torch.utils.data 7 | import torch.backends.cudnn as cudnn 8 | import pdb 9 | 10 | import conf 11 | from opts_lift import Opts 12 | from datasets_lift.h36m17 import H36M17 13 | from datasets_lift.mpiinf import MPIINF 14 | from datasets_lift.h36m17_mpiinf import H36M17_MPIINF 15 | from models.resnet import ResNet 16 | from train_lift import train, val, test 17 | from utils.logger import Logger 18 | 19 | def main(): 20 | # for repeatable experiments 21 | cudnn.benchmark = False 22 | cudnn.deterministic = True 23 | np.random.seed(0) 24 | torch.manual_seed(0) 25 | torch.cuda.manual_seed(0) 26 | 27 | # options 28 | opt = Opts().parse() 29 | 30 | # dataset loader (train) 31 | if opt.dataset_train == 'h36m': 32 | train_loader = torch.utils.data.DataLoader( 33 | H36M17(opt.protocol, 'train', True, opt.scale, opt.noise, opt.std_train, opt.std_test, opt.noise_path), 34 | batch_size = opt.batch_size, 35 | shuffle = True, 36 | num_workers = int(conf.num_threads)) 37 | elif opt.dataset_train == 'inf': 38 | train_loader = torch.utils.data.DataLoader( 39 | MPIINF('train', opt.noise, opt.std_train, opt.std_test, opt.noise_path), 40 | batch_size = opt.batch_size, 41 | shuffle = True, 42 | num_workers = int(conf.num_threads)) 43 | elif opt.dataset_train == 'h36m_inf': 44 | train_loader = torch.utils.data.DataLoader( 45 | H36M17_MPIINF('train', opt), 46 | batch_size = opt.batch_size, 47 | shuffle = True, 48 | num_workers = int(conf.num_threads)) 49 | else: 50 | raise ValueError('unsupported dataset %s' % opt.dataset_train) 51 | 52 | # dataset loader (valid) 53 | if opt.dataset_test == 'h36m': 54 | val_loader = torch.utils.data.DataLoader( 55 | H36M17(opt.protocol, 'val', False, False, opt.noise, opt.std_train, opt.std_test), 56 | batch_size = opt.batch_size, 57 | shuffle = False, 58 | num_workers = int(conf.num_threads)) 59 | elif opt.dataset_test == 'inf': 60 | val_loader = torch.utils.data.DataLoader( 61 | MPIINF('val', opt.noise, opt.std_train, opt.std_test), 62 | batch_size = opt.batch_size, 63 | shuffle = False, 64 | num_workers = int(conf.num_threads)) 65 | else: 66 | raise ValueError('unsupported dataset %s' % opt.dataset_test) 67 | 68 | # model 69 | if opt.network == 'resnet': 70 | model = ResNet(opt.mode, conf.num_joints, opt.num_layers, opt.num_features).cuda() 71 | else: 72 | raise ValueError('unsupported model %s' % opt.network) 73 | 74 | # multi-gpu 75 | if opt.multi_gpu == True: 76 | model = torch.nn.DataParallel(model, device_ids=[0, 1, 2]) 77 | else: 78 | model = torch.nn.DataParallel(model, device_ids=[0]) 79 | 80 | # optimizer 81 | if opt.opt_method == 'rmsprop': 82 | optimizer = torch.optim.RMSprop(model.parameters(), lr = opt.lr) 83 | elif opt.opt_method == 'adam': 84 | optimizer = torch.optim.Adam(model.parameters(), lr = opt.lr) 85 | else: 86 | raise ValueError('unsupported optimizer %s' % opt.opt_method) 87 | 88 | # scheduler 89 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[200, 300], gamma=0.1) 90 | 91 | # log 92 | log = [] 93 | log.append([]) # epoch 94 | log.append([]) # cost (train) 95 | log.append([]) # error3d1 (train) 96 | log.append([]) # error3d2 (train) 97 | log.append([]) # cost (val) 98 | log.append([]) # error3d1 (val) 99 | log.append([]) # error3d2 (val) 100 | 101 | # load model 102 | idx_start = opt.num_epochs 103 | while idx_start > 0: 104 | file_name = os.path.join(opt.save_dir, 'model_{}.pth'.format(idx_start)) 105 | if os.path.exists(file_name): 106 | state = torch.load(file_name) 107 | model.load_state_dict(state['model']) 108 | optimizer.load_state_dict(state['optimizer']) 109 | scheduler.load_state_dict(state['scheduler']) 110 | log_name = os.path.join(opt.save_dir, 'log_{}.pkl'.format(idx_start)) 111 | if os.path.exists(log_name): 112 | with open(log_name, 'rb') as fin: 113 | log = pickle.load(fin) 114 | break 115 | idx_start -= 1 116 | 117 | # logger 118 | if idx_start == 0: 119 | logger = Logger(opt.save_dir + '/logs') 120 | else: 121 | logger = Logger(opt.save_dir + '/logs', reset=False) 122 | 123 | # train 124 | epoch = idx_start+1 125 | for epoch in range(idx_start+1, opt.num_epochs+1): 126 | # for repeatable experiments 127 | np.random.seed(epoch) 128 | torch.manual_seed(epoch) 129 | torch.cuda.manual_seed(epoch) 130 | 131 | # do scheduler 132 | scheduler.step() 133 | 134 | # perform one epoch of training 135 | cost_train, error3d1_train, error3d2_train = train(epoch, opt, train_loader, model, optimizer) 136 | logger.scalar_summary('cost_train', cost_train, epoch) 137 | logger.scalar_summary('error3d1_train', error3d1_train, epoch) 138 | logger.scalar_summary('error3d2_train', error3d2_train, epoch) 139 | 140 | # perform one epoch of validation 141 | with torch.no_grad(): 142 | cost_val, error3d1_val, error3d2_val = val(epoch, opt, val_loader, model) 143 | logger.scalar_summary('cost_val', cost_val, epoch) 144 | logger.scalar_summary('error3d1_val', error3d1_val, epoch) 145 | logger.scalar_summary('error3d2_val', error3d2_val, epoch) 146 | 147 | # print message to log file 148 | logger.write('%d %1.1e | %.4f %.4f %.4f | %.4f %.4f %.4f\n' % 149 | (epoch, optimizer.param_groups[0]['lr'], cost_train, error3d1_train, error3d2_train, cost_val, error3d1_val, error3d2_val)) 150 | 151 | # 152 | log[0].append(epoch) 153 | log[1].append(cost_train) 154 | log[2].append(error3d1_train) 155 | log[3].append(error3d2_train) 156 | log[4].append(cost_val) 157 | log[5].append(error3d1_val) 158 | log[6].append(error3d2_val) 159 | 160 | # save model 161 | state = { 162 | 'epoch': epoch, 163 | 'model': model.state_dict(), 164 | 'optimizer': optimizer.state_dict(), 165 | 'scheduler': scheduler.state_dict() 166 | } 167 | if epoch % opt.save_intervals == 0: 168 | torch.save(state, os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch))) 169 | log_name = os.path.join(opt.save_dir, 'log_{}.pkl'.format(epoch)) 170 | with open(log_name, 'wb') as fout: 171 | pickle.dump(log, fout) 172 | 173 | logger.close() 174 | 175 | # save final model 176 | file_name = os.path.join(opt.save_dir, 'final_model.pth') 177 | torch.save(state, file_name) 178 | 179 | # save final log 180 | log_name = os.path.join(opt.save_dir, 'final_log.pkl') 181 | with open(log_name, 'wb') as fout: 182 | pickle.dump(log, fout) 183 | 184 | # plotting 185 | x = range(1, opt.num_epochs+1) 186 | cost_train = np.array(log[1]) 187 | error3d1_train = np.array(log[2]) 188 | error3d2_train = np.array(log[3]) 189 | cost_val = np.array(log[4]) 190 | error3d1_val = np.array(log[5]) 191 | error3d2_val = np.array(log[6]) 192 | 193 | fig, ax = plt.subplots() 194 | ax.plot(x, cost_train, 'r') 195 | ax.plot(x, cost_val, 'b') 196 | ax.set(xlabel='epoch', ylabel='cost', title='cost') 197 | plt.legend(('cost_train', 'cost_val')) 198 | ax.grid() 199 | fig.savefig(os.path.join(opt.save_dir, 'cost.png')) 200 | 201 | fig, ax = plt.subplots() 202 | ax.plot(x, error3d1_train, 'r') 203 | ax.plot(x, error3d2_train, 'm') 204 | ax.plot(x, error3d1_val, 'b') 205 | ax.plot(x, error3d2_val, 'c') 206 | ax.set(xlabel='epoch', ylabel='error3d', title='3D error (mm)') 207 | plt.legend(('error3d1_train', 'error3d2_train', 'error3d1_val', 'error3d2_val')) 208 | ax.grid() 209 | fig.savefig(os.path.join(opt.save_dir, 'error3d.png')) 210 | 211 | #--------------------------------------------------------------------------- 212 | # dataset loader (test) 213 | if opt.dataset_test == 'h36m': 214 | test_loader = torch.utils.data.DataLoader( 215 | H36M17(opt.protocol, 'test', True, False, opt.noise, opt.std_train, opt.std_test), 216 | batch_size = opt.batch_size, 217 | shuffle = False, 218 | num_workers = int(conf.num_threads)) 219 | elif opt.dataset_test == 'inf': 220 | test_loader = torch.utils.data.DataLoader( 221 | MPIINF('val', opt.noise, opt.std_train, opt.std_test), 222 | batch_size = opt.batch_size, 223 | shuffle = False, 224 | num_workers = int(conf.num_threads)) 225 | else: 226 | raise ValueError('unsupported dataset %s' % opt.dataset_test) 227 | 228 | # final evaluation 229 | with torch.no_grad(): 230 | cost_final, error3d1_final, error3d2_final = test(epoch, opt, test_loader, model) 231 | 232 | if __name__ == '__main__': 233 | main() 234 | 235 | -------------------------------------------------------------------------------- /src/datasets_lift/h36m17_mpiinf.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data as data 3 | from h5py import File 4 | import conf 5 | import numpy as np 6 | import pdb 7 | 8 | class H36M17_MPIINF(data.Dataset): 9 | def __init__(self, protocol, split, dense=False, noise=0, std_train=0, std_test=0, noise_path=None): 10 | print('==> Initializing fusion %s data' % (split)) 11 | 12 | annot1 = {} 13 | tags = ['idx', 'pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'action', 'subaction', 'camera'] 14 | f = File('%s/h36m/h36m17.h5' % (conf.data_dir), 'r') 15 | for tag in tags: 16 | annot1[tag] = np.asarray(f[tag]).copy() 17 | f.close() 18 | 19 | if dense == False: 20 | idxs = np.mod(annot1['idx'], 50) == 1 21 | idxs = np.arange(annot1['idx'].shape[0])[idxs] 22 | for tag in tags: 23 | annot1[tag] = annot1[tag][idxs] 24 | 25 | idxs = np.full(annot1['idx'].shape[0], False) 26 | subject = subject_list[protocol][1-int(split=='train' or split=='test_train')] 27 | for i in range(len(subject)): 28 | idxs = idxs + (annot1['subject']==subject[i]) 29 | idxs = np.arange(annot1['idx'].shape[0])[idxs] 30 | for tag in tags: 31 | annot1[tag] = annot1[tag][idxs] 32 | 33 | annot2 = {} 34 | tags = ['pose2d', 'pose3d', 'bbox', 'cam_f', 'cam_c', 'subject', 'sequence', 'video'] 35 | f = File('%s/inf/inf_%s.h5' % (conf.data_dir, split), 'r') 36 | for tag in tags: 37 | annot2[tag] = np.asarray(f[tag]).copy() 38 | f.close() 39 | 40 | annot = {} 41 | annot['pose2d'] = np.concatenate((annot1['pose2d'], annot2['pose2d'].astype(np.float32)), axis=0) 42 | annot['pose3d'] = np.concatenate((annot1['pose3d'], annot2['pose3d'].astype(np.float32)), axis=0) 43 | annot['bbox'] = np.concatenate((annot1['bbox'], annot2['bbox']), axis=0) 44 | annot['cam_f'] = np.concatenate((annot1['cam_f'], annot2['cam_f']), axis=0) 45 | annot['cam_c'] = np.concatenate((annot1['cam_c'], annot2['cam_c']), axis=0) 46 | 47 | self.split = split 48 | self.noise = noise 49 | self.std_train = std_train 50 | self.std_test = std_test 51 | self.noise_path = noise_path 52 | self.annot = annot 53 | self.num_samples = self.annot['pose2d'].shape[0] 54 | 55 | # load error statistics 56 | self.load_error_stat() 57 | 58 | print('Load %d fusion %s samples' % (self.num_samples, split)) 59 | 60 | def get_part_info(self, index): 61 | pose2d = self.annot['pose2d'][index].copy() 62 | bbox = self.annot['bbox'][index].copy() 63 | pose3d = self.annot['pose3d'][index].copy() 64 | cam_f = self.annot['cam_f'][index].copy() 65 | cam_c = self.annot['cam_c'][index].copy() 66 | return pose2d, bbox, pose3d, cam_f, cam_c 67 | 68 | def load_error_stat(self): 69 | # load error stat 70 | if self.split == 'train': 71 | if self.noise == 2: 72 | filename = '%s_protocol%d/resnet152-int/fusion/rmsprop_lr1.0e-05_batch48/test_train.pth' % (conf.exp_dir, self.protocol) 73 | result = torch.load(filename) 74 | self.annot['pose2d'] = result['pred'].cpu().numpy() 75 | elif self.noise == 3: 76 | filename = '../pose2d-hmap-resnet/analysis/stat_fusion_protocol0_simple.pth' 77 | result = torch.load(filename) 78 | mean = result['mean'].numpy() / float(conf.res_in - 1) 79 | self.mean = mean[0] 80 | std = result['std'].numpy() / float(conf.res_in - 1) 81 | self.std = std[0] 82 | elif self.noise == 4: 83 | filename = '../pose2d-hmap-resnet/analysis/stat_fusion_protocol0_1d_em1.pth' 84 | result = torch.load(filename) 85 | self.mean = result['mean'].numpy() / float(conf.res_in - 1) 86 | self.std = result['std'].numpy() / float(conf.res_in - 1) 87 | self.weight = result['weight'].numpy() 88 | elif self.noise == 5: 89 | filename = '../pose2d-hmap-resnet/analysis/stat_fusion_protocol0_1d_em2.pth' 90 | result = torch.load(filename) 91 | self.mean1 = result['mean1'].numpy() / float(conf.res_in - 1) 92 | self.std1 = result['std1'].numpy() / float(conf.res_in - 1) 93 | self.mean2 = result['mean2'].numpy() / float(conf.res_in - 1) 94 | self.std2 = result['std2'].numpy() / float(conf.res_in - 1) 95 | self.weight = result['weight'].numpy() 96 | elif self.noise == 6: 97 | filename = '../pose2d-hmap-resnet/analysis/stat_fusion_protocol0_2d_em1.pth' 98 | result = torch.load(filename) 99 | self.mean = result['mean'].numpy() / float(conf.res_in - 1) 100 | self.std = result['std'].numpy() / float(conf.res_in - 1) 101 | self.weight = result['weight'].numpy() 102 | elif self.noise == 7: 103 | filename = '../pose2d-hmap-resnet/analysis/stat_fusion_protocol0_2d_em2.pth' 104 | result = torch.load(filename) 105 | self.mean1 = result['mean1'].numpy() / float(conf.res_in - 1) 106 | self.std1 = result['std1'].numpy() / float(conf.res_in - 1) 107 | self.mean2 = result['mean2'].numpy() / float(conf.res_in - 1) 108 | self.std2 = result['std2'].numpy() / float(conf.res_in - 1) 109 | self.weight = result['weight'].numpy() 110 | elif self.noise == 8: 111 | filename = '%s/%s' % (conf.exp_dir, self.noise_path) 112 | result = torch.load(filename) 113 | self.mean = result['mean'].numpy() / float(conf.res_in - 1) 114 | self.std = result['std'].numpy() / float(conf.res_in - 1) 115 | self.weight = result['weight'].numpy() 116 | 117 | def __getitem__(self, index): 118 | # get 2D/3D pose, bounding box, camera information 119 | pose2d, bbox, pose3d, cam_f, cam_c = self.get_part_info(index) 120 | cam_f = cam_f.astype(np.float32) 121 | cam_c = cam_c.astype(np.float32) 122 | 123 | # original 2D pose 124 | meta2d = pose2d.copy() 125 | 126 | # set 2D pose 127 | if self.noise == 2: 128 | if not self.split == 'train': 129 | pose2d = pose2d - bbox[0:2] 130 | pose2d = pose2d / bbox[2:4] 131 | pose2d = pose2d * float(conf.res_in - 1) 132 | else: 133 | pose2d = pose2d - bbox[0:2] 134 | pose2d = pose2d / bbox[2:4] 135 | if self.split == 'train': 136 | if self.noise == 1: 137 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std_train 138 | if self.noise == 3: 139 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean 140 | elif self.noise == 4: 141 | val = np.random.random((pose2d.shape[0], pose2d.shape[1])) 142 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean) * (val < self.weight) 143 | pose2d += ((np.random.random((pose2d.shape[0], pose2d.shape[1])) * 200.0 - 100.0) / float(conf.res_in - 1)) * (val >= self.weight) 144 | elif self.noise == 5: 145 | val = np.random.random((pose2d.shape[0], pose2d.shape[1])) 146 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std1 + self.mean1) * (val < self.weight) 147 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std2 + self.mean2) * (val >= self.weight) 148 | elif self.noise == 6: 149 | val = np.random.random((pose2d.shape[0])) 150 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean) * (val < self.weight).reshape((pose2d.shape[0], 1)) 151 | pose2d += ((np.random.random((pose2d.shape[0], pose2d.shape[1])) * 100.0 - 50.0) / float(conf.res_in - 1)) * (val >= self.weight).reshape((pose2d.shape[0], 1)) 152 | elif self.noise == 7: 153 | val = np.random.random((pose2d.shape[0])) 154 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std1 + self.mean1) * (val < self.weight).reshape((pose2d.shape[0], 1)) 155 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std2 + self.mean2) * (val >= self.weight).reshape((pose2d.shape[0], 1)) 156 | elif self.noise == 8: 157 | val = np.random.random((pose2d.shape[0])) 158 | pose2d += (np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std + self.mean) * (val < self.weight).reshape((pose2d.shape[0], 1)) 159 | elif self.split == 'test': 160 | if not self.std_test == 0.0: 161 | if (self.std_test > 0.0) and (self.std_test < 1.0): 162 | # gaussian noise 163 | pose2d = pose2d + np.random.randn(pose2d.shape[0], pose2d.shape[1]) * self.std_test 164 | elif (self.std_test > 1.0) and (self.std_test < 2.0): 165 | # uniform noise 166 | val = np.random.random((pose2d.shape[0])) + 1.0 167 | pose2d += ((np.random.random((pose2d.shape[0], pose2d.shape[1])) * 100.0 - 50.0) / float(conf.res_in - 1)) * (val < self.opt.std_test).reshape((pose2d.shape[0], 1)) 168 | pose2d = pose2d * float(conf.res_in - 1) 169 | 170 | # root coordinates 171 | coords_root = pose3d[conf.root].copy() 172 | depth_root = coords_root[2].copy() 173 | depth_root_canonical = coords_root[2].copy() / np.sqrt(np.prod(cam_f)) * conf.f0 174 | 175 | # set 3d pose 176 | pose3d = pose3d - pose3d[conf.root] 177 | pose3d = np.delete(pose3d, (conf.root), axis=0) 178 | 179 | # set data 180 | data = {'pose2d': pose2d, 'bbox': bbox, 181 | 'pose3d': pose3d, 'coords_root': coords_root, 182 | 'depth_root': depth_root, 183 | 'depth_root_canonical': depth_root_canonical, 184 | 'cam_f': cam_f, 'cam_c': cam_c, 185 | 'meta2d': meta2d} 186 | 187 | return data 188 | 189 | def __len__(self): 190 | return self.num_samples 191 | 192 | -------------------------------------------------------------------------------- /src/train_2d.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn as nn 4 | import conf 5 | from utils.utils import AverageMeter 6 | from utils.eval import compute_error, compute_error_direct 7 | from utils.eval import compute_error3d, compute_error3d_pa, compute_error_root 8 | from progress.bar import Bar 9 | import pdb 10 | 11 | import scipy.io as sio 12 | 13 | def weighted_mse_loss(prediction, target, weight=1.0): 14 | return torch.sum(weight*(prediction-target)**2)/prediction.shape[0] 15 | 16 | def weighted_l1_loss(prediction, target, weight=1.0): 17 | return torch.sum(weight*torch.abs(prediction-target))/prediction.shape[0] 18 | 19 | def step(split, epoch, opt, dataLoader, model, optimizer = None): 20 | # training mode 21 | if split == 'train': 22 | model.train() 23 | else: 24 | model.eval() 25 | 26 | # save final predictions 27 | if split == 'test' and opt.save_results == True: 28 | pred2d = [] 29 | gt2d = [] 30 | pred3d = [] 31 | gt3d = [] 32 | if split == 'test_train': 33 | pred = [] 34 | gt = [] 35 | 36 | # initialize evaluations 37 | cost = AverageMeter() 38 | error2d = [] 39 | for i in range(2): error2d.append(AverageMeter()) 40 | error3d = AverageMeter() 41 | error3d_pa = AverageMeter() 42 | error3d_root = AverageMeter() 43 | if split == 'test': 44 | error3dx, error3dy, error3dz = AverageMeter(), AverageMeter(), AverageMeter() 45 | error_action = [] 46 | for i in range(conf.num_actions): error_action.append(AverageMeter()) 47 | 48 | num_iters = len(dataLoader) 49 | bar = Bar('==>', max=num_iters) 50 | 51 | # for each mini-batch 52 | for i, (idx, data) in enumerate(dataLoader): 53 | img = data['inp'].to("cuda") 54 | hmap = data['hmap'].to("cuda") 55 | pose2d = data['pose2d'].to("cuda") 56 | valid2d = data['valid2d'].to("cuda") 57 | valid2d_sum = valid2d.sum().item() 58 | 59 | # for 3d 60 | pose3d = data['pose3d'].to("cuda") 61 | valid3d = data['valid3d'].float().to("cuda") 62 | valid3d_sum = valid3d.sum().item() 63 | bbox = data['bbox'].float().to("cuda") 64 | 65 | # for depth 66 | cam_f = data['cam_f'].to("cuda") 67 | cam_c = data['cam_c'].to("cuda") 68 | meta3d = data['meta3d'].to("cuda") 69 | action = data['action'].to("cuda") 70 | coords_root = data['coords_root'].to("cuda") 71 | depth_root = data['depth_root'].to("cuda") 72 | depth_root_canonical = data['depth_root_canonical'].to("cuda") 73 | 74 | # constants 75 | nb = img.shape[0] 76 | nj = conf.num_joints 77 | 78 | # forward propagation 79 | if split == 'test': 80 | model.module.set_fliptest(opt.fliptest) 81 | if opt.lift3d == 0: 82 | outputs = model(img) 83 | else: 84 | outputs = model([img, bbox, cam_c]) 85 | 86 | # compute cost 87 | loss = 0.0 88 | if opt.lift3d == 0: 89 | if opt.integral == 0: 90 | loss += weighted_mse_loss(outputs[0], hmap, valid2d.reshape(nb, nj, 1, 1)) 91 | else: 92 | loss += weighted_mse_loss(outputs[0], hmap, valid2d.reshape(nb, nj, 1, 1)) 93 | loss += weighted_l1_loss(outputs[1], pose2d, valid2d.reshape(nb, nj, 1))*opt.weight1 94 | else: 95 | loss += weighted_l1_loss(outputs[2], pose3d, valid3d.reshape(nb, 1, 1)) 96 | 97 | # update model parameters with backpropagation 98 | if split == 'train': 99 | optimizer.zero_grad() 100 | loss.backward() 101 | optimizer.step() 102 | 103 | # update evaluations 104 | cost.update(loss.detach().item(), nb) 105 | if opt.lift3d == 0: 106 | if opt.integral == 0: 107 | error2d[0].update(compute_error(outputs[0].detach(), pose2d.detach(), valid2d.detach()), valid2d_sum) 108 | else: 109 | error2d[0].update(compute_error(outputs[0].detach(), pose2d.detach(), valid2d.detach()), valid2d_sum) 110 | error2d[1].update(compute_error_direct(outputs[1].detach(), pose2d.detach(), valid2d.detach()), valid2d_sum) 111 | else: 112 | # unnormalized 2d pose 113 | pred = outputs[1].detach() 114 | pred = pred / 255.0 115 | pred = pred * torch.reshape(bbox[:, 2:4], (nb, 1, 2)) 116 | pred = pred + torch.reshape(bbox[:, 0:2], (nb, 1, 2)) 117 | 118 | # get predicted root coordinates 119 | x = pred[:, conf.root, 0] 120 | y = pred[:, conf.root, 1] 121 | cx = cam_c[:, 0].detach() 122 | cy = cam_c[:, 1].detach() 123 | if opt.lift3d == 1: # Z is not canonical depth 124 | Z = outputs[3].detach() 125 | f = torch.sqrt(torch.prod(cam_f.detach(), 1)) 126 | X = (x-cx)*Z/f 127 | Y = (y-cy)*Z/f 128 | pred_root = torch.cat((X.view(nb,1), Y.view(nb,1), Z.view(nb,1)), 1) 129 | elif opt.lift3d == 2: # Z is canonical depth 130 | Z = outputs[3].detach() 131 | X = (x-cx)*Z/conf.f0 132 | Y = (y-cy)*Z/conf.f0 133 | f = torch.sqrt(torch.prod(cam_f.detach(), 1))/conf.f0 134 | Z = Z*f 135 | pred_root = torch.cat((X.view(nb,1), Y.view(nb,1), Z.view(nb,1)), 1) 136 | 137 | # evaluate 2d estimate 138 | error2d[0].update(compute_error(outputs[0].detach(), pose2d.detach(), valid2d.detach()), valid2d_sum) 139 | error2d[1].update(compute_error_direct(outputs[1].detach(), pose2d.detach(), valid2d.detach()), valid2d_sum) 140 | 141 | # evaluate 3d estimate 142 | error3d.update(compute_error3d(outputs[2].detach(), pose3d.detach(), valid3d.detach())) 143 | error3d_pa.update(compute_error3d_pa(outputs[2].detach(), pose3d.detach())) 144 | 145 | # evaluate root coordinates (MRPE) 146 | error3d_root.update(compute_error_root(pred_root, coords_root.detach())) 147 | error3dx.update(torch.sum(torch.abs(pred_root[:,0]-coords_root[:,0].detach()))/nb) 148 | error3dy.update(torch.sum(torch.abs(pred_root[:,1]-coords_root[:,1].detach()))/nb) 149 | error3dz.update(torch.sum(torch.abs(pred_root[:,2]-coords_root[:,2].detach()))/nb) 150 | 151 | if split == 'test': 152 | for j in range(nb): 153 | if opt.protocol == 1: 154 | error_action[int(action[j].item())-2].update(compute_error3d(outputs[2][j].reshape(1,nj-1,3).detach(), pose3d[j].reshape(1,nj-1,3).detach(), valid3d[j].detach())) 155 | elif opt.protocol == 2: 156 | error_action[int(action[j].item())-2].update(compute_error3d_pa(outputs[2][j].reshape(1,nj-1,3).detach(), pose3d[j].reshape(1,nj-1,3).detach())) 157 | 158 | # for computing error statistics 159 | if split == 'test' and opt.save_results == True: 160 | pred2d.append(outputs[1].detach()) 161 | gt2d.append(pose2d.detach()) 162 | if opt.lift3d != 0: 163 | temp = torch.zeros(nb, nj, 3).to("cuda") 164 | temp[:,:conf.root,:] = outputs[2][:,:conf.root,:] 165 | temp[:,conf.root+1:,:] = outputs[2][:,conf.root:,:] 166 | #temp = temp + torch.reshape(outputs[3], (nb,1,3)) 167 | pred3d.append(temp.detach()) 168 | gt3d.append(meta3d) 169 | 170 | # for computing error statistics 171 | if split == 'test_train': 172 | pred.append(outputs[1].detach()) 173 | gt.append(pose2d.detach()) 174 | 175 | msg = '{split} Epoch: [{0}][{1}/{2}]| Total: {total:} | ETA: {eta:}'.format(epoch, i, num_iters, split=split, total=bar.elapsed_td, eta=bar.eta_td) 176 | if split == 'train': 177 | msg = '{} | LR: {:1.1e}'.format(msg, optimizer.param_groups[0]['lr']) 178 | msg = '{} | Cost {cost.avg:.2f}'.format(msg, cost=cost) 179 | for j in range(2): 180 | msg = '{} | E{:d} {:.2f}'.format(msg, j, error2d[j].avg) 181 | msg = '{} | E3D {:.2f} | E3D_PA {:.2f}'.format(msg, error3d.avg, error3d_pa.avg) 182 | Bar.suffix = msg 183 | bar.next() 184 | 185 | bar.finish() 186 | 187 | # save final test results 188 | if split == 'test' and opt.save_results == True: 189 | pred2d = torch.cat(pred2d, dim=0) 190 | gt2d = torch.cat(gt2d, dim=0) 191 | if opt.lift3d == 0: 192 | result = {'pred2d': pred2d, 'gt2d': gt2d} 193 | else: 194 | pred3d = torch.cat(pred3d, dim=0) 195 | gt3d = torch.cat(gt3d, dim=0) 196 | result = {'pred2d': pred2d, 'gt2d': gt2d, 'pred3d': pred3d, 'gt3d': gt3d} 197 | filename = os.path.join(opt.save_dir, 'result.pth') 198 | torch.save(result, filename) 199 | 200 | if opt.lift3d == 0: 201 | result = {'pred2d': pred2d.cpu().numpy(), 'gt2d': gt2d.cpu().numpy()} 202 | else: 203 | result = {'pred2d': pred2d.cpu().numpy(), 'gt2d': gt2d.cpu().numpy(), 'pred3d': pred3d.cpu().numpy(), 'gt3d': gt3d.cpu().numpy()} 204 | filename = os.path.join(opt.save_dir, 'result.mat') 205 | sio.savemat(filename, result) 206 | 207 | # save train results 208 | if split == 'test_train': 209 | pred = torch.cat(pred, dim=0) 210 | gt = torch.cat(gt, dim=0) 211 | result = {'pred': pred, 'gt': gt} 212 | filename = os.path.join(opt.save_dir, 'test_train.pth') 213 | torch.save(result, filename) 214 | 215 | # 216 | if split == 'test': 217 | if opt.fliptest == True: 218 | file = open(os.path.join(opt.save_dir, 'result_fliptest.txt'), 'w') 219 | else: 220 | file = open(os.path.join(opt.save_dir, 'result.txt'), 'w') 221 | file.write('Loss for test set = {:6f}\n'.format(cost.avg)) 222 | file.write('(Heatmap) 2D pixel error for test set = {:6f}\n'.format(error2d[0].avg)) 223 | file.write('(Coord) 2D pixel error for test set = {:6f}\n'.format(error2d[1].avg)) 224 | file.write('---------------------------------------------------\n') 225 | file.write('3D MPJPE error for test set = {:6f}\n'.format(error3d.avg)) 226 | file.write('3D PA MPJPE error for test set = {:6f}\n'.format(error3d_pa.avg)) 227 | file.write('Global root error for test set = {:6f}\n'.format(error3d_root.avg)) 228 | file.write('---------------------------------------------------\n') 229 | file.write('Root error in X direction for test set = {:6f}\n'.format(error3dx.avg)) 230 | file.write('Root error in Y direction for test set = {:6f}\n'.format(error3dy.avg)) 231 | file.write('Root error in Z direction for test set = {:6f}\n'.format(error3dz.avg)) 232 | file.write('---------------------------------------------------\n') 233 | for i in range(conf.num_actions): 234 | file.write('3D MPJPE error for action %d = %.6f\n' % (i, error_action[i].avg)) 235 | file.close() 236 | 237 | return cost.avg, [error2d[0].avg, error2d[1].avg], error3d.avg, error3d_pa.avg 238 | 239 | def train(epoch, opt, train_loader, model, optimizer): 240 | return step('train', epoch, opt, train_loader, model, optimizer) 241 | 242 | def val(epoch, opt, val_loader, model): 243 | return step('val', epoch, opt, val_loader, model) 244 | 245 | def test(epoch, opt, test_loader, model): 246 | return step('test', epoch, opt, test_loader, model) 247 | 248 | def test_train(epoch, opt, test_train_loader, model): 249 | return step('test_train', epoch, opt, test_train_loader, model) 250 | 251 | -------------------------------------------------------------------------------- /src/main_2d.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import torch 6 | import torch.utils.data 7 | import torch.backends.cudnn as cudnn 8 | import pdb 9 | 10 | import conf 11 | from opts_2d import Opts 12 | from models.resnet2d import ResNet 13 | from models.resnet2d_int import ResNetInt 14 | from models.resnet2d_lift import ResNetLift 15 | from datasets_2d.h36m17 import H36M17 16 | from datasets_2d.mpii import MPII 17 | from datasets_2d.h36m17_mpii import H36M17_MPII 18 | from datasets_2d.mpiinf import MPIINF 19 | from datasets_2d.mpiinf_mpii import MPIINF_MPII 20 | from train_2d import train, val, test, test_train 21 | from utils.logger import Logger 22 | 23 | def main(): 24 | # for repeatable experiments 25 | cudnn.benchmark = False 26 | cudnn.deterministic = True 27 | np.random.seed(0) 28 | torch.manual_seed(0) 29 | torch.cuda.manual_seed(0) 30 | 31 | # options 32 | opt = Opts().parse() 33 | 34 | # model 35 | if opt.lift3d == 0: 36 | if opt.integral == 0: 37 | model = ResNet(opt.network, conf.num_joints).cuda() 38 | else: 39 | if opt.model_2d_path is None: 40 | model = ResNetInt(opt.network, conf.num_joints).cuda() 41 | else: 42 | filename = '%s/%s' % (conf.exp_dir, opt.model_2d_path) 43 | model = ResNetInt(opt.network, conf.num_joints, filename).cuda() 44 | else: 45 | filename_2d = '%s/%s' % (conf.exp_dir, opt.model_2d_path) 46 | filename_lift = '%s/%s' % (conf.exp_dir, opt.model_lift_path) 47 | model = ResNetLift(opt.network, conf.num_joints, opt.num_layers, opt.num_features, opt.mode, \ 48 | filename_2d, filename_lift).cuda() 49 | 50 | # multi-gpu 51 | if opt.multi_gpu == True: 52 | model = torch.nn.DataParallel(model, device_ids=[0, 1, 2]) 53 | else: 54 | model = torch.nn.DataParallel(model, device_ids=[0]) 55 | 56 | # dataset loader (train) 57 | if opt.dataset_train == 'h36m': 58 | train_loader = torch.utils.data.DataLoader( 59 | H36M17(opt.protocol, 'train'), 60 | batch_size = opt.batch_size, 61 | shuffle = True, 62 | num_workers = int(conf.num_threads)) 63 | elif opt.dataset_train == 'mpii': 64 | train_loader = torch.utils.data.DataLoader( 65 | MPII(), 66 | batch_size = opt.batch_size, 67 | shuffle = True, 68 | num_workers = int(conf.num_threads)) 69 | elif opt.dataset_train == 'h36m_mpii': 70 | train_loader = torch.utils.data.DataLoader( 71 | H36M17_MPII(opt.protocol, 'train'), 72 | batch_size = opt.batch_size, 73 | shuffle = True, 74 | num_workers = int(conf.num_threads)) 75 | elif opt.dataset_train == 'inf': 76 | train_loader = torch.utils.data.DataLoader( 77 | MPIINF('train'), 78 | batch_size = opt.batch_size, 79 | shuffle = True, 80 | num_workers = int(conf.num_threads)) 81 | elif opt.dataset_train == 'inf_mpii': 82 | train_loader = torch.utils.data.DataLoader( 83 | MPIINF_MPII('train'), 84 | batch_size = opt.batch_size, 85 | shuffle = True, 86 | num_workers = int(conf.num_threads)) 87 | else: 88 | raise ValueError('unsupported dataset %s' % opt.dataset_train) 89 | 90 | # dataset loader (val) 91 | if opt.dataset_test == 'h36m': 92 | val_loader = torch.utils.data.DataLoader( 93 | H36M17(opt.protocol, 'val'), 94 | batch_size = opt.batch_size, 95 | shuffle = False, 96 | num_workers = int(conf.num_threads)) 97 | elif opt.dataset_test == 'inf': 98 | val_loader = torch.utils.data.DataLoader( 99 | MPIINF('val'), 100 | batch_size = opt.batch_size, 101 | shuffle = False, 102 | num_workers = int(conf.num_threads)) 103 | else: 104 | raise ValueError('unsupported dataset %s' % opt.dataset_test) 105 | 106 | # optimizer 107 | if opt.opt_method == 'rmsprop': 108 | optimizer = torch.optim.RMSprop(filter(lambda p: p.requires_grad, model.parameters()), 109 | lr = opt.lr) 110 | elif opt.opt_method == 'adam': 111 | optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), 112 | lr = opt.lr) 113 | else: 114 | raise ValueError('unsupported optimizer %s' % opt.opt_method) 115 | 116 | # scheduler 117 | if opt.lift3d == 0: 118 | if opt.integral == 0: 119 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30], gamma=0.1) 120 | else: 121 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[], gamma=0.1) 122 | else: 123 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[], gamma=0.1) 124 | 125 | # log 126 | log = [] 127 | log.append([]) # epoch 128 | log.append([]) # cost (train) 129 | log.append([]) # error2d1 (train) 130 | log.append([]) # error2d2 (train) 131 | log.append([]) # cost (val) 132 | log.append([]) # error2d1 (val) 133 | log.append([]) # error2d2 (val) 134 | 135 | # train 136 | if opt.lift3d == 0: 137 | idx_start = opt.num_epochs 138 | while idx_start > 0: 139 | file_name = os.path.join(opt.save_dir, 'model_{}.pth'.format(idx_start)) 140 | if os.path.exists(file_name): 141 | state = torch.load(file_name) 142 | model.load_state_dict(state['model']) 143 | optimizer.load_state_dict(state['optimizer']) 144 | scheduler.load_state_dict(state['scheduler']) 145 | log_name = os.path.join(opt.save_dir, 'log_{}.pkl'.format(idx_start)) 146 | if os.path.exists(log_name): 147 | with open(log_name, 'rb') as fin: 148 | log = pickle.load(fin) 149 | break 150 | idx_start -= 1 151 | 152 | # logger 153 | if idx_start == 0: 154 | logger_error = Logger(opt.save_dir + '/logs_error') 155 | else: 156 | logger_error = Logger(opt.save_dir + '/logs_error', reset=False) 157 | 158 | for epoch in range(idx_start+1, opt.num_epochs+1): 159 | # for repeatable experiments 160 | np.random.seed(epoch) 161 | torch.manual_seed(epoch) 162 | torch.cuda.manual_seed(epoch) 163 | 164 | # do scheduler 165 | scheduler.step() 166 | 167 | # perform one epoch of training 168 | cost_train, error2d_train, error3d_train, error3d_pa_train = train(epoch, opt, train_loader, model, optimizer) 169 | logger_error.scalar_summary('cost_train', cost_train, epoch) 170 | for i in range(2): 171 | logger_error.scalar_summary('error2d%d_train' % i, error2d_train[i], epoch) 172 | logger_error.scalar_summary('error3d_train', error3d_train, epoch) 173 | logger_error.scalar_summary('error3d_pa_train', error3d_pa_train, epoch) 174 | 175 | # perform one epoch of validation 176 | with torch.no_grad(): 177 | cost_val, error2d_val, error3d_val, error3d_pa_val = val(epoch, opt, val_loader, model) 178 | logger_error.scalar_summary('cost_val', cost_val, epoch) 179 | for i in range(2): 180 | logger_error.scalar_summary('error2d%d_val' % i, error2d_val[i], epoch) 181 | logger_error.scalar_summary('error3d_val', error3d_val, epoch) 182 | logger_error.scalar_summary('error3d_pa_val', error3d_pa_val, epoch) 183 | 184 | # print message to log file 185 | msg = '%d %1.1e' % (epoch, optimizer.param_groups[0]['lr']) 186 | msg = '%s | %.4f' % (msg, cost_train) 187 | for i in range(2): msg = '%s %.4f' % (msg, error2d_train[i]) 188 | msg = '%s %.4f %.4f' % (msg, error3d_train, error3d_pa_train) 189 | msg = '%s | %.4f' % (msg, cost_val) 190 | for i in range(2): msg = '%s %.4f' % (msg, error2d_val[i]) 191 | msg = '%s %.4f %.4f' % (msg, error3d_val, error3d_pa_val) 192 | logger_error.write('%s\n' % msg) 193 | 194 | # 195 | log[0].append(epoch) 196 | log[1].append(cost_train) 197 | log[2].append(error2d_train[0]) 198 | log[3].append(error2d_train[1]) 199 | log[4].append(cost_val) 200 | log[5].append(error2d_val[0]) 201 | log[6].append(error2d_val[1]) 202 | 203 | # save model 204 | state = { 205 | 'epoch': epoch, 206 | 'model': model.state_dict(), 207 | 'optimizer': optimizer.state_dict(), 208 | 'scheduler': scheduler.state_dict() 209 | } 210 | if epoch % opt.save_intervals == 0: 211 | torch.save(state, os.path.join(opt.save_dir, 'model_%d.pth' % (epoch))) 212 | log_name = os.path.join(opt.save_dir, 'log_%d.pkl' % (epoch)) 213 | with open(log_name, 'wb') as fout: 214 | pickle.dump(log, fout) 215 | 216 | logger_error.close() 217 | 218 | # save final model 219 | file_name = os.path.join(opt.save_dir, 'final_model.pth') 220 | torch.save(state, file_name) 221 | 222 | # save final log 223 | log_name = os.path.join(opt.save_dir, 'final_log.pkl') 224 | with open(log_name, 'wb') as fout: 225 | pickle.dump(log, fout) 226 | 227 | # plotting 228 | x = range(1, opt.num_epochs+1) 229 | cost_train = np.array(log[1]) 230 | error2d1_train = np.array(log[2]) 231 | error2d2_train = np.array(log[3]) 232 | cost_val = np.array(log[4]) 233 | error2d1_val = np.array(log[5]) 234 | error2d2_val = np.array(log[6]) 235 | 236 | fig, ax = plt.subplots() 237 | ax.plot(x, cost_train, 'r') 238 | ax.plot(x, cost_val, 'b') 239 | ax.set(xlabel='epoch', ylabel='cost', title='cost') 240 | plt.legend(('cost_train', 'cost_val')) 241 | ax.grid() 242 | fig.savefig(os.path.join(opt.save_dir, 'cost.png')) 243 | 244 | fig, ax = plt.subplots() 245 | ax.plot(x, error2d1_train, 'r') 246 | ax.plot(x, error2d2_train, 'm') 247 | ax.plot(x, error2d1_val, 'b') 248 | ax.plot(x, error2d2_val, 'c') 249 | ax.set(xlabel='epoch', ylabel='error2d', title='2D error (pixel)') 250 | plt.legend(('error2d1_train', 'error2d2_train', 'error2d1_val', 'error2d2_val')) 251 | ax.grid() 252 | fig.savefig(os.path.join(opt.save_dir, 'error2d.png')) 253 | 254 | #-------------------------------------------------------------------- 255 | # test loader for final prediction 256 | if opt.dataset_test == 'h36m': 257 | test_loader = torch.utils.data.DataLoader( 258 | H36M17(opt.protocol, 'val'), 259 | batch_size = opt.batch_size, 260 | shuffle = False, 261 | num_workers = int(conf.num_threads)) 262 | elif opt.dataset_test == 'inf': 263 | test_loader = torch.utils.data.DataLoader( 264 | MPIINF('val'), 265 | batch_size = opt.batch_size, 266 | shuffle = False, 267 | num_workers = int(conf.num_threads)) 268 | 269 | # generate final prediction 270 | with torch.no_grad(): 271 | test(opt.num_epochs, opt, test_loader, model) 272 | 273 | # to test on training set 274 | if opt.save_results_train == True: 275 | test_train_loader = torch.utils.data.DataLoader( 276 | H36M17(opt.protocol, 'test_train', dense=True), 277 | batch_size = opt.batch_size, 278 | shuffle = False, 279 | num_workers = int(conf.num_threads)) 280 | 281 | with torch.no_grad(): 282 | test_train(opt.num_epochs, opt, test_train_loader, model) 283 | 284 | if __name__ == '__main__': 285 | main() 286 | 287 | --------------------------------------------------------------------------------