├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── assets └── demo.gif ├── config ├── h2o │ ├── usst_res18_2d.yml │ └── usst_res18_3d.yml ├── usst_res18_2d.yml ├── usst_res18_3d.yml ├── usst_res18_local3d.yml ├── usst_vit_2d.yml ├── usst_vit_3d.yml └── usst_vit_local3d.yml ├── data └── .gitignore ├── exp ├── demo.sh ├── test.sh ├── train.sh └── trainval_h2o.sh ├── preprocess ├── EgoPAT3D │ ├── README.md │ ├── depth_estimation.py │ ├── get_3dhand_trajectory.py │ ├── pc2depth.py │ ├── read_mkv_pyk4a.py │ ├── split_videos.py │ ├── step1_split_videos_with_landmarks.py │ ├── step2_flow_warped_2dhand_trajectory.py │ ├── step3_fetch_depth_traj3d.py │ ├── step4_repair_invalid_depth.py │ ├── step5_compare_depth.py │ ├── step6_compute_odometry.py │ └── visualize_gt.py └── H2O │ ├── get_splits.py │ ├── preprocess.py │ └── visualize.py ├── requirements.txt ├── src ├── EgoPAT3DLoader.py ├── H2OLoader.py ├── Losses.py ├── __init__.py ├── config.py ├── models │ ├── EmbedFrame.py │ ├── EmbedTrajectory.py │ ├── Heads.py │ ├── ProTranV3.py │ ├── TransformerSSM.py │ ├── transformer_layers.py │ └── utils.py ├── optimizers.py ├── runtime.py ├── utils.py ├── utils_h2o.py ├── utils_io.py └── utils_vis.py ├── test.py ├── tools ├── demo.py ├── demo_paper.py ├── illustrate_task.py └── visualize_dataset.py ├── train.py └── trainval_h2o.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/README.md -------------------------------------------------------------------------------- /assets/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/assets/demo.gif -------------------------------------------------------------------------------- /config/h2o/usst_res18_2d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/h2o/usst_res18_2d.yml -------------------------------------------------------------------------------- /config/h2o/usst_res18_3d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/h2o/usst_res18_3d.yml -------------------------------------------------------------------------------- /config/usst_res18_2d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/usst_res18_2d.yml -------------------------------------------------------------------------------- /config/usst_res18_3d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/usst_res18_3d.yml -------------------------------------------------------------------------------- /config/usst_res18_local3d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/usst_res18_local3d.yml -------------------------------------------------------------------------------- /config/usst_vit_2d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/usst_vit_2d.yml -------------------------------------------------------------------------------- /config/usst_vit_3d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/usst_vit_3d.yml -------------------------------------------------------------------------------- /config/usst_vit_local3d.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/config/usst_vit_local3d.yml -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | EgoPAT3D* 2 | depth_vis* 3 | trajectory 4 | iPhone* 5 | H2O* -------------------------------------------------------------------------------- /exp/demo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/exp/demo.sh -------------------------------------------------------------------------------- /exp/test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/exp/test.sh -------------------------------------------------------------------------------- /exp/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/exp/train.sh -------------------------------------------------------------------------------- /exp/trainval_h2o.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/exp/trainval_h2o.sh -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/README.md -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/depth_estimation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/depth_estimation.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/get_3dhand_trajectory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/get_3dhand_trajectory.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/pc2depth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/pc2depth.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/read_mkv_pyk4a.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/read_mkv_pyk4a.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/split_videos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/split_videos.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/step1_split_videos_with_landmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/step1_split_videos_with_landmarks.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/step2_flow_warped_2dhand_trajectory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/step2_flow_warped_2dhand_trajectory.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/step3_fetch_depth_traj3d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/step3_fetch_depth_traj3d.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/step4_repair_invalid_depth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/step4_repair_invalid_depth.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/step5_compare_depth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/step5_compare_depth.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/step6_compute_odometry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/step6_compute_odometry.py -------------------------------------------------------------------------------- /preprocess/EgoPAT3D/visualize_gt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/EgoPAT3D/visualize_gt.py -------------------------------------------------------------------------------- /preprocess/H2O/get_splits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/H2O/get_splits.py -------------------------------------------------------------------------------- /preprocess/H2O/preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/H2O/preprocess.py -------------------------------------------------------------------------------- /preprocess/H2O/visualize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/preprocess/H2O/visualize.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/EgoPAT3DLoader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/EgoPAT3DLoader.py -------------------------------------------------------------------------------- /src/H2OLoader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/H2OLoader.py -------------------------------------------------------------------------------- /src/Losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/Losses.py -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/config.py -------------------------------------------------------------------------------- /src/models/EmbedFrame.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/EmbedFrame.py -------------------------------------------------------------------------------- /src/models/EmbedTrajectory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/EmbedTrajectory.py -------------------------------------------------------------------------------- /src/models/Heads.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/Heads.py -------------------------------------------------------------------------------- /src/models/ProTranV3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/ProTranV3.py -------------------------------------------------------------------------------- /src/models/TransformerSSM.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/TransformerSSM.py -------------------------------------------------------------------------------- /src/models/transformer_layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/transformer_layers.py -------------------------------------------------------------------------------- /src/models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/models/utils.py -------------------------------------------------------------------------------- /src/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/optimizers.py -------------------------------------------------------------------------------- /src/runtime.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/runtime.py -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/utils.py -------------------------------------------------------------------------------- /src/utils_h2o.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/utils_h2o.py -------------------------------------------------------------------------------- /src/utils_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/utils_io.py -------------------------------------------------------------------------------- /src/utils_vis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/src/utils_vis.py -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/test.py -------------------------------------------------------------------------------- /tools/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/tools/demo.py -------------------------------------------------------------------------------- /tools/demo_paper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/tools/demo_paper.py -------------------------------------------------------------------------------- /tools/illustrate_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/tools/illustrate_task.py -------------------------------------------------------------------------------- /tools/visualize_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/tools/visualize_dataset.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/train.py -------------------------------------------------------------------------------- /trainval_h2o.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oppo-us-research/USST/HEAD/trainval_h2o.py --------------------------------------------------------------------------------