├── .gitignore ├── LICENSE ├── README.md ├── data_preprocessing ├── episodes.json ├── package_calvin.py ├── package_rlbench.py ├── preprocess_calvin_instructions.py ├── preprocess_rlbench_instructions.py ├── rearrange_rlbench_demos.py └── rerender_highres_rlbench.py ├── datasets ├── __init__.py ├── dataset_calvin.py ├── dataset_engine.py └── utils.py ├── diffuser_actor ├── __init__.py ├── keypose_optimization │ ├── __init__.py │ └── act3d.py ├── trajectory_optimization │ ├── __init__.py │ └── diffuser_actor.py └── utils │ ├── __init__.py │ ├── clip.py │ ├── converter.py │ ├── encoder.py │ ├── layers.py │ ├── multihead_custom_attention.py │ ├── multihead_flash_attention.py │ ├── position_encodings.py │ ├── resnet.py │ └── utils.py ├── docs ├── DATA_PREPARATION_CALVIN.md ├── DATA_PREPARATION_RLBENCH.md ├── GETTING_STARTED_CALVIN.md ├── GETTING_STARTED_RLBENCH.md └── OVERVIEW.md ├── engine.py ├── environment.yaml ├── fig ├── sota_calvin.png ├── sota_rlbench.png └── teaser.gif ├── main_keypose.py ├── main_trajectory.py ├── main_trajectory_calvin.py ├── online_evaluation_calvin ├── evaluate_model.py ├── evaluate_policy.py ├── evaluate_utils.py └── multistep_sequences.py ├── online_evaluation_rlbench ├── eval_act3d_gnfactor.sh ├── eval_act3d_peract.sh ├── eval_gnfactor.sh ├── eval_peract.sh └── evaluate_policy.py ├── scripts ├── rerender_highres_cameraview.sh ├── test_trajectory_calvin.sh ├── train_act3d_gnfactor.sh ├── train_act3d_peract.sh ├── train_keypose_gnfactor.sh ├── train_keypose_peract.sh ├── train_trajectory_calvin.sh ├── train_trajectory_calvin_nohistory.sh ├── train_trajectory_gnfactor.sh └── train_trajectory_peract.sh ├── setup.py ├── tasks ├── 18_peract_tasks_location_bounds.json ├── 18_peract_tasks_traj_location_bounds.json ├── 74_hiveformer_tasks_location_bounds.json ├── all_82_tasks.csv ├── calvin_rel_traj_location_bounds_task_ABC_D.json ├── hiveformer_74_tasks.csv ├── hiveformer_74_tasks_11_20.csv ├── hiveformer_74_tasks_1_10.csv ├── hiveformer_74_tasks_21_30.csv ├── hiveformer_74_tasks_31_40.csv ├── hiveformer_74_tasks_41_50.csv ├── hiveformer_74_tasks_51_60.csv ├── hiveformer_74_tasks_61_74.csv ├── hiveformer_74_tasks_grouped.txt ├── hiveformer_fix_data_tasks.csv ├── hiveformer_hard_10_demo_tasks.csv ├── hiveformer_high_precision_tasks.csv ├── peract_18_tasks.csv ├── peract_18_tasks_11_18.csv └── peract_18_tasks_1_10.csv └── utils ├── __init__.py ├── common_utils.py ├── pytorch3d_transforms.py ├── utils_with_calvin.py └── utils_with_rlbench.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/README.md -------------------------------------------------------------------------------- /data_preprocessing/episodes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/episodes.json -------------------------------------------------------------------------------- /data_preprocessing/package_calvin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/package_calvin.py -------------------------------------------------------------------------------- /data_preprocessing/package_rlbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/package_rlbench.py -------------------------------------------------------------------------------- /data_preprocessing/preprocess_calvin_instructions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/preprocess_calvin_instructions.py -------------------------------------------------------------------------------- /data_preprocessing/preprocess_rlbench_instructions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/preprocess_rlbench_instructions.py -------------------------------------------------------------------------------- /data_preprocessing/rearrange_rlbench_demos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/rearrange_rlbench_demos.py -------------------------------------------------------------------------------- /data_preprocessing/rerender_highres_rlbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/data_preprocessing/rerender_highres_rlbench.py -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/dataset_calvin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/datasets/dataset_calvin.py -------------------------------------------------------------------------------- /datasets/dataset_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/datasets/dataset_engine.py -------------------------------------------------------------------------------- /datasets/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/datasets/utils.py -------------------------------------------------------------------------------- /diffuser_actor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/__init__.py -------------------------------------------------------------------------------- /diffuser_actor/keypose_optimization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /diffuser_actor/keypose_optimization/act3d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/keypose_optimization/act3d.py -------------------------------------------------------------------------------- /diffuser_actor/trajectory_optimization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /diffuser_actor/trajectory_optimization/diffuser_actor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/trajectory_optimization/diffuser_actor.py -------------------------------------------------------------------------------- /diffuser_actor/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /diffuser_actor/utils/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/clip.py -------------------------------------------------------------------------------- /diffuser_actor/utils/converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/converter.py -------------------------------------------------------------------------------- /diffuser_actor/utils/encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/encoder.py -------------------------------------------------------------------------------- /diffuser_actor/utils/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/layers.py -------------------------------------------------------------------------------- /diffuser_actor/utils/multihead_custom_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/multihead_custom_attention.py -------------------------------------------------------------------------------- /diffuser_actor/utils/multihead_flash_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/multihead_flash_attention.py -------------------------------------------------------------------------------- /diffuser_actor/utils/position_encodings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/position_encodings.py -------------------------------------------------------------------------------- /diffuser_actor/utils/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/resnet.py -------------------------------------------------------------------------------- /diffuser_actor/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/diffuser_actor/utils/utils.py -------------------------------------------------------------------------------- /docs/DATA_PREPARATION_CALVIN.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/docs/DATA_PREPARATION_CALVIN.md -------------------------------------------------------------------------------- /docs/DATA_PREPARATION_RLBENCH.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/docs/DATA_PREPARATION_RLBENCH.md -------------------------------------------------------------------------------- /docs/GETTING_STARTED_CALVIN.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/docs/GETTING_STARTED_CALVIN.md -------------------------------------------------------------------------------- /docs/GETTING_STARTED_RLBENCH.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/docs/GETTING_STARTED_RLBENCH.md -------------------------------------------------------------------------------- /docs/OVERVIEW.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/docs/OVERVIEW.md -------------------------------------------------------------------------------- /engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/engine.py -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/environment.yaml -------------------------------------------------------------------------------- /fig/sota_calvin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/fig/sota_calvin.png -------------------------------------------------------------------------------- /fig/sota_rlbench.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/fig/sota_rlbench.png -------------------------------------------------------------------------------- /fig/teaser.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/fig/teaser.gif -------------------------------------------------------------------------------- /main_keypose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/main_keypose.py -------------------------------------------------------------------------------- /main_trajectory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/main_trajectory.py -------------------------------------------------------------------------------- /main_trajectory_calvin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/main_trajectory_calvin.py -------------------------------------------------------------------------------- /online_evaluation_calvin/evaluate_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_calvin/evaluate_model.py -------------------------------------------------------------------------------- /online_evaluation_calvin/evaluate_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_calvin/evaluate_policy.py -------------------------------------------------------------------------------- /online_evaluation_calvin/evaluate_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_calvin/evaluate_utils.py -------------------------------------------------------------------------------- /online_evaluation_calvin/multistep_sequences.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_calvin/multistep_sequences.py -------------------------------------------------------------------------------- /online_evaluation_rlbench/eval_act3d_gnfactor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_rlbench/eval_act3d_gnfactor.sh -------------------------------------------------------------------------------- /online_evaluation_rlbench/eval_act3d_peract.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_rlbench/eval_act3d_peract.sh -------------------------------------------------------------------------------- /online_evaluation_rlbench/eval_gnfactor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_rlbench/eval_gnfactor.sh -------------------------------------------------------------------------------- /online_evaluation_rlbench/eval_peract.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_rlbench/eval_peract.sh -------------------------------------------------------------------------------- /online_evaluation_rlbench/evaluate_policy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/online_evaluation_rlbench/evaluate_policy.py -------------------------------------------------------------------------------- /scripts/rerender_highres_cameraview.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/rerender_highres_cameraview.sh -------------------------------------------------------------------------------- /scripts/test_trajectory_calvin.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/test_trajectory_calvin.sh -------------------------------------------------------------------------------- /scripts/train_act3d_gnfactor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_act3d_gnfactor.sh -------------------------------------------------------------------------------- /scripts/train_act3d_peract.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_act3d_peract.sh -------------------------------------------------------------------------------- /scripts/train_keypose_gnfactor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_keypose_gnfactor.sh -------------------------------------------------------------------------------- /scripts/train_keypose_peract.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_keypose_peract.sh -------------------------------------------------------------------------------- /scripts/train_trajectory_calvin.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_trajectory_calvin.sh -------------------------------------------------------------------------------- /scripts/train_trajectory_calvin_nohistory.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_trajectory_calvin_nohistory.sh -------------------------------------------------------------------------------- /scripts/train_trajectory_gnfactor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_trajectory_gnfactor.sh -------------------------------------------------------------------------------- /scripts/train_trajectory_peract.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/scripts/train_trajectory_peract.sh -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/setup.py -------------------------------------------------------------------------------- /tasks/18_peract_tasks_location_bounds.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/18_peract_tasks_location_bounds.json -------------------------------------------------------------------------------- /tasks/18_peract_tasks_traj_location_bounds.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/18_peract_tasks_traj_location_bounds.json -------------------------------------------------------------------------------- /tasks/74_hiveformer_tasks_location_bounds.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/74_hiveformer_tasks_location_bounds.json -------------------------------------------------------------------------------- /tasks/all_82_tasks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/all_82_tasks.csv -------------------------------------------------------------------------------- /tasks/calvin_rel_traj_location_bounds_task_ABC_D.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/calvin_rel_traj_location_bounds_task_ABC_D.json -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_11_20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_11_20.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_1_10.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_1_10.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_21_30.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_21_30.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_31_40.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_31_40.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_41_50.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_41_50.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_51_60.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_51_60.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_61_74.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_61_74.csv -------------------------------------------------------------------------------- /tasks/hiveformer_74_tasks_grouped.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_74_tasks_grouped.txt -------------------------------------------------------------------------------- /tasks/hiveformer_fix_data_tasks.csv: -------------------------------------------------------------------------------- 1 | unplug_charger 2 | close_door 3 | open_fridge -------------------------------------------------------------------------------- /tasks/hiveformer_hard_10_demo_tasks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_hard_10_demo_tasks.csv -------------------------------------------------------------------------------- /tasks/hiveformer_high_precision_tasks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/hiveformer_high_precision_tasks.csv -------------------------------------------------------------------------------- /tasks/peract_18_tasks.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/peract_18_tasks.csv -------------------------------------------------------------------------------- /tasks/peract_18_tasks_11_18.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/peract_18_tasks_11_18.csv -------------------------------------------------------------------------------- /tasks/peract_18_tasks_1_10.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/tasks/peract_18_tasks_1_10.csv -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/common_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/utils/common_utils.py -------------------------------------------------------------------------------- /utils/pytorch3d_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/utils/pytorch3d_transforms.py -------------------------------------------------------------------------------- /utils/utils_with_calvin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/utils/utils_with_calvin.py -------------------------------------------------------------------------------- /utils/utils_with_rlbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickgkan/3d_diffuser_actor/HEAD/utils/utils_with_rlbench.py --------------------------------------------------------------------------------