├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── assets └── tsail_rdt.png ├── configs ├── bimanual_video_data.yaml ├── datasets │ └── example.yaml ├── rdt │ └── post_train.yaml └── robots │ ├── eval_bimanual_fr3_config.yaml │ └── eval_bimanual_ur5e_config.yaml ├── data ├── base_dataset.py ├── image_corrupt.py ├── umi │ ├── codecs │ │ └── imagecodecs_numcodecs.py │ ├── common │ │ ├── cv2_util.py │ │ ├── normalize_util.py │ │ ├── pose_repr_util.py │ │ ├── pytorch_util.py │ │ ├── replay_buffer.py │ │ └── sampler.py │ └── pose_util.py ├── umi_video_dataset.py └── utils.py ├── deploy ├── calibration │ ├── base.py │ ├── calibrate_franka.py │ ├── calibrate_ur.py │ ├── compute_calibration_matrix.py │ └── tracker.py ├── collision_utils.py ├── get_camera_serials.py ├── get_gripper_serials.py ├── inference_real_fm.py ├── inference_real_vq.py ├── reset_robot_gripper.py ├── umi │ ├── common │ │ ├── interpolation_util.py │ │ ├── pose_trajectory_interpolator.py │ │ ├── precise_sleep.py │ │ └── timestamp_accumulator.py │ ├── real_world │ │ ├── bimanual_umi_env.py │ │ ├── camera │ │ │ ├── base.py │ │ │ ├── cam.py │ │ │ ├── multi_mvs_cam.py │ │ │ ├── mvs_cam.py │ │ │ └── print_utils.py │ │ ├── franka_interpolation_controller.py │ │ ├── keystroke_counter.py │ │ ├── real_inference_util.py │ │ ├── rtde_interpolation_controller.py │ │ ├── zhixing_controller.py │ │ └── zhixing_driver.py │ └── shared_memory │ │ ├── __init__.py │ │ ├── shared_memory_queue.py │ │ ├── shared_memory_ring_buffer.py │ │ ├── shared_memory_util.py │ │ └── shared_ndarray.py └── vllm_utils.py ├── examples ├── DEPLOYMENT_TIPS.md ├── fr3 │ ├── README.md │ └── launch_franka_server.py └── ur5e │ └── README.md ├── main.py ├── models ├── hub_mixin.py ├── normalizer │ ├── __init__.py │ ├── dict_of_tensor_mixin.py │ └── normalizer.py ├── rdt │ ├── attention.py │ ├── blocks.py │ ├── model.py │ ├── norm.py │ └── pos_emb.py ├── rdt_inferencer.py └── rdt_runner.py ├── rdt ├── dataset.py ├── main.py ├── sample.py └── train.py ├── requirements.txt ├── requirements ├── franka_research_3.txt └── ur5e.txt ├── scripts ├── finetune_full_param.sh ├── finetune_lora.sh ├── finetune_rdt.sh └── zero1.json ├── train.py ├── utils.py ├── vla_trainer.py └── vqvae └── models ├── cnn ├── blocks.py └── model.py ├── multivqvae.py ├── rvq.py ├── vq.py └── vqvae.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/.gitignore -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/README.md -------------------------------------------------------------------------------- /assets/tsail_rdt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/assets/tsail_rdt.png -------------------------------------------------------------------------------- /configs/bimanual_video_data.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/configs/bimanual_video_data.yaml -------------------------------------------------------------------------------- /configs/datasets/example.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/configs/datasets/example.yaml -------------------------------------------------------------------------------- /configs/rdt/post_train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/configs/rdt/post_train.yaml -------------------------------------------------------------------------------- /configs/robots/eval_bimanual_fr3_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/configs/robots/eval_bimanual_fr3_config.yaml -------------------------------------------------------------------------------- /configs/robots/eval_bimanual_ur5e_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/configs/robots/eval_bimanual_ur5e_config.yaml -------------------------------------------------------------------------------- /data/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/base_dataset.py -------------------------------------------------------------------------------- /data/image_corrupt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/image_corrupt.py -------------------------------------------------------------------------------- /data/umi/codecs/imagecodecs_numcodecs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/codecs/imagecodecs_numcodecs.py -------------------------------------------------------------------------------- /data/umi/common/cv2_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/common/cv2_util.py -------------------------------------------------------------------------------- /data/umi/common/normalize_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/common/normalize_util.py -------------------------------------------------------------------------------- /data/umi/common/pose_repr_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/common/pose_repr_util.py -------------------------------------------------------------------------------- /data/umi/common/pytorch_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/common/pytorch_util.py -------------------------------------------------------------------------------- /data/umi/common/replay_buffer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/common/replay_buffer.py -------------------------------------------------------------------------------- /data/umi/common/sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/common/sampler.py -------------------------------------------------------------------------------- /data/umi/pose_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi/pose_util.py -------------------------------------------------------------------------------- /data/umi_video_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/umi_video_dataset.py -------------------------------------------------------------------------------- /data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/data/utils.py -------------------------------------------------------------------------------- /deploy/calibration/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/calibration/base.py -------------------------------------------------------------------------------- /deploy/calibration/calibrate_franka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/calibration/calibrate_franka.py -------------------------------------------------------------------------------- /deploy/calibration/calibrate_ur.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/calibration/calibrate_ur.py -------------------------------------------------------------------------------- /deploy/calibration/compute_calibration_matrix.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/calibration/compute_calibration_matrix.py -------------------------------------------------------------------------------- /deploy/calibration/tracker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/calibration/tracker.py -------------------------------------------------------------------------------- /deploy/collision_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/collision_utils.py -------------------------------------------------------------------------------- /deploy/get_camera_serials.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/get_camera_serials.py -------------------------------------------------------------------------------- /deploy/get_gripper_serials.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/get_gripper_serials.py -------------------------------------------------------------------------------- /deploy/inference_real_fm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/inference_real_fm.py -------------------------------------------------------------------------------- /deploy/inference_real_vq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/inference_real_vq.py -------------------------------------------------------------------------------- /deploy/reset_robot_gripper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/reset_robot_gripper.py -------------------------------------------------------------------------------- /deploy/umi/common/interpolation_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/common/interpolation_util.py -------------------------------------------------------------------------------- /deploy/umi/common/pose_trajectory_interpolator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/common/pose_trajectory_interpolator.py -------------------------------------------------------------------------------- /deploy/umi/common/precise_sleep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/common/precise_sleep.py -------------------------------------------------------------------------------- /deploy/umi/common/timestamp_accumulator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/common/timestamp_accumulator.py -------------------------------------------------------------------------------- /deploy/umi/real_world/bimanual_umi_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/bimanual_umi_env.py -------------------------------------------------------------------------------- /deploy/umi/real_world/camera/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/camera/base.py -------------------------------------------------------------------------------- /deploy/umi/real_world/camera/cam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/camera/cam.py -------------------------------------------------------------------------------- /deploy/umi/real_world/camera/multi_mvs_cam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/camera/multi_mvs_cam.py -------------------------------------------------------------------------------- /deploy/umi/real_world/camera/mvs_cam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/camera/mvs_cam.py -------------------------------------------------------------------------------- /deploy/umi/real_world/camera/print_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/camera/print_utils.py -------------------------------------------------------------------------------- /deploy/umi/real_world/franka_interpolation_controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/franka_interpolation_controller.py -------------------------------------------------------------------------------- /deploy/umi/real_world/keystroke_counter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/keystroke_counter.py -------------------------------------------------------------------------------- /deploy/umi/real_world/real_inference_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/real_inference_util.py -------------------------------------------------------------------------------- /deploy/umi/real_world/rtde_interpolation_controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/rtde_interpolation_controller.py -------------------------------------------------------------------------------- /deploy/umi/real_world/zhixing_controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/zhixing_controller.py -------------------------------------------------------------------------------- /deploy/umi/real_world/zhixing_driver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/real_world/zhixing_driver.py -------------------------------------------------------------------------------- /deploy/umi/shared_memory/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/shared_memory/__init__.py -------------------------------------------------------------------------------- /deploy/umi/shared_memory/shared_memory_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/shared_memory/shared_memory_queue.py -------------------------------------------------------------------------------- /deploy/umi/shared_memory/shared_memory_ring_buffer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/shared_memory/shared_memory_ring_buffer.py -------------------------------------------------------------------------------- /deploy/umi/shared_memory/shared_memory_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/shared_memory/shared_memory_util.py -------------------------------------------------------------------------------- /deploy/umi/shared_memory/shared_ndarray.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/umi/shared_memory/shared_ndarray.py -------------------------------------------------------------------------------- /deploy/vllm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/deploy/vllm_utils.py -------------------------------------------------------------------------------- /examples/DEPLOYMENT_TIPS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/examples/DEPLOYMENT_TIPS.md -------------------------------------------------------------------------------- /examples/fr3/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/examples/fr3/README.md -------------------------------------------------------------------------------- /examples/fr3/launch_franka_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/examples/fr3/launch_franka_server.py -------------------------------------------------------------------------------- /examples/ur5e/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/examples/ur5e/README.md -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/main.py -------------------------------------------------------------------------------- /models/hub_mixin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/hub_mixin.py -------------------------------------------------------------------------------- /models/normalizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/normalizer/__init__.py -------------------------------------------------------------------------------- /models/normalizer/dict_of_tensor_mixin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/normalizer/dict_of_tensor_mixin.py -------------------------------------------------------------------------------- /models/normalizer/normalizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/normalizer/normalizer.py -------------------------------------------------------------------------------- /models/rdt/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt/attention.py -------------------------------------------------------------------------------- /models/rdt/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt/blocks.py -------------------------------------------------------------------------------- /models/rdt/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt/model.py -------------------------------------------------------------------------------- /models/rdt/norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt/norm.py -------------------------------------------------------------------------------- /models/rdt/pos_emb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt/pos_emb.py -------------------------------------------------------------------------------- /models/rdt_inferencer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt_inferencer.py -------------------------------------------------------------------------------- /models/rdt_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/models/rdt_runner.py -------------------------------------------------------------------------------- /rdt/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/rdt/dataset.py -------------------------------------------------------------------------------- /rdt/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/rdt/main.py -------------------------------------------------------------------------------- /rdt/sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/rdt/sample.py -------------------------------------------------------------------------------- /rdt/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/rdt/train.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements/franka_research_3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/requirements/franka_research_3.txt -------------------------------------------------------------------------------- /requirements/ur5e.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/requirements/ur5e.txt -------------------------------------------------------------------------------- /scripts/finetune_full_param.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/scripts/finetune_full_param.sh -------------------------------------------------------------------------------- /scripts/finetune_lora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/scripts/finetune_lora.sh -------------------------------------------------------------------------------- /scripts/finetune_rdt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/scripts/finetune_rdt.sh -------------------------------------------------------------------------------- /scripts/zero1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/scripts/zero1.json -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/train.py -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/utils.py -------------------------------------------------------------------------------- /vla_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vla_trainer.py -------------------------------------------------------------------------------- /vqvae/models/cnn/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vqvae/models/cnn/blocks.py -------------------------------------------------------------------------------- /vqvae/models/cnn/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vqvae/models/cnn/model.py -------------------------------------------------------------------------------- /vqvae/models/multivqvae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vqvae/models/multivqvae.py -------------------------------------------------------------------------------- /vqvae/models/rvq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vqvae/models/rvq.py -------------------------------------------------------------------------------- /vqvae/models/vq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vqvae/models/vq.py -------------------------------------------------------------------------------- /vqvae/models/vqvae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thu-ml/RDT2/HEAD/vqvae/models/vqvae.py --------------------------------------------------------------------------------