├── .env ├── .gitignore ├── .project-root ├── LICENSE.txt ├── README.md ├── assets └── teaser.png ├── common ├── data │ ├── data_utils.py │ ├── datasets.py │ └── mix_utils.py ├── models │ └── model_utils.py └── processors │ ├── preprocessor_utils.py │ └── rgb_preprocessors.py ├── data_preprocessing ├── calvin_to_lmdb.py ├── oxe_dataset_configs.py ├── oxe_to_lmdb.py ├── oxe_to_video.py └── oxe_utils.py ├── latent_motion_tokenizer ├── checkpoints │ ├── latent_motion_tokenizer_trained_on_calvin │ │ └── config.yaml │ └── latent_motion_tokenizer_trained_on_oxe │ │ └── config.yaml ├── configs │ ├── data │ │ ├── calvin.yaml │ │ └── rtx.yaml │ ├── models │ │ └── vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue.yaml │ └── train │ │ ├── data_calvin-vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue-train_lr0.0001_bs256-aug_shiftTrue_resizedCropFalse.yaml │ │ └── data_rtx-vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue-train_lr0.0001_bs256-aug_shiftTrue_resizedCropFalse.yaml ├── src │ ├── models │ │ ├── latent_motion_decoder.py │ │ ├── latent_motion_tokenizer.py │ │ ├── m_former.py │ │ ├── modeling_lmd_vit.py │ │ ├── theia_model.py │ │ ├── timm_dinov2_model.py │ │ └── vector_quantizer.py │ └── trainers │ │ ├── latent_motion_tokenizer_trainer.py │ │ ├── optimizer.py │ │ └── trainer_utils.py └── train │ └── train_latent_motion_tokenizer.py ├── moto_gpt ├── checkpoints │ ├── moto_gpt_finetuned_on_calvin │ │ └── config.yaml │ ├── moto_gpt_finetuned_on_rt1 │ │ └── config.yaml │ ├── moto_gpt_pretrained_on_calvin │ │ └── config.yaml │ └── moto_gpt_pretrained_on_oxe │ │ └── config.yaml ├── configs │ ├── data │ │ ├── calvin.yaml │ │ ├── rt1.yaml │ │ └── rtx.yaml │ ├── models │ │ ├── actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5.yaml │ │ ├── actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5.yaml │ │ ├── actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5.yaml │ │ └── actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5.yaml │ └── train │ │ ├── data_calvin-model_actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse.yaml │ │ ├── data_calvin-model_actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5-train_lr0.0002_bs512-aug_shiftTrue_resizedCropFalse-resume_from_predLatentOnly_calvin_Epoch10.yaml │ │ ├── data_rt1-model_actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse-resume_from_predLatentOnly_oxe_Epoch10.yaml │ │ └── data_rtx-model_actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse.yaml ├── evaluation │ ├── motion_prior_analysis │ │ ├── latent_motion_generation.py │ │ └── sample_data │ │ │ ├── calvin │ │ │ ├── lang_annotations.json │ │ │ └── videos │ │ │ │ ├── task_ABC_D_validation_eps_13126_13190.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_222094_222158.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_2635_2699.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_267_331.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_2786_2850.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_3351_3398.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_5332_5373.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_609_673.mp4 │ │ │ │ ├── task_ABC_D_validation_eps_7754_7818.mp4 │ │ │ │ └── task_ABC_D_validation_eps_850_898.mp4 │ │ │ └── oxe │ │ │ ├── lang_annotations.json │ │ │ └── videos │ │ │ ├── fractal20220817_eps_00083076.mp4 │ │ │ ├── fractal20220817_eps_00083655.mp4 │ │ │ ├── fractal20220817_eps_00084175.mp4 │ │ │ ├── fractal20220817_eps_00084367.mp4 │ │ │ ├── fractal20220817_eps_00084376.mp4 │ │ │ ├── fractal20220817_eps_00085242.mp4 │ │ │ ├── fractal20220817_eps_00085250.mp4 │ │ │ ├── fractal20220817_eps_00085439.mp4 │ │ │ ├── fractal20220817_eps_00086889.mp4 │ │ │ └── fractal20220817_eps_00087067.mp4 │ └── robot_manipulation_benchmarks │ │ ├── calvin │ │ ├── calvin_env_wrapper_raw.py │ │ ├── calvin_utils.py │ │ ├── evaluate_calvin.py │ │ └── fake_dataset │ │ │ ├── training │ │ │ ├── .hydra │ │ │ │ ├── config.yaml │ │ │ │ ├── hydra.yaml │ │ │ │ ├── merged_config.yaml │ │ │ │ └── overrides.yaml │ │ │ └── statistics.yaml │ │ │ └── validation │ │ │ ├── .hydra │ │ │ ├── config.yaml │ │ │ ├── hydra.yaml │ │ │ ├── merged_config.yaml │ │ │ └── overrides.yaml │ │ │ └── statistics.yaml │ │ └── simpler │ │ ├── compute_metrics.py │ │ ├── evaluate_simpler.py │ │ ├── maniskill2_evaluator.py │ │ └── simpler_env_policy_wrapper.py ├── src │ ├── models │ │ ├── mae_model.py │ │ ├── moto_gpt.py │ │ ├── moto_gpt_policy_wraper.py │ │ ├── theia_model.py │ │ ├── timm_dinov2_model.py │ │ └── trajectory_gpt2.py │ └── trainers │ │ ├── moto_gpt_trainer.py │ │ └── trainer_utils.py └── train │ └── train_moto_gpt.py ├── requirements.txt └── scripts ├── download_and_preprocess_calvin_data.sh ├── download_and_preprocess_oxe_data.sh ├── evaluate_moto_gpt_in_calvin.sh ├── evaluate_moto_gpt_in_simpler.sh ├── finetune_moto_gpt_on_calvin.sh ├── finetune_moto_gpt_on_rt1.sh ├── pretrain_moto_gpt_on_calvin.sh ├── pretrain_moto_gpt_on_oxe.sh ├── run_latent_motion_generation.sh ├── train_latent_motion_tokenizer_on_calvin.sh └── train_latent_motion_tokenizer_on_oxe.sh /.env: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/.gitignore -------------------------------------------------------------------------------- /.project-root: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/README.md -------------------------------------------------------------------------------- /assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/assets/teaser.png -------------------------------------------------------------------------------- /common/data/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/common/data/data_utils.py -------------------------------------------------------------------------------- /common/data/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/common/data/datasets.py -------------------------------------------------------------------------------- /common/data/mix_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/common/data/mix_utils.py -------------------------------------------------------------------------------- /common/models/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/common/models/model_utils.py -------------------------------------------------------------------------------- /common/processors/preprocessor_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/common/processors/preprocessor_utils.py -------------------------------------------------------------------------------- /common/processors/rgb_preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/common/processors/rgb_preprocessors.py -------------------------------------------------------------------------------- /data_preprocessing/calvin_to_lmdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/data_preprocessing/calvin_to_lmdb.py -------------------------------------------------------------------------------- /data_preprocessing/oxe_dataset_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/data_preprocessing/oxe_dataset_configs.py -------------------------------------------------------------------------------- /data_preprocessing/oxe_to_lmdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/data_preprocessing/oxe_to_lmdb.py -------------------------------------------------------------------------------- /data_preprocessing/oxe_to_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/data_preprocessing/oxe_to_video.py -------------------------------------------------------------------------------- /data_preprocessing/oxe_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/data_preprocessing/oxe_utils.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/checkpoints/latent_motion_tokenizer_trained_on_calvin/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/checkpoints/latent_motion_tokenizer_trained_on_calvin/config.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/checkpoints/latent_motion_tokenizer_trained_on_oxe/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/checkpoints/latent_motion_tokenizer_trained_on_oxe/config.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/configs/data/calvin.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/configs/data/calvin.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/configs/data/rtx.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/configs/data/rtx.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/configs/models/vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/configs/models/vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/configs/train/data_calvin-vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue-train_lr0.0001_bs256-aug_shiftTrue_resizedCropFalse.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/configs/train/data_calvin-vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue-train_lr0.0001_bs256-aug_shiftTrue_resizedCropFalse.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/configs/train/data_rtx-vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue-train_lr0.0001_bs256-aug_shiftTrue_resizedCropFalse.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/configs/train/data_rtx-vq_size128_dim32_num8_legacyTrue-vision_MaeLarge-decoder_queryFusionModeAdd_Patch196_useMaskFalse-mformer_legacyTrue-train_lr0.0001_bs256-aug_shiftTrue_resizedCropFalse.yaml -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/latent_motion_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/latent_motion_decoder.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/latent_motion_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/latent_motion_tokenizer.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/m_former.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/m_former.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/modeling_lmd_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/modeling_lmd_vit.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/theia_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/theia_model.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/timm_dinov2_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/timm_dinov2_model.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/models/vector_quantizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/models/vector_quantizer.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/trainers/latent_motion_tokenizer_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/trainers/latent_motion_tokenizer_trainer.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/trainers/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/trainers/optimizer.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/src/trainers/trainer_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/src/trainers/trainer_utils.py -------------------------------------------------------------------------------- /latent_motion_tokenizer/train/train_latent_motion_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/latent_motion_tokenizer/train/train_latent_motion_tokenizer.py -------------------------------------------------------------------------------- /moto_gpt/checkpoints/moto_gpt_finetuned_on_calvin/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/checkpoints/moto_gpt_finetuned_on_calvin/config.yaml -------------------------------------------------------------------------------- /moto_gpt/checkpoints/moto_gpt_finetuned_on_rt1/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/checkpoints/moto_gpt_finetuned_on_rt1/config.yaml -------------------------------------------------------------------------------- /moto_gpt/checkpoints/moto_gpt_pretrained_on_calvin/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/checkpoints/moto_gpt_pretrained_on_calvin/config.yaml -------------------------------------------------------------------------------- /moto_gpt/checkpoints/moto_gpt_pretrained_on_oxe/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/checkpoints/moto_gpt_pretrained_on_oxe/config.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/data/calvin.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/data/calvin.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/data/rt1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/data/rt1.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/data/rtx.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/data/rtx.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/models/actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/models/actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/models/actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/models/actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/models/actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/models/actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/models/actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/models/actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/train/data_calvin-model_actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/train/data_calvin-model_actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/train/data_calvin-model_actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5-train_lr0.0002_bs512-aug_shiftTrue_resizedCropFalse-resume_from_predLatentOnly_calvin_Epoch10.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/train/data_calvin-model_actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk5_maskProb0.5-train_lr0.0002_bs512-aug_shiftTrue_resizedCropFalse-resume_from_predLatentOnly_calvin_Epoch10.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/train/data_rt1-model_actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse-resume_from_predLatentOnly_oxe_Epoch10.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/train/data_rt1-model_actPredTrue_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse-resume_from_predLatentOnly_oxe_Epoch10.yaml -------------------------------------------------------------------------------- /moto_gpt/configs/train/data_rtx-model_actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/configs/train/data_rtx-model_actPredFalse_motionPredTrue_visionMaeLarge_seq2_chunk3_maskProb0.5-train_lr0.0001_bs512-aug_shiftTrue_resizedCropFalse.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/latent_motion_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/latent_motion_generation.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/lang_annotations.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/lang_annotations.json -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_13126_13190.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_13126_13190.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_222094_222158.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_222094_222158.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_2635_2699.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_2635_2699.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_267_331.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_267_331.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_2786_2850.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_2786_2850.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_3351_3398.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_3351_3398.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_5332_5373.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_5332_5373.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_609_673.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_609_673.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_7754_7818.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_7754_7818.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_850_898.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/calvin/videos/task_ABC_D_validation_eps_850_898.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/lang_annotations.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/lang_annotations.json -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00083076.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00083076.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00083655.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00083655.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00084175.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00084175.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00084367.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00084367.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00084376.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00084376.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00085242.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00085242.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00085250.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00085250.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00085439.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00085439.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00086889.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00086889.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00087067.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/motion_prior_analysis/sample_data/oxe/videos/fractal20220817_eps_00087067.mp4 -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/calvin_env_wrapper_raw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/calvin_env_wrapper_raw.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/calvin_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/calvin_utils.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/evaluate_calvin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/evaluate_calvin.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/config.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/hydra.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/hydra.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/merged_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/merged_config.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/overrides.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/.hydra/overrides.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/statistics.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/training/statistics.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/config.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/hydra.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/hydra.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/merged_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/merged_config.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/overrides.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/.hydra/overrides.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/statistics.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/calvin/fake_dataset/validation/statistics.yaml -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/compute_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/compute_metrics.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/evaluate_simpler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/evaluate_simpler.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/maniskill2_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/maniskill2_evaluator.py -------------------------------------------------------------------------------- /moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/simpler_env_policy_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/evaluation/robot_manipulation_benchmarks/simpler/simpler_env_policy_wrapper.py -------------------------------------------------------------------------------- /moto_gpt/src/models/mae_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/models/mae_model.py -------------------------------------------------------------------------------- /moto_gpt/src/models/moto_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/models/moto_gpt.py -------------------------------------------------------------------------------- /moto_gpt/src/models/moto_gpt_policy_wraper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/models/moto_gpt_policy_wraper.py -------------------------------------------------------------------------------- /moto_gpt/src/models/theia_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/models/theia_model.py -------------------------------------------------------------------------------- /moto_gpt/src/models/timm_dinov2_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/models/timm_dinov2_model.py -------------------------------------------------------------------------------- /moto_gpt/src/models/trajectory_gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/models/trajectory_gpt2.py -------------------------------------------------------------------------------- /moto_gpt/src/trainers/moto_gpt_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/trainers/moto_gpt_trainer.py -------------------------------------------------------------------------------- /moto_gpt/src/trainers/trainer_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/src/trainers/trainer_utils.py -------------------------------------------------------------------------------- /moto_gpt/train/train_moto_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/moto_gpt/train/train_moto_gpt.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/download_and_preprocess_calvin_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/download_and_preprocess_calvin_data.sh -------------------------------------------------------------------------------- /scripts/download_and_preprocess_oxe_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/download_and_preprocess_oxe_data.sh -------------------------------------------------------------------------------- /scripts/evaluate_moto_gpt_in_calvin.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/evaluate_moto_gpt_in_calvin.sh -------------------------------------------------------------------------------- /scripts/evaluate_moto_gpt_in_simpler.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/evaluate_moto_gpt_in_simpler.sh -------------------------------------------------------------------------------- /scripts/finetune_moto_gpt_on_calvin.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/finetune_moto_gpt_on_calvin.sh -------------------------------------------------------------------------------- /scripts/finetune_moto_gpt_on_rt1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/finetune_moto_gpt_on_rt1.sh -------------------------------------------------------------------------------- /scripts/pretrain_moto_gpt_on_calvin.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/pretrain_moto_gpt_on_calvin.sh -------------------------------------------------------------------------------- /scripts/pretrain_moto_gpt_on_oxe.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/pretrain_moto_gpt_on_oxe.sh -------------------------------------------------------------------------------- /scripts/run_latent_motion_generation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/run_latent_motion_generation.sh -------------------------------------------------------------------------------- /scripts/train_latent_motion_tokenizer_on_calvin.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/train_latent_motion_tokenizer_on_calvin.sh -------------------------------------------------------------------------------- /scripts/train_latent_motion_tokenizer_on_oxe.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TencentARC/Moto/HEAD/scripts/train_latent_motion_tokenizer_on_oxe.sh --------------------------------------------------------------------------------