├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── assets ├── converenge.jpg ├── dmlab_sample.png ├── example │ ├── 5.npz │ └── v_CliffDiving_g08_c03.mp4 ├── extrapolation.png ├── kv_cache_comp.png ├── long_short_term_ctx.jpg ├── long_short_term_ctx.png ├── multi_level_kv_cache.png ├── performance.png └── pipeline.png ├── datasets └── README.md ├── experiments └── pretrained_models │ └── README.md ├── far ├── data │ ├── __init__.py │ ├── bair_dataset.py │ ├── dmlab_dataset.py │ ├── minecraft_dataset.py │ └── ucf_dataset.py ├── losses │ └── lpips.py ├── metrics │ ├── fvd.py │ └── metric.py ├── models │ ├── __init__.py │ ├── autoencoder_dc_model.py │ ├── far_long_model.py │ ├── far_model.py │ └── patch_discriminator.py ├── pipelines │ ├── __init__.py │ └── pipeline_far.py ├── trainers │ ├── __init__.py │ ├── trainer_dcae.py │ └── trainer_far.py └── utils │ ├── ema_util.py │ ├── logger_util.py │ ├── misc.py │ ├── registry.py │ ├── vis_util.py │ └── wandb.py ├── options ├── model_cfg │ ├── dcae │ │ ├── model_16x_c32_config.json │ │ ├── model_32x_c32_config.json │ │ └── model_8x_c32_config.json │ └── far │ │ └── scheduler_config.json ├── test │ └── far │ │ ├── long_video_prediction │ │ ├── FAR_B_Long_dmlab_res64_1000K_bs32.yml │ │ ├── FAR_B_Long_dmlab_res64_1000K_bs32_36to264.yml │ │ ├── FAR_M_Long_minecraft_res128_1000K_bs32.yml │ │ └── FAR_M_Long_minecraft_res128_1000K_bs32_36to264.yml │ │ ├── short_video_prediction │ │ ├── FAR_B_bair_res64_200K_bs32.yml │ │ ├── FAR_B_bair_res64_200K_bs32_2to14.yml │ │ └── FAR_B_ucf101_res64_200K_bs32.yml │ │ └── video_generation │ │ ├── FAR_L_ucf101_cond_res128_400K_bs32.yml │ │ ├── FAR_L_ucf101_cond_res256_400K_bs32.yml │ │ ├── FAR_L_ucf101_uncond_res128_400K_bs32.yml │ │ ├── FAR_L_ucf101_uncond_res256_400K_bs32.yml │ │ ├── FAR_XL_ucf101_cond_res256_400K_bs32.yml │ │ └── FAR_XL_ucf101_uncond_res256_400K_bs32.yml └── train │ ├── dcae │ ├── bair │ │ └── DCAE_bair_res64_bs128_200K.yml │ ├── dmlab │ │ └── DCAE_dmlab_res64_bs128_200K.yml │ ├── minecraft │ │ └── DCAE_minecraft_res128_bs128_200K.yml │ └── ucf101 │ │ ├── DCAE_ucf101_res128_bs128_200K.yml │ │ ├── DCAE_ucf101_res256_bs128_200K.yml │ │ └── DCAE_ucf101_res64_bs128_200K.yml │ └── far │ ├── long_video_prediction │ ├── FAR_B_Long_dmlab_res64_1000K_bs32.yml │ └── FAR_M_Long_minecraft_res128_1000K_bs32.yml │ ├── short_video_prediction │ ├── FAR_B_bair_res64_200K_bs32.yml │ └── FAR_B_ucf101_res64_200K_bs32.yml │ └── video_generation │ ├── FAR_L_ucf101_cond_res128_400K_bs32.yml │ ├── FAR_L_ucf101_cond_res256_400K_bs32.yml │ ├── FAR_L_ucf101_uncond_res128_400K_bs32.yml │ ├── FAR_L_ucf101_uncond_res256_400K_bs32.yml │ ├── FAR_XL_ucf101_cond_res256_400K_bs32.yml │ └── FAR_XL_ucf101_uncond_res256_400K_bs32.yml ├── requirements.txt ├── scripts ├── publish_model.py └── speed_test.py ├── test.py ├── train.py └── train_dcae.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/README.md -------------------------------------------------------------------------------- /assets/converenge.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/converenge.jpg -------------------------------------------------------------------------------- /assets/dmlab_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/dmlab_sample.png -------------------------------------------------------------------------------- /assets/example/5.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/example/5.npz -------------------------------------------------------------------------------- /assets/example/v_CliffDiving_g08_c03.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/example/v_CliffDiving_g08_c03.mp4 -------------------------------------------------------------------------------- /assets/extrapolation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/extrapolation.png -------------------------------------------------------------------------------- /assets/kv_cache_comp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/kv_cache_comp.png -------------------------------------------------------------------------------- /assets/long_short_term_ctx.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/long_short_term_ctx.jpg -------------------------------------------------------------------------------- /assets/long_short_term_ctx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/long_short_term_ctx.png -------------------------------------------------------------------------------- /assets/multi_level_kv_cache.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/multi_level_kv_cache.png -------------------------------------------------------------------------------- /assets/performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/performance.png -------------------------------------------------------------------------------- /assets/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/assets/pipeline.png -------------------------------------------------------------------------------- /datasets/README.md: -------------------------------------------------------------------------------- 1 | # Put downloaded dataset here 2 | -------------------------------------------------------------------------------- /experiments/pretrained_models/README.md: -------------------------------------------------------------------------------- 1 | # Put downloaded pretrained models here 2 | -------------------------------------------------------------------------------- /far/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/data/__init__.py -------------------------------------------------------------------------------- /far/data/bair_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/data/bair_dataset.py -------------------------------------------------------------------------------- /far/data/dmlab_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/data/dmlab_dataset.py -------------------------------------------------------------------------------- /far/data/minecraft_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/data/minecraft_dataset.py -------------------------------------------------------------------------------- /far/data/ucf_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/data/ucf_dataset.py -------------------------------------------------------------------------------- /far/losses/lpips.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/losses/lpips.py -------------------------------------------------------------------------------- /far/metrics/fvd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/metrics/fvd.py -------------------------------------------------------------------------------- /far/metrics/metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/metrics/metric.py -------------------------------------------------------------------------------- /far/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/models/__init__.py -------------------------------------------------------------------------------- /far/models/autoencoder_dc_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/models/autoencoder_dc_model.py -------------------------------------------------------------------------------- /far/models/far_long_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/models/far_long_model.py -------------------------------------------------------------------------------- /far/models/far_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/models/far_model.py -------------------------------------------------------------------------------- /far/models/patch_discriminator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/models/patch_discriminator.py -------------------------------------------------------------------------------- /far/pipelines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/pipelines/__init__.py -------------------------------------------------------------------------------- /far/pipelines/pipeline_far.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/pipelines/pipeline_far.py -------------------------------------------------------------------------------- /far/trainers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/trainers/__init__.py -------------------------------------------------------------------------------- /far/trainers/trainer_dcae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/trainers/trainer_dcae.py -------------------------------------------------------------------------------- /far/trainers/trainer_far.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/trainers/trainer_far.py -------------------------------------------------------------------------------- /far/utils/ema_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/utils/ema_util.py -------------------------------------------------------------------------------- /far/utils/logger_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/utils/logger_util.py -------------------------------------------------------------------------------- /far/utils/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/utils/misc.py -------------------------------------------------------------------------------- /far/utils/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/utils/registry.py -------------------------------------------------------------------------------- /far/utils/vis_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/utils/vis_util.py -------------------------------------------------------------------------------- /far/utils/wandb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/far/utils/wandb.py -------------------------------------------------------------------------------- /options/model_cfg/dcae/model_16x_c32_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/model_cfg/dcae/model_16x_c32_config.json -------------------------------------------------------------------------------- /options/model_cfg/dcae/model_32x_c32_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/model_cfg/dcae/model_32x_c32_config.json -------------------------------------------------------------------------------- /options/model_cfg/dcae/model_8x_c32_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/model_cfg/dcae/model_8x_c32_config.json -------------------------------------------------------------------------------- /options/model_cfg/far/scheduler_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/model_cfg/far/scheduler_config.json -------------------------------------------------------------------------------- /options/test/far/long_video_prediction/FAR_B_Long_dmlab_res64_1000K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/long_video_prediction/FAR_B_Long_dmlab_res64_1000K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/long_video_prediction/FAR_B_Long_dmlab_res64_1000K_bs32_36to264.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/long_video_prediction/FAR_B_Long_dmlab_res64_1000K_bs32_36to264.yml -------------------------------------------------------------------------------- /options/test/far/long_video_prediction/FAR_M_Long_minecraft_res128_1000K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/long_video_prediction/FAR_M_Long_minecraft_res128_1000K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/long_video_prediction/FAR_M_Long_minecraft_res128_1000K_bs32_36to264.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/long_video_prediction/FAR_M_Long_minecraft_res128_1000K_bs32_36to264.yml -------------------------------------------------------------------------------- /options/test/far/short_video_prediction/FAR_B_bair_res64_200K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/short_video_prediction/FAR_B_bair_res64_200K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/short_video_prediction/FAR_B_bair_res64_200K_bs32_2to14.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/short_video_prediction/FAR_B_bair_res64_200K_bs32_2to14.yml -------------------------------------------------------------------------------- /options/test/far/short_video_prediction/FAR_B_ucf101_res64_200K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/short_video_prediction/FAR_B_ucf101_res64_200K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/video_generation/FAR_L_ucf101_cond_res128_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/video_generation/FAR_L_ucf101_cond_res128_400K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/video_generation/FAR_L_ucf101_cond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/video_generation/FAR_L_ucf101_cond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/video_generation/FAR_L_ucf101_uncond_res128_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/video_generation/FAR_L_ucf101_uncond_res128_400K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/video_generation/FAR_L_ucf101_uncond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/video_generation/FAR_L_ucf101_uncond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/video_generation/FAR_XL_ucf101_cond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/video_generation/FAR_XL_ucf101_cond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/test/far/video_generation/FAR_XL_ucf101_uncond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/test/far/video_generation/FAR_XL_ucf101_uncond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/train/dcae/bair/DCAE_bair_res64_bs128_200K.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/dcae/bair/DCAE_bair_res64_bs128_200K.yml -------------------------------------------------------------------------------- /options/train/dcae/dmlab/DCAE_dmlab_res64_bs128_200K.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/dcae/dmlab/DCAE_dmlab_res64_bs128_200K.yml -------------------------------------------------------------------------------- /options/train/dcae/minecraft/DCAE_minecraft_res128_bs128_200K.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/dcae/minecraft/DCAE_minecraft_res128_bs128_200K.yml -------------------------------------------------------------------------------- /options/train/dcae/ucf101/DCAE_ucf101_res128_bs128_200K.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/dcae/ucf101/DCAE_ucf101_res128_bs128_200K.yml -------------------------------------------------------------------------------- /options/train/dcae/ucf101/DCAE_ucf101_res256_bs128_200K.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/dcae/ucf101/DCAE_ucf101_res256_bs128_200K.yml -------------------------------------------------------------------------------- /options/train/dcae/ucf101/DCAE_ucf101_res64_bs128_200K.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/dcae/ucf101/DCAE_ucf101_res64_bs128_200K.yml -------------------------------------------------------------------------------- /options/train/far/long_video_prediction/FAR_B_Long_dmlab_res64_1000K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/long_video_prediction/FAR_B_Long_dmlab_res64_1000K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/long_video_prediction/FAR_M_Long_minecraft_res128_1000K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/long_video_prediction/FAR_M_Long_minecraft_res128_1000K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/short_video_prediction/FAR_B_bair_res64_200K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/short_video_prediction/FAR_B_bair_res64_200K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/short_video_prediction/FAR_B_ucf101_res64_200K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/short_video_prediction/FAR_B_ucf101_res64_200K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/video_generation/FAR_L_ucf101_cond_res128_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/video_generation/FAR_L_ucf101_cond_res128_400K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/video_generation/FAR_L_ucf101_cond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/video_generation/FAR_L_ucf101_cond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/video_generation/FAR_L_ucf101_uncond_res128_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/video_generation/FAR_L_ucf101_uncond_res128_400K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/video_generation/FAR_L_ucf101_uncond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/video_generation/FAR_L_ucf101_uncond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/video_generation/FAR_XL_ucf101_cond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/video_generation/FAR_XL_ucf101_cond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /options/train/far/video_generation/FAR_XL_ucf101_uncond_res256_400K_bs32.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/options/train/far/video_generation/FAR_XL_ucf101_uncond_res256_400K_bs32.yml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/publish_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/scripts/publish_model.py -------------------------------------------------------------------------------- /scripts/speed_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/scripts/speed_test.py -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/test.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/train.py -------------------------------------------------------------------------------- /train_dcae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/FAR/HEAD/train_dcae.py --------------------------------------------------------------------------------