├── .gitignore ├── LICENSE ├── README.md ├── assets ├── paper_demo.png ├── samples_results │ ├── few_shot_pred.gif │ ├── few_shot_pred.mp4 │ ├── one_shot_portrait_pred.gif │ ├── one_shot_portrait_pred.mp4 │ ├── one_shot_pred.gif │ └── one_shot_pred.mp4 ├── single_images │ └── test001.png └── training_samples │ ├── landmarks │ └── RD_Radio1_000_25fps │ │ ├── RD_Radio1_000_25fps_1.txt │ │ ├── RD_Radio1_000_25fps_10.txt │ │ ├── RD_Radio1_000_25fps_2.txt │ │ ├── RD_Radio1_000_25fps_3.txt │ │ ├── RD_Radio1_000_25fps_4.txt │ │ ├── RD_Radio1_000_25fps_5.txt │ │ ├── RD_Radio1_000_25fps_6.txt │ │ ├── RD_Radio1_000_25fps_7.txt │ │ ├── RD_Radio1_000_25fps_8.txt │ │ └── RD_Radio1_000_25fps_9.txt │ └── raw_frames_25fps │ └── RD_Radio1_000_25fps │ ├── RD_Radio1_000_25fps_1.png │ ├── RD_Radio1_000_25fps_10.png │ ├── RD_Radio1_000_25fps_2.png │ ├── RD_Radio1_000_25fps_3.png │ ├── RD_Radio1_000_25fps_4.png │ ├── RD_Radio1_000_25fps_5.png │ ├── RD_Radio1_000_25fps_6.png │ ├── RD_Radio1_000_25fps_7.png │ ├── RD_Radio1_000_25fps_8.png │ └── RD_Radio1_000_25fps_9.png ├── auto_download_ckpt.sh ├── choices.py ├── config.py ├── config_base.py ├── dataset.py ├── dataset_util.py ├── demo.py ├── demo_with_batch.py ├── diffusion ├── __init__.py ├── base.py ├── diffusion.py └── resample.py ├── dist_utils.py ├── experiment.py ├── index.html ├── model ├── __init__.py ├── blocks.py ├── latentnet.py ├── nn.py ├── speech2latent.py ├── unet.py └── unet_autoenc.py ├── renderer.py ├── requirements.txt ├── resources ├── ablation_study │ ├── RD_Radio13_000_25fps.mp4 │ ├── RD_Radio18_000_25fps.mp4 │ ├── RD_Radio21_000_25fps.mp4 │ ├── RD_Radio4_000_25fps.mp4 │ └── RD_Radio8_000_25fps.mp4 ├── few_shot │ ├── test12_d8.mp4 │ ├── test19_driving_audio_6.mp4 │ ├── test3_d6.mp4 │ ├── test5_driving_audio_2.mp4 │ └── test6_driving_audio_5.mp4 ├── one_shot │ ├── RD_Radio11_000_25fps.mp4 │ ├── RD_Radio17_000_25fps.mp4 │ ├── RD_Radio25_000_25fps.mp4 │ ├── RD_Radio29_000_25fps.mp4 │ └── RD_Radio5_000_25fps.mp4 └── reconstruction │ ├── RD_Radio22_000_25fps.mp4 │ ├── RD_Radio42_000_25fps.mp4 │ ├── RD_Radio51_000_25fps.mp4 │ ├── RD_Radio8_000_25fps.mp4 │ └── WRA_PeterRoskam0_000_25fps.mp4 ├── templates.py └── train_stage1.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/README.md -------------------------------------------------------------------------------- /assets/paper_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/paper_demo.png -------------------------------------------------------------------------------- /assets/samples_results/few_shot_pred.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/samples_results/few_shot_pred.gif -------------------------------------------------------------------------------- /assets/samples_results/few_shot_pred.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/samples_results/few_shot_pred.mp4 -------------------------------------------------------------------------------- /assets/samples_results/one_shot_portrait_pred.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/samples_results/one_shot_portrait_pred.gif -------------------------------------------------------------------------------- /assets/samples_results/one_shot_portrait_pred.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/samples_results/one_shot_portrait_pred.mp4 -------------------------------------------------------------------------------- /assets/samples_results/one_shot_pred.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/samples_results/one_shot_pred.gif -------------------------------------------------------------------------------- /assets/samples_results/one_shot_pred.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/samples_results/one_shot_pred.mp4 -------------------------------------------------------------------------------- /assets/single_images/test001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/single_images/test001.png -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_1.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_10.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_10.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_2.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_3.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_3.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_4.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_4.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_5.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_6.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_6.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_7.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_7.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_8.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_8.txt -------------------------------------------------------------------------------- /assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_9.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/landmarks/RD_Radio1_000_25fps/RD_Radio1_000_25fps_9.txt -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_1.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_10.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_2.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_3.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_4.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_5.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_6.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_7.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_8.png -------------------------------------------------------------------------------- /assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/assets/training_samples/raw_frames_25fps/RD_Radio1_000_25fps/RD_Radio1_000_25fps_9.png -------------------------------------------------------------------------------- /auto_download_ckpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/auto_download_ckpt.sh -------------------------------------------------------------------------------- /choices.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/choices.py -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/config.py -------------------------------------------------------------------------------- /config_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/config_base.py -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/dataset.py -------------------------------------------------------------------------------- /dataset_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/dataset_util.py -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/demo.py -------------------------------------------------------------------------------- /demo_with_batch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/demo_with_batch.py -------------------------------------------------------------------------------- /diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/diffusion/__init__.py -------------------------------------------------------------------------------- /diffusion/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/diffusion/base.py -------------------------------------------------------------------------------- /diffusion/diffusion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/diffusion/diffusion.py -------------------------------------------------------------------------------- /diffusion/resample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/diffusion/resample.py -------------------------------------------------------------------------------- /dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/dist_utils.py -------------------------------------------------------------------------------- /experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/experiment.py -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/index.html -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/__init__.py -------------------------------------------------------------------------------- /model/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/blocks.py -------------------------------------------------------------------------------- /model/latentnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/latentnet.py -------------------------------------------------------------------------------- /model/nn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/nn.py -------------------------------------------------------------------------------- /model/speech2latent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/speech2latent.py -------------------------------------------------------------------------------- /model/unet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/unet.py -------------------------------------------------------------------------------- /model/unet_autoenc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/model/unet_autoenc.py -------------------------------------------------------------------------------- /renderer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/renderer.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/requirements.txt -------------------------------------------------------------------------------- /resources/ablation_study/RD_Radio13_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/ablation_study/RD_Radio13_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/ablation_study/RD_Radio18_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/ablation_study/RD_Radio18_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/ablation_study/RD_Radio21_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/ablation_study/RD_Radio21_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/ablation_study/RD_Radio4_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/ablation_study/RD_Radio4_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/ablation_study/RD_Radio8_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/ablation_study/RD_Radio8_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/few_shot/test12_d8.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/few_shot/test12_d8.mp4 -------------------------------------------------------------------------------- /resources/few_shot/test19_driving_audio_6.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/few_shot/test19_driving_audio_6.mp4 -------------------------------------------------------------------------------- /resources/few_shot/test3_d6.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/few_shot/test3_d6.mp4 -------------------------------------------------------------------------------- /resources/few_shot/test5_driving_audio_2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/few_shot/test5_driving_audio_2.mp4 -------------------------------------------------------------------------------- /resources/few_shot/test6_driving_audio_5.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/few_shot/test6_driving_audio_5.mp4 -------------------------------------------------------------------------------- /resources/one_shot/RD_Radio11_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/one_shot/RD_Radio11_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/one_shot/RD_Radio17_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/one_shot/RD_Radio17_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/one_shot/RD_Radio25_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/one_shot/RD_Radio25_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/one_shot/RD_Radio29_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/one_shot/RD_Radio29_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/one_shot/RD_Radio5_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/one_shot/RD_Radio5_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/reconstruction/RD_Radio22_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/reconstruction/RD_Radio22_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/reconstruction/RD_Radio42_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/reconstruction/RD_Radio42_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/reconstruction/RD_Radio51_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/reconstruction/RD_Radio51_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/reconstruction/RD_Radio8_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/reconstruction/RD_Radio8_000_25fps.mp4 -------------------------------------------------------------------------------- /resources/reconstruction/WRA_PeterRoskam0_000_25fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/resources/reconstruction/WRA_PeterRoskam0_000_25fps.mp4 -------------------------------------------------------------------------------- /templates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/templates.py -------------------------------------------------------------------------------- /train_stage1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liutaocode/DiffDub/HEAD/train_stage1.py --------------------------------------------------------------------------------