├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── __init__.py ├── config.py ├── configs └── lipread_config.ini ├── cover.png ├── data ├── LRS3_lists.pkl ├── fixed_displacement_256.npy ├── head_template.obj ├── landmark_embedding.npy ├── list_full_mead_annotated.txt ├── mean_texture.jpg ├── phonemes2visemes.csv ├── texture_data_256.npy ├── uv_face_eye_mask.png └── uv_face_mask.png ├── datasets ├── __init__.py ├── build_datasets.py ├── data_utils.py ├── datasets.py └── extra_datasets.py ├── demo.py ├── get_training_data.sh ├── main.py ├── quick_install.sh ├── requirements.txt ├── samples ├── LRS3 │ ├── 0Fi83BHQsMA_00002.mp4 │ ├── OzA6jRYjVQs_00004.mp4 │ ├── PbgB2TaYhio_00007.mp4 │ └── QoT0S2vu9m4_00002.mp4 ├── MEAD │ ├── M003_level_1_disgusted_015.mp4 │ ├── M026_level_3_disgusted_018.mp4 │ ├── M030_level_3_angry_016.mp4 │ ├── W023_level_2_surprised_010.mp4 │ ├── W036_level_3_happy_007.mp4 │ └── W040_level_3_happy_029.mp4 └── visualizations │ ├── M003_level_1_angry_014_grid.gif │ ├── M003_level_1_angry_014_grid.mp4 │ ├── test_BImnT7lcLDE_00003_grid.gif │ └── test_BImnT7lcLDE_00003_grid.mp4 ├── src ├── __init__.py ├── models │ ├── FLAME.py │ ├── encoders.py │ ├── expression_loss.py │ ├── lbs.py │ └── resnet.py ├── spectre.py ├── trainer_spectre.py └── utils │ ├── lossfunc.py │ ├── renderer.py │ ├── rotation_converter.py │ ├── tensor_cropper.py │ ├── trainer.py │ └── util.py └── utils ├── __init__.py ├── extract_frames_LRS3.py ├── extract_frames_and_audio.py ├── extract_wavs_LRS3.py ├── lipread_utils.py └── run_av_hubert.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/README.md -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/config.py -------------------------------------------------------------------------------- /configs/lipread_config.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/configs/lipread_config.ini -------------------------------------------------------------------------------- /cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/cover.png -------------------------------------------------------------------------------- /data/LRS3_lists.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/LRS3_lists.pkl -------------------------------------------------------------------------------- /data/fixed_displacement_256.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/fixed_displacement_256.npy -------------------------------------------------------------------------------- /data/head_template.obj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/head_template.obj -------------------------------------------------------------------------------- /data/landmark_embedding.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/landmark_embedding.npy -------------------------------------------------------------------------------- /data/list_full_mead_annotated.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/list_full_mead_annotated.txt -------------------------------------------------------------------------------- /data/mean_texture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/mean_texture.jpg -------------------------------------------------------------------------------- /data/phonemes2visemes.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/phonemes2visemes.csv -------------------------------------------------------------------------------- /data/texture_data_256.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/texture_data_256.npy -------------------------------------------------------------------------------- /data/uv_face_eye_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/uv_face_eye_mask.png -------------------------------------------------------------------------------- /data/uv_face_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/data/uv_face_mask.png -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/build_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/datasets/build_datasets.py -------------------------------------------------------------------------------- /datasets/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/datasets/data_utils.py -------------------------------------------------------------------------------- /datasets/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/datasets/datasets.py -------------------------------------------------------------------------------- /datasets/extra_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/datasets/extra_datasets.py -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/demo.py -------------------------------------------------------------------------------- /get_training_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/get_training_data.sh -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/main.py -------------------------------------------------------------------------------- /quick_install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/quick_install.sh -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/requirements.txt -------------------------------------------------------------------------------- /samples/LRS3/0Fi83BHQsMA_00002.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/LRS3/0Fi83BHQsMA_00002.mp4 -------------------------------------------------------------------------------- /samples/LRS3/OzA6jRYjVQs_00004.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/LRS3/OzA6jRYjVQs_00004.mp4 -------------------------------------------------------------------------------- /samples/LRS3/PbgB2TaYhio_00007.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/LRS3/PbgB2TaYhio_00007.mp4 -------------------------------------------------------------------------------- /samples/LRS3/QoT0S2vu9m4_00002.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/LRS3/QoT0S2vu9m4_00002.mp4 -------------------------------------------------------------------------------- /samples/MEAD/M003_level_1_disgusted_015.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/MEAD/M003_level_1_disgusted_015.mp4 -------------------------------------------------------------------------------- /samples/MEAD/M026_level_3_disgusted_018.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/MEAD/M026_level_3_disgusted_018.mp4 -------------------------------------------------------------------------------- /samples/MEAD/M030_level_3_angry_016.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/MEAD/M030_level_3_angry_016.mp4 -------------------------------------------------------------------------------- /samples/MEAD/W023_level_2_surprised_010.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/MEAD/W023_level_2_surprised_010.mp4 -------------------------------------------------------------------------------- /samples/MEAD/W036_level_3_happy_007.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/MEAD/W036_level_3_happy_007.mp4 -------------------------------------------------------------------------------- /samples/MEAD/W040_level_3_happy_029.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/MEAD/W040_level_3_happy_029.mp4 -------------------------------------------------------------------------------- /samples/visualizations/M003_level_1_angry_014_grid.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/visualizations/M003_level_1_angry_014_grid.gif -------------------------------------------------------------------------------- /samples/visualizations/M003_level_1_angry_014_grid.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/visualizations/M003_level_1_angry_014_grid.mp4 -------------------------------------------------------------------------------- /samples/visualizations/test_BImnT7lcLDE_00003_grid.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/visualizations/test_BImnT7lcLDE_00003_grid.gif -------------------------------------------------------------------------------- /samples/visualizations/test_BImnT7lcLDE_00003_grid.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/samples/visualizations/test_BImnT7lcLDE_00003_grid.mp4 -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/models/FLAME.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/models/FLAME.py -------------------------------------------------------------------------------- /src/models/encoders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/models/encoders.py -------------------------------------------------------------------------------- /src/models/expression_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/models/expression_loss.py -------------------------------------------------------------------------------- /src/models/lbs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/models/lbs.py -------------------------------------------------------------------------------- /src/models/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/models/resnet.py -------------------------------------------------------------------------------- /src/spectre.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/spectre.py -------------------------------------------------------------------------------- /src/trainer_spectre.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/trainer_spectre.py -------------------------------------------------------------------------------- /src/utils/lossfunc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/utils/lossfunc.py -------------------------------------------------------------------------------- /src/utils/renderer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/utils/renderer.py -------------------------------------------------------------------------------- /src/utils/rotation_converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/utils/rotation_converter.py -------------------------------------------------------------------------------- /src/utils/tensor_cropper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/utils/tensor_cropper.py -------------------------------------------------------------------------------- /src/utils/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/utils/trainer.py -------------------------------------------------------------------------------- /src/utils/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/src/utils/util.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/extract_frames_LRS3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/utils/extract_frames_LRS3.py -------------------------------------------------------------------------------- /utils/extract_frames_and_audio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/utils/extract_frames_and_audio.py -------------------------------------------------------------------------------- /utils/extract_wavs_LRS3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/utils/extract_wavs_LRS3.py -------------------------------------------------------------------------------- /utils/lipread_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/utils/lipread_utils.py -------------------------------------------------------------------------------- /utils/run_av_hubert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/filby89/spectre/HEAD/utils/run_av_hubert.py --------------------------------------------------------------------------------