├── .github └── workflows │ └── static.yml ├── .gitignore ├── 20words_mean_face.npy ├── DataPreProcess ├── process_lrs23.py └── process_vox2.py ├── LICENSE ├── README.md ├── README_zh-CN.md ├── configs ├── LRS2-IIANet.yml ├── LRS3-IIANet.yml └── Vox2-IIANet.yml ├── figures ├── IIANet-Figure1B.png ├── audio-waves.png ├── overall.gif ├── overall.mp4 ├── results.png ├── separation.gif ├── separation.mp4 └── spec.png ├── files ├── css ├── font-awesome.min.css └── style.css ├── index.html ├── inference.py ├── look2hear ├── datas │ ├── __init__.py │ ├── avspeech_dataset.py │ ├── avspeech_dymanic_dataset.py │ └── transform.py ├── losses │ ├── __init__.py │ ├── matrix.py │ ├── mixit.py │ └── pit_wrapper.py ├── metrics │ ├── __init__.py │ ├── splitwrapper.py │ └── wrapper.py ├── models │ ├── IIANet.py │ ├── __init__.py │ └── base_av_model.py ├── system │ ├── __init__.py │ ├── av_litmodule.py │ ├── comet.py │ ├── core.py │ ├── optimizers.py │ └── tensorboard.py ├── utils │ ├── __init__.py │ ├── lightning_utils.py │ ├── parser_utils.py │ ├── separator.py │ ├── stft.py │ └── torch_utils.py └── videomodels │ ├── __init__.py │ ├── resnet.py │ └── resnet_videomodel.py ├── requirements.txt ├── test.py ├── test_videos └── video.mp4 └── train.py /.github/workflows/static.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/.github/workflows/static.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/.gitignore -------------------------------------------------------------------------------- /20words_mean_face.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/20words_mean_face.npy -------------------------------------------------------------------------------- /DataPreProcess/process_lrs23.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/DataPreProcess/process_lrs23.py -------------------------------------------------------------------------------- /DataPreProcess/process_vox2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/DataPreProcess/process_vox2.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/README.md -------------------------------------------------------------------------------- /README_zh-CN.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/README_zh-CN.md -------------------------------------------------------------------------------- /configs/LRS2-IIANet.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/configs/LRS2-IIANet.yml -------------------------------------------------------------------------------- /configs/LRS3-IIANet.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/configs/LRS3-IIANet.yml -------------------------------------------------------------------------------- /configs/Vox2-IIANet.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/configs/Vox2-IIANet.yml -------------------------------------------------------------------------------- /figures/IIANet-Figure1B.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/IIANet-Figure1B.png -------------------------------------------------------------------------------- /figures/audio-waves.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/audio-waves.png -------------------------------------------------------------------------------- /figures/overall.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/overall.gif -------------------------------------------------------------------------------- /figures/overall.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/overall.mp4 -------------------------------------------------------------------------------- /figures/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/results.png -------------------------------------------------------------------------------- /figures/separation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/separation.gif -------------------------------------------------------------------------------- /figures/separation.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/separation.mp4 -------------------------------------------------------------------------------- /figures/spec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/figures/spec.png -------------------------------------------------------------------------------- /files/css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/files/css -------------------------------------------------------------------------------- /files/font-awesome.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/files/font-awesome.min.css -------------------------------------------------------------------------------- /files/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/files/style.css -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/index.html -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/inference.py -------------------------------------------------------------------------------- /look2hear/datas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/datas/__init__.py -------------------------------------------------------------------------------- /look2hear/datas/avspeech_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/datas/avspeech_dataset.py -------------------------------------------------------------------------------- /look2hear/datas/avspeech_dymanic_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/datas/avspeech_dymanic_dataset.py -------------------------------------------------------------------------------- /look2hear/datas/transform.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/datas/transform.py -------------------------------------------------------------------------------- /look2hear/losses/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/losses/__init__.py -------------------------------------------------------------------------------- /look2hear/losses/matrix.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/losses/matrix.py -------------------------------------------------------------------------------- /look2hear/losses/mixit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/losses/mixit.py -------------------------------------------------------------------------------- /look2hear/losses/pit_wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/losses/pit_wrapper.py -------------------------------------------------------------------------------- /look2hear/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/metrics/__init__.py -------------------------------------------------------------------------------- /look2hear/metrics/splitwrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/metrics/splitwrapper.py -------------------------------------------------------------------------------- /look2hear/metrics/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/metrics/wrapper.py -------------------------------------------------------------------------------- /look2hear/models/IIANet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/models/IIANet.py -------------------------------------------------------------------------------- /look2hear/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/models/__init__.py -------------------------------------------------------------------------------- /look2hear/models/base_av_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/models/base_av_model.py -------------------------------------------------------------------------------- /look2hear/system/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/system/__init__.py -------------------------------------------------------------------------------- /look2hear/system/av_litmodule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/system/av_litmodule.py -------------------------------------------------------------------------------- /look2hear/system/comet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/system/comet.py -------------------------------------------------------------------------------- /look2hear/system/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/system/core.py -------------------------------------------------------------------------------- /look2hear/system/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/system/optimizers.py -------------------------------------------------------------------------------- /look2hear/system/tensorboard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/system/tensorboard.py -------------------------------------------------------------------------------- /look2hear/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/utils/__init__.py -------------------------------------------------------------------------------- /look2hear/utils/lightning_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/utils/lightning_utils.py -------------------------------------------------------------------------------- /look2hear/utils/parser_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/utils/parser_utils.py -------------------------------------------------------------------------------- /look2hear/utils/separator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/utils/separator.py -------------------------------------------------------------------------------- /look2hear/utils/stft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/utils/stft.py -------------------------------------------------------------------------------- /look2hear/utils/torch_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/utils/torch_utils.py -------------------------------------------------------------------------------- /look2hear/videomodels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/videomodels/__init__.py -------------------------------------------------------------------------------- /look2hear/videomodels/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/videomodels/resnet.py -------------------------------------------------------------------------------- /look2hear/videomodels/resnet_videomodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/look2hear/videomodels/resnet_videomodel.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/requirements.txt -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/test.py -------------------------------------------------------------------------------- /test_videos/video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/test_videos/video.mp4 -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JusperLee/IIANet/HEAD/train.py --------------------------------------------------------------------------------