├── eval ├── metric.txt ├── all_scores_tmp.txt ├── lmd │ ├── temp │ │ └── .gitkeep │ ├── checkpoints │ │ └── .gitkeep │ ├── third_part │ │ ├── face3d │ │ │ ├── BFM │ │ │ │ ├── .gitkeep │ │ │ │ └── std_exp.txt │ │ │ ├── models │ │ │ │ └── arcface_torch │ │ │ │ │ ├── configs │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── 3millions.py │ │ │ │ │ ├── speed.py │ │ │ │ │ ├── 3millions_pfc.py │ │ │ │ │ ├── glint360k_mbf.py │ │ │ │ │ ├── glint360k_r100.py │ │ │ │ │ ├── glint360k_r18.py │ │ │ │ │ ├── glint360k_r34.py │ │ │ │ │ ├── glint360k_r50.py │ │ │ │ │ ├── ms1mv3_mbf.py │ │ │ │ │ ├── ms1mv3_r18.py │ │ │ │ │ ├── ms1mv3_r34.py │ │ │ │ │ ├── ms1mv3_r50.py │ │ │ │ │ ├── ms1mv3_r2060.py │ │ │ │ │ └── base.py │ │ │ │ │ ├── docs │ │ │ │ │ ├── modelzoo.md │ │ │ │ │ ├── eval.md │ │ │ │ │ └── install.md │ │ │ │ │ ├── eval │ │ │ │ │ └── __init__.py │ │ │ │ │ ├── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── utils_os.py │ │ │ │ │ ├── utils_config.py │ │ │ │ │ ├── utils_logging.py │ │ │ │ │ └── plot.py │ │ │ │ │ ├── requirement.txt │ │ │ │ │ ├── run.sh │ │ │ │ │ ├── backbones │ │ │ │ │ └── __init__.py │ │ │ │ │ ├── inference.py │ │ │ │ │ ├── losses.py │ │ │ │ │ └── torch2onnx.py │ │ │ ├── util │ │ │ │ ├── __init__.py │ │ │ │ └── generate_list.py │ │ │ ├── options │ │ │ │ ├── __init__.py │ │ │ │ ├── test_options.py │ │ │ │ └── inference_options.py │ │ │ ├── data_preparation.py │ │ │ ├── checkpoints │ │ │ │ └── model_name │ │ │ │ │ └── test_opt.txt │ │ │ └── data │ │ │ │ └── image_folder.py │ │ ├── GPEN │ │ │ ├── face_detect │ │ │ │ ├── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── nms │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── py_cpu_nms.py │ │ │ │ │ └── timer.py │ │ │ │ ├── facemodels │ │ │ │ │ └── __init__.py │ │ │ │ ├── layers │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── modules │ │ │ │ │ │ └── __init__.py │ │ │ │ │ └── functions │ │ │ │ │ │ └── prior_box.py │ │ │ │ ├── .DS_Store │ │ │ │ └── data │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── config.py │ │ │ ├── face_morpher │ │ │ │ ├── .gitignore │ │ │ │ ├── requirements.txt │ │ │ │ ├── setup.cfg │ │ │ │ ├── facemorpher │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── videoer.py │ │ │ │ ├── scripts │ │ │ │ │ ├── make_docs.sh │ │ │ │ │ └── publish_ghpages.sh │ │ │ │ └── setup.py │ │ │ └── face_model │ │ │ │ ├── op │ │ │ │ ├── __init__.py │ │ │ │ ├── fused_bias_act.cpp │ │ │ │ └── upfirdn2d.cpp │ │ │ │ └── face_gan.py │ │ ├── face_detection │ │ │ ├── detection │ │ │ │ ├── __init__.py │ │ │ │ └── sfd │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── sfd_detector.py │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ └── api.py │ │ ├── ganimation_replicate │ │ │ ├── data │ │ │ │ ├── __init__.py │ │ │ │ ├── data_loader.py │ │ │ │ └── celeba.py │ │ │ ├── ckpts │ │ │ │ ├── ganimation │ │ │ │ │ ├── 220419_183211 │ │ │ │ │ │ └── run_script.sh │ │ │ │ │ └── 220419_183229 │ │ │ │ │ │ └── run_script.sh │ │ │ │ └── run_script.sh │ │ │ ├── main.py │ │ │ ├── model │ │ │ │ └── __init__.py │ │ │ ├── LICENSE │ │ │ └── checkpoints │ │ │ │ └── run_script.sh │ │ └── GFPGAN │ │ │ └── gfpgan │ │ │ ├── weights │ │ │ └── README.md │ │ │ ├── version.py │ │ │ ├── __init__.py │ │ │ ├── train.py │ │ │ ├── archs │ │ │ └── __init__.py │ │ │ ├── models │ │ │ └── __init__.py │ │ │ └── data │ │ │ └── __init__.py │ ├── lmd_eval.sh │ ├── lmd_eval_bydir.sh │ └── utils │ │ └── flow_util.py ├── metrics │ ├── encoders │ │ ├── __init__.py │ │ └── w_encoder.py │ └── mtcnn │ │ ├── __init__.py │ │ └── mtcnn_pytorch │ │ ├── __init__.py │ │ └── src │ │ ├── __init__.py │ │ └── visualization_utils.py ├── syncnet_python │ ├── data │ │ └── .gitkeep │ ├── all_scores_tmp.txt │ ├── detectors │ │ ├── s3fd │ │ │ ├── weights │ │ │ │ └── .gitkeep │ │ │ └── __init__.py │ │ ├── __init__.py │ │ └── README.md │ ├── requirements.txt │ ├── download_model.sh │ ├── calculate_scores_single_video.sh │ ├── calculate_scores_real_videos.sh │ ├── .gitignore │ ├── demo_syncnet.py │ ├── demo_feature.py │ ├── LICENSE.md │ ├── run_syncnet.py │ ├── calculate_scores_real_videos.py │ ├── README.md │ └── calculate_scores_LRS.py ├── metric.json ├── au_detection.sh └── calc_FID.py ├── models └── .gitkeep ├── GFPGAN ├── VERSION ├── tests │ ├── data │ │ ├── ffhq_gt.lmdb │ │ │ ├── meta_info.txt │ │ │ ├── data.mdb │ │ │ └── lock.mdb │ │ └── test_ffhq_degradation_dataset.yml │ ├── test_utils.py │ ├── test_arcface_arch.py │ └── test_stylegan2_clean_arch.py ├── gfpgan │ ├── weights │ │ └── README.md │ ├── __init__.py │ ├── train.py │ ├── archs │ │ └── __init__.py │ ├── models │ │ └── __init__.py │ └── data │ │ └── __init__.py ├── MANIFEST.in ├── cog.yaml ├── experiments │ └── pretrained_models │ │ └── README.md ├── .vscode │ └── settings.json ├── setup.cfg └── .pre-commit-config.yaml ├── preprocess_utils ├── data_utils │ ├── face_tracking │ │ ├── __init__.py │ │ ├── 3DMM │ │ │ └── .gitkeep │ │ ├── data_loader.py │ │ ├── convert_BFM.py │ │ └── geo_transform.py │ ├── deepspeech_features │ │ ├── fea_win.py │ │ └── README.md │ └── face_parsing │ │ └── logger.py ├── do_au_detection.sh ├── do_audio_and_image_preprocessing.sh ├── do_vive3d_training.sh └── detect_wav2lip_bbox.py ├── Wav2Lip ├── checkpoints │ └── README.md ├── face_detection │ ├── detection │ │ ├── __init__.py │ │ └── sfd │ │ │ ├── __init__.py │ │ │ └── sfd_detector.py │ ├── README.md │ └── __init__.py ├── filelists │ └── README.md ├── results │ └── README.md ├── requirements.txt ├── models │ ├── __init__.py │ └── conv.py ├── temp │ └── README.md ├── .gitignore └── evaluation │ ├── scores_LSE │ ├── calculate_scores_real_videos.sh │ ├── calculate_scores_real_videos.py │ └── calculate_scores_LRS.py │ └── test_filelists │ ├── ReSyncED │ └── tts_pairs.txt │ └── README.md ├── docs ├── structure.png ├── download_dataset.md └── preprocessing.md ├── eg3d ├── dataset_preprocessing │ ├── ffhq │ │ ├── preprocess.py │ │ ├── preprocess_in_the_wild.py │ │ ├── crop_images.py │ │ ├── crop_images_in_the_wild.py │ │ ├── 3dface2idr_mat.py │ │ └── validate_ffhq.py │ └── shapenet_cars │ │ └── run_me.py ├── .gitmodules ├── .gitignore └── eg3d │ ├── viz │ ├── __init__.py │ ├── backbone_cache_widget.py │ ├── render_type_widget.py │ ├── render_depth_sample_widget.py │ ├── zoom_widget.py │ └── conditioning_pose_widget.py │ ├── metrics │ ├── __init__.py │ ├── inception_score.py │ ├── frechet_inception_distance.py │ └── kernel_inception_distance.py │ ├── training │ ├── __init__.py │ ├── volumetric_rendering │ │ └── __init__.py │ └── crosssection_utils.py │ ├── gui_utils │ └── __init__.py │ ├── torch_utils │ ├── __init__.py │ └── ops │ │ ├── __init__.py │ │ ├── bias_act.h │ │ ├── filtered_lrelu_rd.cu │ │ ├── filtered_lrelu_wr.cu │ │ ├── filtered_lrelu_ns.cu │ │ ├── upfirdn2d.h │ │ └── fma.py │ ├── dnnlib │ └── __init__.py │ └── environment.yml ├── vive3D ├── config.py └── landmark_detector.py ├── .gitignore ├── do_preprocessing.sh ├── do_train.sh ├── demo.sh ├── do_evaluation.sh ├── training ├── __init__.py ├── volumetric_rendering │ └── __init__.py └── crosssection_utils.py ├── torch_utils ├── __init__.py └── ops │ ├── __init__.py │ ├── bias_act.h │ ├── filtered_lrelu_rd.cu │ ├── filtered_lrelu_wr.cu │ ├── filtered_lrelu_ns.cu │ ├── upfirdn2d.h │ └── fma.py └── dnnlib └── __init__.py /eval/metric.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/all_scores_tmp.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/temp/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /GFPGAN/VERSION: -------------------------------------------------------------------------------- 1 | 1.3.8 2 | -------------------------------------------------------------------------------- /eval/lmd/checkpoints/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/metrics/encoders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/metrics/mtcnn/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/syncnet_python/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/syncnet_python/all_scores_tmp.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/BFM/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/metrics/mtcnn/mtcnn_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/syncnet_python/detectors/s3fd/weights/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/face_tracking/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/utils/nms/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/face_tracking/3DMM/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/facemodels/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/syncnet_python/detectors/__init__.py: -------------------------------------------------------------------------------- 1 | from .s3fd import S3FD -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/docs/modelzoo.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/utils/utils_os.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /GFPGAN/tests/data/ffhq_gt.lmdb/meta_info.txt: -------------------------------------------------------------------------------- 1 | 00000000.png (512,512,3) 1 2 | -------------------------------------------------------------------------------- /Wav2Lip/checkpoints/README.md: -------------------------------------------------------------------------------- 1 | Place all your checkpoints (.pth files) here. -------------------------------------------------------------------------------- /Wav2Lip/face_detection/detection/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import FaceDetector -------------------------------------------------------------------------------- /Wav2Lip/filelists/README.md: -------------------------------------------------------------------------------- 1 | Place LRS2 (and any other) filelists here for training. -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | MANIFEST 4 | -------------------------------------------------------------------------------- /Wav2Lip/results/README.md: -------------------------------------------------------------------------------- 1 | Generated results will be placed in this folder by default. -------------------------------------------------------------------------------- /eval/lmd/third_part/face_detection/detection/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import FaceDetector -------------------------------------------------------------------------------- /docs/structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-kaist/Talk3D/HEAD/docs/structure.png -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/ffhq/preprocess.py: -------------------------------------------------------------------------------- 1 | ./Deep3DFaceRecon_pytorch/util/preprocess.py -------------------------------------------------------------------------------- /GFPGAN/gfpgan/weights/README.md: -------------------------------------------------------------------------------- 1 | # Weights 2 | 3 | Put the downloaded weights to this folder. 4 | -------------------------------------------------------------------------------- /Wav2Lip/face_detection/detection/sfd/__init__.py: -------------------------------------------------------------------------------- 1 | from .sfd_detector import SFDDetector as FaceDetector -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .data_loader import create_dataloader -------------------------------------------------------------------------------- /vive3D/config.py: -------------------------------------------------------------------------------- 1 | HOME = '' 2 | PROJECT = f'' 3 | 4 | BOUNDARY_PATH = f'{PROJECT}/boundaries' 5 | 6 | -------------------------------------------------------------------------------- /Wav2Lip/requirements.txt: -------------------------------------------------------------------------------- 1 | librosa 2 | numpy 3 | opencv-contrib-python 4 | opencv-python 5 | tqdm 6 | numba -------------------------------------------------------------------------------- /Wav2Lip/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .wav2lip import Wav2Lip, Wav2Lip_disc_qual 2 | from .syncnet import SyncNet_color -------------------------------------------------------------------------------- /Wav2Lip/temp/README.md: -------------------------------------------------------------------------------- 1 | Temporary files at the time of inference/testing will be saved here. You can ignore them. -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from .functions import * 2 | from .modules import * 3 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy 3 | matplotlib 4 | docopt 5 | dlib 6 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face_detection/detection/sfd/__init__.py: -------------------------------------------------------------------------------- 1 | from .sfd_detector import SFDDetector as FaceDetector -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/weights/README.md: -------------------------------------------------------------------------------- 1 | # Weights 2 | 3 | Put the downloaded weights to this folder. 4 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/requirement.txt: -------------------------------------------------------------------------------- 1 | tensorboard 2 | easydict 3 | mxnet 4 | onnx 5 | sklearn 6 | -------------------------------------------------------------------------------- /GFPGAN/tests/data/ffhq_gt.lmdb/data.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-kaist/Talk3D/HEAD/GFPGAN/tests/data/ffhq_gt.lmdb/data.mdb -------------------------------------------------------------------------------- /GFPGAN/tests/data/ffhq_gt.lmdb/lock.mdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-kaist/Talk3D/HEAD/GFPGAN/tests/data/ffhq_gt.lmdb/lock.mdb -------------------------------------------------------------------------------- /eval/metrics/mtcnn/mtcnn_pytorch/src/__init__.py: -------------------------------------------------------------------------------- 1 | from .visualization_utils import show_bboxes 2 | from .detector import detect_faces 3 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/layers/modules/__init__.py: -------------------------------------------------------------------------------- 1 | from .multibox_loss import MultiBoxLoss 2 | 3 | __all__ = ['MultiBoxLoss'] 4 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/ckpts/ganimation/220419_183211/run_script.sh: -------------------------------------------------------------------------------- 1 | [train][220419_183211]python test.py --data_root . 2 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/ckpts/ganimation/220419_183229/run_script.sh: -------------------------------------------------------------------------------- 1 | [train][220419_183229]python test.py --data_root . 2 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-kaist/Talk3D/HEAD/eval/lmd/third_part/GPEN/face_detect/.DS_Store -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_model/op/__init__.py: -------------------------------------------------------------------------------- 1 | from .fused_act import FusedLeakyReLU, fused_leaky_relu 2 | from .upfirdn2d import upfirdn2d 3 | -------------------------------------------------------------------------------- /eval/syncnet_python/detectors/README.md: -------------------------------------------------------------------------------- 1 | # Face detector 2 | 3 | This face detector is adapted from `https://github.com/cs-giung/face-detection-pytorch`. 4 | -------------------------------------------------------------------------------- /eval/lmd/lmd_eval.sh: -------------------------------------------------------------------------------- 1 | PRED=$1 2 | GT=$2 3 | 4 | cd lmd 5 | python3 lmd_eval.py \ 6 | --pred $PRED \ 7 | --gt $GT \ 8 | --fps 25 \ 9 | 10 | cd .. -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/util/__init__.py: -------------------------------------------------------------------------------- 1 | """This package includes a miscellaneous collection of useful helper functions.""" 2 | from face3d.util import * 3 | -------------------------------------------------------------------------------- /GFPGAN/gfpgan/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .archs import * 3 | from .data import * 4 | from .models import * 5 | from .utils import * 6 | 7 | # from .version import * 8 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .wider_face import WiderFaceDetection, detection_collate 2 | from .data_augment import * 3 | from .config import * 4 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/options/__init__.py: -------------------------------------------------------------------------------- 1 | """This package options includes option modules: training options, test options, and basic options (used in both training and test).""" 2 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/version.py: -------------------------------------------------------------------------------- 1 | # GENERATED VERSION FILE 2 | # TIME: Wed Apr 20 14:43:06 2022 3 | __version__ = '1.3.2' 4 | __gitsha__ = '924ce47' 5 | version_info = (1, 3, 2) 6 | -------------------------------------------------------------------------------- /eval/syncnet_python/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.18.1 2 | scipy>=1.2.1 3 | scenedetect==0.5.1 4 | opencv-contrib-python 5 | python_speech_features 6 | # torch>=1.4.0 7 | # torchvision>=0.5.0 8 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | 3 | from .archs import * 4 | from .data import * 5 | from .models import * 6 | from .utils import * 7 | 8 | # from .version import * 9 | -------------------------------------------------------------------------------- /eg3d/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "dataset_preprocessing/ffhq/Deep3DFaceRecon_pytorch"] 2 | path = dataset_preprocessing/ffhq/Deep3DFaceRecon_pytorch 3 | url = https://github.com/sicxu/Deep3DFaceRecon_pytorch.git 4 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/setup.cfg: -------------------------------------------------------------------------------- 1 | [pep8] 2 | ignore = E111,E114,E226,E302,E41,E121,E701 3 | max-line-length = 100 4 | 5 | [flake8] 6 | ignore = E111,E114,E226,E302,E41,E121,E701 7 | max-line-length = 100 -------------------------------------------------------------------------------- /GFPGAN/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include assets/* 2 | include inputs/* 3 | include scripts/*.py 4 | include inference_gfpgan.py 5 | include VERSION 6 | include LICENSE 7 | include requirements.txt 8 | include gfpgan/weights/README.md 9 | -------------------------------------------------------------------------------- /Wav2Lip/.gitignore: -------------------------------------------------------------------------------- 1 | *.pkl 2 | *.jpg 3 | *.mp4 4 | *.pth 5 | *.pyc 6 | __pycache__ 7 | *.h5 8 | *.avi 9 | *.wav 10 | filelists/*.txt 11 | evaluation/test_filelists/lr*.txt 12 | *.pyc 13 | *.mkv 14 | *.gif 15 | *.webm 16 | *.mp3 17 | -------------------------------------------------------------------------------- /Wav2Lip/face_detection/README.md: -------------------------------------------------------------------------------- 1 | The code for Face Detection in this folder has been taken from the wonderful [face_alignment](https://github.com/1adrianb/face-alignment) repository. This has been modified to take batches of faces at a time. -------------------------------------------------------------------------------- /eval/metric.json: -------------------------------------------------------------------------------- 1 | { 2 | "PSNR": "-1.000", 3 | "SSIM": "-1.000", 4 | "LPIPS": "-1.000", 5 | "ID-SIM": "-1.000", 6 | "FID": "-1.000", 7 | "SYNC": "-1.000", 8 | "LMD": "-1.000", 9 | "AUE": "-1.000" 10 | } -------------------------------------------------------------------------------- /Wav2Lip/face_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __author__ = """Adrian Bulat""" 4 | __email__ = 'adrian.bulat@nottingham.ac.uk' 5 | __version__ = '1.0.1' 6 | 7 | from .api import FaceAlignment, LandmarksType, NetworkSize 8 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face_detection/README.md: -------------------------------------------------------------------------------- 1 | The code for Face Detection in this folder has been taken from the wonderful [face_alignment](https://github.com/1adrianb/face-alignment) repository. This has been modified to take batches of faces at a time. -------------------------------------------------------------------------------- /eval/lmd/third_part/face_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __author__ = """Adrian Bulat""" 4 | __email__ = 'adrian.bulat@nottingham.ac.uk' 5 | __version__ = '1.0.1' 6 | 7 | from .api import FaceAlignment, LandmarksType, NetworkSize 8 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/facemorpher/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Face Morpher module init code 3 | """ 4 | from .morpher import morpher, list_imgpaths 5 | from .averager import averager 6 | 7 | __all__ = ['list_imgpaths', 8 | 'morpher', 9 | 'averager'] 10 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/run.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/ms1mv3_r50 2 | ps -ef | grep "train" | grep -v grep | awk '{print "kill -9 "$2}' | sh 3 | -------------------------------------------------------------------------------- /eval/lmd/lmd_eval_bydir.sh: -------------------------------------------------------------------------------- 1 | save_file="all_scores_"$1".txt" 2 | 3 | rm $save_file 4 | yourfilenames=`ls ../results/$1` 5 | 6 | for eachfile in $yourfilenames 7 | do 8 | echo $eachfile >> $save_file 9 | python3 lmd_eval.py --pred ../results/"$1"/"$eachfile" --gt ../gt/"$eachfile" --fps 25 >> $save_file 10 | done 11 | -------------------------------------------------------------------------------- /GFPGAN/gfpgan/train.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import os.path as osp 3 | from basicsr.train import train_pipeline 4 | 5 | import gfpgan.archs 6 | import gfpgan.data 7 | import gfpgan.models 8 | 9 | if __name__ == '__main__': 10 | root_path = osp.abspath(osp.join(__file__, osp.pardir, osp.pardir)) 11 | train_pipeline(root_path) 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | video/* 2 | savepoints/* 3 | wandb/* 4 | data/* 5 | ori_imgs/* 6 | 7 | __pycache__ 8 | .ipynb_checkpoints 9 | *.mp4 10 | *.png 11 | !structure.png 12 | *.jpg 13 | *.pth 14 | *.pt 15 | *.wav 16 | *.npy 17 | *.pkl 18 | *.ipynb 19 | *.csv 20 | *.mat 21 | *.obj 22 | *.avi 23 | *.model 24 | *.dat 25 | *.bin 26 | do_train2.sh -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/train.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import os.path as osp 3 | from basicsr.train import train_pipeline 4 | 5 | import gfpgan.archs 6 | import gfpgan.data 7 | import gfpgan.models 8 | 9 | if __name__ == '__main__': 10 | root_path = osp.abspath(osp.join(__file__, osp.pardir, osp.pardir)) 11 | train_pipeline(root_path) 12 | -------------------------------------------------------------------------------- /eg3d/.gitignore: -------------------------------------------------------------------------------- 1 | images/ 2 | .DS_Store 3 | .ipynb_checkpoints 4 | data 5 | output 6 | debug 7 | *.pyc 8 | deep-head-pose 9 | EvalImages 10 | cache* 11 | *.pkl 12 | gif 13 | archive 14 | *.ply 15 | eval 16 | out 17 | 18 | # evaluation: 19 | temp/ 20 | shapes/ 21 | imgs/ 22 | vids/ 23 | *.mp4 24 | 25 | stylegan3/results 26 | eg3d/results 27 | eg3d_results 28 | -------------------------------------------------------------------------------- /do_preprocessing.sh: -------------------------------------------------------------------------------- 1 | INPUT_VID=data/May.mp4 2 | IDENTITY=May 3 | ROOT_DIR=/path/to/data/root/directory 4 | 5 | mkdir $ROOT_DIR/$IDENTITY 6 | sh preprocess_utils/do_audio_and_image_preprocessing.sh $INPUT_VID $IDENTITY $ROOT_DIR 7 | sh preprocess_utils/do_au_detection.sh $INPUT_VID $IDENTITY $ROOT_DIR 8 | sh preprocess_utils/do_vive3d_training.sh $INPUT_VID $IDENTITY $ROOT_DIR 9 | -------------------------------------------------------------------------------- /Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.sh: -------------------------------------------------------------------------------- 1 | rm all_scores.txt 2 | yourfilenames=`ls $1` 3 | 4 | for eachfile in $yourfilenames 5 | do 6 | python run_pipeline.py --videofile $1/$eachfile --reference wav2lip --data_dir tmp_dir 7 | python calculate_scores_real_videos.py --videofile $1/$eachfile --reference wav2lip --data_dir tmp_dir >> all_scores.txt 8 | done 9 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Dec 13, 2018 3 | @author: Yuedong Chen 4 | """ 5 | 6 | from options import Options 7 | from solvers import create_solver 8 | 9 | 10 | 11 | 12 | if __name__ == '__main__': 13 | opt = Options().parse() 14 | 15 | solver = create_solver(opt) 16 | solver.run_solver() 17 | 18 | print('[THE END]') -------------------------------------------------------------------------------- /do_train.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=1 python main.py \ 2 | --saveroot_path path/to/saving/directory \ 3 | --data_root_dir path/to/data/root/directory \ 4 | --personal_id May \ 5 | --short_configs test \ 6 | --checkpoint_dir None \ 7 | --use_GFPGAN \ 8 | --do_inference \ 9 | --do_inference_novel \ 10 | --do_inference_OOD \ 11 | --val_max_length -1 \ 12 | --num_gpus 1 \ 13 | --master_port 15379 \ 14 | --inf_camera_type rotation -------------------------------------------------------------------------------- /demo.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=1 python main.py \ 2 | --saveroot_path path/to/saving/directory \ 3 | --data_root_dir path/to/data/root/directory \ 4 | --checkpoint_dir ./May_demo.pt \ 5 | --personal_id May \ 6 | --short_configs test \ 7 | --use_GFPGAN \ 8 | --do_inference \ 9 | --do_inference_novel \ 10 | --do_inference_OOD \ 11 | --val_max_length -1 \ 12 | --num_gpus 1 \ 13 | --master_port 15379 \ 14 | --inf_camera_type rotation \ 15 | --only_do_inference -------------------------------------------------------------------------------- /eval/syncnet_python/download_model.sh: -------------------------------------------------------------------------------- 1 | # SyncNet model 2 | 3 | mkdir data 4 | wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/syncnet_v2.model -O data/syncnet_v2.model 5 | wget http://www.robots.ox.ac.uk/~vgg/software/lipsync/data/example.avi -O data/example.avi 6 | 7 | # For the pre-processing pipeline 8 | mkdir detectors/s3fd/weights 9 | wget https://www.robots.ox.ac.uk/~vgg/software/lipsync/data/sfd_face.pth -O detectors/s3fd/weights/sfd_face.pth -------------------------------------------------------------------------------- /eval/syncnet_python/calculate_scores_single_video.sh: -------------------------------------------------------------------------------- 1 | save_file="all_scores_tmp.txt" 2 | 3 | rm $save_file 4 | 5 | echo dist conf >> $save_file 6 | 7 | eachfile=$1 8 | 9 | echo $eachfile >> $save_file 10 | cd syncnet_python 11 | python run_pipeline.py --videofile $eachfile --reference wav2lip --data_dir tmp_dir 12 | python calculate_scores_real_videos.py --videofile $eachfile --reference wav2lip --data_dir tmp_dir >> $save_file 13 | rm -r tmp_dir 14 | cd .. 15 | -------------------------------------------------------------------------------- /eval/au_detection.sh: -------------------------------------------------------------------------------- 1 | VID_DIR=$1 2 | VID_NAME=$2 3 | LOCAL_DIR=$3 4 | SAVE_DIR=$4 5 | 6 | mkdir data_openface 7 | cp $VID_DIR data_openface 8 | 9 | docker run --rm --ipc=host -v $LOCAL_DIR/data_openface/:/opt/OpenFace/build/bin/facecam_exp benbuleong/openface-cambridge ./FeatureExtraction -f facecam_exp/$VID_NAME -outroot facecam_exp/ -of $SAVE_DIR -q 10 | 11 | # mv $ROOT_DIR/data/$IDENTITY/data_openface/au.csv $SAVE_DIR 12 | # rm -rf $ROOT_DIR/data/$IDENTITY/data_openface -------------------------------------------------------------------------------- /do_evaluation.sh: -------------------------------------------------------------------------------- 1 | # evaluation bash file for metric logging while training 2 | ID=$1 3 | CONFIG=$2 4 | DATADIR=$3 5 | INPUTVID=$4 6 | EVAL_TYPE=$5 7 | INF_TYPE=$6 8 | 9 | cd eval/ 10 | 11 | python compare.py \ 12 | --ID $ID \ 13 | --short_configs $CONFIG \ 14 | --data_dir $DATADIR \ 15 | --input_video $INPUTVID \ 16 | --eval_type $EVAL_TYPE \ 17 | --inf_type $INF_TYPE \ 18 | 19 | cd .. 20 | 21 | # usage : sh do_evaluation obama tmp path/to/dataroot path/to/synth_video.mp4 all novel -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_model import BaseModel 2 | from .ganimation import GANimationModel 3 | from .stargan import StarGANModel 4 | 5 | 6 | 7 | def create_model(opt): 8 | # specify model name here 9 | if opt.model == "ganimation": 10 | instance = GANimationModel() 11 | elif opt.model == "stargan": 12 | instance = StarGANModel() 13 | else: 14 | instance = BaseModel() 15 | instance.initialize(opt) 16 | instance.setup() 17 | return instance 18 | 19 | -------------------------------------------------------------------------------- /eval/syncnet_python/calculate_scores_real_videos.sh: -------------------------------------------------------------------------------- 1 | save_file="all_scores_"$1".txt" 2 | 3 | rm $save_file 4 | 5 | yourfilenames=`ls ../results/$1` 6 | 7 | echo dist conf >> $save_file 8 | 9 | for eachfile in $yourfilenames 10 | do 11 | echo $eachfile >> $save_file 12 | python run_pipeline.py --videofile ../results/$1/$eachfile --reference wav2lip --data_dir tmp_dir 13 | python calculate_scores_real_videos.py --videofile ../results/$1/$eachfile --reference wav2lip --data_dir tmp_dir >> $save_file 14 | rm -r tmp_dir 15 | done 16 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/deepspeech_features/fea_win.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | net_output = np.load('french.ds.npy').reshape(-1, 29) 4 | win_size = 16 5 | zero_pad = np.zeros((int(win_size / 2), net_output.shape[1])) 6 | net_output = np.concatenate((zero_pad, net_output, zero_pad), axis=0) 7 | windows = [] 8 | for window_index in range(0, net_output.shape[0] - win_size, 2): 9 | windows.append(net_output[window_index:window_index + win_size]) 10 | print(np.array(windows).shape) 11 | np.save('aud_french.npy', np.array(windows)) 12 | -------------------------------------------------------------------------------- /docs/download_dataset.md: -------------------------------------------------------------------------------- 1 | We used talking portrait videos from [AD-NeRF](https://github.com/YudongGuo/AD-NeRF), [GeneFace](https://github.com/yerfor/GeneFace) and [HDTF dataset](https://github.com/MRzzm/HDTF). 2 | These are static videos whose average length are about 3~5 minutes. 3 | 4 | You can see an example video with the below line: 5 | 6 | ``` 7 | wget https://github.com/yerfor/GeneFace/releases/download/v1.1.0/May.zip 8 | ``` 9 | 10 | We also used [SynObama](https://grail.cs.washington.edu/projects/AudioToObama/) for cross-driven setting inference. -------------------------------------------------------------------------------- /GFPGAN/gfpgan/archs/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import arch modules for registry 6 | # scan all the files that end with '_arch.py' under the archs folder 7 | arch_folder = osp.dirname(osp.abspath(__file__)) 8 | arch_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(arch_folder) if v.endswith('_arch.py')] 9 | # import all the arch modules 10 | _arch_modules = [importlib.import_module(f'gfpgan.archs.{file_name}') for file_name in arch_filenames] 11 | -------------------------------------------------------------------------------- /GFPGAN/gfpgan/models/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import model modules for registry 6 | # scan all the files that end with '_model.py' under the model folder 7 | model_folder = osp.dirname(osp.abspath(__file__)) 8 | model_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(model_folder) if v.endswith('_model.py')] 9 | # import all the model modules 10 | _model_modules = [importlib.import_module(f'gfpgan.models.{file_name}') for file_name in model_filenames] 11 | -------------------------------------------------------------------------------- /GFPGAN/gfpgan/data/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import dataset modules for registry 6 | # scan all the files that end with '_dataset.py' under the data folder 7 | data_folder = osp.dirname(osp.abspath(__file__)) 8 | dataset_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(data_folder) if v.endswith('_dataset.py')] 9 | # import all the dataset modules 10 | _dataset_modules = [importlib.import_module(f'gfpgan.data.{file_name}') for file_name in dataset_filenames] 11 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/archs/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import arch modules for registry 6 | # scan all the files that end with '_arch.py' under the archs folder 7 | arch_folder = osp.dirname(osp.abspath(__file__)) 8 | arch_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(arch_folder) if v.endswith('_arch.py')] 9 | # import all the arch modules 10 | _arch_modules = [importlib.import_module(f'gfpgan.archs.{file_name}') for file_name in arch_filenames] 11 | -------------------------------------------------------------------------------- /GFPGAN/cog.yaml: -------------------------------------------------------------------------------- 1 | # This file is used for constructing replicate env 2 | image: "r8.im/tencentarc/gfpgan" 3 | 4 | build: 5 | gpu: true 6 | python_version: "3.8" 7 | system_packages: 8 | - "libgl1-mesa-glx" 9 | - "libglib2.0-0" 10 | python_packages: 11 | - "torch==1.7.1" 12 | - "torchvision==0.8.2" 13 | - "numpy==1.21.1" 14 | - "lmdb==1.2.1" 15 | - "opencv-python==4.5.3.56" 16 | - "PyYAML==5.4.1" 17 | - "tqdm==4.62.2" 18 | - "yapf==0.31.0" 19 | - "basicsr==1.4.2" 20 | - "facexlib==0.2.5" 21 | 22 | predict: "cog_predict.py:Predictor" 23 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/models/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import model modules for registry 6 | # scan all the files that end with '_model.py' under the model folder 7 | model_folder = osp.dirname(osp.abspath(__file__)) 8 | model_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(model_folder) if v.endswith('_model.py')] 9 | # import all the model modules 10 | _model_modules = [importlib.import_module(f'gfpgan.models.{file_name}') for file_name in model_filenames] 11 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/scripts/make_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -rf docs 4 | # reStructuredText in python files to rst. Documentation in docs folder 5 | sphinx-apidoc -A "Alyssa Quek" -f -F -o docs facemorpher/ 6 | 7 | cd docs 8 | 9 | # Append module path to end of conf file 10 | echo "" >> conf.py 11 | echo "import os" >> conf.py 12 | echo "import sys" >> conf.py 13 | echo "sys.path.insert(0, os.path.abspath('../'))" >> conf.py 14 | echo "sys.path.insert(0, os.path.abspath('../facemorpher'))" >> conf.py 15 | 16 | # Make sphinx documentation 17 | make html 18 | cd .. 19 | -------------------------------------------------------------------------------- /GFPGAN/tests/data/test_ffhq_degradation_dataset.yml: -------------------------------------------------------------------------------- 1 | name: UnitTest 2 | type: FFHQDegradationDataset 3 | dataroot_gt: tests/data/gt 4 | io_backend: 5 | type: disk 6 | 7 | use_hflip: true 8 | mean: [0.5, 0.5, 0.5] 9 | std: [0.5, 0.5, 0.5] 10 | out_size: 512 11 | 12 | blur_kernel_size: 41 13 | kernel_list: ['iso', 'aniso'] 14 | kernel_prob: [0.5, 0.5] 15 | blur_sigma: [0.1, 10] 16 | downsample_range: [0.8, 8] 17 | noise_range: [0, 20] 18 | jpeg_range: [60, 100] 19 | 20 | # color jitter and gray 21 | color_jitter_prob: 1 22 | color_jitter_shift: 20 23 | color_jitter_pt_prob: 1 24 | gray_prob: 1 25 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GFPGAN/gfpgan/data/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import dataset modules for registry 6 | # scan all the files that end with '_dataset.py' under the data folder 7 | data_folder = osp.dirname(osp.abspath(__file__)) 8 | dataset_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(data_folder) if v.endswith('_dataset.py')] 9 | # import all the dataset modules 10 | _dataset_modules = [importlib.import_module(f'gfpgan.data.{file_name}') for file_name in dataset_filenames] 11 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/face_tracking/data_loader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import numpy as np 4 | 5 | 6 | def load_dir(path, start, end): 7 | lmss = [] 8 | imgs_paths = [] 9 | for i in range(start, end): 10 | if os.path.isfile(os.path.join(path, str(i) + ".lms")): 11 | lms = np.loadtxt(os.path.join(path, str(i) + ".lms"), dtype=np.float32) 12 | lmss.append(lms) 13 | imgs_paths.append(os.path.join(path, str(i) + ".jpg")) 14 | lmss = np.stack(lmss) 15 | lmss = torch.as_tensor(lmss).cuda() 16 | return lmss, imgs_paths 17 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/deepspeech_features/README.md: -------------------------------------------------------------------------------- 1 | # Routines for DeepSpeech features processing 2 | Several routines for [DeepSpeech](https://github.com/mozilla/DeepSpeech) features processing, like speech features generation for [VOCA](https://github.com/TimoBolkart/voca) model. 3 | 4 | ## Installation 5 | 6 | ``` 7 | pip3 install -r requirements.txt 8 | ``` 9 | 10 | ## Usage 11 | 12 | Generate wav files: 13 | ``` 14 | python3 extract_wav.py --in-video= 15 | ``` 16 | 17 | Generate files with DeepSpeech features: 18 | ``` 19 | python3 extract_ds_features.py --input= 20 | ``` 21 | -------------------------------------------------------------------------------- /training/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /eg3d/eg3d/viz/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /torch_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /eg3d/eg3d/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /eg3d/eg3d/training/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /torch_utils/ops/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /eg3d/eg3d/gui_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /training/volumetric_rendering/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty -------------------------------------------------------------------------------- /Wav2Lip/evaluation/test_filelists/ReSyncED/tts_pairs.txt: -------------------------------------------------------------------------------- 1 | adam_1.mp4 andreng_optimization.wav 2 | agad_2.mp4 agad_2.wav 3 | agad_1.mp4 agad_1.wav 4 | agad_3.mp4 agad_3.wav 5 | rms_prop_1.mp4 rms_prop_tts.wav 6 | tf_1.mp4 tf_1.wav 7 | tf_2.mp4 tf_2.wav 8 | andrew_ng_ai_business.mp4 andrewng_business_tts.wav 9 | covid_autopsy_1.mp4 autopsy_tts.wav 10 | news_1.mp4 news_tts.wav 11 | andrew_ng_fund_1.mp4 andrewng_ai_fund.wav 12 | covid_treatments_1.mp4 covid_tts.wav 13 | pytorch_v_tf.mp4 pytorch_vs_tf_eng.wav 14 | pytorch_1.mp4 pytorch.wav 15 | pkb_1.mp4 pkb_1.wav 16 | ss_1.mp4 ss_1.wav 17 | carlsen_1.mp4 carlsen_eng.wav 18 | french.mp4 french.wav -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty 12 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/BFM/std_exp.txt: -------------------------------------------------------------------------------- 1 | 453980 257264 263068 211890 135873 184721 47055.6 72732 62787.4 106226 56708.5 51439.8 34887.1 44378.7 51813.4 31030.7 23354.9 23128.1 19400 21827.6 22767.7 22057.4 19894.3 16172.8 17142.7 10035.3 14727.5 12972.5 10763.8 8953.93 8682.62 8941.81 6342.3 5205.3 7065.65 6083.35 6678.88 4666.63 5082.89 5134.76 4908.16 3964.93 3739.95 3180.09 2470.45 1866.62 1624.71 2423.74 1668.53 1471.65 1194.52 782.102 815.044 835.782 834.937 744.496 575.146 633.76 705.685 753.409 620.306 673.326 766.189 619.866 559.93 357.264 396.472 556.849 455.048 460.592 400.735 326.702 279.428 291.535 326.584 305.664 287.816 283.642 276.19 -------------------------------------------------------------------------------- /eg3d/eg3d/training/volumetric_rendering/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | # empty -------------------------------------------------------------------------------- /preprocess_utils/do_au_detection.sh: -------------------------------------------------------------------------------- 1 | INPUT_VID=$1 2 | IDENTITY=$2 3 | ROOT_DIR=$3 4 | 5 | # If you don't have docker image, download with the below script 6 | # docker pull benbuleong/openface-cambridge 7 | 8 | mkdir $ROOT_DIR/$IDENTITY/data_openface 9 | cp $INPUT_VID $ROOT_DIR/$IDENTITY/data_openface 10 | 11 | docker run --rm --ipc=host -v $ROOT_DIR/$IDENTITY/data_openface/:/opt/OpenFace/build/bin/facecam_exp benbuleong/openface-cambridge ./FeatureExtraction -f facecam_exp/$IDENTITY.mp4 -outroot facecam_exp/ -of au.csv -q 12 | 13 | mv $ROOT_DIR/$IDENTITY/data_openface/au.csv $ROOT_DIR/$IDENTITY/ 14 | rm -rf $ROOT_DIR/$IDENTITY/data_openface -------------------------------------------------------------------------------- /GFPGAN/experiments/pretrained_models/README.md: -------------------------------------------------------------------------------- 1 | # Pre-trained Models and Other Data 2 | 3 | Download pre-trained models and other data. Put them in this folder. 4 | 5 | 1. [Pretrained StyleGAN2 model: StyleGAN2_512_Cmul1_FFHQ_B12G4_scratch_800k.pth](https://github.com/TencentARC/GFPGAN/releases/download/v0.1.0/StyleGAN2_512_Cmul1_FFHQ_B12G4_scratch_800k.pth) 6 | 1. [Component locations of FFHQ: FFHQ_eye_mouth_landmarks_512.pth](https://github.com/TencentARC/GFPGAN/releases/download/v0.1.0/FFHQ_eye_mouth_landmarks_512.pth) 7 | 1. [A simple ArcFace model: arcface_resnet18.pth](https://github.com/TencentARC/GFPGAN/releases/download/v0.1.0/arcface_resnet18.pth) 8 | -------------------------------------------------------------------------------- /dnnlib/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | from .util import EasyDict, make_cache_dir_path 12 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/scripts/publish_ghpages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # delete previous gh-pages 4 | git branch -D gh-pages 5 | git push origin :gh-pages 6 | 7 | git checkout -b gh-pages 8 | git rebase master 9 | git reset HEAD 10 | 11 | # make docs 12 | ./scripts/make_docs.sh 13 | 14 | # Add docs 15 | mv docs/_build/html/*.html . 16 | git add *.html 17 | mv docs/_build/html/*.js . 18 | git add *.js 19 | mv docs/_build/html/_static/ _static 20 | git add _static 21 | 22 | touch .nojekyll 23 | git add .nojekyll 24 | 25 | # Publish to gh-pages 26 | git commit -m "docs" 27 | git push origin gh-pages 28 | 29 | git checkout master 30 | -------------------------------------------------------------------------------- /eg3d/eg3d/dnnlib/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | from .util import EasyDict, make_cache_dir_path 12 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/3millions.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # configs for test speed 4 | 5 | config = edict() 6 | config.loss = "arcface" 7 | config.network = "r50" 8 | config.resume = False 9 | config.output = None 10 | config.embedding_size = 512 11 | config.sample_rate = 1.0 12 | config.fp16 = True 13 | config.momentum = 0.9 14 | config.weight_decay = 5e-4 15 | config.batch_size = 128 16 | config.lr = 0.1 # batch size is 512 17 | 18 | config.rec = "synthetic" 19 | config.num_classes = 300 * 10000 20 | config.num_epoch = 30 21 | config.warmup_epoch = -1 22 | config.decay_epoch = [10, 16, 22] 23 | config.val_targets = [] 24 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/speed.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # configs for test speed 4 | 5 | config = edict() 6 | config.loss = "arcface" 7 | config.network = "r50" 8 | config.resume = False 9 | config.output = None 10 | config.embedding_size = 512 11 | config.sample_rate = 1.0 12 | config.fp16 = True 13 | config.momentum = 0.9 14 | config.weight_decay = 5e-4 15 | config.batch_size = 128 16 | config.lr = 0.1 # batch size is 512 17 | 18 | config.rec = "synthetic" 19 | config.num_classes = 100 * 10000 20 | config.num_epoch = 30 21 | config.warmup_epoch = -1 22 | config.decay_epoch = [10, 16, 22] 23 | config.val_targets = [] 24 | -------------------------------------------------------------------------------- /GFPGAN/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.trimTrailingWhitespace": true, 3 | "editor.wordWrap": "on", 4 | "editor.rulers": [ 5 | 80, 6 | 120 7 | ], 8 | "editor.renderWhitespace": "all", 9 | "editor.renderControlCharacters": true, 10 | "python.formatting.provider": "yapf", 11 | "python.formatting.yapfArgs": [ 12 | "--style", 13 | "{BASED_ON_STYLE = pep8, BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true, SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN = true, COLUMN_LIMIT = 120}" 14 | ], 15 | "python.linting.flake8Enabled": true, 16 | "python.linting.flake8Args": [ 17 | "max-line-length=120" 18 | ], 19 | } 20 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/3millions_pfc.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # configs for test speed 4 | 5 | config = edict() 6 | config.loss = "arcface" 7 | config.network = "r50" 8 | config.resume = False 9 | config.output = None 10 | config.embedding_size = 512 11 | config.sample_rate = 0.1 12 | config.fp16 = True 13 | config.momentum = 0.9 14 | config.weight_decay = 5e-4 15 | config.batch_size = 128 16 | config.lr = 0.1 # batch size is 512 17 | 18 | config.rec = "synthetic" 19 | config.num_classes = 300 * 10000 20 | config.num_epoch = 30 21 | config.warmup_epoch = -1 22 | config.decay_epoch = [10, 16, 22] 23 | config.val_targets = [] 24 | -------------------------------------------------------------------------------- /eval/calc_FID.py: -------------------------------------------------------------------------------- 1 | import os, shutil 2 | import sys 3 | from cleanfid import fid 4 | import argparse 5 | 6 | # Function to calculate the FID score 7 | # Disable 8 | def blockPrint(): 9 | sys.stdout = open(os.devnull, 'w') 10 | 11 | # Restore 12 | def enablePrint(): 13 | sys.stdout = sys.__stdout__ 14 | 15 | 16 | def calculate_fid_score(video_path, data_root, video_id): 17 | # Load video 18 | os.makedirs("FID/temp", exist_ok=True) 19 | cmd = f"ffmpeg -i {video_path} FID/temp/video1_%06d.png" 20 | os.system(cmd) 21 | 22 | score = fid.compute_fid("FID/temp", f"{data_root}/{video_id}/image") 23 | cmd = f"rm -r FID/temp" 24 | os.system(cmd) 25 | return score -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/utils/utils_config.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os.path as osp 3 | 4 | 5 | def get_config(config_file): 6 | assert config_file.startswith('configs/'), 'config file setting must start with configs/' 7 | temp_config_name = osp.basename(config_file) 8 | temp_module_name = osp.splitext(temp_config_name)[0] 9 | config = importlib.import_module("configs.base") 10 | cfg = config.config 11 | config = importlib.import_module("configs.%s" % temp_module_name) 12 | job_cfg = config.config 13 | cfg.update(job_cfg) 14 | if cfg.output is None: 15 | cfg.output = osp.join('work_dirs', temp_module_name) 16 | return cfg -------------------------------------------------------------------------------- /preprocess_utils/data_utils/face_parsing/logger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | 5 | import os.path as osp 6 | import time 7 | import sys 8 | import logging 9 | 10 | import torch.distributed as dist 11 | 12 | 13 | def setup_logger(logpth): 14 | logfile = 'BiSeNet-{}.log'.format(time.strftime('%Y-%m-%d-%H-%M-%S')) 15 | logfile = osp.join(logpth, logfile) 16 | FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s' 17 | log_level = logging.INFO 18 | if dist.is_initialized() and not dist.get_rank()==0: 19 | log_level = logging.ERROR 20 | logging.basicConfig(level=log_level, format=FORMAT, filename=logfile) 21 | logging.root.addHandler(logging.StreamHandler()) 22 | 23 | 24 | -------------------------------------------------------------------------------- /eval/syncnet_python/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | *.pyc 10 | 11 | # Packages # 12 | ############ 13 | # it's better to unpack these files and commit the raw source 14 | # git has its own built in compression methods 15 | *.7z 16 | *.dmg 17 | *.gz 18 | *.iso 19 | *.jar 20 | *.rar 21 | *.tar 22 | *.zip 23 | 24 | # Logs and databases # 25 | ###################### 26 | *.log 27 | *.sql 28 | *.sqlite 29 | 30 | # OS generated files # 31 | ###################### 32 | .DS_Store 33 | .DS_Store? 34 | ._* 35 | .Spotlight-V100 36 | .Trashes 37 | ehthumbs.db 38 | Thumbs.db 39 | 40 | # Specific to this demo # 41 | ######################### 42 | # data/ 43 | protos/ 44 | utils/ 45 | *.pth 46 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/glint360k_mbf.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "mbf" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 0.1 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 2e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/glint360k_r100.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r100" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/glint360k_r18.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r18" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/glint360k_r34.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r34" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/glint360k_r50.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "cosface" 9 | config.network = "r50" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/glint360k" 21 | config.num_classes = 360232 22 | config.num_image = 17091657 23 | config.num_epoch = 20 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [8, 12, 15, 18] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/ms1mv3_mbf.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "mbf" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 2e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 30 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 20, 25] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/ms1mv3_r18.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r18" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/ms1mv3_r34.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r34" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/ms1mv3_r50.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r50" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 128 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/ms1mv3_r2060.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r2060" 10 | config.resume = False 11 | config.output = None 12 | config.embedding_size = 512 13 | config.sample_rate = 1.0 14 | config.fp16 = True 15 | config.momentum = 0.9 16 | config.weight_decay = 5e-4 17 | config.batch_size = 64 18 | config.lr = 0.1 # batch size is 512 19 | 20 | config.rec = "/train_tmp/ms1m-retinaface-t1" 21 | config.num_classes = 93431 22 | config.num_image = 5179510 23 | config.num_epoch = 25 24 | config.warmup_epoch = -1 25 | config.decay_epoch = [10, 16, 22] 26 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 27 | -------------------------------------------------------------------------------- /GFPGAN/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | # line break before binary operator (W503) 4 | W503, 5 | # line break after binary operator (W504) 6 | W504, 7 | max-line-length=120 8 | 9 | [yapf] 10 | based_on_style = pep8 11 | column_limit = 120 12 | blank_line_before_nested_class_or_def = true 13 | split_before_expression_after_opening_paren = true 14 | 15 | [isort] 16 | line_length = 120 17 | multi_line_output = 0 18 | known_standard_library = pkg_resources,setuptools 19 | known_first_party = gfpgan 20 | known_third_party = basicsr,cv2,facexlib,numpy,pytest,torch,torchvision,tqdm,yaml 21 | no_lines_before = STDLIB,LOCALFOLDER 22 | default_section = THIRDPARTY 23 | 24 | [codespell] 25 | skip = .git,./docs/build 26 | count = 27 | quiet-level = 3 28 | 29 | [aliases] 30 | test=pytest 31 | 32 | [tool:pytest] 33 | addopts=tests/ 34 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/docs/eval.md: -------------------------------------------------------------------------------- 1 | ## Eval on ICCV2021-MFR 2 | 3 | coming soon. 4 | 5 | 6 | ## Eval IJBC 7 | You can eval ijbc with pytorch or onnx. 8 | 9 | 10 | 1. Eval IJBC With Onnx 11 | ```shell 12 | CUDA_VISIBLE_DEVICES=0 python onnx_ijbc.py --model-root ms1mv3_arcface_r50 --image-path IJB_release/IJBC --result-dir ms1mv3_arcface_r50 13 | ``` 14 | 15 | 2. Eval IJBC With Pytorch 16 | ```shell 17 | CUDA_VISIBLE_DEVICES=0,1 python eval_ijbc.py \ 18 | --model-prefix ms1mv3_arcface_r50/backbone.pth \ 19 | --image-path IJB_release/IJBC \ 20 | --result-dir ms1mv3_arcface_r50 \ 21 | --batch-size 128 \ 22 | --job ms1mv3_arcface_r50 \ 23 | --target IJBC \ 24 | --network iresnet50 25 | ``` 26 | 27 | ## Inference 28 | 29 | ```shell 30 | python inference.py --weight ms1mv3_arcface_r50/backbone.pth --network r50 31 | ``` 32 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/ckpts/run_script.sh: -------------------------------------------------------------------------------- 1 | [ test][220419_183311]python test.py --data_root . --mode test 2 | [ test][220419_183356]python test.py --data_root . --mode test 3 | [ test][220419_183456]python test.py --data_root . --mode test 4 | [ test][220419_183528]python test.py --data_root . --mode test 5 | [ test][220419_183711]python test.py --data_root . --mode test 6 | [ test][220419_183837]python test.py --data_root . --mode test 7 | [ test][220419_184333]python test.py --data_root . --mode test 8 | [ test][220419_184442]python test.py --data_root . --mode test 9 | [ test][220419_184500]python test.py --data_root . --mode test 10 | [ test][220419_184533]python test.py --data_root . --mode test 11 | [ test][220419_184603]python test.py --data_root . --mode test 12 | [ test][220419_184714]python test.py --data_root . --mode test 13 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/facemorpher/videoer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Create a video with image frames 3 | """ 4 | 5 | import cv2 6 | import numpy as np 7 | 8 | 9 | def check_write_video(func): 10 | def inner(self, *args, **kwargs): 11 | if self.video: 12 | return func(self, *args, **kwargs) 13 | else: 14 | pass 15 | return inner 16 | 17 | 18 | class Video(object): 19 | def __init__(self, filename, fps, w, h): 20 | self.filename = filename 21 | 22 | if filename is None: 23 | self.video = None 24 | else: 25 | fourcc = cv2.VideoWriter_fourcc(*'MJPG') 26 | self.video = cv2.VideoWriter(filename, fourcc, fps, (w, h), True) 27 | 28 | @check_write_video 29 | def write(self, img, num_times=1): 30 | for i in range(num_times): 31 | self.video.write(img[..., :3]) 32 | 33 | @check_write_video 34 | def end(self): 35 | print(self.filename + ' saved') 36 | self.video.release() 37 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_morpher/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | # To test locally: python setup.py sdist bdist_wheel 4 | # To upload to pypi: twine upload dist/* 5 | 6 | setup( 7 | name='facemorpher', 8 | version='5.2.dev0', 9 | author='Alyssa Quek', 10 | author_email='alyssaquek@gmail.com', 11 | description=('Warp, morph and average human faces!'), 12 | keywords='face morphing, averaging, warping', 13 | url='https://github.com/alyssaq/face_morpher', 14 | license='MIT', 15 | packages=find_packages(), 16 | install_requires=[ 17 | 'docopt', 18 | 'numpy', 19 | 'scipy', 20 | 'matplotlib', 21 | 'dlib' 22 | ], 23 | entry_points={'console_scripts': [ 24 | 'facemorpher=facemorpher.morpher:main', 25 | 'faceaverager=facemorpher.averager:main' 26 | ] 27 | }, 28 | data_files=[('readme', ['README.rst'])], 29 | long_description=open('README.rst').read(), 30 | ) 31 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_model/op/fused_bias_act.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | torch::Tensor fused_bias_act_op(const torch::Tensor& input, const torch::Tensor& bias, const torch::Tensor& refer, 5 | int act, int grad, float alpha, float scale); 6 | 7 | #define CHECK_CUDA(x) TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") 8 | #define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") 9 | #define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) 10 | 11 | torch::Tensor fused_bias_act(const torch::Tensor& input, const torch::Tensor& bias, const torch::Tensor& refer, 12 | int act, int grad, float alpha, float scale) { 13 | CHECK_CUDA(input); 14 | CHECK_CUDA(bias); 15 | 16 | return fused_bias_act_op(input, bias, refer, act, grad, alpha, scale); 17 | } 18 | 19 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 20 | m.def("fused_bias_act", &fused_bias_act, "fused bias act (CUDA)"); 21 | } -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/options/test_options.py: -------------------------------------------------------------------------------- 1 | """This script contains the test options for Deep3DFaceRecon_pytorch 2 | """ 3 | 4 | from .base_options import BaseOptions 5 | 6 | 7 | class TestOptions(BaseOptions): 8 | """This class includes test options. 9 | 10 | It also includes shared options defined in BaseOptions. 11 | """ 12 | 13 | def initialize(self, parser): 14 | parser = BaseOptions.initialize(self, parser) # define shared options 15 | parser.add_argument('--phase', type=str, default='test', help='train, val, test, etc') 16 | parser.add_argument('--dataset_mode', type=str, default=None, help='chooses how datasets are loaded. [None | flist]') 17 | parser.add_argument('--img_folder', type=str, default='examples', help='folder for test images.') 18 | 19 | # Dropout and Batchnorm has different behavior during training and test. 20 | self.isTrain = False 21 | return parser 22 | -------------------------------------------------------------------------------- /eval/metrics/mtcnn/mtcnn_pytorch/src/visualization_utils.py: -------------------------------------------------------------------------------- 1 | from PIL import ImageDraw 2 | 3 | 4 | def show_bboxes(img, bounding_boxes, facial_landmarks=[]): 5 | """Draw bounding boxes and facial landmarks. 6 | 7 | Arguments: 8 | img: an instance of PIL.Image. 9 | bounding_boxes: a float numpy array of shape [n, 5]. 10 | facial_landmarks: a float numpy array of shape [n, 10]. 11 | 12 | Returns: 13 | an instance of PIL.Image. 14 | """ 15 | 16 | img_copy = img.copy() 17 | draw = ImageDraw.Draw(img_copy) 18 | 19 | for b in bounding_boxes: 20 | draw.rectangle([ 21 | (b[0], b[1]), (b[2], b[3]) 22 | ], outline='white') 23 | 24 | for p in facial_landmarks: 25 | for i in range(5): 26 | draw.ellipse([ 27 | (p[i] - 1.0, p[i + 5] - 1.0), 28 | (p[i] + 1.0, p[i + 5] + 1.0) 29 | ], outline='blue') 30 | 31 | return img_copy 32 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/backbones/__init__.py: -------------------------------------------------------------------------------- 1 | from .iresnet import iresnet18, iresnet34, iresnet50, iresnet100, iresnet200 2 | from .mobilefacenet import get_mbf 3 | 4 | 5 | def get_model(name, **kwargs): 6 | # resnet 7 | if name == "r18": 8 | return iresnet18(False, **kwargs) 9 | elif name == "r34": 10 | return iresnet34(False, **kwargs) 11 | elif name == "r50": 12 | return iresnet50(False, **kwargs) 13 | elif name == "r100": 14 | return iresnet100(False, **kwargs) 15 | elif name == "r200": 16 | return iresnet200(False, **kwargs) 17 | elif name == "r2060": 18 | from .iresnet2060 import iresnet2060 19 | return iresnet2060(False, **kwargs) 20 | elif name == "mbf": 21 | fp16 = kwargs.get("fp16", False) 22 | num_features = kwargs.get("num_features", 512) 23 | return get_mbf(fp16=fp16, num_features=num_features) 24 | else: 25 | raise ValueError() -------------------------------------------------------------------------------- /Wav2Lip/evaluation/test_filelists/README.md: -------------------------------------------------------------------------------- 1 | This folder contains the filelists for the new evaluation framework proposed in the paper. 2 | 3 | ## Test filelists for LRS2, LRS3, and LRW. 4 | 5 | This folder contains three filelists, each containing a list of names of audio-video pairs from the test sets of LRS2, LRS3, and LRW. The LRS2 and LRW filelists are strictly "Copyright BBC" and can only be used for “non-commercial research by applicants who have an agreement with the BBC to access the Lip Reading in the Wild and/or Lip Reading Sentences in the Wild datasets”. Please follow this link for more details: [https://www.bbc.co.uk/rd/projects/lip-reading-datasets](https://www.bbc.co.uk/rd/projects/lip-reading-datasets). 6 | 7 | 8 | ## ReSynCED benchmark 9 | 10 | The sub-folder `ReSynCED` contains filelists for our own Real-world lip-Sync Evaluation Dataset (ReSyncED). 11 | 12 | 13 | #### Instructions on how to use the above two filelists are available in the README of the parent folder. 14 | -------------------------------------------------------------------------------- /vive3D/landmark_detector.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class LandmarkDetector: 4 | def __init__(self, device='cuda'): 5 | import face_alignment 6 | try: 7 | self.detector = face_alignment.FaceAlignment(face_alignment.LandmarksType.TWO_D, device=f'cuda:{device}') #_2D라고 되어있었음 TWO_D 8 | except: 9 | self.detector = face_alignment.FaceAlignment(face_alignment.LandmarksType.TWO_D, device=str(device)) #_2D라고 되어있었음 TWO_D 10 | 11 | def get_landmarks(self, np_image, get_all=True): 12 | # estimate all 68 landmarks 13 | landmarks_68 = np.array(self.detector.get_landmarks_from_image(np_image)) 14 | if not landmarks_68.all(): 15 | return [] 16 | 17 | if get_all: 18 | return landmarks_68[0, :, :] 19 | else: 20 | keypoint_indices=[30, 8, 45, 36, 64, 60] #nose, chin, l_eye, r_eye, l_mouth, r_mouth 21 | # pick a subset of desired landmarks 22 | return landmarks_68[0, keypoint_indices, :] -------------------------------------------------------------------------------- /preprocess_utils/do_audio_and_image_preprocessing.sh: -------------------------------------------------------------------------------- 1 | # set video directory 2 | INPUT_VID=$1 # Put each mp4 file separately in a folder 3 | IDENTITY=$2 4 | EG3D_PATH=models/ffhq-fixed-triplane512-128.pkl 5 | ROOT_DIR=$3 6 | 7 | 8 | 9 | # process audio (from mp4) 10 | CUDA_VISIBLE_DEVICES=2 python ./preprocess_utils/data_utils/process.py $INPUT_VID $ROOT_DIR/$IDENTITY 11 | 12 | # If you want to process any other audio dataset (OOD) 13 | # WAVS=data/syncobama_B.wav 14 | # python ./preprocess_utils/data_utils/process.py $WAVS --task 2 15 | # python ./preprocess_utils/data_utils/process.py $WAVS2 --task 2 16 | 17 | # crop images and segmentation 18 | CUDA_VISIBLE_DEVICES=2 python ./preprocess_utils/vive3d_cropping.py \ 19 | --source_video $INPUT_VID \ 20 | --generator_path $EG3D_PATH \ 21 | --savepoint_path $ROOT_DIR \ 22 | --device 0 23 | 24 | # detect wav2lip bbox 25 | CUDA_VISIBLE_DEVICES=2 python ./preprocess_utils/detect_wav2lip_bbox.py \ 26 | --root_dir $ROOT_DIR \ 27 | --id $IDENTITY \ 28 | --source_images image 29 | 30 | 31 | -------------------------------------------------------------------------------- /eval/syncnet_python/demo_syncnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess 5 | 6 | from SyncNetInstance import * 7 | 8 | # ==================== LOAD PARAMS ==================== 9 | 10 | 11 | parser = argparse.ArgumentParser(description = "SyncNet"); 12 | 13 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); 14 | parser.add_argument('--batch_size', type=int, default='20', help=''); 15 | parser.add_argument('--vshift', type=int, default='15', help=''); 16 | parser.add_argument('--videofile', type=str, default="data/example.avi", help=''); 17 | parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help=''); 18 | parser.add_argument('--reference', type=str, default="demo", help=''); 19 | 20 | opt = parser.parse_args(); 21 | 22 | 23 | # ==================== RUN EVALUATION ==================== 24 | 25 | s = SyncNetInstance(); 26 | 27 | s.loadParameters(opt.initial_model); 28 | print("Model %s loaded."%opt.initial_model); 29 | 30 | s.evaluate(opt, videofile=opt.videofile) 31 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_model/op/upfirdn2d.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | torch::Tensor upfirdn2d_op(const torch::Tensor& input, const torch::Tensor& kernel, 5 | int up_x, int up_y, int down_x, int down_y, 6 | int pad_x0, int pad_x1, int pad_y0, int pad_y1); 7 | 8 | #define CHECK_CUDA(x) TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") 9 | #define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") 10 | #define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) 11 | 12 | torch::Tensor upfirdn2d(const torch::Tensor& input, const torch::Tensor& kernel, 13 | int up_x, int up_y, int down_x, int down_y, 14 | int pad_x0, int pad_x1, int pad_y0, int pad_y1) { 15 | CHECK_CUDA(input); 16 | CHECK_CUDA(kernel); 17 | 18 | return upfirdn2d_op(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1); 19 | } 20 | 21 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 22 | m.def("upfirdn2d", &upfirdn2d, "upfirdn2d (CUDA)"); 23 | } -------------------------------------------------------------------------------- /eval/syncnet_python/demo_feature.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess 5 | 6 | from SyncNetInstance import * 7 | 8 | # ==================== LOAD PARAMS ==================== 9 | 10 | 11 | parser = argparse.ArgumentParser(description = "SyncNet"); 12 | 13 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); 14 | parser.add_argument('--batch_size', type=int, default='20', help=''); 15 | parser.add_argument('--vshift', type=int, default='15', help=''); 16 | parser.add_argument('--videofile', type=str, default="data/example.avi", help=''); 17 | parser.add_argument('--tmp_dir', type=str, default="data", help=''); 18 | parser.add_argument('--save_as', type=str, default="data/features.pt", help=''); 19 | 20 | opt = parser.parse_args(); 21 | 22 | 23 | # ==================== RUN EVALUATION ==================== 24 | 25 | s = SyncNetInstance(); 26 | 27 | s.loadParameters(opt.initial_model); 28 | print("Model %s loaded."%opt.initial_model); 29 | 30 | feats = s.extract_feature(opt, videofile=opt.videofile) 31 | 32 | torch.save(feats, opt.save_as) 33 | -------------------------------------------------------------------------------- /eg3d/eg3d/environment.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | name: eg3d 12 | channels: 13 | - pytorch 14 | - nvidia 15 | dependencies: 16 | - python >= 3.8 17 | - pip 18 | - numpy>=1.20 19 | - click>=8.0 20 | - pillow=8.3.1 21 | - scipy=1.7.1 22 | - pytorch=1.11.0 23 | - cudatoolkit=11.1 24 | - requests=2.26.0 25 | - tqdm=4.62.2 26 | - ninja=1.10.2 27 | - matplotlib=3.4.2 28 | - imageio=2.9.0 29 | - pip: 30 | - imgui==1.3.0 31 | - glfw==2.2.0 32 | - pyopengl==3.1.5 33 | - imageio-ffmpeg==0.4.3 34 | - pyspng 35 | - psutil 36 | - mrcfile 37 | - tensorboard -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/data/config.py: -------------------------------------------------------------------------------- 1 | # config.py 2 | 3 | cfg_mnet = { 4 | 'name': 'mobilenet0.25', 5 | 'min_sizes': [[16, 32], [64, 128], [256, 512]], 6 | 'steps': [8, 16, 32], 7 | 'variance': [0.1, 0.2], 8 | 'clip': False, 9 | 'loc_weight': 2.0, 10 | 'gpu_train': True, 11 | 'batch_size': 32, 12 | 'ngpu': 1, 13 | 'epoch': 250, 14 | 'decay1': 190, 15 | 'decay2': 220, 16 | 'image_size': 640, 17 | 'pretrain': False, 18 | 'return_layers': {'stage1': 1, 'stage2': 2, 'stage3': 3}, 19 | 'in_channel': 32, 20 | 'out_channel': 64 21 | } 22 | 23 | cfg_re50 = { 24 | 'name': 'Resnet50', 25 | 'min_sizes': [[16, 32], [64, 128], [256, 512]], 26 | 'steps': [8, 16, 32], 27 | 'variance': [0.1, 0.2], 28 | 'clip': False, 29 | 'loc_weight': 2.0, 30 | 'gpu_train': True, 31 | 'batch_size': 24, 32 | 'ngpu': 4, 33 | 'epoch': 100, 34 | 'decay1': 70, 35 | 'decay2': 90, 36 | 'image_size': 840, 37 | 'pretrain': False, 38 | 'return_layers': {'layer2': 1, 'layer3': 2, 'layer4': 3}, 39 | 'in_channel': 256, 40 | 'out_channel': 256 41 | } 42 | 43 | -------------------------------------------------------------------------------- /eval/syncnet_python/LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-present Joon Son Chung. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Yuedong Chen (Donald) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /preprocess_utils/do_vive3d_training.sh: -------------------------------------------------------------------------------- 1 | INPUT_VID=$1 # Put each mp4 file separately in a folder 2 | IDENTITY=$2 3 | EG3D_PATH=models/ffhq-fixed-triplane512-128.pkl 4 | ROOT_DIR=$3 5 | SAVEPOINT=trial 6 | 7 | # train vive3d 8 | # It is recommended to choose images with different facial expressions as much as possible. 9 | # RTX 3090 can hold only 3 frames, but recommended to use 5 frames 10 | CUDA_VISIBLE_DEVICES=2 python personalize_generator.py --source_video $INPUT_VID \ 11 | --generator_path $EG3D_PATH \ 12 | --start_sec 0 \ 13 | --end_sec 10 \ 14 | --frame 0 \ 15 | --frame 85 \ 16 | --frame 110 \ 17 | --device 'cuda:0' \ 18 | --directory_name $SAVEPOINT 19 | 20 | # inference vive3d 21 | # This process takes time. 22 | CUDA_VISIBLE_DEVICES=2 python invert_video.py --source_video $INPUT_VID --savepoint_path ./savepoints/${IDENTITY}_${SAVEPOINT} \ 23 | --source_video $INPUT_VID 24 | 25 | 26 | \cp -f savepoints/${IDENTITY}_${SAVEPOINT}/G_tune.pkl $ROOT_DIR/$IDENTITY 27 | \cp -f savepoints/${IDENTITY}_${SAVEPOINT}/inversion_0-0_angles.pt $ROOT_DIR/$IDENTITY/inversion_0-0_angles.pt 28 | \cp -f savepoints/${IDENTITY}_${SAVEPOINT}/inversion_w_person.pt $ROOT_DIR/$IDENTITY 29 | 30 | # rm -rf savepoints 31 | # rm -rf video -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/inference.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import cv2 4 | import numpy as np 5 | import torch 6 | 7 | from backbones import get_model 8 | 9 | 10 | @torch.no_grad() 11 | def inference(weight, name, img): 12 | if img is None: 13 | img = np.random.randint(0, 255, size=(112, 112, 3), dtype=np.uint8) 14 | else: 15 | img = cv2.imread(img) 16 | img = cv2.resize(img, (112, 112)) 17 | 18 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 19 | img = np.transpose(img, (2, 0, 1)) 20 | img = torch.from_numpy(img).unsqueeze(0).float() 21 | img.div_(255).sub_(0.5).div_(0.5) 22 | net = get_model(name, fp16=False) 23 | net.load_state_dict(torch.load(weight)) 24 | net.eval() 25 | feat = net(img).numpy() 26 | print(feat) 27 | 28 | 29 | if __name__ == "__main__": 30 | parser = argparse.ArgumentParser(description='PyTorch ArcFace Training') 31 | parser.add_argument('--network', type=str, default='r50', help='backbone network') 32 | parser.add_argument('--weight', type=str, default='') 33 | parser.add_argument('--img', type=str, default=None) 34 | args = parser.parse_args() 35 | inference(args.weight, args.network, args.img) 36 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/utils/nms/py_cpu_nms.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------- 2 | # Fast R-CNN 3 | # Copyright (c) 2015 Microsoft 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Ross Girshick 6 | # -------------------------------------------------------- 7 | 8 | import numpy as np 9 | 10 | def py_cpu_nms(dets, thresh): 11 | """Pure Python NMS baseline.""" 12 | x1 = dets[:, 0] 13 | y1 = dets[:, 1] 14 | x2 = dets[:, 2] 15 | y2 = dets[:, 3] 16 | scores = dets[:, 4] 17 | 18 | areas = (x2 - x1 + 1) * (y2 - y1 + 1) 19 | order = scores.argsort()[::-1] 20 | 21 | keep = [] 22 | while order.size > 0: 23 | i = order[0] 24 | keep.append(i) 25 | xx1 = np.maximum(x1[i], x1[order[1:]]) 26 | yy1 = np.maximum(y1[i], y1[order[1:]]) 27 | xx2 = np.minimum(x2[i], x2[order[1:]]) 28 | yy2 = np.minimum(y2[i], y2[order[1:]]) 29 | 30 | w = np.maximum(0.0, xx2 - xx1 + 1) 31 | h = np.maximum(0.0, yy2 - yy1 + 1) 32 | inter = w * h 33 | ovr = inter / (areas[i] + areas[order[1:]] - inter) 34 | 35 | inds = np.where(ovr <= thresh)[0] 36 | order = order[inds + 1] 37 | 38 | return keep 39 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/options/inference_options.py: -------------------------------------------------------------------------------- 1 | from face3d.options.base_options import BaseOptions 2 | 3 | 4 | class InferenceOptions(BaseOptions): 5 | """This class includes test options. 6 | 7 | It also includes shared options defined in BaseOptions. 8 | """ 9 | 10 | def initialize(self, parser): 11 | parser = BaseOptions.initialize(self, parser) # define shared options 12 | parser.add_argument('--phase', type=str, default='test', help='train, val, test, etc') 13 | parser.add_argument('--dataset_mode', type=str, default=None, help='chooses how datasets are loaded. [None | flist]') 14 | 15 | parser.add_argument('--input_dir', type=str, help='the folder of the input files') 16 | parser.add_argument('--keypoint_dir', type=str, help='the folder of the keypoint files') 17 | parser.add_argument('--output_dir', type=str, default='mp4', help='the output dir to save the extracted coefficients') 18 | parser.add_argument('--save_split_files', action='store_true', help='save split files or not') 19 | parser.add_argument('--inference_batch_size', type=int, default=8) 20 | 21 | # Dropout and Batchnorm has different behavior during training and test. 22 | self.isTrain = False 23 | return parser 24 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/utils/timer.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------- 2 | # Fast R-CNN 3 | # Copyright (c) 2015 Microsoft 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Ross Girshick 6 | # -------------------------------------------------------- 7 | 8 | import time 9 | 10 | 11 | class Timer(object): 12 | """A simple timer.""" 13 | def __init__(self): 14 | self.total_time = 0. 15 | self.calls = 0 16 | self.start_time = 0. 17 | self.diff = 0. 18 | self.average_time = 0. 19 | 20 | def tic(self): 21 | # using time.time instead of time.clock because time time.clock 22 | # does not normalize for multithreading 23 | self.start_time = time.time() 24 | 25 | def toc(self, average=True): 26 | self.diff = time.time() - self.start_time 27 | self.total_time += self.diff 28 | self.calls += 1 29 | self.average_time = self.total_time / self.calls 30 | if average: 31 | return self.average_time 32 | else: 33 | return self.diff 34 | 35 | def clear(self): 36 | self.total_time = 0. 37 | self.calls = 0 38 | self.start_time = 0. 39 | self.diff = 0. 40 | self.average_time = 0. 41 | -------------------------------------------------------------------------------- /training/crosssection_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import torch 12 | 13 | def sample_cross_section(G, ws, resolution=256, w=1.2): 14 | axis=0 15 | A, B = torch.meshgrid(torch.linspace(w/2, -w/2, resolution, device=ws.device), torch.linspace(-w/2, w/2, resolution, device=ws.device), indexing='ij') 16 | A, B = A.reshape(-1, 1), B.reshape(-1, 1) 17 | C = torch.zeros_like(A) 18 | coordinates = [A, B] 19 | coordinates.insert(axis, C) 20 | coordinates = torch.cat(coordinates, dim=-1).expand(ws.shape[0], -1, -1) 21 | 22 | sigma = G.sample_mixed(coordinates, torch.randn_like(coordinates), ws)['sigma'] 23 | return sigma.reshape(-1, 1, resolution, resolution) 24 | 25 | # if __name__ == '__main__': 26 | # sample_crossection(None) -------------------------------------------------------------------------------- /eg3d/eg3d/training/crosssection_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import torch 12 | 13 | def sample_cross_section(G, ws, resolution=256, w=1.2): 14 | axis=0 15 | A, B = torch.meshgrid(torch.linspace(w/2, -w/2, resolution, device=ws.device), torch.linspace(-w/2, w/2, resolution, device=ws.device), indexing='ij') 16 | A, B = A.reshape(-1, 1), B.reshape(-1, 1) 17 | C = torch.zeros_like(A) 18 | coordinates = [A, B] 19 | coordinates.insert(axis, C) 20 | coordinates = torch.cat(coordinates, dim=-1).expand(ws.shape[0], -1, -1) 21 | 22 | sigma = G.sample_mixed(coordinates, torch.randn_like(coordinates), ws)['sigma'] 23 | return sigma.reshape(-1, 1, resolution, resolution) 24 | 25 | # if __name__ == '__main__': 26 | # sample_crossection(None) -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/utils/utils_logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | 6 | class AverageMeter(object): 7 | """Computes and stores the average and current value 8 | """ 9 | 10 | def __init__(self): 11 | self.val = None 12 | self.avg = None 13 | self.sum = None 14 | self.count = None 15 | self.reset() 16 | 17 | def reset(self): 18 | self.val = 0 19 | self.avg = 0 20 | self.sum = 0 21 | self.count = 0 22 | 23 | def update(self, val, n=1): 24 | self.val = val 25 | self.sum += val * n 26 | self.count += n 27 | self.avg = self.sum / self.count 28 | 29 | 30 | def init_logging(rank, models_root): 31 | if rank == 0: 32 | log_root = logging.getLogger() 33 | log_root.setLevel(logging.INFO) 34 | formatter = logging.Formatter("Training: %(asctime)s-%(message)s") 35 | handler_file = logging.FileHandler(os.path.join(models_root, "training.log")) 36 | handler_stream = logging.StreamHandler(sys.stdout) 37 | handler_file.setFormatter(formatter) 38 | handler_stream.setFormatter(formatter) 39 | log_root.addHandler(handler_file) 40 | log_root.addHandler(handler_stream) 41 | log_root.info('rank_id: %d' % rank) 42 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | 5 | def get_loss(name): 6 | if name == "cosface": 7 | return CosFace() 8 | elif name == "arcface": 9 | return ArcFace() 10 | else: 11 | raise ValueError() 12 | 13 | 14 | class CosFace(nn.Module): 15 | def __init__(self, s=64.0, m=0.40): 16 | super(CosFace, self).__init__() 17 | self.s = s 18 | self.m = m 19 | 20 | def forward(self, cosine, label): 21 | index = torch.where(label != -1)[0] 22 | m_hot = torch.zeros(index.size()[0], cosine.size()[1], device=cosine.device) 23 | m_hot.scatter_(1, label[index, None], self.m) 24 | cosine[index] -= m_hot 25 | ret = cosine * self.s 26 | return ret 27 | 28 | 29 | class ArcFace(nn.Module): 30 | def __init__(self, s=64.0, m=0.5): 31 | super(ArcFace, self).__init__() 32 | self.s = s 33 | self.m = m 34 | 35 | def forward(self, cosine: torch.Tensor, label): 36 | index = torch.where(label != -1)[0] 37 | m_hot = torch.zeros(index.size()[0], cosine.size()[1], device=cosine.device) 38 | m_hot.scatter_(1, label[index, None], self.m) 39 | cosine.acos_() 40 | cosine[index] += m_hot 41 | cosine.cos_().mul_(self.s) 42 | return cosine 43 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/face_tracking/convert_BFM.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.io import loadmat 3 | 4 | original_BFM = loadmat("3DMM/01_MorphableModel.mat") 5 | sub_inds = np.load("3DMM/topology_info.npy", allow_pickle=True).item()["sub_inds"] 6 | 7 | shapePC = original_BFM["shapePC"] 8 | shapeEV = original_BFM["shapeEV"] 9 | shapeMU = original_BFM["shapeMU"] 10 | texPC = original_BFM["texPC"] 11 | texEV = original_BFM["texEV"] 12 | texMU = original_BFM["texMU"] 13 | 14 | b_shape = shapePC.reshape(-1, 199).transpose(1, 0).reshape(199, -1, 3) 15 | mu_shape = shapeMU.reshape(-1, 3) 16 | 17 | b_tex = texPC.reshape(-1, 199).transpose(1, 0).reshape(199, -1, 3) 18 | mu_tex = texMU.reshape(-1, 3) 19 | 20 | b_shape = b_shape[:, sub_inds, :].reshape(199, -1) 21 | mu_shape = mu_shape[sub_inds, :].reshape(-1) 22 | b_tex = b_tex[:, sub_inds, :].reshape(199, -1) 23 | mu_tex = mu_tex[sub_inds, :].reshape(-1) 24 | 25 | exp_info = np.load("3DMM/exp_info.npy", allow_pickle=True).item() 26 | np.save( 27 | "3DMM/3DMM_info.npy", 28 | { 29 | "mu_shape": mu_shape, 30 | "b_shape": b_shape, 31 | "sig_shape": shapeEV.reshape(-1), 32 | "mu_exp": exp_info["mu_exp"], 33 | "b_exp": exp_info["base_exp"], 34 | "sig_exp": exp_info["sig_exp"], 35 | "mu_tex": mu_tex, 36 | "b_tex": b_tex, 37 | "sig_tex": texEV.reshape(-1), 38 | }, 39 | ) 40 | -------------------------------------------------------------------------------- /docs/preprocessing.md: -------------------------------------------------------------------------------- 1 | To preprocess video dataset, run the bash file below: 2 | 3 | ``` 4 | sh do_preprocessing.sh {path/to/video.mp4} {videoname} {path/to/save/directory} 5 | ``` 6 | 7 | For instance, you can run like below: 8 | 9 | ``` 10 | sh do_preprocessing.sh ./videos/May.mp4 May ./data 11 | ``` 12 | 13 | This preprocessing takes time to find camera params by inversion. 14 | 15 | *Warning* Some videos may not be continuous and could be composed of fragmented segments. In such cases, the VIVE3D inversion process may not accurately predict the correct camera params. We recommend to verify the data before use. 16 | 17 | Also, the inverted frames are saved at `./video` directory. We recommend checking them to ensure that the inversion process was successful 18 | 19 | You can process additional wav files by the following code: 20 | 21 | ``` 22 | python ./preprocess_utils/data_utils/process.py /path/to/audio.wav --task 2 23 | ``` 24 | 25 | ## Quick Start 26 | 27 | We provide a part of demo dataset in this [Links](https://works.do/GMb3yGM). You can skip the `preprocess_utils/do_vive3d_training.sh` in `do_preprocessing.sh`. 28 | Place these files in data directory `dataroot/name`. 29 | 30 | We also provide demo checkpoint in this [Links](https://works.do/FMb9GSa), which is trained on May.mp4. Download this and add this checkpoint directory to `--checkpoint_dir` in the script. 31 | 32 | You can use `--only_do_inference` flag for quick inference. 33 | Please see `demo.sh` for the example. 34 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_detect/layers/functions/prior_box.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from itertools import product as product 3 | import numpy as np 4 | from math import ceil 5 | 6 | 7 | class PriorBox(object): 8 | def __init__(self, cfg, image_size=None, phase='train'): 9 | super(PriorBox, self).__init__() 10 | self.min_sizes = cfg['min_sizes'] 11 | self.steps = cfg['steps'] 12 | self.clip = cfg['clip'] 13 | self.image_size = image_size 14 | self.feature_maps = [[ceil(self.image_size[0]/step), ceil(self.image_size[1]/step)] for step in self.steps] 15 | self.name = "s" 16 | 17 | def forward(self): 18 | anchors = [] 19 | for k, f in enumerate(self.feature_maps): 20 | min_sizes = self.min_sizes[k] 21 | for i, j in product(range(f[0]), range(f[1])): 22 | for min_size in min_sizes: 23 | s_kx = min_size / self.image_size[1] 24 | s_ky = min_size / self.image_size[0] 25 | dense_cx = [x * self.steps[k] / self.image_size[1] for x in [j + 0.5]] 26 | dense_cy = [y * self.steps[k] / self.image_size[0] for y in [i + 0.5]] 27 | for cy, cx in product(dense_cy, dense_cx): 28 | anchors += [cx, cy, s_kx, s_ky] 29 | 30 | # back to torch land 31 | output = torch.Tensor(anchors).view(-1, 4) 32 | if self.clip: 33 | output.clamp_(max=1, min=0) 34 | return output 35 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/util/generate_list.py: -------------------------------------------------------------------------------- 1 | """This script is to generate training list files for Deep3DFaceRecon_pytorch 2 | """ 3 | 4 | import os 5 | 6 | # save path to training data 7 | def write_list(lms_list, imgs_list, msks_list, mode='train',save_folder='datalist', save_name=''): 8 | save_path = os.path.join(save_folder, mode) 9 | if not os.path.isdir(save_path): 10 | os.makedirs(save_path) 11 | with open(os.path.join(save_path, save_name + 'landmarks.txt'), 'w') as fd: 12 | fd.writelines([i + '\n' for i in lms_list]) 13 | 14 | with open(os.path.join(save_path, save_name + 'images.txt'), 'w') as fd: 15 | fd.writelines([i + '\n' for i in imgs_list]) 16 | 17 | with open(os.path.join(save_path, save_name + 'masks.txt'), 'w') as fd: 18 | fd.writelines([i + '\n' for i in msks_list]) 19 | 20 | # check if the path is valid 21 | def check_list(rlms_list, rimgs_list, rmsks_list): 22 | lms_list, imgs_list, msks_list = [], [], [] 23 | for i in range(len(rlms_list)): 24 | flag = 'false' 25 | lm_path = rlms_list[i] 26 | im_path = rimgs_list[i] 27 | msk_path = rmsks_list[i] 28 | if os.path.isfile(lm_path) and os.path.isfile(im_path) and os.path.isfile(msk_path): 29 | flag = 'true' 30 | lms_list.append(rlms_list[i]) 31 | imgs_list.append(rimgs_list[i]) 32 | msks_list.append(rmsks_list[i]) 33 | print(i, rlms_list[i], flag) 34 | return lms_list, imgs_list, msks_list 35 | -------------------------------------------------------------------------------- /preprocess_utils/detect_wav2lip_bbox.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import matplotlib.pyplot as plt 3 | from tqdm import tqdm 4 | from glob import glob 5 | import click 6 | import torchvision 7 | import sys 8 | sys.path.append('./Wav2Lip') 9 | import face_detection 10 | import os 11 | import numpy as np 12 | import cv2 13 | 14 | @click.command() 15 | @click.option('-r', '--root_dir', type=str, help='Path to source video', required=True) 16 | @click.option('--id', type=str, help='Path to source video', required=True) 17 | @click.option('-i', '--source_images', type=str, help='Path to source video', required=True) 18 | @click.option('-o', '--savepoint_path', type=str, help='Savepoint directory', default='wav2lip_bbox') 19 | 20 | def main(**config): 21 | _main(**config) 22 | 23 | def _main(root_dir, 24 | id, 25 | source_images, 26 | savepoint_path): 27 | 28 | images_dir = sorted(glob(os.path.join(root_dir, id, source_images, '*.png'))) 29 | save_root = os.path.join(root_dir, id, savepoint_path) 30 | os.makedirs(save_root, exist_ok=True) 31 | fa = face_detection.FaceAlignment(face_detection.LandmarksType._2D, flip_input=False) 32 | 33 | for image_dir in tqdm(images_dir, desc='preprocessing wav2lip bounding box...'): 34 | name = image_dir.split('/')[-1].split('.')[0] 35 | save_dir = os.path.join(save_root, f'{name}.npy') 36 | image = cv2.imread(image_dir) 37 | results = fa.get_detections_for_image(image) 38 | np.save(save_dir, results) 39 | 40 | 41 | if __name__ == '__main__': 42 | main() -------------------------------------------------------------------------------- /eg3d/eg3d/viz/backbone_cache_widget.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import imgui 12 | from gui_utils import imgui_utils 13 | 14 | #---------------------------------------------------------------------------- 15 | 16 | class BackboneCacheWidget: 17 | def __init__(self, viz): 18 | self.viz = viz 19 | self.cache_backbone = True 20 | 21 | @imgui_utils.scoped_by_object_id 22 | def __call__(self, show=True): 23 | viz = self.viz 24 | 25 | if show: 26 | imgui.text('Cache Backbone') 27 | imgui.same_line(viz.label_w + viz.spacing * 4) 28 | _clicked, self.cache_backbone = imgui.checkbox('##backbonecache', self.cache_backbone) 29 | imgui.same_line(viz.label_w + viz.spacing * 10) 30 | imgui.text('Note that when enabled, you may be unable to view intermediate backbone weights below') 31 | 32 | viz.args.do_backbone_caching = self.cache_backbone 33 | 34 | #---------------------------------------------------------------------------- 35 | -------------------------------------------------------------------------------- /eg3d/eg3d/viz/render_type_widget.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import imgui 12 | from gui_utils import imgui_utils 13 | 14 | #---------------------------------------------------------------------------- 15 | 16 | class RenderTypeWidget: 17 | def __init__(self, viz): 18 | self.viz = viz 19 | self.render_type = 0 20 | self.render_types = ['image', 'image_depth', 'image_raw'] 21 | self.labels = ['RGB Image', 'Depth Image', 'Neural Rendered Image'] 22 | 23 | @imgui_utils.scoped_by_object_id 24 | def __call__(self, show=True): 25 | viz = self.viz 26 | 27 | if show: 28 | imgui.text('Render Type') 29 | imgui.same_line(viz.label_w) 30 | with imgui_utils.item_width(viz.font_size * 10): 31 | _clicked, self.render_type = imgui.combo('', self.render_type, self.labels) 32 | 33 | viz.args.render_type = self.render_types[self.render_type] 34 | 35 | #---------------------------------------------------------------------------- 36 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/data/data_loader.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | from PIL import Image 4 | import random 5 | import numpy as np 6 | import pickle 7 | import torchvision.transforms as transforms 8 | 9 | from .celeba import CelebADataset 10 | 11 | 12 | def create_dataloader(opt): 13 | data_loader = DataLoader() 14 | data_loader.initialize(opt) 15 | return data_loader 16 | 17 | 18 | class DataLoader: 19 | def name(self): 20 | return self.dataset.name() + "_Loader" 21 | 22 | def create_datase(self): 23 | # specify which dataset to load here 24 | loaded_dataset = os.path.basename(self.opt.data_root.strip('/')).lower() 25 | if 'celeba' in loaded_dataset or 'emotion' in loaded_dataset: 26 | dataset = CelebADataset() 27 | else: 28 | dataset = BaseDataset() 29 | dataset.initialize(self.opt) 30 | return dataset 31 | 32 | def initialize(self, opt): 33 | self.opt = opt 34 | self.dataset = self.create_datase() 35 | self.dataloader = torch.utils.data.DataLoader( 36 | self.dataset, 37 | batch_size=opt.batch_size, 38 | shuffle=not opt.serial_batches, 39 | num_workers=int(opt.n_threads) 40 | ) 41 | 42 | def __len__(self): 43 | return min(len(self.dataset), self.opt.max_dataset_size) 44 | 45 | def __iter__(self): 46 | for i, data in enumerate(self.dataloader): 47 | if i * self.opt.batch_size >= self.opt.max_dataset_size: 48 | break 49 | yield data 50 | -------------------------------------------------------------------------------- /torch_utils/ops/bias_act.h: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | //------------------------------------------------------------------------ 14 | // CUDA kernel parameters. 15 | 16 | struct bias_act_kernel_params 17 | { 18 | const void* x; // [sizeX] 19 | const void* b; // [sizeB] or NULL 20 | const void* xref; // [sizeX] or NULL 21 | const void* yref; // [sizeX] or NULL 22 | const void* dy; // [sizeX] or NULL 23 | void* y; // [sizeX] 24 | 25 | int grad; 26 | int act; 27 | float alpha; 28 | float gain; 29 | float clamp; 30 | 31 | int sizeX; 32 | int sizeB; 33 | int stepB; 34 | int loopX; 35 | }; 36 | 37 | //------------------------------------------------------------------------ 38 | // CUDA kernel selection. 39 | 40 | template void* choose_bias_act_kernel(const bias_act_kernel_params& p); 41 | 42 | //------------------------------------------------------------------------ 43 | -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/bias_act.h: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | //------------------------------------------------------------------------ 14 | // CUDA kernel parameters. 15 | 16 | struct bias_act_kernel_params 17 | { 18 | const void* x; // [sizeX] 19 | const void* b; // [sizeB] or NULL 20 | const void* xref; // [sizeX] or NULL 21 | const void* yref; // [sizeX] or NULL 22 | const void* dy; // [sizeX] or NULL 23 | void* y; // [sizeX] 24 | 25 | int grad; 26 | int act; 27 | float alpha; 28 | float gain; 29 | float clamp; 30 | 31 | int sizeX; 32 | int sizeB; 33 | int stepB; 34 | int loopX; 35 | }; 36 | 37 | //------------------------------------------------------------------------ 38 | // CUDA kernel selection. 39 | 40 | template void* choose_bias_act_kernel(const bias_act_kernel_params& p); 41 | 42 | //------------------------------------------------------------------------ 43 | -------------------------------------------------------------------------------- /GFPGAN/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | # flake8 3 | - repo: https://github.com/PyCQA/flake8 4 | rev: 3.8.3 5 | hooks: 6 | - id: flake8 7 | args: ["--config=setup.cfg", "--ignore=W504, W503"] 8 | 9 | # modify known_third_party 10 | - repo: https://github.com/asottile/seed-isort-config 11 | rev: v2.2.0 12 | hooks: 13 | - id: seed-isort-config 14 | 15 | # isort 16 | - repo: https://github.com/timothycrosley/isort 17 | rev: 5.2.2 18 | hooks: 19 | - id: isort 20 | 21 | # yapf 22 | - repo: https://github.com/pre-commit/mirrors-yapf 23 | rev: v0.30.0 24 | hooks: 25 | - id: yapf 26 | 27 | # codespell 28 | - repo: https://github.com/codespell-project/codespell 29 | rev: v2.1.0 30 | hooks: 31 | - id: codespell 32 | 33 | # pre-commit-hooks 34 | - repo: https://github.com/pre-commit/pre-commit-hooks 35 | rev: v3.2.0 36 | hooks: 37 | - id: trailing-whitespace # Trim trailing whitespace 38 | - id: check-yaml # Attempt to load all yaml files to verify syntax 39 | - id: check-merge-conflict # Check for files that contain merge conflict strings 40 | - id: double-quote-string-fixer # Replace double quoted strings with single quoted strings 41 | - id: end-of-file-fixer # Make sure files end in a newline and only a newline 42 | - id: requirements-txt-fixer # Sort entries in requirements.txt and remove incorrect entry for pkg-resources==0.0.0 43 | - id: fix-encoding-pragma # Remove the coding pragma: # -*- coding: utf-8 -*- 44 | args: ["--remove"] 45 | - id: mixed-line-ending # Replace or check mixed line ending 46 | args: ["--fix=lf"] 47 | -------------------------------------------------------------------------------- /GFPGAN/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from facexlib.utils.face_restoration_helper import FaceRestoreHelper 3 | 4 | from gfpgan.archs.gfpganv1_arch import GFPGANv1 5 | from gfpgan.archs.gfpganv1_clean_arch import GFPGANv1Clean 6 | from gfpgan.utils import GFPGANer 7 | 8 | 9 | def test_gfpganer(): 10 | # initialize with the clean model 11 | restorer = GFPGANer( 12 | model_path='experiments/pretrained_models/GFPGANCleanv1-NoCE-C2.pth', 13 | upscale=2, 14 | arch='clean', 15 | channel_multiplier=2, 16 | bg_upsampler=None) 17 | # test attribute 18 | assert isinstance(restorer.gfpgan, GFPGANv1Clean) 19 | assert isinstance(restorer.face_helper, FaceRestoreHelper) 20 | 21 | # initialize with the original model 22 | restorer = GFPGANer( 23 | model_path='experiments/pretrained_models/GFPGANv1.pth', 24 | upscale=2, 25 | arch='original', 26 | channel_multiplier=1, 27 | bg_upsampler=None) 28 | # test attribute 29 | assert isinstance(restorer.gfpgan, GFPGANv1) 30 | assert isinstance(restorer.face_helper, FaceRestoreHelper) 31 | 32 | # ------------------ test enhance ---------------- # 33 | img = cv2.imread('tests/data/gt/00000000.png', cv2.IMREAD_COLOR) 34 | result = restorer.enhance(img, has_aligned=False, paste_back=True) 35 | assert result[0][0].shape == (512, 512, 3) 36 | assert result[1][0].shape == (512, 512, 3) 37 | assert result[2].shape == (1024, 1024, 3) 38 | 39 | # with has_aligned=True 40 | result = restorer.enhance(img, has_aligned=True, paste_back=False) 41 | assert result[0][0].shape == (512, 512, 3) 42 | assert result[1][0].shape == (512, 512, 3) 43 | assert result[2] is None 44 | -------------------------------------------------------------------------------- /eval/syncnet_python/run_syncnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess, pickle, os, gzip, glob 5 | 6 | from SyncNetInstance import * 7 | 8 | # ==================== PARSE ARGUMENT ==================== 9 | 10 | parser = argparse.ArgumentParser(description = "SyncNet"); 11 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); 12 | parser.add_argument('--batch_size', type=int, default='20', help=''); 13 | parser.add_argument('--vshift', type=int, default='15', help=''); 14 | parser.add_argument('--data_dir', type=str, default='data/work', help=''); 15 | parser.add_argument('--videofile', type=str, default='', help=''); 16 | parser.add_argument('--reference', type=str, default='', help=''); 17 | opt = parser.parse_args(); 18 | 19 | setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi')) 20 | setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp')) 21 | setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork')) 22 | setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop')) 23 | 24 | 25 | # ==================== LOAD MODEL AND FILE LIST ==================== 26 | 27 | s = SyncNetInstance(); 28 | 29 | s.loadParameters(opt.initial_model); 30 | print("Model %s loaded."%opt.initial_model); 31 | 32 | flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi')) 33 | flist.sort() 34 | 35 | # ==================== GET OFFSETS ==================== 36 | 37 | dists = [] 38 | for idx, fname in enumerate(flist): 39 | offset, conf, dist = s.evaluate(opt,videofile=fname) 40 | dists.append(dist) 41 | 42 | # ==================== PRINT RESULTS TO FILE ==================== 43 | 44 | with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: 45 | pickle.dump(dists, fil) 46 | -------------------------------------------------------------------------------- /Wav2Lip/models/conv.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn import functional as F 4 | 5 | class Conv2d(nn.Module): 6 | def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): 7 | super().__init__(*args, **kwargs) 8 | self.conv_block = nn.Sequential( 9 | nn.Conv2d(cin, cout, kernel_size, stride, padding), 10 | nn.SyncBatchNorm(cout) 11 | ) 12 | self.act = nn.ReLU() 13 | self.residual = residual 14 | 15 | def forward(self, x): 16 | out = self.conv_block(x) 17 | if self.residual: 18 | out += x 19 | return self.act(out) 20 | 21 | class nonorm_Conv2d(nn.Module): 22 | def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): 23 | super().__init__(*args, **kwargs) 24 | self.conv_block = nn.Sequential( 25 | nn.Conv2d(cin, cout, kernel_size, stride, padding), 26 | ) 27 | self.act = nn.LeakyReLU(0.01, inplace=True) 28 | 29 | def forward(self, x): 30 | out = self.conv_block(x) 31 | return self.act(out) 32 | 33 | class Conv2dTranspose(nn.Module): 34 | def __init__(self, cin, cout, kernel_size, stride, padding, output_padding=0, *args, **kwargs): 35 | super().__init__(*args, **kwargs) 36 | self.conv_block = nn.Sequential( 37 | nn.ConvTranspose2d(cin, cout, kernel_size, stride, padding, output_padding), 38 | nn.BatchNorm2d(cout) 39 | ) 40 | self.act = nn.ReLU() 41 | 42 | def forward(self, x): 43 | out = self.conv_block(x) 44 | return self.act(out) 45 | -------------------------------------------------------------------------------- /eval/syncnet_python/calculate_scores_real_videos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess, pickle, os, gzip, glob 5 | 6 | from SyncNetInstance_calc_scores import * 7 | 8 | # ==================== PARSE ARGUMENT ==================== 9 | 10 | parser = argparse.ArgumentParser(description = "SyncNet"); 11 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); 12 | parser.add_argument('--batch_size', type=int, default='20', help=''); 13 | parser.add_argument('--vshift', type=int, default='15', help=''); 14 | parser.add_argument('--data_dir', type=str, default='data/work', help=''); 15 | parser.add_argument('--videofile', type=str, default='', help=''); 16 | parser.add_argument('--reference', type=str, default='', help=''); 17 | opt = parser.parse_args(); 18 | 19 | setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi')) 20 | setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp')) 21 | setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork')) 22 | setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop')) 23 | 24 | 25 | # ==================== LOAD MODEL AND FILE LIST ==================== 26 | 27 | s = SyncNetInstance(); 28 | 29 | s.loadParameters(opt.initial_model); 30 | #print("Model %s loaded."%opt.initial_model); 31 | 32 | flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi')) 33 | flist.sort() 34 | 35 | # ==================== GET OFFSETS ==================== 36 | 37 | dists = [] 38 | for idx, fname in enumerate(flist): 39 | offset, conf, dist = s.evaluate(opt,videofile=fname) 40 | print (str(dist)+" "+str(conf)) 41 | 42 | # ==================== PRINT RESULTS TO FILE ==================== 43 | 44 | #with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: 45 | # pickle.dump(dists, fil) 46 | -------------------------------------------------------------------------------- /Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess, pickle, os, gzip, glob 5 | 6 | from SyncNetInstance_calc_scores import * 7 | 8 | # ==================== PARSE ARGUMENT ==================== 9 | 10 | parser = argparse.ArgumentParser(description = "SyncNet"); 11 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); 12 | parser.add_argument('--batch_size', type=int, default='20', help=''); 13 | parser.add_argument('--vshift', type=int, default='15', help=''); 14 | parser.add_argument('--data_dir', type=str, default='data/work', help=''); 15 | parser.add_argument('--videofile', type=str, default='', help=''); 16 | parser.add_argument('--reference', type=str, default='', help=''); 17 | opt = parser.parse_args(); 18 | 19 | setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi')) 20 | setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp')) 21 | setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork')) 22 | setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop')) 23 | 24 | 25 | # ==================== LOAD MODEL AND FILE LIST ==================== 26 | 27 | s = SyncNetInstance(); 28 | 29 | s.loadParameters(opt.initial_model); 30 | #print("Model %s loaded."%opt.initial_model); 31 | 32 | flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi')) 33 | flist.sort() 34 | 35 | # ==================== GET OFFSETS ==================== 36 | 37 | dists = [] 38 | for idx, fname in enumerate(flist): 39 | offset, conf, dist = s.evaluate(opt,videofile=fname) 40 | print (str(dist)+" "+str(conf)) 41 | 42 | # ==================== PRINT RESULTS TO FILE ==================== 43 | 44 | #with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: 45 | # pickle.dump(dists, fil) 46 | -------------------------------------------------------------------------------- /eval/metrics/encoders/w_encoder.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch.nn import Conv2d, BatchNorm2d, PReLU, Sequential, Module 4 | 5 | from metrics.encoders.helpers import get_blocks, bottleneck_IR, bottleneck_IR_SE 6 | from models.stylegan2.model import EqualLinear 7 | 8 | 9 | class WEncoder(Module): 10 | def __init__(self, num_layers, mode='ir', opts=None): 11 | super(WEncoder, self).__init__() 12 | print('Using WEncoder') 13 | assert num_layers in [50, 100, 152], 'num_layers should be 50,100, or 152' 14 | assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' 15 | blocks = get_blocks(num_layers) 16 | if mode == 'ir': 17 | unit_module = bottleneck_IR 18 | elif mode == 'ir_se': 19 | unit_module = bottleneck_IR_SE 20 | self.input_layer = Sequential(Conv2d(3, 64, (3, 3), 1, 1, bias=False), 21 | BatchNorm2d(64), 22 | PReLU(64)) 23 | self.output_pool = torch.nn.AdaptiveAvgPool2d((1, 1)) 24 | self.linear = EqualLinear(512, 512, lr_mul=1) 25 | modules = [] 26 | for block in blocks: 27 | for bottleneck in block: 28 | modules.append(unit_module(bottleneck.in_channel, 29 | bottleneck.depth, 30 | bottleneck.stride)) 31 | self.body = Sequential(*modules) 32 | log_size = int(math.log(opts.output_size, 2)) 33 | self.style_count = 2 * log_size - 2 34 | 35 | def forward(self, x): 36 | x = self.input_layer(x) 37 | x = self.body(x) 38 | x = self.output_pool(x) 39 | x = x.view(-1, 512) 40 | x = self.linear(x) 41 | return x.repeat(self.style_count, 1, 1).permute(1, 0, 2) 42 | 43 | -------------------------------------------------------------------------------- /torch_utils/ops/filtered_lrelu_rd.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include "filtered_lrelu.cu" 14 | 15 | // Template/kernel specializations for sign read mode. 16 | 17 | // Full op, 32-bit indexing. 18 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 19 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 20 | 21 | // Full op, 64-bit indexing. 22 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 23 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 24 | 25 | // Activation/signs only for generic variant. 64-bit indexing. 26 | template void* choose_filtered_lrelu_act_kernel(void); 27 | template void* choose_filtered_lrelu_act_kernel(void); 28 | template void* choose_filtered_lrelu_act_kernel(void); 29 | 30 | // Copy filters to constant memory. 31 | template cudaError_t copy_filters(cudaStream_t stream); 32 | -------------------------------------------------------------------------------- /torch_utils/ops/filtered_lrelu_wr.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include "filtered_lrelu.cu" 14 | 15 | // Template/kernel specializations for sign write mode. 16 | 17 | // Full op, 32-bit indexing. 18 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 19 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 20 | 21 | // Full op, 64-bit indexing. 22 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 23 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 24 | 25 | // Activation/signs only for generic variant. 64-bit indexing. 26 | template void* choose_filtered_lrelu_act_kernel(void); 27 | template void* choose_filtered_lrelu_act_kernel(void); 28 | template void* choose_filtered_lrelu_act_kernel(void); 29 | 30 | // Copy filters to constant memory. 31 | template cudaError_t copy_filters(cudaStream_t stream); 32 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/docs/install.md: -------------------------------------------------------------------------------- 1 | ## v1.8.0 2 | ### Linux and Windows 3 | ```shell 4 | # CUDA 11.0 5 | pip --default-timeout=100 install torch==1.8.0+cu111 torchvision==0.9.0+cu111 torchaudio==0.8.0 -f https://download.pytorch.org/whl/torch_stable.html 6 | 7 | # CUDA 10.2 8 | pip --default-timeout=100 install torch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 9 | 10 | # CPU only 11 | pip --default-timeout=100 install torch==1.8.0+cpu torchvision==0.9.0+cpu torchaudio==0.8.0 -f https://download.pytorch.org/whl/torch_stable.html 12 | 13 | ``` 14 | 15 | 16 | ## v1.7.1 17 | ### Linux and Windows 18 | ```shell 19 | # CUDA 11.0 20 | pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html 21 | 22 | # CUDA 10.2 23 | pip install torch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2 24 | 25 | # CUDA 10.1 26 | pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html 27 | 28 | # CUDA 9.2 29 | pip install torch==1.7.1+cu92 torchvision==0.8.2+cu92 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html 30 | 31 | # CPU only 32 | pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html 33 | ``` 34 | 35 | 36 | ## v1.6.0 37 | 38 | ### Linux and Windows 39 | ```shell 40 | # CUDA 10.2 41 | pip install torch==1.6.0 torchvision==0.7.0 42 | 43 | # CUDA 10.1 44 | pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html 45 | 46 | # CUDA 9.2 47 | pip install torch==1.6.0+cu92 torchvision==0.7.0+cu92 -f https://download.pytorch.org/whl/torch_stable.html 48 | 49 | # CPU only 50 | pip install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 51 | ``` -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/filtered_lrelu_rd.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include "filtered_lrelu.cu" 14 | 15 | // Template/kernel specializations for sign read mode. 16 | 17 | // Full op, 32-bit indexing. 18 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 19 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 20 | 21 | // Full op, 64-bit indexing. 22 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 23 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 24 | 25 | // Activation/signs only for generic variant. 64-bit indexing. 26 | template void* choose_filtered_lrelu_act_kernel(void); 27 | template void* choose_filtered_lrelu_act_kernel(void); 28 | template void* choose_filtered_lrelu_act_kernel(void); 29 | 30 | // Copy filters to constant memory. 31 | template cudaError_t copy_filters(cudaStream_t stream); 32 | -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/filtered_lrelu_wr.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include "filtered_lrelu.cu" 14 | 15 | // Template/kernel specializations for sign write mode. 16 | 17 | // Full op, 32-bit indexing. 18 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 19 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 20 | 21 | // Full op, 64-bit indexing. 22 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 23 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 24 | 25 | // Activation/signs only for generic variant. 64-bit indexing. 26 | template void* choose_filtered_lrelu_act_kernel(void); 27 | template void* choose_filtered_lrelu_act_kernel(void); 28 | template void* choose_filtered_lrelu_act_kernel(void); 29 | 30 | // Copy filters to constant memory. 31 | template cudaError_t copy_filters(cudaStream_t stream); 32 | -------------------------------------------------------------------------------- /torch_utils/ops/filtered_lrelu_ns.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include "filtered_lrelu.cu" 14 | 15 | // Template/kernel specializations for no signs mode (no gradients required). 16 | 17 | // Full op, 32-bit indexing. 18 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 19 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 20 | 21 | // Full op, 64-bit indexing. 22 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 23 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 24 | 25 | // Activation/signs only for generic variant. 64-bit indexing. 26 | template void* choose_filtered_lrelu_act_kernel(void); 27 | template void* choose_filtered_lrelu_act_kernel(void); 28 | template void* choose_filtered_lrelu_act_kernel(void); 29 | 30 | // Copy filters to constant memory. 31 | template cudaError_t copy_filters(cudaStream_t stream); 32 | -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/filtered_lrelu_ns.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include "filtered_lrelu.cu" 14 | 15 | // Template/kernel specializations for no signs mode (no gradients required). 16 | 17 | // Full op, 32-bit indexing. 18 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 19 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 20 | 21 | // Full op, 64-bit indexing. 22 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 23 | template filtered_lrelu_kernel_spec choose_filtered_lrelu_kernel(const filtered_lrelu_kernel_params& p, int sharedKB); 24 | 25 | // Activation/signs only for generic variant. 64-bit indexing. 26 | template void* choose_filtered_lrelu_act_kernel(void); 27 | template void* choose_filtered_lrelu_act_kernel(void); 28 | template void* choose_filtered_lrelu_act_kernel(void); 29 | 30 | // Copy filters to constant memory. 31 | template cudaError_t copy_filters(cudaStream_t stream); 32 | -------------------------------------------------------------------------------- /eval/syncnet_python/README.md: -------------------------------------------------------------------------------- 1 | # SyncNet 2 | 3 | This repository contains the demo for the audio-to-video synchronisation network (SyncNet). This network can be used for audio-visual synchronisation tasks including: 4 | 1. Removing temporal lags between the audio and visual streams in a video; 5 | 2. Determining who is speaking amongst multiple faces in a video. 6 | 7 | Please cite the paper below if you make use of the software. 8 | 9 | ## Dependencies 10 | ``` 11 | pip install -r requirements.txt 12 | ``` 13 | 14 | In addition, `ffmpeg` is required. 15 | 16 | 17 | ## Demo 18 | 19 | SyncNet demo: 20 | ``` 21 | python demo_syncnet.py --videofile data/example.avi --tmp_dir /path/to/temp/directory 22 | ``` 23 | 24 | Check that this script returns: 25 | ``` 26 | AV offset: 3 27 | Min dist: 5.353 28 | Confidence: 10.021 29 | ``` 30 | 31 | Full pipeline: 32 | ``` 33 | sh download_model.sh 34 | python run_pipeline.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output 35 | python run_syncnet.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output 36 | python run_visualise.py --videofile /path/to/video.mp4 --reference name_of_video --data_dir /path/to/output 37 | ``` 38 | 39 | Outputs: 40 | ``` 41 | $DATA_DIR/pycrop/$REFERENCE/*.avi - cropped face tracks 42 | $DATA_DIR/pywork/$REFERENCE/offsets.txt - audio-video offset values 43 | $DATA_DIR/pyavi/$REFERENCE/video_out.avi - output video (as shown below) 44 | ``` 45 |

46 | 47 | 48 |

49 | 50 | ## Publications 51 | 52 | ``` 53 | @InProceedings{Chung16a, 54 | author = "Chung, J.~S. and Zisserman, A.", 55 | title = "Out of time: automated lip sync in the wild", 56 | booktitle = "Workshop on Multi-view Lip-reading, ACCV", 57 | year = "2016", 58 | } 59 | ``` 60 | -------------------------------------------------------------------------------- /Wav2Lip/evaluation/scores_LSE/calculate_scores_LRS.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess 5 | import glob 6 | import os 7 | from tqdm import tqdm 8 | 9 | from SyncNetInstance_calc_scores import * 10 | 11 | # ==================== LOAD PARAMS ==================== 12 | 13 | 14 | parser = argparse.ArgumentParser(description = "SyncNet"); 15 | 16 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); 17 | parser.add_argument('--batch_size', type=int, default='20', help=''); 18 | parser.add_argument('--vshift', type=int, default='15', help=''); 19 | parser.add_argument('--data_root', type=str, required=True, help=''); 20 | parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help=''); 21 | parser.add_argument('--reference', type=str, default="demo", help=''); 22 | 23 | opt = parser.parse_args(); 24 | 25 | 26 | # ==================== RUN EVALUATION ==================== 27 | 28 | s = SyncNetInstance(); 29 | 30 | s.loadParameters(opt.initial_model); 31 | #print("Model %s loaded."%opt.initial_model); 32 | path = os.path.join(opt.data_root, "*.mp4") 33 | 34 | all_videos = glob.glob(path) 35 | 36 | prog_bar = tqdm(range(len(all_videos))) 37 | avg_confidence = 0. 38 | avg_min_distance = 0. 39 | 40 | 41 | for videofile_idx in prog_bar: 42 | videofile = all_videos[videofile_idx] 43 | offset, confidence, min_distance = s.evaluate(opt, videofile=videofile) 44 | avg_confidence += confidence 45 | avg_min_distance += min_distance 46 | prog_bar.set_description('Avg Confidence: {}, Avg Minimum Dist: {}'.format(round(avg_confidence / (videofile_idx + 1), 3), round(avg_min_distance / (videofile_idx + 1), 3))) 47 | prog_bar.refresh() 48 | 49 | print ('Average Confidence: {}'.format(avg_confidence/len(all_videos))) 50 | print ('Average Minimum Distance: {}'.format(avg_min_distance/len(all_videos))) 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/configs/base.py: -------------------------------------------------------------------------------- 1 | from easydict import EasyDict as edict 2 | 3 | # make training faster 4 | # our RAM is 256G 5 | # mount -t tmpfs -o size=140G tmpfs /train_tmp 6 | 7 | config = edict() 8 | config.loss = "arcface" 9 | config.network = "r50" 10 | config.resume = False 11 | config.output = "ms1mv3_arcface_r50" 12 | 13 | config.dataset = "ms1m-retinaface-t1" 14 | config.embedding_size = 512 15 | config.sample_rate = 1 16 | config.fp16 = False 17 | config.momentum = 0.9 18 | config.weight_decay = 5e-4 19 | config.batch_size = 128 20 | config.lr = 0.1 # batch size is 512 21 | 22 | if config.dataset == "emore": 23 | config.rec = "/train_tmp/faces_emore" 24 | config.num_classes = 85742 25 | config.num_image = 5822653 26 | config.num_epoch = 16 27 | config.warmup_epoch = -1 28 | config.decay_epoch = [8, 14, ] 29 | config.val_targets = ["lfw", ] 30 | 31 | elif config.dataset == "ms1m-retinaface-t1": 32 | config.rec = "/train_tmp/ms1m-retinaface-t1" 33 | config.num_classes = 93431 34 | config.num_image = 5179510 35 | config.num_epoch = 25 36 | config.warmup_epoch = -1 37 | config.decay_epoch = [11, 17, 22] 38 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 39 | 40 | elif config.dataset == "glint360k": 41 | config.rec = "/train_tmp/glint360k" 42 | config.num_classes = 360232 43 | config.num_image = 17091657 44 | config.num_epoch = 20 45 | config.warmup_epoch = -1 46 | config.decay_epoch = [8, 12, 15, 18] 47 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 48 | 49 | elif config.dataset == "webface": 50 | config.rec = "/train_tmp/faces_webface_112x112" 51 | config.num_classes = 10572 52 | config.num_image = "forget" 53 | config.num_epoch = 34 54 | config.warmup_epoch = -1 55 | config.decay_epoch = [20, 28, 32] 56 | config.val_targets = ["lfw", "cfp_fp", "agedb_30"] 57 | -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/ffhq/preprocess_in_the_wild.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import os 12 | import argparse 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('--indir', type=str, required=True) 15 | args = parser.parse_args() 16 | 17 | # run mtcnn needed for Deep3DFaceRecon 18 | command = "python batch_mtcnn.py --in_root " + args.indir 19 | print(command) 20 | os.system(command) 21 | 22 | out_folder = args.indir.split("/")[-2] if args.indir.endswith("/") else args.indir.split("/")[-1] 23 | 24 | # run Deep3DFaceRecon 25 | os.chdir('Deep3DFaceRecon_pytorch') 26 | command = "python test.py --img_folder=" + args.indir + " --gpu_ids=0 --name=pretrained --epoch=20" 27 | print(command) 28 | os.system(command) 29 | os.chdir('..') 30 | 31 | # crop out the input image 32 | command = "python crop_images_in_the_wild.py --indir=" + args.indir 33 | print(command) 34 | os.system(command) 35 | 36 | # convert the pose to our format 37 | command = f"python 3dface2idr_mat.py --in_root Deep3DFaceRecon_pytorch/checkpoints/pretrained/results/{out_folder}/epoch_20_000000 --out_path {os.path.join(args.indir, 'crop', 'cameras.json')}" 38 | print(command) 39 | os.system(command) 40 | 41 | # additional correction to match the submission version 42 | command = f"python preprocess_face_cameras.py --source {os.path.join(args.indir, 'crop')} --dest {out_folder} --mode orig" 43 | print(command) 44 | os.system(command) -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/shapenet_cars/run_me.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import os 12 | import gdown 13 | import shutil 14 | import tempfile 15 | import subprocess 16 | 17 | 18 | if __name__ == '__main__': 19 | with tempfile.TemporaryDirectory() as working_dir: 20 | download_name = 'cars_train.zip' 21 | url = 'https://drive.google.com/uc?id=1bThUNtIHx4xEQyffVBSf82ABDDh2HlFn' 22 | output_dataset_name = 'cars_128.zip' 23 | 24 | dir_path = os.path.dirname(os.path.realpath(__file__)) 25 | extracted_data_path = os.path.join(working_dir, os.path.splitext(download_name)[0]) 26 | 27 | print("Downloading data...") 28 | zipped_dataset = os.path.join(working_dir, download_name) 29 | gdown.download(url, zipped_dataset, quiet=False) 30 | 31 | print("Unzipping downloaded data...") 32 | shutil.unpack_archive(zipped_dataset, working_dir) 33 | 34 | print("Converting camera parameters...") 35 | cmd = f"python {os.path.join(dir_path, 'preprocess_shapenet_cameras.py')} --source={extracted_data_path}" 36 | subprocess.run([cmd], shell=True) 37 | 38 | print("Creating dataset zip...") 39 | cmd = f"python {os.path.join(dir_path, '../../eg3d', 'dataset_tool.py')}" 40 | cmd += f" --source {extracted_data_path} --dest {output_dataset_name} --resolution 128x128" 41 | subprocess.run([cmd], shell=True) -------------------------------------------------------------------------------- /eg3d/eg3d/viz/render_depth_sample_widget.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import imgui 12 | from gui_utils import imgui_utils 13 | 14 | #---------------------------------------------------------------------------- 15 | 16 | class RenderDepthSampleWidget: 17 | def __init__(self, viz): 18 | self.viz = viz 19 | self.depth_mult = 2 20 | self.depth_importance_mult = 2 21 | self.render_types = [.5, 1, 2, 4] 22 | self.labels = ['0.5x', '1x', '2x', '4x'] 23 | 24 | @imgui_utils.scoped_by_object_id 25 | def __call__(self, show=True): 26 | viz = self.viz 27 | 28 | if show: 29 | imgui.text('Render Type') 30 | imgui.same_line(viz.label_w) 31 | with imgui_utils.item_width(viz.font_size * 4): 32 | _clicked, self.depth_mult = imgui.combo('Depth Sample Multiplier', self.depth_mult, self.labels) 33 | imgui.same_line(viz.label_w + viz.font_size * 16 + viz.spacing * 2) 34 | with imgui_utils.item_width(viz.font_size * 4): 35 | _clicked, self.depth_importance_mult = imgui.combo('Depth Sample Importance Multiplier', self.depth_importance_mult, self.labels) 36 | 37 | viz.args.depth_mult = self.render_types[self.depth_mult] 38 | viz.args.depth_importance_mult = self.render_types[self.depth_importance_mult] 39 | 40 | #---------------------------------------------------------------------------- 41 | -------------------------------------------------------------------------------- /eg3d/eg3d/viz/zoom_widget.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | from inspect import formatargvalues 12 | import numpy as np 13 | import imgui 14 | import dnnlib 15 | from gui_utils import imgui_utils 16 | 17 | #---------------------------------------------------------------------------- 18 | 19 | class ZoomWidget: 20 | def __init__(self, viz): 21 | self.viz = viz 22 | self.fov = 18.837 23 | self.fov_default = 18.837 24 | 25 | @imgui_utils.scoped_by_object_id 26 | def __call__(self, show=True): 27 | viz = self.viz 28 | if show: 29 | imgui.text('FOV') 30 | imgui.same_line(viz.label_w) 31 | with imgui_utils.item_width(viz.font_size * 10): 32 | _changed, self.fov = imgui.slider_float('##fov', self.fov, 12, 45, format='%.2f Degrees') 33 | 34 | imgui.same_line(viz.label_w + viz.font_size * 13 + viz.button_w + viz.spacing * 3) 35 | snapped = round(self.fov) 36 | if imgui_utils.button('Snap', width=viz.button_w, enabled=(self.fov != snapped)): 37 | self.fov = snapped 38 | imgui.same_line() 39 | if imgui_utils.button('Reset', width=-1, enabled=(abs(self.fov - self.fov_default)) > .01): 40 | self.fov = self.fov_default 41 | 42 | viz.args.focal_length = float(1 / (np.tan(self.fov * 3.14159 / 360) * 1.414)) 43 | #---------------------------------------------------------------------------- 44 | -------------------------------------------------------------------------------- /eval/lmd/utils/flow_util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def convert_flow_to_deformation(flow): 4 | r"""convert flow fields to deformations. 5 | 6 | Args: 7 | flow (tensor): Flow field obtained by the model 8 | Returns: 9 | deformation (tensor): The deformation used for warpping 10 | """ 11 | b,c,h,w = flow.shape 12 | flow_norm = 2 * torch.cat([flow[:,:1,...]/(w-1),flow[:,1:,...]/(h-1)], 1) 13 | grid = make_coordinate_grid(flow) 14 | deformation = grid + flow_norm.permute(0,2,3,1) 15 | return deformation 16 | 17 | def make_coordinate_grid(flow): 18 | r"""obtain coordinate grid with the same size as the flow filed. 19 | 20 | Args: 21 | flow (tensor): Flow field obtained by the model 22 | Returns: 23 | grid (tensor): The grid with the same size as the input flow 24 | """ 25 | b,c,h,w = flow.shape 26 | 27 | x = torch.arange(w).to(flow) 28 | y = torch.arange(h).to(flow) 29 | 30 | x = (2 * (x / (w - 1)) - 1) 31 | y = (2 * (y / (h - 1)) - 1) 32 | 33 | yy = y.view(-1, 1).repeat(1, w) 34 | xx = x.view(1, -1).repeat(h, 1) 35 | 36 | meshed = torch.cat([xx.unsqueeze_(2), yy.unsqueeze_(2)], 2) 37 | meshed = meshed.expand(b, -1, -1, -1) 38 | return meshed 39 | 40 | 41 | def warp_image(source_image, deformation): 42 | r"""warp the input image according to the deformation 43 | 44 | Args: 45 | source_image (tensor): source images to be warpped 46 | deformation (tensor): deformations used to warp the images; value in range (-1, 1) 47 | Returns: 48 | output (tensor): the warpped images 49 | """ 50 | _, h_old, w_old, _ = deformation.shape 51 | _, _, h, w = source_image.shape 52 | if h_old != h or w_old != w: 53 | deformation = deformation.permute(0, 3, 1, 2) 54 | deformation = torch.nn.functional.interpolate(deformation, size=(h, w), mode='bilinear') 55 | deformation = deformation.permute(0, 2, 3, 1) 56 | return torch.nn.functional.grid_sample(source_image, deformation) -------------------------------------------------------------------------------- /Wav2Lip/face_detection/detection/sfd/sfd_detector.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | from torch.utils.model_zoo import load_url 4 | 5 | from ..core import FaceDetector 6 | 7 | from .net_s3fd import s3fd 8 | from .bbox import * 9 | from .detect import * 10 | 11 | models_urls = { 12 | 's3fd': 'https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth', 13 | } 14 | 15 | 16 | class SFDDetector(FaceDetector): 17 | def __init__(self, device, path_to_detector=os.path.join(os.path.dirname(os.path.abspath(__file__)), 's3fd.pth'), verbose=False): 18 | super(SFDDetector, self).__init__(device, verbose) 19 | 20 | # Initialise the face detector 21 | if not os.path.isfile(path_to_detector): 22 | model_weights = load_url(models_urls['s3fd']) 23 | else: 24 | model_weights = torch.load(path_to_detector) 25 | 26 | self.face_detector = s3fd() 27 | self.face_detector.load_state_dict(model_weights) 28 | self.face_detector.to(device) 29 | self.face_detector.eval() 30 | 31 | def detect_from_image(self, tensor_or_path): 32 | image = self.tensor_or_path_to_ndarray(tensor_or_path) 33 | 34 | bboxlist = detect(self.face_detector, image, device=self.device) 35 | keep = nms(bboxlist, 0.3) 36 | bboxlist = bboxlist[keep, :] 37 | bboxlist = [x for x in bboxlist if x[-1] > 0.5] 38 | 39 | return bboxlist 40 | 41 | def detect_from_batch(self, images): 42 | bboxlists = batch_detect(self.face_detector, images, device=self.device) 43 | keeps = [nms(bboxlists[:, i, :], 0.3) for i in range(bboxlists.shape[1])] 44 | bboxlists = [bboxlists[keep, i, :] for i, keep in enumerate(keeps)] 45 | bboxlists = [[x for x in bboxlist if x[-1] > 0.5] for bboxlist in bboxlists] 46 | 47 | return bboxlists 48 | 49 | @property 50 | def reference_scale(self): 51 | return 195 52 | 53 | @property 54 | def reference_x_shift(self): 55 | return 0 56 | 57 | @property 58 | def reference_y_shift(self): 59 | return 0 60 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face_detection/detection/sfd/sfd_detector.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | from torch.utils.model_zoo import load_url 4 | 5 | from ..core import FaceDetector 6 | 7 | from .net_s3fd import s3fd 8 | from .bbox import * 9 | from .detect import * 10 | 11 | models_urls = { 12 | 's3fd': 'https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth', 13 | } 14 | 15 | 16 | class SFDDetector(FaceDetector): 17 | def __init__(self, device, path_to_detector='/apdcephfs/share_1290939/shadowcun/pretrained/s3fd.pth', verbose=False): 18 | super(SFDDetector, self).__init__(device, verbose) 19 | 20 | # Initialise the face detector 21 | if not os.path.isfile(path_to_detector): 22 | model_weights = load_url(models_urls['s3fd']) 23 | else: 24 | model_weights = torch.load(path_to_detector) 25 | 26 | self.face_detector = s3fd() 27 | self.face_detector.load_state_dict(model_weights) 28 | self.face_detector.to(device) 29 | self.face_detector.eval() 30 | 31 | def detect_from_image(self, tensor_or_path): 32 | image = self.tensor_or_path_to_ndarray(tensor_or_path) 33 | 34 | bboxlist = detect(self.face_detector, image, device=self.device) 35 | keep = nms(bboxlist, 0.3) 36 | bboxlist = bboxlist[keep, :] 37 | bboxlist = [x for x in bboxlist if x[-1] > 0.5] 38 | 39 | return bboxlist 40 | 41 | def detect_from_batch(self, images): 42 | bboxlists = batch_detect(self.face_detector, images, device=self.device) 43 | keeps = [nms(bboxlists[:, i, :], 0.3) for i in range(bboxlists.shape[1])] 44 | bboxlists = [bboxlists[keep, i, :] for i, keep in enumerate(keeps)] 45 | bboxlists = [[x for x in bboxlist if x[-1] > 0.5] for bboxlist in bboxlists] 46 | 47 | return bboxlists 48 | 49 | @property 50 | def reference_scale(self): 51 | return 195 52 | 53 | @property 54 | def reference_x_shift(self): 55 | return 0 56 | 57 | @property 58 | def reference_y_shift(self): 59 | return 0 60 | -------------------------------------------------------------------------------- /GFPGAN/tests/test_arcface_arch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from gfpgan.archs.arcface_arch import BasicBlock, Bottleneck, ResNetArcFace 4 | 5 | 6 | def test_resnetarcface(): 7 | """Test arch: ResNetArcFace.""" 8 | 9 | # model init and forward (gpu) 10 | if torch.cuda.is_available(): 11 | net = ResNetArcFace(block='IRBlock', layers=(2, 2, 2, 2), use_se=True).cuda().eval() 12 | img = torch.rand((1, 1, 128, 128), dtype=torch.float32).cuda() 13 | output = net(img) 14 | assert output.shape == (1, 512) 15 | 16 | # -------------------- without SE block ----------------------- # 17 | net = ResNetArcFace(block='IRBlock', layers=(2, 2, 2, 2), use_se=False).cuda().eval() 18 | output = net(img) 19 | assert output.shape == (1, 512) 20 | 21 | 22 | def test_basicblock(): 23 | """Test the BasicBlock in arcface_arch""" 24 | block = BasicBlock(1, 3, stride=1, downsample=None).cuda() 25 | img = torch.rand((1, 1, 12, 12), dtype=torch.float32).cuda() 26 | output = block(img) 27 | assert output.shape == (1, 3, 12, 12) 28 | 29 | # ----------------- use the downsmaple module--------------- # 30 | downsample = torch.nn.UpsamplingNearest2d(scale_factor=0.5).cuda() 31 | block = BasicBlock(1, 3, stride=2, downsample=downsample).cuda() 32 | img = torch.rand((1, 1, 12, 12), dtype=torch.float32).cuda() 33 | output = block(img) 34 | assert output.shape == (1, 3, 6, 6) 35 | 36 | 37 | def test_bottleneck(): 38 | """Test the Bottleneck in arcface_arch""" 39 | block = Bottleneck(1, 1, stride=1, downsample=None).cuda() 40 | img = torch.rand((1, 1, 12, 12), dtype=torch.float32).cuda() 41 | output = block(img) 42 | assert output.shape == (1, 4, 12, 12) 43 | 44 | # ----------------- use the downsmaple module--------------- # 45 | downsample = torch.nn.UpsamplingNearest2d(scale_factor=0.5).cuda() 46 | block = Bottleneck(1, 1, stride=2, downsample=downsample).cuda() 47 | img = torch.rand((1, 1, 12, 12), dtype=torch.float32).cuda() 48 | output = block(img) 49 | assert output.shape == (1, 4, 6, 6) 50 | -------------------------------------------------------------------------------- /eval/syncnet_python/calculate_scores_LRS.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #-*- coding: utf-8 -*- 3 | 4 | import time, pdb, argparse, subprocess 5 | import glob 6 | import os 7 | from tqdm import tqdm 8 | 9 | from SyncNetInstance_calc_scores import * 10 | 11 | # ==================== LOAD PARAMS ==================== 12 | 13 | 14 | parser = argparse.ArgumentParser(description = "SyncNet"); 15 | 16 | parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help='') 17 | parser.add_argument('--batch_size', type=int, default='20', help='') 18 | parser.add_argument('--vshift', type=int, default='15', help='') 19 | parser.add_argument('--data_root', type=str, required=True, help='') 20 | parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help='') 21 | parser.add_argument('--reference', type=str, default="demo", help='') 22 | parser.add_argument('--audiofile', type=str, default="", help='') 23 | 24 | opt = parser.parse_args(); 25 | 26 | 27 | # ==================== RUN EVALUATION ==================== 28 | 29 | s = SyncNetInstance(); 30 | 31 | s.loadParameters(opt.initial_model); 32 | #print("Model %s loaded."%opt.initial_model); 33 | path = os.path.join(opt.data_root, "*.mp4") 34 | 35 | all_videos = glob.glob(path) 36 | 37 | prog_bar = tqdm(range(len(all_videos))) 38 | avg_confidence = 0. 39 | avg_min_distance = 0. 40 | 41 | 42 | print("################# eval pred:{0} | gt:{1} #######################".format(opt.data_root, opt.audiofile)) 43 | for videofile_idx in prog_bar: 44 | videofile = all_videos[videofile_idx] 45 | offset, confidence, min_distance = s.evaluate(opt, videofile=videofile, audiofile=opt.audiofile) 46 | avg_confidence += confidence 47 | avg_min_distance += min_distance 48 | prog_bar.set_description('Avg Confidence: {}, Avg Minimum Dist: {}'.format(round(avg_confidence / (videofile_idx + 1), 3), round(avg_min_distance / (videofile_idx + 1), 3))) 49 | prog_bar.refresh() 50 | 51 | print ('Average Confidence: {}'.format(avg_confidence/len(all_videos))) 52 | print ('Average Minimum Distance: {}'.format(avg_min_distance/len(all_videos))) 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/data/celeba.py: -------------------------------------------------------------------------------- 1 | from .base_dataset import BaseDataset 2 | import os 3 | import random 4 | import numpy as np 5 | 6 | 7 | class CelebADataset(BaseDataset): 8 | """docstring for CelebADataset""" 9 | def __init__(self): 10 | super(CelebADataset, self).__init__() 11 | 12 | def initialize(self, opt): 13 | super(CelebADataset, self).initialize(opt) 14 | 15 | def get_aus_by_path(self, img_path): 16 | assert os.path.isfile(img_path), "Cannot find image file: %s" % img_path 17 | img_id = str(os.path.splitext(os.path.basename(img_path))[0]) 18 | return self.aus_dict[img_id] / 5.0 # norm to [0, 1] 19 | 20 | def make_dataset(self): 21 | # return all image full path in a list 22 | imgs_path = [] 23 | assert os.path.isfile(self.imgs_name_file), "%s does not exist." % self.imgs_name_file 24 | with open(self.imgs_name_file, 'r') as f: 25 | lines = f.readlines() 26 | imgs_path = [os.path.join(self.imgs_dir, line.strip()) for line in lines] 27 | imgs_path = sorted(imgs_path) 28 | return imgs_path 29 | 30 | def __getitem__(self, index): 31 | img_path = self.imgs_path[index] 32 | 33 | # load source image 34 | src_img = self.get_img_by_path(img_path) 35 | src_img_tensor = self.img2tensor(src_img) 36 | src_aus = self.get_aus_by_path(img_path) 37 | 38 | # load target image 39 | tar_img_path = random.choice(self.imgs_path) 40 | tar_img = self.get_img_by_path(tar_img_path) 41 | tar_img_tensor = self.img2tensor(tar_img) 42 | tar_aus = self.get_aus_by_path(tar_img_path) 43 | if self.is_train and not self.opt.no_aus_noise: 44 | tar_aus = tar_aus + np.random.uniform(-0.1, 0.1, tar_aus.shape) 45 | 46 | # record paths for debug and test usage 47 | data_dict = {'src_img':src_img_tensor, 'src_aus':src_aus, 'tar_img':tar_img_tensor, 'tar_aus':tar_aus, \ 48 | 'src_path':img_path, 'tar_path':tar_img_path} 49 | 50 | return data_dict 51 | -------------------------------------------------------------------------------- /eval/lmd/third_part/GPEN/face_model/face_gan.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @paper: GAN Prior Embedded Network for Blind Face Restoration in the Wild (CVPR2021) 3 | @author: yangxy (yangtao9009@gmail.com) 4 | ''' 5 | import torch 6 | import os 7 | import cv2 8 | import glob 9 | import numpy as np 10 | from torch import nn 11 | import torch.nn.functional as F 12 | from torchvision import transforms, utils 13 | from face_model.gpen_model import FullGenerator 14 | 15 | class FaceGAN(object): 16 | def __init__(self, base_dir='./', size=512, model=None, channel_multiplier=2, narrow=1, is_norm=True, device='cuda'): 17 | self.mfile = os.path.join(base_dir, model+'.pth') 18 | self.n_mlp = 8 19 | self.device = device 20 | self.is_norm = is_norm 21 | self.resolution = size 22 | self.load_model(channel_multiplier, narrow) 23 | 24 | def load_model(self, channel_multiplier=2, narrow=1): 25 | self.model = FullGenerator(self.resolution, 512, self.n_mlp, channel_multiplier, narrow=narrow, device=self.device) 26 | pretrained_dict = torch.load(self.mfile, map_location=torch.device('cpu')) 27 | self.model.load_state_dict(pretrained_dict) 28 | self.model.to(self.device) 29 | self.model.eval() 30 | 31 | def process(self, img): 32 | img = cv2.resize(img, (self.resolution, self.resolution)) 33 | img_t = self.img2tensor(img) 34 | 35 | with torch.no_grad(): 36 | out, __ = self.model(img_t) 37 | 38 | out = self.tensor2img(out) 39 | 40 | return out 41 | 42 | def img2tensor(self, img): 43 | img_t = torch.from_numpy(img).to(self.device)/255. 44 | if self.is_norm: 45 | img_t = (img_t - 0.5) / 0.5 46 | img_t = img_t.permute(2, 0, 1).unsqueeze(0).flip(1) # BGR->RGB 47 | return img_t 48 | 49 | def tensor2img(self, img_t, pmax=255.0, imtype=np.uint8): 50 | if self.is_norm: 51 | img_t = img_t * 0.5 + 0.5 52 | img_t = img_t.squeeze(0).permute(1, 2, 0).flip(2) # RGB->BGR 53 | img_np = np.clip(img_t.float().cpu().numpy(), 0, 1) * pmax 54 | 55 | return img_np.astype(imtype) 56 | -------------------------------------------------------------------------------- /eg3d/eg3d/metrics/inception_score.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | """Inception Score (IS) from the paper "Improved techniques for training 12 | GANs". Matches the original implementation by Salimans et al. at 13 | https://github.com/openai/improved-gan/blob/master/inception_score/model.py""" 14 | 15 | import numpy as np 16 | from . import metric_utils 17 | 18 | #---------------------------------------------------------------------------- 19 | 20 | def compute_is(opts, num_gen, num_splits): 21 | # Direct TorchScript translation of http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz 22 | detector_url = 'https://api.ngc.nvidia.com/v2/models/nvidia/research/stylegan3/versions/1/files/metrics/inception-2015-12-05.pkl' 23 | detector_kwargs = dict(no_output_bias=True) # Match the original implementation by not applying bias in the softmax layer. 24 | 25 | gen_probs = metric_utils.compute_feature_stats_for_generator( 26 | opts=opts, detector_url=detector_url, detector_kwargs=detector_kwargs, 27 | capture_all=True, max_items=num_gen).get_all() 28 | 29 | if opts.rank != 0: 30 | return float('nan'), float('nan') 31 | 32 | scores = [] 33 | for i in range(num_splits): 34 | part = gen_probs[i * num_gen // num_splits : (i + 1) * num_gen // num_splits] 35 | kl = part * (np.log(part) - np.log(np.mean(part, axis=0, keepdims=True))) 36 | kl = np.mean(np.sum(kl, axis=1)) 37 | scores.append(np.exp(kl)) 38 | return float(np.mean(scores)), float(np.std(scores)) 39 | 40 | #---------------------------------------------------------------------------- 41 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/data_preparation.py: -------------------------------------------------------------------------------- 1 | """This script is the data preparation script for Deep3DFaceRecon_pytorch 2 | """ 3 | 4 | import os 5 | import numpy as np 6 | import argparse 7 | from util.detect_lm68 import detect_68p,load_lm_graph 8 | from util.skin_mask import get_skin_mask 9 | from util.generate_list import check_list, write_list 10 | import warnings 11 | warnings.filterwarnings("ignore") 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('--data_root', type=str, default='datasets', help='root directory for training data') 15 | parser.add_argument('--img_folder', nargs="+", required=True, help='folders of training images') 16 | parser.add_argument('--mode', type=str, default='train', help='train or val') 17 | opt = parser.parse_args() 18 | 19 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 20 | 21 | def data_prepare(folder_list,mode): 22 | 23 | lm_sess,input_op,output_op = load_lm_graph('./checkpoints/lm_model/68lm_detector.pb') # load a tensorflow version 68-landmark detector 24 | 25 | for img_folder in folder_list: 26 | detect_68p(img_folder,lm_sess,input_op,output_op) # detect landmarks for images 27 | get_skin_mask(img_folder) # generate skin attention mask for images 28 | 29 | # create files that record path to all training data 30 | msks_list = [] 31 | for img_folder in folder_list: 32 | path = os.path.join(img_folder, 'mask') 33 | msks_list += ['/'.join([img_folder, 'mask', i]) for i in sorted(os.listdir(path)) if 'jpg' in i or 34 | 'png' in i or 'jpeg' in i or 'PNG' in i] 35 | 36 | imgs_list = [i.replace('mask/', '') for i in msks_list] 37 | lms_list = [i.replace('mask', 'landmarks') for i in msks_list] 38 | lms_list = ['.'.join(i.split('.')[:-1]) + '.txt' for i in lms_list] 39 | 40 | lms_list_final, imgs_list_final, msks_list_final = check_list(lms_list, imgs_list, msks_list) # check if the path is valid 41 | write_list(lms_list_final, imgs_list_final, msks_list_final, mode=mode) # save files 42 | 43 | if __name__ == '__main__': 44 | print('Datasets:',opt.img_folder) 45 | data_prepare([os.path.join(opt.data_root,folder) for folder in opt.img_folder],opt.mode) 46 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/checkpoints/model_name/test_opt.txt: -------------------------------------------------------------------------------- 1 | ----------------- Options --------------- 2 | add_image: True 3 | bfm_folder: BFM 4 | bfm_model: BFM09_model_info.mat 5 | camera_d: 10.0 6 | center: 112.0 7 | checkpoints_dir: ./checkpoints 8 | dataset_mode: None 9 | ddp_port: 12355 10 | display_per_batch: True 11 | epoch: 20 [default: latest] 12 | eval_batch_nums: inf 13 | focal: 1015.0 14 | gpu_ids: 0 15 | inference_batch_size: 8 16 | init_path: checkpoints/init_model/resnet50-0676ba61.pth 17 | input_dir: demo_video [default: None] 18 | isTrain: False [default: None] 19 | keypoint_dir: demo_cctv [default: None] 20 | model: facerecon 21 | name: model_name [default: face_recon] 22 | net_recon: resnet50 23 | output_dir: demo_cctv [default: mp4] 24 | phase: test 25 | save_split_files: False 26 | suffix: 27 | use_ddp: False [default: True] 28 | use_last_fc: False 29 | verbose: False 30 | vis_batch_nums: 1 31 | world_size: 1 32 | z_far: 15.0 33 | z_near: 5.0 34 | ----------------- End ------------------- 35 | -------------------------------------------------------------------------------- /torch_utils/ops/upfirdn2d.h: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include 14 | 15 | //------------------------------------------------------------------------ 16 | // CUDA kernel parameters. 17 | 18 | struct upfirdn2d_kernel_params 19 | { 20 | const void* x; 21 | const float* f; 22 | void* y; 23 | 24 | int2 up; 25 | int2 down; 26 | int2 pad0; 27 | int flip; 28 | float gain; 29 | 30 | int4 inSize; // [width, height, channel, batch] 31 | int4 inStride; 32 | int2 filterSize; // [width, height] 33 | int2 filterStride; 34 | int4 outSize; // [width, height, channel, batch] 35 | int4 outStride; 36 | int sizeMinor; 37 | int sizeMajor; 38 | 39 | int loopMinor; 40 | int loopMajor; 41 | int loopX; 42 | int launchMinor; 43 | int launchMajor; 44 | }; 45 | 46 | //------------------------------------------------------------------------ 47 | // CUDA kernel specialization. 48 | 49 | struct upfirdn2d_kernel_spec 50 | { 51 | void* kernel; 52 | int tileOutW; 53 | int tileOutH; 54 | int loopMinor; 55 | int loopX; 56 | }; 57 | 58 | //------------------------------------------------------------------------ 59 | // CUDA kernel selection. 60 | 61 | template upfirdn2d_kernel_spec choose_upfirdn2d_kernel(const upfirdn2d_kernel_params& p); 62 | 63 | //------------------------------------------------------------------------ 64 | -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/upfirdn2d.h: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: LicenseRef-NvidiaProprietary 4 | * 5 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 6 | * property and proprietary rights in and to this material, related 7 | * documentation and any modifications thereto. Any use, reproduction, 8 | * disclosure or distribution of this material and related documentation 9 | * without an express license agreement from NVIDIA CORPORATION or 10 | * its affiliates is strictly prohibited. 11 | */ 12 | 13 | #include 14 | 15 | //------------------------------------------------------------------------ 16 | // CUDA kernel parameters. 17 | 18 | struct upfirdn2d_kernel_params 19 | { 20 | const void* x; 21 | const float* f; 22 | void* y; 23 | 24 | int2 up; 25 | int2 down; 26 | int2 pad0; 27 | int flip; 28 | float gain; 29 | 30 | int4 inSize; // [width, height, channel, batch] 31 | int4 inStride; 32 | int2 filterSize; // [width, height] 33 | int2 filterStride; 34 | int4 outSize; // [width, height, channel, batch] 35 | int4 outStride; 36 | int sizeMinor; 37 | int sizeMajor; 38 | 39 | int loopMinor; 40 | int loopMajor; 41 | int loopX; 42 | int launchMinor; 43 | int launchMajor; 44 | }; 45 | 46 | //------------------------------------------------------------------------ 47 | // CUDA kernel specialization. 48 | 49 | struct upfirdn2d_kernel_spec 50 | { 51 | void* kernel; 52 | int tileOutW; 53 | int tileOutH; 54 | int loopMinor; 55 | int loopX; 56 | }; 57 | 58 | //------------------------------------------------------------------------ 59 | // CUDA kernel selection. 60 | 61 | template upfirdn2d_kernel_spec choose_upfirdn2d_kernel(const upfirdn2d_kernel_params& p); 62 | 63 | //------------------------------------------------------------------------ 64 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/data/image_folder.py: -------------------------------------------------------------------------------- 1 | """A modified image folder class 2 | 3 | We modify the official PyTorch image folder (https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py) 4 | so that this class can load images from both current directory and its subdirectories. 5 | """ 6 | import numpy as np 7 | import torch.utils.data as data 8 | 9 | from PIL import Image 10 | import os 11 | import os.path 12 | 13 | IMG_EXTENSIONS = [ 14 | '.jpg', '.JPG', '.jpeg', '.JPEG', 15 | '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 16 | '.tif', '.TIF', '.tiff', '.TIFF', 17 | ] 18 | 19 | 20 | def is_image_file(filename): 21 | return any(filename.endswith(extension) for extension in IMG_EXTENSIONS) 22 | 23 | 24 | def make_dataset(dir, max_dataset_size=float("inf")): 25 | images = [] 26 | assert os.path.isdir(dir) or os.path.islink(dir), '%s is not a valid directory' % dir 27 | 28 | for root, _, fnames in sorted(os.walk(dir, followlinks=True)): 29 | for fname in fnames: 30 | if is_image_file(fname): 31 | path = os.path.join(root, fname) 32 | images.append(path) 33 | return images[:min(max_dataset_size, len(images))] 34 | 35 | 36 | def default_loader(path): 37 | return Image.open(path).convert('RGB') 38 | 39 | 40 | class ImageFolder(data.Dataset): 41 | 42 | def __init__(self, root, transform=None, return_paths=False, 43 | loader=default_loader): 44 | imgs = make_dataset(root) 45 | if len(imgs) == 0: 46 | raise(RuntimeError("Found 0 images in: " + root + "\n" 47 | "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) 48 | 49 | self.root = root 50 | self.imgs = imgs 51 | self.transform = transform 52 | self.return_paths = return_paths 53 | self.loader = loader 54 | 55 | def __getitem__(self, index): 56 | path = self.imgs[index] 57 | img = self.loader(path) 58 | if self.transform is not None: 59 | img = self.transform(img) 60 | if self.return_paths: 61 | return img, path 62 | else: 63 | return img 64 | 65 | def __len__(self): 66 | return len(self.imgs) 67 | -------------------------------------------------------------------------------- /preprocess_utils/data_utils/face_tracking/geo_transform.py: -------------------------------------------------------------------------------- 1 | """This module contains functions for geometry transform and camera projection""" 2 | import torch 3 | import torch.nn as nn 4 | import numpy as np 5 | 6 | 7 | def euler2rot(euler_angle): 8 | batch_size = euler_angle.shape[0] 9 | theta = euler_angle[:, 0].reshape(-1, 1, 1) 10 | phi = euler_angle[:, 1].reshape(-1, 1, 1) 11 | psi = euler_angle[:, 2].reshape(-1, 1, 1) 12 | one = torch.ones((batch_size, 1, 1), dtype=torch.float32, device=euler_angle.device) 13 | zero = torch.zeros( 14 | (batch_size, 1, 1), dtype=torch.float32, device=euler_angle.device 15 | ) 16 | rot_x = torch.cat( 17 | ( 18 | torch.cat((one, zero, zero), 1), 19 | torch.cat((zero, theta.cos(), theta.sin()), 1), 20 | torch.cat((zero, -theta.sin(), theta.cos()), 1), 21 | ), 22 | 2, 23 | ) 24 | rot_y = torch.cat( 25 | ( 26 | torch.cat((phi.cos(), zero, -phi.sin()), 1), 27 | torch.cat((zero, one, zero), 1), 28 | torch.cat((phi.sin(), zero, phi.cos()), 1), 29 | ), 30 | 2, 31 | ) 32 | rot_z = torch.cat( 33 | ( 34 | torch.cat((psi.cos(), -psi.sin(), zero), 1), 35 | torch.cat((psi.sin(), psi.cos(), zero), 1), 36 | torch.cat((zero, zero, one), 1), 37 | ), 38 | 2, 39 | ) 40 | return torch.bmm(rot_x, torch.bmm(rot_y, rot_z)) 41 | 42 | 43 | def rot_trans_geo(geometry, rot, trans): 44 | rott_geo = torch.bmm(rot, geometry.permute(0, 2, 1)) + trans.view(-1, 3, 1) 45 | return rott_geo.permute(0, 2, 1) 46 | 47 | 48 | def euler_trans_geo(geometry, euler, trans): 49 | rot = euler2rot(euler) 50 | return rot_trans_geo(geometry, rot, trans) 51 | 52 | 53 | def proj_geo(rott_geo, camera_para): 54 | fx = camera_para[:, 0] 55 | fy = camera_para[:, 0] 56 | cx = camera_para[:, 1] 57 | cy = camera_para[:, 2] 58 | 59 | X = rott_geo[:, :, 0] 60 | Y = rott_geo[:, :, 1] 61 | Z = rott_geo[:, :, 2] 62 | 63 | fxX = fx[:, None] * X 64 | fyY = fy[:, None] * Y 65 | 66 | proj_x = -fxX / Z + cx[:, None] 67 | proj_y = fyY / Z + cy[:, None] 68 | 69 | return torch.cat((proj_x[:, :, None], proj_y[:, :, None], Z[:, :, None]), 2) 70 | -------------------------------------------------------------------------------- /GFPGAN/tests/test_stylegan2_clean_arch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from gfpgan.archs.stylegan2_clean_arch import StyleGAN2GeneratorClean 4 | 5 | 6 | def test_stylegan2generatorclean(): 7 | """Test arch: StyleGAN2GeneratorClean.""" 8 | 9 | # model init and forward (gpu) 10 | if torch.cuda.is_available(): 11 | net = StyleGAN2GeneratorClean( 12 | out_size=32, num_style_feat=512, num_mlp=8, channel_multiplier=1, narrow=0.5).cuda().eval() 13 | style = torch.rand((1, 512), dtype=torch.float32).cuda() 14 | output = net([style], input_is_latent=False) 15 | assert output[0].shape == (1, 3, 32, 32) 16 | assert output[1] is None 17 | 18 | # -------------------- with return_latents ----------------------- # 19 | output = net([style], input_is_latent=True, return_latents=True) 20 | assert output[0].shape == (1, 3, 32, 32) 21 | assert len(output[1]) == 1 22 | # check latent 23 | assert output[1][0].shape == (8, 512) 24 | 25 | # -------------------- with randomize_noise = False ----------------------- # 26 | output = net([style], randomize_noise=False) 27 | assert output[0].shape == (1, 3, 32, 32) 28 | assert output[1] is None 29 | 30 | # -------------------- with truncation = 0.5 and mixing----------------------- # 31 | output = net([style, style], truncation=0.5, truncation_latent=style) 32 | assert output[0].shape == (1, 3, 32, 32) 33 | assert output[1] is None 34 | 35 | # ------------------ test make_noise ----------------------- # 36 | out = net.make_noise() 37 | assert len(out) == 7 38 | assert out[0].shape == (1, 1, 4, 4) 39 | assert out[1].shape == (1, 1, 8, 8) 40 | assert out[2].shape == (1, 1, 8, 8) 41 | assert out[3].shape == (1, 1, 16, 16) 42 | assert out[4].shape == (1, 1, 16, 16) 43 | assert out[5].shape == (1, 1, 32, 32) 44 | assert out[6].shape == (1, 1, 32, 32) 45 | 46 | # ------------------ test get_latent ----------------------- # 47 | out = net.get_latent(style) 48 | assert out.shape == (1, 512) 49 | 50 | # ------------------ test mean_latent ----------------------- # 51 | out = net.mean_latent(2) 52 | assert out.shape == (1, 512) 53 | -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/ffhq/crop_images.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import argparse 12 | import os 13 | import json 14 | 15 | import numpy as np 16 | from PIL import Image 17 | from tqdm import tqdm 18 | from preprocess import align_img 19 | 20 | 21 | if __name__ == '__main__': 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('--indir', type=str, required=True) 24 | parser.add_argument('--outdir', type=str, required=True) 25 | parser.add_argument('--compress_level', type=int, default=0) 26 | args = parser.parse_args() 27 | 28 | with open(os.path.join(args.indir, 'cropping_params.json')) as f: 29 | cropping_params = json.load(f) 30 | 31 | os.makedirs(args.outdir, exist_ok=True) 32 | 33 | for im_path, cropping_dict in tqdm(cropping_params.items()): 34 | im = Image.open(os.path.join(args.indir, im_path)).convert('RGB') 35 | 36 | _, H = im.size 37 | lm = np.array(cropping_dict['lm']) 38 | lm = lm.reshape([-1, 2]) 39 | lm[:, -1] = H - 1 - lm[:, -1] 40 | 41 | _, im_high, _, _, = align_img(im, lm, np.array(cropping_dict['lm3d_std']), target_size=1024., rescale_factor=cropping_dict['rescale_factor']) 42 | 43 | left = int(im_high.size[0]/2 - cropping_dict['center_crop_size']/2) 44 | upper = int(im_high.size[1]/2 - cropping_dict['center_crop_size']/2) 45 | right = left + cropping_dict['center_crop_size'] 46 | lower = upper + cropping_dict['center_crop_size'] 47 | im_cropped = im_high.crop((left, upper, right,lower)) 48 | im_cropped = im_cropped.resize((cropping_dict['output_size'], cropping_dict['output_size']), resample=Image.LANCZOS) 49 | 50 | im_cropped.save(os.path.join(args.outdir, os.path.basename(im_path)), compress_level=args.compress_level) -------------------------------------------------------------------------------- /eg3d/eg3d/metrics/frechet_inception_distance.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | """Frechet Inception Distance (FID) from the paper 12 | "GANs trained by a two time-scale update rule converge to a local Nash 13 | equilibrium". Matches the original implementation by Heusel et al. at 14 | https://github.com/bioinf-jku/TTUR/blob/master/fid.py""" 15 | 16 | import numpy as np 17 | import scipy.linalg 18 | from . import metric_utils 19 | 20 | #---------------------------------------------------------------------------- 21 | 22 | def compute_fid(opts, max_real, num_gen): 23 | # Direct TorchScript translation of http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz 24 | detector_url = 'https://api.ngc.nvidia.com/v2/models/nvidia/research/stylegan3/versions/1/files/metrics/inception-2015-12-05.pkl' 25 | detector_kwargs = dict(return_features=True) # Return raw features before the softmax layer. 26 | 27 | mu_real, sigma_real = metric_utils.compute_feature_stats_for_dataset( 28 | opts=opts, detector_url=detector_url, detector_kwargs=detector_kwargs, 29 | rel_lo=0, rel_hi=0, capture_mean_cov=True, max_items=max_real).get_mean_cov() 30 | 31 | mu_gen, sigma_gen = metric_utils.compute_feature_stats_for_generator( 32 | opts=opts, detector_url=detector_url, detector_kwargs=detector_kwargs, 33 | rel_lo=0, rel_hi=1, capture_mean_cov=True, max_items=num_gen).get_mean_cov() 34 | 35 | if opts.rank != 0: 36 | return float('nan') 37 | 38 | m = np.square(mu_gen - mu_real).sum() 39 | s, _ = scipy.linalg.sqrtm(np.dot(sigma_gen, sigma_real), disp=False) # pylint: disable=no-member 40 | fid = np.real(m + np.trace(sigma_gen + sigma_real - s * 2)) 41 | return float(fid) 42 | 43 | #---------------------------------------------------------------------------- 44 | -------------------------------------------------------------------------------- /eval/syncnet_python/detectors/s3fd/__init__.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import cv2 4 | import torch 5 | from torchvision import transforms 6 | from .nets import S3FDNet 7 | from .box_utils import nms_ 8 | 9 | PATH_WEIGHT = './detectors/s3fd/weights/sfd_face.pth' 10 | img_mean = np.array([104., 117., 123.])[:, np.newaxis, np.newaxis].astype('float32') 11 | 12 | 13 | class S3FD(): 14 | 15 | def __init__(self, device='cuda'): 16 | 17 | tstamp = time.time() 18 | self.device = device 19 | 20 | print('[S3FD] loading with', self.device) 21 | self.net = S3FDNet(device=self.device).to(self.device) 22 | state_dict = torch.load(PATH_WEIGHT, map_location=self.device) 23 | self.net.load_state_dict(state_dict) 24 | self.net.eval() 25 | print('[S3FD] finished loading (%.4f sec)' % (time.time() - tstamp)) 26 | 27 | def detect_faces(self, image, conf_th=0.8, scales=[1]): 28 | 29 | w, h = image.shape[1], image.shape[0] 30 | 31 | bboxes = np.empty(shape=(0, 5)) 32 | 33 | with torch.no_grad(): 34 | for s in scales: 35 | scaled_img = cv2.resize(image, dsize=(0, 0), fx=s, fy=s, interpolation=cv2.INTER_LINEAR) 36 | 37 | scaled_img = np.swapaxes(scaled_img, 1, 2) 38 | scaled_img = np.swapaxes(scaled_img, 1, 0) 39 | scaled_img = scaled_img[[2, 1, 0], :, :] 40 | scaled_img = scaled_img.astype('float32') 41 | scaled_img -= img_mean 42 | scaled_img = scaled_img[[2, 1, 0], :, :] 43 | x = torch.from_numpy(scaled_img).unsqueeze(0).to(self.device) 44 | y = self.net(x) 45 | 46 | detections = y.data 47 | scale = torch.Tensor([w, h, w, h]) 48 | 49 | for i in range(detections.size(1)): 50 | j = 0 51 | while detections[0, i, j, 0] > conf_th: 52 | score = detections[0, i, j, 0] 53 | pt = (detections[0, i, j, 1:] * scale).cpu().numpy() 54 | bbox = (pt[0], pt[1], pt[2], pt[3], score) 55 | bboxes = np.vstack((bboxes, bbox)) 56 | j += 1 57 | 58 | keep = nms_(bboxes, 0.1) 59 | bboxes = bboxes[keep] 60 | 61 | return bboxes 62 | -------------------------------------------------------------------------------- /torch_utils/ops/fma.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | """Fused multiply-add, with slightly faster gradients than `torch.addcmul()`.""" 12 | 13 | import torch 14 | 15 | #---------------------------------------------------------------------------- 16 | 17 | def fma(a, b, c): # => a * b + c 18 | return _FusedMultiplyAdd.apply(a, b, c) 19 | 20 | #---------------------------------------------------------------------------- 21 | 22 | class _FusedMultiplyAdd(torch.autograd.Function): # a * b + c 23 | @staticmethod 24 | def forward(ctx, a, b, c): # pylint: disable=arguments-differ 25 | out = torch.addcmul(c, a, b) 26 | ctx.save_for_backward(a, b) 27 | ctx.c_shape = c.shape 28 | return out 29 | 30 | @staticmethod 31 | def backward(ctx, dout): # pylint: disable=arguments-differ 32 | a, b = ctx.saved_tensors 33 | c_shape = ctx.c_shape 34 | da = None 35 | db = None 36 | dc = None 37 | 38 | if ctx.needs_input_grad[0]: 39 | da = _unbroadcast(dout * b, a.shape) 40 | 41 | if ctx.needs_input_grad[1]: 42 | db = _unbroadcast(dout * a, b.shape) 43 | 44 | if ctx.needs_input_grad[2]: 45 | dc = _unbroadcast(dout, c_shape) 46 | 47 | return da, db, dc 48 | 49 | #---------------------------------------------------------------------------- 50 | 51 | def _unbroadcast(x, shape): 52 | extra_dims = x.ndim - len(shape) 53 | assert extra_dims >= 0 54 | dim = [i for i in range(x.ndim) if x.shape[i] > 1 and (i < extra_dims or shape[i - extra_dims] == 1)] 55 | if len(dim): 56 | x = x.sum(dim=dim, keepdim=True) 57 | if extra_dims: 58 | x = x.reshape(-1, *x.shape[extra_dims+1:]) 59 | assert x.shape == shape 60 | return x 61 | 62 | #---------------------------------------------------------------------------- 63 | -------------------------------------------------------------------------------- /eg3d/eg3d/torch_utils/ops/fma.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | """Fused multiply-add, with slightly faster gradients than `torch.addcmul()`.""" 12 | 13 | import torch 14 | 15 | #---------------------------------------------------------------------------- 16 | 17 | def fma(a, b, c): # => a * b + c 18 | return _FusedMultiplyAdd.apply(a, b, c) 19 | 20 | #---------------------------------------------------------------------------- 21 | 22 | class _FusedMultiplyAdd(torch.autograd.Function): # a * b + c 23 | @staticmethod 24 | def forward(ctx, a, b, c): # pylint: disable=arguments-differ 25 | out = torch.addcmul(c, a, b) 26 | ctx.save_for_backward(a, b) 27 | ctx.c_shape = c.shape 28 | return out 29 | 30 | @staticmethod 31 | def backward(ctx, dout): # pylint: disable=arguments-differ 32 | a, b = ctx.saved_tensors 33 | c_shape = ctx.c_shape 34 | da = None 35 | db = None 36 | dc = None 37 | 38 | if ctx.needs_input_grad[0]: 39 | da = _unbroadcast(dout * b, a.shape) 40 | 41 | if ctx.needs_input_grad[1]: 42 | db = _unbroadcast(dout * a, b.shape) 43 | 44 | if ctx.needs_input_grad[2]: 45 | dc = _unbroadcast(dout, c_shape) 46 | 47 | return da, db, dc 48 | 49 | #---------------------------------------------------------------------------- 50 | 51 | def _unbroadcast(x, shape): 52 | extra_dims = x.ndim - len(shape) 53 | assert extra_dims >= 0 54 | dim = [i for i in range(x.ndim) if x.shape[i] > 1 and (i < extra_dims or shape[i - extra_dims] == 1)] 55 | if len(dim): 56 | x = x.sum(dim=dim, keepdim=True) 57 | if extra_dims: 58 | x = x.reshape(-1, *x.shape[extra_dims+1:]) 59 | assert x.shape == shape 60 | return x 61 | 62 | #---------------------------------------------------------------------------- 63 | -------------------------------------------------------------------------------- /eval/lmd/third_part/ganimation_replicate/checkpoints/run_script.sh: -------------------------------------------------------------------------------- 1 | [ test][220417_224012]python main.py --mode test --data_root datasets/celebA --ckpt_dir checkpoints --load_epoch 30 2 | [ test][220419_184832]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 3 | [ test][220419_185232]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 4 | [ test][220419_185252]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 5 | [ test][220419_185305]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 6 | [ test][220419_185320]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 7 | [ test][220419_185810]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 8 | [ test][220419_190338]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 9 | [ test][220419_190445]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 10 | [ test][220419_190628]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 11 | [ test][220419_195037]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 12 | [ test][220419_200348]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 13 | [ test][220419_200512]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 14 | [ test][220419_200529]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 15 | [ test][220419_200554]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 16 | [ test][220419_200622]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 17 | [ test][220419_200641]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 18 | [ test][220419_200658]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 19 | [ test][220419_200717]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 20 | [ test][220419_200740]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 21 | [ test][220419_200807]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 22 | [ test][220419_213236]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 23 | [ test][220419_213329]python test.py --data_root . --mode test --load_epoch 30 --ckpt_dir checkpoints/ 24 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/utils/plot.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import os 4 | from pathlib import Path 5 | 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | import pandas as pd 9 | from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap 10 | from prettytable import PrettyTable 11 | from sklearn.metrics import roc_curve, auc 12 | 13 | image_path = "/data/anxiang/IJB_release/IJBC" 14 | files = [ 15 | "./ms1mv3_arcface_r100/ms1mv3_arcface_r100/ijbc.npy" 16 | ] 17 | 18 | 19 | def read_template_pair_list(path): 20 | pairs = pd.read_csv(path, sep=' ', header=None).values 21 | t1 = pairs[:, 0].astype(np.int) 22 | t2 = pairs[:, 1].astype(np.int) 23 | label = pairs[:, 2].astype(np.int) 24 | return t1, t2, label 25 | 26 | 27 | p1, p2, label = read_template_pair_list( 28 | os.path.join('%s/meta' % image_path, 29 | '%s_template_pair_label.txt' % 'ijbc')) 30 | 31 | methods = [] 32 | scores = [] 33 | for file in files: 34 | methods.append(file.split('/')[-2]) 35 | scores.append(np.load(file)) 36 | 37 | methods = np.array(methods) 38 | scores = dict(zip(methods, scores)) 39 | colours = dict( 40 | zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) 41 | x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] 42 | tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) 43 | fig = plt.figure() 44 | for method in methods: 45 | fpr, tpr, _ = roc_curve(label, scores[method]) 46 | roc_auc = auc(fpr, tpr) 47 | fpr = np.flipud(fpr) 48 | tpr = np.flipud(tpr) # select largest tpr at same fpr 49 | plt.plot(fpr, 50 | tpr, 51 | color=colours[method], 52 | lw=1, 53 | label=('[%s (AUC = %0.4f %%)]' % 54 | (method.split('-')[-1], roc_auc * 100))) 55 | tpr_fpr_row = [] 56 | tpr_fpr_row.append("%s-%s" % (method, "IJBC")) 57 | for fpr_iter in np.arange(len(x_labels)): 58 | _, min_index = min( 59 | list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) 60 | tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) 61 | tpr_fpr_table.add_row(tpr_fpr_row) 62 | plt.xlim([10 ** -6, 0.1]) 63 | plt.ylim([0.3, 1.0]) 64 | plt.grid(linestyle='--', linewidth=1) 65 | plt.xticks(x_labels) 66 | plt.yticks(np.linspace(0.3, 1.0, 8, endpoint=True)) 67 | plt.xscale('log') 68 | plt.xlabel('False Positive Rate') 69 | plt.ylabel('True Positive Rate') 70 | plt.title('ROC on IJB') 71 | plt.legend(loc="lower right") 72 | print(tpr_fpr_table) 73 | -------------------------------------------------------------------------------- /eval/lmd/third_part/face_detection/api.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import torch 4 | from torch.utils.model_zoo import load_url 5 | from enum import Enum 6 | import numpy as np 7 | import cv2 8 | try: 9 | import urllib.request as request_file 10 | except BaseException: 11 | import urllib as request_file 12 | 13 | from .models import FAN, ResNetDepth 14 | from .utils import * 15 | 16 | 17 | class LandmarksType(Enum): 18 | """Enum class defining the type of landmarks to detect. 19 | 20 | ``_2D`` - the detected points ``(x,y)`` are detected in a 2D space and follow the visible contour of the face 21 | ``_2halfD`` - this points represent the projection of the 3D points into 3D 22 | ``_3D`` - detect the points ``(x,y,z)``` in a 3D space 23 | 24 | """ 25 | _2D = 1 26 | _2halfD = 2 27 | _3D = 3 28 | 29 | 30 | class NetworkSize(Enum): 31 | # TINY = 1 32 | # SMALL = 2 33 | # MEDIUM = 3 34 | LARGE = 4 35 | 36 | def __new__(cls, value): 37 | member = object.__new__(cls) 38 | member._value_ = value 39 | return member 40 | 41 | def __int__(self): 42 | return self.value 43 | 44 | ROOT = os.path.dirname(os.path.abspath(__file__)) 45 | 46 | class FaceAlignment: 47 | def __init__(self, landmarks_type, network_size=NetworkSize.LARGE, 48 | device='cuda', flip_input=False, face_detector='sfd', verbose=False): 49 | self.device = device 50 | self.flip_input = flip_input 51 | self.landmarks_type = landmarks_type 52 | self.verbose = verbose 53 | 54 | network_size = int(network_size) 55 | 56 | if 'cuda' in device: 57 | torch.backends.cudnn.benchmark = True 58 | 59 | # Get the face detector 60 | face_detector_module = __import__('face_detection.detection.' + face_detector, 61 | globals(), locals(), [face_detector], 0) 62 | self.face_detector = face_detector_module.FaceDetector(device=device, verbose=verbose) 63 | 64 | def get_detections_for_batch(self, images): 65 | images = images[..., ::-1] 66 | detected_faces = self.face_detector.detect_from_batch(images.copy()) 67 | results = [] 68 | 69 | for i, d in enumerate(detected_faces): 70 | if len(d) == 0: 71 | results.append(None) 72 | continue 73 | d = d[0] 74 | d = np.clip(d, 0, None) 75 | 76 | x1, y1, x2, y2 = map(int, d[:-1]) 77 | results.append((x1, y1, x2, y2)) 78 | 79 | return results -------------------------------------------------------------------------------- /eval/lmd/third_part/face3d/models/arcface_torch/torch2onnx.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import onnx 3 | import torch 4 | 5 | 6 | def convert_onnx(net, path_module, output, opset=11, simplify=False): 7 | assert isinstance(net, torch.nn.Module) 8 | img = np.random.randint(0, 255, size=(112, 112, 3), dtype=np.int32) 9 | img = img.astype(np.float) 10 | img = (img / 255. - 0.5) / 0.5 # torch style norm 11 | img = img.transpose((2, 0, 1)) 12 | img = torch.from_numpy(img).unsqueeze(0).float() 13 | 14 | weight = torch.load(path_module) 15 | net.load_state_dict(weight) 16 | net.eval() 17 | torch.onnx.export(net, img, output, keep_initializers_as_inputs=False, verbose=False, opset_version=opset) 18 | model = onnx.load(output) 19 | graph = model.graph 20 | graph.input[0].type.tensor_type.shape.dim[0].dim_param = 'None' 21 | if simplify: 22 | from onnxsim import simplify 23 | model, check = simplify(model) 24 | assert check, "Simplified ONNX model could not be validated" 25 | onnx.save(model, output) 26 | 27 | 28 | if __name__ == '__main__': 29 | import os 30 | import argparse 31 | from backbones import get_model 32 | 33 | parser = argparse.ArgumentParser(description='ArcFace PyTorch to onnx') 34 | parser.add_argument('input', type=str, help='input backbone.pth file or path') 35 | parser.add_argument('--output', type=str, default=None, help='output onnx path') 36 | parser.add_argument('--network', type=str, default=None, help='backbone network') 37 | parser.add_argument('--simplify', type=bool, default=False, help='onnx simplify') 38 | args = parser.parse_args() 39 | input_file = args.input 40 | if os.path.isdir(input_file): 41 | input_file = os.path.join(input_file, "backbone.pth") 42 | assert os.path.exists(input_file) 43 | model_name = os.path.basename(os.path.dirname(input_file)).lower() 44 | params = model_name.split("_") 45 | if len(params) >= 3 and params[1] in ('arcface', 'cosface'): 46 | if args.network is None: 47 | args.network = params[2] 48 | assert args.network is not None 49 | print(args) 50 | backbone_onnx = get_model(args.network, dropout=0) 51 | 52 | output_path = args.output 53 | if output_path is None: 54 | output_path = os.path.join(os.path.dirname(__file__), 'onnx') 55 | if not os.path.exists(output_path): 56 | os.makedirs(output_path) 57 | assert os.path.isdir(output_path) 58 | output_file = os.path.join(output_path, "%s.onnx" % model_name) 59 | convert_onnx(backbone_onnx, input_file, output_file, simplify=args.simplify) 60 | -------------------------------------------------------------------------------- /eg3d/eg3d/metrics/kernel_inception_distance.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | """Kernel Inception Distance (KID) from the paper "Demystifying MMD 12 | GANs". Matches the original implementation by Binkowski et al. at 13 | https://github.com/mbinkowski/MMD-GAN/blob/master/gan/compute_scores.py""" 14 | 15 | import numpy as np 16 | from . import metric_utils 17 | 18 | #---------------------------------------------------------------------------- 19 | 20 | def compute_kid(opts, max_real, num_gen, num_subsets, max_subset_size): 21 | # Direct TorchScript translation of http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz 22 | detector_url = 'https://api.ngc.nvidia.com/v2/models/nvidia/research/stylegan3/versions/1/files/metrics/inception-2015-12-05.pkl' 23 | detector_kwargs = dict(return_features=True) # Return raw features before the softmax layer. 24 | 25 | real_features = metric_utils.compute_feature_stats_for_dataset( 26 | opts=opts, detector_url=detector_url, detector_kwargs=detector_kwargs, 27 | rel_lo=0, rel_hi=0, capture_all=True, max_items=max_real).get_all() 28 | 29 | gen_features = metric_utils.compute_feature_stats_for_generator( 30 | opts=opts, detector_url=detector_url, detector_kwargs=detector_kwargs, 31 | rel_lo=0, rel_hi=1, capture_all=True, max_items=num_gen).get_all() 32 | 33 | if opts.rank != 0: 34 | return float('nan') 35 | 36 | n = real_features.shape[1] 37 | m = min(min(real_features.shape[0], gen_features.shape[0]), max_subset_size) 38 | t = 0 39 | for _subset_idx in range(num_subsets): 40 | x = gen_features[np.random.choice(gen_features.shape[0], m, replace=False)] 41 | y = real_features[np.random.choice(real_features.shape[0], m, replace=False)] 42 | a = (x @ x.T / n + 1) ** 3 + (y @ y.T / n + 1) ** 3 43 | b = (x @ y.T / n + 1) ** 3 44 | t += (a.sum() - np.diag(a).sum()) / (m - 1) - b.sum() * 2 / m 45 | kid = t / num_subsets / m 46 | return float(kid) 47 | 48 | #---------------------------------------------------------------------------- 49 | -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/ffhq/crop_images_in_the_wild.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import argparse 12 | import os 13 | from preprocess import align_img 14 | from PIL import Image 15 | import numpy as np 16 | import sys 17 | sys.path.append('Deep3DFaceRecon_pytorch') 18 | from Deep3DFaceRecon_pytorch.util.load_mats import load_lm3d 19 | 20 | 21 | if __name__ == '__main__': 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('--indir', type=str, required=True) 24 | args = parser.parse_args() 25 | 26 | lm_dir = os.path.join(args.indir, "detections") 27 | img_files = sorted([x for x in os.listdir(args.indir) if x.lower().endswith(".png") or x.lower().endswith(".jpg")]) 28 | lm_files = sorted([x for x in os.listdir(lm_dir) if x.endswith(".txt")]) 29 | 30 | lm3d_std = load_lm3d("Deep3DFaceRecon_pytorch/BFM/") 31 | 32 | out_dir = os.path.join(args.indir, "crop") 33 | if not os.path.exists(out_dir): 34 | os.makedirs(out_dir, exist_ok=True) 35 | 36 | for img_file, lm_file in zip(img_files, lm_files): 37 | 38 | img_path = os.path.join(args.indir, img_file) 39 | lm_path = os.path.join(lm_dir, lm_file) 40 | im = Image.open(img_path).convert('RGB') 41 | _,H = im.size 42 | lm = np.loadtxt(lm_path).astype(np.float32) 43 | lm = lm.reshape([-1, 2]) 44 | lm[:, -1] = H - 1 - lm[:, -1] 45 | 46 | target_size = 1024. 47 | rescale_factor = 300 48 | center_crop_size = 700 49 | output_size = 512 50 | 51 | _, im_high, _, _, = align_img(im, lm, lm3d_std, target_size=target_size, rescale_factor=rescale_factor) 52 | 53 | left = int(im_high.size[0]/2 - center_crop_size/2) 54 | upper = int(im_high.size[1]/2 - center_crop_size/2) 55 | right = left + center_crop_size 56 | lower = upper + center_crop_size 57 | im_cropped = im_high.crop((left, upper, right,lower)) 58 | im_cropped = im_cropped.resize((output_size, output_size), resample=Image.LANCZOS) 59 | out_path = os.path.join(out_dir, img_file.split(".")[0] + ".png") 60 | im_cropped.save(out_path) -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/ffhq/3dface2idr_mat.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | 12 | import numpy as np 13 | import os 14 | import torch 15 | import json 16 | import argparse 17 | import scipy.io 18 | import sys 19 | sys.path.append('Deep3DFaceRecon_pytorch') 20 | from Deep3DFaceRecon_pytorch.models.bfm import ParametricFaceModel 21 | 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('--in_root', type=str, default="", help='process folder') 24 | parser.add_argument('--out_path', type=str, default="cameras.json", help='output filename') 25 | args = parser.parse_args() 26 | in_root = args.in_root 27 | 28 | npys = sorted([x for x in os.listdir(in_root) if x.endswith(".mat")]) 29 | 30 | mode = 1 31 | outAll={} 32 | 33 | face_model = ParametricFaceModel(bfm_folder='Deep3DFaceRecon_pytorch/BFM') 34 | 35 | for src_filename in npys: 36 | src = os.path.join(in_root, src_filename) 37 | 38 | dict_load = scipy.io.loadmat(src) 39 | angle = dict_load['angle'] 40 | trans = dict_load['trans'][0] 41 | R = face_model.compute_rotation(torch.from_numpy(angle))[0].numpy() 42 | trans[2] += -10 43 | c = -np.dot(R, trans) 44 | pose = np.eye(4) 45 | pose[:3, :3] = R 46 | 47 | c *= 0.27 # normalize camera radius 48 | c[1] += 0.006 # additional offset used in submission 49 | c[2] += 0.161 # additional offset used in submission 50 | pose[0,3] = c[0] 51 | pose[1,3] = c[1] 52 | pose[2,3] = c[2] 53 | 54 | focal = 2985.29 # = 1015*1024/224*(300/466.285)# 55 | pp = 512#112 56 | w = 1024#224 57 | h = 1024#224 58 | 59 | count = 0 60 | K = np.eye(3) 61 | K[0][0] = focal 62 | K[1][1] = focal 63 | K[0][2] = w/2.0 64 | K[1][2] = h/2.0 65 | K = K.tolist() 66 | 67 | Rot = np.eye(3) 68 | Rot[0, 0] = 1 69 | Rot[1, 1] = -1 70 | Rot[2, 2] = -1 71 | pose[:3, :3] = np.dot(pose[:3, :3], Rot) 72 | 73 | pose = pose.tolist() 74 | out = {} 75 | out["intrinsics"] = K 76 | out["pose"] = pose 77 | outAll[src_filename.replace(".mat", ".png")] = out 78 | 79 | 80 | with open(args.out_path, "w") as outfile: 81 | json.dump(outAll, outfile) 82 | -------------------------------------------------------------------------------- /eg3d/dataset_preprocessing/ffhq/validate_ffhq.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | """ 12 | Usage: python validate_ffhq.py 13 | 14 | Checks in-the-wild images to verify images are complete and uncorrupted. Deletes files that 15 | failed check. After running this script, re-run download_ffhq.py to reacquire failed images. 16 | """ 17 | 18 | 19 | import json 20 | from PIL import Image 21 | import hashlib 22 | import numpy as np 23 | from tqdm import tqdm 24 | import os 25 | import argparse 26 | import sys 27 | 28 | if __name__ == '__main__': 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument('--dataset_json', type=str, default='ffhq-dataset-v2.json') 31 | parser.add_argument('--mode', type=str, default='file', choices=['file', 'pixel']) 32 | args = parser.parse_args() 33 | clean = True 34 | 35 | with open(args.dataset_json) as f: 36 | datasetjson = json.load(f) 37 | 38 | for key, val in tqdm(datasetjson.items()): 39 | file_spec = val['in_the_wild'] 40 | try: 41 | if args.mode == 'file': 42 | with open(file_spec['file_path'], 'rb') as file_to_check: 43 | data = file_to_check.read() 44 | if 'file_md5' in file_spec and hashlib.md5(data).hexdigest() != file_spec['file_md5']: 45 | raise IOError('Incorrect file MD5', file_spec['file_path']) 46 | elif args.mode == 'pixel': 47 | with Image.open(file_spec['file_path']) as image: 48 | if 'pixel_size' in file_spec and list(image.size) != file_spec['pixel_size']: 49 | raise IOError('Incorrect pixel size', file_spec['file_path']) 50 | if 'pixel_md5' in file_spec and hashlib.md5(np.array(image)).hexdigest() != file_spec['pixel_md5']: 51 | raise IOError('Incorrect pixel MD5', file_spec['file_path']) 52 | except IOError: 53 | clean = False 54 | tqdm.write(f"Bad file {file_spec['file_path']}") 55 | if os.path.isfile(file_spec['file_path']): 56 | os.remove(file_spec['file_path']) 57 | 58 | if not clean: 59 | sys.exit(1) -------------------------------------------------------------------------------- /eg3d/eg3d/viz/conditioning_pose_widget.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: LicenseRef-NvidiaProprietary 3 | # 4 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | # property and proprietary rights in and to this material, related 6 | # documentation and any modifications thereto. Any use, reproduction, 7 | # disclosure or distribution of this material and related documentation 8 | # without an express license agreement from NVIDIA CORPORATION or 9 | # its affiliates is strictly prohibited. 10 | 11 | import numpy as np 12 | import imgui 13 | import dnnlib 14 | from gui_utils import imgui_utils 15 | 16 | #---------------------------------------------------------------------------- 17 | 18 | class ConditioningPoseWidget: 19 | def __init__(self, viz): 20 | self.viz = viz 21 | self.pose = dnnlib.EasyDict(yaw=0, pitch=0, anim=False, speed=0.25) 22 | self.pose_def = dnnlib.EasyDict(self.pose) 23 | 24 | def drag(self, dx, dy): 25 | viz = self.viz 26 | self.pose.yaw += -dx / viz.font_size * 3e-2 27 | self.pose.pitch += -dy / viz.font_size * 3e-2 28 | 29 | @imgui_utils.scoped_by_object_id 30 | def __call__(self, show=True): 31 | viz = self.viz 32 | if show: 33 | imgui.text('Cond Pose') 34 | imgui.same_line(viz.label_w) 35 | yaw = self.pose.yaw 36 | pitch = self.pose.pitch 37 | with imgui_utils.item_width(viz.font_size * 5): 38 | changed, (new_yaw, new_pitch) = imgui.input_float2('##frac', yaw, pitch, format='%+.2f', flags=imgui.INPUT_TEXT_ENTER_RETURNS_TRUE) 39 | if changed: 40 | self.pose.yaw = new_yaw 41 | self.pose.pitch = new_pitch 42 | imgui.same_line(viz.label_w + viz.font_size * 13 + viz.spacing * 2) 43 | _clicked, dragging, dx, dy = imgui_utils.drag_button('Drag', width=viz.button_w) 44 | if dragging: 45 | self.drag(dx, dy) 46 | imgui.same_line() 47 | snapped = dnnlib.EasyDict(self.pose, yaw=round(self.pose.yaw, 1), pitch=round(self.pose.pitch, 1)) 48 | if imgui_utils.button('Snap', width=viz.button_w, enabled=(self.pose != snapped)): 49 | self.pose = snapped 50 | imgui.same_line() 51 | if imgui_utils.button('Reset', width=-1, enabled=(self.pose != self.pose_def)): 52 | self.pose = dnnlib.EasyDict(self.pose_def) 53 | 54 | viz.args.conditioning_yaw = self.pose.yaw 55 | viz.args.conditioning_pitch = self.pose.pitch 56 | 57 | #---------------------------------------------------------------------------- 58 | --------------------------------------------------------------------------------