├── .gitignore ├── CONTRIBUTING.md ├── License.txt ├── README.md ├── attribute └── M3DFEL │ ├── README.md │ ├── datasets │ ├── __init__.py │ ├── dataset_DFEW.py │ └── video_transform.py │ ├── figs │ ├── CVPR_architecture.png │ └── CVPR_result.png │ ├── main.py │ ├── models │ ├── M3DFEL.py │ └── __init__.py │ ├── options.py │ ├── requirements.txt │ ├── solver.py │ └── utils.py ├── generation └── uiface │ ├── README.md │ ├── configs │ ├── dataset │ │ └── CASIA_file.yaml │ ├── diffusion │ │ └── ddpm.yaml │ ├── model │ │ └── unet_cond_ca_cpd25_uncond20.yaml │ ├── sample_ddim_config.yaml │ └── train_config.yaml │ ├── diffusion │ ├── __init__.py │ └── ddpm.py │ ├── main.py │ ├── models │ ├── __init__.py │ ├── autoencoder │ │ ├── __init__.py │ │ ├── first_stage_config.yaml │ │ ├── modules.py │ │ ├── quantization.py │ │ └── vqgan.py │ └── diffusion │ │ ├── __init__.py │ │ ├── nn.py │ │ ├── unet.py │ │ └── util.py │ ├── pipeline.png │ ├── requirements.txt │ ├── sample.py │ └── utils │ ├── CASIA_dataset.py │ ├── __init__.py │ ├── checkpoint.py │ ├── colored.py │ ├── ema.py │ └── helpers.py ├── quality ├── README.md ├── backbone │ ├── __init__.py │ ├── loss.py │ └── metric.py ├── demo_imgs │ ├── 1.jpg │ ├── 2.jpg │ └── 3.jpg ├── docs │ ├── framework.png │ └── res.png ├── eval.py ├── generate_pseudo_labels │ ├── extract_embedding │ │ ├── config_test.py │ │ ├── dataset │ │ │ ├── __init__.py │ │ │ └── dataset_txt.py │ │ ├── extract_feats.py │ │ └── model │ │ │ ├── MobileFaceNet_MS1M.pth │ │ │ ├── model.py │ │ │ └── model_mobilefaceNet.py │ ├── gen_datalist.py │ └── gen_pseudo_labels.py ├── local_train.sh ├── train.py └── train_config.py ├── recognition ├── README.md ├── deploy │ ├── CMakeLists.txt │ ├── README.md │ ├── converter │ │ ├── convert_onnx2tnn.sh │ │ ├── export_onnx.py │ │ └── onnx_inference.py │ ├── core │ │ ├── face_feature.cpp │ │ ├── face_feature.h │ │ ├── timer.h │ │ └── util.h │ ├── data │ │ └── brucelee.jpg │ ├── scripts │ │ └── build_linux_x86.sh │ └── test │ │ └── test.cpp ├── doc │ ├── framework.png │ └── logo.png ├── local_train.sh ├── requirements.txt ├── tasks │ ├── dctdp │ │ ├── README.md │ │ ├── doc │ │ │ ├── main.png │ │ │ ├── result1.png │ │ │ ├── result2.png │ │ │ └── result3.png │ │ ├── local_train.sh │ │ ├── model_irse_dct.py │ │ ├── requirements.txt │ │ ├── train.py │ │ ├── train.yaml │ │ └── utils.py │ ├── duetface │ │ ├── README.md │ │ ├── base_task.py │ │ ├── duetface_model.py │ │ ├── local_backbones.py │ │ ├── pfld_model.py │ │ ├── pipeline.png │ │ ├── requirements.txt │ │ ├── train.py │ │ └── train.yaml │ ├── ekd │ │ ├── README.md │ │ ├── dataset.py │ │ ├── distillation │ │ │ ├── cc.py │ │ │ ├── cosine_distance.py │ │ │ ├── darkrank.py │ │ │ ├── ekd.py │ │ │ ├── fitnet.py │ │ │ ├── rkd.py │ │ │ ├── similarity_perserving.py │ │ │ └── simple_kd.py │ │ ├── doc │ │ │ ├── method.png │ │ │ ├── result1.png │ │ │ ├── result2.png │ │ │ └── result3.png │ │ ├── train.py │ │ └── train.yaml │ ├── minusface │ │ ├── README.md │ │ ├── __init__.py │ │ ├── local_train.sh │ │ ├── minusface.py │ │ ├── pipeline.png │ │ ├── requirements.txt │ │ ├── train.py │ │ ├── train.yaml │ │ └── utils.py │ ├── partialface │ │ ├── README.md │ │ ├── __init__.py │ │ ├── base_task.py │ │ ├── local_train.sh │ │ ├── pipeline.png │ │ ├── requirements.txt │ │ ├── train.py │ │ ├── train.yaml │ │ └── utils.py │ └── slerpface │ │ ├── .gitignore │ │ ├── encrypt_template.sh │ │ ├── encrypt_template │ │ └── enrollment.py │ │ ├── extract_template.sh │ │ ├── extract_template │ │ ├── utils.py │ │ └── verification.py │ │ ├── match_template.sh │ │ ├── match_template │ │ └── gen_sim.py │ │ ├── modules │ │ └── model.py │ │ ├── pipeline.png │ │ ├── readme.md │ │ └── requirements.txt ├── test │ ├── IJB_Evaluation.py │ ├── README.md │ ├── __init__.py │ ├── datasets.py │ ├── extract_features.py │ ├── utils.py │ ├── verification.py │ └── verification_rfw.py ├── tools │ ├── convert_new_index.py │ ├── decode.py │ └── img2tfrecord.py ├── torchkit │ ├── __init__.py │ ├── backbone │ │ ├── __init__.py │ │ ├── common.py │ │ ├── fbnets │ │ │ ├── fbnet_builder.py │ │ │ ├── fbnet_modeldef.json │ │ │ └── layers │ │ │ │ ├── __init__.py │ │ │ │ ├── batch_norm.py │ │ │ │ └── misc.py │ │ ├── model_efficientnet.py │ │ ├── model_ghostnet.py │ │ ├── model_irse.py │ │ ├── model_mobilefacenet.py │ │ └── model_resnet.py │ ├── data │ │ ├── __init__.py │ │ ├── dataset.py │ │ ├── example_pb2.py │ │ ├── feature_pb2.py │ │ ├── parser.py │ │ └── sampler.py │ ├── head │ │ ├── __init__.py │ │ ├── distfc │ │ │ ├── arcface.py │ │ │ ├── common.py │ │ │ ├── cosface.py │ │ │ ├── curricularface.py │ │ │ ├── normface.py │ │ │ └── partial_fc.py │ │ └── localfc │ │ │ ├── arcface.py │ │ │ ├── cifp.py │ │ │ ├── common.py │ │ │ ├── cosface.py │ │ │ └── curricularface.py │ ├── hooks │ │ ├── __init__.py │ │ ├── base_hook.py │ │ ├── checkpoint_hook.py │ │ ├── learning_rate_hook.py │ │ ├── log_hook.py │ │ └── summary_hook.py │ ├── loss │ │ ├── __init__.py │ │ ├── ddl.py │ │ └── dist_softmax.py │ ├── task │ │ ├── __init__.py │ │ └── base_task.py │ ├── tfrecord │ │ └── protos │ │ │ ├── example.proto │ │ │ └── feature.proto │ └── util │ │ ├── __init__.py │ │ ├── checkpoint.py │ │ ├── distributed_functions.py │ │ └── utils.py ├── train.py └── train.yaml └── security ├── README.md ├── common ├── __init__.py ├── data │ ├── __init__.py │ ├── base_dataloader.py │ └── base_transform.py ├── losses │ ├── __init__.py │ └── euclidean_loss.py ├── optimizers │ └── __init__.py ├── schedulers │ └── __init__.py ├── task │ ├── __init__.py │ ├── base_task.py │ └── fas │ │ ├── __init__.py │ │ └── modules.py └── utils │ ├── __init__.py │ ├── cli_utils.py │ ├── distribute_utils.py │ ├── face_utils.py │ ├── logger_utils.py │ ├── meters.py │ ├── metrics.py │ ├── misc.py │ ├── model_init.py │ └── parameters.py ├── doc ├── intro.png └── logo.png ├── requirements.txt └── tasks ├── Adv-Attack-Defense ├── Adv-Makeup │ ├── Poisson_Image_Editing │ │ └── poisson_image_editing_makeup.py │ ├── README.md │ ├── configs │ │ └── config.yaml │ ├── datasets │ │ ├── Datasets_Makeup │ │ │ └── landmark_aligned_600.txt │ │ └── dataset.py │ ├── doc │ │ ├── asr.png │ │ ├── logo.png │ │ └── physical_vis_pose.png │ ├── models │ │ ├── __init__.py │ │ ├── makeup_attack.py │ │ ├── networks.py │ │ └── utils.py │ ├── test.py │ └── train.py └── Sibling-Attack │ ├── README.md │ ├── attack.py │ ├── configs │ └── config.yaml │ ├── datasets │ ├── FFHQ-select-aligned-label.txt │ ├── celebaHQ-select-aligned-label.txt │ └── lfw-select-aligned-label.txt │ ├── doc │ ├── gradcam.png │ ├── main_fig.png │ ├── table2.png │ └── table3.png │ ├── models │ └── models.py │ └── utils │ └── utils.py ├── Face-Anti-Spoofing ├── ANRL │ ├── README.md │ ├── configs │ │ ├── ICM2O.yaml │ │ ├── IOM2C.yaml │ │ ├── OCI2M.yaml │ │ └── OCM2I.yaml │ ├── datasets │ │ ├── DG_dataset.py │ │ ├── __init__.py │ │ └── factory.py │ ├── doc │ │ ├── ROC.png │ │ ├── framework.png │ │ ├── results_table.png │ │ └── tSNE.png │ ├── models │ │ ├── __init__.py │ │ ├── base_block.py │ │ └── framework.py │ ├── train.py │ └── train.sh └── DCN │ ├── README.md │ ├── configs │ ├── Oulu_1.yaml │ ├── Oulu_2.yaml │ ├── Oulu_3_1.yaml │ └── Oulu_4_1.yaml │ ├── datasets │ ├── __init__.py │ ├── dataset.py │ ├── factory.py │ └── permutations_30.npy │ ├── doc │ ├── TSNE.png │ ├── table.png │ └── table_2.png │ ├── models │ ├── __init__.py │ ├── base_block.py │ └── framework.py │ ├── train.py │ └── train.sh └── Face-Forgery-Detection ├── DCL ├── README.md ├── configs │ ├── test.yaml │ └── train.yaml ├── datasets │ ├── __init__.py │ ├── celeb_df.py │ ├── factory.py │ ├── ffpp.py │ ├── utils │ │ ├── __init__.py │ │ ├── data_structure.py │ │ ├── random_patch.py │ │ └── srm.py │ └── wild_deepfake.py ├── doc │ └── main.png ├── models │ ├── __init__.py │ ├── base_net.py │ └── dcl.py ├── requirements.txt ├── test.py └── train.py └── STIL ├── README.md ├── configs └── ffpp.yaml ├── datasets ├── __init__.py ├── factory.py └── video_dataset.py ├── doc └── stil.jpg ├── models ├── __init__.py ├── ops.py └── stil.py ├── test.py ├── test.sh ├── train.py ├── train.sh └── utils ├── __init__.py ├── lr_utils.py └── metrics.py /.gitignore: -------------------------------------------------------------------------------- 1 | */__pycache__ 2 | *.pyc 3 | *.log 4 | logs/ 5 | .vscode/ 6 | .idea/ 7 | *.swp 8 | rpf.yaml 9 | *.bak 10 | build/ 11 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Request for contributions 2 | 3 | Please contribute to this repository if any of the following is true: 4 | - You have expertise in community development, communication, or education 5 | - You want open source communities to be more collaborative and inclusive 6 | - You want to help lower the burden to first time contributors 7 | 8 | # How to contribute 9 | 10 | Prerequisites: 11 | 12 | - Familiarity with [pull requests](https://help.github.com/articles/using-pull-requests) and [issues](https://guides.github.com/features/issues/). 13 | - Knowledge of [Markdown](https://help.github.com/articles/markdown-basics/) for editing `.md` documents. 14 | 15 | Please make sure the following conditions are met before raising the issue： 16 | 17 | - It must be a bug or a new function 18 | - It must be a TFace related issue. 19 | - You have been searched in issues, and no similar issue or solution has been found. -------------------------------------------------------------------------------- /attribute/M3DFEL/README.md: -------------------------------------------------------------------------------- 1 | # M3DFEL 2 | Official code for [Rethinking the Learning Paradigm for Dynamic Facial Expression Recognition](https://openaccess.thecvf.com/content/CVPR2023/papers/Wang_Rethinking_the_Learning_Paradigm_for_Dynamic_Facial_Expression_Recognition_CVPR_2023_paper.pdf) (accepted by CVPR2023). 3 | 4 | We propose a weakly supervised approach to model Dynamic Facial Expression Recognition (DFER) as a Multi-Instance Learning (MIL) problem. 5 | 6 | The Multi-3D Dynamic Facial Expression Learning (M3DFEL) framework provides a unified solution to the weakly supervised problem and model the imbalanced short- and long-term temporal relationships in DFER. 7 | 8 | 9 | 10 | ## Results on DFEW 11 | 12 | 13 | 14 | ## Data Preparation 15 | 16 | Download [DFEW](https://dfew-dataset.github.io/) and unzip the files to make sure: 17 | 18 | ``` 19 | . 20 | ├── Clip 21 | │ └── clip_224x224 22 | │ ├── 00001 23 | │ ├── 00002 24 | │ └── ... 25 | └── EmoLabel_DataSplit 26 | ├── test(single-labeled) 27 | └── train(single-labeled) 28 | ``` 29 | 30 | ## Installation 31 | ``` 32 | pip install -r requirements.txt 33 | ``` 34 | 35 | 36 | ## Training 37 | ``` 38 | python main.py --data_root path-to-DFEW 39 | ``` 40 | 41 | ## Citation 42 | If you find our work useful in your research, please consider citing: 43 | ``` 44 | @inproceedings{wang2023rethinking, 45 | title={Rethinking the Learning Paradigm for Dynamic Facial Expression Recognition}, 46 | author={Wang, Hanyang and Li, Bo and Wu, Shuang and Shen, Siyuan and Liu, Feng and Ding, Shouhong and Zhou, Aimin}, 47 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 48 | pages={17958--17968}, 49 | year={2023} 50 | } 51 | ``` -------------------------------------------------------------------------------- /attribute/M3DFEL/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .dataset_DFEW import DFEWDataset 2 | import torch 3 | 4 | 5 | def create_dataloader(args, mode): 6 | """create dataloader according to args and training/testing mode 7 | 8 | Args: 9 | args 10 | mode: String("train" or "test") 11 | 12 | Returns: 13 | dataloader 14 | """ 15 | dataset = DFEWDataset(args, mode) 16 | 17 | dataloader = None 18 | 19 | # return train_dataset or test_dataset according to the mode 20 | if mode == "train": 21 | dataloader = torch.utils.data.DataLoader(dataset, 22 | batch_size=args.batch_size, 23 | shuffle=True, 24 | num_workers=args.workers, 25 | pin_memory=True, 26 | drop_last=True) 27 | elif mode == "test": 28 | dataloader = torch.utils.data.DataLoader(dataset, 29 | batch_size=args.batch_size, 30 | shuffle=False, 31 | num_workers=args.workers, 32 | pin_memory=True) 33 | return dataloader 34 | -------------------------------------------------------------------------------- /attribute/M3DFEL/figs/CVPR_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/attribute/M3DFEL/figs/CVPR_architecture.png -------------------------------------------------------------------------------- /attribute/M3DFEL/figs/CVPR_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/attribute/M3DFEL/figs/CVPR_result.png -------------------------------------------------------------------------------- /attribute/M3DFEL/main.py: -------------------------------------------------------------------------------- 1 | from options import Options 2 | from solver import Solver 3 | 4 | 5 | def main(): 6 | """Run the whole training process through solver 7 | Change the options according to your situation, especially workers, gpu_ids, batch_size and epochs 8 | """ 9 | args = Options().parse() 10 | solver = Solver(args) 11 | solver.run() 12 | 13 | 14 | if __name__ == '__main__': 15 | main() 16 | -------------------------------------------------------------------------------- /attribute/M3DFEL/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .M3DFEL import M3DFEL 2 | 3 | 4 | def create_model(args): 5 | """create model according to args 6 | 7 | Args: 8 | args 9 | """ 10 | model = M3DFEL(args) 11 | 12 | return model 13 | -------------------------------------------------------------------------------- /attribute/M3DFEL/requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | timm 3 | einops 4 | sklearn 5 | seaborn 6 | matplotlib 7 | numpy 8 | torchvision 9 | pillow -------------------------------------------------------------------------------- /attribute/M3DFEL/utils.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | from timm.scheduler.cosine_lr import CosineLRScheduler 4 | from einops import rearrange 5 | 6 | 7 | def build_scheduler(args, optimizer, n_iter_per_epoch): 8 | """Build the scheduler 9 | The CosineLRScheduler schedules every iter in every epoch, so it needs n_iter_per_epoch instead of epochs 10 | 11 | Args: 12 | optimizer 13 | n_iter_per_epoch: 14 | """ 15 | num_steps = int(args.epochs * n_iter_per_epoch) 16 | warmup_steps = int(args.warmup_epochs * n_iter_per_epoch) 17 | 18 | # use the Cosine LR Scheduler for training 19 | lr_scheduler = CosineLRScheduler( 20 | optimizer, 21 | t_initial=num_steps, 22 | lr_min=args.min_lr, 23 | warmup_lr_init=args.warmup_lr, 24 | warmup_t=warmup_steps, 25 | cycle_limit=1, 26 | t_in_epochs=False, 27 | ) 28 | 29 | return lr_scheduler 30 | 31 | 32 | class DMIN(nn.Module): 33 | """The Dynamic Multi Instance Normalization Module 34 | This module dynamicly normlizes the instances and bags 35 | 36 | Args: 37 | optimizer 38 | n_iter_per_epoch: 39 | """ 40 | 41 | def __init__(self, num_features, eps=1e-5, momentum=0.9): 42 | super(DMIN, self).__init__() 43 | 44 | # init the DMIN module 45 | self.eps = eps 46 | self.momentum = momentum 47 | self.weight = nn.Parameter(torch.ones(1, num_features, 1)) 48 | self.bias = nn.Parameter(torch.zeros(1, num_features, 1)) 49 | self.mean_weight = nn.Parameter(torch.ones(2)) 50 | self.var_weight = nn.Parameter(torch.ones(2)) 51 | 52 | self.weight.data.fill_(1) 53 | self.bias.data.zero_() 54 | 55 | def forward(self, x): 56 | 57 | x = rearrange(x, 'b c n -> b n c') 58 | # calculate the mean and var of input 59 | mean_in = x.mean(-1, keepdim=True) 60 | var_in = x.var(-1, keepdim=True) 61 | 62 | mean_ln = mean_in.mean(1, keepdim=True) 63 | temp = var_in + mean_in ** 2 64 | var_ln = temp.mean(1, keepdim=True) - mean_ln ** 2 65 | 66 | softmax = nn.Softmax(0) 67 | mean_weight = softmax(self.mean_weight) 68 | var_weight = softmax(self.var_weight) 69 | 70 | # caculated the weighted mean and var according to the learned weights 71 | mean = mean_weight[0] * mean_in + mean_weight[1] * mean_ln 72 | var = var_weight[0] * var_in + var_weight[1] * var_ln 73 | 74 | # regulize x according to the calculated results 75 | x = (x-mean) / (var+self.eps).sqrt() 76 | x = x * self.weight + self.bias 77 | x = rearrange(x, 'b n c -> b c n') 78 | 79 | return x 80 | -------------------------------------------------------------------------------- /generation/uiface/configs/dataset/CASIA_file.yaml: -------------------------------------------------------------------------------- 1 | constants: 2 | input_channels: 3 3 | n_classes: 0 4 | image_size: 128 5 | 6 | latent_diffusion: True 7 | 8 | dataset: 9 | _target_: utils.CASIA_dataset.SamplesWithEmbeddingsFileDataset 10 | 11 | samples_root: " " 12 | embeddings_file_path: "dataset/elastic_embed.npy" 13 | images_name_file_path: "dataset/image_names.pkl" 14 | sample_file_ending: ".jpg" 15 | sample_loader: 16 | _target_: utils.CASIA_dataset.PILImageLoader 17 | sample_transform: 18 | _target_: torchvision.transforms.Compose 19 | transforms: 20 | - _target_: torchvision.transforms.Resize 21 | size: 22 | - 128 23 | - 128 24 | - _target_: torchvision.transforms.ToTensor 25 | - _target_: torchvision.transforms.RandomHorizontalFlip 26 | p: 0.5 27 | 28 | -------------------------------------------------------------------------------- /generation/uiface/configs/diffusion/ddpm.yaml: -------------------------------------------------------------------------------- 1 | _target_: diffusion.ddpm.DenoisingDiffusionProbabilisticModel 2 | eps_model: ${model} 3 | T: 1000 4 | 5 | schedule_type: "linear" # ['linear', 'cosine', 'cosine_warped] 6 | 7 | schedule_k: 1 8 | schedule_beta_min: 0.000 # if this is too high, noise is added too fast 9 | schedule_beta_max: 0.100 # if this is too high, the cosine schedule does not work 10 | 11 | criterion: 12 | _target_: torch.nn.MSELoss -------------------------------------------------------------------------------- /generation/uiface/configs/model/unet_cond_ca_cpd25_uncond20.yaml: -------------------------------------------------------------------------------- 1 | # PARAMS: 216341187 2 | _target_: models.diffusion.unet.ConditionalUNet 3 | input_channels: 3 4 | initial_channels: 96 5 | channel_multipliers: 6 | - 1 7 | - 2 8 | - 2 9 | - 2 10 | is_attention: 11 | - false 12 | - true 13 | - true 14 | - true 15 | attention_heads: -1 16 | attention_head_channels: 32 17 | n_blocks_per_resolution: 2 18 | 19 | condition_type: "CA" 20 | is_context_conditional: True 21 | n_context_classes: 0 22 | context_input_channels: 512 23 | context_channels: 256 24 | learn_empty_context: True # add learnable embedding to the model for empty context 25 | context_dropout_probability: 0.25 26 | unconditioned_probability: 0.2 27 | 28 | -------------------------------------------------------------------------------- /generation/uiface/configs/sample_ddim_config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - _self_ 3 | - override hydra/hydra_logging: disabled 4 | - override hydra/job_logging: disabled 5 | checkpoint: 6 | path: " " 7 | global_step: null 8 | use_non_ema: True 9 | 10 | diffusion_cfg_path: "./configs/model/unet_cond_ca_cpd25_uncond20.yaml" 11 | 12 | VQEncoder_path: "" 13 | VQDecoder_path: "" 14 | 15 | sampling: 16 | gpu_idx: 0 17 | seed: 41 18 | contexts_file : "" 19 | save_dir: " " 20 | save_mode: rgb 21 | batch_size: 50 22 | n_contexts: 10000 23 | num_workers: 1 24 | pin_memory: true 25 | n_samples_per_context: 50 26 | sample_config: 27 | cfg_scale: 1.0 28 | MSE_th: 0.006 29 | step_th: 500 30 | 31 | hydra: 32 | output_subdir: null 33 | run: 34 | dir: . 35 | -------------------------------------------------------------------------------- /generation/uiface/configs/train_config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - _self_ 3 | - diffusion: ddpm 4 | - model: unet_cond_ca_cpd25_uncond20 5 | - dataset: CASIA_file 6 | 7 | latent_diffusion: true 8 | 9 | constants: 10 | seed: 0 11 | image_size: -1 12 | input_channels: -1 13 | n_classes: 0 14 | 15 | training: 16 | sample_weight_path: null 17 | precision: 16 18 | batch_size: 64 19 | num_workers: 8 20 | pin_memory: true 21 | steps: 900_000 22 | steps_between_sampling: 10000 23 | steps_between_eval: 50000 24 | steps_between_logging: 500 25 | steps_of_checkpoints: 26 | - 100_000 27 | - 250_000 28 | - 400_000 29 | - 600_000 30 | - 800_000 31 | - 900_000 32 | context_dropout: 0 33 | context_permutation: 0.0 34 | ema: 35 | _partial_: true 36 | _target_: utils.ema.EMAModel 37 | inv_gamma: 1.0 38 | power: 0.75 39 | min_value: 0.0 40 | max_value: 0.9999 41 | optimizer: 42 | _partial_: true 43 | _target_: torch.optim.Adam 44 | lr: 1e-4 45 | lr_scheduler: 46 | _partial_: true 47 | _target_: torch.optim.lr_scheduler.CosineAnnealingWarmRestarts 48 | T_0: 10_000 49 | T_mult: 2 50 | checkpoint: 51 | restore: false 52 | path: "" 53 | VQEncoder: "" 54 | VQDecoder: "" 55 | 56 | hydra: 57 | job: 58 | chdir: true 59 | run: 60 | dir: ./logs/${now:%Y-%m-%d}/${now:%H-%M-%S} -------------------------------------------------------------------------------- /generation/uiface/diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/generation/uiface/diffusion/__init__.py -------------------------------------------------------------------------------- /generation/uiface/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/generation/uiface/models/__init__.py -------------------------------------------------------------------------------- /generation/uiface/models/autoencoder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/generation/uiface/models/autoencoder/__init__.py -------------------------------------------------------------------------------- /generation/uiface/models/autoencoder/first_stage_config.yaml: -------------------------------------------------------------------------------- 1 | target: ldm.models.autoencoder.VQModelInterface 2 | params: 3 | embed_dim: 3 4 | n_embed: 8192 5 | ddconfig: 6 | double_z: false 7 | z_channels: 3 8 | resolution: 256 9 | in_channels: 3 10 | out_ch: 3 11 | ch: 128 12 | ch_mult: 13 | - 1 14 | - 2 15 | - 4 16 | num_res_blocks: 2 17 | attn_resolutions: [] 18 | dropout: 0.0 19 | lossconfig: 20 | target: torch.nn.Identity 21 | -------------------------------------------------------------------------------- /generation/uiface/models/autoencoder/vqgan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from models.autoencoder.modules import Decoder, Encoder 3 | from models.autoencoder.quantization import VectorQuantizer2 as VectorQuantizer 4 | from omegaconf import OmegaConf 5 | 6 | 7 | class VQEncoderInterface(torch.nn.Module): 8 | 9 | def __init__(self, first_stage_config_path: str, encoder_state_dict_path: str): 10 | super().__init__() 11 | 12 | embed_dim = 3 13 | 14 | config = OmegaConf.load(first_stage_config_path) 15 | dd_config = config.params.ddconfig 16 | 17 | self.encoder = Encoder(**dd_config) 18 | self.quant_conv = torch.nn.Conv2d(dd_config["z_channels"], embed_dim, 1) 19 | 20 | state_dict = torch.load(encoder_state_dict_path) 21 | self.load_state_dict(state_dict) 22 | 23 | def forward(self, x): 24 | h = self.encoder(x) 25 | h = self.quant_conv(h) 26 | return h 27 | 28 | 29 | class VQDecoderInterface(torch.nn.Module): 30 | 31 | def __init__(self, first_stage_config_path: str, decoder_state_dict_path: str): 32 | super().__init__() 33 | 34 | embed_dim = 3 35 | n_embed = 8192 36 | 37 | config = OmegaConf.load(first_stage_config_path) 38 | dd_config = config.params.ddconfig 39 | 40 | self.decoder = Decoder(**dd_config) 41 | 42 | self.quantize = VectorQuantizer(n_embed, embed_dim, beta=0.25) 43 | self.post_quant_conv = torch.nn.Conv2d(embed_dim, dd_config["z_channels"], 1) 44 | 45 | state_dict = torch.load(decoder_state_dict_path) 46 | self.load_state_dict(state_dict) 47 | 48 | def forward(self, h, force_not_quantize=False): 49 | # also go through quantization layer 50 | if not force_not_quantize: 51 | quant, emb_loss, info = self.quantize(h) 52 | else: 53 | quant = h 54 | quant = self.post_quant_conv(quant) 55 | dec = self.decoder(quant) 56 | return dec 57 | -------------------------------------------------------------------------------- /generation/uiface/models/diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/generation/uiface/models/diffusion/__init__.py -------------------------------------------------------------------------------- /generation/uiface/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/generation/uiface/pipeline.png -------------------------------------------------------------------------------- /generation/uiface/requirements.txt: -------------------------------------------------------------------------------- 1 | einops==0.6.1 2 | hydra-core 3 | matplotlib==3.7.2 4 | numpy==1.24.4 5 | omegaconf==2.3.0 6 | pandas==2.0.3 7 | pyeer==0.5.5 8 | pytorch_lightning==1.5.9 9 | Requests==2.31.0 10 | scikit_learn==1.3.0 11 | scipy 12 | seaborn==0.12.2 13 | scikit-image==0.21.0 14 | tqdm==4.66.1 15 | -------------------------------------------------------------------------------- /generation/uiface/utils/CASIA_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | from typing import Callable 4 | 5 | import numpy as np 6 | import torch 7 | from PIL import Image 8 | from torchvision.datasets import DatasetFolder 9 | 10 | 11 | class PILImageLoader: 12 | 13 | def __call__(self, path): 14 | with open(path, "rb") as f: 15 | img = Image.open(f) 16 | return img.convert("RGB") 17 | 18 | 19 | class SamplesWithEmbeddingsFileDataset(torch.utils.data.Dataset): 20 | 21 | def __init__( 22 | self, 23 | samples_root: str, 24 | embeddings_file_path: str, 25 | images_name_file_path: str, 26 | sample_file_ending: str = ".jpg", 27 | sample_loader: Callable = None, 28 | sample_transform: Callable = None, 29 | ): 30 | super(SamplesWithEmbeddingsFileDataset, self).__init__() 31 | 32 | self.sample_loader = sample_loader 33 | self.transform = sample_transform 34 | self.embeddings_file_path = embeddings_file_path 35 | self.samples_root = samples_root 36 | self.samples = self.build_samples( 37 | embeddings_file_path=embeddings_file_path, 38 | images_name_file_path=images_name_file_path, 39 | sample_root=samples_root, 40 | sample_file_ending=sample_file_ending, 41 | ) 42 | print("Total images:") 43 | print(len(self.samples)) 44 | 45 | @staticmethod 46 | def build_samples( 47 | embeddings_file_path: str, 48 | images_name_file_path: str, 49 | sample_root: str, 50 | sample_file_ending: str, 51 | ): 52 | # normed templates: (n, 512) 53 | content = np.load(embeddings_file_path) 54 | print("CASIA contexts file loaded") 55 | 56 | with open(images_name_file_path, "rb") as f: 57 | image_names = pickle.load(f) 58 | print("CASIA file names file loaded") 59 | 60 | samples = [] 61 | total_images = len(image_names) 62 | for index in range(total_images): 63 | embed = content[index] 64 | image_path = os.path.join(sample_root, image_names[index]) 65 | samples.append((image_path, embed)) 66 | return samples 67 | 68 | def __getitem__(self, index: int): 69 | image_path, embedding_npy = self.samples[index] 70 | image = self.sample_loader(image_path) 71 | embedding = torch.from_numpy(embedding_npy) 72 | if self.transform is not None: 73 | image = self.transform(image) 74 | return image, embedding 75 | 76 | def __len__(self): 77 | return len(self.samples) 78 | -------------------------------------------------------------------------------- /generation/uiface/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/generation/uiface/utils/__init__.py -------------------------------------------------------------------------------- /generation/uiface/utils/checkpoint.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def checkpoint(func, inputs, params, flag): 5 | """ 6 | Evaluate a function without caching intermediate activations, allowing for 7 | reduced memory at the expense of extra compute in the backward pass. 8 | :param func: the function to evaluate. 9 | :param inputs: the argument sequence to pass to `func`. 10 | :param params: a sequence of parameters `func` depends on but does not 11 | explicitly take as arguments. 12 | :param flag: if False, disable gradient checkpointing. 13 | """ 14 | if flag: 15 | args = tuple(inputs) + tuple(params) 16 | return CheckpointFunction.apply(func, len(inputs), *args) 17 | else: 18 | return func(*inputs) 19 | 20 | 21 | class CheckpointFunction(torch.autograd.Function): 22 | @staticmethod 23 | def forward(ctx, run_function, length, *args): 24 | ctx.run_function = run_function 25 | ctx.input_tensors = list(args[:length]) 26 | ctx.input_params = list(args[length:]) 27 | 28 | with torch.no_grad(): 29 | output_tensors = ctx.run_function(*ctx.input_tensors) 30 | return output_tensors 31 | 32 | @staticmethod 33 | def backward(ctx, *output_grads): 34 | ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors] 35 | with torch.enable_grad(): 36 | shallow_copies = [x.view_as(x) for x in ctx.input_tensors] 37 | output_tensors = ctx.run_function(*shallow_copies) 38 | input_grads = torch.autograd.grad( 39 | output_tensors, 40 | ctx.input_tensors + ctx.input_params, 41 | output_grads, 42 | allow_unused=True, 43 | ) 44 | del ctx.input_tensors 45 | del ctx.input_params 46 | del output_tensors 47 | return (None, None) + input_grads 48 | -------------------------------------------------------------------------------- /generation/uiface/utils/colored.py: -------------------------------------------------------------------------------- 1 | # https://github.com/hfeeki/termcolor/blob/master/termcolor.py 2 | 3 | import os 4 | import re 5 | 6 | ATTRIBUTES = dict( 7 | list( 8 | zip( 9 | ["bold", "dark", "", "underline", "blink", "", "reverse", "concealed"], 10 | list(range(1, 9)), 11 | ) 12 | ) 13 | ) 14 | del ATTRIBUTES[""] 15 | 16 | ATTRIBUTES_RE = "\033\\[(?:%s)m" % "|".join(["%d" % v for v in ATTRIBUTES.values()]) 17 | 18 | HIGHLIGHTS = dict( 19 | list( 20 | zip( 21 | [ 22 | "on_grey", 23 | "on_red", 24 | "on_green", 25 | "on_yellow", 26 | "on_blue", 27 | "on_magenta", 28 | "on_cyan", 29 | "on_white", 30 | ], 31 | list(range(40, 48)), 32 | ) 33 | ) 34 | ) 35 | 36 | HIGHLIGHTS_RE = "\033\\[(?:%s)m" % "|".join(["%d" % v for v in HIGHLIGHTS.values()]) 37 | 38 | COLORS = dict( 39 | list( 40 | zip( 41 | [ 42 | "grey", 43 | "red", 44 | "green", 45 | "yellow", 46 | "blue", 47 | "magenta", 48 | "cyan", 49 | "white", 50 | ], 51 | list(range(30, 38)), 52 | ) 53 | ) 54 | ) 55 | 56 | COLORS_RE = "\033\\[(?:%s)m" % "|".join(["%d" % v for v in COLORS.values()]) 57 | 58 | RESET = "\033[0m" 59 | RESET_RE = "\033\\[0m" 60 | 61 | 62 | def colored(text, color=None, on_color=None, attrs=None): 63 | if os.getenv("ANSI_COLORS_DISABLED") is None: 64 | fmt_str = "\033[%dm%s" 65 | if color is not None: 66 | text = re.sub(COLORS_RE + "(.*?)" + RESET_RE, r"\1", text) 67 | text = fmt_str % (COLORS[color], text) 68 | if on_color is not None: 69 | text = re.sub(HIGHLIGHTS_RE + "(.*?)" + RESET_RE, r"\1", text) 70 | text = fmt_str % (HIGHLIGHTS[on_color], text) 71 | if attrs is not None: 72 | text = re.sub(ATTRIBUTES_RE + "(.*?)" + RESET_RE, r"\1", text) 73 | for attr in attrs: 74 | text = fmt_str % (ATTRIBUTES[attr], text) 75 | return text + RESET 76 | else: 77 | return text 78 | -------------------------------------------------------------------------------- /quality/README.md: -------------------------------------------------------------------------------- 1 | # SDD-FIQA: Unsupervised Face Image Quality Assessment with Similarity Distribution Distance 2 | 3 | ## Introduction 4 | #### 5 | In recent years, Face Image Quality Assessment (FIQA) has become an indispensable part of the face recognition system to guarantee the stability and reliability of recognition performance in an unconstrained scenario. In this work, we argue that a high-quality face image should be similar to its intra-class samples and dissimilar to its inter-class samples. Thus, we propose a novel unsupervised FIQA method that incorporates Similarity Distribution Distance for Face Image Quality Assessment (SDD-FIQA). Our method generates quality pseudo-labels by calculating the Wasserstein Distance (WD) between the intra-class and inter-class similarity distributions. With these quality pseudo-labels, we are capable of training a regression network for quality prediction. Extensive experiments on benchmark datasets demonstrate that the proposed SDD-FIQA surpasses the state-of-the-arts by an impressive margin. Meanwhile, our method shows good generalization across different recognition systems. 6 | 7 | 8 | 9 | ## Generation of Quality Pseudo-Labels 10 | 2. Run './generate_pseudo_labels/gen_datalist.py' to obtain the data list. 11 | 3. Run './generate_pseudo_labels/extract_embedding/extract_feats.py' to extract the embeddings of face image. 12 | 4. Run './generate_pseudo_labels/gen_pseudo_labels.py' to calculate the quality pseudo-labels. 13 | 14 | 15 | ## Training of Quality Regression Model 16 | 1. Replace the data path with your local path on `train_confing.py`. 17 | 2. Run local_train.sh. 18 | 19 | ## Prediction of FIQA 20 | Run './eval.py' to predict face quality score. 21 | We provide the pre-trained model on the refined MS1M dataset with IR50: [googledrive](https://drive.google.com/file/d/1AM0iWVfSVWRjCriwZZ3FXiUGbcDzkF25/view?usp=sharing) 22 | 23 | ## Results 24 | 25 | 26 | ## Citing this Repository 27 | If you find this code useful in your research, please consider citing us: 28 | ``` 29 | @InProceedings{SDD-FIQA2021, 30 | author={Ou, Fu-Zhao and Chen, Xingyu and Zhang, Ruixin and Huang, Yuge and Li, Shaoxin and Li, Jilin and Li, Yong and Cao, Liujuan and Wang, Yuan-Gen}, 31 | title = {{SDD-FIQA}: Unsupervised Face Image Quality Assessment with Similarity Distribution Distance}, 32 | booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 33 | year = {2021}, 34 | } 35 | 36 | ``` 37 | -------------------------------------------------------------------------------- /quality/backbone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/backbone/__init__.py -------------------------------------------------------------------------------- /quality/backbone/loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | # focal loss for training recognition model 6 | class FocalLoss(nn.Module): 7 | ''' 8 | focal loss for training recognition model 9 | ''' 10 | def __init__(self, gamma=0, eps=1e-7): 11 | super(FocalLoss, self).__init__() 12 | self.gamma = gamma 13 | self.eps = eps 14 | self.ce = torch.nn.CrossEntropyLoss() 15 | def forward(self, input, target): 16 | logp = self.ce(input, target) 17 | p = torch.exp(-logp) 18 | loss = (1 - p) ** self.gamma * logp 19 | return loss.mean() 20 | -------------------------------------------------------------------------------- /quality/demo_imgs/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/demo_imgs/1.jpg -------------------------------------------------------------------------------- /quality/demo_imgs/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/demo_imgs/2.jpg -------------------------------------------------------------------------------- /quality/demo_imgs/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/demo_imgs/3.jpg -------------------------------------------------------------------------------- /quality/docs/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/docs/framework.png -------------------------------------------------------------------------------- /quality/docs/res.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/docs/res.png -------------------------------------------------------------------------------- /quality/eval.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as opti 6 | from tqdm import tqdm 7 | import torchvision.transforms as T 8 | from generate_pseudo_labels.extract_embedding.model import model 9 | import numpy as np 10 | from scipy import stats 11 | import pdb 12 | from PIL import Image 13 | 14 | 15 | def read_img(imgPath): # read image & data pre-process 16 | data = torch.randn(1, 3, 112, 112) 17 | transform = T.Compose([ 18 | T.Resize((112, 112)), 19 | T.ToTensor(), 20 | T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), 21 | ]) 22 | img = Image.open(imgPath).convert("RGB") 23 | data[0, :, :, :] = transform(img) 24 | return data 25 | 26 | 27 | def network(eval_model, device): 28 | net = model.R50([112, 112], use_type="Qua").to(device) 29 | net_dict = net.state_dict() 30 | data_dict = { 31 | key.replace('module.', ''): value for key, value in torch.load(eval_model, map_location=device).items()} 32 | net_dict.update(data_dict) 33 | net.load_state_dict(net_dict) 34 | net.eval() 35 | return net 36 | 37 | if __name__ == "__main__": 38 | imgpath = './demo_imgs/1.jpg' # [1,2,3.jpg] 39 | device = 'cpu' # 'cpu' or 'cuda:x' 40 | eval_model = './model/SDD_FIQA_checkpoints_r50.pth' # checkpoint 41 | net = network(eval_model, device) 42 | input_data = read_img(imgpath) 43 | pred_score = net(input_data).data.cpu().numpy().squeeze() 44 | print(f"Quality score = {pred_score}") 45 | -------------------------------------------------------------------------------- /quality/generate_pseudo_labels/extract_embedding/config_test.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as T 3 | 4 | 5 | class Config: 6 | # dataset 7 | data_root = '' 8 | img_list = '../DATA.labelpath' 9 | eval_model = './model/MobileFaceNet_MS1M.pth' 10 | outfile = '../feats_npy/Embedding_Features.npy' 11 | # data preprocess 12 | transform = T.Compose([ 13 | T.Resize((112, 112)), 14 | T.ToTensor(), 15 | T.Normalize(mean=[0.5,0.5,0.5], std=[0.5,0.5,0.5]), 16 | ]) 17 | # network settings 18 | backbone = 'MFN' # [MFN, R_50] 19 | device = 'cuda:0' if torch.cuda.is_available() else 'cpu' 20 | multi_GPUs = [0,1,2,3,4,5,6] 21 | embedding_size = 512 22 | batch_size = 2000 23 | pin_memory = True 24 | num_workers = 4 25 | config = Config() 26 | -------------------------------------------------------------------------------- /quality/generate_pseudo_labels/extract_embedding/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/generate_pseudo_labels/extract_embedding/dataset/__init__.py -------------------------------------------------------------------------------- /quality/generate_pseudo_labels/extract_embedding/model/MobileFaceNet_MS1M.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/quality/generate_pseudo_labels/extract_embedding/model/MobileFaceNet_MS1M.pth -------------------------------------------------------------------------------- /quality/generate_pseudo_labels/gen_datalist.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data import DataLoader 2 | from torchvision.datasets import ImageFolder 3 | from tqdm import tqdm 4 | 5 | 6 | def gentxt(data_root, outfile): # generate data file via traveling the target dataset 7 | ''' 8 | Use ImageFolder method to travel the target dataset 9 | Save to two files including ".label" and ".labelpath" 10 | ''' 11 | # output file1 12 | outfile1 = open(outfile, 'w') 13 | # output file2 14 | outfile2 = open(outfile+'path', 'w') 15 | data = ImageFolder(data_root) 16 | count = 0 17 | tqdm(data.imgs) 18 | # travel the target dataset 19 | for index, value in enumerate(data.imgs): 20 | count += 1 21 | img = '/'+'/'.join(value[0].split('/')[-2:]) 22 | outfile1.write(img + '\n') 23 | outfile2.write(value[0] + '\t' + str(value[1]) + '\n') 24 | return count 25 | 26 | 27 | if __name__ == "__main__": # obtain data list 28 | ''' 29 | This method is to obtain data list from dataset 30 | and save to txt files 31 | ''' 32 | outfile = './DATA.label' 33 | data_root = '/data/home/fuzhaoou/Dataset/MS1M/MS1M_arcface_112' 34 | print(gentxt(data_root, outfile)) 35 | -------------------------------------------------------------------------------- /quality/local_train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -d "logs" ]; then 4 | mkdir logs 5 | fi 6 | export CUDA_VISIBLE_DEVICES='0,1,2,3,4,5,6,7' 7 | nohup python -u train.py > logs/$(date +%F-%H-%M-%S).log 2>&1 & 8 | -------------------------------------------------------------------------------- /quality/train_config.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as T 3 | 4 | 5 | class Config: 6 | # dataset 7 | img_list = './generate_pseudo_labels/labels/pseudo_quality_score.txt' 8 | finetuning_model = './generate_pseudo_labels/extract_embedding/model/MobileFaceNet_MS1M.pth' 9 | # save settings 10 | checkpoints = "./checkpoints/MS1M_Quality_Regression/S1" 11 | checkpoints_name = "MFN" 12 | # data preprocess 13 | transform = T.Compose([ 14 | T.RandomHorizontalFlip(), 15 | T.ToTensor(), 16 | T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), 17 | ]) 18 | # training settings 19 | device = 'cuda:0' if torch.cuda.is_available() else 'cpu' 20 | seed = 0 21 | multi_GPUs = [0,1,2,3,4,5,6,7] 22 | backbone = 'MFN' # [MFN, R_50] 23 | pin_memory = True 24 | num_workers = 12 25 | batch_size = 5000 26 | epoch = 20 27 | lr = 0.0001 28 | stepLR = [5, 10] 29 | weight_decay = 0.0005 30 | display = 100 31 | saveModel_epoch = 1 32 | loss = 'SmoothL1' # ['L1', 'L2', 'SmoothL1'] 33 | 34 | config = Config() 35 | -------------------------------------------------------------------------------- /recognition/deploy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.1) 2 | project(facefeature-tnn CXX) 3 | 4 | set(CMAKE_BUILD_TYPE Release) 5 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -std=c++11") 6 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fPIC -pthread") 7 | 8 | option(TNN_OPENVINO_ENABLE "with openvino lib" ON) 9 | option(LOG_ENABLE "with log print" OFF) 10 | 11 | message(${TNN_LIB_DIR}) 12 | message(${TNN_INCLUDE_DIR}) 13 | message(${OpenCV_DIR}) 14 | message(${GFLAGS_DIR}) 15 | 16 | if (${TNN_OPENVINO_ENABLE}) 17 | add_definitions(-D_OPENVINO_) 18 | endif() 19 | 20 | if (${LOG_ENABLE}) 21 | add_definitions(-D_LOG_) 22 | endif() 23 | 24 | find_package(OpenCV 3 REQUIRED) 25 | include_directories(${OpenCV_INCLUDE_DIRS}) 26 | 27 | include_directories(${TNN_INCLUDE_DIR}) 28 | link_directories(${TNN_LIB_DIR}) 29 | link_libraries(TNN) 30 | 31 | add_subdirectory(${GFLAGS_DIR} ${GFLAGS_DIR}) 32 | get_target_property(GFLAGS_INCLUDE_DIRS gflags INTERFACE_INCLUDE_DIRECTORIES) 33 | include_directories(BEFORE "${GFLAGS_INCLUDE_DIRS}") 34 | link_libraries(gflags) 35 | 36 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}/core/) 37 | file(GLOB_RECURSE 38 | ALL_SOURCE_FILES 39 | ${CMAKE_CURRENT_SOURCE_DIR}/core/**.cpp 40 | ) 41 | 42 | add_library(FaceFeature SHARED ${ALL_SOURCE_FILES}) 43 | add_executable(test test/test.cpp) 44 | target_link_libraries(test FaceFeature ${OpenCV_LIBS}) 45 | 46 | -------------------------------------------------------------------------------- /recognition/deploy/README.md: -------------------------------------------------------------------------------- 1 | ## Deploy 2 | 3 | ### Introduction 4 | After finishing the train, we deploy the model based on [TNN](https://github.com/Tencent/TNN) framework. TNN is a high-performance, lightweight neural network inference framework open sourced by Tencent Youtu Lab, and supports for ARM CPU/X86 CPU/NV GPU/ devcies. 5 | 6 | ### Preparation 7 | 8 | Download TNN source and compile core library and model-converter tools, please refer to the official guides, [Compile converter](https://github.com/Tencent/TNN/blob/master/doc/en/user/convert_en.md), [Compile TNN](https://github.com/Tencent/TNN/blob/master/doc/en/user/compile_en.md) 9 | 10 | ### Convert 11 | 1. Convert PyTorch checkpoint to ONNX model 12 | ``` 13 | python3 converter/export_onnx.py -h 14 | ``` 15 | The output shows 16 | ``` bash 17 | usage: export_onnx.py [-h] --ckpt_path CKPT_PATH --onnx_name ONNX_NAME 18 | --model_name MODEL_NAME 19 | 20 | export pytorch model to onnx 21 | 22 | optional arguments: 23 | -h, --help show this help message and exit 24 | --ckpt_path CKPT_PATH 25 | --onnx_name ONNX_NAME 26 | --model_name MODEL_NAME 27 | ``` 28 | 29 | 2. Convert ONNX model to TNN model 30 | 31 | ``` bash 32 | ./converter/convert_onnx2tnn.sh 33 | ``` 34 | 35 | ### Compile 36 | 37 | 1. Linux x86 38 | 39 | Please compile TNN by using `build_linux_openvino.sh`. We suggest using the OpenVINO backend in x86 cpus for high performance and TNN integrates the OpenVINO interfaces. 40 | Run below scripts to compile and test deploy codes. 41 | ``` 42 | ./scripts/build_linux_x86.sh 43 | 44 | ./build/test -h 45 | -h 46 | -img 47 | -proto 48 | -model 49 | 50 | ``` 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /recognition/deploy/converter/convert_onnx2tnn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# != 2 ]; then 4 | echo "$0 " 5 | exit 0 6 | fi 7 | 8 | if [ -z $TNN_ROOT ]; then 9 | echo "Please set env [TNN_ROOT]"; 10 | exit -1; 11 | fi 12 | 13 | tnn_converter_dir=${TNN_ROOT}tools/convert2tnn/ 14 | onnx_path=$(readlink -f "$1") 15 | echo $onnx_path 16 | 17 | output_dir=$(readlink -f "$2") 18 | echo $output_dir 19 | cd $tnn_converter_dir 20 | python converter.py onnx2tnn $onnx_path -in input:1,3,112,112 -o $output_dir 21 | -------------------------------------------------------------------------------- /recognition/deploy/converter/export_onnx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import argparse 4 | import torch.onnx 5 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..')) 6 | from torchkit.backbone import get_model 7 | from torchkit.util.utils import l2_norm 8 | 9 | 10 | parser = argparse.ArgumentParser(description='export pytorch model to onnx') 11 | parser.add_argument('--ckpt_path', default=None, type=str, required=True, help='') 12 | parser.add_argument('--onnx_name', default=None, type=str, required=True, help='') 13 | parser.add_argument('--model_name', default=None, type=str, required=True, help='') 14 | args = parser.parse_args() 15 | 16 | 17 | def main(): 18 | net = get_model(args.model_name) 19 | input_size = [112, 112] 20 | torch_model = net(input_size) 21 | if not os.path.isfile(args.ckpt_path): 22 | print("Invalid ckpt path: %s" % args.ckpt_path) 23 | return 24 | torch_model.load_state_dict(torch.load(args.ckpt_path)) 25 | torch_model.eval() 26 | 27 | batch_size = 1 28 | x = torch.randn(batch_size, 3, input_size[0], input_size[1], requires_grad=True) 29 | torch_out = torch_model(x) 30 | torch_out = l2_norm(torch_out) 31 | torch.onnx.export(torch_model, 32 | x, 33 | "%s.onnx" % args.onnx_name, 34 | export_params=True, 35 | opset_version=10, 36 | do_constant_folding=True, 37 | input_names = ['input'], 38 | output_names = ['output']) 39 | print("finished") 40 | 41 | 42 | if __name__ == '__main__': 43 | main() 44 | -------------------------------------------------------------------------------- /recognition/deploy/converter/onnx_inference.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import cv2 3 | import numpy as np 4 | import onnxruntime 5 | import torchvision.transforms as transforms 6 | 7 | 8 | parser = argparse.ArgumentParser(description='onnx inference') 9 | parser.add_argument('--onnx_path', default=None, type=str, required=True, help='') 10 | args = parser.parse_args() 11 | 12 | 13 | def l2_norm(x): 14 | """ l2 normalize 15 | """ 16 | output = x / np.linalg.norm(x) 17 | return output 18 | 19 | 20 | def to_numpy(tensor): 21 | return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() 22 | 23 | 24 | def get_test_transform(): 25 | test_transform = transforms.Compose([ 26 | transforms.ToTensor(), 27 | transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])]) 28 | return test_transform 29 | 30 | 31 | def main(): 32 | img = cv2.imread("brucelee.jpg") 33 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 34 | img = get_test_transform()(img) 35 | img = img.unsqueeze_(0) 36 | print(img.shape) 37 | onnx_path = args.onnx_path 38 | session = onnxruntime.InferenceSession(onnx_path) 39 | inputs = {session.get_inputs()[0].name: to_numpy(img)} 40 | outs = session.run(None, inputs)[0] 41 | print(outs.shape) 42 | outs = l2_norm(outs).squeeze() 43 | ss = "" 44 | for x in outs: 45 | ss = ss + "%.6f " % x 46 | print(ss) 47 | 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /recognition/deploy/core/face_feature.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_FEATURE_H_ 2 | #define _FACE_FEATURE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | class TNNInstance{ 15 | public: 16 | TNNInstance() {} 17 | ~TNNInstance() {} 18 | TNN_NS::Status Init(const std::string&, const std::string& model); 19 | TNN_NS::Status Forward(TNN_NS::Mat& input_mat, std::vector& feature); 20 | TNN_NS::DimsVector GetInputDims(std::string layer_name = ""); 21 | TNN_NS::DimsVector GetOutputDims(std::string layer_name = ""); 22 | 23 | private: 24 | std::shared_ptr instance_ = nullptr; 25 | std::map input_dims_; 26 | std::map output_dims_; 27 | std::map> output_mats_; 28 | }; 29 | 30 | #endif // _FACE_FEATURE_H_ -------------------------------------------------------------------------------- /recognition/deploy/core/timer.h: -------------------------------------------------------------------------------- 1 | #ifndef _TIMER_H_ 2 | #define _TIMER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using std::chrono::time_point; 11 | using std::chrono::system_clock; 12 | using std::chrono::duration_cast; 13 | using std::chrono::microseconds; 14 | 15 | class Timer { 16 | public: 17 | explicit Timer(std::string timer_info) { 18 | timer_info_ = timer_info; 19 | Reset(); 20 | } 21 | void Start() { 22 | start_ = system_clock::now(); 23 | } 24 | void Stop() { 25 | stop_ = system_clock::now(); 26 | float delta = duration_cast(stop_ - start_).count() / 1000.0f; 27 | min_ = static_cast(fmin(min_, delta)); 28 | max_ = static_cast(fmax(max_, delta)); 29 | sum_ += delta; 30 | count_++; 31 | } 32 | void Reset() { 33 | min_ = FLT_MAX; 34 | max_ = FLT_MIN; 35 | sum_ = 0.0f; 36 | count_ = 0; 37 | stop_ = start_ = system_clock::now(); 38 | } 39 | void Print() { 40 | char min_str[16]; 41 | snprintf(min_str, 16, "%6.3f", min_); 42 | char max_str[16]; 43 | snprintf(max_str, 16, "%6.3f", max_); 44 | char avg_str[16]; 45 | snprintf(avg_str, 16, "%6.3f", sum_ / static_cast(count_)); 46 | printf("%-15s cost: min = %-8s ms | max = %-8s ms | avg = %-8s ms \n", timer_info_.c_str(), 47 | min_str, max_str, avg_str); 48 | } 49 | 50 | private: 51 | float min_; 52 | float max_; 53 | float sum_; 54 | std::string timer_info_; 55 | time_point start_; 56 | time_point stop_; 57 | int count_; 58 | }; 59 | 60 | #endif // _TIMER_H_ -------------------------------------------------------------------------------- /recognition/deploy/core/util.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_H_ 2 | #define _UTIL_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | inline bool CheckResult(std::string desc, TNN_NS::Status result) { 10 | if (result != 0) { 11 | #ifdef _LOG_ 12 | fprintf(stderr, "[%s] failed: %s \n", desc.c_str(), result.description().c_str()); 13 | #endif 14 | return false; 15 | } else { 16 | #ifdef _LOG_ 17 | fprintf(stdout, "[%s] success! \n", desc.c_str()); 18 | #endif 19 | return true; 20 | } 21 | } 22 | 23 | inline float norm(std::vector &fin) { 24 | float sum = 0.f; 25 | for (float v : fin) { 26 | sum += v * v; 27 | } 28 | return sqrt(sum); 29 | } 30 | 31 | inline std::vector l2_normlize(std::vector & fin) { 32 | float div = norm(fin); 33 | std::vector vout; 34 | for (auto it = fin.begin(); it != fin.end(); it++) { 35 | float value = (*it) / div; 36 | vout.push_back(value); 37 | } 38 | return vout; 39 | } 40 | 41 | inline int ReadFile(const std::string& file_path, std::string &data) { 42 | std::ifstream file(file_path, std::ios::binary); 43 | if (!file) { 44 | fprintf(stderr, ("file %s not exist\n", file_path.c_str())); 45 | return -1; 46 | } 47 | 48 | file.seekg(0, std::ios::end); 49 | std::streampos fileSize = file.tellg(); 50 | file.seekg(0, std::ios::beg); 51 | if (fileSize <= 0) { 52 | fprintf(stderr, ("file %s is empty\n", file_path.c_str())); 53 | return -2; 54 | } 55 | 56 | data = std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); 57 | return 0; 58 | } 59 | 60 | #endif -------------------------------------------------------------------------------- /recognition/deploy/data/brucelee.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/deploy/data/brucelee.jpg -------------------------------------------------------------------------------- /recognition/deploy/scripts/build_linux_x86.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z $TNN_ROOT ]; then 4 | echo "Please set env [TNN_ROOT]"; 5 | exit -1; 6 | fi 7 | 8 | if [ -z $OpenCV_DIR ]; then 9 | echo "Please set env [OpenCV_DIR]"; 10 | exit -1; 11 | fi 12 | 13 | TNN_LIB_DIR=${TNN_ROOT}/scripts/x86_linux_release/lib/ 14 | TNN_INCLUDE_DIR=${TNN_ROOT}/scripts/x86_linux_release/include/ 15 | GFLAGS_DIR=${TNN_ROOT}/third_party/gflags/ 16 | OpenCV_DIR=${OpenCV_DIR} 17 | 18 | 19 | if [ ! -d ${TNN_LIB_PATH} ]; then 20 | echo "TNN not build success, please build TNN first"; 21 | exit -2; 22 | fi 23 | 24 | rm -rf ./build 25 | mkdir build 26 | cd build 27 | cmake3 .. \ 28 | -DTNN_LIB_DIR=${TNN_LIB_DIR} \ 29 | -DTNN_INCLUDE_DIR=${TNN_INCLUDE_DIR} \ 30 | -DOpenCV_DIR=${OpenCV_DIR} \ 31 | -DGFLAGS_DIR=${GFLAGS_DIR} \ 32 | -DTNN_OPENVINO_ENABLE=ON \ 33 | -DLOG_ENABLE=OFF 34 | make 35 | -------------------------------------------------------------------------------- /recognition/deploy/test/test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "gflags/gflags.h" 8 | #include "face_feature.h" 9 | #include "util.h" 10 | #include "timer.h" 11 | 12 | using namespace std; 13 | using namespace cv; 14 | 15 | DEFINE_string(img, "./data/brucelee.jpg", "input img"); 16 | DEFINE_string(proto, "", "tnn proto path"); 17 | DEFINE_string(model, "", "tnn model path"); 18 | DEFINE_bool(h, false, "help"); 19 | 20 | void ShowUsage() { 21 | printf(" -h \n"); 22 | printf(" -img \n"); 23 | printf(" -proto \n"); 24 | printf(" -model \n"); 25 | } 26 | 27 | bool ParseAndCheckCommandLine(int argc, char* argv[]) { 28 | gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); 29 | if (FLAGS_h) { 30 | ShowUsage(); 31 | return false; 32 | } 33 | if (FLAGS_proto.empty()) { 34 | printf("tnn proto path is not set\n"); 35 | ShowUsage(); 36 | return false; 37 | } 38 | 39 | if (FLAGS_model.empty()) { 40 | printf("tnn model path is not set\n"); 41 | ShowUsage(); 42 | return false; 43 | } 44 | return true; 45 | } 46 | 47 | int main(int argc, char* argv[]) { 48 | if (!ParseAndCheckCommandLine(argc, argv)) 49 | return -1; 50 | 51 | Mat img = imread(FLAGS_img); 52 | cvtColor(img, img, cv::COLOR_BGR2RGB); 53 | printf("image width: %d, height: %d\n", img.cols, img.rows); 54 | string proto_path = FLAGS_proto; 55 | string model_path = FLAGS_model; 56 | 57 | string proto, model; 58 | int ret = 0; 59 | ret = ReadFile(proto_path, proto); 60 | if (ret) return ret; 61 | ret = ReadFile(model_path, model); 62 | if (ret) return ret; 63 | 64 | TNNInstance* instance = new TNNInstance(); 65 | TNN_NS::Status status; 66 | status = instance->Init(proto, model); 67 | if (status != 0) { 68 | return status; 69 | } 70 | TNN_NS::DimsVector input_dims = instance->GetInputDims(); 71 | TNN_NS::Mat input_mat = TNN_NS::Mat(TNN_NS::DEVICE_NAIVE, TNN_NS::N8UC3, input_dims, img.data); 72 | 73 | vector feature; 74 | status = instance->Forward(input_mat, feature); 75 | if (status != 0) { 76 | return status; 77 | } 78 | for (float i : feature) 79 | printf("%f ", i); 80 | printf("\n"); 81 | 82 | Timer timer("feature"); 83 | 84 | for (size_t i = 0; i < 100; i++) { 85 | timer.Start(); 86 | status = instance->Forward(input_mat, feature); 87 | if (status != 0) 88 | return status; 89 | timer.Stop(); 90 | } 91 | timer.Print(); 92 | 93 | return 0; 94 | } 95 | -------------------------------------------------------------------------------- /recognition/doc/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/doc/framework.png -------------------------------------------------------------------------------- /recognition/doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/doc/logo.png -------------------------------------------------------------------------------- /recognition/local_train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -d "logs" ]; then 4 | mkdir logs 5 | fi 6 | 7 | if [ ! -d "ckpt" ]; then 8 | mkdir ckpt 9 | fi 10 | 11 | export CUDA_VISIBLE_DEVICES='0,1,2,3,4,5,6,7' 12 | nohup python -u -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 train.py > logs/$(date +%F-%H-%M-%S).log 2>&1 & -------------------------------------------------------------------------------- /recognition/requirements.txt: -------------------------------------------------------------------------------- 1 | dareblopy -------------------------------------------------------------------------------- /recognition/tasks/dctdp/README.md: -------------------------------------------------------------------------------- 1 | # Privacy-Preserving Face Recognition with Learnable Privacy Budgets in Frequency Domain 2 | 3 | ## Introduction 4 | 5 | Face recognition technology has been used in many fields due to its high recognition accuracy, including the face unlocking of mobile devices, community access control systems, and city surveillance. As the current high accuracy is guaranteed by very deep network structures, facial images often need to be transmitted to third-party servers with high computational power for inference. However, facial images visually reveal the user's identity information. In this process, both untrusted service providers and malicious users can significantly increase the risk of a personal privacy breach. Current privacy-preserving approaches to face recognition are often accompanied by many side effects, such as a significant increase in inference time or a noticeable decrease in recognition accuracy. This paper proposes a privacy-preserving face recognition method using differential privacy in the frequency domain. Due to the utilization of differential privacy, it offers a guarantee of privacy in theory. Meanwhile, the loss of accuracy is very slight. This method first converts the original image to the frequency domain and removes the direct component termed DC. Then a privacy budget allocation method can be learned based on the loss of the back-end face recognition network within the differential privacy framework. Finally, it adds the corresponding noise to the frequency domain features. Our method performs very well with several classical face recognition test sets according to the extensive experiments. 6 | 7 | ## Method 8 | 9 | 10 | 11 | 12 | ## Experiments 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | ## Citation 23 | If you find this code useful in your research, please consider citing us: 24 | ``` 25 | @article{ji2022privacy, 26 | title={Privacy-Preserving Face Recognition with Learnable Privacy Budgets in Frequency Domain}, 27 | author={Ji, Jiazhen and Wang, Huan and Huang, Yuge and Wu, Jiaxiang and Xu, Xingkun and Ding, Shouhong and Zhang, ShengChuan and Cao, Liujuan and Ji, Rongrong}, 28 | journal={arXiv preprint arXiv:2207.07316}, 29 | year={2022} 30 | } 31 | -------------------------------------------------------------------------------- /recognition/tasks/dctdp/doc/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/dctdp/doc/main.png -------------------------------------------------------------------------------- /recognition/tasks/dctdp/doc/result1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/dctdp/doc/result1.png -------------------------------------------------------------------------------- /recognition/tasks/dctdp/doc/result2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/dctdp/doc/result2.png -------------------------------------------------------------------------------- /recognition/tasks/dctdp/doc/result3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/dctdp/doc/result3.png -------------------------------------------------------------------------------- /recognition/tasks/dctdp/local_train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -d "logs" ]; then 4 | mkdir logs 5 | fi 6 | 7 | if [ ! -d "ckpt" ]; then 8 | mkdir ckpt 9 | fi 10 | 11 | export CUDA_VISIBLE_DEVICES='0' 12 | nohup python3 -u -m torch.distributed.launch --nproc_per_node=1 --nnodes=1 train.py > logs/$(date +%F-%H-%M-%S).log 2>&1 & 13 | -------------------------------------------------------------------------------- /recognition/tasks/dctdp/requirements.txt: -------------------------------------------------------------------------------- 1 | dareblopy 2 | torchjpeg 3 | torchsummary 4 | -------------------------------------------------------------------------------- /recognition/tasks/dctdp/train.yaml: -------------------------------------------------------------------------------- 1 | SEED: 1337 # random seed for reproduce results 2 | DATA_ROOT: '' # to fill, the parent root where your train/val/test data are stored 3 | INDEX_ROOT: '' #to fill 4 | DATASETS: # the dataset index name 5 | - name: TFR-refined_ms1m_112_shuf #TFR-vggface2_shuf 6 | batch_size: 64 7 | weight: 1.0 8 | scale: 64 9 | margin: 0.4 10 | 11 | BACKBONE_RESUME: "" 12 | HEAD_RESUME: "" 13 | META_RESUME: "" 14 | 15 | # BACKBONE_NAME: 'EfficientNetB0' 16 | INPUT_SIZE: [112, 112] # support: [112, 112] and [224, 224] 17 | BACKBONE_NAME: 'IR_50' # Must be a model of the IR series 18 | EMBEDDING_SIZE: 512 # feature dimension 19 | 20 | MODEL_ROOT: './ckpt/' # the root to buffer your checkpoints 21 | LOG_ROOT: './tensorboard' # the root to log your train/val status 22 | 23 | DIST_FC: true 24 | HEAD_NAME: "ArcFace" # support: ['ArcFace', 'CurricularFace', 'CosFace'] 25 | LOSS_NAME: 'DistCrossEntropy' # support: ['DistCrossEntropy', 'Softmax'] 26 | 27 | RGB_MEAN: [0.5, 0.5, 0.5] # for normalize inputs to [-1, 1] 28 | RGB_STD: [0.5, 0.5, 0.5] 29 | 30 | LRS: [0.1, 0.01, 0.001, 0.0001] # initial LR 31 | LRS_NOISE: [0.1, 0.01, 0.001, 0.0001] 32 | 33 | WARMUP_STEP: -1 34 | STAGES: [10, 18, 22] # epoch stages to decay learning rate 35 | 36 | START_EPOCH: 0 # start epoch 37 | NUM_EPOCH: 24 # total epoch number 38 | SAVE_EPOCHS: [10, 18, 22, 24] 39 | 40 | WEIGHT_DECAY: 0.0005 # do not apply to batch_norm parameters 41 | MOMENTUM: 0.9 42 | 43 | WORLD_SIZE: 1 44 | RANK: 0 45 | LOCAL_RANK: 0 46 | DIST_BACKEND: 'nccl' 47 | DIST_URL: 'env://' 48 | 49 | NUM_WORKERS: 8 50 | 51 | AMP: True # fp16 for backbone 52 | -------------------------------------------------------------------------------- /recognition/tasks/dctdp/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torchjpeg import dct 5 | 6 | def images_to_batch(x): 7 | x = (x + 1) / 2 * 255 8 | x = F.interpolate(x, scale_factor=8, mode='bilinear', align_corners=True) 9 | if x.shape[1] != 3: 10 | print("Wrong input, Channel should equals to 3") 11 | return 12 | x = dct.to_ycbcr(x) # comvert RGB to YCBCR 13 | x -= 128 14 | bs, ch, h, w = x.shape 15 | block_num = h // 8 16 | x = x.view(bs * ch, 1, h, w) 17 | x = F.unfold(x, kernel_size=(8, 8), dilation=1, padding=0, 18 | stride=(8, 8)) 19 | x = x.transpose(1, 2) 20 | x = x.view(bs, ch, -1, 8, 8) 21 | dct_block = dct.block_dct(x) 22 | dct_block = dct_block.view(bs, ch, block_num, block_num, 64).permute(0, 1, 4, 2, 3) 23 | dct_block = dct_block[:, :, 1:, :, :] # remove DC 24 | dct_block = dct_block.reshape(bs, -1, block_num, block_num) 25 | return dct_block 26 | 27 | 28 | class NoisyActivation(nn.Module): 29 | def __init__(self, input_shape=112, budget_mean=4, sensitivity=None): 30 | super(NoisyActivation, self).__init__() 31 | self.h, self.w = input_shape, input_shape 32 | if sensitivity is None: 33 | sensitivity = torch.ones([189, self.h, self.w]).cuda() 34 | self.sensitivity = sensitivity.reshape(189 * self.h * self.w) 35 | self.given_locs = torch.zeros((189, self.h, self.w)) 36 | size = self.given_locs.shape 37 | self.budget = budget_mean * 189 * self.h * self.w 38 | self.locs = nn.Parameter(torch.Tensor(size).copy_(self.given_locs)) 39 | self.rhos = nn.Parameter(torch.zeros(size)) 40 | self.laplace = torch.distributions.laplace.Laplace(0, 1) 41 | self.rhos.requires_grad = True 42 | self.locs.requires_grad = True 43 | 44 | def scales(self): 45 | softmax = nn.Softmax() 46 | return (self.sensitivity / (softmax(self.rhos.reshape(189 * self.h * self.w)) 47 | * self.budget)).reshape(189, self.h, self.w) 48 | 49 | def sample_noise(self): 50 | epsilon = self.laplace.sample(self.rhos.shape).cuda() 51 | return self.locs + self.scales() * epsilon 52 | 53 | def forward(self, input): 54 | noise = self.sample_noise() 55 | output = input + noise 56 | return output 57 | 58 | def aux_loss(self): 59 | scale = self.scales() 60 | loss = -1.0 * torch.log(scale.mean()) 61 | return loss 62 | -------------------------------------------------------------------------------- /recognition/tasks/duetface/base_task.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import torch 4 | import torch.nn.init as init 5 | from tasks.duetface.duetface_model import DuetFaceModel 6 | 7 | from ..head import get_head 8 | from ..util import get_class_split 9 | 10 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')) 11 | 12 | # ================================================ 13 | # do NOT run this script directly 14 | # append the following make_interactive_models() function to base_task.py 15 | # right after make_models() 16 | # ================================================ 17 | 18 | 19 | def make_interactive_models(self): 20 | sub_channels = self.cfg['SUB_CHS'] 21 | main_feature_size = self.cfg['FEATURE_SIZE'] 22 | sub_feature_size = self.cfg['SUB_FEATURE_SIZE'] 23 | 24 | num_sub_channels = len(sub_channels) * 3 25 | self.backbone = DuetFaceModel(num_sub_channels, main_feature_size, sub_feature_size, 26 | main_model_name=self.cfg['BACKBONE_NAME'], 27 | sub_model_name=self.cfg['SUB_BACKBONE_NAME']) 28 | 29 | # load pre-trained client-side model from a given checkpoint 30 | if self.cfg['LOAD_CKPT']: 31 | ckpt_path = self.cfg['CKPT_PATH'] 32 | if not os.path.exists(ckpt_path): 33 | raise RuntimeError("%s not exists" % ckpt_path) 34 | 35 | model_dict = self.backbone.sub_model.state_dict() 36 | pretrained_dict = torch.load(ckpt_path) 37 | pretrained_dict = {k.split('.', 1)[1]: v for k, v in pretrained_dict.items() if 38 | k.split('.', 1)[1] in model_dict} 39 | 40 | model_dict.update(pretrained_dict) 41 | self.backbone.sub_model.load_state_dict(model_dict) 42 | 43 | self.backbone = self.backbone.cuda() 44 | # logging.info("DuetFace Backbone Generated") 45 | 46 | # make heads, the rest are identical to that in make_model() 47 | embedding_size = self.cfg['EMBEDDING_SIZE'] 48 | self.class_shards = [] 49 | metric = get_head(self.cfg['HEAD_NAME'], dist_fc=self.dist_fc) 50 | 51 | for name, branch in self.branches.items(): 52 | class_num = self.class_nums[name] 53 | class_shard = get_class_split(class_num, self.world_size) 54 | self.class_shards.append(class_shard) 55 | # logging.info('Split FC: {}'.format(class_shard)) 56 | 57 | init_value = torch.FloatTensor(embedding_size, class_num) 58 | init.normal_(init_value, std=0.01) 59 | head = metric(in_features=embedding_size, 60 | gpu_index=self.rank, 61 | weight_init=init_value, 62 | class_split=class_shard, 63 | scale=branch.scale, 64 | margin=branch.margin) 65 | del init_value 66 | head = head.cuda() 67 | self.heads[name] = head 68 | -------------------------------------------------------------------------------- /recognition/tasks/duetface/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/duetface/pipeline.png -------------------------------------------------------------------------------- /recognition/tasks/duetface/requirements.txt: -------------------------------------------------------------------------------- 1 | dareblopy 2 | torchjpeg 3 | torchsummary 4 | shapely -------------------------------------------------------------------------------- /recognition/tasks/duetface/train.yaml: -------------------------------------------------------------------------------- 1 | SEED: 1337 # random seed for reproduce results 2 | DATA_ROOT: '' # to fill, the parent root where your train/val/test data are stored 3 | INDEX_ROOT: '' # to fill 4 | DATASETS: # the dataset index name 5 | - name: # to fill, the dataset name 6 | batch_size: 64 7 | weight: 1.0 8 | scale: 64 9 | margin: 0.5 10 | 11 | BACKBONE_RESUME: "" 12 | HEAD_RESUME: "" 13 | META_RESUME: "" 14 | 15 | INPUT_SIZE: [ 112, 112 ] 16 | BACKBONE_NAME: 'IR_18' # IR_50 17 | EMBEDDING_SIZE: 512 # feature dimension 18 | 19 | MODEL_ROOT: './ckpt/' # the root to buffer your checkpoints 20 | LOG_ROOT: './tensorboard' # the root to log your train/val status 21 | 22 | DIST_FC: true 23 | HEAD_NAME: "ArcFace" # support: ['ArcFace', 'CurricularFace', 'CosFace'] 24 | LOSS_NAME: 'DistCrossEntropy' # support: ['DistCrossEntropy', 'Softmax'] 25 | 26 | RGB_MEAN: [ 0.5, 0.5, 0.5 ] 27 | RGB_STD: [ 0.5, 0.5, 0.5 ] 28 | 29 | LRS: [ 0.1, 0.01, 0.001, 0.0001 ] 30 | WARMUP_STEP: -1 31 | STAGES: [ 10, 18, 22 ] 32 | 33 | START_EPOCH: 0 34 | NUM_EPOCH: 24 35 | SAVE_EPOCHS: [ 16, 20, 24 ] 36 | 37 | WEIGHT_DECAY: 0.0005 38 | MOMENTUM: 0.9 39 | 40 | WORLD_SIZE: 1 41 | RANK: 0 42 | LOCAL_RANK: 0 43 | DIST_BACKEND: 'nccl' 44 | DIST_URL: 'env://' 45 | 46 | NUM_WORKERS: 8 47 | 48 | AMP: false # fp16 for backbone 49 | 50 | 51 | # DuetFace configurations 52 | MODE: INT # mode of training, support: RGB (baseline), INT 53 | SUB_CHS: [ 0, 1, 2, 3, 4, 5, 8, 9, 16, 24 ] # channels assigned to clients 54 | SUB_BACKBONE_NAME: 'MobileFaceNet' # client-side backbone 55 | FEATURE_SIZE: 512 # output feature size of server-side model 56 | SUB_FEATURE_SIZE: 512 # feature size of client-side model 57 | SUB_WARMUP_STEP: 0 # pre-training steps of client-side model, set to 0 if load checkpoint 58 | LOAD_CKPT: true # for end-to-end training, set to true to load following checkpoints 59 | CKPT_PATH: '' # to fill, checkpoint path of pre-trained client-side model 60 | LANDMARK_CKPT_PATH: '' # to fill, checkpoint path of PFLD facial landmark detector 61 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/README.md: -------------------------------------------------------------------------------- 1 | # Evaluation-oriented knowledge distillation for deep face recognition 2 | 3 | ## Introduction 4 | 5 | Knowledge distillation (KD) is a widely-used technique that utilizes large networks to improve the performance of compact models. Previous KD approaches usually aim to guide the student to mimic the teacher’s behavior completely in the representation space. However, such one-to-one corresponding constraints may lead to inflexible knowledge transfer from the teacher to the student, especially those with low model capacities. Inspired by the ultimate goal of KD methods, we propose a novel Evaluation oriented KD method (EKD) for deep face recognition to directly reduce the performance gap between the teacher and student models during training. Specifically, we adopt the commonly used evaluation metrics in face recognition, i.e., False Positive Rate (FPR) and True Positive Rate (TPR) as the performance indicator. According to the evaluation protocol, the critical pair relations that cause the TPR and FPR difference between the teacher and student models are selected. Then, the critical relations in the student are constrained to approximate the corresponding ones in the teacher by a novel rank-based loss function, giving more flexibility to the student with low capacity. Extensive experimental results on popular benchmarks demonstrate the superiority of our EKD over state-of-the-art competitors. 6 | 7 | ## Method 8 | 9 | 10 | 11 | 12 | ## Experiments 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | ## Citation 23 | If you find this code useful in your research, please consider citing us: 24 | ``` 25 | @inproceedings{ 26 | title={Evaluation-oriented knowledge distillation for deep face recognition}, 27 | author={Yuge Huang, Jiaxiang Wu, Xingkun Xu, Shouhong Ding}, 28 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 29 | year={2022} 30 | } 31 | ``` -------------------------------------------------------------------------------- /recognition/tasks/ekd/dataset.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | from collections import defaultdict 4 | from torch.utils.data.sampler import Sampler 5 | 6 | 7 | def create_label2index(dataset): 8 | """ create label2index map for BalancedBatchSampler, 9 | dataset is a SingleDataset object 10 | """ 11 | label2index = defaultdict(list) 12 | for i, sample in enumerate(dataset.inputs): 13 | label = sample[-1] 14 | label2index[label].append(i) 15 | return label2index 16 | 17 | 18 | class BalancedBatchSampler(Sampler): 19 | """ BalancedBatchSampler class 20 | Each batch includes `general_batch_size` general samples and `balanced_batch_size` balanced samples, 21 | general samples are directly randomly sampled, But balanced samples means picking `subsample_size` 22 | samples for each label. 23 | """ 24 | def __init__(self, labels2index, general_batch_size, 25 | balanced_batch_size, world_size, 26 | total_size, subsample_size=4): 27 | self.general_batch_size = general_batch_size 28 | self.balanced_batch_size = balanced_batch_size 29 | self.subsample_size = subsample_size 30 | self.labels2index = labels2index 31 | self.total_size = total_size 32 | self.world_size = world_size 33 | 34 | self.samples = [] 35 | for _, inds in self.labels2index.items(): 36 | self.samples.extend(inds) 37 | 38 | def __iter__(self): 39 | for _ in range(self.__len__()): 40 | inds = [] 41 | # random sample 42 | random_samples = random.sample(self.samples, self.general_batch_size) 43 | inds.extend(random_samples) 44 | 45 | # balanced sample 46 | sample_labels = random.sample(self.labels2index.keys(), 47 | self.balanced_batch_size // self.subsample_size) 48 | for each_label in sample_labels: 49 | each_label_indexes = self.labels2index[each_label] 50 | if len(each_label_indexes) > self.subsample_size: 51 | each_label_inds = random.sample(each_label_indexes, self.subsample_size) 52 | elif len(each_label_indexes) == self.subsample_size: 53 | each_label_inds = each_label_indexes 54 | else: 55 | repeat_inds = each_label_indexes * math.ceil(self.subsample_size / len(each_label_indexes)) 56 | each_label_inds = repeat_inds[:self.subsample_size] 57 | inds.extend(each_label_inds) 58 | yield inds 59 | 60 | def __len__(self): 61 | return self.total_size // self.general_batch_size // self.world_size 62 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/cc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class CC(nn.Module): 8 | """ Correlation Congruence for Knowledge Distillation. ICCV 2019 9 | """ 10 | def __init__(self, gamma=0.4, p=2): 11 | super().__init__() 12 | self.gamma = gamma 13 | self.p = p 14 | 15 | def forward(self, feat_s, feat_t): 16 | corr_mat_s = self.get_correlation_matrix(feat_s) 17 | corr_mat_t = self.get_correlation_matrix(feat_t) 18 | loss = F.mse_loss(corr_mat_s, corr_mat_t) 19 | return loss 20 | 21 | def get_correlation_matrix(self, feat): 22 | feat = F.normalize(feat) 23 | sim_mat = torch.matmul(feat, feat.t()) 24 | corr_mat = torch.zeros_like(sim_mat) 25 | for p in range(self.p): 26 | corr_mat += math.exp(-2*self.gamma) * (2*self.gamma)**p / \ 27 | math.factorial(p) * torch.pow(sim_mat, p) 28 | return corr_mat 29 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/cosine_distance.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class CosineDistance(nn.Module): 7 | """ Feature-Feature cosine distance 8 | """ 9 | 10 | def __init__(self): 11 | super(CosineDistance, self).__init__() 12 | 13 | def forward(self, f_s, f_t): 14 | batch_size = f_s.size(0) 15 | # just to make sure the tensor is 2D 16 | f_s = f_s.view(batch_size, -1) 17 | f_t = f_t.view(batch_size, -1) 18 | # normalize the feature 19 | f_s = F.normalize(f_s) 20 | f_t = F.normalize(f_t) 21 | 22 | distance = F.cosine_similarity(f_s, f_t, dim=1, eps=1e-8) 23 | loss = 1 - torch.mean(distance) 24 | return loss 25 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/darkrank.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | def euclidean_dist(x, y): 7 | """ 8 | Args: 9 | x: pytorch Variable, with shape [m, d] 10 | y: pytorch Variable, with shape [n, d] 11 | Returns: 12 | dist: pytorch Variable, with shape [m, n] 13 | """ 14 | 15 | m, n = x.size(0), y.size(0) 16 | xx = torch.pow(x, 2).sum(1, keepdim=True).expand(m, n) 17 | yy = torch.pow(y, 2).sum(1, keepdim=True).expand(n, m).t() 18 | dist = xx + yy 19 | dist = torch.addmm(dist, x, y.t(), beta=1, alpha=-2) 20 | dist = dist.clamp(min=1e-12).sqrt() # for numerical stability 21 | return dist 22 | 23 | 24 | class HardDarkRank(nn.Module): 25 | """ DarkRank: Accelerating Deep Metric Learning via Cross Sample Similarities Transfer. AAAI 2018 26 | """ 27 | def __init__(self, alpha=3, beta=3, permute_len=3): 28 | super().__init__() 29 | self.alpha = alpha 30 | self.beta = beta 31 | self.permute_len = permute_len 32 | 33 | def forward(self, g_s, g_t): 34 | g_s = F.normalize(g_s) 35 | g_t = F.normalize(g_t) 36 | 37 | score_teacher = -1 * self.alpha * \ 38 | euclidean_dist(g_t, g_t).pow(self.beta) 39 | score_student = -1 * self.alpha * \ 40 | euclidean_dist(g_s, g_s).pow(self.beta) 41 | 42 | permute_idx = score_teacher.sort(dim=1, descending=True)[ 43 | 1][:, 1:(self.permute_len+1)] 44 | ordered_student = torch.gather(score_student, 1, permute_idx) 45 | 46 | log_prob = (ordered_student - torch.stack([torch.logsumexp( 47 | ordered_student[:, i:], dim=1) for i in range(permute_idx.size(1))], dim=1)).sum(dim=1) 48 | loss = (-1 * log_prob).mean() 49 | return loss 50 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/fitnet.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class HintLoss(nn.Module): 5 | """ Fitnets: hints for thin deep nets, ICLR 2015 6 | """ 7 | def __init__(self): 8 | super().__init__() 9 | self.crit = nn.MSELoss() 10 | 11 | def forward(self, f_s, f_t): 12 | loss = self.crit(f_s, f_t) 13 | return loss 14 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/rkd.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class RKDLoss(nn.Module): 7 | """ Relational Knowledge Disitllation, CVPR2019 8 | """ 9 | def __init__(self, w_d=25, w_a=50): 10 | super().__init__() 11 | self.w_d = w_d 12 | self.w_a = w_a 13 | 14 | def forward(self, f_s, f_t): 15 | student = f_s.view(f_s.shape[0], -1) 16 | teacher = f_t.view(f_t.shape[0], -1) 17 | 18 | # RKD distance loss 19 | with torch.no_grad(): 20 | t_d = self.pdist(teacher, squared=False) 21 | mean_td = t_d[t_d > 0].mean() 22 | t_d = t_d / mean_td 23 | 24 | d = self.pdist(student, squared=False) 25 | mean_d = d[d > 0].mean() 26 | d = d / mean_d 27 | 28 | loss_d = F.smooth_l1_loss(d, t_d) 29 | 30 | # RKD Angle loss 31 | with torch.no_grad(): 32 | td = (teacher.unsqueeze(0) - teacher.unsqueeze(1)) 33 | norm_td = F.normalize(td, p=2, dim=2) 34 | t_angle = torch.bmm(norm_td, norm_td.transpose(1, 2)).view(-1) 35 | 36 | sd = (student.unsqueeze(0) - student.unsqueeze(1)) 37 | norm_sd = F.normalize(sd, p=2, dim=2) 38 | s_angle = torch.bmm(norm_sd, norm_sd.transpose(1, 2)).view(-1) 39 | 40 | loss_a = F.smooth_l1_loss(s_angle, t_angle) 41 | 42 | loss = self.w_d * loss_d + self.w_a * loss_a 43 | 44 | return loss 45 | 46 | @staticmethod 47 | def pdist(e, squared=False, eps=1e-12): 48 | e_square = e.pow(2).sum(dim=1) 49 | prod = e @ e.t() 50 | res = (e_square.unsqueeze(1) + e_square.unsqueeze(0) - 2 * prod).clamp(min=eps) 51 | 52 | if not squared: 53 | res = res.sqrt() 54 | 55 | res = res.clone() 56 | res[range(len(e)), range(len(e))] = 0 57 | return res 58 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/similarity_perserving.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class SimPerserving(nn.Module): 7 | """ Similarity-Preserving Knowledge Distillation, ICCV2019 8 | """ 9 | 10 | def __init__(self): 11 | super().__init__() 12 | 13 | def forward(self, g_s, g_t): 14 | return self.similarity_loss(g_s, g_t) 15 | 16 | # different from the origin paper, here use the embedding feature 17 | def similarity_loss(self, f_s, f_t): 18 | batch_size = f_s.size(0) 19 | # just to make sure the tensor is 2D 20 | f_s = f_s.view(batch_size, -1) 21 | f_t = f_t.view(batch_size, -1) 22 | # normalize the feature 23 | f_s = F.normalize(f_s) 24 | f_t = F.normalize(f_t) 25 | s_similarity = torch.mm(f_s, torch.t(f_s)) 26 | t_similarity = torch.mm(f_t, torch.t(f_t)) 27 | 28 | diff_similarity = s_similarity - t_similarity 29 | diff_norm = torch.norm(diff_similarity) 30 | loss = diff_norm * diff_norm / (batch_size * batch_size) 31 | 32 | return loss 33 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/distillation/simple_kd.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | 4 | 5 | class DistillKL(nn.Module): 6 | """ Distilling the Knowledge in a Neural Network, NIPSW 2015 7 | """ 8 | def __init__(self, t=4): 9 | super().__init__() 10 | self.t = t 11 | 12 | def forward(self, y_s, y_t): 13 | p_s = F.log_softmax(y_s/self.t, dim=1) 14 | p_t = F.softmax(y_t/self.t, dim=1) 15 | loss = F.kl_div(p_s, p_t, size_average=False) * (self.t**2) / y_s.shape[0] 16 | return loss 17 | -------------------------------------------------------------------------------- /recognition/tasks/ekd/doc/method.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/ekd/doc/method.png -------------------------------------------------------------------------------- /recognition/tasks/ekd/doc/result1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/ekd/doc/result1.png -------------------------------------------------------------------------------- /recognition/tasks/ekd/doc/result2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/ekd/doc/result2.png -------------------------------------------------------------------------------- /recognition/tasks/ekd/doc/result3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/ekd/doc/result3.png -------------------------------------------------------------------------------- /recognition/tasks/ekd/train.yaml: -------------------------------------------------------------------------------- 1 | SEED: 1337 # random seed for reproduce results 2 | DATA_ROOT: '/youtu-face/ata_root/remote_tfrecord' # the parent root where your train/val/test data are stored 3 | INDEX_ROOT: '/youtu-face/train_data/' 4 | DATASETS: # the dataset index name 5 | - name: TFR-refined_ms1m_112_shuf 6 | batch_size: 64 7 | weight: 1.0 8 | scale: 64 9 | margin: 0.4 10 | 11 | T_BACKBONE_RESUME: "" 12 | T_BACKBONE_NAME: 'IR_50' 13 | 14 | BACKBONE_RESUME: "" 15 | HEAD_RESUME: "" 16 | BACKBONE_NAME: 'MobileFaceNet' 17 | HEAD_NAME: "ArcFace" 18 | 19 | META_RESUME: "" 20 | 21 | GENERAL_BATCH_SIZE: 32 22 | BALANCED_BATCH_SIZE: 32 23 | 24 | INPUT_SIZE: [112, 112] # support: [112, 112] and [224, 224] 25 | EMBEDDING_SIZE: 512 # feature dimension 26 | 27 | MODEL_ROOT: './ckpt/' # the root to buffer your checkpoints 28 | LOG_ROOT: './tensorboard' # the root to log your train/val status 29 | 30 | DIST_FC: true 31 | 32 | RGB_MEAN: [0.5, 0.5, 0.5] # for normalize inputs to [-1, 1] 33 | RGB_STD: [0.5, 0.5, 0.5] 34 | 35 | LRS: [0.1, 0.01, 0.001, 0.0001] # initial LR 36 | WARMUP_STEP: -1 37 | STAGES: [10, 18, 22] # epoch stages to decay learning rate 38 | 39 | START_EPOCH: 0 # start epoch 40 | NUM_EPOCH: 26 # total epoch number 41 | SAVE_EPOCHS: [1, 26] 42 | 43 | WEIGHT_DECAY: 0.0005 # do not apply to batch_norm parameters 44 | MOMENTUM: 0.9 45 | 46 | WORLD_SIZE: 1 47 | RANK: 0 48 | LOCAL_RANK: 0 49 | DIST_BACKEND: 'nccl' 50 | DIST_URL: 'env://' 51 | 52 | NUM_WORKERS: 8 53 | -------------------------------------------------------------------------------- /recognition/tasks/minusface/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/minusface/__init__.py -------------------------------------------------------------------------------- /recognition/tasks/minusface/local_train.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES='0' 2 | python3 -u -m torch.distributed.launch --nproc_per_node=1 --nnodes=1 train.py 3 | -------------------------------------------------------------------------------- /recognition/tasks/minusface/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/minusface/pipeline.png -------------------------------------------------------------------------------- /recognition/tasks/minusface/requirements.txt: -------------------------------------------------------------------------------- 1 | dareblopy 2 | torchjpeg -------------------------------------------------------------------------------- /recognition/tasks/minusface/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..')) 5 | from tasks.partialface.train import TrainTask 6 | 7 | # parts of MinusFace's code requires the prior repository of PartialFace to sustain 8 | # please refer to ../partialface for details 9 | 10 | def main(): 11 | task_dir = os.path.dirname(os.path.abspath(__file__)) 12 | task = TrainTask(os.path.join(task_dir, 'train.yaml')) 13 | task.init_env() 14 | task.train() 15 | 16 | 17 | if __name__ == '__main__': 18 | main() 19 | -------------------------------------------------------------------------------- /recognition/tasks/minusface/train.yaml: -------------------------------------------------------------------------------- 1 | SEED: 1337 # random seed for reproduce results 2 | DATA_ROOT: '' # [fill in this blank] the parent directory where your train/val/test data are stored 3 | INDEX_ROOT: '' # [fill in this blank] the parent directory for index 4 | DATASETS: 5 | - name: # [fill in this blank] the name of your dataset 6 | batch_size: 64 7 | weight: 1.0 8 | scale: 64 9 | margin: 0.5 10 | 11 | BACKBONE_RESUME: "" 12 | HEAD_RESUME: "" 13 | META_RESUME: "" 14 | 15 | INPUT_SIZE: [ 112, 112 ] 16 | BACKBONE_NAME: 'IR_18' # support: ['IR_18', 'IR_50'] 17 | EMBEDDING_SIZE: 512 18 | 19 | MODEL_ROOT: './ckpt/' # the root to buffer your checkpoints 20 | LOG_ROOT: './tensorboard' # the root to log your train/val status 21 | 22 | DIST_FC: true 23 | HEAD_NAME: "ArcFace" # support: ['ArcFace', 'CurricularFace', 'CosFace'] 24 | LOSS_NAME: 'DistCrossEntropy' # support: ['DistCrossEntropy', 'Softmax'] 25 | 26 | RGB_MEAN: [ 0.5, 0.5, 0.5 ] # for normalize inputs to [-1, 1] 27 | RGB_STD: [ 0.5, 0.5, 0.5 ] 28 | 29 | LRS: [ 0.01, 0.001, 0.0001, 0.00001 ] 30 | WARMUP_STEP: -1 31 | STAGES: [ 10, 18, 22 ] 32 | 33 | START_EPOCH: 0 # start epoch 34 | NUM_EPOCH: 24 # total epoch number 35 | SAVE_EPOCHS: [ 1, 10, 16, 20, 24 ] 36 | 37 | WEIGHT_DECAY: 0.0005 # do not apply to batch_norm parameters 38 | MOMENTUM: 0.9 39 | 40 | WORLD_SIZE: 1 41 | RANK: 0 42 | LOCAL_RANK: 0 43 | DIST_BACKEND: 'nccl' 44 | DIST_URL: 'env://' 45 | 46 | NUM_WORKERS: 8 47 | 48 | AMP: false # fp16 for backbone 49 | 50 | # MinusFace 51 | METHOD: MinusFace 52 | TASK: stage1 # toy, stage1, stage2 53 | NUM_AUG: 3 # multiplier for data augmentation 54 | TASK_BACKBONE: 'IR_18' # IR_18, IR_50 55 | PRETRAIN_CKPT: '' # [fill in this blank] to train the recognition model requires pretrained MinusFace checkpoint 56 | TASK_VER: 3 -------------------------------------------------------------------------------- /recognition/tasks/partialface/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_task import LocalBaseTask 2 | from .utils import form_training_batch -------------------------------------------------------------------------------- /recognition/tasks/partialface/local_train.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES='0' 2 | python -u -m torch.distributed.launch --nproc_per_node=1 --nnodes=1 train.py 3 | -------------------------------------------------------------------------------- /recognition/tasks/partialface/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/partialface/pipeline.png -------------------------------------------------------------------------------- /recognition/tasks/partialface/requirements.txt: -------------------------------------------------------------------------------- 1 | dareblopy 2 | torchjpeg -------------------------------------------------------------------------------- /recognition/tasks/partialface/train.yaml: -------------------------------------------------------------------------------- 1 | SEED: 1337 # random seed for reproduce results 2 | DATA_ROOT: '' # [fill in this blank] the parent directory where your train/val/test data are stored 3 | INDEX_ROOT: '' # [fill in this blank] the parent directory for index 4 | DATASETS: 5 | - name: # [fill in this blank] the name of your dataset 6 | batch_size: 64 7 | weight: 1.0 8 | scale: 64 9 | margin: 0.5 10 | 11 | BACKBONE_RESUME: "" 12 | HEAD_RESUME: "" 13 | META_RESUME: "" 14 | 15 | INPUT_SIZE: [ 112, 112 ] 16 | BACKBONE_NAME: 'IR_18' # support: ['IR_18', 'IR_50'] 17 | EMBEDDING_SIZE: 512 18 | 19 | MODEL_ROOT: './ckpt/' # the root to buffer your checkpoints 20 | LOG_ROOT: './tensorboard' # the root to log your train/val status 21 | 22 | DIST_FC: true 23 | HEAD_NAME: "ArcFace" # support: ['ArcFace', 'CurricularFace', 'CosFace'] 24 | LOSS_NAME: 'DistCrossEntropy' # support: ['DistCrossEntropy', 'Softmax'] 25 | 26 | RGB_MEAN: [ 0.5, 0.5, 0.5 ] # for normalize inputs to [-1, 1] 27 | RGB_STD: [ 0.5, 0.5, 0.5 ] 28 | 29 | LRS: [ 0.1, 0.01, 0.001, 0.0001 ] # initial LR 30 | WARMUP_STEP: -1 31 | STAGES: [ 10, 18, 22 ] # epoch stages to decay learning rate 32 | 33 | START_EPOCH: 0 # start epoch 34 | NUM_EPOCH: 24 # total epoch number 35 | SAVE_EPOCHS: [ 1, 10, 16, 20, 24 ] 36 | 37 | 38 | WEIGHT_DECAY: 0.0005 # do not apply to batch_norm parameters 39 | MOMENTUM: 0.9 40 | 41 | WORLD_SIZE: 1 42 | RANK: 0 43 | LOCAL_RANK: 0 44 | DIST_BACKEND: 'nccl' 45 | DIST_URL: 'env://' 46 | 47 | NUM_WORKERS: 8 48 | 49 | AMP: false # fp16 for backbone 50 | 51 | # PartialFace 52 | METHOD: PartialFace 53 | NUM_AUG: 3 # r, multiplier for data augmentation 54 | NUM_CHS: 9 # s, channels within each rank, 9 by default 55 | # To modify S, P, please set them explicitly in partialface/utils.py 56 | -------------------------------------------------------------------------------- /recognition/tasks/slerpface/.gitignore: -------------------------------------------------------------------------------- 1 | templates 2 | ckpt 3 | pair_list 4 | 5 | # Python 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | *.so 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # Virtual Environment 28 | venv/ 29 | env/ 30 | ENV/ 31 | .env 32 | 33 | # IDE 34 | .idea/ 35 | .vscode/ 36 | *.swp 37 | *.swo 38 | 39 | # Logs 40 | *.log 41 | logs/ 42 | 43 | # Local configuration 44 | config.local.yaml 45 | *.local.json 46 | *.local.yml 47 | -------------------------------------------------------------------------------- /recognition/tasks/slerpface/encrypt_template.sh: -------------------------------------------------------------------------------- 1 | python3 -m tasks.slerpface.encrypt_template.enrollment \ 2 | --templates_folder ./tasks/slerpface/templates \ 3 | --key CFP \ 4 | --alpha 0.9 \ 5 | --drop_rate 0.5 \ 6 | --group_size 16 -------------------------------------------------------------------------------- /recognition/tasks/slerpface/extract_template.sh: -------------------------------------------------------------------------------- 1 | ### get SlerpFace model features ### 2 | python3 -um tasks.slerpface.extract_template.verification \ 3 | --model_path=./tasks/slerpface/ckpt/Backbone_Epoch_24_checkpoint.pth \ 4 | --data_root=/remote-home/share/yxmi/datasets/val_data \ 5 | --backbone=IR_50 \ 6 | --batch_size=512 \ 7 | --output_dir=./tasks/slerpface/templates \ 8 | --gpu_ids=0 -------------------------------------------------------------------------------- /recognition/tasks/slerpface/match_template.sh: -------------------------------------------------------------------------------- 1 | python3 -m tasks.slerpface.match_template.gen_sim \ 2 | --templates_folder ./tasks/slerpface/templates \ 3 | --key CFP \ 4 | --alpha 0.9 \ 5 | --group_size 16 -------------------------------------------------------------------------------- /recognition/tasks/slerpface/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/tasks/slerpface/pipeline.png -------------------------------------------------------------------------------- /recognition/tasks/slerpface/requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=1.7.1 2 | torchvision>=0.8.2 3 | numpy>=1.19.2 4 | scipy>=1.6.0 5 | pillow>=8.0.0 6 | tqdm>=4.50.0 7 | -------------------------------------------------------------------------------- /recognition/test/README.md: -------------------------------------------------------------------------------- 1 | ### Verification test 2 | 3 | LFW, CFP-FP, CPLFW, AgeDB, CALFW are popular verification datasets in face recognition task, below is test steps. 4 | 5 | 1. Download the bin files and save into `val_data_dir`, the test checkpoint path is `ckpt_path`. 6 | 2. Run test codes： 7 | ``` bash 8 | export CUDA_VISIBLE_DEVICES='0' 9 | python -u verification.py --ckpt_path=$ckpt_path --data_root=$val_data_dir 10 | ``` 11 | 12 | RFW is the common test datasets for fairness, the test code is `verification_rfw.py` 13 | 14 | ### 1:1 test 15 | 16 | IJB-B and IJB-C are most common large-scale face 1:1 test protocols. 17 | 1. Download the raw image data and meta files, saved into `data_root` 18 | 2. Run test codes： 19 | ``` bash 20 | # extract face features 21 | export CUDA_VISIBLE_DEVICES='0,1,2,3' 22 | python -u extract_features.py --ckpt_path=$ckpt_path --backbone=$backbone_name --gpu_ids=$gpu_ids \ 23 | --batch_size=512 --data_root=$data_root \ 24 | --filename_list=$pts_score_file \ 25 | --output_dir=$output_dir 26 | # evaluation 27 | python -u IJB_Evaluation.py --dataset=$dataset --meta_dir=$meta_dir \ 28 | --feature="${output_dir}"/"feature.npy" --face_scores=${output_dir}/"faceness_scores.npy" \ 29 | --output_name=${output_dir}/"similarity.npy" 30 | 31 | ``` -------------------------------------------------------------------------------- /recognition/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/recognition/test/__init__.py -------------------------------------------------------------------------------- /recognition/tools/convert_new_index.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | 4 | 5 | def parse_args(): 6 | parser = argparse.ArgumentParser( 7 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 8 | description='convert training index file') 9 | parser.add_argument('--old', default=None, type=str, required=True, 10 | help='path to old training list') 11 | parser.add_argument('--tfr_index', default=None, type=str, required=True, 12 | help='path to tfrecord index file') 13 | parser.add_argument('--new', default=None, type=str, required=True, 14 | help='path to new training list') 15 | args = parser.parse_args() 16 | return args 17 | 18 | 19 | def build_dict(tfr_index): 20 | d = {} 21 | print("reading {}".format(tfr_index)) 22 | tfr_name = os.path.basename(tfr_index).replace('.index', '') 23 | with open(tfr_index, 'r') as f: 24 | for line in f: 25 | file_name, shard_index, offset = line.rstrip().split('\t') 26 | d[file_name] = '{}\t{}\t{}'.format(tfr_name, shard_index, offset) 27 | print("build dict done") 28 | return d 29 | 30 | 31 | def convert(index_file, d, out_index_file): 32 | print("write to new index file {}".format(out_index_file)) 33 | with open(index_file, 'r') as f, open(out_index_file, 'w') as out_f: 34 | for line in f: 35 | if '\t' in line: 36 | file_name, label = line.rstrip().split('\t') 37 | else: 38 | file_name, label = line.rstrip().split(' ') 39 | tfr_string = d.get(file_name, None) 40 | if tfr_string is None: 41 | print(file_name + " failed") 42 | continue 43 | out_f.write(tfr_string + '\t{}'.format(label) + '\n') 44 | 45 | 46 | def main(): 47 | args = parse_args() 48 | d = build_dict(args.tfr_index) 49 | convert(args.old, d, args.new) 50 | 51 | 52 | if __name__ == '__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /recognition/tools/decode.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')) 4 | import argparse 5 | import struct 6 | from torchkit.data import example_pb2 7 | 8 | 9 | def parse_args(): 10 | parser = argparse.ArgumentParser( 11 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 12 | description='decode tfrecord') 13 | parser.add_argument('--tfrecords_dir', default=None, type=str, required=True, 14 | help='path to the output of tfrecords dir path') 15 | parser.add_argument('--output_dir', default=None, type=str, required=True, 16 | help='path to the output of decoded imgs') 17 | parser.add_argument('--limit', default=10, type=int, required=True, 18 | help='limit num of decoded samples') 19 | 20 | args = parser.parse_args() 21 | return args 22 | 23 | 24 | def parser(feature_list): 25 | for key, feature in feature_list: 26 | if key == 'image': 27 | image_raw = feature.bytes_list.value[0] 28 | return image_raw 29 | raise ValueError("No key=image in feature list") 30 | 31 | 32 | def get_record(record_file, offset): 33 | with open(record_file, 'rb') as ifs: 34 | ifs.seek(offset) 35 | byte_len_crc = ifs.read(12) 36 | proto_len = struct.unpack('Q', byte_len_crc[:8])[0] 37 | # proto,crc 38 | pb_data = ifs.read(proto_len) 39 | if len(pb_data) < proto_len: 40 | print("read pb_data err,proto_len:%s pb_data len:%s" % (proto_len, len(pb_data))) 41 | return None 42 | example = example_pb2.Example() 43 | example.ParseFromString(pb_data) 44 | # keep key value in order 45 | feature = sorted(example.features.feature.items()) 46 | record = parser(feature) 47 | return record 48 | 49 | 50 | def main(): 51 | args = parse_args() 52 | tfrecords_dir = os.path.normpath(args.tfrecords_dir) 53 | tfrecords_name = tfrecords_dir.split('/')[-1] 54 | output_dir = args.output_dir 55 | if not os.path.isdir(output_dir): 56 | os.makedirs(output_dir) 57 | limit = args.limit 58 | print(tfrecords_dir) 59 | print(tfrecords_name) 60 | 61 | index_file = os.path.join(tfrecords_dir, '%s.index' % tfrecords_name) 62 | with open(index_file, 'r') as f: 63 | for line_i, line in enumerate(f): 64 | if line_i >= limit: 65 | break 66 | raw_img_path, tfr_idx, tfr_offset = line.rstrip().split('\t') 67 | record_file = os.path.join(tfrecords_dir, "%s-%05d.tfrecord" % (tfrecords_name, int(tfr_idx))) 68 | image_raw = get_record(record_file, int(tfr_offset)) 69 | img_path = os.path.join(output_dir, os.path.basename(raw_img_path)) 70 | f = open(img_path, 'wb') 71 | f.write(image_raw) 72 | f.close() 73 | 74 | if __name__ == "__main__": 75 | main() 76 | -------------------------------------------------------------------------------- /recognition/torchkit/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.2.0_beta' 2 | -------------------------------------------------------------------------------- /recognition/torchkit/backbone/__init__.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from .model_resnet import ResNet_50, ResNet_101, ResNet_152 3 | from .model_irse import IR_18, IR_34, IR_50, IR_101, IR_152, IR_200 4 | from .model_irse import IR_SE_50, IR_SE_101, IR_SE_152, IR_SE_200 5 | from .model_mobilefacenet import MobileFaceNet 6 | from .model_efficientnet import EfficientNetB0, EfficientNetB1 7 | from .model_ghostnet import GhostNet 8 | from .fbnets.fbnet_builder import get_fbnet_model 9 | 10 | _model_dict = { 11 | 'ResNet_50': ResNet_50, 12 | 'ResNet_101': ResNet_101, 13 | 'ResNet_152': ResNet_152, 14 | 'IR_18': IR_18, 15 | 'IR_34': IR_34, 16 | 'IR_50': IR_50, 17 | 'IR_101': IR_101, 18 | 'IR_152': IR_152, 19 | 'IR_200': IR_200, 20 | 'IR_SE_50': IR_SE_50, 21 | 'IR_SE_101': IR_SE_101, 22 | 'IR_SE_152': IR_SE_152, 23 | 'IR_SE_200': IR_SE_200, 24 | 'MobileFaceNet': MobileFaceNet, 25 | 'EfficientNetB0': EfficientNetB0, 26 | 'EfficientNetB1': EfficientNetB1, 27 | 'GhostNet': GhostNet, 28 | 'fbnet_a': partial(get_fbnet_model, "fbnet_a"), 29 | 'fbnet_b': partial(get_fbnet_model, "fbnet_b"), 30 | 'fbnet_c': partial(get_fbnet_model, "fbnet_c"), 31 | } 32 | 33 | 34 | def get_model(key): 35 | """ Get different backbone network by key, 36 | support ResNet50, ResNet_101, ResNet_152 37 | IR_18, IR_34, IR_50, IR_101, IR_152, IR_200, 38 | IR_SE_50, IR_SE_101, IR_SE_152, IR_SE_200, 39 | EfficientNetB0, EfficientNetB1. 40 | MobileFaceNet, FBNets. 41 | """ 42 | if key in _model_dict.keys(): 43 | return _model_dict[key] 44 | else: 45 | raise KeyError("not support model {}".format(key)) 46 | -------------------------------------------------------------------------------- /recognition/torchkit/backbone/fbnets/layers/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .batch_norm import FrozenBatchNorm2d 4 | from .misc import Conv2d 5 | from .misc import ConvTranspose2d 6 | from .misc import BatchNorm2d 7 | from .misc import interpolate 8 | from .misc import _NewEmptyTensorOp 9 | 10 | 11 | __all__ = ["Conv2d", "ConvTranspose2d", "interpolate", 12 | "BatchNorm2d", "FrozenBatchNorm2d", "_NewEmptyTensorOp" 13 | ] 14 | -------------------------------------------------------------------------------- /recognition/torchkit/backbone/fbnets/layers/batch_norm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | class FrozenBatchNorm2d(nn.Module): 5 | """ 6 | BatchNorm2d where the batch statistics and the affine parameters 7 | are fixed 8 | """ 9 | 10 | def __init__(self, n): 11 | super(FrozenBatchNorm2d, self).__init__() 12 | self.register_buffer("weight", torch.ones(n)) 13 | self.register_buffer("bias", torch.zeros(n)) 14 | self.register_buffer("running_mean", torch.zeros(n)) 15 | self.register_buffer("running_var", torch.ones(n)) 16 | 17 | def forward(self, x): 18 | scale = self.weight * self.running_var.rsqrt() 19 | bias = self.bias - self.running_mean * scale 20 | scale = scale.reshape(1, -1, 1, 1) 21 | bias = bias.reshape(1, -1, 1, 1) 22 | return x * scale + bias 23 | -------------------------------------------------------------------------------- /recognition/torchkit/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .dataset import SingleDataset, MultiDataset 2 | from .parser import IndexParser, ImgSampleParser, TFRecordSampleParser 3 | from .sampler import MultiDistributedSampler 4 | 5 | __all__ = [ 6 | 'SingleDataset', 7 | 'MultiDataset', 8 | 'IndexParser', 9 | 'ImgSampleParser', 10 | 'TFRecordSampleParser', 11 | 'MultiDistributedSampler', 12 | ] 13 | -------------------------------------------------------------------------------- /recognition/torchkit/data/parser.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import dareblopy as db 4 | from . import example_pb2 5 | 6 | 7 | class IndexParser(object): 8 | """ Class for Line parser. 9 | """ 10 | def __init__(self) -> None: 11 | self.sample_num = 0 12 | self.class_num = 0 13 | 14 | def __call__(self, line): 15 | line_s = line.rstrip().split('\t') 16 | if len(line_s) == 2: 17 | # Default line format 18 | img_path, label = line_s 19 | label = int(label) 20 | self.sample_num += 1 21 | self.class_num = max(self.class_num, label) 22 | return (img_path, label) 23 | elif len(line_s) == 4: 24 | # IndexTFRDataset line format 25 | tfr_name, tfr_index, tfr_offset, label = line_s 26 | label = int(label) 27 | tfr_file = "{0}/{0}-{1:05d}.tfrecord".format(tfr_name, int(tfr_index)) 28 | tfr_offset = int(tfr_offset) 29 | self.sample_num += 1 30 | self.class_num = max(self.class_num, label) 31 | return (tfr_file, tfr_offset, label) 32 | else: 33 | raise RuntimeError("IndexParser line length %d not supported" % len(line_s)) 34 | 35 | def reset(self): 36 | self.sample_num = 0 37 | self.class_num = 0 38 | 39 | 40 | class ImgSampleParser(object): 41 | """ Class for Image Sample parser 42 | """ 43 | def __init__(self, transform) -> None: 44 | self.transform = transform 45 | 46 | def __call__(self, path, label): 47 | image = cv2.imread(path) 48 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 49 | if self.transform is not None: 50 | image = self.transform(image) 51 | return image, label 52 | 53 | 54 | class TFRecordSampleParser(object): 55 | """ Class for TFRecord Sample parser 56 | """ 57 | def __init__(self, transform) -> None: 58 | self.transform = transform 59 | self.file_readers = dict() 60 | 61 | def __call__(self, record_path, offset, label): 62 | rr = self.file_readers.get(record_path, None) 63 | if rr is None: 64 | rr = db.RecordReader(record_path) 65 | self.file_readers[record_path] = rr 66 | pb_data = rr.read_record(offset) 67 | example = example_pb2.Example() 68 | example.ParseFromString(pb_data) 69 | image_raw = example.features.feature['image'].bytes_list.value[0] 70 | image = cv2.imdecode(np.frombuffer(image_raw, np.uint8), cv2.IMREAD_COLOR) 71 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 72 | if self.transform is not None: 73 | image = self.transform(image) 74 | return image, label 75 | -------------------------------------------------------------------------------- /recognition/torchkit/head/__init__.py: -------------------------------------------------------------------------------- 1 | def get_head(key, dist_fc): 2 | """ Get different classification head functions by key, support NormFace CosFace, ArcFace, CurricularFace. 3 | If distfc is True, the weight is splited equally into all gpus and calculated in parallel 4 | """ 5 | if dist_fc: 6 | from .distfc.arcface import ArcFace 7 | from .distfc.cosface import CosFace 8 | from .distfc.curricularface import CurricularFace 9 | from .distfc.normface import NormFace 10 | from .distfc.partial_fc import PartialFC 11 | _head_dict = { 12 | 'CosFace': CosFace, 13 | 'ArcFace': ArcFace, 14 | 'CurricularFace': CurricularFace, 15 | 'NormFace': NormFace, 16 | 'PartialFC': PartialFC, 17 | } 18 | else: 19 | from .localfc.cosface import CosFace 20 | from .localfc.arcface import ArcFace 21 | from .localfc.curricularface import CurricularFace 22 | from .localfc.cifp import Cifp 23 | _head_dict = { 24 | 'CosFace': CosFace, 25 | 'ArcFace': ArcFace, 26 | 'CurricularFace': CurricularFace, 27 | 'Cifp': Cifp, 28 | } 29 | if key in _head_dict.keys(): 30 | return _head_dict[key] 31 | else: 32 | raise KeyError("not support head {}".format(key)) 33 | -------------------------------------------------------------------------------- /recognition/torchkit/head/distfc/arcface.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torchkit.util.utils import l2_norm 4 | from .common import CommonFace 5 | 6 | 7 | class ArcFace(CommonFace): 8 | """ Implement of ArcFace (https://arxiv.org/pdf/1801.07698v1.pdf) 9 | """ 10 | def __init__(self, 11 | in_features, 12 | gpu_index, 13 | weight_init, 14 | class_split, 15 | scale=64.0, 16 | margin=0.5, 17 | easy_margin=False): 18 | """ Args: 19 | in_features: size of input features 20 | gpu_index: gpu worker index for model parallel 21 | weight_init: weight initial value 22 | class_split: class num shards 23 | scale: scale of input feature 24 | margin: margin 25 | """ 26 | super(ArcFace, self).__init__(in_features, gpu_index, weight_init, class_split) 27 | self.scale = scale 28 | self.margin = margin 29 | self.easy_margin = easy_margin 30 | self.cos_m = math.cos(margin) 31 | self.sin_m = math.sin(margin) 32 | self.theta = math.cos(math.pi - margin) 33 | self.sinmm = math.sin(math.pi - margin) * margin 34 | 35 | def forward(self, embeddings, labels): 36 | index, part_labels, cos_theta, original_logits = self._calc_logits(embeddings, labels) 37 | target_logit = cos_theta[index, part_labels[index].view(-1)] 38 | 39 | sin_theta = torch.sqrt(1.0 - torch.pow(target_logit, 2)) 40 | cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m # cos(target+margin) 41 | if self.easy_margin: 42 | final_target_logit = torch.where(target_logit > 0, cos_theta_m, 43 | target_logit) 44 | else: 45 | final_target_logit = torch.where(target_logit > self.theta, 46 | cos_theta_m, 47 | target_logit - self.sinmm) 48 | 49 | cos_theta[index, part_labels[index].view(-1)] = final_target_logit 50 | cos_theta = cos_theta * self.scale 51 | 52 | return cos_theta, part_labels, original_logits * self.scale 53 | -------------------------------------------------------------------------------- /recognition/torchkit/head/distfc/common.py: -------------------------------------------------------------------------------- 1 | from itertools import accumulate 2 | import logging 3 | import torch 4 | import torch.nn as nn 5 | from torch.nn import Parameter 6 | from torchkit.util.utils import l2_norm 7 | 8 | 9 | class CommonFace(nn.Module): 10 | """ CommonFace head 11 | """ 12 | def __init__(self, 13 | in_features, 14 | gpu_index, 15 | weight_init, 16 | class_split): 17 | """ Args: 18 | in_features: size of input features 19 | gpu_index: gpu worker index for model parallel 20 | weight_init: weight initial value 21 | class_split: class num shards 22 | """ 23 | super(CommonFace, self).__init__() 24 | self.in_features = in_features 25 | self.gpu_index = gpu_index 26 | self.out_features = class_split[gpu_index] 27 | self.shard_start = [] 28 | self.shard_start.append(0) 29 | self.shard_start.extend(accumulate(class_split)) 30 | logging.info('FC Start Point: {}'.format(self.shard_start)) 31 | 32 | select_weight_init = weight_init[:, self.shard_start[self.gpu_index]: 33 | self.shard_start[self.gpu_index + 1]] 34 | 35 | self.kernel = Parameter(select_weight_init.clone()) 36 | 37 | def _calc_logits(self, embeddings, labels): 38 | """ calculate original logits 39 | """ 40 | embeddings = l2_norm(embeddings, axis=1) 41 | kernel_norm = l2_norm(self.kernel, axis=0) 42 | cos_theta = torch.mm(embeddings, kernel_norm) 43 | cos_theta = cos_theta.clamp(-1, 1) 44 | with torch.no_grad(): 45 | original_logits = cos_theta.clone() 46 | labels = labels.view(-1, 1) 47 | part_labels = self._generate_part_labels(labels) 48 | index = torch.where(part_labels != -1)[0] 49 | return index, part_labels, cos_theta, original_logits 50 | 51 | def _generate_part_labels(self, labels): 52 | with torch.no_grad(): 53 | part_labels = labels.clone() 54 | shad_start = self.shard_start[self.gpu_index] 55 | shad_end = self.shard_start[self.gpu_index + 1] 56 | label_mask = torch.ge(part_labels, shad_start) & torch.lt(part_labels, shad_end) 57 | 58 | part_labels[~label_mask] = -1 59 | part_labels[label_mask] -= shad_start 60 | 61 | return part_labels 62 | 63 | def forward(self, embeddings, labels): 64 | raise NotImplementedError() 65 | -------------------------------------------------------------------------------- /recognition/torchkit/head/distfc/cosface.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchkit.util.utils import l2_norm 3 | from .common import CommonFace 4 | 5 | 6 | class CosFace(CommonFace): 7 | """ Implement of CosFace (https://arxiv.org/abs/1801.09414) 8 | """ 9 | def __init__(self, 10 | in_features, 11 | gpu_index, 12 | weight_init, 13 | class_split, 14 | scale=64.0, 15 | margin=0.4): 16 | """ Args: 17 | in_features: size of input features 18 | gpu_index: gpu worker index for model parallel 19 | weight_init: weight initial value 20 | class_split: class num shards 21 | scale: scale of input feature 22 | margin: margin 23 | """ 24 | super(CosFace, self).__init__(in_features, gpu_index, weight_init, class_split) 25 | 26 | self.scale = scale 27 | self.margin = margin 28 | 29 | def forward(self, embeddings, labels): 30 | index, part_labels, cos_theta, original_logits = self._calc_logits(embeddings, labels) 31 | target_logit = cos_theta[index, part_labels[index].view(-1)] 32 | 33 | final_target_logit = target_logit - self.margin 34 | 35 | cos_theta[index, part_labels[index].view(-1)] = final_target_logit 36 | cos_theta = cos_theta * self.scale 37 | 38 | return cos_theta, part_labels, original_logits * self.scale 39 | -------------------------------------------------------------------------------- /recognition/torchkit/head/distfc/curricularface.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch.distributed import ReduceOp 4 | from torchkit.util.utils import l2_norm 5 | from .common import CommonFace 6 | 7 | 8 | class CurricularFace(CommonFace): 9 | """ Implement of CurricularFace (https://arxiv.org/abs/2004.00288) 10 | """ 11 | def __init__(self, 12 | in_features, 13 | gpu_index, 14 | weight_init, 15 | class_split, 16 | scale=64.0, 17 | margin=0.5, 18 | alpha=0.1): 19 | """ Args: 20 | in_features: size of input features 21 | gpu_index: gpu worker index for model parallel 22 | weight_init: weight initial value 23 | class_split: class num shards 24 | scale: scale of input feature 25 | margin: margin 26 | alpha: alpha 27 | """ 28 | super(CurricularFace, self).__init__(in_features, gpu_index, weight_init, class_split) 29 | self.scale = scale 30 | self.margin = margin 31 | self.alpha = alpha 32 | self.cos_m = math.cos(margin) 33 | self.sin_m = math.sin(margin) 34 | self.theta = math.cos(math.pi - margin) 35 | self.sinmm = math.sin(math.pi - margin) * margin 36 | self.register_buffer('t', torch.zeros(1)) 37 | 38 | def forward(self, embeddings, labels): 39 | index, part_labels, cos_theta, original_logits = self._calc_logits(embeddings, labels) 40 | target_logit = torch.zeros(embeddings.size(0), device=embeddings.device) 41 | 42 | target_logit[index] = cos_theta[index, part_labels[index].view(-1)] 43 | 44 | torch.distributed.all_reduce(target_logit, ReduceOp.SUM) 45 | 46 | sin_theta = torch.sqrt(1.0 - torch.pow(target_logit, 2)) 47 | cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m # cos(target+margin) 48 | 49 | hard_sample_mask = cos_theta > cos_theta_m.view(-1, 1) 50 | # print('hard_sample_mask', hard_sample_mask.size()) 51 | hard_example = cos_theta[hard_sample_mask] 52 | final_target_logit = torch.where(target_logit > self.theta, 53 | cos_theta_m, 54 | target_logit - self.sinmm) 55 | with torch.no_grad(): 56 | self.t = target_logit.mean() * self.alpha + (1 - self.alpha) * self.t 57 | cos_theta[hard_sample_mask] = hard_example * (self.t + hard_example) 58 | cos_theta[index, part_labels[index].view(-1)] = final_target_logit[index] 59 | cos_theta = cos_theta * self.scale 60 | return cos_theta, part_labels, original_logits * self.scale 61 | -------------------------------------------------------------------------------- /recognition/torchkit/head/distfc/normface.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchkit.util.utils import l2_norm 3 | from .common import CommonFace 4 | 5 | 6 | class NormFace(CommonFace): 7 | """ Implement of NormFace 8 | """ 9 | def __init__(self, 10 | in_features, 11 | gpu_index, 12 | weight_init, 13 | class_split, 14 | scale=64.): 15 | """ Args: 16 | in_features: size of input features 17 | gpu_index: gpu worker index for model parallel 18 | weight_init: weight initial value 19 | class_split: class num shards 20 | scale: scale of input feature 21 | """ 22 | super(NormFace).__init__(in_features, gpu_index, weight_init, class_split) 23 | self.scale = scale 24 | 25 | def forward(self, inputs, labels): 26 | inputs_norm = l2_norm(inputs, axis=1) 27 | kernel_norm = l2_norm(self.kernel, axis=0) 28 | cos_theta = torch.mm(inputs_norm, kernel_norm) 29 | cos_theta = cos_theta.clamp(-1, 1) 30 | output = cos_theta * self.scale 31 | 32 | return output 33 | -------------------------------------------------------------------------------- /recognition/torchkit/head/localfc/arcface.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | import math 4 | import torch 5 | import torch.nn as nn 6 | from torch.nn import Parameter 7 | from torchkit.head.localfc.common import calc_logits 8 | 9 | 10 | class ArcFace(nn.Module): 11 | """ Implement of ArcFace (https://arxiv.org/pdf/1801.07698v1.pdf) 12 | """ 13 | def __init__(self, 14 | in_features, 15 | out_features, 16 | scale=64.0, 17 | margin=0.5, 18 | easy_margin=False): 19 | """ Args: 20 | in_features: size of input features 21 | out_features: size of output features 22 | scale: scale of input feature 23 | margin: margin 24 | """ 25 | super(ArcFace, self).__init__() 26 | self.in_features = in_features 27 | self.out_features = out_features 28 | 29 | self.scale = scale 30 | self.margin = margin 31 | 32 | self.kernel = Parameter(torch.FloatTensor(in_features, out_features)) 33 | # nn.init.xavier_uniform_(self.kernel) 34 | nn.init.normal_(self.kernel, std=0.01) 35 | # init.kaiming_uniform_(self.kernel, a=math.sqrt(5)) 36 | 37 | self.easy_margin = easy_margin 38 | self.cos_m = math.cos(margin) 39 | self.sin_m = math.sin(margin) 40 | self.th = math.cos(math.pi - margin) 41 | self.mm = math.sin(math.pi - margin) * margin 42 | 43 | def forward(self, embeddings, labels): 44 | cos_theta, origin_cos = calc_logits(embeddings, self.kernel) 45 | target_logit = cos_theta[torch.arange(0, embeddings.size(0)), labels].view(-1, 1) 46 | 47 | sin_theta = torch.sqrt(1.0 - torch.pow(target_logit, 2)) 48 | cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m # cos(target+margin) 49 | if self.easy_margin: 50 | final_target_logit = torch.where(target_logit > 0, cos_theta_m, target_logit) 51 | else: 52 | final_target_logit = torch.where(target_logit > self.th, cos_theta_m, target_logit - self.mm) 53 | 54 | cos_theta.scatter_(1, labels.view(-1, 1).long(), final_target_logit) 55 | output = cos_theta * self.scale 56 | 57 | return output, origin_cos * self.scale 58 | -------------------------------------------------------------------------------- /recognition/torchkit/head/localfc/common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torchkit.util.utils import l2_norm 4 | 5 | 6 | def calc_logits(embeddings, kernel): 7 | """ calculate original logits 8 | """ 9 | embeddings = l2_norm(embeddings, axis=1) 10 | kernel_norm = l2_norm(kernel, axis=0) 11 | cos_theta = torch.mm(embeddings, kernel_norm) 12 | cos_theta = cos_theta.clamp(-1, 1) # for numerical stability 13 | with torch.no_grad(): 14 | origin_cos = cos_theta.clone() 15 | return cos_theta, origin_cos 16 | -------------------------------------------------------------------------------- /recognition/torchkit/head/localfc/cosface.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | import math 4 | import torch 5 | import torch.nn as nn 6 | from torch.nn import Parameter 7 | from torchkit.head.localfc.common import calc_logits 8 | 9 | 10 | class CosFace(nn.Module): 11 | """ Implement of CosFace (https://arxiv.org/abs/1801.09414) 12 | 13 | """ 14 | def __init__(self, 15 | in_features, 16 | out_features, 17 | scale=64.0, 18 | margin=0.40): 19 | """ Args: 20 | in_features: size of each input features 21 | out_features: size of each output features 22 | scale: norm of input feature 23 | margin: margin 24 | """ 25 | super(CosFace, self).__init__() 26 | self.in_features = in_features 27 | self.out_features = out_features 28 | 29 | self.scale = scale 30 | self.margin = margin 31 | 32 | self.kernel = Parameter(torch.FloatTensor(in_features, out_features)) 33 | # nn.init.xavier_uniform_(self.kernel) 34 | nn.init.normal_(self.kernel, std=0.01) 35 | # init.kaiming_uniform_(self.kernel, a=math.sqrt(5)) 36 | 37 | def forward(self, embeddings, labels): 38 | cos_theta, origin_cos = calc_logits(embeddings, self.kernel) 39 | target_logit = cos_theta[torch.arange(0, embeddings.size(0)), labels].view(-1, 1) 40 | 41 | final_target_logit = target_logit - self.margin 42 | 43 | cos_theta.scatter_(1, labels.view(-1, 1).long(), final_target_logit) 44 | output = cos_theta * self.scale 45 | 46 | return output, origin_cos * self.scale 47 | -------------------------------------------------------------------------------- /recognition/torchkit/head/localfc/curricularface.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | import torch 4 | import torch.nn as nn 5 | from torch.nn import Parameter 6 | import math 7 | from torchkit.util.utils import l2_norm 8 | from torchkit.head.localfc.common import calc_logits 9 | 10 | 11 | class CurricularFace(nn.Module): 12 | """ Implement of CurricularFace (https://arxiv.org/abs/2004.00288) 13 | """ 14 | 15 | def __init__(self, 16 | in_features, 17 | out_features, 18 | scale=64.0, 19 | margin=0.5, 20 | alpha=0.1): 21 | """ Args: 22 | in_features: size of each input features 23 | out_features: size of each output features 24 | scale: norm of input feature 25 | margin: margin 26 | """ 27 | super(CurricularFace, self).__init__() 28 | self.in_features = in_features 29 | self.out_features = out_features 30 | self.margin = margin 31 | self.scale = scale 32 | self.alpha = alpha 33 | self.cos_m = math.cos(margin) 34 | self.sin_m = math.sin(margin) 35 | self.threshold = math.cos(math.pi - margin) 36 | self.mm = math.sin(math.pi - margin) * margin 37 | self.kernel = Parameter(torch.Tensor(in_features, out_features)) 38 | self.register_buffer('t', torch.zeros(1)) 39 | nn.init.normal_(self.kernel, std=0.01) 40 | 41 | def forward(self, embeddings, labels): 42 | cos_theta, origin_cos = calc_logits(embeddings, self.kernel) 43 | target_logit = cos_theta[torch.arange(0, embeddings.size(0)), labels].view(-1, 1) 44 | 45 | sin_theta = torch.sqrt(1.0 - torch.pow(target_logit, 2)) 46 | cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m # cos(target+margin) 47 | mask = cos_theta > cos_theta_m 48 | final_target_logit = torch.where(target_logit > self.threshold, cos_theta_m, target_logit - self.mm) 49 | 50 | hard_example = cos_theta[mask] 51 | with torch.no_grad(): 52 | self.t = target_logit.mean() * self.alpha + (1 - self.alpha) * self.t 53 | cos_theta[mask] = hard_example * (self.t + hard_example) 54 | cos_theta.scatter_(1, labels.view(-1, 1).long(), final_target_logit) 55 | output = cos_theta * self.scale 56 | 57 | return output, origin_cos * self.scale 58 | -------------------------------------------------------------------------------- /recognition/torchkit/hooks/__init__.py: -------------------------------------------------------------------------------- 1 | from .checkpoint_hook import CheckpointHook 2 | from .log_hook import LogHook 3 | from .summary_hook import SummaryHook 4 | from .learning_rate_hook import LearningRateHook 5 | 6 | __all__ = [ 7 | 'CheckpointHook', 8 | 'LogHook', 9 | 'SummaryHook', 10 | 'LearningRateHook', 11 | ] 12 | -------------------------------------------------------------------------------- /recognition/torchkit/hooks/base_hook.py: -------------------------------------------------------------------------------- 1 | class Hook(object): 2 | """ BaseClass for Hook 3 | """ 4 | stages = ('before_run', 'before_train_epoch', 'before_train_iter', 5 | 'after_train_iter', 'after_train_epoch', 'after_run') 6 | 7 | def before_run(self, *args): 8 | pass 9 | 10 | def before_train_epoch(self, *args): 11 | pass 12 | 13 | def before_train_iter(self, *args): 14 | pass 15 | 16 | def after_train_iter(self, *args): 17 | pass 18 | 19 | def after_train_epoch(self, *args): 20 | pass 21 | 22 | def after_run(self, *args): 23 | pass 24 | -------------------------------------------------------------------------------- /recognition/torchkit/hooks/checkpoint_hook.py: -------------------------------------------------------------------------------- 1 | from .base_hook import Hook 2 | 3 | 4 | class CheckpointHook(Hook): 5 | """ CheckpointHook 6 | """ 7 | def __init__(self, save_epochs): 8 | self.save_epochs = save_epochs 9 | 10 | def before_run(self, task): 11 | task.load_pretrain_model() 12 | 13 | def after_train_epoch(self, task, epoch): 14 | if epoch + 1 in self.save_epochs: 15 | task.save_ckpt(epoch + 1) 16 | -------------------------------------------------------------------------------- /recognition/torchkit/hooks/learning_rate_hook.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from bisect import bisect 3 | from .base_hook import Hook 4 | 5 | 6 | def set_optimizer_lr(optimizer, lr): 7 | if isinstance(optimizer, dict): 8 | backbone_opt, head_opts = optimizer['backbone'], optimizer['heads'] 9 | for param_group in backbone_opt.param_groups: 10 | param_group['lr'] = lr 11 | for _, head_opt in head_opts.items(): 12 | for param_group in head_opt.param_groups: 13 | param_group['lr'] = lr 14 | else: 15 | for param_group in optimizer.param_groups: 16 | param_group['lr'] = lr 17 | 18 | 19 | def warm_up_lr(step, warmup_step, init_lr, optimizer): 20 | """ Warm up learning rate when batch step below warmup steps 21 | """ 22 | 23 | lr = step * init_lr / warmup_step 24 | if step % 500 == 0: 25 | logging.info("Current step {}, learning rate {}".format(step, lr)) 26 | 27 | set_optimizer_lr(optimizer, lr) 28 | 29 | 30 | def adjust_lr(epoch, learning_rates, stages, optimizer): 31 | """ Decay the learning rate based on schedule 32 | """ 33 | 34 | pos = bisect(stages, epoch) 35 | lr = learning_rates[pos] 36 | logging.info("Current epoch {}, learning rate {}".format(epoch + 1, lr)) 37 | 38 | set_optimizer_lr(optimizer, lr) 39 | 40 | 41 | class LearningRateHook(Hook): 42 | """ LearningRate Hook, adjust learning rate in training 43 | """ 44 | def __init__(self, 45 | learning_rates, 46 | stages, 47 | warmup_step): 48 | """ Create a ``LearningRateHook`` object 49 | 50 | Args: 51 | learning_rates: all learning rates value 52 | stages: learning rate adjust stages value 53 | warmup_step: step num of warmup 54 | """ 55 | 56 | self.learning_rates = learning_rates 57 | self.stages = stages 58 | if len(self.learning_rates) != len(self.stages) + 1: 59 | raise RuntimeError("Learning_rates size should be one larger than stages size") 60 | self.init_lr = self.learning_rates[0] 61 | self.warmup_step = warmup_step 62 | 63 | def before_train_iter(self, task, step, epoch): 64 | global_step = epoch * task.step_per_epoch + step 65 | if self.warmup_step > 0 and global_step <= self.warmup_step: 66 | warm_up_lr(global_step, self.warmup_step, self.init_lr, task.opt) 67 | 68 | def before_train_epoch(self, task, epoch): 69 | adjust_lr(epoch, self.learning_rates, self.stages, task.opt) 70 | -------------------------------------------------------------------------------- /recognition/torchkit/hooks/log_hook.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from ..util import AverageMeter 3 | from .base_hook import Hook 4 | 5 | 6 | class LogHook(Hook): 7 | """ LogHook, print log info in training 8 | """ 9 | def __init__(self, freq, rank): 10 | """ Create a LogHook object 11 | 12 | Args: 13 | freq: step interval 14 | rank: work rank in ddp 15 | """ 16 | 17 | self.freq = freq 18 | self.rank = rank 19 | 20 | def before_train_epoch(self, task, epoch): 21 | task.log_buffer.clear() 22 | 23 | def after_train_iter(self, task, step, epoch): 24 | """ Print log info after every training step 25 | """ 26 | 27 | if self.rank != 0: 28 | return 29 | if step == 0 or (step + 1) % self.freq == 0: 30 | time_cost = task.log_buffer['time_cost'] 31 | logging.info("Epoch {} / {}, batch {} / {}, {:.4f} sec/batch".format( 32 | epoch + 1, task.epoch_num, step + 1, task.step_per_epoch, time_cost)) 33 | 34 | log_str = " " * 25 35 | for k, v in task.log_buffer.items(): 36 | if k == 'time_cost': 37 | continue 38 | if isinstance(v, list): 39 | s = ', '.join(['%.6f' % x.val for x in v]) 40 | elif isinstance(v, AverageMeter): 41 | s = '%.6f' % v.val 42 | else: 43 | s = str(v) 44 | log_str += '%s = [%s] ' % (k, s) 45 | print(log_str) 46 | -------------------------------------------------------------------------------- /recognition/torchkit/hooks/summary_hook.py: -------------------------------------------------------------------------------- 1 | from torch.utils.tensorboard.writer import SummaryWriter 2 | from ..util import AverageMeter 3 | from .base_hook import Hook 4 | 5 | 6 | class SummaryHook(Hook): 7 | """ SummaryHook, write tensorboard summery in training 8 | """ 9 | def __init__(self, log_root, freq, rank): 10 | """ Create A SummaryHook object 11 | 12 | Args: 13 | log_root: tensorboard summary root path 14 | freq: step interval 15 | rank: gpu rank 16 | """ 17 | 18 | self.writer = SummaryWriter(log_root) if rank == 0 else None 19 | self.freq = freq 20 | 21 | def after_train_iter(self, task, step, epoch): 22 | if self.writer is None: 23 | return 24 | if step == 0 or (step + 1) % self.freq == 0: 25 | global_step = step + epoch * task.step_per_epoch 26 | scalars = task.summary.get('scalars', {}) 27 | for k, v in scalars.items(): 28 | if isinstance(v, list): 29 | for i, x in enumerate(v): 30 | self.writer.add_scalar('%s_%d' % (k, i), x.val, 31 | global_step=global_step) 32 | elif isinstance(v, AverageMeter): 33 | self.writer.add_scalar(k, v.val, global_step=global_step) 34 | 35 | histograms = task.summary.get('histograms', {}) 36 | for k, v in histograms.items(): 37 | if isinstance(v, list): 38 | for i, x in enumerate(v): 39 | self.writer.add_histogram('%s_%d' % (k, i), x.val, 40 | global_step=global_step) 41 | elif isinstance(v, AverageMeter): 42 | self.writer.add_histogram(k, v.val, global_step=global_step) 43 | 44 | def after_run(self, *args): 45 | if self.writer: 46 | self.writer.close() 47 | -------------------------------------------------------------------------------- /recognition/torchkit/loss/__init__.py: -------------------------------------------------------------------------------- 1 | from torch.nn import CrossEntropyLoss 2 | from .dist_softmax import DistCrossEntropy 3 | from .ddl import DDL 4 | 5 | _loss_dict = { 6 | 'Softmax': CrossEntropyLoss(), 7 | 'DistCrossEntropy': DistCrossEntropy(), 8 | 'DDL': DDL() 9 | } 10 | 11 | 12 | def get_loss(key): 13 | """ Get different training loss functions by key, 14 | support Softmax(distfc = False), DistCrossEntropy (distfc = True), and DDL. 15 | """ 16 | if key in _loss_dict.keys(): 17 | return _loss_dict[key] 18 | else: 19 | raise KeyError("not support loss {}".format(key)) 20 | -------------------------------------------------------------------------------- /recognition/torchkit/loss/dist_softmax.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.distributed as dist 4 | from torch.distributed import ReduceOp 5 | 6 | 7 | class DistCrossEntropyFunc(torch.autograd.Function): 8 | """ CrossEntropy loss is calculated in parallel, allreduce all logits into single gpu and calculate loss. 9 | Implemented of ArcFace (https://arxiv.org/pdf/1801.07698v1.pdf): 10 | """ 11 | @staticmethod 12 | def forward(ctx, logit_part, part_labels): 13 | ctx.batch_size = logit_part.size(0) 14 | # for numerical stability 15 | logit_part_max, _ = torch.max(logit_part, dim=1, keepdim=True) 16 | dist.all_reduce(logit_part_max, ReduceOp.MAX) 17 | logit_part = logit_part - logit_part_max 18 | 19 | # get exp sum 20 | exp_logit = torch.exp(logit_part) 21 | exp_sum = torch.sum(exp_logit, dim=1, keepdim=True) 22 | torch.distributed.all_reduce(exp_sum, ReduceOp.SUM) 23 | log_exp_sum = torch.log(exp_sum) 24 | log_softmax = logit_part - log_exp_sum 25 | ctx.log_softmax = log_softmax 26 | index = torch.where(part_labels != -1)[0] 27 | label_mask = torch.zeros(index.size()[0], 28 | log_softmax.size()[1], 29 | device=log_softmax.device) 30 | label_mask.scatter_(1, part_labels[index], 1) 31 | ctx.label_mask = label_mask 32 | ctx.index = index 33 | 34 | loss = torch.zeros(log_softmax.size()[0], 1, device=log_softmax.device) 35 | loss[index] = log_softmax[index].gather(1, part_labels[index]) 36 | torch.distributed.all_reduce(loss, ReduceOp.SUM) 37 | loss = -torch.mean(loss) 38 | 39 | return loss 40 | 41 | @staticmethod 42 | def backward(ctx, loss_grad): 43 | """ 44 | Args: 45 | loss_grad (torch.Tensor): gradient backward by last layer 46 | Returns: 47 | gradients for each input in forward function 48 | `None` gradients for one-hot label 49 | """ 50 | logit_grad = torch.exp(ctx.log_softmax) 51 | logit_grad[ctx.index] -= ctx.label_mask 52 | logit_grad = loss_grad.item() * logit_grad / ctx.batch_size 53 | return logit_grad, None 54 | 55 | 56 | class DistCrossEntropy(nn.Module): 57 | def __init__(self): 58 | super(DistCrossEntropy, self).__init__() 59 | 60 | def forward(self, logit_part, label_part): 61 | return DistCrossEntropyFunc.apply(logit_part, label_part) 62 | -------------------------------------------------------------------------------- /recognition/torchkit/task/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_task import BaseTask 2 | 3 | __all__ = ['BaseTask'] 4 | -------------------------------------------------------------------------------- /recognition/torchkit/tfrecord/protos/example.proto: -------------------------------------------------------------------------------- 1 | // Protocol messages for describing input data Examples for machine learning 2 | // model training or inference. 3 | syntax = "proto3"; 4 | 5 | import "feature.proto"; 6 | option cc_enable_arenas = true; 7 | 8 | package face; 9 | 10 | message Example { 11 | Features features = 1; 12 | }; 13 | 14 | message SequenceExample { 15 | Features context = 1; 16 | FeatureLists feature_lists = 2; 17 | }; 18 | -------------------------------------------------------------------------------- /recognition/torchkit/tfrecord/protos/feature.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option cc_enable_arenas = true; 3 | 4 | package face; 5 | 6 | // Containers to hold repeated fundamental values. 7 | message BytesList { 8 | repeated bytes value = 1; 9 | } 10 | message FloatList { 11 | repeated float value = 1 [packed = true]; 12 | } 13 | message Int64List { 14 | repeated int64 value = 1 [packed = true]; 15 | } 16 | 17 | // Containers for non-sequential data. 18 | message Feature { 19 | // Each feature can be exactly one kind. 20 | oneof kind { 21 | BytesList bytes_list = 1; 22 | FloatList float_list = 2; 23 | Int64List int64_list = 3; 24 | } 25 | }; 26 | 27 | message Features { 28 | // Map from feature name to feature. 29 | map feature = 1; 30 | }; 31 | 32 | message FeatureList { 33 | repeated Feature feature = 1; 34 | }; 35 | 36 | message FeatureLists { 37 | // Map from feature name to feature list. 38 | map feature_list = 1; 39 | }; 40 | -------------------------------------------------------------------------------- /recognition/torchkit/util/__init__.py: -------------------------------------------------------------------------------- 1 | from .checkpoint import CkptLoader, CkptSaver 2 | from .utils import AverageMeter, Timer 3 | from .utils import separate_irse_bn_paras, separate_resnet_bn_paras 4 | from .utils import load_config, get_class_split 5 | from .utils import accuracy_dist, accuracy 6 | from .distributed_functions import AllGather 7 | 8 | __all__ = [ 9 | 'CkptLoader', 10 | 'CkptSaver', 11 | 'AverageMeter', 12 | 'Timer', 13 | 'separate_irse_bn_paras', 14 | 'separate_resnet_bn_paras', 15 | 'load_config', 16 | 'get_class_split', 17 | 'accuracy_dist', 18 | 'accuracy', 19 | 'AllGather', 20 | ] 21 | -------------------------------------------------------------------------------- /recognition/torchkit/util/distributed_functions.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributed as dist 3 | from torch.autograd import Function 4 | from torch.distributed import ReduceOp 5 | 6 | 7 | class AllGatherFunc(Function): 8 | """ AllGather op with gradient backward 9 | """ 10 | @staticmethod 11 | def forward(ctx, tensor, world_size): 12 | gather_list = [torch.zeros_like(tensor) for _ in range(world_size)] 13 | dist.all_gather(gather_list, tensor) 14 | return tuple(gather_list) 15 | 16 | @staticmethod 17 | def backward(ctx, *grads): 18 | world_size = dist.get_world_size() 19 | rank = dist.get_rank() 20 | grad_list = list(grads) 21 | grad_out = torch.zeros_like(grad_list[rank], requires_grad=True) 22 | dist.reduce_scatter(grad_out, grad_list, op=ReduceOp.SUM) 23 | # Gradient correction for DistCrossEntropy 24 | grad_out = grad_out * world_size 25 | return (grad_out, None) 26 | 27 | 28 | AllGather = AllGatherFunc.apply 29 | -------------------------------------------------------------------------------- /recognition/train.yaml: -------------------------------------------------------------------------------- 1 | SEED: 1337 # random seed for reproduce results 2 | DATA_ROOT: '/youtu-face/ata_root/remote_tfrecord' # the parent root where your train/val/test data are stored 3 | INDEX_ROOT: '/youtu-face/train_data/' 4 | DATASETS: # the dataset index name 5 | - name: TFR-deepglint_360k_new 6 | batch_size: 128 7 | weight: 1.0 8 | scale: 64 9 | margin: 0.4 10 | 11 | BACKBONE_RESUME: "" 12 | HEAD_RESUME: "" 13 | META_RESUME: "" 14 | 15 | # BACKBONE_NAME: 'EfficientNetB0' 16 | INPUT_SIZE: [112, 112] # support: [112, 112] and [224, 224] 17 | BACKBONE_NAME: 'IR_50' 18 | EMBEDDING_SIZE: 512 # feature dimension 19 | 20 | MODEL_ROOT: './ckpt/' # the root to buffer your checkpoints 21 | LOG_ROOT: './tensorboard' # the root to log your train/val status 22 | 23 | DIST_FC: true 24 | HEAD_NAME: "PartialFC" # support: ['ArcFace', 'CurricularFace', 'CosFace'] 25 | LOSS_NAME: 'DistCrossEntropy' # support: ['DistCrossEntropy', 'Softmax'] 26 | 27 | RGB_MEAN: [0.5, 0.5, 0.5] # for normalize inputs to [-1, 1] 28 | RGB_STD: [0.5, 0.5, 0.5] 29 | 30 | LRS: [0.1, 0.01, 0.001, 0.0001, 0.00001] # initial LR 31 | WARMUP_STEP: -1 32 | STAGES: [8, 12, 15, 18] # epoch stages to decay learning rate 33 | 34 | START_EPOCH: 0 # start epoch 35 | NUM_EPOCH: 20 # total epoch number 36 | SAVE_EPOCHS: [1, 15, 18, 20] 37 | 38 | WEIGHT_DECAY: 0.0005 # do not apply to batch_norm parameters 39 | MOMENTUM: 0.9 40 | 41 | WORLD_SIZE: 1 42 | RANK: 0 43 | LOCAL_RANK: 0 44 | DIST_BACKEND: 'nccl' 45 | DIST_URL: 'env://' 46 | 47 | NUM_WORKERS: 8 48 | 49 | AMP: true # fp16 for backbone 50 | -------------------------------------------------------------------------------- /security/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/common/__init__.py -------------------------------------------------------------------------------- /security/common/data/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - create_base_transforms (function): 4 | create base transforms for training, validation and testing 5 | - create_base_dataloader (function): 6 | create base dataloader for training, validation and testing 7 | """ 8 | from .base_transform import create_base_transforms 9 | from .base_dataloader import create_base_dataloader 10 | -------------------------------------------------------------------------------- /security/common/data/base_dataloader.py: -------------------------------------------------------------------------------- 1 | import torch.utils.data as data 2 | 3 | 4 | def create_base_dataloader(args, dataset, split): 5 | """Base data loader 6 | 7 | Args: 8 | args: Dataset config args 9 | split (string): Load "train", "val" or "test" 10 | 11 | Returns: 12 | [dataloader]: Corresponding Dataloader 13 | """ 14 | sampler = None 15 | if args.distributed: 16 | sampler = data.distributed.DistributedSampler(dataset) 17 | 18 | shuffle = True if sampler is None and split == 'train' else False 19 | batch_size = getattr(args, split).batch_size 20 | num_workers = args.num_workers if 'num_workers' in args else 8 21 | 22 | dataloader = data.DataLoader(dataset, 23 | batch_size=batch_size, 24 | shuffle=shuffle, 25 | sampler=sampler, 26 | num_workers=num_workers, 27 | pin_memory=True, 28 | drop_last=True) 29 | return dataloader 30 | -------------------------------------------------------------------------------- /security/common/data/base_transform.py: -------------------------------------------------------------------------------- 1 | import albumentations as alb 2 | from albumentations.pytorch.transforms import ToTensorV2 3 | 4 | 5 | def create_base_transforms(args, split='train'): 6 | """Base data transformation 7 | 8 | Args: 9 | args: Data transformation args 10 | split (str, optional): Defaults to 'train'. 11 | 12 | Returns: 13 | [transform]: Data transform 14 | """ 15 | num_segments = args.num_segments if 'num_segments' in args else 1 16 | additional_targets = {} 17 | for i in range(1, num_segments): 18 | additional_targets[f'image{i}'] = 'image' 19 | 20 | if split == 'train': 21 | base_transform = alb.Compose([ 22 | alb.HorizontalFlip(), 23 | alb.Resize(args.image_size, args.image_size), 24 | alb.Normalize(mean=args.mean, std=args.std), 25 | ToTensorV2(), 26 | ], additional_targets=additional_targets) 27 | 28 | elif split == 'val': 29 | base_transform = alb.Compose([ 30 | alb.Resize(args.image_size, args.image_size), 31 | alb.Normalize(mean=args.mean, std=args.std), 32 | ToTensorV2(), 33 | ], additional_targets=additional_targets) 34 | 35 | elif split == 'test': 36 | base_transform = alb.Compose([ 37 | alb.Resize(args.image_size, args.image_size), 38 | alb.Normalize(mean=args.mean, std=args.std), 39 | ToTensorV2(), 40 | ], additional_targets=additional_targets) 41 | 42 | return base_transform 43 | -------------------------------------------------------------------------------- /security/common/losses/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the loss functions used in the project: 3 | - pytorch official losses 4 | - euclidean_loss 5 | """ 6 | from torch.nn.modules.loss import * 7 | from .euclidean_loss import EuclideanLoss 8 | -------------------------------------------------------------------------------- /security/common/losses/euclidean_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class EuclideanLoss(nn.Module): 6 | '''Compute euclidean distance between two tensors 7 | ''' 8 | 9 | def __init__(self, reduction=None): 10 | super(EuclideanLoss, self).__init__() 11 | self.reduction = reduction 12 | 13 | def forward(self, x, y): 14 | 15 | n = x.size(0) 16 | m = n 17 | d = x.size(1) 18 | y = y.unsqueeze(0).expand(n, d) 19 | 20 | x = x.unsqueeze(1).expand(n, m, d) 21 | y = y.unsqueeze(0).expand(n, m, d) 22 | 23 | if self.reduction == 'mean': 24 | return torch.pow(x - y, 2).mean() 25 | 26 | elif self.reduction == 'sum': 27 | return torch.pow(x - y, 2).sum() 28 | -------------------------------------------------------------------------------- /security/common/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the optimizers used in the project: 3 | - pytorch official optimizers 4 | """ 5 | from torch.optim import * 6 | -------------------------------------------------------------------------------- /security/common/schedulers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the schedulers used in the project: 3 | - timm schedulers 4 | """ 5 | from timm.scheduler import * 6 | -------------------------------------------------------------------------------- /security/common/task/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - BaseTask (class): 4 | base task for training, validation and testing 5 | """ 6 | from .base_task import BaseTask 7 | -------------------------------------------------------------------------------- /security/common/task/fas/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - test_module (class): 4 | function for validation and testing 5 | """ 6 | from .modules import test_module 7 | -------------------------------------------------------------------------------- /security/common/task/fas/modules.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import numpy as np 4 | from tqdm import tqdm 5 | 6 | import torch 7 | 8 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..')) 9 | from common.utils import * 10 | 11 | 12 | def test_module(model, test_data_loaders, forward_function, device='cuda', distributed=False): 13 | """Test module for Face Anti-spoofing 14 | 15 | Args: 16 | model (nn.module): fas model 17 | test_data_loaders (torch.dataloader): list of test data loaders 18 | forward_function (function): model forward function 19 | device (str, optional): Defaults to 'cuda'. 20 | distributed (bool, optional): whether to use distributed training. Defaults to False. 21 | 22 | Returns: 23 | y_preds (list): predictions 24 | y_trues (list): ground truth labels 25 | """ 26 | prob_dict = {} 27 | label_dict = {} 28 | 29 | y_preds = [] 30 | y_trues = [] 31 | 32 | model.eval() 33 | for loaders in test_data_loaders: 34 | for iter, datas in enumerate(tqdm(loaders)): 35 | with torch.no_grad(): 36 | images = datas[0].to(device) 37 | targets = datas[1].to(device) 38 | map_GT = datas[2].to(device) 39 | img_path = datas[3] 40 | probs = forward_function(images) 41 | 42 | if not distributed: 43 | probs = probs.cpu().data.numpy() 44 | label = targets.cpu().data.numpy() 45 | 46 | for i in range(len(probs)): 47 | # the image of the same video share the same video_path 48 | video_path = img_path[i].rsplit('/', 1)[0] 49 | if (video_path in prob_dict.keys()): 50 | prob_dict[video_path].append(probs[i]) 51 | label_dict[video_path].append(label[i]) 52 | else: 53 | prob_dict[video_path] = [] 54 | label_dict[video_path] = [] 55 | prob_dict[video_path].append(probs[i]) 56 | label_dict[video_path].append(label[i]) 57 | else: 58 | y_preds.extend(probs) 59 | y_trues.extend(targets) 60 | 61 | if not distributed: 62 | y_preds = [] 63 | y_trues = [] 64 | for key in prob_dict.keys(): 65 | # calculate the scores in video-level via averaging the scores of the images from the same videos 66 | avg_single_video_prob = sum(prob_dict[key]) / len(prob_dict[key]) 67 | avg_single_video_label = sum(label_dict[key]) / len(label_dict[key]) 68 | y_preds = np.append(y_preds, avg_single_video_prob) 69 | y_trues = np.append(y_trues, avg_single_video_label) 70 | 71 | return y_preds, y_trues 72 | -------------------------------------------------------------------------------- /security/common/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - parameters: (deprecated) 4 | - cli_utils: 5 | client parameters utils 6 | - logger_utils: 7 | logger utils 8 | - distribute_utils: 9 | tensor reduce and gather utils in distributed training 10 | - face_utils: 11 | face crop functions 12 | - misc: 13 | training misc utils 14 | - meters: 15 | training meters 16 | - metrics: 17 | calculate metrics 18 | - model_init: 19 | model weight initialization functions 20 | """ 21 | from .parameters import get_parameters 22 | from .cli_utils import get_params 23 | from .logger_utils import get_logger 24 | from .distribute_utils import reduce_tensor, gather_tensor 25 | from .face_utils import add_face_margin, get_face_box 26 | from .misc import set_seed, setup, init_exam_dir, init_wandb_workspace, save_test_results 27 | from .meters import AverageMeter, ProgressMeter 28 | from .metrics import find_best_threshold, cal_metrics 29 | from .model_init import * 30 | -------------------------------------------------------------------------------- /security/common/utils/distribute_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributed as dist 3 | 4 | 5 | def reduce_tensor(tensor, mean=True): 6 | """Reduce tensor in the distributed settting. 7 | 8 | Args: 9 | tensor (torch.tensor): 10 | Input torch tensor to reduce. 11 | mean (bool, optional): 12 | Whether to apply mean. Defaults to True. 13 | 14 | Returns: 15 | [torch.tensor]: Returned reduced torch tensor or. 16 | """ 17 | rt = tensor.clone() # The function operates in-place. 18 | dist.all_reduce(rt, op=dist.ReduceOp.SUM) 19 | if mean: 20 | rt /= dist.get_world_size() 21 | return rt 22 | 23 | 24 | def gather_tensor(inp, world_size=None, dist_=True, to_numpy=False): 25 | """Gather tensor in the distributed setting. 26 | 27 | Args: 28 | inp (torch.tensor): 29 | Input torch tensor to gather. 30 | world_size (int, optional): 31 | Dist world size. Defaults to None. If None, world_size = dist.get_world_size(). 32 | dist_ (bool, optional): 33 | Whether to use all_gather method to gather all the tensors. Defaults to True. 34 | to_numpy (bool, optional): 35 | Whether to return numpy array. Defaults to False. 36 | 37 | Returns: 38 | (torch.tensor || numpy.ndarray): Returned tensor or numpy array. 39 | """ 40 | inp = torch.stack(inp) 41 | if dist_: 42 | if world_size is None: 43 | world_size = dist.get_world_size() 44 | gather_inp = [torch.ones_like(inp) for _ in range(world_size)] 45 | dist.all_gather(gather_inp, inp) 46 | gather_inp = torch.cat(gather_inp) 47 | else: 48 | gather_inp = inp 49 | 50 | if to_numpy: 51 | gather_inp = gather_inp.cpu().numpy() 52 | 53 | return gather_inp 54 | -------------------------------------------------------------------------------- /security/common/utils/face_utils.py: -------------------------------------------------------------------------------- 1 | """Crop face from image via landmarks 2 | """ 3 | 4 | 5 | def add_face_margin(x, y, w, h, margin=0.5): 6 | """Add marigin to face bounding box 7 | """ 8 | x_marign = int(w * margin / 2) 9 | y_marign = int(h * margin / 2) 10 | 11 | x1 = x - x_marign 12 | x2 = x + w + x_marign 13 | y1 = y - y_marign 14 | y2 = y + h + y_marign 15 | 16 | return x1, x2, y1, y2 17 | 18 | 19 | def get_face_box(img, landmarks, margin): 20 | """Get faca bounding box from landmarks 21 | 22 | Args: 23 | img (np.array): input image 24 | landmarks (np.array): face landmarks 25 | margin (float): margin for face box 26 | 27 | Returns: 28 | list: face bouding box 29 | """ 30 | # load the positions of five landmarks 31 | x_list = [ 32 | int(float(landmarks[6])), 33 | int(float(landmarks[8])), 34 | int(float(landmarks[10])), 35 | int(float(landmarks[12])), 36 | int(float(landmarks[14])) 37 | ] 38 | y_list = [ 39 | int(float(landmarks[7])), 40 | int(float(landmarks[9])), 41 | int(float(landmarks[11])), 42 | int(float(landmarks[13])), 43 | int(float(landmarks[15])) 44 | ] 45 | 46 | x, y = min(x_list), min(y_list) 47 | w, h = max(x_list) - x, max(y_list) - y 48 | 49 | side = w if w > h else h 50 | 51 | # add margin 52 | x1, x2, y1, y2 = add_face_margin(x, y, side, side, margin) 53 | max_h, max_w = img.shape[:2] 54 | x1 = max(0, x1) 55 | y1 = max(0, y1) 56 | x2 = min(max_w, x2) 57 | y2 = min(max_h, y2) 58 | 59 | return x1, x2, y1, y2 60 | -------------------------------------------------------------------------------- /security/common/utils/meters.py: -------------------------------------------------------------------------------- 1 | class AverageMeter(object): 2 | ''' 3 | The Class AverageMeter record the metrics during the training process 4 | Examples: 5 | >>> acces = AverageMeter('_Acc', ':.5f') 6 | >>> acc = (prediction == labels).float().mean() 7 | >>> acces.update(acc) 8 | ''' 9 | def __init__(self, name='metric', fmt=':f'): 10 | self.name = name 11 | self.fmt = fmt 12 | self.reset() 13 | 14 | def reset(self): 15 | self.val = 0 16 | self.avg = 0 17 | self.sum = 0 18 | self.count = 0 19 | 20 | def update(self, val, n=1): 21 | self.val = val 22 | self.sum += val * n 23 | self.count += n 24 | self.avg = self.sum / self.count 25 | 26 | def __str__(self): 27 | fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' 28 | return fmtstr.format(**self.__dict__) 29 | 30 | 31 | class ProgressMeter(object): 32 | ''' 33 | The ProgressMeter to record all AverageMeter and print the results 34 | Examples: 35 | >>> acces = AverageMeter('_Acc', ':.5f') 36 | >>> progress = ProgressMeter(epoch_size, [acces]) 37 | >>> progress.display(iterations) 38 | ''' 39 | def __init__(self, num_batches, meters, prefix=""): 40 | self.batch_fmtstr = self._get_batch_fmtstr(num_batches) 41 | self.meters = meters 42 | self.prefix = prefix 43 | 44 | def display(self, batch): 45 | entries = [self.prefix + self.batch_fmtstr.format(batch)] 46 | entries += [str(meter) for meter in self.meters] 47 | # print('\t'.join(entries)) 48 | return '\t'.join(entries) 49 | 50 | def _get_batch_fmtstr(self, num_batches): 51 | num_digits = len(str(num_batches // 1)) 52 | fmt = '{:' + str(num_digits) + 'd}' 53 | return '[' + fmt + '/' + fmt.format(num_batches) + ']' 54 | -------------------------------------------------------------------------------- /security/common/utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from easydict import EasyDict 3 | from sklearn.metrics import roc_curve, auc, confusion_matrix 4 | from scipy.optimize import brentq 5 | from scipy.interpolate import interp1d 6 | 7 | 8 | def find_best_threshold(y_trues, y_preds): 9 | ''' 10 | This function is utilized to find the threshold corresponding to the best ACER 11 | Args: 12 | y_trues (list): the list of the ground-truth labels, which contains the int data 13 | y_preds (list): the list of the predicted results, which contains the float data 14 | ''' 15 | print("Finding best threshold...") 16 | best_thre = 0.5 17 | best_metrics = None 18 | candidate_thres = list(np.unique(np.sort(y_preds))) 19 | for thre in candidate_thres: 20 | metrics = cal_metrics(y_trues, y_preds, threshold=thre) 21 | if best_metrics is None: 22 | best_metrics = metrics 23 | best_thre = thre 24 | elif metrics.ACER < best_metrics.ACER: 25 | best_metrics = metrics 26 | best_thre = thre 27 | print(f"Best threshold is {best_thre}") 28 | return best_thre, best_metrics 29 | 30 | 31 | def cal_metrics(y_trues, y_preds, threshold=0.5): 32 | ''' 33 | This function is utilized to calculate the performance of the methods 34 | Args: 35 | y_trues (list): the list of the ground-truth labels, which contains the int data 36 | y_preds (list): the list of the predicted results, which contains the float data 37 | threshold (float, optional): 38 | 'best': calculate the best results 39 | 'auto': calculate the results corresponding to the thresholds of EER 40 | float: calculate the results of the specific thresholds 41 | ''' 42 | 43 | metrics = EasyDict() 44 | 45 | fpr, tpr, thresholds = roc_curve(y_trues, y_preds) 46 | metrics.AUC = auc(fpr, tpr) 47 | 48 | metrics.EER = brentq(lambda x: 1. - x - interp1d(fpr, tpr)(x), 0., 1.) 49 | metrics.Thre = float(interp1d(fpr, thresholds)(metrics.EER)) 50 | 51 | if threshold == 'best': 52 | _, best_metrics = find_best_threshold(y_trues, y_preds) 53 | return best_metrics 54 | 55 | elif threshold == 'auto': 56 | threshold = metrics.Thre 57 | 58 | prediction = (np.array(y_preds) > threshold).astype(int) 59 | 60 | res = confusion_matrix(y_trues, prediction, labels=[0, 1]) 61 | TP, FN = res[0, :] 62 | FP, TN = res[1, :] 63 | metrics.ACC = (TP + TN) / len(y_trues) 64 | 65 | TP_rate = float(TP / (TP + FN)) 66 | TN_rate = float(TN / (TN + FP)) 67 | 68 | metrics.APCER = float(FP / (TN + FP)) 69 | metrics.BPCER = float(FN / (FN + TP)) 70 | metrics.ACER = (metrics.APCER + metrics.BPCER) / 2 71 | 72 | return metrics 73 | -------------------------------------------------------------------------------- /security/common/utils/misc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import wandb 4 | import shutil 5 | import time 6 | import datetime 7 | import warnings 8 | import torch 9 | import numpy as np 10 | 11 | 12 | def set_seed(SEED): 13 | """This function set the random seed for the training process 14 | 15 | Args: 16 | SEED (int): the random seed 17 | """ 18 | if SEED: 19 | random.seed(SEED) 20 | np.random.seed(SEED) 21 | torch.manual_seed(SEED) 22 | torch.cuda.manual_seed(SEED) 23 | torch.cuda.manual_seed_all(SEED) 24 | torch.backends.cudnn.deterministic = True 25 | 26 | 27 | def setup(cfg): 28 | if getattr(cfg, 'torch_home', None): 29 | os.environ['TORCH_HOME'] = cfg.torch_home 30 | warnings.filterwarnings("ignore") 31 | seed = cfg.seed 32 | set_seed(seed) 33 | 34 | 35 | def init_exam_dir(cfg): 36 | if cfg.local_rank == 0: 37 | if not os.path.exists(cfg.exam_dir): 38 | os.makedirs(cfg.exam_dir) 39 | ckpt_dir = os.path.join(cfg.exam_dir, 'ckpt') 40 | if not os.path.exists(ckpt_dir): 41 | os.makedirs(ckpt_dir) 42 | 43 | 44 | def init_wandb_workspace(cfg): 45 | """This function initializes the wandb workspace 46 | """ 47 | if cfg.wandb.name is None: 48 | cfg.wandb.name = cfg.config.split('/')[-1].replace('.yaml', '') 49 | wandb.init(**cfg.wandb) 50 | allow_val_change = False if cfg.wandb.resume is None else True 51 | wandb.config.update(cfg, allow_val_change) 52 | wandb.save(cfg.config) 53 | if cfg.debug or wandb.run.dir == '/tmp': 54 | cfg.exam_dir = 'wandb/debug' 55 | if os.path.exists(cfg.exam_dir): 56 | shutil.rmtree(cfg.exam_dir) 57 | os.makedirs(cfg.exam_dir, exist_ok=True) 58 | else: 59 | cfg.exam_dir = os.path.dirname(wandb.run.dir) 60 | os.makedirs(os.path.join(cfg.exam_dir, 'ckpts'), exist_ok=True) 61 | return cfg 62 | 63 | 64 | def save_test_results(img_paths, y_preds, y_trues, filename='results.log'): 65 | assert len(y_trues) == len(y_preds) == len(img_paths) 66 | 67 | with open(filename, 'w') as f: 68 | for i in range(len(img_paths)): 69 | print(img_paths[i], end=' ', file=f) 70 | print(y_preds[i], file=f) 71 | print(y_trues[i], end=' ', file=f) 72 | -------------------------------------------------------------------------------- /security/common/utils/parameters.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | from omegaconf import OmegaConf 4 | 5 | 6 | def get_parameters(): 7 | """define the parameter for training 8 | 9 | Args: 10 | --config (string): the path of config files 11 | --distributed (int): train the model in the mode of DDP or Not, default: 1 12 | --local_rank (int): define the rank of this process 13 | --world_size (int): define the Number of GPU 14 | """ 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('-c', '--config', type=str, default='configs/train.yaml') 17 | parser.add_argument('--distributed', type=int, default=1) 18 | parser.add_argument('--local_rank', type=int, default=0) 19 | parser.add_argument('--world_size', type=int, default=1) 20 | parser.add_argument('--sync-bn', action='store_true', default=False) 21 | parser.add_argument('--debug', action='store_true', default=False) 22 | args = parser.parse_args() 23 | 24 | _C = OmegaConf.load(args.config) 25 | _C.merge_with(vars(args)) 26 | 27 | if _C.debug: 28 | _C.train.epochs = 2 29 | 30 | return _C 31 | 32 | 33 | if __name__ == '__main__': 34 | args = get_parameters() 35 | print(args) 36 | -------------------------------------------------------------------------------- /security/doc/intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/doc/intro.png -------------------------------------------------------------------------------- /security/doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/doc/logo.png -------------------------------------------------------------------------------- /security/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/requirements.txt -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/configs/config.yaml: -------------------------------------------------------------------------------- 1 | checkpoint_dir: '' 2 | data_dir: './datasets/Datasets_Makeup' 3 | model_dir: './models_meta_tar_ir152' 4 | after_dir: 'after_aligned_600' 5 | before_dir: 'before_aligned_600' 6 | log_dir: 'log' 7 | lmk_name: 'landmark_aligned_600.pk' 8 | use_se: True 9 | pretrained: False 10 | train_model_name_list: ['irse50', 'facenet', 'mobile_face'] 11 | val_model_name_list: ['ir152'] 12 | 13 | lr: 0.001 14 | update_lr_m: 0.001 15 | epoch_steps: 300 16 | gpu: 0 17 | n_threads: 8 18 | batch_size: 4 19 | input_dim: 3 20 | 21 | resize_size: [420, 160] 22 | eye_area: [9, 10, 11, 19, 84, 29, 79, 28, 24, 73, 70, 75, 74, 13, 15, 14, 22] 23 | mean: [0.5, 0.5, 0.5] 24 | std: [0.5, 0.5, 0.5] 25 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/datasets/Datasets_Makeup/landmark_aligned_600.txt: -------------------------------------------------------------------------------- 1 | # Save your own landmarks as the pikle file! 2 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/doc/asr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Adv-Makeup/doc/asr.png -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Adv-Makeup/doc/logo.png -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/doc/physical_vis_pose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Adv-Makeup/doc/physical_vis_pose.png -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/models/__init__.py: -------------------------------------------------------------------------------- 1 | """MakeupAttack model definition 2 | """ 3 | from .makeup_attack import MakeupAttack 4 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Adv-Makeup/train.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | import os 4 | import torchvision 5 | import argparse 6 | from omegaconf import OmegaConf 7 | 8 | from models.makeup_attack import * 9 | from datasets.dataset import * 10 | 11 | args = OmegaConf.load('./configs/config.yaml') 12 | device = torch.device('cuda:{}'.format(args.gpu)) if args.gpu >= 0 else torch.device('cpu') 13 | 14 | 15 | def model_save(model, ep, target_name): 16 | if (ep + 1) % 50 == 0: 17 | os.makedirs(os.path.join(args.model_dir, target_name.split('.')[0]), exist_ok=True) 18 | print('--- save the model @ ep %d ---' % (ep)) 19 | # Save the params of encoder in the generator 20 | torch.save(model.enc.state_dict(), 21 | '%s/%05d_enc.pth' % (os.path.join(args.model_dir, target_name.split('.')[0]), ep)) 22 | # Save the params of decoder in the generator 23 | torch.save(model.dec.state_dict(), 24 | '%s/%05d_dec.pth' % (os.path.join(args.model_dir, target_name.split('.')[0]), ep)) 25 | # Save the params of discriminator 26 | torch.save(model.discr.state_dict(), 27 | '%s/%05d_discr.pth' % (os.path.join(args.model_dir, target_name.split('.')[0]), ep)) 28 | 29 | 30 | def train_single_epoch(model, train_loader, target_name): 31 | for it, data in enumerate(train_loader): 32 | # Un-makeup eye area images (attacker's) 33 | before_img = data[0].to(device).detach() 34 | # Real-world makeup eye area images 35 | after_img = data[1].to(device).detach() 36 | # Un-makeup images' path 37 | before_path = data[2] 38 | 39 | # Update the eye makeup discriminator 40 | model.update_discr(before_img, after_img) 41 | # Update the eye makeup generator 42 | model.update_gen(before_img, before_path, target_name) 43 | 44 | 45 | def train(model, train_loader, target_name): 46 | for ep in range(args.epoch_steps): 47 | # Re-init all the params for each training epoch 48 | model.res_init(ep) 49 | 50 | # Update a single epoch 51 | train_single_epoch(model, train_loader, target_name) 52 | 53 | # Save the visualized images during training and output the training logs 54 | model.visualization(ep, len(train_loader)) 55 | 56 | # Save model 57 | model_save(model, ep, target_name) 58 | 59 | 60 | def main(): 61 | for target_name in os.listdir(args.data_dir + '/target_aligned_600'): 62 | print("Target: %s" % (target_name)) 63 | # Initialize the Adv-Makeup, networks and optimizers 64 | model = MakeupAttack(args) 65 | # Initialize the data-loader 66 | dataset = dataset_makeup(args) 67 | train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=True, 68 | num_workers=args.n_threads) 69 | train(model, train_loader, target_name) 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/configs/config.yaml: -------------------------------------------------------------------------------- 1 | attack: 2 | gpu: 0 3 | outer_loops: 200 4 | alpha: 0.01 5 | eps: 0.15 6 | gamma: 0.1 7 | inner_loops: 5 8 | 9 | dataset: 10 | dataset_name: celebaHQ 11 | dataset_dir: datasets/celebaHQ-select-aligned 12 | dataset_txt: datasets/celebaHQ-select-aligned-label.txt 13 | input_size: [112, 112] 14 | mean: [0.5, 0.5, 0.5] 15 | std: [0.5, 0.5, 0.5] 16 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/datasets/FFHQ-select-aligned-label.txt: -------------------------------------------------------------------------------- 1 | aligned_00140.jpg 2 | aligned_00239.jpg 3 | aligned_01239.jpg 4 | aligned_02849.jpg 5 | aligned_02959.jpg 6 | aligned_03812.jpg 7 | aligned_03936.jpg 8 | aligned_04988.jpg 9 | aligned_05267.jpg 10 | aligned_05516.jpg 11 | aligned_05953.jpg 12 | aligned_07421.jpg 13 | aligned_07649.jpg 14 | aligned_10028.jpg 15 | aligned_10152.jpg 16 | aligned_10370.jpg 17 | aligned_10917.jpg 18 | aligned_11054.jpg 19 | aligned_11669.jpg 20 | aligned_11773.jpg 21 | aligned_11910.jpg 22 | aligned_13291.jpg 23 | aligned_13869.jpg 24 | aligned_14007.jpg 25 | aligned_14560.jpg 26 | aligned_14902.jpg 27 | aligned_16142.jpg 28 | aligned_16272.jpg 29 | aligned_17356.jpg 30 | aligned_18320.jpg 31 | aligned_20966.jpg 32 | aligned_21871.jpg 33 | aligned_22324.jpg 34 | aligned_23683.jpg 35 | aligned_24093.jpg 36 | aligned_25703.jpg 37 | aligned_25706.jpg 38 | aligned_25942.jpg 39 | aligned_26130.jpg 40 | aligned_27132.jpg 41 | aligned_28311.jpg 42 | aligned_29525.jpg 43 | aligned_29785.jpg 44 | aligned_30934.jpg 45 | aligned_31001.jpg 46 | aligned_31538.jpg 47 | aligned_32107.jpg 48 | aligned_32376.jpg 49 | aligned_32857.jpg 50 | aligned_32994.jpg 51 | aligned_33143.jpg 52 | aligned_33331.jpg 53 | aligned_33892.jpg 54 | aligned_34914.jpg 55 | aligned_35351.jpg 56 | aligned_35381.jpg 57 | aligned_37009.jpg 58 | aligned_37210.jpg 59 | aligned_37364.jpg 60 | aligned_37567.jpg 61 | aligned_37588.jpg 62 | aligned_37992.jpg 63 | aligned_38159.jpg 64 | aligned_38221.jpg 65 | aligned_39010.jpg 66 | aligned_39406.jpg 67 | aligned_39710.jpg 68 | aligned_40229.jpg 69 | aligned_41278.jpg 70 | aligned_41331.jpg 71 | aligned_41992.jpg 72 | aligned_42232.jpg 73 | aligned_42587.jpg 74 | aligned_42768.jpg 75 | aligned_42847.jpg 76 | aligned_43993.jpg 77 | aligned_44402.jpg 78 | aligned_44415.jpg 79 | aligned_44979.jpg 80 | aligned_45467.jpg 81 | aligned_47472.jpg 82 | aligned_47840.jpg 83 | aligned_48643.jpg 84 | aligned_49079.jpg 85 | aligned_51627.jpg 86 | aligned_52221.jpg 87 | aligned_52268.jpg 88 | aligned_53710.jpg 89 | aligned_54446.jpg 90 | aligned_54732.jpg 91 | aligned_54869.jpg 92 | aligned_55209.jpg 93 | aligned_56084.jpg 94 | aligned_56164.jpg 95 | aligned_56594.jpg 96 | aligned_57225.jpg 97 | aligned_57680.jpg 98 | aligned_57926.jpg 99 | aligned_58539.jpg 100 | aligned_58727.jpg 101 | aligned_60641.jpg 102 | aligned_60748.jpg 103 | aligned_62266.jpg 104 | aligned_62760.jpg 105 | aligned_63613.jpg 106 | aligned_64876.jpg 107 | aligned_65104.jpg 108 | aligned_65826.jpg 109 | aligned_69731.jpg 110 | aligned_69832.jpg 111 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/datasets/celebaHQ-select-aligned-label.txt: -------------------------------------------------------------------------------- 1 | aligned_000004.jpg 2 | aligned_000009.jpg 3 | aligned_000014.jpg 4 | aligned_000016.jpg 5 | aligned_000025.jpg 6 | aligned_000030.jpg 7 | aligned_000044.jpg 8 | aligned_000051.jpg 9 | aligned_000052.jpg 10 | aligned_000074.jpg 11 | aligned_012853.jpg 12 | aligned_087916.jpg 13 | aligned_100434.jpg 14 | aligned_105409.jpg 15 | aligned_127741.jpg 16 | aligned_195210.jpg 17 | aligned_199359.jpg 18 | aligned_003644.jpg 19 | aligned_008938.jpg 20 | aligned_009032.jpg 21 | aligned_019197.jpg 22 | aligned_020432.jpg 23 | aligned_024581.jpg 24 | aligned_025036.jpg 25 | aligned_031553.jpg 26 | aligned_032421.jpg 27 | aligned_036170.jpg 28 | aligned_044363.jpg 29 | aligned_053200.jpg 30 | aligned_057862.jpg 31 | aligned_063305.jpg 32 | aligned_078168.jpg 33 | aligned_085135.jpg 34 | aligned_088664.jpg 35 | aligned_094973.jpg 36 | aligned_105634.jpg 37 | aligned_111062.jpg 38 | aligned_111787.jpg 39 | aligned_131067.jpg 40 | aligned_136378.jpg 41 | aligned_144204.jpg 42 | aligned_147809.jpg 43 | aligned_151944.jpg 44 | aligned_152193.jpg 45 | aligned_154232.jpg 46 | aligned_159405.jpg 47 | aligned_163521.jpg 48 | aligned_167947.jpg 49 | aligned_186090.jpg 50 | aligned_186382.jpg 51 | aligned_186913.jpg 52 | aligned_187454.jpg 53 | aligned_192699.jpg 54 | aligned_193013.jpg 55 | aligned_195919.jpg 56 | aligned_196129.jpg 57 | aligned_200381.jpg 58 | aligned_201253.jpg 59 | aligned_000618.jpg 60 | aligned_001422.jpg 61 | aligned_001435.jpg 62 | aligned_002768.jpg 63 | aligned_004839.jpg 64 | aligned_005675.jpg 65 | aligned_006592.jpg 66 | aligned_007102.jpg 67 | aligned_008484.jpg 68 | aligned_012069.jpg 69 | aligned_012902.jpg 70 | aligned_014841.jpg 71 | aligned_015855.jpg 72 | aligned_015915.jpg 73 | aligned_016010.jpg 74 | aligned_016346.jpg 75 | aligned_017793.jpg 76 | aligned_017927.jpg 77 | aligned_019238.jpg 78 | aligned_019712.jpg 79 | aligned_023907.jpg 80 | aligned_025389.jpg 81 | aligned_025883.jpg 82 | aligned_025919.jpg 83 | aligned_026031.jpg 84 | aligned_027520.jpg 85 | aligned_027677.jpg 86 | aligned_027920.jpg 87 | aligned_028423.jpg 88 | aligned_029772.jpg 89 | aligned_030535.jpg 90 | aligned_030720.jpg 91 | aligned_031349.jpg 92 | aligned_032755.jpg 93 | aligned_033108.jpg 94 | aligned_033274.jpg 95 | aligned_036555.jpg 96 | aligned_038443.jpg 97 | aligned_038582.jpg 98 | aligned_038869.jpg 99 | aligned_039855.jpg 100 | aligned_039957.jpg 101 | aligned_041523.jpg 102 | aligned_042262.jpg 103 | aligned_043323.jpg 104 | aligned_044826.jpg 105 | aligned_046483.jpg 106 | aligned_047019.jpg 107 | aligned_047467.jpg 108 | aligned_048460.jpg 109 | aligned_048781.jpg 110 | aligned_049081.jpg 111 | -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/gradcam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/gradcam.png -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/main_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/main_fig.png -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/table2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/table2.png -------------------------------------------------------------------------------- /security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/table3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Adv-Attack-Defense/Sibling-Attack/doc/table3.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/configs/ICM2O.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: DG_Dataset 7 | DG_Dataset: 8 | use_LMDB: False 9 | LMDB_root: /path/to/lmdb_database 10 | train_pos_list1_path: /path/to/Replay_Real_list 11 | train_neg_list1_path: /path/to/Replay_Fake_list 12 | train_pos_list2_path: /path/to/CASIA_Real_list 13 | train_neg_list2_path: /path/to/CASIA_Fake_list 14 | train_pos_list3_path: /path/to/MSU_Real_list 15 | train_neg_list3_path: /path/to/MSU_Fake_list 16 | val_pos_list_path: /path/to/Oulu_Real_list 17 | val_neg_list_path: /path/to/Oulu_Fake_list 18 | test_pos_list_path: /path/to/Oulu_Real_list 19 | test_neg_list_path: /path/to/Oulu_Fake_list 20 | img_mode: rgb_hsv 21 | depth_map: True 22 | depth_map_size: 32 23 | crop_face_from_5points: False 24 | margin: 2 25 | return_path: True 26 | 27 | transform: 28 | image_size: 256 29 | mean: [0.5, 0.5, 0.5] 30 | std: [0.5, 0.5, 0.5] 31 | num_segments: 2 32 | 33 | model: 34 | name: Framework 35 | ckpt_path: 36 | resume: 37 | params: 38 | in_ch: 6 39 | mid_ch: 384 40 | AdapNorm: True 41 | AdapNorm_attention_flag: 1layer 42 | model_initial: kaiming 43 | 44 | loss: 45 | name: CrossEntropyLoss 46 | weight: 1.0 47 | params: 48 | reduction: mean 49 | 50 | loss_2: 51 | name: MSELoss 52 | weight: 0.1 53 | params: 54 | reduction: mean 55 | 56 | loss_3: 57 | name: EuclideanLoss 58 | domain_weight: 0.001 59 | discri_weight: 0.001 60 | params: 61 | reduction: mean 62 | 63 | optimizer: 64 | name: optimizer_name 65 | params: 66 | lr: learning_rate 67 | weight_decay: weight_decay_rate 68 | 69 | scheduler: 70 | name: scheduler_name 71 | params: 72 | decay_t: decay_t 73 | decay_rate: decay_rate 74 | 75 | train: 76 | epochs: 50 77 | batch_size: batchsize 78 | print_interval: 10 79 | metasize: 2 80 | meta_step_size: 0.0001 81 | 82 | val: 83 | batch_size: batchsize 84 | 85 | test: 86 | batch_size: batchsize 87 | record_results: True 88 | 89 | wandb: 90 | project: ANRL 91 | group: ICMtoO 92 | mode: offline 93 | job_type: 94 | id: 95 | resume: 96 | save_code: True 97 | name: 98 | notes: 99 | 100 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/configs/IOM2C.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: DG_Dataset 7 | DG_Dataset: 8 | use_LMDB: False 9 | LMDB_root: /path/to/lmdb_database 10 | train_pos_list1_path: /path/to/Replay_Real_list 11 | train_neg_list1_path: /path/to/Replay_Fake_list 12 | train_pos_list2_path: /path/to/Oulu_Real_list 13 | train_neg_list2_path: /path/to/Oulu_Fake_list 14 | train_pos_list3_path: /path/to/MSU_Real_list 15 | train_neg_list3_path: /path/to/MSU_Fake_list 16 | val_pos_list_path: /path/to/CASIA_Real_list 17 | val_neg_list_path: /path/to/CASIA_Fake_list 18 | test_pos_list_path: /path/to/CASIA_Real_list 19 | test_neg_list_path: /path/to/CASIA_Fake_list 20 | img_mode: rgb_hsv 21 | depth_map: True 22 | depth_map_size: 32 23 | crop_face_from_5points: False 24 | margin: 2 25 | return_path: True 26 | 27 | transform: 28 | image_size: 256 29 | mean: [0.5, 0.5, 0.5] 30 | std: [0.5, 0.5, 0.5] 31 | num_segments: 2 32 | 33 | model: 34 | name: Framework 35 | ckpt_path: 36 | resume: 37 | params: 38 | in_ch: 6 39 | mid_ch: 384 40 | AdapNorm: True 41 | AdapNorm_attention_flag: 1layer 42 | model_initial: kaiming 43 | 44 | loss: 45 | name: CrossEntropyLoss 46 | weight: 1.0 47 | params: 48 | reduction: mean 49 | 50 | loss_2: 51 | name: MSELoss 52 | weight: 0.1 53 | params: 54 | reduction: mean 55 | 56 | loss_3: 57 | name: EuclideanLoss 58 | domain_weight: 0.001 59 | discri_weight: 0.001 60 | params: 61 | reduction: mean 62 | 63 | optimizer: 64 | name: optimizer_name 65 | params: 66 | lr: learning_rate 67 | weight_decay: weight_decay_rate 68 | 69 | scheduler: 70 | name: scheduler_name 71 | params: 72 | decay_t: decay_t 73 | decay_rate: decay_rate 74 | 75 | train: 76 | epochs: 50 77 | batch_size: batchsize 78 | print_interval: 10 79 | metasize: 2 80 | meta_step_size: 0.0001 81 | 82 | val: 83 | batch_size: batchsize 84 | 85 | test: 86 | batch_size: batchsize 87 | record_results: True 88 | 89 | wandb: 90 | project: ANRL 91 | group: IOMtoC 92 | mode: offline 93 | job_type: 94 | id: 95 | resume: 96 | save_code: True 97 | name: 98 | notes: 99 | 100 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/configs/OCI2M.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: DG_Dataset 7 | DG_Dataset: 8 | train_pos_list1_path: /path/to/Oulu_Real_list 9 | train_neg_list1_path: /path/to/Oulu_Fake_list 10 | train_pos_list2_path: /path/to/CASIA_Real_list 11 | train_neg_list2_path: /path/to/CASIA_Fake_list 12 | train_pos_list3_path: /path/to/Replay_Real_list 13 | train_neg_list3_path: /path/to/Replay_Fake_list 14 | val_pos_list_path: /path/to/MSU_Real_list 15 | val_neg_list_path: /path/to/MSU_Fake_list 16 | test_pos_list_path: /path/to/MSU_Real_list 17 | test_neg_list_path: /path/to/MSU_Fake_list 18 | img_mode: rgb_hsv 19 | depth_map: True 20 | depth_map_size: 32 21 | rcrop_face_from_5points: False 22 | margin: 2 23 | return_path: True 24 | 25 | transform: 26 | image_size: 256 27 | mean: [0.5, 0.5, 0.5] 28 | std: [0.5, 0.5, 0.5] 29 | num_segments: 2 30 | 31 | model: 32 | name: Framework 33 | ckpt_path: 34 | resume: 35 | params: 36 | in_ch: 6 37 | mid_ch: 384 38 | AdapNorm: True 39 | AdapNorm_attention_flag: 1layer 40 | model_initial: kaiming 41 | 42 | loss: 43 | name: CrossEntropyLoss 44 | weight: 1.0 45 | params: 46 | reduction: mean 47 | 48 | loss_2: 49 | name: MSELoss 50 | weight: 0.1 51 | params: 52 | reduction: mean 53 | 54 | loss_3: 55 | name: EuclideanLoss 56 | domain_weight: 0.001 57 | discri_weight: 0.001 58 | params: 59 | reduction: mean 60 | 61 | optimizer: 62 | name: optimizer_name 63 | params: 64 | lr: learning_rate 65 | weight_decay: weight_decay_rate 66 | 67 | scheduler: 68 | name: scheduler_name 69 | params: 70 | decay_t: decay_t 71 | decay_rate: decay_rate 72 | 73 | train: 74 | epochs: 50 75 | batch_size: batchsize 76 | print_interval: 10 77 | metasize: 2 78 | meta_step_size: 0.0001 79 | 80 | val: 81 | batch_size: batchsize 82 | 83 | test: 84 | batch_size: batchsize 85 | record_results: True 86 | 87 | wandb: 88 | project: ANRL 89 | group: OCI2M 90 | mode: offline 91 | job_type: 92 | id: 93 | resume: 94 | save_code: True 95 | name: 96 | notes: 97 | 98 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/configs/OCM2I.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: DG_Dataset 7 | DG_Dataset: 8 | train_pos_list1_path: /path/to/Oulu_Real_list 9 | train_neg_list1_path: /path/to/Oulu_Fake_list 10 | train_pos_list2_path: /path/to/CASIA_Real_list 11 | train_neg_list2_path: /path/to/CASIA_Fake_list 12 | train_pos_list3_path: /path/to/MSU_Real_list 13 | train_neg_list3_path: /path/to/MSU_Real_list 14 | val_pos_list_path: /path/to/Replay_Real_list 15 | val_neg_list_path: /path/to/Replay_Fake_list 16 | test_pos_list_path: /path/to/Replay_Real_list 17 | test_neg_list_path: /path/to/Replay_Fake_list 18 | img_mode: rgb_hsv 19 | depth_map: True 20 | depth_map_size: 32 21 | return_path: False 22 | 23 | transform: 24 | image_size: 256 25 | mean: [0.5, 0.5, 0.5] 26 | std: [0.5, 0.5, 0.5] 27 | num_segments: 2 28 | 29 | model: 30 | name: Framework 31 | ckpt_path: 32 | resume: 33 | params: 34 | in_ch: 6 35 | mid_ch: 384 36 | AdapNorm: True 37 | AdapNorm_attention_flag: 1layer 38 | model_initial: kaiming 39 | 40 | loss: 41 | name: CrossEntropyLoss 42 | weight: 1.0 43 | params: 44 | reduction: mean 45 | 46 | loss_2: 47 | name: MSELoss 48 | weight: 0.1 49 | params: 50 | reduction: mean 51 | 52 | loss_3: 53 | name: EuclideanLoss 54 | domain_weight: 0.001 55 | discri_weight: 0.001 56 | params: 57 | reduction: mean 58 | 59 | optimizer: 60 | name: optimizer_name 61 | params: 62 | lr: learning_rate 63 | weight_decay: weight_decay_rate 64 | 65 | scheduler: 66 | name: scheduler_name 67 | params: 68 | decay_t: decay_t 69 | decay_rate: decay_rate 70 | 71 | train: 72 | epochs: 50 73 | batch_size: batchsize 74 | print_interval: 10 75 | metasize: 2 76 | meta_step_size: 0.0001 77 | 78 | val: 79 | batch_size: batchsize 80 | 81 | test: 82 | batch_size: batchsize 83 | record_results: True 84 | 85 | wandb: 86 | project: ANRL 87 | group: OCMtoI 88 | mode: offline 89 | job_type: 90 | id: 91 | resume: 92 | save_code: True 93 | name: 94 | notes: 95 | 96 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | create_dataloader: defines the dataloader 3 | ''' 4 | from .factory import create_dataloader 5 | 6 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/datasets/factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import albumentations as alb 4 | 5 | from .DG_dataset import DG_Dataset 6 | 7 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..', '..', '..')) 8 | from common.data import create_base_transforms, create_base_dataloader 9 | 10 | 11 | def create_dg_data_transforms(args, split='train'): 12 | ''' 13 | define the domain generalization transforms accoding to different parameters 14 | Args: 15 | args ([type]): contain the specific parmaters 16 | split (str, optinal): 17 | 'train': to generate the domain generalization transforms for training 18 | 'val': to generate the domain generalization transforms for validation 19 | 'test': to generaate the domain generalization transforms for testing 20 | ''' 21 | base_transform = create_base_transforms(args, split=split) 22 | if split == 'train': 23 | dg_aug_transform = alb.Compose([ 24 | alb.HueSaturationValue(p=0.1), 25 | ]) 26 | data_transform = alb.Compose([*dg_aug_transform, *base_transform]) 27 | 28 | else: 29 | data_transform = base_transform 30 | 31 | return data_transform 32 | 33 | 34 | def create_dataloader(args, split='train', category=None, print_info=False): 35 | kwargs = getattr(args.dataset, args.dataset.name) 36 | dg_transform = create_dg_data_transforms(args.transform, split=split) 37 | dg_dataset = eval(args.dataset.name)(transform=dg_transform, 38 | split=split, 39 | category=category, 40 | print_info=print_info, 41 | **kwargs) 42 | dg_dataloader = create_base_dataloader(args, dg_dataset, split=split) 43 | return dg_dataloader 44 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/doc/ROC.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/ANRL/doc/ROC.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/doc/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/ANRL/doc/framework.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/doc/results_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/ANRL/doc/results_table.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/doc/tSNE.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/ANRL/doc/tSNE.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/models/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module: 3 | FeatEmbedder: Classify the input features 4 | Shape: 5 | - Input: (B, 384, 32, 32) 6 | - Output: (B, 2) 7 | FeatExtractor: Extract the features from the input images 8 | Shape: 9 | - Input: (B, 6, 256, 256) 10 | - Ouput: (B, 384, 32, 32) 11 | DepthEstmator: generate the depth map for the input features 12 | Shape: 13 | - Input: (B, 382, 32, 32) 14 | - Ouput: (B, 1, 32, 32) 15 | Framork: Contain the above three modules 16 | Shape: 17 | - Input: (B, 6, 256, 256) 18 | - Output: 19 | (B, 2) 20 | (B, 1, 32, 32) 21 | (B, 384, 32, 32) 22 | ''' 23 | from .framework import Framework, FeatEmbedder, FeatExtractor, DepthEstmator 24 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/ANRL/train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | if [ $# -lt 3 ]; then 4 | echo "USAGE: sh ./train.sh GPU_NUM CONFIG_PATH PORT" 5 | exit -1 6 | fi 7 | 8 | python3 -u -m torch.distributed.launch --nproc_per_node=$1 --master_port $3 ./train.py -c $2 9 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/configs/Oulu_1.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: FASDataset 7 | FASDataset: 8 | use_LMDB: False 9 | LMDB_root: /path/to/lmdb_database 10 | train_pos_list_path: /path/to/Oulu_Protocol_1_train_pos_list 11 | train_neg_list_path: /path/to/Oulu_Protocol_1_train_neg_list 12 | val_pos_list_path: /path/to/Oulu_Protocol_1_val_pos_list 13 | val_neg_list_path: /path/to/Oulu_Protocol_1_val_neg_list 14 | test_pos_list_path: /path/to/Oulu_Protocol_1_test_pos_list 15 | test_neg_1_list_path: /path/to/Oulu_Protocol_1_test_neg_1_list 16 | test_neg_2_list_path: /path/to/Oulu_Protocol_1_test_neg_2_list 17 | img_mode: rgb_hsv 18 | reflection_map: True 19 | reflection_map_size: 32 20 | img_size: 255 21 | crop_face_from_5points: False 22 | margin: 2 23 | return_path: True 24 | permutation_path: /path/to/permutations_30.npy 25 | grid_size: 3 26 | 27 | transform: 28 | image_size: 255 29 | mean: [0.5, 0.5, 0.5] 30 | std: [0.5, 0.5, 0.5] 31 | num_segments: 2 32 | 33 | model: 34 | name: DCN 35 | ckpt_path: 36 | resume: 37 | params: 38 | in_ch: 6 39 | 40 | loss: 41 | name: MSELoss 42 | weight: 1.0 43 | params: 44 | reduction: mean 45 | 46 | loss_2: 47 | name: MSELoss 48 | weight: 0.1 49 | params: 50 | reduction: mean 51 | 52 | optimizer: 53 | name: optimizer_name 54 | params: 55 | lr: learning_rate 56 | weight_decay: weight_decay_rate 57 | 58 | scheduler: 59 | name: scheduler_name 60 | params: 61 | decay_t: decay_t 62 | decay_rate: decay_rate 63 | 64 | train: 65 | epochs: 50 66 | batch_size: batchsize 67 | print_interval: 10 68 | 69 | val: 70 | batch_size: batchsize 71 | 72 | test: 73 | batch_size: batchsize 74 | record_results: True 75 | 76 | wandb: 77 | project: DCN 78 | group: Oulu_1 79 | mode: offline 80 | job_type: 81 | id: 82 | resume: 83 | save_code: True 84 | name: 85 | notes: 86 | 87 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/configs/Oulu_2.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: FASDataset 7 | FASDataset: 8 | use_LMDB: False 9 | LMDB_root: /path/to/lmdb_database 10 | train_pos_list_path: /path/to/Oulu_Protocol_1_train_pos_list 11 | train_neg_list_path: /path/to/Oulu_Protocol_1_train_neg_list 12 | val_pos_list_path: /path/to/Oulu_Protocol_1_val_pos_list 13 | val_neg_list_path: /path/to/Oulu_Protocol_1_val_neg_list 14 | test_pos_list_path: /path/to/Oulu_Protocol_1_test_pos_list 15 | test_neg_1_list_path: /path/to/Oulu_Protocol_1_test_neg_1_list 16 | test_neg_2_list_path: /path/to/Oulu_Protocol_1_test_neg_2_list 17 | img_mode: rgb_hsv 18 | reflection_map: True 19 | reflection_map_size: 32 20 | img_size: 255 21 | crop_face_from_5points: False 22 | margin: 2 23 | return_path: True 24 | permutation_path: /path/to/permutations_30.npy 25 | grid_size: 3 26 | 27 | transform: 28 | image_size: 255 29 | mean: [0.5, 0.5, 0.5] 30 | std: [0.5, 0.5, 0.5] 31 | num_segments: 2 32 | 33 | model: 34 | name: DCN 35 | ckpt_path: 36 | resume: 37 | params: 38 | in_ch: 6 39 | 40 | loss: 41 | name: MSELoss 42 | weight: 1.0 43 | params: 44 | reduction: mean 45 | 46 | loss_2: 47 | name: MSELoss 48 | weight: 0.1 49 | params: 50 | reduction: mean 51 | 52 | optimizer: 53 | name: optimizer_name 54 | params: 55 | lr: learning_rate 56 | weight_decay: weight_decay_rate 57 | 58 | scheduler: 59 | name: scheduler_name 60 | params: 61 | decay_t: decay_t 62 | decay_rate: decay_rate 63 | 64 | train: 65 | epochs: 50 66 | batch_size: batchsize 67 | print_interval: 10 68 | 69 | val: 70 | batch_size: batchsize 71 | 72 | test: 73 | batch_size: batchsize 74 | record_results: True 75 | 76 | wandb: 77 | project: DCN 78 | group: Oulu_2 79 | mode: offline 80 | job_type: 81 | id: 82 | resume: 83 | save_code: True 84 | name: 85 | notes: 86 | 87 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/configs/Oulu_3_1.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: FASDataset 7 | FASDataset: 8 | use_LMDB: False 9 | LMDB_root: /path/to/lmdb_database 10 | train_pos_list_path: /path/to/Oulu_Protocol_1_train_pos_list 11 | train_neg_list_path: /path/to/Oulu_Protocol_1_train_neg_list 12 | val_pos_list_path: /path/to/Oulu_Protocol_1_val_pos_list 13 | val_neg_list_path: /path/to/Oulu_Protocol_1_val_neg_list 14 | test_pos_list_path: /path/to/Oulu_Protocol_1_test_pos_list 15 | test_neg_1_list_path: /path/to/Oulu_Protocol_1_test_neg_1_list 16 | test_neg_2_list_path: /path/to/Oulu_Protocol_1_test_neg_2_list 17 | img_mode: rgb_hsv 18 | reflection_map: True 19 | reflection_map_size: 32 20 | img_size: 255 21 | crop_face_from_5points: False 22 | margin: 2 23 | return_path: True 24 | permutation_path: /path/to/permutations_30.npy 25 | grid_size: 3 26 | 27 | transform: 28 | image_size: 255 29 | mean: [0.5, 0.5, 0.5] 30 | std: [0.5, 0.5, 0.5] 31 | num_segments: 2 32 | 33 | model: 34 | name: DCN 35 | ckpt_path: 36 | resume: 37 | params: 38 | in_ch: 6 39 | 40 | loss: 41 | name: MSELoss 42 | weight: 1.0 43 | params: 44 | reduction: mean 45 | 46 | loss_2: 47 | name: MSELoss 48 | weight: 0.1 49 | params: 50 | reduction: mean 51 | 52 | optimizer: 53 | name: optimizer_name 54 | params: 55 | lr: learning_rate 56 | weight_decay: weight_decay_rate 57 | 58 | scheduler: 59 | name: scheduler_name 60 | params: 61 | decay_t: decay_t 62 | decay_rate: decay_rate 63 | 64 | train: 65 | epochs: 50 66 | batch_size: batchsize 67 | print_interval: 10 68 | 69 | val: 70 | batch_size: batchsize 71 | 72 | test: 73 | batch_size: batchsize 74 | record_results: True 75 | 76 | wandb: 77 | project: DCN 78 | group: Oulu_3_1 79 | mode: offline 80 | job_type: 81 | id: 82 | resume: 83 | save_code: True 84 | name: 85 | notes: 86 | 87 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/configs/Oulu_4_1.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: /path/to/torch 3 | num_workers: 10 4 | 5 | dataset: 6 | name: FASDataset 7 | FASDataset: 8 | use_LMDB: False 9 | LMDB_root: /path/to/lmdb_database 10 | train_pos_list_path: /path/to/Oulu_Protocol_1_train_pos_list 11 | train_neg_list_path: /path/to/Oulu_Protocol_1_train_neg_list 12 | val_pos_list_path: /path/to/Oulu_Protocol_1_val_pos_list 13 | val_neg_list_path: /path/to/Oulu_Protocol_1_val_neg_list 14 | test_pos_list_path: /path/to/Oulu_Protocol_1_test_pos_list 15 | test_neg_1_list_path: /path/to/Oulu_Protocol_1_test_neg_1_list 16 | test_neg_2_list_path: /path/to/Oulu_Protocol_1_test_neg_2_list 17 | img_mode: rgb_hsv 18 | reflection_map: True 19 | reflection_map_size: 32 20 | img_size: 255 21 | crop_face_from_5points: False 22 | margin: 2 23 | return_path: True 24 | permutation_path: /path/to/permutations_30.npy 25 | grid_size: 3 26 | 27 | transform: 28 | image_size: 255 29 | mean: [0.5, 0.5, 0.5] 30 | std: [0.5, 0.5, 0.5] 31 | num_segments: 2 32 | 33 | model: 34 | name: DCN 35 | ckpt_path: 36 | resume: 37 | params: 38 | in_ch: 6 39 | 40 | loss: 41 | name: MSELoss 42 | weight: 1.0 43 | params: 44 | reduction: mean 45 | 46 | loss_2: 47 | name: MSELoss 48 | weight: 0.1 49 | params: 50 | reduction: mean 51 | 52 | optimizer: 53 | name: optimizer_name 54 | params: 55 | lr: learning_rate 56 | weight_decay: weight_decay_rate 57 | 58 | scheduler: 59 | name: scheduler_name 60 | params: 61 | decay_t: decay_t 62 | decay_rate: decay_rate 63 | 64 | train: 65 | epochs: 50 66 | batch_size: batchsize 67 | print_interval: 10 68 | 69 | val: 70 | batch_size: batchsize 71 | 72 | test: 73 | batch_size: batchsize 74 | record_results: True 75 | 76 | wandb: 77 | project: DCN 78 | group: Oulu_4_1 79 | mode: offline 80 | job_type: 81 | id: 82 | resume: 83 | save_code: True 84 | name: 85 | notes: 86 | 87 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | create_dataloader: defines the dataloader 3 | ''' 4 | from .factory import create_dataloader 5 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/datasets/factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import albumentations as alb 4 | 5 | from .dataset import FASDataset 6 | 7 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..', '..', '..')) 8 | from common.data import create_base_transforms, create_base_dataloader 9 | 10 | 11 | def create_data_transforms(args, split='train'): 12 | ''' 13 | define the transforms accoding to different parameters 14 | Args: 15 | args ([type]): contain the specific parmaters 16 | split (str, optinal): 17 | 'train': to generate the transforms for training 18 | 'val': to generate the transforms for validation 19 | 'test': to generaate the transforms for testing 20 | ''' 21 | base_transform = create_base_transforms(args, split=split) 22 | if split == 'train': 23 | aug_transform = alb.Compose([ 24 | alb.HueSaturationValue(p=0.1), 25 | ]) 26 | data_transform = alb.Compose([*aug_transform, *base_transform]) 27 | 28 | else: 29 | data_transform = base_transform 30 | 31 | return data_transform 32 | 33 | 34 | def create_dataloader(args, split='train', category=None, print_info=False): 35 | kwargs = getattr(args.dataset, args.dataset.name) 36 | transform = create_data_transforms(args.transform, split=split) 37 | dataset = eval(args.dataset.name)(transform=transform, 38 | split=split, 39 | category=category, 40 | print_info=print_info, 41 | **kwargs) 42 | dataloader = create_base_dataloader(args, dataset, split=split) 43 | return dataloader 44 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/datasets/permutations_30.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/DCN/datasets/permutations_30.npy -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/doc/TSNE.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/DCN/doc/TSNE.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/doc/table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/DCN/doc/table.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/doc/table_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Anti-Spoofing/DCN/doc/table_2.png -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/models/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module: 3 | DCN: contain the different modules 4 | Shape: 5 | - Input: (B, 6, 255, 255) 6 | - Ouput: 7 | (B, 1, 32, 32) 8 | (B, 9, 9) 9 | ''' 10 | from .framework import DCN 11 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/models/base_block.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import numpy as np 4 | 5 | import torch 6 | from torch import nn 7 | import torch.nn.functional as F 8 | 9 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..', '..', '..')) 10 | from common.utils.model_init import init_weights 11 | 12 | 13 | class Conv2dBlock(nn.Module): 14 | ''' 15 | Args: 16 | First_block (bool): 17 | 'True': indicate the modules is the first block and the input channels is 64 18 | ''' 19 | 20 | def __init__(self, First_block=False): 21 | super(Conv2dBlock, self).__init__() 22 | self.net = [] 23 | if First_block: 24 | self.net += [nn.Conv2d(64, 128, 3, 1, 1, bias=True)] 25 | else: 26 | self.net += [nn.Conv2d(128, 128, 3, 1, 1, bias=True)] 27 | 28 | self.net += [nn.ELU()] 29 | self.net += [nn.BatchNorm2d(128)] 30 | 31 | self.net += [nn.Conv2d(128, 196, 3, 1, 1, bias=True)] 32 | self.net += [nn.ELU()] 33 | self.net += [nn.BatchNorm2d(196)] 34 | 35 | self.net += [nn.Conv2d(196, 128, 3, 1, 1, bias=True)] 36 | self.net += [nn.ELU()] 37 | self.net += [nn.BatchNorm2d(128)] 38 | 39 | self.net += [nn.AvgPool2d(2, stride=2)] 40 | 41 | self.net = nn.Sequential(*self.net) 42 | init_weights(self.net, init_type='kaiming') 43 | 44 | def forward(self, x): 45 | return self.net(x) 46 | -------------------------------------------------------------------------------- /security/tasks/Face-Anti-Spoofing/DCN/train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | if [ $# -lt 3 ]; then 4 | echo "USAGE: sh ./train.sh GPU_NUM CONFIG_PATH PORT" 5 | exit -1 6 | fi 7 | 8 | python3 -u -m torch.distributed.launch --nproc_per_node=$1 --master_port $3 ./train.py -c $2 -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/configs/test.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: 'PATH TO YOUR TORCH HOME' 3 | froze: False 4 | 5 | dataset: 6 | name: 'celeb_df' 7 | pair: False 8 | ffpp: 9 | data_root: 'PATH TO YOUR DATA ROOT' 10 | data_types: '' 11 | compressions: 'c23' 12 | num_frames: 50 13 | methods: 14 | has_mask: False 15 | mask_size: 10 16 | pair: True 17 | alb: True 18 | celeb_df: 19 | data_root: 'PATH TO YOUR DATA ROOT' 20 | data_types: 'images_v1' 21 | num_frames: 25 22 | sample: 0 23 | 24 | wild_deepfake: 25 | root: 'PATH TO YOUR DATA ROOT' 26 | 27 | 28 | 29 | 30 | test: 31 | batch_size: 32 32 | threshold: 0.5 33 | record_results: True 34 | 35 | 36 | 37 | train: 38 | batch_size: 32 39 | threshold: 0.5 40 | record_results: True -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/configs/train.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: 'PATH TO YOUR TORCH HOME' 3 | froze: False 4 | use_amp: False 5 | dataset: 6 | name: 'FaceForensics' 7 | moco: True 8 | FaceForensics: 9 | data_root: 'PATH TO YOUR DATA ROOT' 10 | data_types: '' 11 | compressions: 'c23' 12 | num_frames: 50 13 | methods: 14 | has_mask: True 15 | mask_size: 10 16 | pair: True 17 | corr_pair: False 18 | random_patch: 3 19 | srm: 0.5 20 | diff_frame: 0.5 21 | image_size: 299 22 | CelebDF: 23 | data_root: 'PATH TO YOUR DATA ROOT' 24 | data_types: '' 25 | num_frames: 25 26 | pair: True 27 | random_patch: 3 28 | sample: 0 29 | 30 | 31 | transform: 32 | image_size: 299 33 | mean: [0.5, 0.5, 0.5] 34 | std: [0.5, 0.5, 0.5] 35 | 36 | 37 | model: 38 | name: 'DCL' 39 | ckpt_path: 40 | params: 41 | dim: 100 42 | K: 32768 43 | m: 0.999 44 | T: 0.07 45 | threshold: 5 46 | 47 | base_model: 48 | name: BinaryClassifier 49 | params: 50 | encoder: 'tf_efficientnet_b4_ns' 51 | num_classes: 2 52 | drop_rate: 0.2 53 | has_feature: True 54 | pretrained: True 55 | 56 | 57 | loss: 58 | name: CrossEntropyLoss 59 | weight: 1.0 60 | params: 61 | reduction: mean 62 | 63 | 64 | optimizer: 65 | name: 'Adam' 66 | params: 67 | lr: 2.0e-4 68 | weight_decay: 1.0e-5 69 | 70 | 71 | scheduler: 72 | name: StepLRScheduler 73 | params: 74 | decay_t: 10 75 | decay_rate: 0.1 76 | 77 | 78 | train: 79 | warmup: 3 80 | epochs: 30 81 | batch_size: 32 82 | print_interval: 100 83 | val_interval: 1 84 | 85 | 86 | val: 87 | batch_size: 32 88 | 89 | 90 | test: 91 | batch_size: 32 92 | record_results: True 93 | 94 | 95 | wandb: 96 | project: Face-Forgery-Detection 97 | group: DCL 98 | save_code: True 99 | name: 100 | resume: -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - FaceForensics (class): 4 | dataset class for FaceForensics++ dataset. 5 | - CelebDF (class): 6 | dataset class for CelebDF dataset. 7 | - WildDeepfake (class): 8 | dataset class for WildDeepfake dataset. 9 | - create_dataloader (function): 10 | helper function to set up train, val and test dataloader 11 | """ 12 | from .factory import create_dataloader 13 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/datasets/factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import albumentations as alb 4 | 5 | from .ffpp import FaceForensics 6 | from .celeb_df import CelebDF 7 | from .wild_deepfake import WildDeepfake 8 | 9 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..', '..', '..')) 10 | 11 | from common.data import create_base_transforms, create_base_dataloader 12 | 13 | 14 | def create_data_transforms(args, split='train'): 15 | """Create data transofrms 16 | 17 | Args: 18 | args: data transforms configs 19 | split (str, optional): split for train, val or test. Defaults to 'train'. 20 | 21 | Returns: 22 | albumentation: pytorch data augmentations 23 | """ 24 | base_transform = create_base_transforms(args, split=split) 25 | 26 | if split == 'train': 27 | aug_transform = alb.Compose([ 28 | alb.Rotate(limit=30), 29 | alb.Cutout(1, 25, 25, p=0.1), 30 | alb.RandomResizedCrop(256, 256, scale=(0.5, 1.0), p=0.5), 31 | alb.Resize(args.image_size, args.image_size), 32 | alb.HorizontalFlip(), 33 | alb.ToGray(p=0.1), 34 | alb.GaussNoise(p=0.1), 35 | alb.OneOf([ 36 | alb.RandomBrightnessContrast(), 37 | alb.FancyPCA(), 38 | alb.HueSaturationValue(), 39 | ], p=0.7), 40 | alb.GaussianBlur(blur_limit=3, p=0.05), 41 | ]) 42 | data_transform = alb.Compose([*aug_transform, *base_transform]) 43 | 44 | elif split == 'val': 45 | data_transform = base_transform 46 | 47 | elif split == 'test': 48 | data_transform = base_transform 49 | 50 | return data_transform 51 | 52 | 53 | def create_dataloader(args, split='train'): 54 | """create data loader 55 | 56 | Args: 57 | args: data loader configs 58 | split (str, optional): split for train, val or test. Defaults to 'train'. 59 | 60 | Returns: 61 | [type]: [description] 62 | """ 63 | dataset_params = getattr(args.dataset, args.dataset.name) 64 | transform = create_data_transforms(args.transform, split=split) 65 | dataset = eval(args.dataset.name)(transform=transform, split=split, **dataset_params) 66 | dataloader = create_base_dataloader(args, dataset, split=split) 67 | return dataloader 68 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/datasets/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - data_structure (function): 4 | helper function to set up FaceForenisics++ dataset 5 | - SRM (function): 6 | SRM related weights and convolutional layers 7 | - RandomPatch (class): 8 | random patch data augmentation 9 | """ 10 | from .data_structure import * 11 | from .srm import setup_srm_weights, setup_srm_layer 12 | from .random_patch import RandomPatch 13 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/doc/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Forgery-Detection/DCL/doc/main.png -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/models/__init__.py: -------------------------------------------------------------------------------- 1 | """model zoo""" 2 | from .base_net import BinaryClassifier 3 | from .dcl import DCL 4 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/models/base_net.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import timm 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | __all__ = ['BinaryClassifier'] 9 | 10 | MODEL_DICTS = {} 11 | MODEL_DICTS.update(timm.models.__dict__) 12 | 13 | 14 | class BinaryClassifier(nn.Module): 15 | def __init__(self, encoder, num_classes=2, drop_rate=0.2, has_feature=False, pretrained=False, **kwargs) -> None: 16 | """Base binary classifier 17 | Args: 18 | encoder ([nn.Module]): Backbone of the DCL 19 | num_classes (int, optional): Defaults to 2. 20 | drop_rate (float, optional): Defaults to 0.2. 21 | has_feature (bool, optional): Wthether to return feature maps. Defaults to False. 22 | pretrained (bool, optional): Whether to use a pretrained model. Defaults to False. 23 | """ 24 | super().__init__() 25 | self.encoder = MODEL_DICTS[encoder](pretrained=pretrained, **kwargs) 26 | for k, v in kwargs.items(): 27 | setattr(self, k, v) 28 | 29 | if hasattr(self.encoder, 'get_classifier'): 30 | self.num_features = self.encoder.get_classifier().in_features 31 | else: 32 | self.num_features = self.encoder.last_channel 33 | 34 | self.global_pool = nn.AdaptiveAvgPool2d((1, 1)) 35 | self.dropout = nn.Dropout(drop_rate) 36 | self.has_feature = has_feature 37 | self.feature_squeeze = nn.Conv2d(self.num_features, 1, 1) 38 | self.fc = nn.Linear(self.num_features, num_classes) 39 | 40 | def forward(self, x): 41 | featuremap = self.encoder.forward_features(x) 42 | x = self.global_pool(featuremap).flatten(1) 43 | x = self.dropout(x) 44 | x = self.fc(x) 45 | if self.has_feature: 46 | return x, featuremap 47 | return x 48 | 49 | 50 | if __name__ == '__main__': 51 | name = "tf_efficientnet_b4_ns" 52 | device = 'cpu' 53 | model = BinaryClassifier(name) 54 | model = model.to(device) 55 | model.eval() 56 | with torch.no_grad(): 57 | inputs = torch.rand(4, 3, 224, 224) 58 | inputs = inputs.to(device) 59 | out = model(inputs) 60 | print(out.shape) 61 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/DCL/requirements.txt: -------------------------------------------------------------------------------- 1 | albumentations==0.5.1 2 | easydict==1.9 3 | timm==0.4.12 4 | omegaconf==2.0.5 5 | nni==2.1 6 | tensorboardX==2.1 7 | wandb 8 | loguru 9 | rich 10 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/configs/ffpp.yaml: -------------------------------------------------------------------------------- 1 | seed: 1234 2 | torch_home: 3 | 4 | method: Deepfakes # should be one of ['Deepfakes', 'Face2Face', 'FaceSwap', 'NeuralTextures'] 5 | compression: c23 # should be one of ['c23', 'c40'] 6 | exam_dir: data_${method}_${compression} 7 | 8 | transform_params: 9 | image_size: 224 10 | mean: [0.485, 0.456, 0.406] 11 | std: [0.229, 0.224, 0.225] 12 | 13 | train: 14 | batch_size: 10 15 | num_workers: 8 16 | print_info_step_freq: 1 17 | max_epoches: 100 18 | use_warmup: True 19 | warmup_epochs: 1 20 | dataset: 21 | name: FFPP_Dataset 22 | params: 23 | root: data/ffpp_videos 24 | face_info_path: data/ffpp_face_rects.pkl 25 | method: ${method} 26 | compression: ${compression} 27 | split: train 28 | num_segments: 8 29 | sparse_span: 150 30 | 31 | test: 32 | batch_size: 10 33 | num_workers: 8 34 | dataset: 35 | name: FFPP_Dataset 36 | params: 37 | root: data/ffpp_videos 38 | face_info_path: data/ffpp_face_rects.pkl 39 | method: ${method} 40 | compression: ${compression} 41 | split: val 42 | num_segments: 16 43 | sparse_span: 150 44 | 45 | model: 46 | name: STIL_Model 47 | params: 48 | num_class: 2 49 | num_segment: 8 50 | resume: 51 | only_resume_model: False 52 | not_resume_layer_names: 53 | 54 | optimizer: 55 | name: Adam 56 | params: 57 | lr: 0.001 58 | weight_decay: 1.0e-5 59 | 60 | loss: 61 | name: CrossEntropyLoss 62 | params: 63 | 64 | scheduler: 65 | name: piecewise 66 | decay_epochs: [5, 20] 67 | decay_rates: [0.1, 0.05] 68 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the following main classes/functions: 3 | - FFPP_Dataset (class): 4 | dataset class for ff++ dataset. 5 | - get_dataloader (function): 6 | helper function to set up train/test dataloader 7 | """ 8 | from .video_dataset import * 9 | from .factory import * 10 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/datasets/factory.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from omegaconf import OmegaConf 4 | import torch.utils.data as data 5 | 6 | sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..', '..', '..')) 7 | from common.data import create_base_transforms, create_base_dataloader 8 | 9 | from .video_dataset import * 10 | 11 | 12 | def get_dataloader(args, split): 13 | """Set dataloader. 14 | 15 | Args: 16 | args (object): Args load from get_params function. 17 | split (str): One of ['train', 'test'] 18 | """ 19 | transform = create_base_transforms(args.transform_params, split=split) 20 | 21 | dataset_cfg = getattr(args, split).dataset 22 | dataset_params = OmegaConf.to_container(dataset_cfg.params, resolve=True) 23 | dataset_params['transform'] = transform 24 | 25 | _dataset = eval(dataset_cfg.name)(**dataset_params) 26 | 27 | _dataloader = create_base_dataloader(args, _dataset, split=split) 28 | 29 | return _dataloader 30 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/doc/stil.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/TFace/4939d8edc6c800ce7de1a0a1dcd1b7cf98c18031/security/tasks/Face-Forgery-Detection/STIL/doc/stil.jpg -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/models/__init__.py: -------------------------------------------------------------------------------- 1 | """STIL model definition 2 | """ 3 | from .stil import STIL_Model 4 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/models/stil.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import numpy as np 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | try: 8 | from models.ops import * 9 | except: 10 | from ops import * 11 | 12 | 13 | class STIL_Model(nn.Module): 14 | def __init__(self, 15 | num_class=2, 16 | num_segment=8, 17 | add_softmax=False, 18 | **kwargs): 19 | """ Model Builder for STIL model. 20 | STIL: Spatiotemporal Inconsistency Learning for DeepFake Video Detection (https://arxiv.org/abs/2109.01860) 21 | 22 | Args: 23 | num_class (int, optional): Number of classes. Defaults to 2. 24 | num_segment (int, optional): Number of segments (frames) fed to the model. Defaults to 8. 25 | add_softmax (bool, optional): Whether to add softmax layer at the end. Defaults to False. 26 | """ 27 | super().__init__() 28 | 29 | self.num_class = num_class 30 | self.num_segment = num_segment 31 | 32 | self.add_softmax = add_softmax 33 | 34 | self.build_model() 35 | 36 | 37 | def build_model(self): 38 | """ 39 | Construct the model. 40 | """ 41 | self.base_model = scnet50_v1d(self.num_segment, pretrained=True) 42 | 43 | fc_feature_dim = self.base_model.fc.in_features 44 | self.base_model.fc = nn.Linear(fc_feature_dim, self.num_class) 45 | 46 | if self.add_softmax: 47 | self.softmax_layer = nn.Softmax(dim=1) 48 | 49 | 50 | def forward(self, x): 51 | """Forward pass of the model. 52 | 53 | Args: 54 | x (torch.tensor): input tensor of shape (n, t*c, h, w). n is the batch_size, t is num_segment 55 | """ 56 | # img channel default to 3 57 | img_channel = 3 58 | 59 | # x: [n, tc, h, w] -> [nt, c, h, w] 60 | # out: [nt, num_class] 61 | out = self.base_model( 62 | x.view((-1, img_channel) + x.size()[2:]) 63 | ) 64 | 65 | out = out.view(-1, self.num_segment, self.num_class) # [n, t, num_class] 66 | out = out.mean(1, keepdim=False) # [n, num_class] 67 | 68 | if self.add_softmax: 69 | out = self.softmax_layer(out) 70 | 71 | return out 72 | 73 | 74 | def set_segment(self, num_segment): 75 | """Change num_segment of the model. 76 | Useful when the train and test want to feed different number of frames. 77 | 78 | Args: 79 | num_segment (int): New number of segments. 80 | """ 81 | self.num_segment = num_segment 82 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/test.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | # Manipulation method. One of ['Deepfakes', 'Face2Face', 'FaceSwap', 'NeuralTextures']. 4 | METHOD=Deepfakes 5 | # video compression rate. One of ['c23', 'c40']. 6 | COMPRESSION=c23 7 | # model path 8 | MODEL_PATH=xxx.tar 9 | 10 | python3 -m torch.distributed.launch --nproc_per_node=8 --master_port 12345 test.py -c configs/ffpp.yaml --method ${METHOD} --compression ${COMPRESSION} --model.resume ${MODEL_PATH} -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/train.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | # Manipulation method. One of ['Deepfakes', 'Face2Face', 'FaceSwap', 'NeuralTextures']. 4 | METHOD=Deepfakes 5 | # video compression rate. One of ['c23', 'c40']. 6 | COMPRESSION=c23 7 | 8 | python3 -m torch.distributed.launch --nproc_per_node=8 --master_port 12345 train.py -c configs/ffpp.yaml --method ${METHOD} --compression ${COMPRESSION} -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """Utils for STIL 2 | """ 3 | from .lr_utils import lr_tuner 4 | from .metrics import compute_metrics 5 | -------------------------------------------------------------------------------- /security/tasks/Face-Forgery-Detection/STIL/utils/metrics.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | 5 | def compute_metrics(model_outputs, labels): 6 | """ 7 | Compute the accuracy metrics. 8 | """ 9 | real_probs = F.softmax(model_outputs, dim=1)[:, 0] 10 | bin_preds = (real_probs <= 0.5).int() 11 | bin_labels = (labels != 0).int() 12 | 13 | real_cnt = (bin_labels == 0).sum() 14 | fake_cnt = (bin_labels == 1).sum() 15 | 16 | acc = (bin_preds == bin_labels).float().mean() 17 | 18 | real_acc = (bin_preds == bin_labels)[torch.where(bin_labels == 0)].sum() / (real_cnt + 1e-12) 19 | fake_acc = (bin_preds == bin_labels)[torch.where(bin_labels == 1)].sum() / (fake_cnt + 1e-12) 20 | 21 | return acc.item(), real_acc.item(), fake_acc.item(), real_cnt.item(), fake_cnt.item() 22 | --------------------------------------------------------------------------------