├── LICENSE ├── README.md ├── model ├── config.py ├── dataset.py ├── loss.py ├── model.py └── utils.py └── train.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Michigan State University 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Tree Learning for Zero-shot Face Anti-Spoofing 2 | Yaojie Liu, Joel Stehouwer, Amin Jourabloo, Xiaoming Liu 3 | 4 | ![alt text](https://yaojieliu.github.io/images/cvpr19.png) 5 | 6 | ## Setup 7 | Install the Tensorflow 2.0. 8 | 9 | ## Training 10 | To run the training code: 11 | python train.py 12 | 13 | 14 | ## Acknowledge 15 | Please cite the paper: 16 | 17 | @inproceedings{cvpr19yaojie, 18 | title={Deep Tree Learning for Zero-shot Face Anti-Spoofing}, 19 | author={Yaojie Liu, Joel Stehouwer, Amin Jourabloo, Xiaoming Liu}, 20 | booktitle={In Proceeding of IEEE Computer Vision and Pattern Recognition (CVPR 2019)}, 21 | address={Long Beach, CA}, 22 | year={2019} 23 | } 24 | 25 | If you have any question, please contact: [Yaojie Liu](liuyaoj1@msu.edu) 26 | 27 | -------------------------------------------------------------------------------- /model/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 2 | # 3 | # Yaojie Liu, Amin Jourabloo, Xiaoming Liu, Michigan State University 4 | # 5 | # All Rights Reserved. 6 | # 7 | # This research is based upon work supported by the Office of the Director of 8 | # National Intelligence (ODNI), Intelligence Advanced Research Projects Activity 9 | # (IARPA), via IARPA R&D Contract No. 2017-17020200004. The views and 10 | # conclusions contained herein are those of the authors and should not be 11 | # interpreted as necessarily representing the official policies or endorsements, 12 | # either expressed or implied, of the ODNI, IARPA, or the U.S. Government. The 13 | # U.S. Government is authorized to reproduce and distribute reprints for 14 | # Governmental purposes not withstanding any copyright annotation thereon. 15 | # ============================================================================== 16 | """ 17 | DTN for Zero-shot Face Anti-spoofing 18 | Base Configurations class. 19 | 20 | """ 21 | import numpy as np 22 | import tensorflow as tf 23 | 24 | # Base Configuration Class 25 | class Config(object): 26 | """Base configuration class. For custom configurations, create a 27 | sub-class that inherits from this one and override properties 28 | that need to be changed. 29 | """ 30 | # GPU Usage 31 | GPU_USAGE = 1 32 | 33 | # Log and Model Storage Default 34 | LOG_DIR = './log/DTN' 35 | 36 | # Input Data Meta 37 | IMAGE_SIZE = 256 38 | MAP_SIZE = 64 39 | 40 | TRU_PARAMETERS = { 41 | "alpha": 1e-3, 42 | "beta": 1e-2, 43 | "mu_update_rate": 1e-3, 44 | } 45 | 46 | # Training Meta 47 | STEPS_PER_EPOCH = 1000 48 | MAX_EPOCH = 40 49 | NUM_EPOCHS_PER_DECAY = 12.0 # Epochs after which learning rate decays 50 | BATCH_SIZE = 32 51 | LEARNING_RATE = 0.001 # Initial learning rate. 52 | LEARNING_MOMENTUM = 0.999 # The decay to use for the moving average. 53 | 54 | def __init__(self): 55 | """Set values of computed attributes.""" 56 | gpus = tf.config.experimental.list_physical_devices('GPU') 57 | if gpus: 58 | try: 59 | tf.config.experimental.set_memory_growth(gpus[self.GPU_USAGE], True) 60 | tf.config.experimental.set_visible_devices(gpus[self.GPU_USAGE], 'GPU') 61 | logical_gpus = tf.config.experimental.list_logical_devices('GPU') 62 | print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPU") 63 | except RuntimeError as e: 64 | # Memory growth must be set before GPUs have been initialized 65 | print(e) 66 | 67 | def display(self): 68 | """Display Configuration values.""" 69 | print("\nConfigurations:") 70 | for a in dir(self): 71 | if not a.startswith("__") and not callable(getattr(self, a)): 72 | print("{:30} {}".format(a, getattr(self, a))) 73 | print("\n") 74 | -------------------------------------------------------------------------------- /model/dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 2 | # 3 | # Yaojie Liu, Joel Stehouwer, Amin Jourabloo, Xiaoming Liu, Michigan State University 4 | # 5 | # All Rights Reserved. 6 | # 7 | # This research is based upon work supported by the Office of the Director of 8 | # National Intelligence (ODNI), Intelligence Advanced Research Projects Activity 9 | # (IARPA), via IARPA R&D Contract No. 2017-17020200004. The views and 10 | # conclusions contained herein are those of the authors and should not be 11 | # interpreted as necessarily representing the official policies or endorsements, 12 | # either expressed or implied, of the ODNI, IARPA, or the U.S. Government. The 13 | # U.S. Government is authorized to reproduce and distribute reprints for 14 | # Governmental purposes not withstanding any copyright annotation thereon. 15 | # ============================================================================== 16 | """ 17 | DTN for Zero-shot Face Anti-spoofing 18 | Data Loading class. 19 | 20 | """ 21 | import tensorflow as tf 22 | import numpy as np 23 | import glob 24 | 25 | 26 | class Dataset(): 27 | def __init__(self, config, mode): 28 | self.config = config 29 | if self.config.MODE == 'training': 30 | self.input_tensors = self.inputs_for_training(mode) 31 | else: 32 | self.input_tensors, self.name_list = self.inputs_for_testing() 33 | self.feed = iter(self.input_tensors) 34 | 35 | def inputs_for_training(self, mode): 36 | autotune = tf.data.experimental.AUTOTUNE 37 | if mode == 'train': 38 | data_dir = self.config.DATA_DIR 39 | else: 40 | data_dir = self.config.DATA_DIR_VAL 41 | data_samples = [] 42 | for _dir in data_dir: 43 | _list = glob.glob(_dir+'/*.dat') 44 | data_samples += _list 45 | shuffle_buffer_size = len(data_samples) 46 | dataset = tf.data.Dataset.from_tensor_slices(data_samples) 47 | dataset = dataset.shuffle(shuffle_buffer_size).repeat(-1) 48 | dataset = dataset.map(map_func=self.parse_fn, num_parallel_calls=autotune) 49 | dataset = dataset.batch(batch_size=self.config.BATCH_SIZE).prefetch(buffer_size=autotune) 50 | return dataset 51 | 52 | def inputs_for_testing(self): 53 | autotune = tf.data.experimental.AUTOTUNE 54 | data_dir = self.config.DATA_DIR 55 | data_samples = [] 56 | for _dir in data_dir: 57 | _list = sorted(glob.glob(_dir + '/*.dat')) 58 | data_samples += _list 59 | dataset = tf.data.Dataset.from_tensor_slices(data_samples) 60 | dataset = dataset.map(map_func=self.parse_fn, num_parallel_calls=autotune) 61 | dataset = dataset.batch(batch_size=self.config.BATCH_SIZE).prefetch(buffer_size=autotune) 62 | return dataset, data_samples 63 | 64 | def parse_fn(self, file): 65 | config = self.config 66 | image_size = config.IMAGE_SIZE 67 | dmap_size = config.MAP_SIZE 68 | label_size = 1 69 | 70 | def _parse_function(_file): 71 | _file = _file.decode('UTF-8') 72 | image_bytes = image_size * image_size * 3 73 | dmap_bytes = dmap_size * dmap_size 74 | bin = np.fromfile(_file, dtype='uint8') 75 | image = np.transpose(bin[0:image_bytes].reshape((3, image_size, image_size)) / 255, (1, 2, 0)) 76 | dmap = np.transpose(bin[image_bytes:image_bytes+dmap_bytes].reshape((1, dmap_size, dmap_size)) / 255, (1, 2, 0)) 77 | label = bin[image_bytes+dmap_bytes:image_bytes+dmap_bytes+label_size] / 1 78 | dmap1 = dmap * (1-label) 79 | dmap2 = np.ones_like(dmap) * label 80 | dmap = np.concatenate([dmap1, dmap2], axis=2) 81 | 82 | return image.astype(np.float32), dmap.astype(np.float32), label.astype(np.float32) 83 | 84 | image_ts, dmap_ts, label_ts = tf.numpy_function(_parse_function, [file], [tf.float32, tf.float32, tf.float32]) 85 | image_ts = tf.ensure_shape(image_ts, [config.IMAGE_SIZE, config.IMAGE_SIZE, 3]) 86 | dmap_ts = tf.ensure_shape(dmap_ts, [config.MAP_SIZE, config.MAP_SIZE, 2]) 87 | label_ts = tf.ensure_shape(label_ts, [1]) 88 | return image_ts, dmap_ts, label_ts 89 | -------------------------------------------------------------------------------- /model/loss.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 2 | # 3 | # Yaojie Liu, Joel Stehouwer, Amin Jourabloo, Xiaoming Liu, Michigan State University 4 | # 5 | # All Rights Reserved. 6 | # 7 | # This research is based upon work supported by the Office of the Director of 8 | # National Intelligence (ODNI), Intelligence Advanced Research Projects Activity 9 | # (IARPA), via IARPA R&D Contract No. 2017-17020200004. The views and 10 | # conclusions contained herein are those of the authors and should not be 11 | # interpreted as necessarily representing the official policies or endorsements, 12 | # either expressed or implied, of the ODNI, IARPA, or the U.S. Government. The 13 | # U.S. Government is authorized to reproduce and distribute reprints for 14 | # Governmental purposes not withstanding any copyright annotation thereon. 15 | # ============================================================================== 16 | """ 17 | DTN for Zero-shot Face Anti-spoofing 18 | Losses class. 19 | 20 | """ 21 | import tensorflow as tf 22 | import tensorflow.keras.layers as layers 23 | import numpy as np 24 | 25 | 26 | def l1_loss(x, y, mask=None): 27 | xshape = x.shape 28 | if mask is not None: 29 | loss = tf.reduce_mean(tf.reshape(tf.abs(x-y), [xshape[0], -1]), axis=1, keepdims=True) 30 | loss = tf.reduce_sum(loss * mask) / (tf.reduce_sum(mask) + 1e-8) 31 | else: 32 | loss = tf.reduce_mean(tf.abs(x-y)) 33 | return loss 34 | 35 | 36 | def l2_loss(x, y, mask=None): 37 | xshape = x.shape 38 | if mask is None: 39 | loss = tf.reduce_mean(tf.reshape(tf.square(x-y), [xshape[0], -1]), axis=1, keepdims=True) 40 | loss = tf.reduce_sum(loss * mask) / (tf.reduce_sum(mask) + 1e-8) 41 | else: 42 | loss = tf.reduce_mean(tf.square(x-y)) 43 | return loss 44 | 45 | 46 | def leaf_l1_loss(xlist, y, masklist): 47 | loss_list = [] 48 | xshape = xlist[0].shape 49 | for x, mask in zip(xlist, masklist): 50 | loss = tf.reduce_mean(tf.reshape(tf.abs(x-y), [xshape[0], -1]), axis=1) 51 | # tag of spoof 52 | tag = tf.reduce_sum(mask[:, 0]) 53 | tag = tag / (tag + 1e-8) 54 | # spoof 55 | spoof_loss = tf.reduce_sum(loss * mask[:, 0]) / (tf.reduce_sum(mask[:, 0]) + 1e-8) 56 | 57 | # live 58 | live_loss = tf.reduce_sum(loss * mask[:, 1]) / (tf.reduce_sum(mask[:, 1]) + 1e-8) 59 | 60 | total_loss = (spoof_loss + live_loss)/2 61 | loss_list.append(total_loss*tag) 62 | loss = tf.reduce_mean(loss_list) 63 | return loss 64 | 65 | 66 | def leaf_l2_loss(xlist, y, masklist): 67 | loss_list = [] 68 | for x, mask in zip(xlist, masklist): 69 | xshape = x.shape 70 | print(x.shape, y.shape, mask.shape) 71 | input() 72 | # spoof 73 | spoof_loss = tf.reduce_mean(tf.reshape(tf.square(x-y), [xshape[0], -1]), axis=1) 74 | spoof_loss = tf.reduce_sum(loss * mask[:, 0]) / (tf.reduce_sum(mask[:, 0]) + 1e-8) 75 | # live 76 | live_loss = tf.reduce_mean(tf.reshape(tf.square(x - y), [xshape[0], -1]), axis=1) 77 | live_loss = tf.reduce_sum(loss * mask[:, 1]) / (tf.reduce_sum(mask[:, 1]) + 1e-8) 78 | loss = spoof_loss + live_loss 79 | loss_list.append(loss) 80 | 81 | return loss 82 | 83 | def leaf_l1_score(xlist, masklist, ch=None): 84 | loss_list = [] 85 | xshape = xlist[0].shape 86 | scores = [] 87 | for x, mask in zip(xlist, masklist): 88 | if ch is not None: 89 | score = tf.reduce_mean(tf.reshape(tf.abs(x[:, :, :, ch]), [xshape[0], -1]), axis=1) 90 | else: 91 | score = tf.reduce_mean(tf.reshape(tf.abs(x), [xshape[0], -1]), axis=1) 92 | spoof_score = score * mask[:, 0] 93 | scores.append(spoof_score) 94 | loss = np.sum(np.stack(scores, axis=1), axis=1) 95 | return loss -------------------------------------------------------------------------------- /model/model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 2 | # 3 | # Yaojie Liu, Joel Stehouwer, Amin Jourabloo, Xiaoming Liu, Michigan State University 4 | # 5 | # All Rights Reserved. 6 | # 7 | # This research is based upon work supported by the Office of the Director of 8 | # National Intelligence (ODNI), Intelligence Advanced Research Projects Activity 9 | # (IARPA), via IARPA R&D Contract No. 2017-17020200004. The views and 10 | # conclusions contained herein are those of the authors and should not be 11 | # interpreted as necessarily representing the official policies or endorsements, 12 | # either expressed or implied, of the ODNI, IARPA, or the U.S. Government. The 13 | # U.S. Government is authorized to reproduce and distribute reprints for 14 | # Governmental purposes not withstanding any copyright annotation thereon. 15 | # ============================================================================== 16 | """ 17 | DTN for Zero-shot Face Anti-spoofing 18 | DTN Model class. 19 | 20 | """ 21 | 22 | from __future__ import absolute_import 23 | from __future__ import division 24 | from __future__ import print_function 25 | 26 | import os 27 | import time 28 | import math 29 | import tensorflow as tf 30 | import tensorflow.keras.backend as K 31 | import numpy as np 32 | import cv2 33 | from model.utils import CRU, TRU, SFL, Conv, Downsample, Error 34 | from model.loss import l1_loss, l2_loss, leaf_l1_loss, leaf_l2_loss, leaf_l1_score 35 | 36 | 37 | ############################################################ 38 | # Plot the figures 39 | ############################################################ 40 | def plotResults(fname, result_list): 41 | columm = [] 42 | for fig in result_list: 43 | shape = fig.shape 44 | fig = fig.numpy() 45 | row = [] 46 | for idx in range(shape[0]): 47 | item = fig[idx, :, :, :] 48 | if item.shape[2] == 1: 49 | item = np.concatenate([item, item, item], axis=2) 50 | item = cv2.cvtColor(cv2.resize(item, (128, 128)), cv2.COLOR_RGB2BGR) 51 | row.append(item) 52 | row = np.concatenate(row, axis=1) 53 | columm.append(row) 54 | columm = np.concatenate(columm, axis=0) 55 | img = np.uint8(columm * 255) 56 | cv2.imwrite(fname, img) 57 | 58 | 59 | ############################################################ 60 | # Deep Tree Network 61 | ############################################################ 62 | class DTN(tf.keras.models.Model): 63 | def __init__(self, filters, config): 64 | super(DTN, self).__init__() 65 | self.config = config 66 | layer = [1, 2, 4, 8, 16] 67 | self.conv1 = Conv(filters, 5, apply_batchnorm=False) 68 | # CRU 69 | self.cru0 = CRU(filters) 70 | self.cru1 = CRU(filters) 71 | self.cru2 = CRU(filters) 72 | self.cru3 = CRU(filters) 73 | self.cru4 = CRU(filters) 74 | self.cru5 = CRU(filters) 75 | self.cru6 = CRU(filters) 76 | # TRU 77 | alpha = config.TRU_PARAMETERS['alpha'] 78 | beta = config.TRU_PARAMETERS['beta'] 79 | self.tru0 = TRU(filters, '1', alpha, beta) 80 | self.tru1 = TRU(filters, '2', alpha, beta) 81 | self.tru2 = TRU(filters, '3', alpha, beta) 82 | self.tru3 = TRU(filters, '4', alpha, beta) 83 | self.tru4 = TRU(filters, '5', alpha, beta) 84 | self.tru5 = TRU(filters, '6', alpha, beta) 85 | self.tru6 = TRU(filters, '7', alpha, beta) 86 | # SFL 87 | self.sfl0 = SFL(filters) 88 | self.sfl1 = SFL(filters) 89 | self.sfl2 = SFL(filters) 90 | self.sfl3 = SFL(filters) 91 | self.sfl4 = SFL(filters) 92 | self.sfl5 = SFL(filters) 93 | self.sfl6 = SFL(filters) 94 | self.sfl7 = SFL(filters) 95 | 96 | @tf.function 97 | def call(self, x, label, training): 98 | if training: 99 | mask_spoof = label 100 | mask_live = 1 - label 101 | else: 102 | mask_spoof = tf.ones_like(label) 103 | mask_live = tf.zeros_like(label) 104 | ''' Tree Level 1 ''' 105 | x = self.conv1(x, training) 106 | x_cru0 = self.cru0(x) 107 | x_tru0, route_value0, tru0_loss = self.tru0(x_cru0, mask_spoof, training) 108 | 109 | ''' Tree Level 2 ''' 110 | x_cru00 = self.cru1(x_cru0, training) 111 | x_cru01 = self.cru2(x_cru0, training) 112 | x_tru00, route_value00, tru00_loss = self.tru1(x_cru00, x_tru0[0], training) 113 | x_tru01, route_value01, tru01_loss = self.tru2(x_cru01, x_tru0[1], training) 114 | 115 | ''' Tree Level 3 ''' 116 | x_cru000 = self.cru3(x_cru00, training) 117 | x_cru001 = self.cru4(x_cru00, training) 118 | x_cru010 = self.cru5(x_cru01, training) 119 | x_cru011 = self.cru6(x_cru01, training) 120 | x_tru000, route_value000, tru000_loss = self.tru3(x_cru000, x_tru00[0], training) 121 | x_tru001, route_value001, tru001_loss = self.tru4(x_cru001, x_tru00[1], training) 122 | x_tru010, route_value010, tru010_loss = self.tru5(x_cru010, x_tru01[0], training) 123 | x_tru011, route_value011, tru011_loss = self.tru6(x_cru011, x_tru01[1], training) 124 | 125 | ''' Tree Level 4 ''' 126 | map0, cls0 = self.sfl0(x_cru000, training) 127 | map1, cls1 = self.sfl1(x_cru000, training) 128 | map2, cls2 = self.sfl2(x_cru001, training) 129 | map3, cls3 = self.sfl3(x_cru001, training) 130 | map4, cls4 = self.sfl4(x_cru010, training) 131 | map5, cls5 = self.sfl5(x_cru010, training) 132 | map6, cls6 = self.sfl6(x_cru011, training) 133 | map7, cls7 = self.sfl7(x_cru011, training) 134 | ''' Output ''' 135 | maps = [map0, map1, map2, map3, map4, map5, map6, map7] 136 | clss = [cls0, cls1, cls2, cls3, cls4, cls5, cls6, cls7] 137 | route_value = [route_value0, route_value00, route_value01, 138 | route_value000, route_value001, route_value010, route_value011] 139 | x_tru0000 = tf.concat([x_tru000[0], mask_live], axis=1) 140 | x_tru0001 = tf.concat([x_tru000[1], mask_live], axis=1) 141 | x_tru0010 = tf.concat([x_tru001[0], mask_live], axis=1) 142 | x_tru0011 = tf.concat([x_tru001[1], mask_live], axis=1) 143 | x_tru0100 = tf.concat([x_tru010[0], mask_live], axis=1) 144 | x_tru0101 = tf.concat([x_tru010[1], mask_live], axis=1) 145 | x_tru0110 = tf.concat([x_tru011[0], mask_live], axis=1) 146 | x_tru0111 = tf.concat([x_tru011[1], mask_live], axis=1) 147 | leaf_node_mask = [x_tru0000, x_tru0001, x_tru0010, x_tru0011, x_tru0100, x_tru0101, x_tru0110, x_tru0111] 148 | 149 | if training: 150 | # for the training 151 | route_loss = [tru0_loss[0], tru00_loss[0], tru01_loss[0], 152 | tru000_loss[0], tru001_loss[0], tru010_loss[0], tru011_loss[0]] 153 | recon_loss = [tru0_loss[1], tru00_loss[1], tru01_loss[1], 154 | tru000_loss[1], tru001_loss[1], tru010_loss[1], tru011_loss[1]] 155 | mu_update = [self.tru0.project.mu_of_visit+0, 156 | self.tru1.project.mu_of_visit+0, 157 | self.tru2.project.mu_of_visit+0, 158 | self.tru3.project.mu_of_visit+0, 159 | self.tru4.project.mu_of_visit+0, 160 | self.tru5.project.mu_of_visit+0, 161 | self.tru6.project.mu_of_visit+0] 162 | eigenvalue = [self.tru0.project.eigenvalue, 163 | self.tru1.project.eigenvalue, 164 | self.tru2.project.eigenvalue, 165 | self.tru3.project.eigenvalue, 166 | self.tru4.project.eigenvalue, 167 | self.tru5.project.eigenvalue, 168 | self.tru6.project.eigenvalue] 169 | trace = [self.tru0.project.trace, 170 | self.tru1.project.trace, 171 | self.tru2.project.trace, 172 | self.tru3.project.trace, 173 | self.tru4.project.trace, 174 | self.tru5.project.trace, 175 | self.tru6.project.trace] 176 | 177 | return maps, clss, route_value, leaf_node_mask, [route_loss, recon_loss], mu_update, eigenvalue, trace 178 | else: 179 | return maps, clss, route_value, leaf_node_mask 180 | 181 | 182 | class Model: 183 | def __init__(self, config): 184 | self.config = config 185 | # model 186 | self.dtn = DTN(32, config) 187 | # model optimizer 188 | self.dtn_op = tf.compat.v1.train.AdamOptimizer(config.LEARNING_RATE, beta1=0.5) 189 | # model losses 190 | self.depth_map_loss = Error() 191 | self.class_loss = Error() 192 | self.route_loss = Error() 193 | self.uniq_loss = Error() 194 | # model saving setting 195 | self.last_epoch = 0 196 | self.checkpoint_manager = [] 197 | 198 | def compile(self): 199 | checkpoint_dir = self.config.LOG_DIR 200 | checkpoint = tf.train.Checkpoint(dtn=self.dtn, 201 | dtn_optimizer=self.dtn_op) 202 | self.checkpoint_manager = tf.train.CheckpointManager(checkpoint, checkpoint_dir, max_to_keep=30) 203 | last_checkpoint = self.checkpoint_manager.latest_checkpoint 204 | checkpoint.restore(last_checkpoint) 205 | if last_checkpoint: 206 | self.last_epoch = int(last_checkpoint.split('-')[-1]) 207 | print("Restored from {}".format(last_checkpoint)) 208 | else: 209 | print("Initializing from scratch.") 210 | 211 | def train(self, train, val=None): 212 | config = self.config 213 | step_per_epoch = config.STEPS_PER_EPOCH 214 | step_per_epoch_val = config.STEPS_PER_EPOCH_VAL 215 | epochs = config.MAX_EPOCH 216 | 217 | # data stream 218 | it = train.feed 219 | global_step = self.last_epoch * step_per_epoch 220 | if val is not None: 221 | it_val = val.feed 222 | for epoch in range(self.last_epoch, epochs): 223 | start = time.time() 224 | # define the 225 | self.dtn_op = tf.compat.v1.train.AdamOptimizer(config.LEARNING_RATE, beta1=0.5) 226 | ''' train phase''' 227 | for step in range(step_per_epoch): 228 | depth_map_loss, class_loss, route_loss, uniq_loss, spoof_counts, eigenvalue, trace, _to_plot =\ 229 | self.train_one_step(next(it), global_step, True) 230 | # display loss 231 | global_step += 1 232 | print('Epoch {:d}-{:d}/{:d}: Map:{:.3g}, Cls:{:.3g}, Route:{:.3g}({:3.3f}, {:3.3f}), Uniq:{:.3g}, ' 233 | 'Counts:[{:d},{:d},{:d},{:d},{:d},{:d},{:d},{:d}] '. 234 | format(epoch + 1, step + 1, step_per_epoch, 235 | self.depth_map_loss(depth_map_loss), 236 | self.class_loss(class_loss), 237 | self.route_loss(route_loss), eigenvalue, trace, 238 | self.uniq_loss(uniq_loss), 239 | spoof_counts[0], spoof_counts[1], spoof_counts[2], spoof_counts[3], 240 | spoof_counts[4], spoof_counts[5], spoof_counts[6], spoof_counts[7],), end='\r') 241 | # plot the figure 242 | '''if (step + 1) % 400 == 0: 243 | fname = self.config.LOG_DIR + '/epoch-' + str(epoch + 1) + '-train-' + str(step + 1) + '.png' 244 | plotResults(fname, _to_plot)''' 245 | 246 | # save the model 247 | if (epoch + 1) % 1 == 0: 248 | self.checkpoint_manager.save(checkpoint_number=epoch + 1) 249 | print('\n', end='\r') 250 | 251 | ''' eval phase''' 252 | if val is not None: 253 | for step in range(step_per_epoch_val): 254 | depth_map_loss, class_loss, route_loss, uniq_loss, spoof_counts, eigenvalue, trace, _to_plot =\ 255 | self.train_one_step(next(it_val), global_step, False) 256 | # display something 257 | print(' Val-{:d}/{:d}: Map:{:.3g}, Cls:{:.3g}, Route:{:.3g}({:3.3f}, {:3.3f}), Uniq:{:.3g}, ' 258 | 'Counts:[{:d},{:d},{:d},{:d},{:d},{:d},{:d},{:d}] '. 259 | format(step + 1, step_per_epoch_val, 260 | self.depth_map_loss(depth_map_loss, val=1), 261 | self.class_loss(class_loss, val=1), 262 | self.route_loss(route_loss, val=1), eigenvalue, trace, 263 | self.recon_loss(uniq_loss, val=1), 264 | spoof_counts[0], spoof_counts[1], spoof_counts[2], spoof_counts[3], 265 | spoof_counts[4], spoof_counts[5], spoof_counts[6], spoof_counts[7], ), end='\r') 266 | # plot the figure 267 | '''if (step + 1) % 100 == 0: 268 | fname = self.config.LOG_DIR + '/epoch-' + str(epoch + 1) + '-val-' + str(step+1) + '.png' 269 | plotResults(fname, _to_plot)''' 270 | self.depth_map_loss.reset() 271 | self.class_loss.reset() 272 | self.route_loss.reset() 273 | self.uniq_loss.reset() 274 | 275 | # time of one epoch 276 | print('\n Time taken for epoch {} is {:3g} sec'.format(epoch + 1, time.time() - start)) 277 | return 0 278 | 279 | def train_one_step(self, data_batch, step, training): 280 | dtn = self.dtn 281 | dtn_op = self.dtn_op 282 | image, dmap, labels = data_batch 283 | with tf.GradientTape() as tape: 284 | dmap_pred, cls_pred, route_value, leaf_node_mask, tru_loss, mu_update, eigenvalue, trace =\ 285 | dtn(image, labels, True) 286 | 287 | # supervised feature loss 288 | depth_map_loss = leaf_l1_loss(dmap_pred, tf.image.resize(dmap, [32, 32]), leaf_node_mask) 289 | class_loss = leaf_l1_loss(cls_pred, labels, leaf_node_mask) 290 | supervised_loss = depth_map_loss + 0.001*class_loss 291 | 292 | # unsupervised tree loss 293 | route_loss = tf.reduce_mean(tf.stack(tru_loss[0], axis=0) * [1., 0.5, 0.5, 0.25, 0.25, 0.25, 0.25]) 294 | uniq_loss = tf.reduce_mean(tf.stack(tru_loss[1], axis=0) * [1., 0.5, 0.5, 0.25, 0.25, 0.25, 0.25]) 295 | eigenvalue = np.mean(np.stack(eigenvalue, axis=0) * [1., 0.5, 0.5, 0.25, 0.25, 0.25, 0.25]) 296 | trace = np.mean(np.stack(trace, axis=0) * [1., 0.5, 0.5, 0.25, 0.25, 0.25, 0.25]) 297 | unsupervised_loss = 2*route_loss + 0.001*uniq_loss 298 | 299 | # total loss 300 | if step > 10000: 301 | loss = supervised_loss + unsupervised_loss 302 | else: 303 | loss = supervised_loss 304 | 305 | if training: 306 | # back-propagate 307 | gradients = tape.gradient(loss, dtn.variables) 308 | dtn_op.apply_gradients(zip(gradients, dtn.variables)) 309 | 310 | # Update mean values for each tree node 311 | mu_update_rate = self.config.TRU_PARAMETERS["mu_update_rate"] 312 | mu = [dtn.tru0.project.mu, dtn.tru1.project.mu, dtn.tru2.project.mu, dtn.tru3.project.mu, 313 | dtn.tru4.project.mu, dtn.tru5.project.mu, dtn.tru6.project.mu] 314 | for mu, mu_of_visit in zip(mu, mu_update): 315 | if step == 0: 316 | update_mu = mu_of_visit 317 | else: 318 | update_mu = mu_of_visit * mu_update_rate + mu * (1 - mu_update_rate) 319 | K.set_value(mu, update_mu) 320 | 321 | # leaf counts 322 | spoof_counts = [] 323 | for leaf in leaf_node_mask: 324 | spoof_count = tf.reduce_sum(leaf[:, 0]).numpy() 325 | spoof_counts.append(int(spoof_count)) 326 | 327 | _to_plot = [image, dmap, dmap_pred[0]] 328 | 329 | return depth_map_loss, class_loss, route_loss, uniq_loss, spoof_counts, eigenvalue, trace, _to_plot -------------------------------------------------------------------------------- /model/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 2 | # 3 | # Yaojie Liu, Joel Stehouwer, Amin Jourabloo, Xiaoming Liu, Michigan State University 4 | # 5 | # All Rights Reserved. 6 | # 7 | # This research is based upon work supported by the Office of the Director of 8 | # National Intelligence (ODNI), Intelligence Advanced Research Projects Activity 9 | # (IARPA), via IARPA R&D Contract No. 2017-17020200004. The views and 10 | # conclusions contained herein are those of the authors and should not be 11 | # interpreted as necessarily representing the official policies or endorsements, 12 | # either expressed or implied, of the ODNI, IARPA, or the U.S. Government. The 13 | # U.S. Government is authorized to reproduce and distribute reprints for 14 | # Governmental purposes not withstanding any copyright annotation thereon. 15 | # ============================================================================== 16 | """ 17 | DTN for Zero-shot Face Anti-spoofing 18 | DTN Basic Blocks class. 19 | 20 | """ 21 | import tensorflow as tf 22 | import tensorflow.keras.layers as layers 23 | import numpy as np 24 | from model.loss import l1_loss, l2_loss 25 | 26 | 27 | class Error: 28 | def __init__(self): 29 | self.value = 0 30 | self.value_val = 0 31 | self.step = 0 32 | self.step_val = 0 33 | 34 | def __call__(self, update, val=0): 35 | if val == 1: 36 | self.value_val += update 37 | self.step_val += 1 38 | return self.value_val / self.step_val 39 | else: 40 | self.value += update 41 | self.step += 1 42 | return self.value / self.step 43 | 44 | def reset(self): 45 | self.value = 0 46 | self.value_val = 0 47 | self.step = 0 48 | self.step_val = 0 49 | 50 | 51 | class Linear(layers.Layer): 52 | def __init__(self, idx, alpha, beta, input_dim=32): 53 | super(Linear, self).__init__() 54 | initializer = tf.random_normal_initializer(0., 0.02) 55 | initializer0 = tf.zeros_initializer() 56 | self.v = tf.Variable(initial_value=initializer(shape=(1, input_dim), dtype='float32'), 57 | trainable=True, name='tru/v/'+idx) 58 | self.mu = tf.Variable(initial_value=initializer0(shape=(1, input_dim), dtype='float32'), 59 | trainable=True, name='tru/mu/'+idx) 60 | # training hyper-parameters 61 | self.alpha = alpha 62 | self.beta = beta 63 | # mean, eigenvalue and trace for each mini-batch 64 | self.mu_of_visit = 0 65 | self.eigenvalue = 0. 66 | self.trace = 0. 67 | 68 | def call(self, x, mask, training): 69 | norm_v = self.v / (tf.norm(self.v) + 1e-8) 70 | norm_v_t = tf.transpose(norm_v, [1, 0]) 71 | num_of_visit = tf.reduce_sum(mask) 72 | 73 | if training and num_of_visit > 1: 74 | # use only the visiting samples 75 | index = tf.where(tf.greater(mask[:, 0], tf.constant(0.))) 76 | index_not = tf.where(tf.equal(mask[:, 0], tf.constant(0.))) 77 | x_sub = tf.gather_nd(x, index) - tf.stop_gradient(self.mu) 78 | x_not = tf.gather_nd(x, index_not) 79 | x_sub_t = tf.transpose(x_sub, [1, 0]) 80 | 81 | # compute the covariance matrix, eigenvalue, and the trace 82 | covar = tf.matmul(x_sub_t, x_sub) / num_of_visit 83 | eigenvalue = tf.reshape(tf.matmul(tf.matmul(norm_v, covar), norm_v_t), []) 84 | trace = tf.linalg.trace(covar) 85 | # compute the route loss 86 | # print(tf.exp(-self.alpha * eigenvalue), self.beta * trace) 87 | route_loss = tf.exp(-self.alpha * eigenvalue) + self.beta * trace 88 | uniq_loss = -tf.reduce_mean(tf.square(tf.matmul(x_sub, norm_v_t))) + \ 89 | tf.reduce_mean(tf.square(tf.matmul(x_not, norm_v_t))) 90 | # compute mean and response for this batch 91 | self.mu_of_visit = tf.reduce_mean(x_sub, axis=0, keepdims=True) 92 | self.eigenvalue = eigenvalue 93 | self.trace = trace 94 | x -= tf.stop_gradient(self.mu_of_visit) 95 | route_value = tf.matmul(x, norm_v_t) 96 | else: 97 | self.mu_of_visit = self.mu 98 | self.eigenvalue = 0. 99 | self.trace = 0. 100 | x -= self.mu 101 | route_value = tf.matmul(x, norm_v_t) 102 | route_loss = 0. 103 | uniq_loss = 0. 104 | 105 | return route_value, route_loss, uniq_loss 106 | 107 | 108 | class Downsample(tf.keras.Model): 109 | def __init__(self, filters, size, padding='SAME', apply_batchnorm=True): 110 | super(Downsample, self).__init__() 111 | self.apply_batchnorm = apply_batchnorm 112 | initializer = tf.random_normal_initializer(0., 0.02) 113 | filters = int(filters) 114 | self.conv1 = layers.Conv2D(filters, 115 | (size, size), 116 | strides=2, 117 | padding=padding, 118 | kernel_initializer=initializer, 119 | use_bias=False) 120 | if self.apply_batchnorm: 121 | self.batchnorm = tf.keras.layers.BatchNormalization() 122 | 123 | def call(self, x, training): 124 | x = self.conv1(x) 125 | if self.apply_batchnorm: 126 | x = self.batchnorm(x, training=training) 127 | x = tf.nn.leaky_relu(x) 128 | return x 129 | 130 | 131 | class Upsample(tf.keras.Model): 132 | def __init__(self, filters, size, apply_dropout=False): 133 | super(Upsample, self).__init__() 134 | self.apply_dropout = apply_dropout 135 | initializer = tf.random_normal_initializer(0., 0.02) 136 | filters = int(filters) 137 | self.up_conv = tf.keras.layers.Conv2DTranspose(filters, 138 | (size, size), 139 | strides=2, 140 | padding='same', 141 | kernel_initializer=initializer, 142 | use_bias=False) 143 | self.batchnorm = tf.keras.layers.BatchNormalization() 144 | if self.apply_dropout: 145 | self.dropout = tf.keras.layers.Dropout(0.5) 146 | 147 | def call(self, x, training): 148 | x = self.up_conv(x) 149 | x = self.batchnorm(x, training=training) 150 | if self.apply_dropout: 151 | x = self.dropout(x, training=training) 152 | x = tf.nn.leaky_relu(x) 153 | return x 154 | 155 | 156 | class Conv(tf.keras.Model): 157 | def __init__(self, filters, size, stride=1, activation=True, padding='SAME', apply_batchnorm=True): 158 | super(Conv, self).__init__() 159 | self.apply_batchnorm = apply_batchnorm 160 | self.activation = activation 161 | initializer = tf.random_normal_initializer(0., 0.02) 162 | filters = int(filters) 163 | self.conv1 = layers.Conv2D(filters, 164 | (size, size), 165 | strides=stride, 166 | padding=padding, 167 | kernel_initializer=initializer, 168 | use_bias=False) 169 | if self.apply_batchnorm: 170 | self.batchnorm = layers.BatchNormalization() 171 | 172 | def call(self, x, training): 173 | x = self.conv1(x) 174 | if self.apply_batchnorm: 175 | x = self.batchnorm(x, training=training) 176 | if self.activation: 177 | x = tf.nn.leaky_relu(x) 178 | return x 179 | 180 | 181 | class Dense(tf.keras.Model): 182 | def __init__(self, filters, activation=True, apply_batchnorm=True, apply_dropout=False): 183 | super(Dense, self).__init__() 184 | self.apply_batchnorm = apply_batchnorm 185 | self.activation = activation 186 | self.apply_dropout = apply_dropout 187 | initializer = tf.random_normal_initializer(0., 0.02) 188 | filters = int(filters) 189 | self.dense = layers.Dense(filters, 190 | kernel_initializer=initializer, 191 | use_bias=False) 192 | if self.apply_batchnorm: 193 | self.batchnorm = layers.BatchNormalization() 194 | if self.apply_dropout: 195 | self.dropout = tf.keras.layers.Dropout(0.3) 196 | 197 | def call(self, x, training): 198 | x = self.dense(x) 199 | if self.apply_batchnorm: 200 | x = self.batchnorm(x, training=training) 201 | if self.activation: 202 | x = tf.nn.leaky_relu(x) 203 | if self.apply_dropout: 204 | x = self.dropout(x, training=training) 205 | return x 206 | 207 | 208 | class CRU(tf.keras.Model): 209 | 210 | def __init__(self, filters, size=3, stride=2, apply_batchnorm=True): 211 | super(CRU, self).__init__() 212 | self.apply_batchnorm = apply_batchnorm 213 | self.stride = stride 214 | initializer = tf.random_normal_initializer(0., 0.02) 215 | filters = int(filters) 216 | 217 | self.conv1 = layers.Conv2D(filters, 218 | (size, size), 219 | strides=1, 220 | padding='SAME', 221 | kernel_initializer=initializer, 222 | use_bias=False) 223 | self.conv2 = layers.Conv2D(filters, 224 | (size, size), 225 | strides=1, 226 | padding='SAME', 227 | kernel_initializer=initializer, 228 | use_bias=False) 229 | self.conv3 = layers.Conv2D(filters, 230 | (size, size), 231 | strides=1, 232 | padding='SAME', 233 | kernel_initializer=initializer, 234 | use_bias=False) 235 | self.conv4 = layers.Conv2D(filters, 236 | (size, size), 237 | strides=1, 238 | padding='SAME', 239 | kernel_initializer=initializer, 240 | use_bias=False) 241 | 242 | self.batchnorm1 = tf.keras.layers.BatchNormalization() 243 | self.batchnorm2 = tf.keras.layers.BatchNormalization() 244 | self.batchnorm3 = tf.keras.layers.BatchNormalization() 245 | self.batchnorm4 = tf.keras.layers.BatchNormalization() 246 | 247 | def call(self, x, training): 248 | # first residual block 249 | _x = self.conv1(x) 250 | _x = self.batchnorm1(_x, training=training) 251 | _x = tf.nn.leaky_relu(_x) 252 | _x = self.conv2(_x) 253 | _x = self.batchnorm2(_x, training=training) 254 | _x = x + _x 255 | x = tf.nn.leaky_relu(_x) 256 | 257 | # second residual block 258 | _x = self.conv3(x) 259 | _x = self.batchnorm3(_x, training=training) 260 | _x = tf.nn.leaky_relu(_x) 261 | _x = self.conv4(_x) 262 | _x = self.batchnorm4(_x, training=training) 263 | _x = x + _x 264 | x = tf.nn.leaky_relu(_x) 265 | 266 | if self.stride > 1: 267 | x = tf.nn.max_pool(x, 3, 2, padding='SAME') 268 | return x 269 | 270 | 271 | class TRU(tf.keras.Model): 272 | 273 | def __init__(self, filters, idx, alpha=1e-3, beta=1e-4, size=3, apply_batchnorm=True): 274 | super(TRU, self).__init__() 275 | self.apply_batchnorm = apply_batchnorm 276 | # variables 277 | self.conv1 = Downsample(filters, size) 278 | self.conv2 = Downsample(filters, size) 279 | self.conv3 = Downsample(filters, size) 280 | self.flatten = layers.Flatten() 281 | self.project = Linear(idx, alpha, beta, input_dim=2048) 282 | 283 | 284 | def call(self, x, mask, training): 285 | # Downsampling 286 | x_small = self.conv1(x, training=training) 287 | depth = 0 288 | if x_small.shape[1] > 16: 289 | x_small = self.conv2(x_small, training=training) 290 | depth += 1 291 | if x_small.shape[1] > 16: 292 | x_small = self.conv3(x_small, training=training) 293 | depth += 1 294 | x_small_shape = x_small.shape 295 | x_flatten = self.flatten(tf.nn.avg_pool(x_small, ksize=3, strides=2, padding='SAME')) 296 | 297 | # PCA Projection 298 | route_value, route_loss, uniq_loss = self.project(x_flatten, mask, training=training) 299 | 300 | # Generate the splitting mask 301 | mask_l = mask * tf.cast(tf.greater_equal(route_value, tf.constant(0.)), tf.float32) 302 | mask_r = mask * tf.cast(tf.less(route_value, tf.constant(0.)), tf.float32) 303 | 304 | return [mask_l, mask_r], route_value, [route_loss, uniq_loss] 305 | 306 | 307 | class SFL(tf.keras.Model): 308 | 309 | def __init__(self, filters, size=3, apply_batchnorm=True): 310 | super(SFL, self).__init__() 311 | self.apply_batchnorm = apply_batchnorm 312 | # depth map 313 | self.cru1 = CRU(filters, size, stride=1) 314 | self.conv1 = Conv(2, size, activation=False, apply_batchnorm=False) 315 | 316 | # class 317 | self.conv2 = Downsample(filters*1, size) 318 | self.conv3 = Downsample(filters*1, size) 319 | self.conv4 = Downsample(filters*2, size) 320 | self.conv5 = Downsample(filters*4, 4, padding='VALID') 321 | self.flatten = layers.Flatten() 322 | self.fc1 = Dense(256) 323 | self.fc2 = Dense(1, activation=False, apply_batchnorm=False) 324 | 325 | self.dropout = tf.keras.layers.Dropout(0.3) 326 | 327 | def call(self, x, training): 328 | # depth map branch 329 | xd = self.cru1(x) 330 | xd = self.conv1(xd) 331 | dmap = tf.nn.sigmoid(xd) 332 | # class branch 333 | x = self.conv2(x) # 16*16*32 334 | x = self.conv3(x) # 8*8*64 335 | x = self.conv4(x) # 4*4*128 336 | x = self.conv5(x) # 1*1*256 337 | x = self.flatten(x) 338 | x = self.dropout(x, training=training) 339 | x = self.fc1(x) 340 | cls = self.fc2(x) 341 | return dmap, cls 342 | 343 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 2 | # 3 | # Yaojie Liu, Amin Jourabloo, Xiaoming Liu, Michigan State University 4 | # 5 | # All Rights Reserved. 6 | # 7 | # This research is based upon work supported by the Office of the Director of 8 | # National Intelligence (ODNI), Intelligence Advanced Research Projects Activity 9 | # (IARPA), via IARPA R&D Contract No. 2017-17020200004. The views and 10 | # conclusions contained herein are those of the authors and should not be 11 | # interpreted as necessarily representing the official policies or endorsements, 12 | # either expressed or implied, of the ODNI, IARPA, or the U.S. Government. The 13 | # U.S. Government is authorized to reproduce and distribute reprints for 14 | # Governmental purposes not withstanding any copyright annotation thereon. 15 | # ============================================================================== 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import sys 21 | from datetime import datetime 22 | import time 23 | 24 | import tensorflow as tf 25 | from model.dataset import Dataset 26 | from model.config import Config 27 | from model.model import Model 28 | 29 | 30 | def main(argv=None): 31 | # Configurations 32 | config = Config() 33 | config.DATA_DIR = ['/data/'] 34 | config.LOG_DIR = './log/model' 35 | config.MODE = 'training' 36 | config.STEPS_PER_EPOCH_VAL = 180 37 | config.display() 38 | 39 | # Get images and labels. 40 | dataset_train = Dataset(config, 'train') 41 | # Build a Graph 42 | model = Model(config) 43 | 44 | # Train the model 45 | model.compile() 46 | model.train(dataset_train, None) 47 | 48 | main() 49 | --------------------------------------------------------------------------------