├── requirements.txt ├── LICENSE ├── results_analysis.py ├── face_prepare_ytfdb.py ├── ecg_test_triplet_model.py ├── face_test_triplet_model.py ├── ecg_test_secure_model.py ├── ecg_prepare_uoftdb.py ├── face_test_secure_model.py ├── ecg_train_triplet_model.py ├── ecg_train_securetl_model.py ├── ecg_train_securetl_linkability_model.py ├── face_train_triplet_model.py ├── face_train_securetl_model.py ├── face_train_securetl_linkability_model.py ├── losses.py ├── README.md ├── dataset.py ├── trainer.py ├── eval.py ├── models.py └── aux_functions.py /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2020.12.5 2 | chardet==3.0.4 3 | cycler==0.10.0 4 | entropy-estimators==0.0.1 5 | facenet-pytorch==2.3.0 6 | idna==2.10 7 | joblib==0.17.0 8 | kiwisolver==1.3.1 9 | matplotlib==2.2.2 10 | numpy==1.19.4 11 | opencv-python==4.2.0.32 12 | Pillow==8.0.1 13 | pyparsing==2.4.7 14 | python-dateutil==2.8.1 15 | pytz==2020.4 16 | requests==2.25.0 17 | scikit-learn==0.22.2.post1 18 | scipy==1.5.4 19 | six==1.15.0 20 | torch==1.4.0 21 | torchvision==0.5.0 22 | urllib3==1.26.2 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 João Tiago Ribeiro Pinto 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /results_analysis.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: results_analysis.py 5 | - Reads the results pickle file saved by [trait]_test_triplet_model.py or 6 | [trait]_test_secure_model.py and prints several performance and security 7 | metrics, also plotting cancelability, linkability, ROC, and DET curves. 8 | 9 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 10 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 11 | IEEE Transactions on Biometrics, Behavior, and Identity Science 12 | 13 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 14 | ''' 15 | 16 | import numpy as np 17 | import pickle as pk 18 | import aux_functions as af 19 | 20 | 21 | RESULTS_FILE = "model_results.pk" # results file saved by [trait]_test_triplet_model.py or [trait]_test_secure_model.py 22 | SECURE = True # True (model trained with SecureTL) or False (model trained with original Triplet Loss) 23 | 24 | results = pk.load(open(RESULTS_FILE, 'rb')) 25 | 26 | # FMR, FNMR, EER, Cancelability, and Non-Linkability: 27 | if SECURE: 28 | af.plot_perf_vs_canc_curves(results, title='My Method', figsize=[7.0, 4.0], savefile='performance_curve.pdf') 29 | af.plot_dsys(results, title='My Method', figsize=[7.0, 4.0], savefile='dsys_curve.pdf') 30 | else: 31 | af.plot_perf_curves(results[0], title='My Method', figsize=[7.0, 4.0], savefile='performance_curve.pdf') 32 | 33 | # ROC and DET Curves: 34 | af.plot_roc([results[0]['roc']], ['My Method'], 'ROC Curves', figsize=[7.0, 4.0], savefile='roc_curves.pdf') # May plot for more than one method 35 | af.plot_det([results[0]['roc']], ['My Method'], 'DET Curves', figsize=[7.0, 4.0], savefile='det_curves.pdf') # May plot for more than one method 36 | -------------------------------------------------------------------------------- /face_prepare_ytfdb.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: face_prepare_ytfdb.py 5 | - Takes the YouTube Faces aligned images database and prepares everything to 6 | be used in the Secure Triplet Loss training and experiments. 7 | 8 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 9 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 10 | IEEE Transactions on Biometrics, Behavior, and Identity Science 11 | 12 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 13 | ''' 14 | 15 | import os 16 | import numpy as np 17 | import aux_functions as af 18 | 19 | 20 | DIR = 'original_directory' # YTF Aligned-Images directory 21 | SAVE_DIR = 'destination_directory' # Where to save the prepared images 22 | 23 | np.random.seed(42) # Ensuring the same datasets everytime 24 | 25 | train_identities = np.array(sorted(os.listdir(DIR))[0:500]) # Using first 500 identities for training 26 | test_identities = np.array(sorted(os.listdir(DIR))[500:]) # Using the remaining identities for testing 27 | 28 | # Processing training data: 29 | train_triplets = list() 30 | for ii in range(len(train_identities)): 31 | train_triplets.append(af.generate_id_triplets(train_identities[ii], DIR, SAVE_DIR, train_identities, n_triplets=10)) 32 | print('Completed train triplets for identity no.', ii+1) 33 | train_triplets = np.concatenate(train_triplets) 34 | np.save('face_train_data.npy', train_triplets) 35 | 36 | # Processing testing data: 37 | test_triplets = list() 38 | for jj in range(len(test_identities)): 39 | test_triplets.append(af.generate_id_triplets(test_identities[jj], DIR, SAVE_DIR, test_identities, n_triplets=10)) 40 | print('Completed test triplets for identity no.', jj+1) 41 | test_triplets = np.concatenate(test_triplets) 42 | np.save('face_test_data.npy', test_triplets) -------------------------------------------------------------------------------- /ecg_test_triplet_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: ecg_test_triplet_model.py 5 | - Uses the ECG test data to evaluate models trained with the original triplet loss. 6 | 7 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 8 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 9 | IEEE Transactions on Biometrics, Behavior, and Identity Science 10 | 11 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 12 | ''' 13 | 14 | import os 15 | import torch 16 | import numpy as np 17 | import pickle as pk 18 | from models import TripletModel, TripletECGNetwork 19 | from losses import TripletLoss 20 | from dataset import TripletECGDataset 21 | from torch.utils.data import DataLoader 22 | from eval import evaluate_triplet_model 23 | 24 | 25 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 26 | 27 | MODEL = 'model_name' 28 | TEST_DATA = 'ecg_test_data.pickle' 29 | 30 | BATCH_SIZE = 32 31 | 32 | print('Testing model: ' + MODEL) 33 | 34 | # Preparing the dataset 35 | testset = TripletECGDataset(TEST_DATA) 36 | test_loader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4) 37 | 38 | # Creating the model 39 | network = TripletECGNetwork().to(DEVICE) 40 | model = TripletModel(network) 41 | 42 | # Loading saved weights 43 | model.load_state_dict(torch.load(MODEL + '.pth', map_location=DEVICE)) 44 | model = model.to(DEVICE) 45 | 46 | # Evaluating the model on test data 47 | output = evaluate_triplet_model(model, test_loader, BATCH_SIZE, DEVICE, N=10000, debug=True) 48 | 49 | # Saving the results to a pickle 50 | with open(os.path.basename(MODEL) + '_results.pk', 'wb') as hf: 51 | pk.dump(output, hf) 52 | 53 | # Printing the main results 54 | print('EER {:.4f} at threshold {:.4f}'.format(output[0]['eer'][1], output[0]['eer'][0])) 55 | print('Privacy Leakage Rate {:.4f}'.format(output[1]['plr'])) 56 | 57 | -------------------------------------------------------------------------------- /face_test_triplet_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: face_test_triplet_model.py 5 | - Uses the face test data to evaluate models trained with the original triplet loss. 6 | 7 | REQUIRES: 8 | - facenet_pytorch package by Tim Esler 9 | (https://github.com/timesler/facenet-pytorch) 10 | 11 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 12 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 13 | IEEE Transactions on Biometrics, Behavior, and Identity Science 14 | 15 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 16 | ''' 17 | 18 | import os 19 | import torch 20 | import numpy as np 21 | import pickle as pk 22 | from models import TripletModel, TripletFaceNetwork 23 | from losses import TripletLoss 24 | from dataset import TripletFaceDataset 25 | from facenet_pytorch import InceptionResnetV1 26 | from torch.utils.data import DataLoader 27 | from eval import evaluate_triplet_model 28 | 29 | 30 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 31 | 32 | MODEL = 'model_name' 33 | TEST_DATA = 'face_test_data.npy' 34 | 35 | BATCH_SIZE = 32 36 | 37 | print('Testing model: ' + MODEL) 38 | 39 | # Preparing the dataset 40 | testset = TripletFaceDataset(TEST_DATA) 41 | test_loader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4) 42 | 43 | # Creating the model 44 | pretrained = InceptionResnetV1(pretrained='vggface2') 45 | network = TripletFaceNetwork(pretrained).to(DEVICE) 46 | model = TripletModel(network) 47 | 48 | # Loading saved weights 49 | model.load_state_dict(torch.load(MODEL + '.pth', map_location=DEVICE)) 50 | model = model.to(DEVICE) 51 | 52 | # Evaluating the model on test data 53 | output = evaluate_triplet_model(model, test_loader, BATCH_SIZE, DEVICE, N=10000, debug=True) 54 | 55 | # Saving the results to a pickle 56 | with open(os.path.basename(MODEL) + '_results.pk', 'wb') as hf: 57 | pk.dump(output, hf) 58 | 59 | # Printing the main results 60 | print('EER {:.4f} at threshold {:.4f}'.format(output[0]['eer'][1], output[0]['eer'][0])) 61 | print('Privacy Leakage Rate {:.4f}'.format(output[1]['plr'])) 62 | 63 | -------------------------------------------------------------------------------- /ecg_test_secure_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: ecg_test_secure_model.py 5 | - Uses the ECG test data to evaluate models trained with any of the Secure 6 | Triplet Loss formulations, with or without linkability. 7 | 8 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 9 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 10 | IEEE Transactions on Biometrics, Behavior, and Identity Science 11 | 12 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 13 | ''' 14 | 15 | import os 16 | import torch 17 | import numpy as np 18 | import pickle as pk 19 | from models import SecureModel, SecureECGNetwork 20 | from losses import SecureTripletLoss 21 | from dataset import SecureECGDataset 22 | from trainer import train_secure_triplet_model 23 | from torch.utils.data import DataLoader 24 | from eval import evaluate_secure_model 25 | 26 | 27 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 28 | 29 | MODEL = 'model_name' 30 | TEST_DATA = 'ecg_test_data.pickle' 31 | 32 | BATCH_SIZE = 32 33 | 34 | print('Testing model: ' + MODEL) 35 | 36 | # Preparing the dataset 37 | testset = SecureECGDataset(TEST_DATA) 38 | test_loader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4) 39 | 40 | # Creating the model 41 | network = SecureECGNetwork().to(DEVICE) 42 | model = SecureModel(network) 43 | 44 | # Loading saved weights 45 | model.load_state_dict(torch.load(MODEL + '.pth', map_location=DEVICE)) 46 | model = model.to(DEVICE) 47 | 48 | # Evaluating the model on test data 49 | output = evaluate_secure_model(model, test_loader, BATCH_SIZE, DEVICE, N=10000, debug=True) 50 | 51 | # Saving the results to a pickle 52 | with open(os.path.basename(MODEL) + '_results.pk', 'wb') as hf: 53 | pk.dump(output, hf) 54 | 55 | # Printing the main results 56 | print('EER {:.4f} at threshold {:.4f} :: Canc_EER {:.4f} at threshold {:.4f} :: d_sys {:.4f}'.format(output[0]['eer'][1], output[0]['eer'][0], output[1]['eer'][1], output[1]['eer'][0], output[2][0])) 57 | print('Privacy Leakage Rate {:.4f}, Secrecy Leakage {:.4f}'.format(output[3]['plr'], output[3]['sl'])) 58 | 59 | 60 | -------------------------------------------------------------------------------- /ecg_prepare_uoftdb.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: ecg_prepare_uoftdb.py 5 | - Takes the University of Toronto ECG Database (UofTDB) and prepares everything to 6 | be used in the Secure Triplet Loss training and experiments. 7 | 8 | Attention: For best results, the original .mat file that carries all UofTDB data has 9 | been divided into multiple .txt files, with names 'uoftdb_SUB_SES_FILE.txt', where 10 | SUB is the subject ID, SES is the session number, and FILE is the posture number 11 | from the respective session. 12 | 13 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 14 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 15 | IEEE Transactions on Biometrics, Behavior, and Identity Science 16 | 17 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 18 | ''' 19 | 20 | import pickle 21 | import numpy as np 22 | import aux_functions as af 23 | 24 | 25 | N_TRAIN = 100000 # Number of triplets for the train set 26 | N_TEST = 10000 # Number of triplets for the test set 27 | 28 | UOFTDB_PATH = 'UofTDB/txt' # Path to UofTDB path with recordings divided by .txt files 29 | 30 | SAVE_TRAIN = 'ecg_train_data.pickle' 31 | SAVE_TEST = 'ecg_test_data.pickle' 32 | 33 | fs = 200.0 # Sampling frequency of data 34 | 35 | # Dividing subjects for training and for testing 36 | train_data = af.extract_data(UOFTDB_PATH, range(921, 1021), fs=200.0) 37 | test_data = af.extract_data(UOFTDB_PATH, range(921), fs=200.0) 38 | 39 | # Preparing data for a deep neural network 40 | X_train_a, y_train_a = af.prepare_for_dnn(train_data['X_anchors'], train_data['y_anchors']) 41 | X_train_r, y_train_r = af.prepare_for_dnn(train_data['X_remaining'], train_data['y_remaining']) 42 | 43 | X_test_a, y_test_a = af.prepare_for_dnn(test_data['X_anchors'], test_data['y_anchors']) 44 | X_test_r, y_test_r = af.prepare_for_dnn(test_data['X_remaining'], test_data['y_remaining']) 45 | 46 | train_triplets = af.generate_triplets(X_train_a, y_train_a, X_train_r, y_train_r, N=N_TRAIN) 47 | test_triplets = af.generate_triplets(X_test_a, y_test_a, X_test_r, y_test_r, N=N_TEST) 48 | 49 | with open(SAVE_TRAIN, 'wb') as handle: 50 | pickle.dump(train_triplets, handle) 51 | 52 | with open(SAVE_TEST, 'wb') as handle: 53 | pickle.dump(test_triplets, handle) -------------------------------------------------------------------------------- /face_test_secure_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: face_test_secure_model.py 5 | - Uses the face test data to evaluate models trained with any of the Secure 6 | Triplet Loss formulations, with or without linkability. 7 | 8 | REQUIRES: 9 | - facenet_pytorch package by Tim Esler 10 | (https://github.com/timesler/facenet-pytorch) 11 | 12 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 13 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 14 | IEEE Transactions on Biometrics, Behavior, and Identity Science 15 | 16 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 17 | ''' 18 | 19 | import os 20 | import torch 21 | import numpy as np 22 | import pickle as pk 23 | from models import SecureModel, SecureFaceNetwork 24 | from losses import SecureTripletLoss 25 | from dataset import SecureFaceDataset 26 | from trainer import train_secure_triplet_model 27 | from torch.utils.data import DataLoader 28 | from eval import evaluate_secure_model 29 | from facenet_pytorch import InceptionResnetV1 30 | 31 | 32 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 33 | 34 | MODEL = 'model_name' 35 | TEST_SET = 'face_test_data.npy' 36 | 37 | BATCH_SIZE = 32 38 | 39 | print('Testing model: ' + MODEL) 40 | 41 | # Preparing the dataset 42 | testset = SecureFaceDataset(TEST_SET) 43 | test_loader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4) 44 | 45 | # Creating the model 46 | pretrained = InceptionResnetV1(pretrained='vggface2') 47 | network = SecureFaceNetwork(pretrained).to(DEVICE) 48 | model = SecureModel(network) 49 | 50 | # Locading saved weights 51 | model.load_state_dict(torch.load(MODEL + '.pth', map_location=DEVICE)) 52 | model = model.to(DEVICE) 53 | 54 | # Evaluating the model on test data 55 | output = evaluate_secure_model(model, test_loader, BATCH_SIZE, DEVICE, debug=True, N=10000, output_shape=100) 56 | 57 | # Saving the results to a pickle on the 'results' folder 58 | with open(os.path.basename(MODEL) + '_results.pk', 'wb') as hf: 59 | pk.dump(output, hf) 60 | 61 | # Printing the main results 62 | print('EER {:.4f} at threshold {:.4f} :: Canc_EER {:.4f} at threshold {:.4f} :: d_sys {:.4f}'.format(output[0]['eer'][1], output[0]['eer'][0], output[1]['eer'][1], output[1]['eer'][0], output[2][0])) 63 | print('Privacy Leakage Rate {:.4f}, Secrecy Leakage {:.4f}'.format(output[3]['plr'], output[3]['sl'])) 64 | 65 | 66 | -------------------------------------------------------------------------------- /ecg_train_triplet_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: ecg_train_triplet_model.py 5 | - Used to train a model with ECG data, with the original triplet loss. 6 | 7 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 8 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 9 | IEEE Transactions on Biometrics, Behavior, and Identity Science 10 | 11 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 12 | ''' 13 | 14 | import os 15 | import torch 16 | import numpy as np 17 | import pickle as pk 18 | from models import TripletModel, TripletECGNetwork 19 | from losses import TripletLoss 20 | from dataset import TripletECGDataset 21 | from trainer import train_triplet_model 22 | from torch.utils.data import DataLoader 23 | 24 | 25 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 26 | 27 | SAVE_MODEL = 'model_name' 28 | TRAIN_DATA = 'ecg_train_data.pickle' 29 | 30 | LEARN_RATE = 1e-4 # learning rate 31 | REG = 0.001 # L2 regularization hyperparameter 32 | 33 | N_EPOCHS = 250 34 | BATCH_SIZE = 32 35 | VALID_SPLIT = .2 36 | 37 | print('Training model: ' + SAVE_MODEL) 38 | 39 | # Preparing and dividing the dataset 40 | trainset = TripletECGDataset(TRAIN_DATA) 41 | 42 | dataset_size = len(trainset) # number of samples in training + validation sets 43 | indices = list(range(dataset_size)) 44 | split = int(np.floor(VALID_SPLIT * dataset_size)) # number of samples in validation set 45 | np.random.seed(42) 46 | np.random.shuffle(indices) 47 | train_indices, valid_indices = indices[split:], indices[:split] 48 | 49 | train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices) 50 | valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indices) 51 | 52 | train_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=train_sampler) 53 | valid_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=valid_sampler) 54 | 55 | # Creating the network and the model 56 | network = TripletECGNetwork().to(DEVICE) 57 | model = TripletModel(network) 58 | loss = TripletLoss(margin=1.0) 59 | optimizer = torch.optim.Adam(model.parameters(), lr=LEARN_RATE, weight_decay=REG) 60 | 61 | # Training the model 62 | train_hist, valid_hist = train_triplet_model(model, loss, optimizer, train_loader, N_EPOCHS, BATCH_SIZE, DEVICE, patience=10, valid_loader=valid_loader, filename=SAVE_MODEL) 63 | 64 | 65 | -------------------------------------------------------------------------------- /ecg_train_securetl_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: ecg_train_securetl_model.py 5 | - Used to train a model with ECG data, with the original formulation of the Secure Triplet Loss. 6 | 7 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 8 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 9 | IEEE Transactions on Biometrics, Behavior, and Identity Science 10 | 11 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 12 | ''' 13 | 14 | import os 15 | import torch 16 | import numpy as np 17 | import pickle as pk 18 | from models import SecureModel, SecureECGNetwork 19 | from losses import SecureTripletLoss 20 | from dataset import SecureECGDataset 21 | from trainer import train_secure_triplet_model 22 | from torch.utils.data import DataLoader 23 | 24 | 25 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 26 | 27 | SAVE_MODEL = 'model_name' 28 | TRAIN_DATA = 'ecg_train_data.pickle' 29 | 30 | LEARN_RATE = 1e-4 # learning rate 31 | REG = 0 # L2 regularization hyperparameter 32 | 33 | N_EPOCHS = 100 34 | BATCH_SIZE = 32 35 | VALID_SPLIT = .2 36 | 37 | print('Training model: ' + SAVE_MODEL) 38 | 39 | # Preparing and dividing the dataset 40 | trainset = SecureECGDataset(TRAIN_DATA) 41 | 42 | dataset_size = len(trainset) # number of samples in training + validation sets 43 | indices = list(range(dataset_size)) 44 | split = int(np.floor(VALID_SPLIT * dataset_size)) # number of samples in validation set 45 | np.random.seed(42) 46 | np.random.shuffle(indices) 47 | train_indices, valid_indices = indices[split:], indices[:split] 48 | 49 | train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices) 50 | valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indices) 51 | 52 | train_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=train_sampler) 53 | valid_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=valid_sampler) 54 | 55 | # Creating the network and the model 56 | network = SecureECGNetwork().to(DEVICE) 57 | model = SecureModel(network) 58 | loss = SecureTripletLoss(margin=1.0) 59 | optimizer = torch.optim.Adam(model.parameters(), lr=LEARN_RATE, weight_decay=REG) 60 | 61 | # Training the model 62 | train_hist, valid_hist = train_secure_triplet_model(model, loss, optimizer, train_loader, N_EPOCHS, BATCH_SIZE, DEVICE, patience=10, valid_loader=valid_loader, filename=SAVE_MODEL) 63 | 64 | -------------------------------------------------------------------------------- /ecg_train_securetl_linkability_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: ecg_train_securetl_linkability_model.py 5 | - Used to train a model with ECG data, with the proposed Secure Triplet Loss with linkability 6 | based on either Kullback-Leibler Divergence (KLD) or simple statistics (SL). 7 | 8 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 9 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 10 | IEEE Transactions on Biometrics, Behavior, and Identity Science 11 | 12 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 13 | ''' 14 | 15 | import os 16 | import torch 17 | import numpy as np 18 | import pickle as pk 19 | from models import SecureModel, SecureECGNetwork 20 | from losses import SecureTripletLossKLD, SecureTripletLossSL 21 | from dataset import SecureECGDataset 22 | from trainer import train_secure_triplet_model 23 | from torch.utils.data import DataLoader 24 | 25 | 26 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 27 | 28 | SAVE_MODEL = 'model_name' 29 | TRAIN_DATA = 'ecg_train_data.pickle' 30 | 31 | LEARN_RATE = 1e-4 # learning rate 32 | REG = 0 # L2 regularization hyperparameter 33 | 34 | N_EPOCHS = 250 35 | BATCH_SIZE = 32 36 | VALID_SPLIT = .2 37 | 38 | GAMMA = 0.9 # Set the gamma parameter here 39 | 40 | # Choose one of the loss functions below: 41 | loss = SecureTripletLossKLD(margin=1.0, gamma=GAMMA) 42 | #loss = SecureTripletLossSL(margin=1.0, gamma=GAMMA) 43 | 44 | print('Training model: ' + SAVE_MODEL) 45 | 46 | # Preparing and dividing the dataset 47 | trainset = SecureECGDataset(TRAIN_DATA) 48 | 49 | dataset_size = len(trainset) # number of samples in training + validation sets 50 | indices = list(range(dataset_size)) 51 | split = int(np.floor(VALID_SPLIT * dataset_size)) # number of samples in validation set 52 | np.random.seed(42) 53 | np.random.shuffle(indices) 54 | train_indices, valid_indices = indices[split:], indices[:split] 55 | 56 | train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices) 57 | valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indices) 58 | 59 | train_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=train_sampler) 60 | valid_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=valid_sampler) 61 | 62 | # Creating the network and the model 63 | network = SecureECGNetwork().to(DEVICE) 64 | model = SecureModel(network) 65 | optimizer = torch.optim.Adam(model.parameters(), lr=LEARN_RATE, weight_decay=REG) 66 | 67 | # Training the model 68 | train_hist, valid_hist = train_secure_triplet_model(model, loss, optimizer, train_loader, N_EPOCHS, BATCH_SIZE, DEVICE, patience=25, valid_loader=valid_loader, filename=SAVE_MODEL) 69 | 70 | 71 | -------------------------------------------------------------------------------- /face_train_triplet_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: face_train_triplet_model.py 5 | - Used to train a model with face data, with the original triplet loss. 6 | 7 | REQUIRES: 8 | - facenet_pytorch package by Tim Esler 9 | (https://github.com/timesler/facenet-pytorch) 10 | 11 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 12 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 13 | IEEE Transactions on Biometrics, Behavior, and Identity Science 14 | 15 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 16 | ''' 17 | 18 | import os 19 | import torch 20 | import numpy as np 21 | import pickle as pk 22 | from models import TripletModel, TripletFaceNetwork 23 | from losses import TripletLoss 24 | from dataset import TripletFaceDataset 25 | from trainer import train_triplet_model 26 | from torch.utils.data import DataLoader 27 | from facenet_pytorch import InceptionResnetV1 28 | 29 | 30 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 31 | 32 | SAVE_MODEL = 'model_name' 33 | TRAIN_SET = 'face_train_data.npy' 34 | 35 | LEARN_RATE = 1e-4 # learning rate 36 | REG = 0.001 # L2 regularization hyperparameter 37 | 38 | N_EPOCHS = 500 39 | BATCH_SIZE = 32 40 | VALID_SPLIT = .2 41 | PATIENCE = 50 42 | 43 | print('Training model: ' + SAVE_MODEL) 44 | 45 | # Preparing and dividing the dataset 46 | trainset = TripletFaceDataset(TRAIN_SET) 47 | 48 | dataset_size = len(trainset) # number of samples in training + validation sets 49 | indices = list(range(dataset_size)) 50 | split = int(np.floor(VALID_SPLIT * dataset_size)) # number of samples in validation set 51 | np.random.seed(42) 52 | np.random.shuffle(indices) 53 | train_indices, valid_indices = indices[split:], indices[:split] 54 | 55 | train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices) 56 | valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indices) 57 | 58 | train_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=train_sampler) 59 | valid_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=valid_sampler) 60 | 61 | # Creating the network and the model 62 | pretrained = InceptionResnetV1(pretrained='vggface2') 63 | network = TripletFaceNetwork(pretrained, dropout_prob=0.6).to(DEVICE) 64 | network.freeze_parameters() # Freezing all parameters except the fully-connected layer(s) 65 | model = TripletModel(network) 66 | loss = TripletLoss(margin=1.0) 67 | optimizer = torch.optim.Adam(model.parameters(), lr=LEARN_RATE, weight_decay=REG) 68 | 69 | # Training the model 70 | train_hist, valid_hist = train_triplet_model(model, loss, optimizer, train_loader, N_EPOCHS, BATCH_SIZE, DEVICE, patience=PATIENCE, valid_loader=valid_loader, filename=SAVE_MODEL) 71 | 72 | 73 | -------------------------------------------------------------------------------- /face_train_securetl_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: face_train_securetl_model.py 5 | - Used to train a model with face data, with the original formulation of the Secure Triplet Loss. 6 | 7 | REQUIRES: 8 | - facenet_pytorch package by Tim Esler 9 | (https://github.com/timesler/facenet-pytorch) 10 | 11 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 12 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 13 | IEEE Transactions on Biometrics, Behavior, and Identity Science 14 | 15 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 16 | ''' 17 | 18 | import os 19 | import torch 20 | import numpy as np 21 | import pickle as pk 22 | from models import SecureModel, SecureFaceNetwork 23 | from losses import SecureTripletLoss 24 | from dataset import SecureFaceDataset 25 | from trainer import train_secure_triplet_model 26 | from torch.utils.data import DataLoader 27 | from facenet_pytorch import InceptionResnetV1 28 | 29 | 30 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 31 | 32 | SAVE_MODEL = 'model_name' 33 | TRAIN_SET = 'face_train_data.npy' 34 | 35 | LEARN_RATE = 1e-4 # learning rate 36 | REG = 0.001 # L2 regularization hyperparameter 37 | 38 | N_EPOCHS = 500 39 | BATCH_SIZE = 32 40 | VALID_SPLIT = .2 41 | PATIENCE = 50 42 | 43 | print('Training model: ' + SAVE_MODEL) 44 | 45 | # Preparing and dividing the dataset 46 | trainset = SecureFaceDataset(TRAIN_SET) 47 | 48 | dataset_size = len(trainset) # number of samples in training + validation sets 49 | indices = list(range(dataset_size)) 50 | split = int(np.floor(VALID_SPLIT * dataset_size)) # number of samples in validation set 51 | np.random.seed(42) 52 | np.random.shuffle(indices) 53 | train_indices, valid_indices = indices[split:], indices[:split] 54 | 55 | train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices) 56 | valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indices) 57 | 58 | train_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=train_sampler) 59 | valid_loader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=valid_sampler) 60 | 61 | # Creating the network and the model 62 | pretrained = InceptionResnetV1(pretrained='vggface2') 63 | network = SecureFaceNetwork(pretrained, dropout_prob=0.5).to(DEVICE) 64 | network.freeze_parameters() # Freezing all parameters except the fully-connected layer(s) 65 | model = SecureModel(network) 66 | loss = SecureTripletLoss(margin=1.0) 67 | optimizer = torch.optim.Adam(model.parameters(), lr=LEARN_RATE, weight_decay=REG) 68 | 69 | # Training the model 70 | train_hist, valid_hist = train_secure_triplet_model(model, loss, optimizer, train_loader, N_EPOCHS, BATCH_SIZE, DEVICE, patience=PATIENCE, valid_loader=valid_loader, filename=SAVE_MODEL) 71 | 72 | -------------------------------------------------------------------------------- /face_train_securetl_linkability_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: face_train_securetl_linkability_model.py 5 | - Used to train a model with face data, with the proposed Secure Triplet Loss with linkability 6 | based on either Kullback-Leibler Divergence (KLD) or simple statistics (SL). 7 | 8 | REQUIRES: 9 | - facenet_pytorch package by Tim Esler 10 | (https://github.com/timesler/facenet-pytorch) 11 | 12 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 13 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 14 | IEEE Transactions on Biometrics, Behavior, and Identity Science 15 | 16 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 17 | ''' 18 | 19 | import os 20 | import torch 21 | import numpy as np 22 | import pickle as pk 23 | from models import SecureModel, SecureFaceNetwork 24 | from losses import SecureTripletLossKLD, SecureTripletLossSL 25 | from dataset import SecureFaceDataset 26 | from trainer import train_secure_triplet_model 27 | from torch.utils.data import DataLoader 28 | from facenet_pytorch import InceptionResnetV1 29 | 30 | 31 | DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu' 32 | 33 | SAVE_MODEL = 'model_name' 34 | TRAIN_SET = 'face_train_data.npy' 35 | 36 | LEARN_RATE = 1e-4 # learning rate 37 | REG = 0.001 # L2 regularization hyperparameter 38 | 39 | N_EPOCHS = 250 40 | BATCH_SIZE = 32 41 | VALID_SPLIT = .2 42 | PATIENCE = 25 43 | 44 | GAMMA = 0.9 # Set the gamma parameter here 45 | 46 | # Choose one of the loss functions below: 47 | loss = SecureTripletLossKLD(margin=1.0, gamma=GAMMA) 48 | #loss = SecureTripletLossSL(margin=1.0, gamma=GAMMA) 49 | 50 | print('Training model: ' + SAVE_MODEL) 51 | 52 | # Preparing and dividing the dataset 53 | trainset = SecureFaceDataset(TRAIN_SET) 54 | 55 | dataset_size = len(trainset) # number of samples in training + validation sets 56 | indices = list(range(dataset_size)) 57 | split = int(np.floor(VALID_SPLIT * dataset_size)) # number of samples in validation set 58 | np.random.seed(42) 59 | np.random.shuffle(indices) 60 | train_indices, valid_indices = indices[split:], indices[:split] 61 | 62 | train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices) 63 | valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indices) 64 | 65 | train_loader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=train_sampler) 66 | valid_loader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, sampler=valid_sampler) 67 | 68 | # Creating the network and the model 69 | pretrained = InceptionResnetV1(pretrained='vggface2') 70 | network = SecureFaceNetwork(pretrained, dropout_prob=0.5).to(DEVICE) 71 | network.freeze_parameters() # Freezing all parameters except the fully-connected layer(s) 72 | model = SecureModel(network) 73 | optimizer = torch.optim.Adam(model.parameters(), lr=LEARN_RATE, weight_decay=REG) 74 | 75 | # Training the model 76 | train_hist, valid_hist = train_secure_triplet_model(model, loss, optimizer, train_loader, N_EPOCHS, BATCH_SIZE, DEVICE, patience=PATIENCE, valid_loader=valid_loader, filename=SAVE_MODEL) 77 | -------------------------------------------------------------------------------- /losses.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: losses.py 5 | - Defines the loss function classes to be used during training: the original 6 | triplet loss [1], the original formulation of the Secure Triplet Loss [2], 7 | and the proposed SecureTL w/KLD and SecureTL w/SL. 8 | 9 | References: 10 | [1] G. Chechik et al., "Large scale online learning of image similarity 11 | through ranking", JMLR 11, pp. 1109–1135, 2010. 12 | [2] JR Pinto et al., "Secure Triplet Loss for End-to-End Deep Biometrics", 13 | in IWBF 2020. 14 | 15 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 16 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 17 | IEEE Transactions on Biometrics, Behavior, and Identity Science 18 | 19 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 20 | ''' 21 | 22 | import torch 23 | import torch.nn as nn 24 | import torch.nn.functional as F 25 | from torch.distributions.normal import Normal 26 | from torch.distributions.kl import kl_divergence 27 | 28 | 29 | class TripletLoss(nn.Module): 30 | # Original Triplet Loss 31 | 32 | def __init__(self, margin): 33 | super(TripletLoss, self).__init__() 34 | self.margin = margin 35 | 36 | def forward(self, anchor, positive, negative, size_average=True): 37 | # Computing loss value based on a batch of triplet embeddings. 38 | distance_positive = torch.norm(anchor - positive, dim=1) 39 | distance_negative = torch.norm(anchor - negative, dim=1) 40 | losses = F.relu(distance_positive - distance_negative + self.margin) 41 | return losses.mean() if size_average else losses.sum() 42 | 43 | 44 | class SecureTripletLoss(nn.Module): 45 | # SecureTL (original formulation w/out linkability) 46 | 47 | def __init__(self, margin): 48 | super(SecureTripletLoss, self).__init__() 49 | self.margin = margin 50 | 51 | def forward(self, anchor, positive1, positive2, negative1, negative2, size_average=True): 52 | # Computing loss value based on a batch of secure triplet embeddings. 53 | dSP = torch.norm(anchor - positive1, dim=1) 54 | dSN = torch.norm(anchor - negative1, dim=1) 55 | dDP = torch.norm(anchor - positive2, dim=1) 56 | dDN = torch.norm(anchor - negative2, dim=1) 57 | dneg = torch.min(dSN, dDP) 58 | dneg = torch.min(dneg, dDN) 59 | losses = F.relu(dSP - dneg + self.margin) 60 | return losses.mean() if size_average else losses.sum() 61 | 62 | 63 | class SecureTripletLossKLD(nn.Module): 64 | # SecureTL w/KLD (with linkability through Kullback-Leibler Divergence) 65 | 66 | def __init__(self, margin, gamma): 67 | super(SecureTripletLossKLD, self).__init__() 68 | self.margin = margin 69 | self.gamma = gamma 70 | 71 | def forward(self, anchor, positive1, positive2, negative1, negative2): 72 | # Computing loss value based on a batch of secure triplet embeddings. 73 | 74 | # Original Secure Triplet Loss component: 75 | dSP = torch.norm(anchor - positive1, dim=1) 76 | dSN = torch.norm(anchor - negative1, dim=1) 77 | dDP = torch.norm(anchor - positive2, dim=1) 78 | dDN = torch.norm(anchor - negative2, dim=1) 79 | dneg = torch.min(dSN, dDP) 80 | dneg = torch.min(dneg, dDN) 81 | losses = F.relu(dSP - dneg + self.margin) 82 | loss = losses.mean() 83 | 84 | # Linkability component: 85 | dist_DP = Normal(torch.mean(dDP).view(1), torch.var(dDP).view(1)) 86 | dist_DN = Normal(torch.mean(dDN).view(1), torch.var(dDN).view(1)) 87 | linkability = kl_divergence(dist_DP, dist_DN) 88 | 89 | return self.gamma * loss + (1 - self.gamma) * linkability 90 | 91 | 92 | class SecureTripletLossSL(nn.Module): 93 | # SecureTL w/SL (with linkability through Simple Statistics) 94 | 95 | def __init__(self, margin, gamma): 96 | super(SecureTripletLossSL, self).__init__() 97 | self.margin = margin 98 | self.gamma = gamma 99 | 100 | def forward(self, anchor, positive1, positive2, negative1, negative2): 101 | # Computing loss value based on a batch of secure triplet embeddings. 102 | 103 | # Original Secure Triplet Loss part 104 | dSP = torch.norm(anchor - positive1, dim=1) 105 | dSN = torch.norm(anchor - negative1, dim=1) 106 | dDP = torch.norm(anchor - positive2, dim=1) 107 | dDN = torch.norm(anchor - negative2, dim=1) 108 | dneg = torch.min(dSN, dDP) 109 | dneg = torch.min(dneg, dDN) 110 | losses = F.relu(dSP - dneg + self.margin) 111 | loss = losses.mean() 112 | 113 | # Linkability part 114 | mean_diff = torch.norm(torch.mean(dDP) - torch.mean(dDN)) 115 | stdv_diff = torch.norm(torch.std(dDP) - torch.std(dDN)) 116 | linkability = mean_diff + stdv_diff 117 | 118 | return self.gamma * loss + (1 - self.gamma) * linkability 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SecureTL Project Repository 2 | 3 | **Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics** 4 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 5 | *INESC TEC and Universidade do Porto, Portugal* 6 | joao.t.pinto@inesctec.pt 7 | 8 | ## Summary 9 | This repository contains the code used for our papers on the Secure Triplet Loss approach for biometric template security in end-to-end deep models. In our first paper (1), we proposed the Secure Triplet Loss, based on the original Triplet Loss (2), as a way to achieve template cancelability in deep end-to-end models without separate encryption processes. Although we succeeded in our goal, the method presented the drawback of high template linkability. Hence, in our second paper (3), we reformulated the Secure Triplet Loss to address this problem, by adding a linkability-measuring component based on Kullback-Leibler Divergence or template distance statistics. We evaluated the proposed method on biometric verification with ECG and face, and it was successful in training secure biometric models from scratch and adapting a pretrained model to make it secure. 10 | 11 | If you want to know more about this, or if you use our code, please read and cite these papers: 12 | 13 | **J. R. Pinto, M. V. Correia, and J. S. Cardoso, "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics", in *IEEE Transactions on Biometrics, Behavior, and Identity Science,* 3(2): pp. 180-189, 2021.** 14 | [[link]](https://ieeexplore.ieee.org/document/9302588) [[pdf]](https://jtrpinto.github.io/files/pdf/jpinto2021tbiom.pdf) [[bib]](https://jtrpinto.github.io/files/bibtex/jpinto2021tbiom.bib) 15 | 16 | **J. R. Pinto, J. S. Cardoso, and M. V. Correia, "Secure Triplet Loss for End-to-End Deep Biometrics", in *8th International Workshop on Biometrics and Forensics (IWBF 2020),* 2020.** 17 | [[link]](https://ieeexplore.ieee.org/document/9107958) [[pdf]](https://jtrpinto.github.io/files/pdf/jpinto2020iwbf.pdf) [[bib]](https://jtrpinto.github.io/files/bibtex/jpinto2020iwbf1.bib) 18 | 19 | ## Description 20 | This repository includes the python scripts used to train and test models with both the original triplet loss and the proposed Secure Triplet Loss. It includes scripts prepared for both face and ECG biometric verification, which used, respectively, the YouTube Faces database (YTF) (4) and the University of Toronto ECG Database (UofTDB) (5). To ensure no data from YTF or UofTDB are redistributed here, this repository does not include trained models, scores, predictions, or any other data. Nevertheless, the scripts are prepared so that anyone with access to the databases should be able to replicate our results exactly: 21 | 1. Use *face_prepare_ytfdb.py* or *ecg_prepare_uoftdb.py* to prepare the face or ECG databases; 22 | 2. Use *[trait]_train_triplet_model.py* to train a model with the original triplet loss; 23 | 3. Use *[trait]_train_securetl_model.py* to train a model with the first formulation of the SecureTL; 24 | 4. Use *[trait]_train_securetl_linkability_model.py* to train a model with the second formulation of the Secure TL (with Linkability); 25 | 5. Use *[trait]_test_triplet_model.py* to test a model trained with the original triplet loss; 26 | 6. Use *[trait]_test_secure_model.py* to test a model trained with SecureTL; 27 | 7. Use *results_analysis.py* to print and plot various performance and security metrics of your model. 28 | 29 | *[trait]* can be either *face* or *ecg*. Do not forget to set the needed variables at the beginning of each script. 30 | 31 | ## Setup 32 | To run our code, download or clone this repository and use *requirements.txt* to set up a pip virtual environment with the needed dependencies. You will also need the data from the YTF and UofTDB databases. The YTF aligned-images dataset can be requested on the [YouTube Faces DB website](https://www.cs.tau.ac.il/~wolf/ytfaces/). To get the UofTDB data, you should contact the [BioSec.Lab at the University of Toronto](https://www.comm.utoronto.ca/~biometrics/). 33 | 34 | ## Acknowledgements 35 | This work was financed by the ERDF - European Regional Development Fund through the Operational Programme for Competitiveness and Internationalization - COMPETE 2020 Programme and by National Funds through the Portuguese funding agency, FCT - Fundação para a Ciência e a Tecnologia within project "POCI-01-0145-FEDER-030707", and within the PhD grant "SFRH/BD/137720/2018". The authors wish to acknowledge the creators of the UofTDB (University of Toronto, Canada), and the YouTube Faces (Tel Aviv University, Israel) databases, essential for this work. 36 | 37 | ## References 38 | (1) Pinto, J.R.; Cardoso, J.S.; Correia, M.V.: Secure Triplet Loss for End-to-End Deep Biometrics. 8th International Workshop on Biometrics and Forensics (IWBF 2020), 2020. 39 | (2) Chechik, G.; Sharma, V.; Shalit, U.; Bengio, S.: Large scale onlinelearning of image similarity through ranking. Journal of Machine Learning Research, 11:1109-1135, 2010. 40 | (3) Pinto, J.R.; Correia, M.V.; Cardoso, J.S.: J. R. Pinto, M. V. Correia, and J. S. Cardoso, "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics". IEEE Transactions on Biometrics, Behavior, and Identity Science, 3(2):180-189, 2021. 41 | (4) Wolf, L.; Hassner, T.; Maoz, I.: Face Recognition in Unconstrained Videos with Matched Background Similarity. IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2011. 42 | (5) Wahabi, S.; Pouryayevali, S.; Hari, S.; Hatzinakos, D.: On Evaluating ECG Biometric Systems: Session-Dependence and Body Posture. IEEE Transactions on Information Forensics and Security, 9(11):2002–2013, Nov 2014. 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: dataset.py 5 | - Uses torch's Dataset class to import the Triplet or Secure datasets of ECG 6 | or Face to be used to train or test the models. 7 | 8 | Info on the dataset files needed: 9 | 10 | TripletFaceDataset and SecureFaceDataset require a numpy file, 11 | e.g. "face_data.npy", which stores a Nx3 or Nx5 numpy array, 12 | where the three first columns are the paths to the prepared images 13 | serving as anchor, positive, and negative samples, respectively. 14 | The fourth and fifth columns are the cancelability keys 1 and 2, 15 | respectively, not required for TripletFaceDataset. 16 | See "face_prepare_ytfdb.py" for more details. 17 | 18 | TripletECGDataset and SecureECGDataset require a pickle file, 19 | e.g. "ecg_data.pickle", which stores a tuple of six numpy arrays. 20 | The first three numpy arrays correspond to the anchors, the positives, 21 | and the negatives, respectively. The fourth array includes the IDs of 22 | the anchors on each triplet. The fifth and sixth arrays correspond to 23 | the cancelable keys of each triplet, k1 and k2, respectively. 24 | See "ecg_prepare_uoftdb.py" for more details. 25 | 26 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 27 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 28 | IEEE Transactions on Biometrics, Behavior, and Identity Science 29 | 30 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 31 | ''' 32 | 33 | import torch 34 | import numpy as np 35 | import pickle as pk 36 | from PIL import Image 37 | from torchvision import transforms 38 | from torch.utils.data import Dataset 39 | 40 | 41 | class SecureECGDataset(Dataset): 42 | # Used to load ECG triplet data, including cancellability keys 43 | # for the SecureTL method. 44 | 45 | def __init__(self, data_path): 46 | with open(data_path, 'rb') as handle: 47 | data = pk.load(handle) 48 | self.xA = data[0] 49 | self.xP = data[1] 50 | self.xN = data[2] 51 | self.k1 = data[4] 52 | self.k2 = data[5] 53 | self.sample_shape = (1, 1000) 54 | 55 | def __getitem__(self, index): 56 | # Load anchor, positive, negative, and the two keys for a given index 57 | xA = self.xA[index].reshape(self.sample_shape) 58 | xP = self.xP[index].reshape(self.sample_shape) 59 | xN = self.xN[index].reshape(self.sample_shape) 60 | k1 = self.k1[index] 61 | k2 = self.k2[index] 62 | return (xA, xP, xN, k1, k2) 63 | 64 | def __len__(self): 65 | return len(self.xA) 66 | 67 | 68 | class TripletECGDataset(Dataset): 69 | # Used to load ECG triplet data for the original triplet loss. 70 | 71 | def __init__(self, data_path): 72 | with open(data_path, 'rb') as handle: 73 | data = pk.load(handle) 74 | self.xA = data[0] 75 | self.xP = data[1] 76 | self.xN = data[2] 77 | self.sample_shape = (1, 1000) 78 | 79 | def __getitem__(self, index): 80 | # Load anchor, positive, and negative for a given index 81 | xA = self.xA[index].reshape(self.sample_shape) 82 | xP = self.xP[index].reshape(self.sample_shape) 83 | xN = self.xN[index].reshape(self.sample_shape) 84 | return (xA, xP, xN) 85 | 86 | def __len__(self): 87 | # Number of triplets on the dataset 88 | return len(self.xA) 89 | 90 | 91 | class SecureFaceDataset(Dataset): 92 | # Used to load face triplet data, including cancellability keys 93 | # for the SecureTL method. 94 | 95 | def __init__(self, dataset): 96 | dset = np.load(dataset, allow_pickle=True) 97 | self.anchors = dset[:,0] 98 | self.positives = dset[:,1] 99 | self.negatives = dset[:,2] 100 | self.k1 = dset[:,3] 101 | self.k2 = dset[:,4] 102 | 103 | def __normalise__(self, img): 104 | # Normalise image pixel intensities 105 | return (img - 127.5) / 128.0 106 | 107 | def __loadimage__(self, imgpath): 108 | # Load image, apply d.a., transform to tensor, and normalise 109 | img = Image.open(imgpath) 110 | tf = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5), 111 | np.float32, 112 | transforms.ToTensor(), 113 | self.__normalise__]) # Adapted from facenet_pytorch 114 | img = tf(img) 115 | return img 116 | 117 | def __getitem__(self, index): 118 | # Load anchor, positive, negative, and the two keys for a given index 119 | xA = self.__loadimage__(self.anchors[index]) 120 | xP = self.__loadimage__(self.positives[index]) 121 | xN = self.__loadimage__(self.negatives[index]) 122 | k1 = self.k1[index] 123 | k2 = self.k2[index] 124 | return (xA, xP, xN, k1, k2) 125 | 126 | def __len__(self): 127 | # Number of triplets on the dataset 128 | return len(self.anchors) 129 | 130 | 131 | class TripletFaceDataset(Dataset): 132 | # Used to load face triplet data for the original triplet loss. 133 | 134 | def __init__(self, dataset): 135 | dset = np.load(dataset, allow_pickle=True) 136 | self.anchors = dset[:,0] 137 | self.positives = dset[:,1] 138 | self.negatives = dset[:,2] 139 | 140 | def __normalise__(self, img): 141 | return (img - 127.5) / 128.0 142 | 143 | def __loadimage__(self, imgpath): 144 | img = Image.open(imgpath) 145 | tf = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5), 146 | np.float32, 147 | transforms.ToTensor(), 148 | self.__normalise__]) # Adapted from facenet_pytorch 149 | img = tf(img) 150 | return img 151 | 152 | def __getitem__(self, index): 153 | # Load anchor, positive, and negative for a given index 154 | xA = self.__loadimage__(self.anchors[index]) 155 | xP = self.__loadimage__(self.positives[index]) 156 | xN = self.__loadimage__(self.negatives[index]) 157 | return (xA, xP, xN) 158 | 159 | def __len__(self): 160 | # Number of triplets on the dataset 161 | return len(self.anchors) 162 | -------------------------------------------------------------------------------- /trainer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: trainer.py 5 | - Defines the training routines for the original triplet loss and for the proposed 6 | Secure Triplet Loss. 7 | 8 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 9 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 10 | IEEE Transactions on Biometrics, Behavior, and Identity Science 11 | 12 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 13 | ''' 14 | 15 | import torch 16 | import numpy as np 17 | import sys 18 | import pickle 19 | from eval import compute_distances_triplet, compute_distances_secure, triplet_metrics, secure_metrics 20 | 21 | 22 | def train_triplet_model(model, loss_fn, optimizer, train_loader, n_epochs, batch_size, device, patience=1, valid_loader=None, filename=None): 23 | train_hist = [] 24 | train_eer = [] 25 | valid_hist = [] 26 | valid_eer = [] 27 | 28 | # For early stopping: 29 | plateau = 0 30 | best_valid_loss = None 31 | 32 | for epoch in range(n_epochs): 33 | print('Epoch {}'.format(epoch + 1)) 34 | 35 | # training loop 36 | model.train() # set model to training mode (affects dropout and batch norm.) 37 | for i, (xA, xP, xN) in enumerate(train_loader): 38 | # copy the mini-batch to GPU 39 | xA = xA.to(device, dtype=torch.float) 40 | xP = xP.to(device, dtype=torch.float) 41 | xN = xN.to(device, dtype=torch.float) 42 | 43 | yA, yP, yN = model(xA, xP, xN) # forward pass 44 | loss = loss_fn(yA, yP, yN) # compute the loss 45 | optimizer.zero_grad() # set all gradients to zero (otherwise they are accumulated) 46 | loss.backward() # backward pass (i.e. compute gradients) 47 | optimizer.step() # update the parameters 48 | 49 | # display the mini-batch loss 50 | sys.stdout.write("\r" + '........mini-batch no. {} loss: {:.4f}'.format(i+1, loss.item())) 51 | sys.stdout.flush() 52 | 53 | if torch.isnan(loss): 54 | print('NaN loss. Terminating train.') 55 | return [], [] 56 | 57 | # compute the training and validation losses to monitor the training progress (optional) 58 | print() 59 | with torch.no_grad(): # now we are doing inference only, so we do not need gradients 60 | model.eval() # set model to inference mode (affects dropout and batch norm.) 61 | 62 | train_loss = 0. 63 | distances = np.zeros((2, 0)) 64 | for i, (xA, xP, xN) in enumerate(train_loader): 65 | # copy the mini-batch to GPU 66 | xA = xA.to(device, dtype=torch.float) 67 | xP = xP.to(device, dtype=torch.float) 68 | xN = xN.to(device, dtype=torch.float) 69 | 70 | yA, yP, yN = model(xA, xP, xN) # forward pass 71 | train_loss += loss_fn(yA, yP, yN) # accumulate the loss of the mini-batch 72 | distances = np.concatenate((distances, compute_distances_triplet(yA, yP, yN)), axis=1) 73 | train_loss /= i + 1 74 | train_hist.append(train_loss.item()) 75 | _, t_eer = triplet_metrics(distances) 76 | train_eer.append(t_eer) 77 | print('....train loss: {:.4f} :: EER {:.4f}'.format(train_loss.item(), t_eer)) 78 | 79 | if valid_loader is None: 80 | print() 81 | continue 82 | 83 | valid_loss = 0. 84 | distances = np.zeros((2, 0)) 85 | for i, (xA, xP, xN) in enumerate(valid_loader): 86 | # copy the mini-batch to GPU 87 | xA = xA.to(device, dtype=torch.float) 88 | xP = xP.to(device, dtype=torch.float) 89 | xN = xN.to(device, dtype=torch.float) 90 | 91 | yA, yP, yN = model(xA, xP, xN) # forward pass 92 | valid_loss += loss_fn(yA, yP, yN) # accumulate the loss of the mini-batch 93 | distances = np.concatenate((distances, compute_distances_triplet(yA, yP, yN)), axis=1) 94 | valid_loss /= i + 1 95 | valid_hist.append(valid_loss.item()) 96 | _, v_eer = triplet_metrics(distances) 97 | valid_eer.append(v_eer) 98 | print('....valid loss: {:.4f} :: EER {:.4f}'.format(valid_loss.item(), v_eer)) 99 | 100 | if best_valid_loss is None: 101 | best_valid_loss = v_eer 102 | torch.save(model.state_dict(), filename + '.pth') 103 | with open(filename + '_trainhist.pk', 'wb') as hf: 104 | pickle.dump({'loss': train_hist, 'eer': train_eer}, hf) 105 | with open(filename + '_validhist.pk', 'wb') as hf: 106 | pickle.dump({'loss': valid_hist, 'eer': valid_eer}, hf) 107 | print('....Saving...') 108 | elif v_eer < best_valid_loss: 109 | best_valid_loss = v_eer 110 | torch.save(model.state_dict(), filename + '.pth') 111 | with open(filename + '_trainhist.pk', 'wb') as hf: 112 | pickle.dump({'loss': train_hist, 'eer': train_eer}, hf) 113 | with open(filename + '_validhist.pk', 'wb') as hf: 114 | pickle.dump({'loss': valid_hist, 'eer': valid_eer}, hf) 115 | plateau = 0 116 | print('....Saving...') 117 | else: 118 | plateau += 1 119 | if plateau >= patience: 120 | print('....Early stopping the train.') 121 | return train_hist, valid_hist 122 | 123 | return train_hist, valid_hist 124 | 125 | 126 | def train_secure_triplet_model(model, loss_fn, optimizer, train_loader, n_epochs, batch_size, device, patience=1, valid_loader=None, filename=None): 127 | train_hist = [] 128 | train_eer = [] 129 | train_canc = [] 130 | train_dsys = [] 131 | valid_hist = [] 132 | valid_eer = [] 133 | valid_canc = [] 134 | valid_dsys = [] 135 | 136 | # For early stopping: 137 | plateau = 0 138 | best_valid_loss = None 139 | 140 | for epoch in range(n_epochs): 141 | print('Epoch {}'.format(epoch + 1)) 142 | 143 | # training loop 144 | model.train() # set model to training mode (affects dropout and batch norm.) 145 | for i, (xA, xP, xN, k1, k2) in enumerate(train_loader): 146 | # copy the mini-batch to GPU 147 | xA = xA.to(device, dtype=torch.float) 148 | xP = xP.to(device, dtype=torch.float) 149 | xN = xN.to(device, dtype=torch.float) 150 | k1 = k1.to(device, dtype=torch.float) 151 | k2 = k2.to(device, dtype=torch.float) 152 | 153 | yA, yP1, yP2, yN1, yN2 = model(xA, xP, xN, k1, k2) # forward pass 154 | loss = loss_fn(yA, yP1, yP2, yN1, yN2) # compute the loss 155 | optimizer.zero_grad() # set all gradients to zero (otherwise they are accumulated) 156 | loss.backward() # backward pass (i.e. compute gradients) 157 | optimizer.step() # update the parameters 158 | 159 | # display the mini-batch loss 160 | sys.stdout.write("\r" + '........mini-batch no. {} loss: {:.4f}'.format(i + 1, loss.item())) 161 | sys.stdout.flush() 162 | 163 | if torch.isnan(loss): 164 | print('NaN loss. Terminating train.') 165 | return [], [] 166 | 167 | # compute the training and validation losses to monitor the training progress (optional) 168 | print() 169 | with torch.no_grad(): # now we are doing inference only, so we do not need gradients 170 | model.eval() # set model to inference mode (affects dropout and batch norm.) 171 | 172 | train_loss = 0. 173 | distances = np.zeros((4, 0)) 174 | for i, (xA, xP, xN, k1, k2) in enumerate(train_loader): 175 | # copy the mini-batch to GPU 176 | xA = xA.to(device, dtype=torch.float) 177 | xP = xP.to(device, dtype=torch.float) 178 | xN = xN.to(device, dtype=torch.float) 179 | k1 = k1.to(device, dtype=torch.float) 180 | k2 = k2.to(device, dtype=torch.float) 181 | 182 | yA, yP1, yP2, yN1, yN2 = model(xA, xP, xN, k1, k2) # forward pass 183 | train_loss += loss_fn(yA, yP1, yP2, yN1, yN2) # accumulate the loss of the mini-batch 184 | distances = np.concatenate((distances, compute_distances_secure(yA, yP1, yP2, yN1, yN2)), axis=1) 185 | train_loss /= i + 1 186 | train_hist.append(train_loss.item()) 187 | _, t_eer, _, t_canc, t_dsys = secure_metrics(distances) 188 | train_eer.append(t_eer) 189 | train_canc.append(t_canc) 190 | train_dsys.append(t_dsys) 191 | print('....train loss: {:.4f} :: EER {:.4f} :: Canc_EER {:.4f} :: D_sys {:.4f}'.format(train_loss.item(), t_eer, t_canc, t_dsys)) 192 | 193 | if valid_loader is None: 194 | print() 195 | continue 196 | 197 | valid_loss = 0. 198 | distances = np.zeros((4, 0)) 199 | for i, (xA, xP, xN, k1, k2) in enumerate(valid_loader): 200 | # copy the mini-batch to GPU 201 | xA = xA.to(device, dtype=torch.float) 202 | xP = xP.to(device, dtype=torch.float) 203 | xN = xN.to(device, dtype=torch.float) 204 | k1 = k1.to(device, dtype=torch.float) 205 | k2 = k2.to(device, dtype=torch.float) 206 | 207 | yA, yP1, yP2, yN1, yN2 = model(xA, xP, xN, k1, k2) # forward pass 208 | valid_loss += loss_fn(yA, yP1, yP2, yN1, yN2) # accumulate the loss of the mini-batch 209 | distances = np.concatenate((distances, compute_distances_secure(yA, yP1, yP2, yN1, yN2)), axis=1) 210 | valid_loss /= i + 1 211 | valid_hist.append(valid_loss.item()) 212 | _, v_eer, _, v_canc, v_dsys = secure_metrics(distances) 213 | valid_eer.append(v_eer) 214 | valid_canc.append(v_canc) 215 | valid_dsys.append(v_dsys) 216 | print('....valid loss: {:.4f} :: EER {:.4f} :: Canc_EER {:.4f} :: D_sys {:.4f}'.format(valid_loss.item(), v_eer, v_canc, v_dsys)) 217 | 218 | # Saving best model and early stopping: 219 | if best_valid_loss is None: 220 | best_valid_loss = v_eer 221 | torch.save(model.state_dict(), filename + '.pth') 222 | with open(filename + '_trainhist.pk', 'wb') as hf: 223 | pickle.dump({'loss': train_hist, 'eer': train_eer, 'canc': train_canc, 'dsys': train_dsys}, hf) 224 | with open(filename + '_validhist.pk', 'wb') as hf: 225 | pickle.dump({'loss': valid_hist, 'eer': valid_eer, 'canc': valid_canc, 'dsys': valid_dsys}, hf) 226 | print('....Saving...') 227 | elif v_eer < best_valid_loss: 228 | best_valid_loss = v_eer 229 | torch.save(model.state_dict(), filename + '.pth') 230 | with open(filename + '_trainhist.pk', 'wb') as hf: 231 | pickle.dump({'loss': train_hist, 'eer': train_eer, 'canc': train_canc, 'dsys': train_dsys}, hf) 232 | with open(filename + '_validhist.pk', 'wb') as hf: 233 | pickle.dump({'loss': valid_hist, 'eer': valid_eer, 'canc': valid_canc, 'dsys': valid_dsys}, hf) 234 | plateau = 0 235 | print('....Saving...') 236 | else: 237 | plateau += 1 238 | if plateau >= patience: 239 | print('....Early stopping the train.') 240 | return train_hist, valid_hist 241 | 242 | return train_hist, valid_hist 243 | -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: eval.py 5 | - Defines various functions to evaluate verification performance, cancellability, 6 | non-linkability, and non-invertibility of the models. 7 | 8 | REQUIRES: 9 | - entropy_estimators package by Paul Brodersen 10 | (https://github.com/paulbrodersen/entropy_estimators) 11 | 12 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 13 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 14 | IEEE Transactions on Biometrics, Behavior, and Identity Science 15 | 16 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 17 | ''' 18 | 19 | import torch 20 | import numpy as np 21 | import pickle as pk 22 | import scipy.stats as stats 23 | from entropy_estimators import continuous 24 | 25 | 26 | def get_triplet_outputs(model, data_loader, batch_size, device, output_shape): 27 | # Gets triplet embeddings from a dataset (given by the data_loader), using 28 | # a TripletModel, to be used for evaluation. 29 | # Note: input_A is returned as it is needed for non-invertibility evaluation. 30 | input_A = list() 31 | output_A = np.zeros((0, output_shape)) 32 | output_P = np.zeros((0, output_shape)) 33 | output_N = np.zeros((0, output_shape)) 34 | with torch.no_grad(): # we do not need gradients 35 | model.eval() # set the model to inference mode 36 | for i, (xA, xP, xN) in enumerate(data_loader): 37 | # copy the mini-batch to GPU 38 | xA = xA.to(device, dtype=torch.float) 39 | xP = xP.to(device, dtype=torch.float) 40 | xN = xN.to(device, dtype=torch.float) 41 | yA, yP, yN = model(xA, xP, xN) 42 | output_A = np.concatenate((output_A, yA.cpu().numpy())) 43 | output_P = np.concatenate((output_P, yP.cpu().numpy())) 44 | output_N = np.concatenate((output_N, yN.cpu().numpy())) 45 | input_A.append(xA.cpu().numpy()) 46 | input_A = np.concatenate(input_A) 47 | return output_A, output_P, output_N, input_A 48 | 49 | 50 | def get_secure_outputs(model, data_loader, batch_size, device, output_shape): 51 | # Gets secure embeddings from a dataset (given by the data_loader), using 52 | # a SecureModel, to be used for evaluation. 53 | # Note: input_A and input_k1 are returned as they are needed for 54 | # non-invertibility evaluation. 55 | input_A = list() 56 | input_k1 = list() 57 | output_A = np.zeros((0, output_shape)) 58 | output_P1 = np.zeros((0, output_shape)) 59 | output_P2 = np.zeros((0, output_shape)) 60 | output_N1 = np.zeros((0, output_shape)) 61 | output_N2 = np.zeros((0, output_shape)) 62 | with torch.no_grad(): # we do not need gradients 63 | model.eval() # set the model to inference mode 64 | for i, (xA, xP, xN, k1, k2) in enumerate(data_loader): 65 | # copy the mini-batch to GPU 66 | xA = xA.to(device, dtype=torch.float) 67 | xP = xP.to(device, dtype=torch.float) 68 | xN = xN.to(device, dtype=torch.float) 69 | k1 = k1.to(device, dtype=torch.float) 70 | k2 = k2.to(device, dtype=torch.float) 71 | yA, yP1, yP2, yN1, yN2 = model(xA, xP, xN, k1, k2) 72 | output_A = np.concatenate((output_A, yA.cpu().numpy())) 73 | output_P1 = np.concatenate((output_P1, yP1.cpu().numpy())) 74 | output_P2 = np.concatenate((output_P2, yP2.cpu().numpy())) 75 | output_N1 = np.concatenate((output_N1, yN1.cpu().numpy())) 76 | output_N2 = np.concatenate((output_N2, yN2.cpu().numpy())) 77 | input_A.append(xA.cpu().numpy()) 78 | input_k1.append(k1.cpu().numpy()) 79 | input_A = np.concatenate(input_A) 80 | input_k1 = np.concatenate(input_k1) 81 | return output_A, output_P1, output_P2, output_N1, output_N2, input_A, input_k1 82 | 83 | 84 | def compute_distances_triplet(yA, yP, yN): 85 | # Computes distances between triplet embeddings. 86 | yA = yA.cpu().numpy() 87 | dp = normalised_distance(yA, yP.cpu().numpy()) 88 | dn = normalised_distance(yA, yN.cpu().numpy()) 89 | return np.array([dp, dn]) 90 | 91 | 92 | def compute_distances_secure(yA, yP1, yP2, yN1, yN2): 93 | # Computes distances between secure triplet embeddings. 94 | yA = yA.cpu().numpy() 95 | dsp = normalised_distance(yA, yP1.cpu().numpy()) 96 | ddp = normalised_distance(yA, yP2.cpu().numpy()) 97 | dsn = normalised_distance(yA, yN1.cpu().numpy()) 98 | ddn = normalised_distance(yA, yN2.cpu().numpy()) 99 | return np.array([dsp, ddp, dsn, ddn]) 100 | 101 | 102 | def evaluate_triplet_model(model, data_loader, batch_size, device, debug=False, output_shape=100, N=1000, save_embeddings=False): 103 | # Gets triplets from a data_loader and sends to triplet_metrics to compute triplet loss metrics. 104 | yA, yP, yN, xA = get_triplet_outputs(model, data_loader, batch_size, device, output_shape) 105 | if save_embeddings: 106 | with open('embeddings.pk', 'wb') as hf: 107 | pk.dump((yA, yP, yN), hf) 108 | positives = normalised_distance(yA, yP) 109 | negatives = normalised_distance(yA, yN) 110 | distances = np.array([positives, negatives]) 111 | out = (triplet_metrics(distances, debug=debug, N=N),) # Transformed into a tuple 112 | out += (triplet_it_measures(xA, yA, subset=1000),) 113 | return out 114 | 115 | 116 | def evaluate_secure_model(model, data_loader, batch_size, device, debug=False, output_shape=100, N=1000, save_embeddings=False): 117 | # Gets secure triplets from a data_loader and sends to secure_metrics to compute SecureTL metrics. 118 | yA, yP1, yP2, yN1, yN2, xA, k1 = get_secure_outputs(model, data_loader, batch_size, device, output_shape) 119 | if save_embeddings: 120 | with open('embeddings.pk', 'wb') as hf: 121 | pk.dump((yA, yP1, yP2, yN1, yN2), hf) 122 | positives_samekey = normalised_distance(yA, yP1) 123 | positives_diffkey = normalised_distance(yA, yP2) 124 | negatives_samekey = normalised_distance(yA, yN1) 125 | negatives_diffkey = normalised_distance(yA, yN2) 126 | distances = np.array([positives_samekey, positives_diffkey, negatives_samekey, negatives_diffkey]) 127 | out = secure_metrics(distances, debug=debug, N=N) 128 | out += (secure_it_measures(xA, yA, k1, subset=1000),) 129 | return out 130 | 131 | 132 | def triplet_metrics(distances, debug=False, N=1000): 133 | # Computes EER based on triplet loss distances. 134 | predictions = np.concatenate((distances[0], distances[1])) 135 | labels = np.concatenate((np.zeros((distances.shape[1],)), np.ones((distances.shape[1],)))) 136 | result = evaluate_eer(predictions, labels, n=N) 137 | if debug: 138 | return result 139 | else: 140 | return result['eer'] 141 | 142 | 143 | def secure_metrics(distances, debug=False, N=1000): 144 | # Computes performance EER, cancelability EER, and linkability metrics based on SecureTL distances 145 | # Performance EER (dSP, dSN). 146 | predictions = np.concatenate((distances[0], distances[2])) 147 | labels = np.concatenate((np.zeros((distances.shape[1],)), np.ones((distances.shape[1],)))) 148 | result_eer = evaluate_eer(predictions, labels, n=N) 149 | # Cancelability EER (dSP, dDP) 150 | predictions = np.concatenate((distances[0], distances[1])) 151 | labels = np.concatenate((np.zeros((distances.shape[1],)), np.ones((distances.shape[1],)))) 152 | result_cancelability = evaluate_eer(predictions, labels, n=N) 153 | if debug: 154 | # Linkability (dDP, dDN) 155 | link = linkability(distances[1], distances[3], N=N, debug=True) 156 | return result_eer, result_cancelability, link 157 | else: 158 | # Linkability (dDP, dDN) 159 | d_sys = linkability(distances[1], distances[3], N=N) 160 | return result_eer['eer'][0], result_eer['eer'][1], result_cancelability['eer'][0], result_cancelability['eer'][1], d_sys 161 | 162 | 163 | def evaluate_eer(predictions, labels, pos_label=0, smin=0, smax=1, 164 | n=1000, positive='lower'): 165 | roc = roc_curve(labels, predictions, pos_label=pos_label, smin=smin, 166 | smax=smax, n=n, positive=positive) 167 | eer = determine_equal_error_rate(roc) 168 | return {'eer': eer, 'roc': roc} 169 | 170 | 171 | def linkability(mated, non_mated, N=1000, debug=False): 172 | # Computes D_s and D_sys based on mated and non-mated distances, to evaluate 173 | # template linkability. Set debug=False to return only D_sys, or debug=True 174 | # to return also D_s, p_sHm, and p_sHnm. 175 | # Based on M. Gomez-Barrero et al., "Unlinkable and irreversible biometric 176 | # template protection based on bloom filters", Information Sciences 370-371, 177 | # pp. 18–32, 2016. 178 | 179 | s_Hm = sorted(mated) # dDP 180 | s_Hnm = sorted(non_mated) # dDN 181 | 182 | p_sHm = stats.norm.pdf(np.linspace(0, 1, num=N), np.mean(s_Hm), np.std(s_Hm)) 183 | p_sHnm = stats.norm.pdf(np.linspace(0, 1, num=N), np.mean(s_Hnm), np.std(s_Hnm)) + 1e-15 # Avoid divide by zero 184 | 185 | # Compute D_{<->}(s) 186 | lr_s = p_sHm/p_sHnm 187 | d_s = np.maximum((np.power(np.exp(-(lr_s - 1)) + 1, -1) - 0.5) * 2, 0) 188 | 189 | # Compute D_{<->}^{sys} 190 | d_sys = np.trapz(d_s * p_sHm, dx=1.0/N) 191 | 192 | # Only return more than 193 | if debug: 194 | return d_sys, d_s, p_sHm, p_sHnm 195 | else: 196 | return d_sys 197 | 198 | 199 | def triplet_it_measures(xA, yA, k=10, subset=None): 200 | # Uses the entropy_estimators package to estimate information 201 | # theoretical security metrics. 202 | np.random.seed(42) # Ensuring reproducibility 203 | if subset is not None: # subset chooses only N random samples to speed up the process 204 | indices = np.random.choice(range(len(xA)), size=subset, replace=False) 205 | input = xA[indices] 206 | output = yA[indices] 207 | else: 208 | input = xA 209 | output = yA 210 | input = np.reshape(input, (len(input), -1)) 211 | mi_xy = continuous.get_mi(input, output, k=k) 212 | h_x = continuous.get_h(input, k=k) 213 | out = {'plr': 1.0 - mi_xy/h_x} 214 | return out 215 | 216 | 217 | def secure_it_measures(xA, yA, k1, k=10, subset=None): 218 | # Uses the entropy_estimators package to estimate information 219 | # theoretical security metrics. 220 | np.random.seed(42) # Ensuring reproducibility 221 | if subset is not None: # subset chooses only N random samples to speed up the process 222 | indices = np.random.choice(range(len(xA)), size=subset, replace=False) 223 | input = xA[indices] 224 | output = yA[indices] 225 | keys = k1[indices] 226 | else: 227 | input = xA 228 | output = yA 229 | keys = k1 230 | input = np.reshape(input, (len(input), -1)) 231 | mi_xy = continuous.get_mi(input, output, k=k) 232 | h_x = continuous.get_h(input, k=k) 233 | mi_ky = continuous.get_mi(keys, output, k=k) 234 | out = {'plr': 1.0 - mi_xy/h_x, 'sl': mi_ky} 235 | return out 236 | 237 | 238 | def determine_equal_error_rate(roc): 239 | # Computes the EER for a certain Receiver Operating Characteristic curve. 240 | fpr, fnr, thr = roc 241 | diff = fpr - fnr 242 | abs_diff = np.absolute(diff) 243 | min_diff = np.min(abs_diff) 244 | min_loc = np.argmin(abs_diff) 245 | if min_diff == 0: 246 | return (thr[min_loc], fpr[min_loc]) 247 | elif diff[0] * diff[-1] > 0: 248 | print('Warning! FPR and FNR lines do not intersect.') 249 | return (np.nan, np.nan) 250 | else: 251 | if diff[np.argmin(abs_diff)] > 0: 252 | idx1 = min_loc - 1 253 | idx2 = min_loc 254 | else: 255 | idx1 = min_loc 256 | idx2 = min_loc + 1 257 | if diff[idx1] * diff[idx2] >= 0: 258 | idx1, idx2 = escape_parallel_plateaux(diff) 259 | A1 = [thr[idx1], fpr[idx1]] 260 | B1 = [thr[idx1], fnr[idx1]] 261 | A2 = [thr[idx2], fpr[idx2]] 262 | B2 = [thr[idx2], fnr[idx2]] 263 | lA = line(A1, A2) 264 | lB = line(B1, B2) 265 | eer = intersection(lA, lB) 266 | if eer is False: 267 | return (np.nan, np.nan) 268 | else: 269 | return eer 270 | 271 | 272 | def roc_curve(labels, predictions, pos_label=0, smin=0, smax=1, 273 | n=1000, positive='lower'): 274 | # Computes a Receiver Operating Characteristic curve based on 275 | # true labels and prediction scores. 276 | positives = predictions[labels == pos_label] 277 | negatives = predictions[labels != pos_label] 278 | n_positives = len(positives) 279 | n_negatives = len(negatives) 280 | fpr = np.zeros((n,)) 281 | fnr = np.zeros((n,)) 282 | thr = np.linspace(smin, smax, num=n) 283 | if positive == 'lower': 284 | for tt in range(n): 285 | fpr[tt] = np.sum(negatives <= thr[tt])/n_negatives 286 | fnr[tt] = np.sum(positives > thr[tt])/n_positives 287 | elif positive == 'higher': 288 | for tt in range(n): 289 | fpr[tt] = np.sum(negatives >= thr[tt])/n_negatives 290 | fnr[tt] = np.sum(positives < thr[tt])/n_positives 291 | else: 292 | raise ValueError('Parameter \'positive\' can only be \'lower\' or \'higher\'.') 293 | return fpr, fnr, thr 294 | 295 | 296 | def normalised_distance(yA, yX): 297 | # Normalised Euclidean distance between two embeddings. 298 | var_A = np.var(yA, axis=1) 299 | var_X = np.var(yX, axis=1) 300 | var_AX = np.var(yA-yX, axis=1) 301 | dist = 0.5 * var_AX / (var_A + var_X) # d = 0.5 * var(x - y) / (var(x) + var(y)) 302 | return dist 303 | 304 | 305 | def line(p1, p2): 306 | # Computes line coefficients for intersection determination. 307 | A = (p1[1] - p2[1]) 308 | B = (p2[0] - p1[0]) 309 | C = (p1[0]*p2[1] - p2[0]*p1[1]) 310 | return A, B, -C 311 | 312 | 313 | def intersection(L1, L2): 314 | # Returns the intersection point between two lines L1 and L2. 315 | D = L1[0]*L2[1] - L1[1]*L2[0] 316 | Dx = L1[2]*L2[1] - L1[1]*L2[2] 317 | Dy = L1[0]*L2[2] - L1[2]*L2[0] 318 | if D != 0: 319 | x = Dx / D 320 | y = Dy / D 321 | return x, y 322 | else: 323 | return False 324 | 325 | 326 | def escape_parallel_plateaux(diff): 327 | # Procedure to escale parallel plateaux. 328 | diff_t = diff[1:len(diff)] * diff[0:len(diff)-1] 329 | idx1 = np.argmin(diff_t) 330 | idx2 = np.argmin(diff_t) + 1 331 | return (idx1, idx2) 332 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: models.py 5 | - Defines the models: the unsecure models of ECG/face that can be trained with 6 | the original triplet loss; and the secure models (with cancellability keys) 7 | that can be trained with the proposed Secure Triplet Loss. The ECG model is 8 | based on our prior work [1] and the face model is based on the Inception 9 | ResNet [2]. 10 | 11 | References: 12 | [1] JR Pinto and JS Cardoso, "A end-to-end convolutional neural network for 13 | ECG based biometric authentication", in BTAS 2019. 14 | [2] C. Szegedy et al., "Inceptionv4, Inception-ResNet and the Impact of 15 | Residual Connections on Learning", in AAAI 2017. 16 | 17 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 18 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 19 | IEEE Transactions on Biometrics, Behavior, and Identity Science 20 | 21 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 22 | ''' 23 | 24 | import torch 25 | from torch import nn 26 | from torch.nn import functional as F 27 | 28 | 29 | class TripletECGNetwork(nn.Module): 30 | # Defines the ECG network that processes a 31 | # single biometric sample. Is used with TripletModel 32 | # for training with the original triplet loss. 33 | 34 | def __init__(self, dropout_prob=0.5): 35 | # Defining the structure of the ECG network. 36 | super(TripletECGNetwork, self).__init__() 37 | self.convnet = nn.Sequential(nn.Conv1d(1, 16, 5), 38 | nn.ReLU(), 39 | nn.MaxPool1d(3, stride=3), 40 | nn.Conv1d(16, 16, 5), 41 | nn.ReLU(), 42 | nn.MaxPool1d(3, stride=3), 43 | nn.Conv1d(16, 32, 5), 44 | nn.ReLU(), 45 | nn.MaxPool1d(3, stride=3), 46 | nn.Conv1d(32, 32, 5), 47 | nn.ReLU(), 48 | nn.MaxPool1d(3, stride=3) ) 49 | 50 | self.dropout = nn.Sequential(nn.Dropout(p=dropout_prob)) 51 | 52 | self.fc = nn.Sequential(nn.Linear(320, 100), 53 | nn.ReLU(), 54 | nn.Linear(100, 100), 55 | nn.ReLU() ) 56 | 57 | def forward(self, x): 58 | # Network's inference routine. 59 | h = self.convnet(x) 60 | h = h.view(h.size()[0], -1) 61 | h = self.dropout(h) 62 | output = self.fc(h) 63 | return output 64 | 65 | def get_embedding(self, x): 66 | # To get an embedding (template). 67 | return self.forward(x) 68 | 69 | 70 | class TripletFaceNetwork(nn.Module): 71 | # Defines the face network that processes a 72 | # single biometric sample. Is used with TripletModel 73 | # for training with the original triplet loss. 74 | 75 | def __init__(self, pretrained_model, dropout_prob=0.5): 76 | # Defining the structure of the network, based on 77 | # the Inception-ResNet model with pretrained weights. 78 | super(TripletFaceNetwork, self).__init__() 79 | self.conv2d_1a = pretrained_model.conv2d_1a 80 | self.conv2d_2a = pretrained_model.conv2d_2a 81 | self.conv2d_2b = pretrained_model.conv2d_2b 82 | self.maxpool_3a = pretrained_model.maxpool_3a 83 | self.conv2d_3b = pretrained_model.conv2d_3b 84 | self.conv2d_4a = pretrained_model.conv2d_4a 85 | self.conv2d_4b = pretrained_model.conv2d_4b 86 | self.repeat_1 = pretrained_model.repeat_1 87 | self.mixed_6a = pretrained_model.mixed_6a 88 | self.repeat_2 = pretrained_model.repeat_2 89 | self.mixed_7a = pretrained_model.mixed_7a 90 | self.repeat_3 = pretrained_model.repeat_3 91 | self.block8 = pretrained_model.block8 92 | self.avgpool_1a = pretrained_model.avgpool_1a 93 | self.relu = nn.Sequential(nn.ReLU()) 94 | self.dropout = nn.Dropout(dropout_prob) 95 | self.fc1 = nn.Sequential(nn.Linear(1792, 100), nn.ReLU()) 96 | self.fc2 = nn.Sequential(nn.Linear(100, 100), nn.ReLU()) 97 | 98 | def forward(self, x): 99 | # Network's inference routine. 100 | x = self.conv2d_1a(x) 101 | x = self.conv2d_2a(x) 102 | x = self.conv2d_2b(x) 103 | x = self.maxpool_3a(x) 104 | x = self.conv2d_3b(x) 105 | x = self.conv2d_4a(x) 106 | x = self.conv2d_4b(x) 107 | x = self.repeat_1(x) 108 | x = self.mixed_6a(x) 109 | x = self.repeat_2(x) 110 | x = self.mixed_7a(x) 111 | x = self.repeat_3(x) 112 | x = self.block8(x) 113 | x = self.avgpool_1a(x) 114 | x = self.relu(x) 115 | x = self.dropout(x) 116 | x = x.view(x.shape[0], -1) 117 | x = self.fc1(x) 118 | x = self.fc2(x) 119 | return x 120 | 121 | def get_embedding(self, x): 122 | # To get an embedding (template). 123 | return self.forward(x) 124 | 125 | def freeze_parameters(self): 126 | # Freezes parameters in the first part of the 127 | # network to retain VGGFace2 pretrained weights. 128 | for param in self.conv2d_1a.parameters(): 129 | param.requires_grad = False 130 | for param in self.conv2d_2a.parameters(): 131 | param.requires_grad = False 132 | for param in self.conv2d_2b.parameters(): 133 | param.requires_grad = False 134 | for param in self.conv2d_3b.parameters(): 135 | param.requires_grad = False 136 | for param in self.conv2d_4a.parameters(): 137 | param.requires_grad = False 138 | for param in self.conv2d_4b.parameters(): 139 | param.requires_grad = False 140 | for param in self.repeat_1.parameters(): 141 | param.requires_grad = False 142 | for param in self.mixed_6a.parameters(): 143 | param.requires_grad = False 144 | for param in self.repeat_2.parameters(): 145 | param.requires_grad = False 146 | for param in self.mixed_7a.parameters(): 147 | param.requires_grad = False 148 | for param in self.repeat_3.parameters(): 149 | param.requires_grad = False 150 | 151 | def unfreeze_parameters(self): 152 | # Unfreezes the first part of the network. 153 | for param in self.conv2d_1a.parameters(): 154 | param.requires_grad = True 155 | for param in self.conv2d_2a.parameters(): 156 | param.requires_grad = True 157 | for param in self.conv2d_2b.parameters(): 158 | param.requires_grad = True 159 | for param in self.conv2d_3b.parameters(): 160 | param.requires_grad = True 161 | for param in self.conv2d_4a.parameters(): 162 | param.requires_grad = True 163 | for param in self.conv2d_4b.parameters(): 164 | param.requires_grad = True 165 | for param in self.repeat_1.parameters(): 166 | param.requires_grad = True 167 | for param in self.mixed_6a.parameters(): 168 | param.requires_grad = True 169 | for param in self.repeat_2.parameters(): 170 | param.requires_grad = True 171 | for param in self.mixed_7a.parameters(): 172 | param.requires_grad = True 173 | for param in self.repeat_3.parameters(): 174 | param.requires_grad = True 175 | 176 | 177 | class SecureECGNetwork(nn.Module): 178 | # Defines the ECG secure network that processes a 179 | # single biometric sample and a key. Is used with 180 | # SecureModel for training with Secure Triplet Loss. 181 | 182 | def __init__(self): 183 | # Defining the structure of the ECG secure network. 184 | super(SecureECGNetwork, self).__init__() 185 | self.convnet = nn.Sequential(nn.Conv1d(1, 16, 5), 186 | nn.ReLU(), 187 | nn.MaxPool1d(3, stride=3), 188 | nn.Conv1d(16, 16, 5), 189 | nn.ReLU(), 190 | nn.MaxPool1d(3, stride=3), 191 | nn.Conv1d(16, 32, 5), 192 | nn.ReLU(), 193 | nn.MaxPool1d(3, stride=3), 194 | nn.Conv1d(32, 32, 5), 195 | nn.ReLU(), 196 | nn.MaxPool1d(3, stride=3) ) 197 | 198 | self.dropout = nn.Sequential(nn.Dropout(p=DROPOUT)) 199 | 200 | self.fc = nn.Sequential(nn.Linear(420, 100), 201 | nn.ReLU(), 202 | nn.Linear(100, 100), 203 | nn.ReLU() ) 204 | 205 | def forward(self, x, k): 206 | # Network's inference routine. 207 | h = self.convnet(x) 208 | h = h.view(h.size()[0], -1) 209 | h = self.dropout(h) 210 | h = torch.cat((h, k), dim=1) 211 | output = self.fc(h) 212 | return output 213 | 214 | def get_embedding(self, x, k): 215 | # To get a secure embedding (template). 216 | return self.forward(x, k) 217 | 218 | 219 | class SecureFaceNetwork(nn.Module): 220 | # Defines the face secure network that processes a 221 | # single biometric sample and a key. Is used with 222 | # SecureModel for training with Secure Triplet Loss. 223 | 224 | def __init__(self, pretrained_model, dropout_prob=0.5): 225 | # Defining the structure of the secure network, based on 226 | # the Inception-ResNet model with pretrained weights. 227 | super(SecureFaceNetwork, self).__init__() 228 | self.conv2d_1a = pretrained_model.conv2d_1a 229 | self.conv2d_2a = pretrained_model.conv2d_2a 230 | self.conv2d_2b = pretrained_model.conv2d_2b 231 | self.maxpool_3a = pretrained_model.maxpool_3a 232 | self.conv2d_3b = pretrained_model.conv2d_3b 233 | self.conv2d_4a = pretrained_model.conv2d_4a 234 | self.conv2d_4b = pretrained_model.conv2d_4b 235 | self.repeat_1 = pretrained_model.repeat_1 236 | self.mixed_6a = pretrained_model.mixed_6a 237 | self.repeat_2 = pretrained_model.repeat_2 238 | self.mixed_7a = pretrained_model.mixed_7a 239 | self.repeat_3 = pretrained_model.repeat_3 240 | self.block8 = pretrained_model.block8 241 | self.avgpool_1a = pretrained_model.avgpool_1a 242 | self.relu = nn.Sequential(nn.ReLU()) 243 | self.dropout = nn.Dropout(dropout_prob) 244 | self.fc1 = nn.Sequential(nn.Linear(1892, 100), nn.ReLU()) 245 | self.fc2 = nn.Sequential(nn.Linear(100, 100), nn.ReLU()) 246 | 247 | def forward(self, x, k): 248 | # Network's inference routine. 249 | x = self.conv2d_1a(x) 250 | x = self.conv2d_2a(x) 251 | x = self.conv2d_2b(x) 252 | x = self.maxpool_3a(x) 253 | x = self.conv2d_3b(x) 254 | x = self.conv2d_4a(x) 255 | x = self.conv2d_4b(x) 256 | x = self.repeat_1(x) 257 | x = self.mixed_6a(x) 258 | x = self.repeat_2(x) 259 | x = self.mixed_7a(x) 260 | x = self.repeat_3(x) 261 | x = self.block8(x) 262 | x = self.avgpool_1a(x) 263 | x = self.relu(x) 264 | x = self.dropout(x) 265 | x = x.view(x.shape[0], -1) 266 | x = torch.cat((x, k), dim=1) 267 | x = self.fc1(x) 268 | x = self.fc2(x) 269 | return x 270 | 271 | def get_embedding(self, x, k): 272 | # To get a secure embedding (template). 273 | return self.forward(x, k) 274 | 275 | def freeze_parameters(self): 276 | # Freezes parameters in the first part of the 277 | # network to retain VGGFace2 pretrained weights. 278 | for param in self.conv2d_1a.parameters(): 279 | param.requires_grad = False 280 | for param in self.conv2d_2a.parameters(): 281 | param.requires_grad = False 282 | for param in self.conv2d_2b.parameters(): 283 | param.requires_grad = False 284 | for param in self.conv2d_3b.parameters(): 285 | param.requires_grad = False 286 | for param in self.conv2d_4a.parameters(): 287 | param.requires_grad = False 288 | for param in self.conv2d_4b.parameters(): 289 | param.requires_grad = False 290 | for param in self.repeat_1.parameters(): 291 | param.requires_grad = False 292 | for param in self.mixed_6a.parameters(): 293 | param.requires_grad = False 294 | for param in self.repeat_2.parameters(): 295 | param.requires_grad = False 296 | for param in self.mixed_7a.parameters(): 297 | param.requires_grad = False 298 | for param in self.repeat_3.parameters(): 299 | param.requires_grad = False 300 | 301 | def unfreeze_parameters(self): 302 | # Unfreezes the first part of the network. 303 | for param in self.conv2d_1a.parameters(): 304 | param.requires_grad = True 305 | for param in self.conv2d_2a.parameters(): 306 | param.requires_grad = True 307 | for param in self.conv2d_2b.parameters(): 308 | param.requires_grad = True 309 | for param in self.conv2d_3b.parameters(): 310 | param.requires_grad = True 311 | for param in self.conv2d_4a.parameters(): 312 | param.requires_grad = True 313 | for param in self.conv2d_4b.parameters(): 314 | param.requires_grad = True 315 | for param in self.repeat_1.parameters(): 316 | param.requires_grad = True 317 | for param in self.mixed_6a.parameters(): 318 | param.requires_grad = True 319 | for param in self.repeat_2.parameters(): 320 | param.requires_grad = True 321 | for param in self.mixed_7a.parameters(): 322 | param.requires_grad = True 323 | for param in self.repeat_3.parameters(): 324 | param.requires_grad = True 325 | 326 | 327 | class TripletModel(nn.Module): 328 | # Defines the model to be trained with the 329 | # original triplet loss. Can be based on either 330 | # TripletECGNetwork or TripletFaceNetwork. 331 | 332 | def __init__(self, network): 333 | super(TripletModel, self).__init__() 334 | self.network = network 335 | 336 | def forward(self, xA, xP, xN): 337 | # Triplet inference routine. 338 | output_a = self.network(xA) 339 | output_p = self.network(xP) 340 | output_n = self.network(xN) 341 | return output_a, output_p, output_n 342 | 343 | def get_embedding(self, x): 344 | # To get an embedding (template). 345 | return self.network(x) 346 | 347 | 348 | class SecureModel(nn.Module): 349 | # Defines the model to be trained with the 350 | # Secure Triplet Loss. Can be based on either 351 | # SecureECGNetwork or SecureFaceNetwork. 352 | 353 | def __init__(self, network): 354 | super(SecureModel, self).__init__() 355 | self.network = network 356 | 357 | def forward(self, xA, xP, xN, k1, k2): 358 | # Secure triplet inference routine. 359 | output_a = self.network(xA, k1) 360 | output_p1 = self.network(xP, k1) 361 | output_p2 = self.network(xP, k2) 362 | output_n1 = self.network(xN, k1) 363 | output_n2 = self.network(xN, k2) 364 | return output_a, output_p1, output_p2, output_n1, output_n2 365 | 366 | def get_embedding(self, x, k): 367 | # To get a secure embedding (template). 368 | return self.network(x, k) 369 | -------------------------------------------------------------------------------- /aux_functions.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Secure Triplet Loss Project Repository (https://github.com/jtrpinto/SecureTL) 3 | 4 | File: aux_functions.py 5 | - Miscellaneous functions used to load and process face and ECG data, and to 6 | compute and plot several performance and security metrics. 7 | 8 | "Secure Triplet Loss: Achieving Cancelability and Non-Linkability in End-to-End Deep Biometrics" 9 | João Ribeiro Pinto, Miguel V. Correia, and Jaime S. Cardoso 10 | IEEE Transactions on Biometrics, Behavior, and Identity Science 11 | 12 | joao.t.pinto@inesctec.pt | https://jtrpinto.github.io 13 | ''' 14 | 15 | import matplotlib 16 | matplotlib.use('agg') # Do not try to show any figures 17 | 18 | import os 19 | import cv2 20 | import warnings 21 | import numpy as np 22 | from PIL import Image 23 | from copy import deepcopy 24 | from matplotlib import pyplot as pl 25 | from sklearn.preprocessing import normalize 26 | 27 | 28 | # AUXILIARY FUNCTIONS FOR THE FACE DATA 29 | 30 | def process_image(origin, destination, id_name): 31 | # Used to open images and store them, prepared to be used, 32 | # in the destination directory. 33 | img_name = os.path.basename(origin) 34 | out_path = os.path.join(destination, id_name + '_' + img_name) 35 | if os.path.exists(out_path): 36 | return out_path 37 | else: 38 | img = Image.open(origin) 39 | shp = img.size 40 | img = np.array(img)[int(0.15*shp[0]):int(0.85*shp[0]), int(0.15*shp[1]):int(0.85*shp[1])] 41 | img = cv2.resize(img, (160, 160), interpolation=cv2.INTER_AREA) 42 | img = Image.fromarray(img) 43 | img.save(out_path) 44 | return out_path 45 | 46 | def generate_id_triplets(identity, db_dir, save_dir, all_identities, n_triplets=100): 47 | # Generates n triplets for a given identity. 48 | triplets = list() 49 | id_dir = os.path.join(db_dir, identity) 50 | 51 | for nn in range(n_triplets): 52 | videos = os.listdir(id_dir) 53 | 54 | # If there is more than one folder for this ID, take the anchor and 55 | # positive images from two different folders: 56 | if len(videos) > 1: 57 | a_p_vids = np.random.choice(videos, size=2, replace=False) 58 | a_vid_dir = os.path.join(id_dir, a_p_vids[0]) 59 | anchor = os.path.join(a_vid_dir, np.random.choice(os.listdir(a_vid_dir))) 60 | p_vid_dir = os.path.join(id_dir, a_p_vids[1]) 61 | positive = os.path.join(p_vid_dir, np.random.choice(os.listdir(p_vid_dir))) 62 | # Otherwise, choose both images randomly within the same folder: 63 | else: 64 | video_dir = os.path.join(id_dir, videos[0]) 65 | frames = os.listdir(video_dir) 66 | a_p = np.random.choice(frames, size=2, replace=False) 67 | anchor = os.path.join(video_dir, a_p[0]) 68 | positive = os.path.join(video_dir, a_p[1]) 69 | 70 | # Choosing randomly a negative image, from a different identity: 71 | neg_id = np.random.choice(all_identities[np.argwhere(all_identities != identity)[:,0]]) 72 | neg_id_dir = os.path.join(db_dir, neg_id) 73 | neg_vids = os.listdir(neg_id_dir) 74 | neg_v = np.random.choice(neg_vids) 75 | neg_v_dir = os.path.join(neg_id_dir, neg_v) 76 | neg_frames = os.listdir(neg_v_dir) 77 | negative = os.path.join(neg_v_dir, np.random.choice(neg_frames)) 78 | 79 | # Take the images and process them to be used later: 80 | anchor_path = process_image(anchor, save_dir, identity) 81 | positive_path = process_image(positive, save_dir, identity) 82 | negative_path = process_image(negative, save_dir, neg_id) 83 | 84 | # Generate two keys: 85 | keys = np.random.randint(0, high=2, size=(2, 100)) 86 | key1, key2 = normalize(keys, norm='l2', axis=1) 87 | 88 | # Save the triplets info: 89 | triplets.append([anchor_path, positive_path, negative_path, key1, key2]) 90 | return triplets 91 | 92 | 93 | 94 | # AUXILIARY FUNCIONS FOR THE ECG DATA 95 | 96 | def load_uoftdb_recording(path, subject, session, filename): 97 | with warnings.catch_warnings(): 98 | warnings.simplefilter("ignore") 99 | signal = np.loadtxt(path + 'uoftdb_' + subject + '_' + session + '_' + filename + '.txt') 100 | return {'signal': signal, 'fs': 200.0} 101 | 102 | 103 | def getSegments(path, subjects, length, step): 104 | uoftdb_ses = ['1', '2', '3', '4', '5', '6'] 105 | uoftdb_fil = ['1', '2', '3', '4', '5'] 106 | 107 | segments = list() 108 | y = list() 109 | t = list() 110 | 111 | for sub in subjects: 112 | file_matrix = np.zeros((6, 5)) 113 | for ses in uoftdb_ses: 114 | for fil in uoftdb_fil: 115 | data = load_uoftdb_recording(path, str(sub), ses, fil) 116 | file_matrix[int(ses)-1, int(fil)-1] = len(data['signal']) 117 | for kk in np.arange(length, len(data['signal']), step): 118 | segments.append(data['signal'][kk-length:kk]) 119 | y.append(sub) 120 | return {'segments': segments, 'labels': y, 'times': t} 121 | 122 | 123 | def extract_data(path, subjects, fs=200.0): 124 | no_templ = 26 # To reserve first 30 seconds for anchors 125 | 126 | data = getSegments(path, subjects, int(fs * 5), int(fs * 1)) 127 | 128 | data_train_x = list() 129 | data_train_y = list() 130 | train_subs = {} 131 | 132 | for kk in range(len(data['labels'])): 133 | if str(data['labels'][kk]) not in train_subs: 134 | train_subs[str(data['labels'][kk])] = 0 135 | if train_subs[str(data['labels'][kk])] < no_templ: 136 | data_train_x.append(data['segments'][kk]) 137 | data_train_y.append(data['labels'][kk]) 138 | train_subs[str(data['labels'][kk])] += 1 139 | 140 | data = getSegments(path, subjects, int(fs * 5), int(fs * 1)) 141 | 142 | data_test_x = list() 143 | data_test_y = list() 144 | test_subs = {} 145 | 146 | for kk in range(len(data['labels'])): 147 | if str(data['labels'][kk]) not in test_subs: 148 | test_subs[str(data['labels'][kk])] = 0 149 | if test_subs[str(data['labels'][kk])] >= no_templ + 4: 150 | data_test_x.append(data['segments'][kk]) 151 | data_test_y.append(data['labels'][kk]) 152 | test_subs[str(data['labels'][kk])] += 1 153 | 154 | train_x = np.array(data_train_x) 155 | test_x = np.array(data_test_x) 156 | 157 | return {'X_anchors': train_x, 'y_anchors': data_train_y, 'X_remaining': test_x, 'y_remaining': data_test_y} 158 | 159 | 160 | def random_triplets(X_anchor, y_anchor, X_remaining, y_remaining, size=1000): 161 | ind = np.random.choice(range(len(X_anchor)), size=size, replace=True) 162 | random_Xa = X_anchor[ind, :, :] # Anchors 163 | random_ya = y_anchor[ind] 164 | random_Xp = np.zeros(random_Xa.shape) # Positives 165 | random_Xn = np.zeros(random_Xa.shape) # Negatives 166 | 167 | for ii in range(len(random_Xa)): 168 | positive_indices = np.argwhere(y_remaining == random_ya[ii]) 169 | negative_indices = np.argwhere(y_remaining != random_ya[ii]) 170 | 171 | rand_p = positive_indices[np.random.randint(0, len(positive_indices))] 172 | rand_n = negative_indices[np.random.randint(0, len(negative_indices))] 173 | 174 | random_Xp[ii, :, :] = X_remaining[rand_p, :, :] 175 | random_Xn[ii, :, :] = X_remaining[rand_n, :, :] 176 | 177 | return random_Xa, random_Xp, random_Xn, random_ya 178 | 179 | 180 | def generate_keys(n_keys, key_length): 181 | keys = np.random.randint(0, high=2, size=(n_keys, key_length)) 182 | keys = normalize(keys, norm='l2', axis=1) 183 | return keys 184 | 185 | 186 | def generate_triplets(X_anchors, y_anchors, X_remaining, y_remaining, N=1000): 187 | xA, xP, xN, yD = random_triplets(X_anchors, y_anchors, X_remaining, y_remaining, size=N) 188 | k1 = generate_keys(N, 100) 189 | k2 = generate_keys(N, 100) 190 | return [xA, xP, xN, yD, k1, k2] 191 | 192 | 193 | def normalise_templates(templates): 194 | out = deepcopy(templates) 195 | for ii in range(len(out)): 196 | numer = np.subtract(out[ii], np.mean(out[ii])) 197 | denum = np.std(out[ii]) 198 | out[ii] = np.divide(numer, denum) 199 | return out 200 | 201 | 202 | def prepare_for_dnn(X, y, z_score_normalise=True): 203 | if z_score_normalise: 204 | X = normalise_templates(X) 205 | X_cnn = X.reshape(X.shape + (1,)) 206 | y_cnn = np.asarray(y, dtype='float') 207 | return X_cnn, y_cnn 208 | 209 | 210 | 211 | # AUXILIARY FUNCTIONS FOR PLOTTING RESULTS 212 | 213 | def cancelability_fmr_at_eer(canc_fmr, thresholds, eer_thr): 214 | aux = np.abs(thresholds - eer_thr) 215 | idx = np.argmin(aux) 216 | if thresholds[idx] == eer_thr: 217 | return canc_fmr[idx] 218 | elif thresholds[idx] > eer_thr: 219 | d_before = np.abs(thresholds[idx-1] - eer_thr) 220 | d_after = np.abs(thresholds[idx] - eer_thr) 221 | d_total = d_before + d_after 222 | return d_before*canc_fmr[idx]/d_total + d_after*canc_fmr[idx-1]/d_total 223 | elif thresholds[idx] < eer_thr: 224 | d_before = np.abs(thresholds[idx] - eer_thr) 225 | d_after = np.abs(thresholds[idx+1] - eer_thr) 226 | d_total = d_before + d_after 227 | return d_after*canc_fmr[idx]/d_total + d_before*canc_fmr[idx+1]/d_total 228 | 229 | 230 | def fnmr_at_fmr(fnmr, fmr, reference_fmr=0.01): 231 | aux = np.abs(fmr - reference_fmr) 232 | idx = np.argmin(aux) 233 | if fmr[idx] == reference_fmr: 234 | return fnmr[idx] 235 | elif fmr[idx] > reference_fmr: 236 | d_before = np.abs(fmr[idx-1] - reference_fmr) 237 | d_after = np.abs(fmr[idx] - reference_fmr) 238 | d_total = d_before + d_after 239 | return d_before*fnmr[idx]/d_total + d_after*fnmr[idx-1]/d_total 240 | elif fmr[idx] < reference_fmr: 241 | d_before = np.abs(fmr[idx] - reference_fmr) 242 | d_after = np.abs(fmr[idx+1] - reference_fmr) 243 | d_total = d_before + d_after 244 | return d_after*fnmr[idx]/d_total + d_before*fnmr[idx+1]/d_total 245 | 246 | 247 | def plot_perf_curves(results, title=None, figsize=[6.4, 4.0], savefile=None): 248 | pl.figure(figsize=figsize) 249 | pl.plot(np.linspace(0, 1, len(results['roc'][1])), results['roc'][1], 'r-', label=r'$FNMR$') 250 | pl.plot(np.linspace(0, 1, len(results['roc'][0])), results['roc'][0], 'b-', label=r'$FMR$') 251 | pl.scatter(results['eer'][0], results['eer'][1], color='black', marker='o', label=r'$EER$') 252 | pl.legend() 253 | pl.xlabel(r'Threshold ($t$)') 254 | pl.ylabel(r'Error Rate') 255 | pl.xlim([-0.02, 1.02]) 256 | pl.ylim([-0.02, 1.02]) 257 | if title is not None: 258 | pl.title(title) 259 | print(title) 260 | print('EER', results['eer'][1]) 261 | print('FNMR@0.1%FMR', fnmr_at_fmr(results['roc'][1], results['roc'][0], reference_fmr=0.001)) 262 | print('FNMR@1%FMR', fnmr_at_fmr(results['roc'][1], results['roc'][0], reference_fmr=0.01)) 263 | print() 264 | pl.tight_layout() 265 | pl.grid(which='both') 266 | if savefile is not None: 267 | pl.savefig(savefile) 268 | else: 269 | pl.show() 270 | 271 | 272 | def plot_perf_vs_canc_curves(results, smin=0, smax=1, title=None, figsize=[6.4, 4.0], savefile=None): 273 | eer = results[0] 274 | canc = results[1] 275 | fmrc_eer = cancelability_fmr_at_eer(canc['roc'][0], canc['roc'][2], eer['eer'][0]) 276 | pl.figure(figsize=figsize) 277 | pl.plot(np.linspace(smin, smax, len(eer['roc'][1])), eer['roc'][1], 'r-', label=r'$FNMR$') 278 | pl.plot(np.linspace(smin, smax, len(eer['roc'][0])), eer['roc'][0], 'b-', label=r'$FMR_V$') 279 | pl.plot(np.linspace(smin, smax, len(canc['roc'][0])), canc['roc'][0], 'g-', label=r'$FMR_C$') 280 | pl.scatter(eer['eer'][0], eer['eer'][1], color='black', marker='o', label=r'$EER$') 281 | pl.scatter(eer['eer'][0], fmrc_eer, color='green', marker='o', label=r'$FMR_C@EER$') 282 | pl.legend() 283 | pl.xlabel(r'Threshold ($t$)') 284 | pl.ylabel(r'Error Rate') 285 | pl.xlim([smin - 0.02, smax + 0.02]) 286 | pl.ylim([smin - 0.02, smax + 0.02]) 287 | if title is not None: 288 | pl.title(title) 289 | print(title) 290 | print('EER', eer['eer'][1]) 291 | print('FNMR@0.1%FMR', fnmr_at_fmr(eer['roc'][1], eer['roc'][0], reference_fmr=0.001)) 292 | print('FNMR@1%FMR', fnmr_at_fmr(eer['roc'][1], eer['roc'][0], reference_fmr=0.01)) 293 | print('FMR_C@EER', fmrc_eer) 294 | pl.tight_layout() 295 | pl.grid(which='both') 296 | if savefile is not None: 297 | pl.savefig(savefile) 298 | else: 299 | pl.show() 300 | 301 | 302 | def plot_dsys(results, smin=0, smax=1, title=None, figsize=[6.4, 4.0], savefile=None): 303 | d_s = results[2][1] 304 | p_sHm = results[2][2] 305 | p_sHnm = results[2][3] 306 | N = len(d_s) 307 | fig = pl.figure(figsize=figsize) 308 | if title is not None: 309 | pl.title(title) 310 | print('d_sys', results[2][0]) 311 | print() 312 | ax = fig.add_subplot() 313 | ax.set_xlabel(r'Distance score ($d$)') 314 | ax.set_xlim([smin-0.02, smax+0.02]) 315 | ax.set_ylabel(r'Probability density ($p$)') 316 | max_y = max(np.max(p_sHm), np.max(p_sHnm)) 317 | ax.set_ylim([-0.02*max_y, 1.02*max_y]) 318 | c1 = ax.plot(np.linspace(smin, smax, num=N), p_sHm, 'g', label=r'$p(d|H_{m})$') 319 | c2 = ax.plot(np.linspace(smin, smax, num=N), p_sHnm, 'r', label=r'$p(d|H_{nm})$') 320 | ax.tick_params(axis='y') 321 | ax2 = ax.twinx() # instantiate a second axes that shares the same x-axis 322 | ax2.set_ylabel(r'$D_{\leftrightarrow}(d)$') 323 | ax2.set_ylim([-0.02, 1.02]) 324 | c3 = ax2.plot(np.linspace(smin, smax, num=N), d_s, 'b', label=r'$D_\leftrightarrow(d)$') 325 | ax2.tick_params(axis='y') 326 | lns = c1 + c2 + c3 327 | labs = [l.get_label() for l in lns] 328 | ax.legend(lns, labs, loc=0) 329 | fig.tight_layout() 330 | if savefile is not None: 331 | pl.savefig(savefile) 332 | else: 333 | pl.show() 334 | 335 | 336 | def plot_roc(rocs, names, title=None, figsize=[6.4, 4.0], savefile=None): 337 | fig = pl.figure(figsize=figsize) 338 | ax = fig.add_subplot() 339 | axins = ax.inset_axes([0.4, 0.35, 0.55, 0.55]) 340 | x1, x2, y1, y2 = 0.05, 0.25, 0.75, 0.95 341 | axins.set_xlim(x1, x2) 342 | axins.set_ylim(y1, y2) 343 | for ii in range(len(rocs)): 344 | ax.plot(rocs[ii][0], 1 - rocs[ii][1], label=names[ii]) 345 | axins.plot(rocs[ii][0], 1 - rocs[ii][1], label=names[ii]) 346 | pl.legend() 347 | ax.set_xlabel(r'$FMR$') 348 | ax.set_ylabel(r'$1-FNMR$') 349 | ax.set_xlim([-0.02, 1.02]) 350 | ax.set_ylim([-0.02, 1.02]) 351 | axins.set_xticks([0.05, 0.10, 0.15, 0.20, 0.25]) 352 | axins.set_yticks([0.75, 0.80, 0.85, 0.90, 0.95]) 353 | ax.indicate_inset_zoom(axins) 354 | if title is not None: 355 | pl.title(title) 356 | fig.tight_layout() 357 | if savefile is not None: 358 | pl.savefig(savefile) 359 | else: 360 | pl.show() 361 | 362 | def plot_det(rocs, names, title=None, figsize=[6.4, 4.0], savefile=None): 363 | fig = pl.figure(figsize=figsize) 364 | ax = fig.add_subplot() 365 | for ii in range(len(rocs)): 366 | ax.loglog(rocs[ii][0], rocs[ii][1], label=names[ii]) 367 | #axins.loglog(rocs[ii][0], rocs[ii][1], label=names[ii]) 368 | pl.legend(loc='lower left') 369 | pl.grid(which='both', alpha=0.3) 370 | ax.set_xlabel(r'$FMR$') 371 | ax.set_ylabel(r'$FNMR$') 372 | ax.set_xlim([0.001, 1.0]) 373 | ax.set_ylim([0.001, 1.0]) 374 | if title is not None: 375 | pl.title(title) 376 | fig.tight_layout() 377 | if savefile is not None: 378 | pl.savefig(savefile) 379 | else: 380 | pl.show() 381 | --------------------------------------------------------------------------------