├── model ├── __init__.py ├── linear.py ├── adv_gan.py └── resnet.py ├── utils ├── __init__.py ├── load_model.py ├── patch_utils.py ├── load_data.py ├── fr_util.py ├── nce.py ├── knn.py └── predict.py ├── requirements.txt ├── image └── pipeline.png ├── LICENSE ├── train_down_classifier.py ├── README.md ├── test_down_classifier.py ├── gan_per_attack.py └── gan_pat_attack.py /model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.23.3 2 | pandas==0.25.2 3 | tqdm==4.64.1 4 | typing==3.7.4.3 -------------------------------------------------------------------------------- /image/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CGCL-codes/AdvEncoder/HEAD/image/pipeline.png -------------------------------------------------------------------------------- /model/linear.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import torch.nn.functional as F 3 | 4 | class NonLinearClassifier(nn.Module): 5 | def __init__(self, feat_dim=512, num_classes=10): 6 | super(NonLinearClassifier, self).__init__() 7 | self.fc1 = nn.Linear(feat_dim, feat_dim) 8 | self.dropout = nn.Dropout(p=0.2) 9 | self.fc2 = nn.Linear(feat_dim, feat_dim) 10 | self.fc3 = nn.Linear(feat_dim, num_classes) 11 | 12 | def forward(self, features): 13 | features = F.relu(self.fc1(features)) 14 | features = F.relu(self.fc2(self.dropout(features))) 15 | return self.fc3(features) 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 CGCL-codes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /utils/load_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from pathlib import Path 4 | import torch.nn as nn 5 | from model.resnet import resnet18 6 | 7 | # victim model source zoo: https://github.com/vturrisi/solo-learn 8 | 9 | def load_victim(args): 10 | 11 | if args.pre_dataset == 'cifar10': 12 | num_classes = 10 13 | victim_path = os.path.join('victims', 'cifar10', str(args.victim)) 14 | encoder_path = [Path(victim_path) / ckpt for ckpt in os.listdir(Path(victim_path)) if ckpt.endswith(".ckpt")][0] 15 | model = resnet18(num_classes=num_classes) 16 | checkpoint = torch.load(encoder_path) 17 | state_dict = checkpoint['state_dict'] 18 | 19 | new_ckpt = dict() 20 | for k, value in state_dict.items(): 21 | if k.startswith('backbone'): 22 | new_ckpt[k.replace('backbone.', '')] = value 23 | elif k.startswith('classifier'): 24 | new_ckpt[k.replace('classifier', 'fc')] = value 25 | if True: 26 | model.conv1 = torch.nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, 27 | bias=False) 28 | 29 | model.load_state_dict(new_ckpt) 30 | model.fc = nn.Identity() 31 | model.maxpool = nn.Identity() 32 | 33 | return model 34 | 35 | -------------------------------------------------------------------------------- /utils/patch_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | def patch_initialization(args, patch_type='rectangle'): 5 | noise_percentage = args.noise_percentage 6 | if args.dataset == 'cifar10': 7 | image_size = (3, 64, 64) 8 | else: image_size = (3, 64, 64) 9 | if patch_type == 'rectangle': 10 | mask_length = int((noise_percentage * image_size[1] * image_size[2])**0.5) 11 | patch = np.random.rand(image_size[0], mask_length, mask_length) 12 | return patch 13 | 14 | def mask_generation(args, patch): 15 | if args.dataset == 'cifar10': 16 | image_size = (3, 64, 64) 17 | else: image_size = (3, 64, 64) 18 | applied_patch = np.zeros(image_size) 19 | x_location = image_size[1] - 14 - patch.shape[1] 20 | y_location = image_size[1] - 14 - patch.shape[2] 21 | 22 | applied_patch[:, x_location: x_location + patch.shape[1], y_location: y_location + patch.shape[2]] = patch 23 | mask = applied_patch.copy() 24 | mask[mask != 0] = 1.0 25 | return mask, applied_patch ,x_location, y_location 26 | 27 | 28 | def clamp_patch(args, patch): 29 | # # Cifar 10 30 | if args.dataset == 'cifar10': 31 | mean = (0.485, 0.456, 0.406) 32 | std = (0.229, 0.224, 0.225) 33 | elif args.dataset == 'stl10': 34 | mean = (0.44087798, 0.42790666, 0.38678814) 35 | std = (0.25507198, 0.24801506, 0.25641308) 36 | elif args.dataset == 'gtsrb': 37 | mean = (0.44087798, 0.42790666, 0.38678814) 38 | std = (0.25507198, 0.24801506, 0.25641308) 39 | elif args.dataset == 'imagenet': 40 | mean = (0.485, 0.456, 0.406) 41 | std = (0.229, 0.224, 0.225) 42 | else: 43 | mean = (0.485, 0.456, 0.406) 44 | std = (0.229, 0.224, 0.225) 45 | 46 | min_in = np.array([0, 0, 0]) 47 | max_in = np.array([1, 1, 1]) 48 | min_out, max_out = np.min((min_in - mean) / std), np.max((max_in - mean) / std) 49 | patch = torch.clamp(patch, min=min_out, max=max_out) 50 | return patch -------------------------------------------------------------------------------- /utils/load_data.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as transforms 3 | from torchvision import datasets 4 | from torch.utils.data import DataLoader 5 | 6 | def load_data(data, bs): 7 | 8 | transform_ = transforms.Compose([transforms.Resize((64, 64)), transforms.ToTensor()]) 9 | 10 | if data == 'cifar10': 11 | train_dataset = datasets.CIFAR10('../Dataset/data/cifar10/', train=True, download=False, transform=transform_) 12 | test_dataset = datasets.CIFAR10('../Dataset/data/cifar10/', train=False, download=False, transform=transform_) 13 | 14 | elif data == 'stl10': 15 | train_dataset = datasets.STL10('../Dataset/data/stl10', split="train", download=True, transform=transform_) 16 | test_dataset = datasets.STL10('../Dataset/data/stl10', split="test", download=True, transform=transform_) 17 | 18 | elif data == 'gtsrb': 19 | train_dataset = datasets.ImageFolder('../Dataset/GTSRB/Train/', transform = transform_) 20 | train_dataset, test_dataset = torch.utils.data.random_split(train_dataset, [30000, len(train_dataset)-30000]) 21 | 22 | print('Train dataset: %d, Test dataset: %d'%(len(train_dataset),len(test_dataset))) 23 | 24 | train_loader = DataLoader(train_dataset, batch_size=bs, num_workers=5, drop_last=False, shuffle=True) 25 | test_loader = DataLoader(test_dataset, batch_size=bs, num_workers=5, drop_last=False, shuffle = False) 26 | 27 | return train_loader, test_loader 28 | 29 | 30 | 31 | def normalzie(args, x): 32 | 33 | if args.dataset == 'cifar10': 34 | mean = (0.485, 0.456, 0.406) 35 | std = (0.229, 0.224, 0.225) 36 | normalizer = transforms.Normalize(mean=mean, std=std) 37 | return normalizer(x) 38 | 39 | elif args.dataset == 'stl10': 40 | mean = (0.44087798, 0.42790666, 0.38678814) 41 | std = (0.25507198, 0.24801506, 0.25641308) 42 | normalizer = transforms.Normalize(mean=mean, std=std) 43 | return normalizer(x) 44 | 45 | elif args.dataset == 'gtsrb': 46 | mean = (0.44087798, 0.42790666, 0.38678814) 47 | std = (0.25507198, 0.24801506, 0.25641308) 48 | normalizer = transforms.Normalize(mean=mean, std=std) 49 | return normalizer(x) 50 | 51 | else: 52 | return x 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /utils/fr_util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | import torch.distributed as dist 5 | 6 | def distance(i, j, imageSize, r): 7 | dis = np.sqrt((i - imageSize / 2) ** 2 + (j - imageSize / 2) ** 2) 8 | if dis < r: 9 | return 1.0 10 | else: 11 | return 0 12 | 13 | def mask_radial(img, r): 14 | rows, cols = img.shape 15 | mask = torch.zeros((rows, cols)) 16 | for i in range(rows): 17 | for j in range(cols): 18 | mask[i, j] = distance(i, j, imageSize=rows, r=r) 19 | return mask.cuda() 20 | 21 | def generate_high(Images, r): 22 | # Image: bsxcxhxw, input batched images 23 | # r: int, radius 24 | mask = mask_radial(torch.zeros([Images.shape[2], Images.shape[3]]), r) 25 | bs, c, h, w = Images.shape 26 | x = Images.reshape([bs * c, h, w]) 27 | fd = torch.fft.fftshift(torch.fft.fftn(x, dim=(-2, -1))) 28 | mask = mask.unsqueeze(0).repeat([bs * c, 1, 1]) 29 | fd = fd * (1.-mask) 30 | fd = torch.fft.ifftn(torch.fft.ifftshift(fd), dim=(-2, -1)) 31 | fd = torch.real(fd) 32 | fd = fd.reshape([bs, c, h, w]) 33 | return fd 34 | 35 | def get_rank(): 36 | if dist.is_available() and dist.is_initialized(): 37 | return dist.get_rank() 38 | return 0 39 | 40 | class GatherLayer(torch.autograd.Function): 41 | """ 42 | Gathers tensors from all process and supports backward propagation 43 | for the gradients across processes. 44 | """ 45 | 46 | @staticmethod 47 | def forward(ctx, x): 48 | if dist.is_available() and dist.is_initialized(): 49 | output = [torch.zeros_like(x) for _ in range(dist.get_world_size())] 50 | dist.all_gather(output, x) 51 | else: 52 | output = [x] 53 | return tuple(output) 54 | 55 | @staticmethod 56 | def backward(ctx, *grads): 57 | if dist.is_available() and dist.is_initialized(): 58 | all_gradients = torch.stack(grads) 59 | dist.all_reduce(all_gradients) 60 | grad_out = all_gradients[get_rank()] 61 | else: 62 | grad_out = grads[0] 63 | return grad_out 64 | 65 | 66 | def gather(X, dim=0): 67 | """Gathers tensors from all processes, supporting backward propagation.""" 68 | return torch.cat(GatherLayer.apply(X), dim=dim) 69 | 70 | -------------------------------------------------------------------------------- /model/adv_gan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class Generator(torch.nn.Module): 4 | def __init__(self, input_dim, num_filters, output_dim, batch_size): 5 | super(Generator, self).__init__() 6 | 7 | # Hidden layers 8 | self.hidden_layer = torch.nn.Sequential() 9 | for i in range(len(num_filters)): 10 | # Deconvolutional layer 11 | if i == 0: 12 | deconv = torch.nn.ConvTranspose2d(input_dim, num_filters[i], kernel_size=4, stride=1, padding=0) 13 | else: 14 | deconv = torch.nn.ConvTranspose2d(num_filters[i-1], num_filters[i], kernel_size=4, stride=2, padding=1) 15 | 16 | deconv_name = 'deconv' + str(i + 1) 17 | self.hidden_layer.add_module(deconv_name, deconv) 18 | 19 | # Initializer 20 | torch.nn.init.normal(deconv.weight, mean=0.0, std=0.02) 21 | torch.nn.init.constant(deconv.bias, 0.0) 22 | 23 | # Batch normalization 24 | bn_name = 'bn' + str(i + 1) 25 | self.hidden_layer.add_module(bn_name, torch.nn.BatchNorm2d(num_filters[i])) 26 | 27 | # Activation 28 | act_name = 'act' + str(i + 1) 29 | self.hidden_layer.add_module(act_name, torch.nn.ReLU()) 30 | 31 | # Output layer 32 | self.output_layer = torch.nn.Sequential() 33 | # Deconvolutional layer 34 | out = torch.nn.ConvTranspose2d(num_filters[i], output_dim, kernel_size=4, stride=2, padding=1) 35 | self.output_layer.add_module('out', out) 36 | # Initializer 37 | torch.nn.init.normal(out.weight, mean=0.0, std=0.02) 38 | torch.nn.init.constant(out.bias, 0.0) 39 | # Activation 40 | self.output_layer.add_module('act', torch.nn.Tanh()) 41 | 42 | # Residual layer 43 | self.residual_layer = torch.nn.Sequential() 44 | res = torch.nn.Conv2d(in_channels=batch_size,out_channels=1,kernel_size=3,stride=1,padding=1) 45 | self.residual_layer.add_module('res', res) 46 | torch.nn.init.normal(res.weight, mean=0.0, std=0.02) 47 | torch.nn.init.constant(res.bias, 0.0) 48 | 49 | def forward(self, x): 50 | h = self.hidden_layer(x) 51 | out = self.output_layer(h) 52 | out_ = out.permute(1,0,2,3) 53 | adv_out = self.residual_layer(out_) 54 | adv_out_ = adv_out.permute(1,0,2,3) 55 | return adv_out_ 56 | -------------------------------------------------------------------------------- /train_down_classifier.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import argparse 4 | import time 5 | import random 6 | import json 7 | import numpy as np 8 | from utils.load_model import load_victim 9 | from utils.load_data import load_data, normalzie 10 | from utils.predict import accuracy, test, make_print_to_file 11 | from model.linear import NonLinearClassifier 12 | 13 | 14 | def arg_parse(): 15 | parser = argparse.ArgumentParser(description='Train downstream models using of the pre-trained encoder') 16 | parser.add_argument('--seed', default=100, type=int, help='which seed the code runs on') 17 | parser.add_argument('--gpu', default='1', type=str, help='which gpu the code runs on') 18 | parser.add_argument('--dataset', default='stl10', choices=['cifar10', 'stl10', 'gtsrb', 'imagenet']) 19 | parser.add_argument('--batch_size', type=int, default=256) 20 | parser.add_argument('--epochs', default=20, type=int) 21 | parser.add_argument('--pre_dataset', default='cifar10', choices=['cifar10', 'imagenet']) 22 | parser.add_argument('--victim', default='simclr', choices=['simclr', 'byol', 'barlow_twins', 'deepclusterv2', 'dino', 'mocov3', 'mocov2plus', 'nnclr', 'ressl', 'simsiam', 'supcon', 'swav', 'bivcreg', 'vicreg', 'wmse']) 23 | args = parser.parse_args() 24 | return args 25 | 26 | def classify(args, encoder): 27 | data = args.dataset 28 | train_loader, test_loader = load_data(data, args.batch_size) 29 | 30 | # save uap result 31 | uap_save_path = os.path.join('victims', str(args.pre_dataset), str(args.victim), 'clean_model', str(args.dataset)) 32 | 33 | if not os.path.exists(uap_save_path): 34 | os.makedirs(uap_save_path) 35 | 36 | # downstream task 37 | if args.dataset == 'imagenet': 38 | num_classes = 100 39 | args.epochs = 50 40 | elif args.dataset == 'gtsrb': 41 | num_classes = 43 42 | else: 43 | num_classes = 10 44 | 45 | F = NonLinearClassifier(feat_dim=512, num_classes=num_classes) 46 | F.cuda() 47 | encoder.cuda() 48 | 49 | # classifier 50 | my_optimizer = torch.optim.Adam(F.parameters(), lr=0.005, weight_decay=0.0008) 51 | my_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=my_optimizer, gamma=0.96) 52 | criterion = torch.nn.CrossEntropyLoss().cuda() 53 | 54 | F.train() 55 | encoder.eval() 56 | 57 | for epoch in range(args.epochs): 58 | start = time.time() 59 | top1_train_accuracy = 0 60 | for counter, (x_batch, y_batch) in enumerate(train_loader): 61 | my_optimizer.zero_grad() 62 | x_batch = x_batch.cuda() 63 | y_batch = y_batch.cuda() 64 | h = encoder(normalzie(args, x_batch)) 65 | downstream_input = h.view(h.size(0), -1) 66 | logits = F(downstream_input) 67 | loss = criterion(logits, y_batch) 68 | top1 = accuracy(logits, y_batch, topk=(1,)) 69 | top1_train_accuracy += top1[0] 70 | loss.backward() 71 | my_optimizer.step() 72 | 73 | end = time.time() 74 | F.train() 75 | clean_acc_t1, clean_acc_t5 = test(args, encoder, F, test_loader, data) 76 | torch.save(F, '{}/{}'.format(uap_save_path, str(args.victim) + '_' +str(args.pre_dataset) + '_' + str(args.dataset) + '_' + str( 77 | round(clean_acc_t1, 4)) + '_' + str(epoch + 1) + '.pth')) 78 | 79 | my_lr_scheduler.step() 80 | top1_train_accuracy /= (counter + 1) 81 | print('Epoch [%d/%d], Top1 train acc: %.4f, Top1 test acc: %.4f, Time: %.4f' 82 | % (epoch + 1, args.epochs, top1_train_accuracy.item(), clean_acc_t1,(end - start))) 83 | 84 | 85 | return clean_acc_t1, clean_acc_t5 86 | 87 | def main(): 88 | args = arg_parse() 89 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 90 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu 91 | random.seed(args.seed) 92 | os.environ['PYTHONHASHSEED'] = str(args.seed) 93 | np.random.seed(args.seed) 94 | torch.manual_seed(args.seed) 95 | torch.cuda.manual_seed(args.seed) 96 | torch.backends.cudnn.deterministic = True 97 | torch.backends.cudnn.benchmark = True 98 | torch.set_printoptions(profile="full") 99 | torch.cuda.synchronize() 100 | 101 | # Logging 102 | log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'down_test', "clean_model", str(args.victim), str(args.dataset)) 103 | if not os.path.exists(log_save_path): 104 | os.makedirs(log_save_path) 105 | 106 | now_time = make_print_to_file(path=log_save_path) 107 | 108 | if not os.path.exists(log_save_path): 109 | os.makedirs(log_save_path) 110 | 111 | # Dump args 112 | with open(log_save_path + '/args.json', 'w') as fid: 113 | json.dump(args.__dict__, fid, indent=2) 114 | 115 | model = load_victim(args) 116 | 117 | print('Day: %s, Target encoder:%s, Downstream task:%s'% (now_time, args.victim, args.dataset)) 118 | print("###################################### Test Attack! ######################################") 119 | 120 | clean_acc_t1, clean_acc_t5 = classify(args, model) 121 | print('Clean downstream accuracy: %.4f%%'% (clean_acc_t1)) 122 | 123 | 124 | if __name__ == "__main__": 125 | main() 126 | -------------------------------------------------------------------------------- /utils/nce.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | __all__ = ['InfoNCE', 'info_nce'] 6 | 7 | 8 | class InfoNCE(nn.Module): 9 | """ 10 | Calculates the InfoNCE loss for self-supervised learning. 11 | This contrastive loss enforces the embeddings of similar (positive) samples to be close 12 | and those of different (negative) samples to be distant. 13 | A query embedding is compared with one positive key and with one or more negative keys. 14 | 15 | References: 16 | https://arxiv.org/abs/1807.03748v2 17 | https://arxiv.org/abs/2010.05113 18 | 19 | Args: 20 | temperature: Logits are divided by temperature before calculating the cross entropy. 21 | reduction: Reduction method applied to the output. 22 | Value must be one of ['none', 'sum', 'mean']. 23 | See torch.nn.functional.cross_entropy for more details about each option. 24 | negative_mode: Determines how the (optional) negative_keys are handled. 25 | Value must be one of ['paired', 'unpaired']. 26 | If 'paired', then each query sample is paired with a number of negative keys. 27 | Comparable to a triplet loss, but with multiple negatives per sample. 28 | If 'unpaired', then the set of negative keys are all unrelated to any positive key. 29 | 30 | Input shape: 31 | query: (N, D) Tensor with query samples (e.g. embeddings of the input). 32 | positive_key: (N, D) Tensor with positive samples (e.g. embeddings of augmented input). 33 | negative_keys (optional): Tensor with negative samples (e.g. embeddings of other inputs) 34 | If negative_mode = 'paired', then negative_keys is a (N, M, D) Tensor. 35 | If negative_mode = 'unpaired', then negative_keys is a (M, D) Tensor. 36 | If None, then the negative keys for a sample are the positive keys for the other samples. 37 | 38 | Returns: 39 | Value of the InfoNCE Loss. 40 | 41 | Examples: 42 | >>> loss = InfoNCE() 43 | >>> batch_size, num_negative, embedding_size = 32, 48, 128 44 | >>> query = torch.randn(batch_size, embedding_size) 45 | >>> positive_key = torch.randn(batch_size, embedding_size) 46 | >>> negative_keys = torch.randn(num_negative, embedding_size) 47 | >>> output = loss(query, positive_key, negative_keys) 48 | """ 49 | 50 | def __init__(self, temperature=0.1, reduction='mean', negative_mode='unpaired'): 51 | super().__init__() 52 | self.temperature = temperature 53 | self.reduction = reduction 54 | self.negative_mode = negative_mode 55 | 56 | def forward(self, query, positive_key, negative_keys=None): 57 | return info_nce(query, positive_key, negative_keys, 58 | temperature=self.temperature, 59 | reduction=self.reduction, 60 | negative_mode=self.negative_mode) 61 | 62 | 63 | def info_nce(query, positive_key, negative_keys=None, temperature=0.1, reduction='mean', negative_mode='unpaired'): 64 | # Check input dimensionality. 65 | if query.dim() != 2: 66 | raise ValueError(' must have 2 dimensions.') 67 | if positive_key.dim() != 2: 68 | raise ValueError(' must have 2 dimensions.') 69 | if negative_keys is not None: 70 | if negative_mode == 'unpaired' and negative_keys.dim() != 2: 71 | raise ValueError(" must have 2 dimensions if == 'unpaired'.") 72 | if negative_mode == 'paired' and negative_keys.dim() != 3: 73 | raise ValueError(" must have 3 dimensions if == 'paired'.") 74 | 75 | # Check matching number of samples. 76 | if len(query) != len(positive_key): 77 | raise ValueError(' and must must have the same number of samples.') 78 | if negative_keys is not None: 79 | if negative_mode == 'paired' and len(query) != len(negative_keys): 80 | raise ValueError("If negative_mode == 'paired', then must have the same number of samples as .") 81 | 82 | # Embedding vectors should have same number of components. 83 | if query.shape[-1] != positive_key.shape[-1]: 84 | raise ValueError('Vectors of and should have the same number of components.') 85 | if negative_keys is not None: 86 | if query.shape[-1] != negative_keys.shape[-1]: 87 | raise ValueError('Vectors of and should have the same number of components.') 88 | 89 | # Normalize to unit vectors 90 | query, positive_key, negative_keys = normalize(query, positive_key, negative_keys) 91 | if negative_keys is not None: 92 | # Explicit negative keys 93 | 94 | # Cosine between positive pairs 95 | positive_logit = torch.sum(query * positive_key, dim=1, keepdim=True) 96 | 97 | if negative_mode == 'unpaired': 98 | # Cosine between all query-negative combinations 99 | negative_logits = query @ transpose(negative_keys) 100 | 101 | elif negative_mode == 'paired': 102 | query = query.unsqueeze(1) 103 | negative_logits = query @ transpose(negative_keys) 104 | negative_logits = negative_logits.squeeze(1) 105 | 106 | # First index in last dimension are the positive samples 107 | logits = torch.cat([positive_logit, negative_logits], dim=1) 108 | labels = torch.zeros(len(logits), dtype=torch.long, device=query.device) 109 | else: 110 | # Negative keys are implicitly off-diagonal positive keys. 111 | 112 | # Cosine between all combinations 113 | logits = query @ transpose(positive_key) 114 | 115 | # Positive keys are the entries on the diagonal 116 | labels = torch.arange(len(query), device=query.device) 117 | 118 | return F.cross_entropy(logits / temperature, labels, reduction=reduction) 119 | 120 | 121 | def transpose(x): 122 | return x.transpose(-2, -1) 123 | 124 | 125 | def normalize(*xs): 126 | return [None if x is None else F.normalize(x, dim=-1) for x in xs] 127 | -------------------------------------------------------------------------------- /utils/knn.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | import torch 3 | import torch.nn.functional as F 4 | # from metric import Metric 5 | 6 | from typing import Dict, List, Sequence 7 | 8 | def accuracy_at_k( 9 | outputs: torch.Tensor, targets: torch.Tensor, top_k: Sequence[int] = (1, 5) 10 | ) -> Sequence[int]: 11 | """Computes the accuracy over the k top predictions for the specified values of k. 12 | 13 | Args: 14 | outputs (torch.Tensor): output of a classifier (logits or probabilities). 15 | targets (torch.Tensor): ground truth labels. 16 | top_k (Sequence[int], optional): sequence of top k values to compute the accuracy over. 17 | Defaults to (1, 5). 18 | 19 | Returns: 20 | Sequence[int]: accuracies at the desired k. 21 | """ 22 | 23 | with torch.no_grad(): 24 | maxk = max(top_k) 25 | batch_size = targets.size(0) 26 | 27 | _, pred = outputs.topk(maxk, 1, True, True) 28 | pred = pred.t() 29 | correct = pred.eq(targets.view(1, -1).expand_as(pred)) 30 | 31 | res = [] 32 | for k in top_k: 33 | correct_k = correct[:k].contiguous().view(-1).float().sum(0, keepdim=True) 34 | res.append(correct_k.mul_(100.0 / batch_size)) 35 | return res 36 | 37 | 38 | def weighted_mean(outputs: List[Dict], key: str, batch_size_key: str) -> float: 39 | """Computes the mean of the values of a key weighted by the batch size. 40 | 41 | Args: 42 | outputs (List[Dict]): list of dicts containing the outputs of a validation step. 43 | key (str): key of the metric of interest. 44 | batch_size_key (str): key of batch size values. 45 | 46 | Returns: 47 | float: weighted mean of the values of a key 48 | """ 49 | 50 | value = 0 51 | n = 0 52 | for out in outputs: 53 | value += out[batch_size_key] * out[key] 54 | n += out[batch_size_key] 55 | value = value / n 56 | return value.squeeze(0) 57 | 58 | 59 | class WeightedKNNClassifier(object): 60 | def __init__( 61 | self, 62 | k: int = 20, 63 | T: float = 0.07, 64 | max_distance_matrix_size: int = int(5e6), 65 | distance_fx: str = "cosine", 66 | epsilon: float = 0.00001, 67 | dist_sync_on_step: bool = False, 68 | ): 69 | """Implements the weighted k-NN classifier used for evaluation. 70 | 71 | Args: 72 | k (int, optional): number of neighbors. Defaults to 20. 73 | T (float, optional): temperature for the exponential. Only used with cosine 74 | distance. Defaults to 0.07. 75 | max_distance_matrix_size (int, optional): maximum number of elements in the 76 | distance matrix. Defaults to 5e6. 77 | distance_fx (str, optional): Distance function. Accepted arguments: "cosine" or 78 | "euclidean". Defaults to "cosine". 79 | epsilon (float, optional): Small value for numerical stability. Only used with 80 | euclidean distance. Defaults to 0.00001. 81 | dist_sync_on_step (bool, optional): whether to sync distributed values at every 82 | step. Defaults to False. 83 | """ 84 | 85 | super().__init__(dist_sync_on_step=dist_sync_on_step, compute_on_step=False) 86 | 87 | self.k = k 88 | self.T = T 89 | self.max_distance_matrix_size = max_distance_matrix_size 90 | self.distance_fx = distance_fx 91 | self.epsilon = epsilon 92 | 93 | self.add_state("train_features", default=[], persistent=False) 94 | self.add_state("train_targets", default=[], persistent=False) 95 | self.add_state("test_features", default=[], persistent=False) 96 | self.add_state("test_targets", default=[], persistent=False) 97 | 98 | def update( 99 | self, 100 | train_features: torch.Tensor = None, 101 | train_targets: torch.Tensor = None, 102 | test_features: torch.Tensor = None, 103 | test_targets: torch.Tensor = None, 104 | ): 105 | """Updates the memory banks. If train (test) features are passed as input, the 106 | corresponding train (test) targets must be passed as well. 107 | 108 | Args: 109 | train_features (torch.Tensor, optional): a batch of train features. Defaults to None. 110 | train_targets (torch.Tensor, optional): a batch of train targets. Defaults to None. 111 | test_features (torch.Tensor, optional): a batch of test features. Defaults to None. 112 | test_targets (torch.Tensor, optional): a batch of test targets. Defaults to None. 113 | """ 114 | assert (train_features is None) == (train_targets is None) 115 | assert (test_features is None) == (test_targets is None) 116 | 117 | if train_features is not None: 118 | assert train_features.size(0) == train_targets.size(0) 119 | self.train_features.append(train_features.detach()) 120 | self.train_targets.append(train_targets.detach()) 121 | 122 | if test_features is not None: 123 | assert test_features.size(0) == test_targets.size(0) 124 | self.test_features.append(test_features.detach()) 125 | self.test_targets.append(test_targets.detach()) 126 | 127 | @torch.no_grad() 128 | def compute(self) -> Tuple[float]: 129 | """Computes weighted k-NN accuracy @1 and @5. If cosine distance is selected, 130 | the weight is computed using the exponential of the temperature scaled cosine 131 | distance of the samples. If euclidean distance is selected, the weight corresponds 132 | to the inverse of the euclidean distance. 133 | 134 | Returns: 135 | Tuple[float]: k-NN accuracy @1 and @5. 136 | """ 137 | 138 | train_features = torch.cat(self.train_features) 139 | train_targets = torch.cat(self.train_targets) 140 | test_features = torch.cat(self.test_features) 141 | test_targets = torch.cat(self.test_targets) 142 | 143 | if self.distance_fx == "cosine": 144 | train_features = F.normalize(train_features) 145 | test_features = F.normalize(test_features) 146 | 147 | num_classes = torch.unique(test_targets).numel() 148 | num_train_images = train_targets.size(0) 149 | num_test_images = test_targets.size(0) 150 | num_train_images = train_targets.size(0) 151 | chunk_size = min( 152 | max(1, self.max_distance_matrix_size // num_train_images), 153 | num_test_images, 154 | ) 155 | k = min(self.k, num_train_images) 156 | 157 | top1, top5, total = 0.0, 0.0, 0 158 | retrieval_one_hot = torch.zeros(k, num_classes).to(train_features.device) 159 | for idx in range(0, num_test_images, chunk_size): 160 | # get the features for test images 161 | features = test_features[idx : min((idx + chunk_size), num_test_images), :] 162 | targets = test_targets[idx : min((idx + chunk_size), num_test_images)] 163 | batch_size = targets.size(0) 164 | 165 | # calculate the dot product and compute top-k neighbors 166 | if self.distance_fx == "cosine": 167 | similarities = torch.mm(features, train_features.t()) 168 | elif self.distance_fx == "euclidean": 169 | similarities = 1 / (torch.cdist(features, train_features) + self.epsilon) 170 | else: 171 | raise NotImplementedError 172 | 173 | similarities, indices = similarities.topk(k, largest=True, sorted=True) 174 | candidates = train_targets.view(1, -1).expand(batch_size, -1) 175 | retrieved_neighbors = torch.gather(candidates, 1, indices) 176 | 177 | retrieval_one_hot.resize_(batch_size * k, num_classes).zero_() 178 | retrieval_one_hot.scatter_(1, retrieved_neighbors.view(-1, 1), 1) 179 | 180 | if self.distance_fx == "cosine": 181 | similarities = similarities.clone().div_(self.T).exp_() 182 | 183 | probs = torch.sum( 184 | torch.mul( 185 | retrieval_one_hot.view(batch_size, -1, num_classes), 186 | similarities.view(batch_size, -1, 1), 187 | ), 188 | 1, 189 | ) 190 | _, predictions = probs.sort(1, True) 191 | 192 | # find the predictions that match the target 193 | correct = predictions.eq(targets.data.view(-1, 1)) 194 | top1 = top1 + correct.narrow(1, 0, 1).sum().item() 195 | top5 = ( 196 | top5 + correct.narrow(1, 0, min(5, k, correct.size(-1))).sum().item() 197 | ) # top5 does not make sense if k < 5 198 | total += targets.size(0) 199 | 200 | top1 = top1 * 100.0 / total 201 | top5 = top5 * 100.0 / total 202 | 203 | self.reset() 204 | 205 | return top1, top5 206 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AdvEncoder 2 | The implementation of our ICCV 2023 paper "Downstream-agnostic Adversarial Examples" 3 | 4 | ![Python 3.8](https://img.shields.io/badge/python-3.8-green.svg?style=plastic) 5 | ![Pytorch 1.8.0](https://img.shields.io/badge/pytorch-1.8.0-red.svg?style=plastic) 6 | 7 | ## Abstract 8 | Self-supervised learning usually uses a large amount of unlabeled data to pre-train an encoder which can be used as a general-purpose feature extractor, such that downstream users only need to perform fine-tuning operations to enjoy the benefit of ``large model". Despite this promising prospect, the security of pre-trained encoder has not been thoroughly investigated yet, especially when the pre-trained encoder is publicly available for commercial use. 9 | 10 | In this paper, we propose AdvEncoder, the first framework for generating downstream-agnostic universal adversarial examples based on the pre-trained encoder. AdvEncoder aims to construct a universal adversarial perturbation or patch for a set of natural images that can fool all the downstream tasks inheriting the victim pre-trained encoder. Unlike traditional adversarial example works, the pre-trained encoder only outputs feature vectors rather than classification labels. Therefore, we first exploit the high frequency component information of the image to guide the generation of adversarial examples. Then we design a generative attack framework to construct adversarial perturbations/patches by learning the distribution of the attack surrogate dataset to improve their attack success rates and transferability. Our results show that an attacker can successfully attack downstream tasks without knowing either the pre-training dataset or the downstream dataset. We also tailor four defenses for pre-trained encoders, the results of which further prove the attack ability of AdvEncoder. 11 | 12 | 13 | 14 | ## Latest Update 15 | **2023/8/24** We have released the official implementation code. 16 | 17 | ## Setup 18 | - **Get code** 19 | ```shell 20 | git clone https://github.com/CGCL-codes/AdvEncoder.git 21 | ``` 22 | 23 | - **Build environment** 24 | ```shell 25 | cd AdvEncoder 26 | # use anaconda to build environment 27 | conda create -n AdvEncoder python=3.8 28 | conda activate AdvEncoder 29 | # install packages 30 | pip install -r requirements.txt 31 | ``` 32 | - **The final project should be like this:** 33 | ```shell 34 | AdvEncoder 35 | └- utils 36 | └- predict 37 | └- model 38 | └- adv_gan 39 | └- data 40 | └- victims 41 | └- cifar10 (pre-training dataset) 42 | └- simclr 43 | └- simclr-cifar10-b30xch14-ep=999.ckpt 44 | └- clean_model 45 | └- output 46 | └- ... 47 | ``` 48 | 49 | - **Download Victim Pre-trained Encoders** 50 | - All of our pre-trained encoders were obtained from the [solo-learn](https://github.com/vturrisi/solo-learn) repository, and some missing pre-trained encoders were trained by us based on their code. 51 | - Please move the downloaded pre-trained encoder into /victims/[pre-dataset]/[method]. 52 | 53 | 54 | ### CIFAR-10 55 | 56 | | Method | Backbone | Epochs | Acc@1 | Acc@5 | Checkpoint | 57 | |--------------|:--------:|:------:|:--------------:|:--------------:|:----------:| 58 | | Barlow Twins | ResNet18 | 1000 | 92.10 | 99.73 | [Link](https://drive.google.com/drive/folders/1L5RAM3lCSViD2zEqLtC-GQKVw6mxtxJ_?usp=sharing) | 59 | | BYOL | ResNet18 | 1000 | 92.58 | 99.79 | [Link](https://drive.google.com/drive/folders/1KxeYAEE7Ev9kdFFhXWkPZhG-ya3_UwGP?usp=sharing) | 60 | |DeepCluster V2| ResNet18 | 1000 | 88.85 | 99.58 | [Link](https://drive.google.com/drive/folders/1tkEbiDQ38vZaQUsT6_vEpxbDxSUAGwF-?usp=sharing) | 61 | | DINO | ResNet18 | 1000 | 89.52 | 99.71 | [Link](https://drive.google.com/drive/folders/1vyqZKUyP8sQyEyf2cqonxlGMbQC-D1Gi?usp=sharing) | 62 | | MoCo V2+ | ResNet18 | 1000 | 92.94 | 99.79 | [Link](https://drive.google.com/drive/folders/1ruNFEB3F-Otxv2Y0p62wrjA4v5Fr2cKC?usp=sharing) | 63 | | MoCo V3 | ResNet18 | 1000 | 93.10 | 99.80 | [Link](https://drive.google.com/drive/folders/1KwZTshNEpmqnYJcmyYPvfIJ_DNwqtAVj?usp=sharing) | 64 | | NNCLR | ResNet18 | 1000 | 91.88 | 99.78 | [Link](https://drive.google.com/drive/folders/1xdCzhvRehPmxinphuiZqFlfBwfwWDcLh?usp=sharing) | 65 | | ReSSL | ResNet18 | 1000 | 90.63 | 99.62 | [Link](https://drive.google.com/drive/folders/1jrFcztY2eO_fG98xPshqOD15pDIhLXp-?usp=sharing) | 66 | | SimCLR | ResNet18 | 1000 | 90.74 | 99.75 | [Link](https://drive.google.com/drive/folders/1mcvWr8P2WNJZ7TVpdLHA_Q91q4VK3y8O?usp=sharing) | 67 | | SupCon | ResNet18 | 1000 | 93.82 | 99.65 | [Link](https://drive.google.com/drive/folders/1VwZ9TrJXCpnxyo7P_l397yGrGH-DAUv-?usp=sharing) | 68 | | SwAV | ResNet18 | 1000 | 89.17 | 99.68 | [Link](https://drive.google.com/drive/folders/1nlJH4Ljm8-5fOIeAaKppQT6gtsmmW1T0?usp=sharing) | 69 | | VIbCReg | ResNet18 | 1000 | 91.18 | 99.74 | [Link](https://drive.google.com/drive/folders/1XvxUOnLPZlC_-OkeuO7VqXT7z9_tNVk7?usp=sharing) | 70 | | VICReg | ResNet18 | 1000 | 92.07 | 99.74 | [Link](https://drive.google.com/drive/folders/159ZgCxocB7aaHxwNDubnAWU71zXV9hn-?usp=sharing) | 71 | | W-MSE | ResNet18 | 1000 | 88.67 | 99.68 | [Link](https://drive.google.com/drive/folders/1xPCiULzQ4JCmhrTsbxBp9S2jRZ01KiVM?usp=sharing) | 72 | 73 | 74 | ### ImageNet-100 75 | 76 | | Method | Backbone | Epochs | Acc@1 | Acc@5| Checkpoint | 77 | |-------------------------|:--------:|:------:|:--------------:|:---------------:|:----------:| 78 | | Barlow Twins| ResNet18 | 400 | 80.38 | 95.28 | [Link](https://drive.google.com/drive/folders/1rj8RbER9E71mBlCHIZEIhKPUFn437D5O?usp=sharing) | 79 | | BYOL | ResNet18 | 400 | 80.16 | 95.02 | [Link](https://drive.google.com/drive/folders/1riOLjMawD_znO4HYj8LBN2e1X4jXpDE1?usp=sharing) | 80 | | DeepCluster V2 | ResNet18 | 400 |75.36 | 93.22 | [Link](https://drive.google.com/drive/folders/1d5jPuavrQ7lMlQZn5m2KnN5sPMGhHFo8?usp=sharing) | 81 | | DINO | ResNet18 | 400 | 74.84 | 92.92 | [Link](https://drive.google.com/drive/folders/1NtVvRj-tQJvrMxRlMtCJSAecQnYZYkqs?usp=sharing) | 82 | | MoCo V2+ | ResNet18 | 400 | 78.20 | 95.50 | [Link](https://drive.google.com/drive/folders/1ItYBtMJ23Yh-Rhrvwjm4w1waFfUGSoKX?usp=sharing) | 83 | | MoCo V3 | ResNet18 | 400 | 80.36 | 95.18 | [Link](https://drive.google.com/drive/folders/15J0JiZsQAsrQler8mbbio-desb_nVoD1?usp=sharing) | 84 | | NNCLR | ResNet18 | 400 | 79.80 | 95.28 | [Link](https://drive.google.com/drive/folders/1QMkq8w3UsdcZmoNUIUPgfSCAZl_LSNjZ?usp=sharing) | 85 | | ReSSL | ResNet18 | 400 | 76.92 | 94.20 | [Link](https://drive.google.com/drive/folders/1urWIFACLont4GAduis6l0jcEbl080c9U?usp=sharing) | 86 | | SimCLR | ResNet18 | 400 | 77.64 | 94.06 | [Link](https://drive.google.com/drive/folders/1yxAVKnc8Vf0tDfkixSB5mXe7dsA8Ll37?usp=sharing) | 87 | | SupCon | ResNet18 | 400 | 84.40 | 95.72 | [Link](https://drive.google.com/drive/folders/1BzR0nehkCKpnLhi-oeDynzzUcCYOCUJi?usp=sharing) | 88 | | SwAV | ResNet18 | 400 | 74.04 | 92.70 | [Link](https://drive.google.com/drive/folders/1VWCMM69sokzjVoPzPSLIsUy5S2Rrm1xJ?usp=sharing) | 89 | | VIbCReg | ResNet18 | 400 | 79.86 | 94.98 | [Link](https://drive.google.com/drive/folders/1Q06hH18usvRwj2P0bsmoCkjNUX_0syCK?usp=sharing) | 90 | | VICReg | ResNet18 | 400 | 79.22 | 95.06 | [Link](https://drive.google.com/drive/folders/1uWWR5VBUru8vaHaGeLicS6X3R4CfZsr2?usp=sharing) | 91 | | W-MSE | ResNet18 | 400 | 67.60 | 90.94 | [Link](https://drive.google.com/drive/folders/1TxubagNV4z5Qs7SqbBcyRHWGKevtFO5l?usp=sharing) | 92 | 93 | 94 | 95 | 96 | ## Quick Start 97 | - **Train AdvEnoder-PER** 98 | ```shell 99 | python gan_per_attack.py # results saved in /output/[pre-dataset]/uap_results/gan_per 100 | ``` 101 | - **Train AdvEnoder-PAT** 102 | ```shell 103 | python gan_pat_attack.py # results saved in /output/[pre-dataset]/uap_results/gan_patch 104 | ``` 105 | - **Train downstream classifiter** 106 | ```shell 107 | python train_down_classifier.py # clean models saved in /victims/[pre-dataset]/[victim-encoder]/clean_model 108 | ``` 109 | - **Test performance of AdvEncoder** 110 | ```shell 111 | python test_down_classifier.py # results saved in /output/[pre-dataset]/log/down_test 112 | ``` 113 | 114 | ## Acknowledge 115 | 116 | We greatly appreciate the immense contribution of the [solo-learn](https://github.com/vturrisi/solo-learn) team in the field of self-supervised learning by open-sourcing their repository. 117 | 118 | ## BibTeX 119 | If you find AdvEncoder both interesting and helpful, please consider citing us in your research or publications: 120 | ```bibtex 121 | @inproceedings{zhou2023advencoder, 122 | title={Downstream-agnostic Adversarial Examples}, 123 | author={Zhou, Ziqi and Hu, Shengshan and Zhao, Ruizhi and Wang, Qian and Zhang, Leo Yu and Hou, Junhui and Jin, Hai }, 124 | booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV'23)}, 125 | year={2023} 126 | } 127 | ``` 128 | -------------------------------------------------------------------------------- /test_down_classifier.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import torch 4 | import argparse 5 | import random 6 | import json 7 | import numpy as np 8 | from utils.load_model import load_victim 9 | from utils.load_data import load_data 10 | from pathlib import Path 11 | from utils.predict import test, adv_test, fr_test, make_print_to_file 12 | from model.linear import NonLinearClassifier 13 | 14 | def arg_parse(): 15 | parser = argparse.ArgumentParser(description='Test the performance of advencoder') 16 | parser.add_argument('--seed', default=100, type=int, help='which seed the code runs on') 17 | parser.add_argument('--gpu', default='0', type=str, help='which gpu the code runs on') 18 | parser.add_argument('--dataset', default='stl10', choices=['cifar10', 'stl10', 'gtsrb', 'imagenet']) 19 | parser.add_argument('--batch_size', type=int, default=256) 20 | parser.add_argument('--type', default='gan_per', choices=['gan_patch', 'gan_per']) 21 | parser.add_argument('--pre_dataset', default='imagenet', choices=['cifar10', 'imagenet']) 22 | parser.add_argument('--victim', default='mocov2plus', choices=['simclr', 'byol', 'barlow_twins', 'deepclusterv2', 'dino', 'mocov3', 'mocov2plus', 'nnclr', 'ressl', 'simsiam', 'supcon', 'swav', 'vibcreg', 'vicreg', 'wmse']) 23 | parser.add_argument('--sup_dataset', default='cifar10') 24 | parser.add_argument('--criterion', default='nce', choices=['cos', 'nt', 'nce']) 25 | parser.add_argument('--noise_percentage', type=float, default=0.03) 26 | parser.add_argument('--eps', type=int, default=10) 27 | args = parser.parse_args() 28 | return args 29 | 30 | def classify(args, encoder, F, uap): 31 | data = args.dataset 32 | # log_save_path = path 33 | train_loader, test_loader = load_data(data, args.batch_size) 34 | 35 | results = {'clean_acc_t1': [], 'adv_acc_t1': [], 'decline_t1': [], 'clean_acc_t5': [], 'adv_acc_t5': [], 36 | 'decline_t5': [], 'attack_success_rate': []} 37 | 38 | F.cuda() 39 | encoder.cuda() 40 | encoder.eval() 41 | clean_acc_t1, clean_acc_t5 = test(args, encoder, F, test_loader, data) 42 | adv_acc_t1, adv_acc_t5 = adv_test(args, encoder, F, test_loader, uap, data) 43 | attack_success_rate = fr_test(args, encoder, F, test_loader, uap, data) 44 | decline_t1 = ((clean_acc_t1 - adv_acc_t1) / clean_acc_t1) * 100 45 | decline_t5 = ((clean_acc_t5 - adv_acc_t5) / clean_acc_t5) * 100 46 | results['clean_acc_t1'].append(clean_acc_t1) 47 | results['clean_acc_t5'].append(clean_acc_t5) 48 | results['decline_t1'].append(decline_t1) 49 | results['adv_acc_t1'].append(adv_acc_t1) 50 | results['adv_acc_t5'].append(adv_acc_t5) 51 | results['decline_t5'].append(decline_t5) 52 | results['attack_success_rate'].append(attack_success_rate) 53 | 54 | print('Top1 test acc: %.4f, Top1 Adv_test acc: %.4f, Fooling rate: %.4f' 55 | % (clean_acc_t1, adv_acc_t1, attack_success_rate)) 56 | 57 | return clean_acc_t1, adv_acc_t1, decline_t1, attack_success_rate, clean_acc_t5, adv_acc_t5, decline_t5 58 | 59 | def main(): 60 | 61 | args = arg_parse() 62 | # Set the seed and determine the GPU 63 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 64 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu 65 | random.seed(args.seed) 66 | os.environ['PYTHONHASHSEED'] = str(args.seed) 67 | np.random.seed(args.seed) 68 | torch.manual_seed(args.seed) 69 | torch.cuda.manual_seed(args.seed) 70 | torch.backends.cudnn.deterministic = True 71 | torch.backends.cudnn.benchmark = True 72 | torch.set_printoptions(profile="full") 73 | torch.cuda.synchronize() 74 | 75 | args.eps = args.eps / 255 76 | 77 | # Logging 78 | log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'down_test', str(args.type), str(args.victim), str(args.sup_dataset), str(args.dataset)) 79 | if not os.path.exists(log_save_path): 80 | os.makedirs(log_save_path) 81 | 82 | now_time = make_print_to_file(path=log_save_path) 83 | 84 | if not os.path.exists(log_save_path): 85 | os.makedirs(log_save_path) 86 | 87 | # Dump args 88 | with open(log_save_path + '/args.json', 'w') as fid: 89 | json.dump(args.__dict__, fid, indent=2) 90 | 91 | # load encoder 92 | model = load_victim(args) 93 | 94 | # downstream task 95 | if args.dataset == 'imagenet': 96 | num_classes = 100 97 | elif args.dataset == 'gtsrb': 98 | num_classes = 43 99 | else: 100 | num_classes = 10 101 | 102 | classifier = torch.nn.DataParallel(NonLinearClassifier(feat_dim=512, num_classes=num_classes)) 103 | 104 | # load classifier 105 | classifier_path = os.path.join('victims', str(args.pre_dataset), str(args.victim), 'clean_model', str(args.dataset)) 106 | if args.dataset == 'imagenet': 107 | encoder_path = [Path(classifier_path) / ckpt for ckpt in os.listdir(Path(classifier_path)) if ckpt.endswith("50.pth")][0] 108 | else: 109 | encoder_path = [Path(classifier_path) / ckpt for ckpt in os.listdir(Path(classifier_path)) if ckpt.endswith("20.pth")][0] 110 | 111 | checkpoint = torch.load(encoder_path) 112 | classifier.load_state_dict(checkpoint) 113 | 114 | 115 | print('Day: %s, Target encoder:%s, Attack type: %s, Downstream task:%s'% (now_time, args.victim, args.type, args.dataset)) 116 | print("###################################### Test Attack! ######################################") 117 | 118 | if args.type == 'gan_per': 119 | uap_load_path = os.path.join('output', str(args.pre_dataset), 'uap_results', str(args.type), str(args.victim), 120 | str(args.sup_dataset), str(args.criterion), str(args.eps)) 121 | uap_path = [Path(uap_load_path) / ckpt for ckpt in os.listdir(Path(uap_load_path)) if ckpt.endswith("20.pt")][0] 122 | 123 | elif args.type == 'gan_pat': 124 | uap_load_path = os.path.join('output', str(args.pre_dataset), 'uap_results', str(args.type), str(args.victim), 125 | str(args.sup_dataset), str(args.criterion), str(args.noise_percentage)) 126 | uap_path = [Path(uap_load_path) / ckpt for ckpt in os.listdir(Path(uap_load_path)) if ckpt.endswith("20.pt")][0] 127 | 128 | 129 | UAP = torch.load(uap_path) 130 | 131 | clean_acc_t1, adv_acc_t1, decline_t1, attack_success_rate, clean_acc_t5, adv_acc_t5, decline_t5 = classify(args, model, classifier, UAP) 132 | 133 | print('Clean downstream accuracy: %.4f%%'% (clean_acc_t1)) 134 | print('Adv downstream accuracy: %.4f%%'% (adv_acc_t1)) 135 | print('Decline accuracy rate: %.4f%%' % (decline_t1)) 136 | print('Downstream fooling rate: %.4f%%' % (attack_success_rate)) 137 | 138 | final_log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'down_test', str(args.type)) 139 | if not os.path.exists(final_log_save_path): 140 | os.makedirs(final_log_save_path) 141 | 142 | final_result = [] 143 | 144 | if args.type == 'gan_per': 145 | final_result_ = {"now_time": now_time, 146 | "attack_type": args.type, 147 | "victim": str(args.victim), 148 | "pre_dataset": str(args.pre_dataset), 149 | "sup_dataset": str(args.sup_dataset), 150 | "down_dataset": str(args.dataset), 151 | "criterion": str(args.criterion), 152 | "eps": str(round(args.eps, 4)), 153 | "clean_acc_t1": round(clean_acc_t1, 4), 154 | "clean_acc_t5": round(clean_acc_t5, 4), 155 | "decline_t1": round(decline_t1, 4), 156 | "adv_acc_t1": round(adv_acc_t1, 4), 157 | "adv_acc_t5": round(adv_acc_t5, 4), 158 | "decline_t5": round(decline_t5, 4), 159 | "attack_success_rate": round(attack_success_rate, 4)} 160 | final_result.append(final_result_) 161 | header = ["now_time", "attack_type", "victim", "pre_dataset", "sup_dataset", "down_dataset", "criterion", "eps", "clean_acc_t1", "clean_acc_t5", "decline_t1", 162 | "adv_acc_t1", "adv_acc_t5", "decline_t5", "attack_success_rate"] 163 | 164 | 165 | elif args.type == 'gan_patch': 166 | final_result_ = {"now_time": now_time, 167 | "attack_type": args.type, 168 | "victim": str(args.victim), 169 | "pre_dataset": str(args.pre_dataset), 170 | "sup_dataset": str(args.sup_dataset), 171 | "down_dataset": str(args.dataset), 172 | "criterion": str(args.criterion), 173 | "noise_percentage": str(round(args.noise_percentage, 4)), 174 | "clean_acc_t1": round(clean_acc_t1, 4), 175 | "clean_acc_t5": round(clean_acc_t5, 4), 176 | "decline_t1": round(decline_t1, 4), 177 | "adv_acc_t1": round(adv_acc_t1, 4), 178 | "adv_acc_t5": round(adv_acc_t5, 4), 179 | "decline_t5": round(decline_t5, 4), 180 | "attack_success_rate": round(attack_success_rate, 4)} 181 | final_result.append(final_result_) 182 | header = ["now_time", "attack_type", "victim", "pre_dataset", "sup_dataset", "down_dataset", "criterion", "noise_percentage", "clean_acc_t1","clean_acc_t5", 183 | "decline_t1", "adv_acc_t1", "adv_acc_t5", "decline_t5", "attack_success_rate"] 184 | 185 | with open(final_log_save_path + '/all_final_results.csv', 'a', newline='') as f: 186 | writer = csv.DictWriter(f, fieldnames=header) 187 | writer.writeheader() 188 | writer.writerows(final_result) 189 | 190 | if __name__ == "__main__": 191 | main() 192 | -------------------------------------------------------------------------------- /gan_per_attack.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import torch 4 | import random 5 | import json 6 | import argparse 7 | import numpy as np 8 | import pandas as pd 9 | import torch.nn as nn 10 | import csv 11 | from torch.autograd import Variable 12 | from model.adv_gan import Generator 13 | from utils.nce import InfoNCE 14 | from utils.load_model import load_victim 15 | from utils.load_data import load_data, normalzie 16 | from utils.predict import knn_per_fr, make_print_to_file 17 | from utils.fr_util import generate_high 18 | import warnings 19 | warnings.filterwarnings("ignore") 20 | 21 | def arg_parse(): 22 | parser = argparse.ArgumentParser(description="AdvEncoder-PER") 23 | parser.add_argument('--seed', default=100, type=int, help='which seed the code runs on') 24 | parser.add_argument('--gpu', default='0, 1', type=str, help='which gpu the code runs on') 25 | parser.add_argument('--pre_dataset', default='cifar10',choices=['cifar10', 'imagenet']) 26 | parser.add_argument('--dataset', default='imagenet',choices=['cifar10', 'stl10', 'gtsrb', 'imagenet', 'minst', 'fashion-mnist']) 27 | parser.add_argument('--victim', default='simclr', choices=['simclr', 'byol', 'barlow_twins', 'deepclusterv2', 'dino', 'mocov3', 'mocov2plus', 'nnclr', 'ressl', 'supcon', 'swav', 'vibcreg', 'vicreg', 'wmse']) 28 | parser.add_argument('--criterion', default='nce', choices=['cos', 'nt', 'nce']) 29 | parser.add_argument('--alpha', type=int, default=5) 30 | parser.add_argument('--batch_size', type=int, default=256) 31 | parser.add_argument('--epochs', type=int, default=20) 32 | parser.add_argument('--eps', type=int, default= 10) 33 | parser.add_argument('--temperature', default=0.5, type=float, help='temperature') 34 | parser.add_argument('--radius', type=int, default=8, help='radius of low freq images') 35 | args = parser.parse_args() 36 | return args 37 | 38 | def uap_dcgan_attack(args, train_loader, test_loader, model): 39 | # init the GAN 40 | G_input_dim = 100 41 | G_output_dim = 3 42 | num_filters = [1024, 512, 256, 128] 43 | learning_rate = 0.0002 44 | betas = (0.5, 0.999) 45 | 46 | # save results 47 | results = {'clean_acc_t1': [], 'adv_acc_t1': [], 'decline_t1': [], 'clean_acc_t5': [], 'adv_acc_t5': [], 'decline_t5': [], 'fooling_rate': [], 'time': []} 48 | epoch_start = 0 49 | 50 | G = Generator(G_input_dim, num_filters, G_output_dim, args.batch_size) 51 | 52 | model.eval() 53 | G.cuda() 54 | 55 | # criterion_l2 56 | criterion_l2 = torch.nn.MSELoss() 57 | 58 | # criterion_contrastive 59 | if args.criterion == 'cos': 60 | criterion_contrastive = nn.CosineSimilarity(dim=0, eps=1e-6) 61 | elif args.criterion == 'nce': 62 | criterion_contrastive = InfoNCE() 63 | 64 | # Optimizers 65 | G_optimizer = torch.optim.Adam(G.parameters(), lr=learning_rate, betas=betas) 66 | 67 | # Training GAN 68 | # define a global fix noise z 69 | z = torch.randn(args.batch_size, G_input_dim).view(-1, G_input_dim, 1, 1) 70 | z = Variable(z.cuda()) 71 | 72 | for epoch in range(args.epochs): 73 | 74 | # init the start time 75 | start = time.time() 76 | 77 | for i, (images, _) in enumerate(train_loader): 78 | 79 | x = Variable(images.cuda()) 80 | new_shape = x.shape 81 | 82 | uap_noise = G(z).squeeze() 83 | uap_noise = torch.clamp(uap_noise, -args.eps, args.eps) 84 | uap_noise.cuda() 85 | 86 | # fake image 87 | f_x = x + uap_noise.expand(new_shape) 88 | 89 | x.cuda() 90 | f_x.cuda() 91 | 92 | # l_{2} loss 93 | reconstruction_loss = criterion_l2(f_x, x) 94 | 95 | clean_output = model(normalzie(args, x)) 96 | per_output = model(normalzie(args, f_x).cuda()) 97 | 98 | 99 | # adv loss 100 | if args.criterion == 'cos': 101 | adv_loss = criterion_contrastive(clean_output, per_output).mean() 102 | else: 103 | adv_loss_pos = criterion_contrastive(clean_output, per_output).mean() 104 | adv_loss = -adv_loss_pos 105 | 106 | # HFC loss 107 | clean_hfc = generate_high(x, r=args.radius) 108 | per_hfc = generate_high(f_x, r=args.radius) 109 | HFC_loss = criterion_l2(clean_hfc, per_hfc) 110 | HFC_loss = - HFC_loss 111 | # lack D 112 | G_loss = args.alpha * adv_loss + reconstruction_loss + HFC_loss 113 | 114 | # Back propagation 115 | G.zero_grad() 116 | G_loss.backward() 117 | G_optimizer.step() 118 | 119 | 120 | if i % 1 == 0: 121 | print('Epoch [%d/%d], Step [%d/%d], Adv_loss: %.4f, L2_loss: %.4f, HFC_loss: %.4f, G_loss: %.4f' 122 | % (epoch + 1, args.epochs, i + 1, len(train_loader), adv_loss.item(), reconstruction_loss.item(), HFC_loss.item(), G_loss.item())) 123 | end = time.time() 124 | run_time = end - start 125 | 126 | 127 | # caculate the acc decline 128 | clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, fooling_rate = knn_per_fr(args, train_loader, test_loader, 129 | model, uap_noise.cuda()) 130 | 131 | decline_t1 = ((clean_acc_t1 - adv_acc_t1) / clean_acc_t1) * 100 132 | decline_t5 = ((clean_acc_t5 - adv_acc_t5) / clean_acc_t5) * 100 133 | 134 | print('############################## Epoch [%d/%d], Clean_acc: %.4f%%, Per_acc: %.4f%%, Decline Rate: %.4f%%, Fooling Rate: %.4f%%! ##############################' % ( 135 | epoch + 1, args.epochs, clean_acc_t1, adv_acc_t1, decline_t1, fooling_rate), end='\n') 136 | 137 | results['clean_acc_t1'].append(clean_acc_t1) 138 | results['clean_acc_t5'].append(clean_acc_t5) 139 | results['decline_t1'].append(decline_t1) 140 | results['adv_acc_t1'].append(adv_acc_t1) 141 | results['adv_acc_t5'].append(adv_acc_t5) 142 | results['decline_t5'].append(decline_t5) 143 | results['fooling_rate'].append(fooling_rate) 144 | results['time'].append(run_time) 145 | 146 | # Save statistics 147 | data_frame = pd.DataFrame(data=results, index=range(epoch_start, epoch + 1)) 148 | data_frame.to_csv(log_save_path + '/log.csv', index_label='epoch') 149 | 150 | # save uap result 151 | uap_save_path = os.path.join('output', str(args.pre_dataset), 'uap_results', 'gan_per', str(args.victim), str(args.dataset), 152 | str(args.criterion), str(args.eps)) 153 | 154 | if not os.path.exists(uap_save_path): 155 | os.makedirs(uap_save_path) 156 | torch.save(uap_noise.cpu().data, '{}/{}'.format(uap_save_path, 'uap_gan_' + str(args.dataset) + '_' + str( 157 | round(decline_t1, 4)) + '_' + str(round(fooling_rate, 4)) + '_' + str(epoch + 1) + '.pt')) 158 | 159 | return clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, fooling_rate, decline_t1, decline_t5 160 | 161 | 162 | 163 | if __name__ == '__main__': 164 | 165 | args = arg_parse() 166 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 167 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu 168 | random.seed(args.seed) 169 | os.environ['PYTHONHASHSEED'] = str(args.seed) 170 | np.random.seed(args.seed) 171 | torch.manual_seed(args.seed) 172 | torch.cuda.manual_seed(args.seed) 173 | torch.backends.cudnn.deterministic = True 174 | torch.backends.cudnn.benchmark = True 175 | torch.set_printoptions(profile="full") 176 | args.eps = args.eps / 255 177 | 178 | # Logging 179 | log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'gan_per', str(args.victim), str(args.dataset), str(args.criterion), str(args.eps)) 180 | if not os.path.exists(log_save_path): 181 | os.makedirs(log_save_path) 182 | 183 | # Dump args 184 | with open(log_save_path + '/args.json', 'w') as fid: 185 | json.dump(args.__dict__, fid, indent=2) 186 | 187 | now_time = make_print_to_file(path = log_save_path) 188 | 189 | print("######################################################### Attack Setting ######################################################### ") 190 | print('Day: %s, Pre-trained Encoder: %s, Dataset: %s, Adv_criterion: %s, Epsilon: %.2f' % (now_time, args.victim, args.dataset, args.criterion, args.eps)) 191 | 192 | # load the pre-trained model 193 | model = torch.nn.DataParallel(load_victim(args)) 194 | 195 | 196 | # load the data 197 | train_loader, test_loader = load_data(args.dataset, args.batch_size) 198 | 199 | model.cuda() 200 | for param in model.parameters(): 201 | param.requires_grad = False 202 | 203 | # output the basic information 204 | print("########################################################## Attack Start! ########################################################## ") 205 | 206 | clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, fooling_rate, decline_t1, decline_t5 = uap_dcgan_attack(args, train_loader, test_loader, model) 207 | 208 | 209 | final_log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'gan_per') 210 | if not os.path.exists(final_log_save_path): 211 | os.makedirs(final_log_save_path) 212 | 213 | final_result = [] 214 | final_result_ = {"now_time": now_time, "victim": str(args.victim), "pre_dataset": str(args.pre_dataset), "sup_dataset": str(args.dataset), 215 | "criterion": str(args.criterion), "eps": str(round(args.eps, 4)), "clean_acc_t1": round(clean_acc_t1, 4), 216 | "clean_acc_t5": round(clean_acc_t5, 4), "decline_t1": round(decline_t1, 4), "adv_acc_t1": round(adv_acc_t1, 4), 217 | "adv_acc_t5": round(adv_acc_t5, 4), "decline_t5": round(decline_t5, 4), "fooling_rate": round(fooling_rate, 4)} 218 | final_result.append(final_result_) 219 | 220 | header = ["now_time", "victim", "pre_dataset", "sup_dataset", "criterion", "eps", "clean_acc_t1", "clean_acc_t5", "decline_t1", 221 | "adv_acc_t1", "adv_acc_t5", "decline_t5", "fooling_rate"] 222 | 223 | with open(final_log_save_path + '/all_final_results.csv', 'a', newline='') as f: 224 | writer = csv.DictWriter(f, fieldnames=header) 225 | writer.writeheader() 226 | writer.writerows(final_result) 227 | -------------------------------------------------------------------------------- /gan_pat_attack.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import csv 4 | import torch 5 | import random 6 | import json 7 | import argparse 8 | import numpy as np 9 | import pandas as pd 10 | import torch.nn as nn 11 | from torch.autograd import Variable 12 | from model.adv_gan import Generator 13 | from utils.nce import InfoNCE 14 | from utils.load_model import load_victim 15 | from utils.load_data import load_data, normalzie 16 | from utils.predict import knn_patch_fr, make_print_to_file 17 | from utils.fr_util import generate_high 18 | from utils.patch_utils import patch_initialization, mask_generation, clamp_patch 19 | 20 | def arg_parse(): 21 | parser = argparse.ArgumentParser(description="AdvEncoder-PAT") 22 | parser.add_argument('--seed', default=100, type=int, help='which seed the code runs on') 23 | parser.add_argument('--gpu', default='1', type=str, help='which gpu the code runs on') 24 | parser.add_argument('--pre_dataset', default='cifar10',choices=['cifar10', 'imagenet']) 25 | parser.add_argument('--dataset', default='cifar10',choices=['cifar10', 'stl10', 'gtsrb', 'imagenet', 'minst', 'fashion-mnist']) 26 | parser.add_argument('--victim', default='simclr', choices=['simclr', 'byol', 'barlow_twins', 'deepclusterv2', 'dino', 'mocov3', 'mocov2plus', 'nnclr', 'ressl', 'simsiam', 'supcon', 'swav', 'vibcreg', 'vicreg', 'wmse']) 27 | parser.add_argument('--criterion', default='nce', choices=['cos', 'nt', 'nce']) 28 | parser.add_argument('--alpha', type=int, default=5) 29 | parser.add_argument('--batch_size', type=int, default=256) 30 | parser.add_argument('--epochs', type=int, default=20) 31 | parser.add_argument('--noise_percentage', type=float, default=0.03) # patch shape 32 | parser.add_argument('--temperature', default=0.5, type=float, help='temperature') 33 | parser.add_argument('--radius', type=int, default=8, help='radius of low freq images') 34 | args = parser.parse_args() 35 | return args 36 | 37 | def uap_dcgan_attack(args, train_loader, test_loader, model, mask): 38 | # init the GAN 39 | G_input_dim = 100 40 | G_output_dim = 3 41 | num_filters = [1024, 512, 256, 128] 42 | learning_rate = 0.0002 43 | betas = (0.5, 0.999) 44 | 45 | # save results 46 | results = {'clean_acc_t1': [], 'adv_acc_t1': [], 'decline_t1': [], 'clean_acc_t5': [], 'adv_acc_t5': [], 'decline_t5': [], 'attack_success_rate': [], 'time': []} 47 | epoch_start = 0 48 | 49 | G = Generator(G_input_dim, num_filters, G_output_dim, args.batch_size) 50 | model.eval() 51 | G.cuda() 52 | 53 | 54 | # criterion_l2 55 | criterion_l2 = torch.nn.MSELoss() 56 | 57 | # criterion_contrastive 58 | if args.criterion == 'cos': 59 | criterion_contrastive = nn.CosineSimilarity(dim=0, eps=1e-6) 60 | elif args.criterion == 'nce': 61 | criterion_contrastive = InfoNCE() 62 | 63 | # Optimizers 64 | G_optimizer = torch.optim.Adam(G.parameters(), lr=learning_rate, betas=betas) 65 | 66 | # Training GAN 67 | # define a global fix noise z 68 | z = torch.randn(args.batch_size, G_input_dim).view(-1, G_input_dim, 1, 1) 69 | z = Variable(z.cuda()) 70 | 71 | for epoch in range(args.epochs): 72 | 73 | # init the start time 74 | start = time.time() 75 | 76 | for i, (images, _) in enumerate(train_loader): 77 | 78 | x = Variable(images.cuda()) 79 | new_shape = x.shape 80 | 81 | uap_noise = G(z).squeeze() 82 | uap_noise = clamp_patch(args, uap_noise) 83 | uap_noise.cuda() 84 | 85 | # fake image 86 | f_x = torch.mul(mask.type(torch.FloatTensor), 87 | uap_noise.type(torch.FloatTensor)) + torch.mul( 88 | 1 - mask.expand(new_shape).type(torch.FloatTensor), x.type(torch.FloatTensor)) 89 | 90 | x.cuda() 91 | f_x.cuda() 92 | 93 | # l_{2} loss 94 | reconstruction_loss = criterion_l2(f_x.cuda(), x.cuda()) 95 | 96 | # adv loss 97 | clean_output = model(normalzie(args, x)) 98 | per_output = model(normalzie(args, f_x).cuda()) 99 | 100 | if args.criterion == 'cos': 101 | adv_loss = criterion_contrastive(clean_output, per_output).mean() 102 | else: 103 | adv_loss_pos = criterion_contrastive(clean_output, per_output).mean() 104 | adv_loss = -adv_loss_pos 105 | 106 | # HFC loss 107 | clean_hfc = generate_high(x, r=args.radius) 108 | per_hfc = generate_high(f_x.cuda(), r=args.radius) 109 | HFC_loss = criterion_l2(clean_hfc, per_hfc) 110 | HFC_loss = - HFC_loss 111 | # lack D 112 | G_loss = args.alpha * adv_loss + reconstruction_loss + HFC_loss 113 | 114 | # Back propagation 115 | G.zero_grad() 116 | G_loss.backward() 117 | G_optimizer.step() 118 | 119 | 120 | if i % 1 == 0: 121 | print('Epoch [%d/%d], Step [%d/%d], Adv_loss: %.4f, L2_loss: %.4f, HFC_loss: %.4f, G_loss: %.4f' 122 | % (epoch + 1, args.epochs, i + 1, len(train_loader), adv_loss.item(), reconstruction_loss.item(), HFC_loss.item(), G_loss.item())) 123 | 124 | 125 | end = time.time() 126 | run_time = end - start 127 | 128 | # caculate the acc decline 129 | clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, attack_success_rate = knn_patch_fr(args, train_loader, test_loader, model, uap_noise, mask) 130 | 131 | decline_t1 = ((clean_acc_t1 - adv_acc_t1) / clean_acc_t1) * 100 132 | decline_t5 = ((clean_acc_t5 - adv_acc_t5) / clean_acc_t5) * 100 133 | 134 | print('############################## Epoch [%d/%d], Clean_acc: %.4f%%, Per_acc: %.4f%%, Decline Rate: %.4f%%, Fooling Rate: %.4f%%! ##############################' % ( 135 | epoch + 1, args.epochs, clean_acc_t1, adv_acc_t1, decline_t1, attack_success_rate), end='\n') 136 | 137 | results['clean_acc_t1'].append(clean_acc_t1) 138 | results['clean_acc_t5'].append(clean_acc_t5) 139 | results['decline_t1'].append(decline_t1) 140 | results['adv_acc_t1'].append(adv_acc_t1) 141 | results['adv_acc_t5'].append(adv_acc_t5) 142 | results['decline_t5'].append(decline_t5) 143 | results['attack_success_rate'].append(attack_success_rate) 144 | results['time'].append(run_time) 145 | 146 | # Save statistics 147 | data_frame = pd.DataFrame(data=results, index=range(epoch_start, epoch + 1)) 148 | data_frame.to_csv(log_save_path + '/log.csv', index_label='epoch') 149 | 150 | # save uap result 151 | uap_save_path = os.path.join('output', str(args.pre_dataset), 'uap_results', 'gan_patch', str(args.victim), str(args.dataset), 152 | str(args.criterion), str(args.noise_percentage)) 153 | 154 | if not os.path.exists(uap_save_path): 155 | os.makedirs(uap_save_path) 156 | torch.save(uap_noise.cpu().data, '{}/{}'.format(uap_save_path, 'uap_patch_gan_' + str(args.dataset) + '_' + str( 157 | round(decline_t1, 4)) + '_' + str(round(attack_success_rate, 4)) + '_' + str(epoch + 1) + '.pt')) 158 | 159 | return clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, attack_success_rate, decline_t1, decline_t5, uap_noise 160 | 161 | 162 | 163 | if __name__ == '__main__': 164 | 165 | args = arg_parse() 166 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 167 | os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu 168 | random.seed(args.seed) 169 | os.environ['PYTHONHASHSEED'] = str(args.seed) 170 | np.random.seed(args.seed) 171 | torch.manual_seed(args.seed) 172 | torch.cuda.manual_seed(args.seed) 173 | torch.backends.cudnn.deterministic = True 174 | torch.backends.cudnn.benchmark = True 175 | torch.set_printoptions(profile="full") 176 | 177 | 178 | # Logging 179 | log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'gan_patch', str(args.victim), str(args.dataset), str(args.criterion), str(args.noise_percentage)) 180 | if not os.path.exists(log_save_path): 181 | os.makedirs(log_save_path) 182 | 183 | # Dump args 184 | with open(log_save_path + '/args.json', 'w') as fid: 185 | json.dump(args.__dict__, fid, indent=2) 186 | 187 | now_time = make_print_to_file(path = log_save_path) 188 | 189 | print("######################################################### Attack Setting ######################################################### ") 190 | print('Day: %s, Pre-trained Encoder: %s, Dataset: %s, Adv_criterion: %s, Noise_percentage: %.2f' % (now_time, args.victim, args.dataset, args.criterion, args.noise_percentage)) 191 | 192 | # load the pre-trained model 193 | model = torch.nn.DataParallel(load_victim(args)) 194 | 195 | # load the data 196 | train_loader, test_loader = load_data(args.dataset, args.batch_size) 197 | 198 | model.cuda() 199 | for param in model.parameters(): 200 | param.requires_grad = False 201 | 202 | # init patch 203 | patch = patch_initialization(args) 204 | mask, applied_patch, x, y = mask_generation(args, patch) 205 | applied_patch = torch.from_numpy(applied_patch) 206 | mask = torch.from_numpy(mask) 207 | mask.cuda() 208 | 209 | clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, attack_success_rate = knn_patch_fr(args, train_loader, test_loader, 210 | model, applied_patch, mask) 211 | 212 | print(clean_acc_t1) 213 | # output the basic information 214 | print("########################################################## Attack Start! ########################################################## ") 215 | 216 | clean_acc_t1, clean_acc_t5, adv_acc_t1, adv_acc_t5, attack_success_rate, decline_t1, decline_t5, uap_noise = uap_dcgan_attack(args, train_loader, test_loader, model, mask) 217 | 218 | 219 | final_log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'gan_patch') 220 | if not os.path.exists(final_log_save_path): 221 | os.makedirs(final_log_save_path) 222 | 223 | final_result = [] 224 | final_result_ = {"now_time": now_time, "victim": str(args.victim), "pre_dataset": str(args.pre_dataset), "sup_dataset": str(args.dataset), 225 | "criterion": str(args.criterion), "noise_percentage": str(round(args.noise_percentage, 4)), "clean_acc_t1": round(clean_acc_t1, 4), 226 | "clean_acc_t5": round(clean_acc_t5, 4), "decline_t1": round(decline_t1, 4), "adv_acc_t1": round(adv_acc_t1, 4), 227 | "adv_acc_t5": round(adv_acc_t5, 4), "decline_t5": round(decline_t5, 4), "attack_success_rate": round(attack_success_rate, 4)} 228 | final_result.append(final_result_) 229 | 230 | header = ["now_time", "victim", "pre_dataset", "sup_dataset", "criterion", "noise_percentage", "clean_acc_t1", "clean_acc_t5", "decline_t1", 231 | "adv_acc_t1", "adv_acc_t5", "decline_t5", "attack_success_rate"] 232 | 233 | with open(final_log_save_path + '/all_final_results.csv', 'a', newline='') as f: 234 | writer = csv.DictWriter(f, fieldnames=header) 235 | writer.writeheader() 236 | writer.writerows(final_result) 237 | -------------------------------------------------------------------------------- /utils/predict.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import datetime 4 | import torch 5 | from tqdm import tqdm 6 | import torch.nn.functional as F 7 | from utils.load_data import normalzie 8 | from utils.patch_utils import patch_initialization, mask_generation 9 | 10 | def make_print_to_file(path='./'): 11 | ''' 12 | path， it is a path for save your log about fuction print 13 | example: 14 | use make_print_to_file() and the all the information of funtion print , will be write in to a log file 15 | :return: 16 | ''' 17 | class Logger(object): 18 | def __init__(self, filename="Default.log", path="./"): 19 | self.terminal = sys.stdout 20 | self.log = open(os.path.join(path, filename), "w", encoding='utf8', ) 21 | 22 | def write(self, message): 23 | self.terminal.write(message) 24 | self.log.write(message) 25 | 26 | def flush(self): 27 | pass 28 | 29 | fileName = datetime.datetime.now().strftime('%Y_%m_%d') 30 | sys.stdout = Logger(fileName + '.log', path=path) 31 | return fileName 32 | 33 | 34 | def knn_per_fr(args, train_loader, test_loader, model, p, k=20, T=0.07): 35 | # extract train features 36 | train_features, train_targets = extract_features(args, train_loader, model) 37 | 38 | # extract test features 39 | test_features, test_targets = extract_features(args, test_loader, model) 40 | 41 | # extract per_test features 42 | p_test_features, p_test_targets = extract_per_features(args, test_loader, model, p) 43 | 44 | max_distance_matrix_size = int(5e6) 45 | train_features = F.normalize(train_features) 46 | test_features = F.normalize(test_features) 47 | p_test_features = F.normalize(p_test_features) 48 | 49 | num_classes = torch.unique(test_targets).numel() 50 | num_test_images = test_targets.size(0) 51 | num_p_test_images = p_test_targets.size(0) 52 | num_train_images = train_targets.size(0) 53 | chunk_size = min( 54 | max(1, max_distance_matrix_size // num_train_images), 55 | num_test_images, 56 | ) 57 | 58 | k = min(k, num_train_images) 59 | 60 | # test clean 61 | top1, top5, total = 0.0, 0.0, 0 62 | retrieval_one_hot = torch.zeros(k, num_classes).to(train_features.device) 63 | p_top1, p_top5, p_total = 0.0, 0.0, 0 64 | p_retrieval_one_hot = torch.zeros(k, num_classes).to(train_features.device) 65 | fr = 0.0 66 | 67 | for idx in range(0, num_test_images, chunk_size): 68 | # get the features for test images 69 | features = test_features[idx: min((idx + chunk_size), num_test_images), :] 70 | targets = test_targets[idx: min((idx + chunk_size), num_test_images)] 71 | batch_size = targets.size(0) 72 | 73 | # calculate the dot product and compute top-k neighbors 74 | similarities = torch.mm(features, train_features.t()) 75 | 76 | similarities, indices = similarities.topk(k, largest=True, sorted=True) 77 | candidates = train_targets.view(1, -1).expand(batch_size, -1) 78 | retrieved_neighbors = torch.gather(candidates, 1, indices) 79 | 80 | retrieval_one_hot.resize_(batch_size * k, num_classes).zero_() 81 | retrieval_one_hot.scatter_(1, retrieved_neighbors.view(-1, 1), 1) 82 | 83 | similarities = similarities.clone().div_(T).exp_() 84 | 85 | probs = torch.sum( 86 | torch.mul( 87 | retrieval_one_hot.view(batch_size, -1, num_classes), 88 | similarities.view(batch_size, -1, 1), 89 | ), 90 | 1, 91 | ) 92 | _, predictions = probs.sort(1, True) 93 | 94 | # get the features for test images 95 | p_features = p_test_features[idx: min((idx + chunk_size), num_p_test_images), :] 96 | p_targets = p_test_targets[idx: min((idx + chunk_size), num_p_test_images)] 97 | p_batch_size = p_targets.size(0) 98 | 99 | # calculate the dot product and compute top-k neighbors 100 | p_similarities = torch.mm(p_features, train_features.t()) 101 | 102 | p_similarities, p_indices = p_similarities.topk(k, largest=True, sorted=True) 103 | p_candidates = train_targets.view(1, -1).expand(p_batch_size, -1) 104 | p_retrieved_neighbors = torch.gather(p_candidates, 1, p_indices) 105 | 106 | p_retrieval_one_hot.resize_(p_batch_size * k, num_classes).zero_() 107 | p_retrieval_one_hot.scatter_(1, p_retrieved_neighbors.view(-1, 1), 1) 108 | p_similarities = p_similarities.clone().div_(T).exp_() 109 | 110 | p_probs = torch.sum( 111 | torch.mul( 112 | p_retrieval_one_hot.view(p_batch_size, -1, num_classes), 113 | p_similarities.view(p_batch_size, -1, 1), 114 | ), 115 | 1, 116 | ) 117 | _, p_predictions = p_probs.sort(1, True) 118 | 119 | # find the predictions that match the target 120 | correct = predictions.eq(targets.data.view(-1, 1)) 121 | top1 = top1 + correct.narrow(1, 0, 1).sum().item() 122 | top5 = (top5 + correct.narrow(1, 0, 123 | min(5, k, correct.size(-1))).sum().item()) # top5 does not make sense if k < 5 124 | total += targets.size(0) 125 | 126 | # find the predictions that match the target 127 | p_correct = p_predictions.eq(p_targets.data.view(-1, 1)) 128 | p_top1 = p_top1 + p_correct.narrow(1, 0, 1).sum().item() 129 | p_top5 = (p_top5 + p_correct.narrow(1, 0, min(5, k, p_correct.size( 130 | -1))).sum().item()) # top5 does not make sense if k < 5 131 | 132 | p_total += p_targets.size(0) 133 | 134 | fr += predictions.eq(p_predictions).narrow(1, 0, 1).sum().item() 135 | 136 | top1 = top1 * 100.0 / total 137 | top5 = top5 * 100.0 / total 138 | 139 | p_top1 = p_top1 * 100.0 / p_total 140 | p_top5 = p_top5 * 100.0 / p_total 141 | 142 | attack_success_rate = (total - fr) * 100.0 / float(total) 143 | 144 | return top1, top5, p_top1, p_top5, attack_success_rate 145 | 146 | 147 | def knn_patch_fr(args, train_loader, test_loader, model, p, mask, k=20, T=0.07): 148 | # extract train features 149 | train_features, train_targets = extract_features(args, train_loader, model) 150 | 151 | # extract test features 152 | test_features, test_targets = extract_features(args, test_loader, model) 153 | 154 | # extract per_test features 155 | p_test_features, p_test_targets = extract_patch_features(args, test_loader, model, p, mask) 156 | 157 | max_distance_matrix_size = int(5e6) 158 | train_features = F.normalize(train_features) 159 | test_features = F.normalize(test_features) 160 | p_test_features = F.normalize(p_test_features) 161 | 162 | num_classes = torch.unique(test_targets).numel() 163 | num_test_images = test_targets.size(0) 164 | num_p_test_images = p_test_targets.size(0) 165 | num_train_images = train_targets.size(0) 166 | chunk_size = min( 167 | max(1, max_distance_matrix_size // num_train_images), 168 | num_test_images, 169 | ) 170 | 171 | k = min(k, num_train_images) 172 | 173 | # test clean 174 | top1, top5, total = 0.0, 0.0, 0 175 | retrieval_one_hot = torch.zeros(k, num_classes).to(train_features.device) 176 | p_top1, p_top5, p_total = 0.0, 0.0, 0 177 | p_retrieval_one_hot = torch.zeros(k, num_classes).to(train_features.device) 178 | fr = 0.0 179 | 180 | for idx in range(0, num_test_images, chunk_size): 181 | # get the features for test images 182 | features = test_features[idx: min((idx + chunk_size), num_test_images), :] 183 | targets = test_targets[idx: min((idx + chunk_size), num_test_images)] 184 | batch_size = targets.size(0) 185 | 186 | # calculate the dot product and compute top-k neighbors 187 | similarities = torch.mm(features, train_features.t()) 188 | 189 | similarities, indices = similarities.topk(k, largest=True, sorted=True) 190 | candidates = train_targets.view(1, -1).expand(batch_size, -1) 191 | retrieved_neighbors = torch.gather(candidates, 1, indices) 192 | 193 | retrieval_one_hot.resize_(batch_size * k, num_classes).zero_() 194 | retrieval_one_hot.scatter_(1, retrieved_neighbors.view(-1, 1), 1) 195 | 196 | similarities = similarities.clone().div_(T).exp_() 197 | 198 | probs = torch.sum( 199 | torch.mul( 200 | retrieval_one_hot.view(batch_size, -1, num_classes), 201 | similarities.view(batch_size, -1, 1), 202 | ), 203 | 1, 204 | ) 205 | _, predictions = probs.sort(1, True) 206 | 207 | # get the features for test images 208 | p_features = p_test_features[idx: min((idx + chunk_size), num_p_test_images), :] 209 | p_targets = p_test_targets[idx: min((idx + chunk_size), num_p_test_images)] 210 | p_batch_size = p_targets.size(0) 211 | 212 | # calculate the dot product and compute top-k neighbors 213 | p_similarities = torch.mm(p_features, train_features.t()) 214 | 215 | p_similarities, p_indices = p_similarities.topk(k, largest=True, sorted=True) 216 | p_candidates = train_targets.view(1, -1).expand(p_batch_size, -1) 217 | p_retrieved_neighbors = torch.gather(p_candidates, 1, p_indices) 218 | 219 | p_retrieval_one_hot.resize_(p_batch_size * k, num_classes).zero_() 220 | p_retrieval_one_hot.scatter_(1, p_retrieved_neighbors.view(-1, 1), 1) 221 | p_similarities = p_similarities.clone().div_(T).exp_() 222 | 223 | p_probs = torch.sum( 224 | torch.mul( 225 | p_retrieval_one_hot.view(p_batch_size, -1, num_classes), 226 | p_similarities.view(p_batch_size, -1, 1), 227 | ), 228 | 1, 229 | ) 230 | _, p_predictions = p_probs.sort(1, True) 231 | 232 | # find the predictions that match the target 233 | correct = predictions.eq(targets.data.view(-1, 1)) 234 | top1 = top1 + correct.narrow(1, 0, 1).sum().item() 235 | top5 = (top5 + correct.narrow(1, 0, 236 | min(5, k, correct.size(-1))).sum().item()) # top5 does not make sense if k < 5 237 | total += targets.size(0) 238 | 239 | # find the predictions that match the target 240 | p_correct = p_predictions.eq(p_targets.data.view(-1, 1)) 241 | p_top1 = p_top1 + p_correct.narrow(1, 0, 1).sum().item() 242 | p_top5 = (p_top5 + p_correct.narrow(1, 0, min(5, k, p_correct.size( 243 | -1))).sum().item()) # top5 does not make sense if k < 5 244 | 245 | p_total += p_targets.size(0) 246 | 247 | fr += predictions.eq(p_predictions).narrow(1, 0, 1).sum().item() 248 | 249 | top1 = top1 * 100.0 / total 250 | top5 = top5 * 100.0 / total 251 | 252 | p_top1 = p_top1 * 100.0 / p_total 253 | p_top5 = p_top5 * 100.0 / p_total 254 | 255 | attack_success_rate = (total - fr) * 100.0 / float(total) 256 | return top1, top5, p_top1, p_top5, attack_success_rate 257 | 258 | 259 | @torch.no_grad() 260 | def extract_features(args, loader, model): 261 | model.eval() 262 | backbone_features, labels = [], [] 263 | for im, lab in tqdm(loader): 264 | im = im.cuda(non_blocking=True) 265 | lab = lab.cuda(non_blocking=True) 266 | outs = model(normalzie(args,im)) 267 | backbone_features.append(outs) 268 | labels.append(lab) 269 | model.train() 270 | backbone_features = torch.cat(backbone_features) 271 | labels = torch.cat(labels) 272 | return backbone_features, labels 273 | 274 | @torch.no_grad() 275 | def extract_patch_features(args, loader, model, p, mask): 276 | model.eval() 277 | backbone_features, labels = [], [] 278 | for im, lab in tqdm(loader): 279 | new_shape = im.shape 280 | im = torch.mul(mask.type(torch.FloatTensor), p.type(torch.FloatTensor)) + torch.mul( 281 | 1 - mask.expand(new_shape).type(torch.FloatTensor), im.type(torch.FloatTensor)) 282 | 283 | im = im.cuda(non_blocking=True) 284 | lab = lab.cuda(non_blocking=True) 285 | outs = model(normalzie(args, im)) 286 | backbone_features.append(outs) 287 | labels.append(lab) 288 | model.train() 289 | backbone_features = torch.cat(backbone_features) 290 | labels = torch.cat(labels) 291 | return backbone_features, labels 292 | 293 | @torch.no_grad() 294 | def extract_per_features(args, loader, model, p): 295 | model.eval() 296 | backbone_features, labels = [], [] 297 | for im, lab in tqdm(loader): 298 | new_shape = im.shape 299 | im = im.cuda(non_blocking=True) 300 | lab = lab.cuda(non_blocking=True) 301 | im = im + p.expand(new_shape) 302 | outs = model(normalzie(args, im)) 303 | backbone_features.append(outs) 304 | labels.append(lab) 305 | model.train() 306 | backbone_features = torch.cat(backbone_features) 307 | labels = torch.cat(labels) 308 | return backbone_features, labels 309 | 310 | def accuracy(output, target, topk=(1,)): 311 | with torch.no_grad(): 312 | 313 | maxk = max(topk) 314 | batch_size = target.size(0) 315 | 316 | _, pred = output.topk(maxk, 1, True, True) 317 | 318 | pred = pred.t() 319 | 320 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 321 | # correct = pred.eq(target.expand_as(pred)) 322 | 323 | res = [] 324 | for k in topk: 325 | correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) 326 | res.append(correct_k.mul_(100.0 / batch_size)) 327 | return res 328 | 329 | def attack_success_rate(clean_output, per_output, target): 330 | with torch.no_grad(): 331 | # maxk = max(topk) 332 | batch_size = target.size(0) 333 | 334 | _, c_pred = clean_output.max(1) 335 | c_pred = c_pred.t() 336 | 337 | _, p_pred = per_output.max(1) 338 | p_pred = p_pred.t() 339 | 340 | attack_success_rate = float(torch.sum(c_pred != p_pred)) / float(batch_size) 341 | 342 | return attack_success_rate * 100 343 | 344 | def test(args, encoder, classifier, test_loader, data): 345 | top1_accuracy = 0 346 | top5_accuracy = 0 347 | 348 | classifier.eval() 349 | 350 | with torch.no_grad(): 351 | for counter, (x_batch, y_batch) in enumerate(tqdm(test_loader)): 352 | x_batch = x_batch.cuda() 353 | y_batch = y_batch.cuda() 354 | h = encoder(normalzie(args, x_batch)) 355 | x_in = h.view(h.size(0), -1) 356 | logits = classifier(x_in) 357 | top1, top5 = accuracy(logits, y_batch, topk=(1, 5)) 358 | top1_accuracy += top1[0] 359 | top5_accuracy += top5[0] 360 | 361 | top1_accuracy /= (counter + 1) 362 | top5_accuracy /= (counter + 1) 363 | 364 | return top1_accuracy.item(), top5_accuracy.item() 365 | 366 | 367 | def adv_test(args, encoder, classifier, test_loader, uap, data): 368 | top1_accuracy = 0 369 | top5_accuracy = 0 370 | 371 | classifier.eval() 372 | 373 | with torch.no_grad(): 374 | for counter, (x_batch, y_batch) in enumerate(tqdm(test_loader)): 375 | new_shape = x_batch.shape 376 | if args.type == 'gan_patch': 377 | patch = patch_initialization(args) 378 | mask, applied_patch, x, y = mask_generation(args, patch) 379 | mask = torch.from_numpy(mask) 380 | x_batch = torch.mul(mask.type(torch.FloatTensor), uap.type(torch.FloatTensor)) + torch.mul(1 - mask.expand(new_shape).type(torch.FloatTensor), x_batch.type(torch.FloatTensor)) 381 | 382 | else: 383 | x_batch = x_batch.cuda() + uap.expand(new_shape).cuda() 384 | 385 | x_batch = x_batch.cuda() 386 | y_batch = y_batch.cuda() 387 | 388 | h = encoder(normalzie(args, x_batch)) 389 | x_in = h.view(h.size(0), -1) 390 | logits = classifier(x_in) 391 | # print(counter) 392 | top1, top5 = accuracy(logits, y_batch, topk=(1, 5)) 393 | top1_accuracy += top1[0] 394 | top5_accuracy += top5[0] 395 | 396 | top1_accuracy /= (counter + 1) 397 | top5_accuracy /= (counter + 1) 398 | 399 | return top1_accuracy.item(), top5_accuracy.item() 400 | 401 | def fr_test(args, encoder, classifier, test_loader, uap, data): 402 | fr = 0 403 | classifier.eval() 404 | 405 | with torch.no_grad(): 406 | for counter, (x_batch, y_batch) in enumerate(tqdm(test_loader)): 407 | 408 | new_shape = x_batch.shape 409 | 410 | clean_x_batch = x_batch.clone() 411 | clean_x_batch = clean_x_batch.cuda() 412 | y_batch = y_batch.cuda() 413 | 414 | if args.type == 'gan_patch': 415 | patch = patch_initialization(args) 416 | mask, applied_patch, x, y = mask_generation(args, patch) 417 | # applied_patch = torch.from_numpy(applied_patch) 418 | mask = torch.from_numpy(mask) 419 | per_x_batch = torch.mul(mask.type(torch.FloatTensor), uap.type(torch.FloatTensor)) + torch.mul( 420 | 1 - mask.expand(new_shape).type(torch.FloatTensor), x_batch.type(torch.FloatTensor)) 421 | else: 422 | per_x_batch = x_batch.cuda() + uap.expand(new_shape).cuda() 423 | 424 | per_x_batch = per_x_batch.cuda() 425 | c_h = encoder(normalzie(args, clean_x_batch)) 426 | p_h = encoder(normalzie(args, per_x_batch)) 427 | 428 | c_x_in = c_h.view(c_h.size(0), -1) 429 | p_x_in = p_h.view(p_h.size(0), -1) 430 | 431 | c_logits = classifier(c_x_in) 432 | p_logits = classifier(p_x_in) 433 | 434 | fooling_ra = attack_success_rate(c_logits, p_logits, y_batch) 435 | fr += fooling_ra 436 | 437 | fr /= (counter + 1) 438 | 439 | return fr 440 | -------------------------------------------------------------------------------- /model/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import Tensor 3 | import torch.nn as nn 4 | from typing import Type, Any, Callable, Union, List, Optional 5 | 6 | 7 | __all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 8 | 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', 9 | 'wide_resnet50_2', 'wide_resnet101_2'] 10 | 11 | model_urls = { 12 | 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 13 | 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 14 | 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 15 | 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 16 | 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 17 | 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', 18 | 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', 19 | 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', 20 | 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', 21 | } 22 | 23 | def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: 24 | """3x3 convolution with padding""" 25 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 26 | padding=dilation, groups=groups, bias=False, dilation=dilation) 27 | 28 | 29 | def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: 30 | """1x1 convolution""" 31 | return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) 32 | 33 | 34 | class BasicBlock(nn.Module): 35 | expansion: int = 1 36 | 37 | def __init__( 38 | self, 39 | inplanes: int, 40 | planes: int, 41 | stride: int = 1, 42 | downsample: Optional[nn.Module] = None, 43 | groups: int = 1, 44 | base_width: int = 64, 45 | dilation: int = 1, 46 | norm_layer: Optional[Callable[..., nn.Module]] = None 47 | ) -> None: 48 | super(BasicBlock, self).__init__() 49 | if norm_layer is None: 50 | norm_layer = nn.BatchNorm2d 51 | if groups != 1 or base_width != 64: 52 | raise ValueError('BasicBlock only supports groups=1 and base_width=64') 53 | if dilation > 1: 54 | raise NotImplementedError("Dilation > 1 not supported in BasicBlock") 55 | # Both self.conv1 and self.downsample layers downsample the input when stride != 1 56 | self.conv1 = conv3x3(inplanes, planes, stride) 57 | self.bn1 = norm_layer(planes) 58 | self.relu = nn.ReLU(inplace=True) 59 | self.conv2 = conv3x3(planes, planes) 60 | self.bn2 = norm_layer(planes) 61 | self.downsample = downsample 62 | self.stride = stride 63 | 64 | def forward(self, x: Tensor) -> Tensor: 65 | identity = x 66 | 67 | out = self.conv1(x) 68 | out = self.bn1(out) 69 | out = self.relu(out) 70 | 71 | out = self.conv2(out) 72 | out = self.bn2(out) 73 | 74 | if self.downsample is not None: 75 | identity = self.downsample(x) 76 | 77 | out += identity 78 | out = self.relu(out) 79 | 80 | return out 81 | 82 | 83 | class Bottleneck(nn.Module): 84 | # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) 85 | # while original implementation places the stride at the first 1x1 convolution(self.conv1) 86 | # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. 87 | # This variant is also known as ResNet V1.5 and improves accuracy according to 88 | # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. 89 | 90 | expansion: int = 4 91 | 92 | def __init__( 93 | self, 94 | inplanes: int, 95 | planes: int, 96 | stride: int = 1, 97 | downsample: Optional[nn.Module] = None, 98 | groups: int = 1, 99 | base_width: int = 64, 100 | dilation: int = 1, 101 | norm_layer: Optional[Callable[..., nn.Module]] = None 102 | ) -> None: 103 | super(Bottleneck, self).__init__() 104 | if norm_layer is None: 105 | norm_layer = nn.BatchNorm2d 106 | width = int(planes * (base_width / 64.)) * groups 107 | # Both self.conv2 and self.downsample layers downsample the input when stride != 1 108 | self.conv1 = conv1x1(inplanes, width) 109 | self.bn1 = norm_layer(width) 110 | self.conv2 = conv3x3(width, width, stride, groups, dilation) 111 | self.bn2 = norm_layer(width) 112 | self.conv3 = conv1x1(width, planes * self.expansion) 113 | self.bn3 = norm_layer(planes * self.expansion) 114 | self.relu = nn.ReLU(inplace=True) 115 | self.downsample = downsample 116 | self.stride = stride 117 | 118 | def forward(self, x: Tensor) -> Tensor: 119 | identity = x 120 | 121 | out = self.conv1(x) 122 | out = self.bn1(out) 123 | out = self.relu(out) 124 | 125 | out = self.conv2(out) 126 | out = self.bn2(out) 127 | out = self.relu(out) 128 | 129 | out = self.conv3(out) 130 | out = self.bn3(out) 131 | 132 | if self.downsample is not None: 133 | identity = self.downsample(x) 134 | 135 | out += identity 136 | out = self.relu(out) 137 | 138 | return out 139 | 140 | 141 | class ResNet(nn.Module): 142 | 143 | def __init__( 144 | self, 145 | block: Type[Union[BasicBlock, Bottleneck]], 146 | layers: List[int], 147 | num_classes: int = 1000, 148 | zero_init_residual: bool = False, 149 | groups: int = 1, 150 | width_per_group: int = 64, 151 | replace_stride_with_dilation: Optional[List[bool]] = None, 152 | norm_layer: Optional[Callable[..., nn.Module]] = None 153 | ) -> None: 154 | super(ResNet, self).__init__() 155 | if norm_layer is None: 156 | norm_layer = nn.BatchNorm2d 157 | self._norm_layer = norm_layer 158 | 159 | self.inplanes = 64 160 | self.dilation = 1 161 | if replace_stride_with_dilation is None: 162 | # each element in the tuple indicates if we should replace 163 | # the 2x2 stride with a dilated convolution instead 164 | replace_stride_with_dilation = [False, False, False] 165 | if len(replace_stride_with_dilation) != 3: 166 | raise ValueError("replace_stride_with_dilation should be None " 167 | "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) 168 | self.groups = groups 169 | self.base_width = width_per_group 170 | self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, 171 | bias=False) 172 | self.bn1 = norm_layer(self.inplanes) 173 | self.relu = nn.ReLU(inplace=True) 174 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 175 | self.layer1 = self._make_layer(block, 64, layers[0]) 176 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2, 177 | dilate=replace_stride_with_dilation[0]) 178 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2, 179 | dilate=replace_stride_with_dilation[1]) 180 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2, 181 | dilate=replace_stride_with_dilation[2]) 182 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 183 | self.fc = nn.Linear(512 * block.expansion, num_classes) 184 | 185 | for m in self.modules(): 186 | if isinstance(m, nn.Conv2d): 187 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 188 | elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): 189 | nn.init.constant_(m.weight, 1) 190 | nn.init.constant_(m.bias, 0) 191 | 192 | # Zero-initialize the last BN in each residual branch, 193 | # so that the residual branch starts with zeros, and each residual block behaves like an identity. 194 | # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 195 | if zero_init_residual: 196 | for m in self.modules(): 197 | if isinstance(m, Bottleneck): 198 | nn.init.constant_(m.bn3.weight, 0) # type: ignore[arg-type] 199 | elif isinstance(m, BasicBlock): 200 | nn.init.constant_(m.bn2.weight, 0) # type: ignore[arg-type] 201 | 202 | def _make_layer(self, block: Type[Union[BasicBlock, Bottleneck]], planes: int, blocks: int, 203 | stride: int = 1, dilate: bool = False) -> nn.Sequential: 204 | norm_layer = self._norm_layer 205 | downsample = None 206 | previous_dilation = self.dilation 207 | if dilate: 208 | self.dilation *= stride 209 | stride = 1 210 | if stride != 1 or self.inplanes != planes * block.expansion: 211 | downsample = nn.Sequential( 212 | conv1x1(self.inplanes, planes * block.expansion, stride), 213 | norm_layer(planes * block.expansion), 214 | ) 215 | 216 | layers = [] 217 | layers.append(block(self.inplanes, planes, stride, downsample, self.groups, 218 | self.base_width, previous_dilation, norm_layer)) 219 | self.inplanes = planes * block.expansion 220 | for _ in range(1, blocks): 221 | layers.append(block(self.inplanes, planes, groups=self.groups, 222 | base_width=self.base_width, dilation=self.dilation, 223 | norm_layer=norm_layer)) 224 | 225 | return nn.Sequential(*layers) 226 | 227 | def _forward_impl(self, x: Tensor) -> Tensor: 228 | # See note [TorchScript super()] 229 | x = self.conv1(x) 230 | x = self.bn1(x) 231 | x = self.relu(x) 232 | x = self.maxpool(x) 233 | 234 | x = self.layer1(x) 235 | x = self.layer2(x) 236 | x = self.layer3(x) 237 | x = self.layer4(x) 238 | 239 | x = self.avgpool(x) 240 | x = torch.flatten(x, 1) 241 | x = self.fc(x) 242 | return x 243 | 244 | def forward(self, x: Tensor) -> Tensor: 245 | return self._forward_impl(x) 246 | 247 | 248 | def adv_forward_impl(self, x: Tensor) -> Tensor: 249 | # See note [TorchScript super()] 250 | x = self.conv1(x) 251 | x = self.bn1(x) 252 | x = self.relu(x) 253 | x = self.maxpool(x) 254 | 255 | x1 = self.layer1(x) 256 | x2 = self.layer2(x1) 257 | x3 = self.layer3(x2) 258 | x4 = self.layer4(x3) 259 | 260 | x1 = self.avgpool(x1) 261 | x1 = torch.flatten(x1, 1) 262 | x1 = self.fc(x1) 263 | 264 | x2 = self.avgpool(x2) 265 | x2 = torch.flatten(x2, 1) 266 | x2 = self.fc(x2) 267 | 268 | x3 = self.avgpool(x3) 269 | x3 = torch.flatten(x3, 1) 270 | x3 = self.fc(x3) 271 | 272 | x4 = self.avgpool(x4) 273 | x4 = torch.flatten(x4, 1) 274 | x4 = self.fc(x4) 275 | 276 | 277 | x1 = torch.cat((x1, x1, x1, x1, x1, x1, x1, x1), dim=1) 278 | x2 = torch.cat((x2, x2, x2, x2) , dim=1) 279 | x3 = torch.cat((x3, x3), dim=1) 280 | x5 = torch.cat((x1, x2, x3, x4), dim=1) 281 | 282 | return x5 283 | 284 | def adv_forward(self, x: Tensor) -> Tensor: 285 | return self.adv_forward_impl(x) 286 | 287 | 288 | def _resnet( 289 | arch: str, 290 | block: Type[Union[BasicBlock, Bottleneck]], 291 | layers: List[int], 292 | pretrained: bool, 293 | progress: bool, 294 | **kwargs: Any 295 | ) -> ResNet: 296 | model = ResNet(block, layers, **kwargs) 297 | # if pretrained: 298 | # state_dict = load_state_dict_from_url(model_urls[arch], 299 | # progress=progress) 300 | # model.load_state_dict(state_dict) 301 | return model 302 | 303 | def resnet18(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 304 | r"""ResNet-18 model from 305 | `"Deep Residual Learning for Image Recognition" `_. 306 | 307 | Args: 308 | pretrained (bool): If True, returns a model pre-trained on ImageNet 309 | progress (bool): If True, displays a progress bar of the download to stderr 310 | """ 311 | return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, 312 | **kwargs) 313 | 314 | 315 | def resnet34(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 316 | r"""ResNet-34 model from 317 | `"Deep Residual Learning for Image Recognition" `_. 318 | 319 | Args: 320 | pretrained (bool): If True, returns a model pre-trained on ImageNet 321 | progress (bool): If True, displays a progress bar of the download to stderr 322 | """ 323 | return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, 324 | **kwargs) 325 | 326 | 327 | def resnet50(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 328 | r"""ResNet-50 model from 329 | `"Deep Residual Learning for Image Recognition" `_. 330 | 331 | Args: 332 | pretrained (bool): If True, returns a model pre-trained on ImageNet 333 | progress (bool): If True, displays a progress bar of the download to stderr 334 | """ 335 | return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, 336 | **kwargs) 337 | 338 | 339 | def resnet101(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 340 | r"""ResNet-101 model from 341 | `"Deep Residual Learning for Image Recognition" `_. 342 | 343 | Args: 344 | pretrained (bool): If True, returns a model pre-trained on ImageNet 345 | progress (bool): If True, displays a progress bar of the download to stderr 346 | """ 347 | return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, 348 | **kwargs) 349 | 350 | 351 | def resnet152(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 352 | r"""ResNet-152 model from 353 | `"Deep Residual Learning for Image Recognition" `_. 354 | 355 | Args: 356 | pretrained (bool): If True, returns a model pre-trained on ImageNet 357 | progress (bool): If True, displays a progress bar of the download to stderr 358 | """ 359 | return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, 360 | **kwargs) 361 | 362 | 363 | def resnext50_32x4d(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 364 | r"""ResNeXt-50 32x4d model from 365 | `"Aggregated Residual Transformation for Deep Neural Networks" `_. 366 | 367 | Args: 368 | pretrained (bool): If True, returns a model pre-trained on ImageNet 369 | progress (bool): If True, displays a progress bar of the download to stderr 370 | """ 371 | kwargs['groups'] = 32 372 | kwargs['width_per_group'] = 4 373 | return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], 374 | pretrained, progress, **kwargs) 375 | 376 | 377 | def resnext101_32x8d(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 378 | r"""ResNeXt-101 32x8d model from 379 | `"Aggregated Residual Transformation for Deep Neural Networks" `_. 380 | 381 | Args: 382 | pretrained (bool): If True, returns a model pre-trained on ImageNet 383 | progress (bool): If True, displays a progress bar of the download to stderr 384 | """ 385 | kwargs['groups'] = 32 386 | kwargs['width_per_group'] = 8 387 | return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], 388 | pretrained, progress, **kwargs) 389 | 390 | 391 | def wide_resnet50_2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 392 | r"""Wide ResNet-50-2 model from 393 | `"Wide Residual Networks" `_. 394 | 395 | The model is the same as ResNet except for the bottleneck number of channels 396 | which is twice larger in every block. The number of channels in outer 1x1 397 | convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 398 | channels, and in Wide ResNet-50-2 has 2048-1024-2048. 399 | 400 | Args: 401 | pretrained (bool): If True, returns a model pre-trained on ImageNet 402 | progress (bool): If True, displays a progress bar of the download to stderr 403 | """ 404 | kwargs['width_per_group'] = 64 * 2 405 | return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], 406 | pretrained, progress, **kwargs) 407 | 408 | 409 | def wide_resnet101_2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: 410 | r"""Wide ResNet-101-2 model from 411 | `"Wide Residual Networks" `_. 412 | 413 | The model is the same as ResNet except for the bottleneck number of channels 414 | which is twice larger in every block. The number of channels in outer 1x1 415 | convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 416 | channels, and in Wide ResNet-50-2 has 2048-1024-2048. 417 | 418 | Args: 419 | pretrained (bool): If True, returns a model pre-trained on ImageNet 420 | progress (bool): If True, displays a progress bar of the download to stderr 421 | """ 422 | kwargs['width_per_group'] = 64 * 2 423 | return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], 424 | pretrained, progress, **kwargs) 425 | --------------------------------------------------------------------------------