├── Hessian ├── README.md ├── curves.py ├── evalacc.py ├── hessian_eig_driver.py ├── hessianflow │ ├── __init__.py │ ├── eigen.py │ ├── optimizer │ │ ├── __init__.py │ │ ├── absa.py │ │ ├── baseline.py │ │ ├── optm_utils.py │ │ └── progressbar.py │ └── utils.py ├── models │ ├── c1.py │ ├── convfc.py │ ├── resnet.py │ └── vgg.py ├── train.py ├── train_resnet.py └── utils.py ├── LICENSE ├── README.md ├── backdoor ├── backdoor-cifar │ ├── AttackPGD.py │ ├── README.md │ ├── aver_weights.py │ ├── aver_weights_case2.py │ ├── curves.py │ ├── data.py │ ├── eval_curve.py │ ├── evalacc.py │ ├── evalacc2.py │ ├── models │ │ ├── __init__.py │ │ ├── convfc.py │ │ ├── preresnet.py │ │ ├── vgg.py │ │ ├── vggW.py │ │ └── wide_resnet.py │ ├── save_model.py │ ├── test_curve.py │ ├── train.py │ ├── train_baseline.py │ ├── train_baseline_from_scratch.py │ ├── train_poison.py │ ├── train_poison_same_poisoned_data.py │ └── utils.py ├── backdoor-svhn │ ├── AttackPGD.py │ ├── README.md │ ├── aver_weights.py │ ├── aver_weights_case2.py │ ├── aver_weights_case3.py │ ├── curves.py │ ├── data.py │ ├── eval_curve.py │ ├── evalacc.py │ ├── evalacc2.py │ ├── models │ │ ├── __init__.py │ │ ├── convfc.py │ │ ├── preresnet.py │ │ ├── vgg.py │ │ ├── vggW.py │ │ └── wide_resnet.py │ ├── save_model.py │ ├── test_curve.py │ ├── train.py │ ├── train_baseline.py │ ├── train_baseline_from_scratch.py │ ├── train_poison.py │ ├── train_poison_same_poisoned_data.py │ └── utils.py ├── prune_cifar │ ├── README.md │ ├── compute_flops.py │ ├── evalacc.py │ ├── evalacc2.py │ ├── main.py │ ├── main_B.py │ ├── main_E.py │ ├── main_finetune.py │ ├── models │ │ ├── __init__.py │ │ ├── resnet.py │ │ └── vgg.py │ ├── res110prune.py │ ├── res56prune.py │ ├── utils.py │ └── vggprune.py └── prune_svhn │ ├── README.md │ ├── compute_flops.py │ ├── evalacc.py │ ├── evalacc2.py │ ├── main.py │ ├── main_B.py │ ├── main_E.py │ ├── main_finetune.py │ ├── models │ ├── __init__.py │ ├── resnet.py │ └── vgg.py │ ├── res110prune.py │ ├── res56prune.py │ ├── utils.py │ └── vggprune.py ├── error-injection ├── injection_cifar │ ├── AttackPGD.py │ ├── README.md │ ├── aver_weights_case2.py │ ├── curves.py │ ├── data.py │ ├── data1.py │ ├── eval_curve.py │ ├── evalacc.py │ ├── evalacc2.py │ ├── evalacc2_injection.py │ ├── evalacc_injection.py │ ├── evalacc_injection_VGG.py │ ├── models │ │ ├── __init__.py │ │ ├── convfc.py │ │ ├── preresnet.py │ │ ├── vgg.py │ │ ├── vggW.py │ │ └── wide_resnet.py │ ├── save_model.py │ ├── test_curve.py │ ├── train.py │ ├── train_baseline.py │ ├── train_injection.py │ └── utils.py └── injection_svhn │ ├── AttackPGD.py │ ├── README.md │ ├── aver_weights_case2.py │ ├── curves.py │ ├── data.py │ ├── data0.py │ ├── data1.py │ ├── eval_curve.py │ ├── evalacc.py │ ├── evalacc2.py │ ├── evalacc2_injection.py │ ├── evalacc_injection.py │ ├── evalacc_injection_VGG.py │ ├── models │ ├── __init__.py │ ├── convfc.py │ ├── preresnet.py │ ├── vgg.py │ ├── vggW.py │ └── wide_resnet.py │ ├── save_model.py │ ├── test_curve.py │ ├── train.py │ ├── train_baseline.py │ ├── train_injection.py │ └── utils.py └── evasion_attack ├── AttackPGD.py ├── README.md ├── connect.py ├── curves.py ├── data.py ├── eval_curve.py ├── eval_curve_robustness.py ├── models ├── __init__.py ├── convfc.py ├── preresnet.py ├── vgg.py ├── vggW.py └── wide_resnet.py ├── save_model.py ├── test_curve.py ├── train.py ├── train_robust_connection.py └── utils.py /Hessian/README.md: -------------------------------------------------------------------------------- 1 | ## Hessian Flow: A Library for Hessian Based Algorithms in Machine Learning 2 | 3 | HessianFlow is a pytorch library for Hessian based analysis of neural network models that could be used in conjunction with pytorch. 4 | The library currently supports utility functions to compute Hessian spectrum of different neural network 5 | models. 6 | 7 | ## ABSA: Adaptive Batch Size with Adversarial training: 8 | This method uses second order information to adaptively increase batch size when SGD training gets to flatter 9 | landscapes. Example codes to run with ABSA: 10 | 11 | python train.py --name cifar10 --epochs 90 --lr 0.02 --lr-decay 0.2 --lr-decay-epoch 30 60 --arch c1 12 | 13 | python train.py --arch ResNet --lr 0.1 --lr-decay 0.2 --lr-decay-epoch 30 60 80 --large-ratio 128 --method absa 14 | 15 | 16 | ## Track Eigen During the whole training procedure of ResNet on cifar10 17 | One could simply modify the above example to track the spectrum of the Hessian throughout training: 18 | 19 | python train_resnet.pt --name cifar10 --epoch 160 --lr 0.1 --lr-decay 0.2 --lr-decay-epoch 60 90 120 20 | 21 | ## Hessian Spectrum Computation 22 | After installing pytoch, the following command trains a ResNet on Cifar-10 and prints the dominant Hessian eigenvalue at every epoch: 23 | 24 | python train_resnet.py --name cifar10 --epoch 160 --lr 0.1 --lr-decay 0.2 --lr-decay-epoch 60 90 120 25 | 26 | You can also use the library to compute Hessian spectrum at different snapshots after training is finished. For instance, here we first 27 | train a simple network: 28 | 29 | python train.py --name cifar10 --epochs 90 --lr 0.02 --lr-decay 0.2 --lr-decay-epoch 30 60 --arch c1 30 | 31 | And afterwards we load the checkpoint and compute the Hessian spectrum at that point: 32 | 33 | python hessian_eig_driver.py --name cifar10 --arch c1 --resume model_param.pkl --eigen-type full 34 | 35 | 36 | 37 | 38 | 39 | ### Evaluate the Hessian of the models on the path 40 | 41 | first use the save_models.py to obtain 11 models on the model connection. Then get the Hessian of each model with the above method. 42 | -------------------------------------------------------------------------------- /Hessian/hessian_eig_driver.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | import argparse 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.optim as optim 8 | from torchvision import datasets, transforms 9 | from torch.autograd import Variable 10 | import torch.backends.cudnn as cudnn 11 | 12 | from utils import * 13 | from models.c1 import * 14 | from models.convfc import * 15 | from models.vgg import * 16 | import hessianflow as hf 17 | import hessianflow.optimizer as hf_optm 18 | 19 | # Training settings 20 | parser = argparse.ArgumentParser(description = 'PyTorch Example') 21 | parser.add_argument('--name', type = str, default = 'cifar10', metavar = 'N', 22 | help='dataset') 23 | parser.add_argument('--batch-size', type = int, default = 128, metavar = 'N', 24 | help='input batch size for training (default: 64)') 25 | parser.add_argument('--test-batch-size', type = int, default = 200, metavar = 'N', 26 | help='input batch size for testing (default: 1000)') 27 | 28 | parser.add_argument('--seed', type = int, default = 1, metavar = 'S', 29 | help = 'random seed (default: 1)') 30 | parser.add_argument('--arch', type = str, default = 'VGG16', 31 | help = 'choose the archtecure') 32 | 33 | parser.add_argument('--eigen-type', type = str, default = 'full', 34 | help = 'full dataset of subset') 35 | parser.add_argument('--resume', type = str, default = '../modelconnect/VGG16-128-256_robust_split/checkpoint-10.pt', 36 | #'../adversarial_train/checkpoint-CFC-robust/checkpoint-180.pt', 37 | help = 'choose the archtecure') 38 | 39 | args = parser.parse_args() 40 | # set random seed to reproduce the work 41 | torch.manual_seed(args.seed) 42 | torch.cuda.manual_seed(args.seed) 43 | 44 | # get dataset 45 | train_loader, test_loader = getData(name = args.name, train_bs = args.batch_size, test_bs = args.test_batch_size) 46 | 47 | # get model and optimizer 48 | model_list = { 49 | 'c1':c1_model(), 50 | 'c2':c2_model(), 51 | 'CFC': ConvFC.base(num_classes=10), 52 | 'VGG16': VGG16.base(num_classes=10), 53 | } 54 | 55 | #model = VGG16.base(num_classes=10) 56 | #model = model.to('cuda') 57 | checkpoint = torch.load(args.resume) 58 | model = model_list[args.arch].cuda() 59 | #model = torch.nn.DataParallel(model) 60 | model.cuda() 61 | cudnn.benchmark = True 62 | #model = torch.nn.DataParallel(model) 63 | model.load_state_dict(checkpoint['model_state']) 64 | 65 | print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) 66 | 67 | criterion = nn.CrossEntropyLoss() 68 | 69 | 70 | if args.eigen_type == 'full': 71 | # compute the eigen information based on the whole testing data set 72 | # eigenvalue, eigenvector = hf.get_eigen_full_dataset(model, test_loader, criterion, cuda = True, maxIter = 10, tol = 1e-2) 73 | 74 | # eigenvalue, eigenvector = hf.get_eigen_full_dataset_with_inputs(model, test_loader, criterion, cuda = True, maxIter = 10, tol = 1e-2) 75 | 76 | eigenvalue, eigenvector = hf.get_eigen_full_dataset_only_inputs(model, test_loader, criterion, cuda = True, maxIter = 10, tol = 1e-2) 77 | 78 | 79 | elif args.eigen_type == 'approximate': 80 | # here we choose a random batch from test_loader to compute approximating eigen information 81 | get_data = True 82 | for data, target in test_loader: 83 | # finish the for loop otherwise there is a warning 84 | if get_data: 85 | inputs = data 86 | targets = target 87 | get_data = False 88 | 89 | inputs.requires_grad = True 90 | 91 | eigenvalue, eigenvector = hf.get_eigen(model, inputs, targets, criterion, cuda = True, maxIter = 10, tol = 1e-2) 92 | # eigenvalue, eigenvector = hf.get_eigen_with_inputs(model, inputs, targets, criterion, cuda=True, maxIter=10, tol=1e-2) 93 | 94 | # eigenvalue, eigenvector = hf.get_eigen_only_inputs(model, inputs, targets, criterion, cuda=True, maxIter=10, tol=1e-2) 95 | 96 | print('Eigenvalue is: %.5f' % eigenvalue) 97 | -------------------------------------------------------------------------------- /Hessian/hessianflow/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | name = 'Hessian Flow' 3 | 4 | from .eigen import * 5 | -------------------------------------------------------------------------------- /Hessian/hessianflow/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | from .baseline import baseline 2 | from .absa import absa 3 | -------------------------------------------------------------------------------- /Hessian/hessianflow/optimizer/baseline.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | 10 | from .progressbar import progress_bar 11 | from .optm_utils import exp_lr_scheduler, test 12 | 13 | # import hessianflow 14 | 15 | def baseline(model, train_loader, test_loader, criterion, optimizer, epochs, lr_decay_epoch, 16 | lr_decay_ratio, batch_size = 128, max_large_ratio = 1, cuda = True): 17 | """ 18 | baseline method training, i,e, vanilla training schedule 19 | """ 20 | 21 | inner_loop = 0 22 | num_updates = 0 23 | large_ratio = max_large_ratio 24 | # assert that shuffle is set for train_loader 25 | # assert and explain large ratio 26 | # assert that the train_loader is always set with a small batch size if not print error/warning telling 27 | # the user to instead use large_ratio 28 | for epoch in range(1, epochs + 1): 29 | print('\nCurrent Epoch: ', epoch) 30 | print('\nTraining') 31 | train_loss = 0. 32 | total_num = 0. 33 | correct = 0. 34 | 35 | for batch_idx, (data, target) in enumerate(train_loader): 36 | if target.size(0) < 128: 37 | continue 38 | model.train() 39 | # gather input and target for large batch training 40 | inner_loop += 1 41 | # get small model update 42 | if cuda: 43 | data, target = data.cuda(), target.cuda() 44 | output = model(data) 45 | loss = criterion(output, target)/float(large_ratio) 46 | loss.backward() 47 | train_loss += loss.item()*target.size(0)*float(large_ratio) 48 | total_num += target.size(0) 49 | _, predicted = output.max(1) 50 | correct += predicted.eq(target).sum().item() 51 | 52 | if inner_loop % large_ratio == 0: 53 | num_updates += 1 54 | optimizer.step() 55 | inner_loop = 0 56 | optimizer.zero_grad() 57 | 58 | progress_bar(batch_idx, len(train_loader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' % 59 | (train_loss / total_num, 100. * correct / total_num, correct, total_num)) 60 | 61 | if epoch in lr_decay_epoch: 62 | exp_lr_scheduler(optimizer, decay_ratio=lr_decay_ratio) 63 | 64 | test(model, test_loader) 65 | return model, num_updates, epoch 66 | -------------------------------------------------------------------------------- /Hessian/hessianflow/optimizer/optm_utils.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import print_function 3 | import numpy as np 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.optim as optim 8 | from torchvision import datasets, transforms 9 | from torch.autograd import Variable 10 | from .progressbar import progress_bar 11 | 12 | 13 | def fgsm(model, data, target, eps, cuda = True): 14 | """Generate an adversarial pertubation using the fast gradient sign method. 15 | 16 | Args: 17 | data: input image to perturb 18 | """ 19 | model.eval() 20 | if cuda: 21 | data, target = data.cuda(), target.cuda() 22 | data.requires_grad = True 23 | model.zero_grad() 24 | output = model(data) 25 | loss = F.cross_entropy(output, target) 26 | loss.backward(create_graph = False) 27 | pertubation = eps * torch.sign(data.grad.data) 28 | x_fgsm = data.data + pertubation 29 | X_adv = torch.clamp(x_fgsm, torch.min(data.data), torch.max(data.data)) 30 | 31 | return X_adv.cpu() 32 | 33 | def exp_lr_scheduler(optimizer, decay_ratio = 0.1): 34 | """ 35 | Decay learning rate by a factor of lr_decay 36 | """ 37 | for param_group in optimizer.param_groups: 38 | param_group['lr'] *= decay_ratio 39 | return optimizer 40 | 41 | 42 | def test(model, test_loader): 43 | """ 44 | Evaluation the performance of model on test_loader 45 | """ 46 | print('\nTesting') 47 | model.eval() 48 | correct = 0 49 | total = 0 50 | with torch.no_grad(): 51 | for batch_idx, (inputs, targets) in enumerate(test_loader): 52 | inputs, targets = inputs.cuda(), targets.cuda() 53 | outputs = model(inputs) 54 | _, predicted = outputs.max(1) 55 | total += targets.size(0) 56 | correct += predicted.eq(targets).sum().item() 57 | 58 | progress_bar(batch_idx, len(test_loader), 'Acc: %.3f%% (%d/%d)' 59 | % (100. * correct/total, correct, total)) 60 | 61 | return correct * 100 / total 62 | -------------------------------------------------------------------------------- /Hessian/hessianflow/optimizer/progressbar.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import sys 4 | import time 5 | import math 6 | 7 | ######## fancy progress bar 8 | try: 9 | _, term_width = os.popen('stty size', 'r').read().split() 10 | except: 11 | term_width = 100 12 | term_width = int(term_width) 13 | 14 | 15 | TOTAL_BAR_LENGTH = 65. 16 | last_time = time.time() 17 | begin_time = last_time 18 | def progress_bar(current, total, msg=None): 19 | global last_time, begin_time 20 | if current == 0: 21 | begin_time = time.time() # Reset for new bar. 22 | 23 | cur_len = int(TOTAL_BAR_LENGTH*current/total) 24 | rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1 25 | 26 | sys.stdout.write(' [') 27 | for i in range(cur_len): 28 | sys.stdout.write('=') 29 | sys.stdout.write('>') 30 | for i in range(rest_len): 31 | sys.stdout.write('.') 32 | sys.stdout.write(']') 33 | 34 | cur_time = time.time() 35 | step_time = cur_time - last_time 36 | last_time = cur_time 37 | tot_time = cur_time - begin_time 38 | 39 | L = [] 40 | L.append(' Step: %s' % format_time(step_time)) 41 | L.append(' | Tot: %s' % format_time(tot_time)) 42 | if msg: 43 | L.append(' | ' + msg) 44 | 45 | msg = ''.join(L) 46 | sys.stdout.write(msg) 47 | for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3): 48 | sys.stdout.write(' ') 49 | 50 | # Go back to the center of the bar. 51 | for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2): 52 | sys.stdout.write('\b') 53 | sys.stdout.write(' %d/%d ' % (current+1, total)) 54 | 55 | if current < total-1: 56 | sys.stdout.write('\r') 57 | else: 58 | sys.stdout.write('\n') 59 | sys.stdout.flush() 60 | 61 | def format_time(seconds): 62 | days = int(seconds / 3600/24) 63 | seconds = seconds - days*3600*24 64 | hours = int(seconds / 3600) 65 | seconds = seconds - hours*3600 66 | minutes = int(seconds / 60) 67 | seconds = seconds - minutes*60 68 | secondsf = int(seconds) 69 | seconds = seconds - secondsf 70 | millis = int(seconds*1000) 71 | 72 | f = '' 73 | i = 1 74 | if days > 0: 75 | f += str(days) + 'D' 76 | i += 1 77 | if hours > 0 and i <= 2: 78 | f += str(hours) + 'h' 79 | i += 1 80 | if minutes > 0 and i <= 2: 81 | f += str(minutes) + 'm' 82 | i += 1 83 | if secondsf > 0 and i <= 2: 84 | f += str(secondsf) + 's' 85 | i += 1 86 | if millis > 0 and i <= 2: 87 | f += str(millis) + 'ms' 88 | i += 1 89 | if f == '': 90 | f = '0ms' 91 | return f 92 | -------------------------------------------------------------------------------- /Hessian/hessianflow/utils.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import math 5 | from torch.autograd import Variable 6 | import numpy as np 7 | 8 | 9 | def group_product(xs, ys): 10 | """ 11 | the inner product of two lists of variables xs,ys 12 | :param xs: 13 | :param ys: 14 | :return: 15 | """ 16 | return sum([torch.sum(x * y) for (x, y) in zip(xs, ys)]) 17 | 18 | def group_add(params, update, alpha=1): 19 | """ 20 | params = params + update*alpha 21 | :param params: list of variable 22 | :param update: list of data 23 | :return: 24 | """ 25 | for i,p in enumerate(params): 26 | params[i].data.add_(update[i] * alpha) 27 | return params 28 | 29 | def normalization(v): 30 | """ 31 | normalization of a list of vectors 32 | return: normalized vectors v 33 | """ 34 | s = group_product(v,v) 35 | s = s ** 0.5 36 | s = s.cpu().item() 37 | v = [vi / (s + 1e-6) for vi in v] 38 | return v 39 | 40 | 41 | def get_params_grad(model): 42 | """ 43 | get model parameters and corresponding gradients 44 | """ 45 | params = [] 46 | grads = [] 47 | for param in model.parameters(): 48 | params.append(param) 49 | if param.grad is None: 50 | continue 51 | grads.append(param.grad + 0.) 52 | return params, grads 53 | 54 | def get_params_grad_with_inputs(model, inputs): 55 | """ 56 | get model parameters and corresponding gradients 57 | """ 58 | params = [] 59 | grads = [] 60 | for param in model.parameters(): 61 | params.append(param) 62 | if param.grad is None: 63 | continue 64 | grads.append(param.grad + 0.) 65 | 66 | params.append(inputs) 67 | grads.append(inputs.grad) 68 | return params, grads 69 | 70 | 71 | def get_params_grad_only_inputs(inputs): 72 | """ 73 | get model parameters and corresponding gradients 74 | """ 75 | params = [] 76 | grads = [] 77 | # for param in model.parameters(): 78 | # params.append(param) 79 | # if param.grad is None: 80 | # continue 81 | # grads.append(param.grad + 0.) 82 | 83 | params.append(inputs) 84 | grads.append(inputs.grad) 85 | return params, grads 86 | 87 | 88 | def hessian_vector_product(gradsH, params, v): 89 | """ 90 | compute the hessian vector product of Hv, where 91 | gradsH is the gradient at the current point, 92 | params is the corresponding variables, 93 | v is the vector. 94 | """ 95 | hv = torch.autograd.grad(gradsH, params, grad_outputs = v, only_inputs = True, retain_graph = True) 96 | return hv 97 | 98 | -------------------------------------------------------------------------------- /Hessian/models/c1.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from __future__ import print_function 4 | import numpy as np 5 | import argparse 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | from torchvision import datasets, transforms 11 | from torch.autograd import Variable 12 | 13 | class c1_model(nn.Module): 14 | 15 | def __init__(self, num_classes=10): 16 | super(c1_model, self).__init__() 17 | self.conv1=nn.Conv2d(3, 64, kernel_size=5, stride=1, padding=2)# 32x32x3 -> 32x32x64 18 | self.bn1=nn.BatchNorm2d(64) 19 | self.conv2=nn.Conv2d(64, 64, kernel_size=5, stride =1, padding=2)# 16x16x64 -> 16x16x64 20 | self.bn2=nn.BatchNorm2d(64) 21 | self.fc1= nn.Linear(64*8*8, 384) 22 | self.fc2=nn.Linear(384,192) 23 | self.fc3=nn.Linear(192,num_classes) 24 | 25 | 26 | def forward(self, x): 27 | x = F.max_pool2d(self.bn1(F.relu(self.conv1(x))),3,2,1) 28 | x = F.max_pool2d(self.bn2(F.relu(self.conv2(x))),3,2,1) 29 | x = x.view(x.size(0), -1) 30 | x = F.relu(self.fc1(x)) 31 | x = F.relu(self.fc2(x)) 32 | x = self.fc3(x) 33 | return x 34 | 35 | 36 | 37 | class c2_model(nn.Module): 38 | def __init__(self): 39 | super(c2_model, self).__init__() 40 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3) 41 | self.bn1 = nn.BatchNorm2d(64) 42 | self.conv2 = nn.Conv2d(64, 64, kernel_size=3) 43 | self.bn2 = nn.BatchNorm2d(64) 44 | self.conv3 = nn.Conv2d(64,128, kernel_size=3) 45 | self.bn3 = nn.BatchNorm2d(128) 46 | self.conv4 = nn.Conv2d(128,128, kernel_size=3) 47 | self.bn4 = nn.BatchNorm2d(128) 48 | self.conv_drop = nn.Dropout2d() 49 | self.fc1 = nn.Linear(128*5*5, 256) 50 | self.fc2 = nn.Linear(256, 256) 51 | self.fc3 = nn.Linear(256, 10) 52 | self.drop = nn.Dropout() 53 | 54 | def forward(self, x): 55 | x = self.bn1(F.relu(self.conv1(x))) 56 | x = F.max_pool2d(self.bn2(F.relu(self.conv2(x))) ,2) 57 | x = self.bn3(F.relu(self.conv3(x))) 58 | x = F.max_pool2d(self.bn4(F.relu(self.conv4(x))) ,2) 59 | #x = self.conv_drop(x) 60 | x = x.view(-1, 128*5*5) 61 | x = F.relu(self.fc1(x)) 62 | x = F.relu(self.fc2(x)) 63 | x = self.fc3(x) 64 | return x 65 | -------------------------------------------------------------------------------- /Hessian/models/convfc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | 4 | import curves 5 | 6 | __all__ = [ 7 | 'ConvFC', 8 | ] 9 | 10 | 11 | class ConvFCBase(nn.Module): 12 | 13 | def __init__(self, num_classes): 14 | super(ConvFCBase, self).__init__() 15 | self.conv_part = nn.Sequential( 16 | nn.Conv2d(3, 32, kernel_size=5, padding=2), 17 | nn.ReLU(True), 18 | nn.MaxPool2d(kernel_size=3, stride=2), 19 | nn.Conv2d(32, 64, kernel_size=5, padding=2), 20 | nn.ReLU(True), 21 | nn.MaxPool2d(3, 2), 22 | nn.Conv2d(64, 128, kernel_size=5, padding=2), 23 | nn.ReLU(True), 24 | nn.MaxPool2d(3, 2), 25 | ) 26 | self.fc_part = nn.Sequential( 27 | nn.Linear(1152, 1000), 28 | nn.ReLU(True), 29 | nn.Linear(1000, 1000), 30 | nn.ReLU(True), 31 | nn.Linear(1000, num_classes) 32 | ) 33 | 34 | # Initialize weights 35 | for m in self.conv_part.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 38 | m.weight.data.normal_(0, math.sqrt(2. / n)) 39 | m.bias.data.zero_() 40 | 41 | def forward(self, x): 42 | x = self.conv_part(x) 43 | x = x.view(x.size(0), -1) 44 | x = self.fc_part(x) 45 | return x 46 | 47 | 48 | class ConvFCCurve(nn.Module): 49 | def __init__(self, num_classes, fix_points): 50 | super(ConvFCCurve, self).__init__() 51 | self.conv1 = curves.Conv2d(3, 32, kernel_size=5, padding=2, fix_points=fix_points) 52 | self.relu1 = nn.ReLU(True) 53 | self.max_pool1 = nn.MaxPool2d(kernel_size=3, stride=2) 54 | 55 | self.conv2 = curves.Conv2d(32, 64, kernel_size=5, padding=2, fix_points=fix_points) 56 | self.relu2 = nn.ReLU(True) 57 | self.max_pool2 = nn.MaxPool2d(3, 2) 58 | 59 | self.conv3 = curves.Conv2d(64, 128, kernel_size=5, padding=2, fix_points=fix_points) 60 | self.relu3 = nn.ReLU(True) 61 | self.max_pool3 = nn.MaxPool2d(3, 2) 62 | 63 | self.fc4 = curves.Linear(1152, 1000, fix_points=fix_points) 64 | self.relu4 = nn.ReLU(True) 65 | 66 | self.fc5 = curves.Linear(1000, 1000, fix_points=fix_points) 67 | self.relu5 = nn.ReLU(True) 68 | 69 | self.fc6 = curves.Linear(1000, num_classes, fix_points=fix_points) 70 | 71 | # Initialize weights 72 | for m in self.modules(): 73 | if isinstance(m, curves.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | for i in range(m.num_bends): 76 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 77 | getattr(m, 'bias_%d' % i).data.zero_() 78 | 79 | def forward(self, x, coeffs_t): 80 | x = self.conv1(x, coeffs_t) 81 | x = self.relu1(x) 82 | x = self.max_pool1(x) 83 | 84 | x = self.conv2(x, coeffs_t) 85 | x = self.relu2(x) 86 | x = self.max_pool2(x) 87 | 88 | x = self.conv3(x, coeffs_t) 89 | x = self.relu3(x) 90 | x = self.max_pool3(x) 91 | 92 | x = x.view(x.size(0), -1) 93 | 94 | x = self.fc4(x, coeffs_t) 95 | x = self.relu4(x) 96 | 97 | x = self.fc5(x, coeffs_t) 98 | x = self.relu5(x) 99 | 100 | x = self.fc6(x, coeffs_t) 101 | 102 | return x 103 | 104 | 105 | class ConvFC: 106 | base = ConvFCBase 107 | curve = ConvFCCurve 108 | kwargs = {} 109 | -------------------------------------------------------------------------------- /Hessian/models/resnet.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | import torch.nn as nn 5 | import math 6 | 7 | 8 | __all__ = ['resnet'] 9 | 10 | def conv3x3(in_planes, out_planes, stride=1): 11 | "3x3 convolution with padding" 12 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 13 | padding=1, bias=False) 14 | 15 | 16 | class BasicBlock(nn.Module): 17 | expansion = 1 18 | 19 | def __init__(self, inplanes, planes, stride=1, downsample=None): 20 | super(BasicBlock, self).__init__() 21 | self.conv1 = conv3x3(inplanes, planes, stride) 22 | self.bn1 = nn.BatchNorm2d(planes) 23 | self.relu = nn.ReLU(inplace=True) 24 | self.conv2 = conv3x3(planes, planes) 25 | self.bn2 = nn.BatchNorm2d(planes) 26 | self.downsample = downsample 27 | self.stride = stride 28 | 29 | def forward(self, x): 30 | residual = x 31 | 32 | out = self.conv1(x) 33 | out = self.bn1(out) 34 | out = self.relu(out) 35 | 36 | out = self.conv2(out) 37 | out = self.bn2(out) 38 | 39 | if self.downsample is not None: 40 | residual = self.downsample(x) 41 | 42 | out += residual 43 | out = self.relu(out) 44 | 45 | return out 46 | 47 | 48 | class Bottleneck(nn.Module): 49 | expansion = 4 50 | 51 | def __init__(self, inplanes, planes, stride=1, downsample=None): 52 | super(Bottleneck, self).__init__() 53 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 54 | self.bn1 = nn.BatchNorm2d(planes) 55 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, 56 | padding=1, bias=False) 57 | self.bn2 = nn.BatchNorm2d(planes) 58 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 59 | self.bn3 = nn.BatchNorm2d(planes * 4) 60 | self.relu = nn.ReLU(inplace=True) 61 | self.downsample = downsample 62 | self.stride = stride 63 | 64 | def forward(self, x): 65 | residual = x 66 | 67 | out = self.conv1(x) 68 | out = self.bn1(out) 69 | out = self.relu(out) 70 | 71 | out = self.conv2(out) 72 | out = self.bn2(out) 73 | out = self.relu(out) 74 | 75 | out = self.conv3(out) 76 | out = self.bn3(out) 77 | 78 | if self.downsample is not None: 79 | residual = self.downsample(x) 80 | 81 | out += residual 82 | out = self.relu(out) 83 | 84 | return out 85 | 86 | ALPHA_=1 87 | class ResNet(nn.Module): 88 | 89 | def __init__(self, depth, num_classes=10): 90 | super(ResNet, self).__init__() 91 | # Model type specifies number of layers for CIFAR-10 model 92 | assert (depth - 2) % 6 == 0, 'depth should be 6n+2' 93 | n = (depth - 2) // 6 94 | 95 | block = Bottleneck if depth >=44 else BasicBlock 96 | 97 | self.inplanes = 16*ALPHA_ 98 | self.conv1 = nn.Conv2d(3, 16*ALPHA_, kernel_size=3, padding=1, 99 | bias=False) 100 | self.bn1 = nn.BatchNorm2d(16*ALPHA_) 101 | self.relu = nn.ReLU(inplace=True) 102 | self.layer1 = self._make_layer(block, 16*ALPHA_, n) 103 | self.layer2 = self._make_layer(block, 32*ALPHA_, n, stride=2) 104 | self.layer3 = self._make_layer(block, 64*ALPHA_, n, stride=2) 105 | self.avgpool = nn.AvgPool2d(8) 106 | self.fc = nn.Linear(64 * ALPHA_* block.expansion, num_classes) 107 | 108 | for m in self.modules(): 109 | if isinstance(m, nn.Conv2d): 110 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 111 | m.weight.data.normal_(0, math.sqrt(2. / n)) 112 | elif isinstance(m, nn.BatchNorm2d): 113 | m.weight.data.fill_(1) 114 | m.bias.data.zero_() 115 | 116 | def _make_layer(self, block, planes, blocks, stride=1): 117 | downsample = None 118 | if stride != 1 or self.inplanes != planes * block.expansion: 119 | downsample = nn.Sequential( 120 | nn.Conv2d(self.inplanes, planes * block.expansion, 121 | kernel_size=1, stride=stride, bias=False), 122 | nn.BatchNorm2d(planes * block.expansion), 123 | ) 124 | 125 | layers = [] 126 | layers.append(block(self.inplanes, planes, stride, downsample)) 127 | self.inplanes = planes * block.expansion 128 | for i in range(1, blocks): 129 | layers.append(block(self.inplanes, planes)) 130 | 131 | return nn.Sequential(*layers) 132 | 133 | def forward(self, x): 134 | x = self.conv1(x) 135 | x = self.bn1(x) 136 | x = self.relu(x) # 32x32 137 | 138 | x = self.layer1(x) # 32x32 139 | x = self.layer2(x) # 16x16 140 | x = self.layer3(x) # 8x8 141 | 142 | x = self.avgpool(x) 143 | x = x.view(x.size(0), -1) 144 | x = self.fc(x) 145 | 146 | return x 147 | 148 | 149 | def resnet(**kwargs): 150 | """ 151 | Constructs a ResNet model. 152 | """ 153 | return ResNet(**kwargs) 154 | -------------------------------------------------------------------------------- /Hessian/models/vgg.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import math 4 | import torch.nn as nn 5 | 6 | import curves 7 | 8 | __all__ = ['VGG16', 'VGG16BN', 'VGG19', 'VGG19BN'] 9 | 10 | config = { 11 | 16: [[64, 64], [128, 128], [256, 256, 256], [512, 512, 512], [512, 512, 512]], 12 | 19: [[64, 64], [128, 128], [256, 256, 256, 256], [512, 512, 512, 512], [512, 512, 512, 512]], 13 | } 14 | 15 | 16 | def make_layers(config, batch_norm=False, fix_points=None): 17 | layer_blocks = nn.ModuleList() 18 | activation_blocks = nn.ModuleList() 19 | poolings = nn.ModuleList() 20 | 21 | kwargs = dict() 22 | conv = nn.Conv2d 23 | bn = nn.BatchNorm2d 24 | if fix_points is not None: 25 | kwargs['fix_points'] = fix_points 26 | conv = curves.Conv2d 27 | bn = curves.BatchNorm2d 28 | 29 | in_channels = 3 30 | for sizes in config: 31 | layer_blocks.append(nn.ModuleList()) 32 | activation_blocks.append(nn.ModuleList()) 33 | for channels in sizes: 34 | layer_blocks[-1].append(conv(in_channels, channels, kernel_size=3, padding=1, **kwargs)) 35 | if batch_norm: 36 | layer_blocks[-1].append(bn(channels, **kwargs)) 37 | activation_blocks[-1].append(nn.ReLU(inplace=True)) 38 | in_channels = channels 39 | poolings.append(nn.MaxPool2d(kernel_size=2, stride=2)) 40 | return layer_blocks, activation_blocks, poolings 41 | 42 | 43 | class VGGBase(nn.Module): 44 | def __init__(self, num_classes, depth=16, batch_norm=False): 45 | super(VGGBase, self).__init__() 46 | layer_blocks, activation_blocks, poolings = make_layers(config[depth], batch_norm) 47 | self.layer_blocks = layer_blocks 48 | self.activation_blocks = activation_blocks 49 | self.poolings = poolings 50 | 51 | self.classifier = nn.Sequential( 52 | nn.Dropout(), 53 | nn.Linear(512, 512), 54 | nn.ReLU(inplace=True), 55 | nn.Dropout(), 56 | nn.Linear(512, 512), 57 | nn.ReLU(inplace=True), 58 | nn.Linear(512, num_classes), 59 | ) 60 | 61 | for m in self.modules(): 62 | if isinstance(m, nn.Conv2d): 63 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 64 | m.weight.data.normal_(0, math.sqrt(2. / n)) 65 | m.bias.data.zero_() 66 | 67 | def forward(self, x): 68 | for layers, activations, pooling in zip(self.layer_blocks, self.activation_blocks, 69 | self.poolings): 70 | for layer, activation in zip(layers, activations): 71 | x = layer(x) 72 | x = activation(x) 73 | x = pooling(x) 74 | x = x.view(x.size(0), -1) 75 | x = self.classifier(x) 76 | return x 77 | 78 | 79 | class VGGCurve(nn.Module): 80 | def __init__(self, num_classes, fix_points, depth=16, batch_norm=False): 81 | super(VGGCurve, self).__init__() 82 | layer_blocks, activation_blocks, poolings = make_layers(config[depth], 83 | batch_norm, 84 | fix_points=fix_points) 85 | self.layer_blocks = layer_blocks 86 | self.activation_blocks = activation_blocks 87 | self.poolings = poolings 88 | 89 | self.dropout1 = nn.Dropout() 90 | self.fc1 = curves.Linear(512, 512, fix_points=fix_points) 91 | self.relu1 = nn.ReLU(inplace=True) 92 | self.dropout2 = nn.Dropout() 93 | self.fc2 = curves.Linear(512, 512, fix_points=fix_points) 94 | self.relu2 = nn.ReLU(inplace=True) 95 | self.fc3 = curves.Linear(512, num_classes, fix_points=fix_points) 96 | 97 | # Initialize weights 98 | aa = self.modules() 99 | for m in self.modules(): 100 | if isinstance(m, curves.Conv2d): 101 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 102 | for i in range(m.num_bends): 103 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 104 | getattr(m, 'bias_%d' % i).data.zero_() 105 | 106 | def forward(self, x, coeffs_t): 107 | for layers, activations, pooling in zip(self.layer_blocks, self.activation_blocks, 108 | self.poolings): 109 | for layer, activation in zip(layers, activations): 110 | x = layer(x, coeffs_t) 111 | x = activation(x) 112 | x = pooling(x) 113 | x = x.view(x.size(0), -1) 114 | 115 | x = self.dropout1(x) 116 | x = self.fc1(x, coeffs_t) 117 | x = self.relu1(x) 118 | 119 | x = self.dropout2(x) 120 | x = self.fc2(x, coeffs_t) 121 | x = self.relu2(x) 122 | 123 | x = self.fc3(x, coeffs_t) 124 | 125 | return x 126 | 127 | 128 | class VGG16: 129 | base = VGGBase 130 | curve = VGGCurve 131 | kwargs = { 132 | 'depth': 16, 133 | 'batch_norm': False 134 | } 135 | 136 | 137 | class VGG16BN: 138 | base = VGGBase 139 | curve = VGGCurve 140 | kwargs = { 141 | 'depth': 16, 142 | 'batch_norm': True 143 | } 144 | 145 | 146 | class VGG19: 147 | base = VGGBase 148 | curve = VGGCurve 149 | kwargs = { 150 | 'depth': 19, 151 | 'batch_norm': False 152 | } 153 | 154 | 155 | class VGG19BN: 156 | base = VGGBase 157 | curve = VGGCurve 158 | kwargs = { 159 | 'depth': 19, 160 | 'batch_norm': True 161 | } 162 | -------------------------------------------------------------------------------- /Hessian/train.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import print_function 3 | import numpy as np 4 | import argparse 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import torch.optim as optim 9 | from torchvision import datasets, transforms 10 | from torch.autograd import Variable 11 | from models.c1 import * 12 | from models.resnet import * 13 | from models.vgg import * 14 | from models.convfc import * 15 | from utils import * 16 | 17 | import hessianflow as hf 18 | import hessianflow.optimizer as hf_optm 19 | import sys 20 | import os 21 | 22 | # Training settings 23 | parser = argparse.ArgumentParser(description='PyTorch Example') 24 | parser.add_argument('--name', type = str, default = 'cifar10', metavar = 'N', 25 | help = 'dataset') 26 | parser.add_argument('--dir', type=str, default='VGG16/', metavar='DIR', 27 | help='training directory (default: /tmp/curve/)') 28 | parser.add_argument('--batch-size', type = int, default = 128, metavar = 'B', 29 | help = 'input batch size for training (default: 64)') 30 | parser.add_argument('--test-batch-size', type=int, default=200, metavar='TBS', 31 | help = 'input batch size for testing (default: 1000)') 32 | parser.add_argument('--epochs', type = int, default = 10, metavar = 'E', 33 | help = 'number of epochs to train (default: 10)') 34 | 35 | parser.add_argument('--lr', type = float, default = 0.1, metavar = 'LR', 36 | help = 'learning rate (default: 0.01)') 37 | parser.add_argument('--lr-decay', type = float, default = 0.2, 38 | help = 'learning rate ratio') 39 | parser.add_argument('--lr-decay-epoch', type = int, nargs = '+', default = [30, 60, 90], 40 | help = 'Decrease learning rate at these epochs.') 41 | 42 | 43 | parser.add_argument('--seed', type = int, default = 1, metavar = 'S', 44 | help = 'random seed (default: 1)') 45 | parser.add_argument('--arch', type = str, default = 'CFC', 46 | help = 'choose the archtecure') 47 | parser.add_argument('--large-ratio', type = int, default = 1, 48 | help = 'large ratio') 49 | parser.add_argument('--depth', type = int, default = 20, 50 | help = 'choose the depth of resnet') 51 | 52 | parser.add_argument('--method', type = str, default = 'sgd', 53 | help = 'choose the method to train you model') 54 | 55 | args = parser.parse_args() 56 | # set random seed to reproduce the work 57 | os.makedirs(args.dir, exist_ok=True) 58 | with open(os.path.join(args.dir, 'command.sh'), 'w') as f: 59 | f.write(' '.join(sys.argv)) 60 | f.write('\n') 61 | torch.manual_seed(args.seed) 62 | torch.cuda.manual_seed(args.seed) 63 | 64 | 65 | # get dataset 66 | train_loader, test_loader = getData(name = args.name, train_bs = args.batch_size, test_bs = args.test_batch_size) 67 | 68 | transform_train = transforms.Compose([ 69 | transforms.ToTensor(), 70 | ]) 71 | 72 | trainset = datasets.CIFAR10(root='../data', train = True, download = True, transform = transform_train) 73 | hessian_loader = torch.utils.data.DataLoader(trainset, batch_size = 128, shuffle = True) 74 | 75 | 76 | # get model and optimizer 77 | model_list = { 78 | 'CFC': ConvFC.base(num_classes=10), 79 | 'c1':c1_model(), 80 | 'ResNet': resnet(depth = args.depth), 81 | 'VGG16': VGG16.base(num_classes=10), 82 | } 83 | 84 | model = model_list[args.arch].cuda() 85 | model = torch.nn.DataParallel(model) 86 | 87 | criterion = nn.CrossEntropyLoss() 88 | optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9) 89 | 90 | 91 | 92 | ########### training 93 | if args.method == 'absa': 94 | model, num_updates=hf_optm.absa(model, train_loader, hessian_loader, test_loader, criterion, optimizer, args.epochs, args.lr_decay_epoch, args.lr_decay, 95 | batch_size = args.batch_size, max_large_ratio = args.large_ratio, adv_ratio = 0.2, eps = 0.005, cuda = True, print_flag = True) 96 | elif args.method == 'sgd': 97 | model, num_updates, epoc = hf_optm.baseline(model, train_loader, test_loader, criterion, optimizer, args.epochs, args.lr_decay_epoch, 98 | args.lr_decay, batch_size = args.batch_size, max_large_ratio = args.large_ratio, cuda = True) 99 | 100 | 101 | save_checkpoint( 102 | args.dir, 103 | args.epochs, 104 | model_state=model.state_dict(), 105 | optimizer_state=optimizer.state_dict() 106 | ) 107 | -------------------------------------------------------------------------------- /Hessian/train_resnet.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | import argparse 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.optim as optim 8 | from torchvision import datasets, transforms 9 | from torch.autograd import Variable 10 | from models.resnet import resnet 11 | from utils import * 12 | 13 | import hessianflow as hf 14 | import hessianflow.optimizer.optm_utils as hf_optm_utils 15 | import hessianflow.optimizer.progressbar as hf_optm_pgb 16 | 17 | 18 | # Training settings 19 | parser = argparse.ArgumentParser(description='PyTorch Example') 20 | parser.add_argument('--name', type = str, default = 'cifar10', metavar = 'N', 21 | help = 'dataset') 22 | parser.add_argument('--batch-size', type = int, default = 128, metavar = 'B', 23 | help = 'input batch size for training (default: 64)') 24 | parser.add_argument('--test-batch-size', type=int, default=200, metavar='TBS', 25 | help = 'input batch size for testing (default: 1000)') 26 | parser.add_argument('--epochs', type = int, default = 160, metavar = 'E', 27 | help = 'number of epochs to train (default: 10)') 28 | 29 | parser.add_argument('--lr', type = float, default = 0.1, metavar = 'LR', 30 | help = 'learning rate (default: 0.01)') 31 | parser.add_argument('--lr-decay', type = float, default = 0.1, 32 | help = 'learning rate ratio') 33 | parser.add_argument('--lr-decay-epoch', type = int, nargs = '+', default = [60,120], 34 | help = 'Decrease learning rate at these epochs.') 35 | 36 | 37 | parser.add_argument('--seed', type = int, default = 1, metavar = 'S', 38 | help = 'random seed (default: 1)') 39 | parser.add_argument('--weight-decay', '--wd', default = 5e-4, type = float, 40 | metavar = 'W', help = 'weight decay (default: 5e-4)') 41 | parser.add_argument('--large-ratio', type = int, default = 1, 42 | help = 'large ratio') 43 | parser.add_argument('--depth', type = int, default = 20, 44 | help = 'choose the depth of resnet') 45 | 46 | 47 | parser.add_argument('--eigen-type', type = str, default = 'approximate', 48 | help = 'full dataset of subset') 49 | 50 | args = parser.parse_args() 51 | args.cuda = True 52 | # set random seed to reproduce the work 53 | torch.manual_seed(args.seed) 54 | torch.cuda.manual_seed(args.seed) 55 | 56 | 57 | # get dataset 58 | train_loader, test_loader = getData(name = args.name, train_bs = args.batch_size, test_bs = args.test_batch_size) 59 | 60 | # get model and optimizer 61 | 62 | model = resnet(depth=args.depth).cuda() 63 | model = torch.nn.DataParallel(model) 64 | 65 | criterion = nn.CrossEntropyLoss() 66 | optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.weight_decay) 67 | 68 | 69 | inner_loop = 0 70 | num_updates = 0 71 | # assert that shuffle is set for train_loader 72 | # assert and explain large ratio 73 | # assert that the train_loader is always set with a small batch size if not print error/warning telling 74 | # the user to instead use large_ratio 75 | for epoch in range(1, args.epochs + 1): 76 | print('\nCurrent Epoch: ', epoch) 77 | print('\nTraining') 78 | train_loss = 0. 79 | total_num = 0. 80 | correct = 0. 81 | 82 | for batch_idx, (data, target) in enumerate(train_loader): 83 | if target.size(0) < 128: 84 | continue 85 | model.train() 86 | # gather input and target for large batch training 87 | inner_loop += 1 88 | # get small model update 89 | if args.cuda: 90 | data, target = data.cuda(), target.cuda() 91 | output = model(data) 92 | loss = criterion(output, target)/float(args.large_ratio) 93 | loss.backward() 94 | train_loss += loss.item()*target.size(0)*float(args.large_ratio) 95 | total_num += target.size(0) 96 | _, predicted = output.max(1) 97 | correct += predicted.eq(target).sum().item() 98 | 99 | if inner_loop % args.large_ratio == 0: 100 | num_updates += 1 101 | optimizer.step() 102 | inner_loop = 0 103 | optimizer.zero_grad() 104 | 105 | hf_optm_pgb.progress_bar(batch_idx, len(train_loader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' 106 | % (train_loss / total_num, 107 | 100. * correct / total_num, correct, total_num)) 108 | 109 | 110 | if args.eigen_type == 'full': 111 | # compute the eigen information based on the whole testing data set 112 | eigenvalue, eigenvector = hf.get_eigen_full_dataset(model, test_loader, criterion, cuda = True, maxIter = 10, tol = 1e-2) 113 | elif args.eigen_type == 'approximate': 114 | # here we choose a random batch from test_loader to compute approximating eigen information 115 | get_data = True 116 | for data, target in test_loader: 117 | # finish the for loop otherwise there is a warning 118 | if get_data: 119 | inputs = data 120 | targets = target 121 | get_data = False 122 | 123 | eigenvalue, eigenvector = hf.get_eigen(model, inputs, targets, criterion, cuda = True, maxIter = 10, tol = 1e-2) 124 | print('\nCurrent Eigenvalue based on Test Dataset: %0.2f' % eigenvalue) 125 | 126 | if epoch in args.lr_decay_epoch: 127 | exp_lr_scheduler(optimizer, decay_ratio=args.lr_decay) 128 | 129 | hf_optm_utils.test(model, test_loader) 130 | -------------------------------------------------------------------------------- /Hessian/utils.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | import os 10 | 11 | def getData(name = 'cifar10', train_bs = 128, test_bs = 1000): 12 | if name == 'svhn': 13 | train_loader = torch.utils.data.DataLoader( 14 | datasets.SVHN('../data', split = 'extra', download = True, 15 | transform = transforms.Compose([ 16 | transforms.ToTensor() 17 | ])), 18 | batch_size = train_bs, shuffle = True) 19 | test_loader = torch.utils.data.DataLoader( 20 | datasets.SVHN('../data', split = 'test', download = True, transform = transforms.Compose([ 21 | transforms.ToTensor() 22 | ])), 23 | batch_size = test_bs, shuffle=False) 24 | 25 | if name == 'mnist': 26 | 27 | train_loader = torch.utils.data.DataLoader( 28 | datasets.MNIST('../data', train = True, download = True, 29 | transform = transforms.Compose([ 30 | transforms.ToTensor(), 31 | transforms.Normalize((0.1307,), (0.3081,)) 32 | ])), 33 | batch_size = train_bs, shuffle = True) 34 | test_loader = torch.utils.data.DataLoader( 35 | datasets.MNIST('../data', train = False, transform = transforms.Compose([ 36 | transforms.ToTensor(), 37 | transforms.Normalize((0.1307,), (0.3081,)) 38 | ])), 39 | batch_size = test_bs, shuffle = False) 40 | 41 | 42 | if name == 'cifar10': 43 | transform_train = transforms.Compose([ 44 | #transforms.RandomCrop(32, padding = 4), 45 | #transforms.RandomHorizontalFlip(), 46 | transforms.ToTensor(), 47 | #transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 48 | ]) 49 | 50 | transform_test = transforms.Compose([ 51 | transforms.ToTensor(), 52 | #transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 53 | ]) 54 | 55 | trainset = datasets.CIFAR10(root='../data', train = True, download = True, transform = transform_train) 56 | train_loader = torch.utils.data.DataLoader(trainset, batch_size = train_bs, shuffle = True) 57 | 58 | testset = datasets.CIFAR10(root='../data', train = False, download = False, transform = transform_test) 59 | test_loader = torch.utils.data.DataLoader(testset, batch_size = test_bs, shuffle = False) 60 | 61 | if name == 'cifar100': 62 | 63 | transform_train = transforms.Compose([ 64 | transforms.RandomCrop(32, padding = 4), 65 | transforms.RandomHorizontalFlip(), 66 | transforms.ToTensor(), 67 | transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 68 | ]) 69 | 70 | transform_test = transforms.Compose([ 71 | transforms.ToTensor(), 72 | transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 73 | ]) 74 | 75 | trainset = datasets.CIFAR100(root='../data', train = True, download = True, transform = transform_train) 76 | train_loader = torch.utils.data.DataLoader(trainset, batch_size = train_bs, shuffle = True) 77 | 78 | testset = datasets.CIFAR100(root='../data', train = False, download = False, transform = transform_test) 79 | test_loader = torch.utils.data.DataLoader(testset, batch_size = test_bs, shuffle = False) 80 | 81 | if name == 'tinyimagenet': 82 | normalize = transforms.Normalize(mean = [0.44785526394844055, 0.41693055629730225, 0.36942949891090393], 83 | std = [0.2928885519504547, 0.28230994939804077, 0.2889912724494934]) 84 | train_dataset = datasets.ImageFolder( 85 | '../data/tiny-imagenet-200/train', 86 | transforms.Compose([ 87 | transforms.RandomCrop(64, padding = 4), 88 | transforms.RandomHorizontalFlip(), 89 | transforms.ToTensor(), 90 | normalize, 91 | ])) 92 | 93 | train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = train_bs, shuffle = True, num_workers = 4, pin_memory = False) 94 | 95 | test_dataset = datasets.ImageFolder( 96 | '../data/tiny-imagenet-200/val', 97 | transforms.Compose([ 98 | transforms.ToTensor(), 99 | normalize, 100 | ])) 101 | 102 | test_loader = torch.utils.data.DataLoader(test_dataset, batch_size = test_bs, shuffle = False) 103 | 104 | return train_loader, test_loader 105 | 106 | def save_checkpoint(dir, epoch, name='checkpoint', **kwargs): 107 | state = { 108 | 'epoch': epoch, 109 | } 110 | state.update(kwargs) 111 | filepath = os.path.join(dir, '%s-%d.pt' % (name, epoch)) 112 | torch.save(state, filepath) 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # model-sanitization 2 | Codes for reproducing the results of the paper "Bridging Mode Connectivity in Loss Landscapes and Adversarial Robustness" published at ICLR 2020 https://openreview.net/forum?id=SJgwzCEKwH 3 | 4 | Pleas go to the corresponding directory for more information 5 | 6 | ## Cite 7 | ``` 8 | @inproceedings{ 9 | Zhao2020Bridging, 10 | title={Bridging Mode Connectivity in Loss Landscapes and Adversarial Robustness}, 11 | author={Pu Zhao and Pin-Yu Chen and Payel Das and Karthikeyan Natesan Ramamurthy and Xue Lin}, 12 | booktitle={International Conference on Learning Representations}, 13 | year={2020}, 14 | url={https://openreview.net/forum?id=SJgwzCEKwH} 15 | } 16 | ``` 17 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/AttackPGD.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | 6 | 7 | class AttackPGD(nn.Module): 8 | def __init__(self, basic_net, config): 9 | super(AttackPGD, self).__init__() 10 | self.basic_net = basic_net 11 | self.rand = config['random_start'] 12 | self.step_size = config['step_size'] 13 | self.epsilon = config['epsilon'] 14 | self.num_steps = config['num_steps'] 15 | assert config['loss_func'] == 'xent', 'Only xent supported for now.' 16 | 17 | def forward(self, inputs, targets, t=None): 18 | x = inputs.detach() 19 | if t is None: 20 | t = inputs.data.new(1).uniform_(0.0,1.0) 21 | if self.rand: 22 | x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon) 23 | for i in range(self.num_steps): 24 | x.requires_grad_() 25 | with torch.enable_grad(): 26 | logits = self.basic_net(x, t=t) 27 | loss = F.cross_entropy(logits, targets, size_average=False) 28 | grad = torch.autograd.grad(loss, [x])[0] 29 | x = x.detach() + self.step_size*torch.sign(grad.detach()) 30 | x = torch.min(torch.max(x, inputs - self.epsilon), inputs + self.epsilon) 31 | x = torch.clamp(x, 0, 1) 32 | # t1 = self.basic_net(x, t=t) 33 | # t2 = F.cross_entropy(t1, targets, size_average=False) 34 | return self.basic_net(x, t=t), x -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/aver_weights.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_5_2bad_testset_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='VGG16_poi_single_target_5_2bad_testset_split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | print('Resume training from %d' % 0) 53 | resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | checkpoint = torch.load(resume) 55 | basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | print('Resume training from %d' % 1) 58 | resume = args.ckpt+'checkpoint-%d.pt' % 1 59 | checkpoint = torch.load(resume) 60 | basic_model1.load_state_dict(checkpoint['model_state']) 61 | 62 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 63 | model_ave.cuda() 64 | 65 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 66 | model_noise.cuda() 67 | 68 | 69 | parameters0 = list( basic_model0.parameters() ) 70 | parameters1 = list( basic_model1.parameters() ) 71 | 72 | model_noise_parameters = list(model_noise.parameters()) 73 | 74 | model_ave_parameters = list(model_ave.parameters()) 75 | 76 | variance = [] 77 | 78 | for j in range(0, len(parameters0)): 79 | model_ave_parameters[j].data.copy_( 1 / 2 * parameters0[j].data + 1 / 2 * parameters1[j].data ) 80 | variance.append( torch.abs(1 / 2 * parameters1[j].data - 1 / 2 * parameters0[j].data ) ) 81 | 82 | 83 | for k in range(421, 440): 84 | 85 | for j in range(0, len(parameters0)): 86 | variance[j] = variance[j].to('cpu') 87 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 88 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 89 | 90 | print('saving model %d', k ) 91 | utils.save_checkpoint( 92 | args.dir, 93 | k, 94 | model_state=model_noise.state_dict(), 95 | ) 96 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/aver_weights_case2.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_5_2bad_testset_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='PreResNet110', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='VGG16_poi_single_target_5_2bad_testset_split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | #print('Resume training from %d' % 0) 53 | #resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | #checkpoint = torch.load(resume) 55 | #basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | #print('Resume training from %d' % 10) 58 | #resume = args.ckpt+'checkpoint-%d.pt' % 10 59 | #checkpoint = torch.load(resume) 60 | #basic_model1.load_state_dict(checkpoint['model_state']) 61 | 62 | 63 | print('Resume training from %d' % 0) 64 | resume = 'Res_single_true_10_same1/checkpoint-100.pt' 65 | checkpoint = torch.load(resume) 66 | basic_model0.load_state_dict(checkpoint['model_state']) 67 | 68 | print('Resume training from %d' % 10) 69 | resume = 'Res_single_true_10_same2/checkpoint-100.pt' 70 | checkpoint = torch.load(resume) 71 | basic_model1.load_state_dict(checkpoint['model_state']) 72 | 73 | 74 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 75 | model_ave.cuda() 76 | 77 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 78 | model_noise.cuda() 79 | 80 | 81 | parameters0 = list( basic_model0.parameters() ) 82 | parameters1 = list( basic_model1.parameters() ) 83 | 84 | model_noise_parameters = list(model_noise.parameters()) 85 | 86 | model_ave_parameters = list(model_ave.parameters()) 87 | 88 | variance = [] 89 | 90 | for j in range(0, len(parameters0)): 91 | model_ave_parameters[j].data.copy_( parameters1[j].data ) 92 | variance.append( torch.abs( parameters0[j].data - parameters1[j].data ) * 1.2 ) 93 | 94 | for k in range(121, 140): 95 | 96 | for j in range(0, len(parameters0)): 97 | variance[j] = variance[j].to('cpu') 98 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 99 | # pertur = torch.normal(torch.zeros(parameters0[j].shape) - variance[j], torch.zeros(parameters0[j].shape) + variance[j] ).cuda() 100 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 101 | 102 | print('saving model %d' % k ) 103 | utils.save_checkpoint( 104 | args.dir, 105 | k, 106 | model_state=model_noise.state_dict(), 107 | ) 108 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/evalacc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import numpy as np 8 | import torchvision 9 | import torchvision.transforms as transforms 10 | import data 11 | import os 12 | import argparse 13 | import utils 14 | from tqdm import tqdm 15 | from models import * 16 | import models 17 | 18 | 19 | parser = argparse.ArgumentParser(description='DNN curve training') 20 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_1_2bad_testset_split', metavar='DIR', 21 | help='training directory (default: /tmp/curve/)') 22 | 23 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 24 | help='dataset name (default: CIFAR10)') 25 | parser.add_argument('--use_test', action='store_true', default=True, 26 | help='switches between validation and test set (default: validation)') 27 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 28 | help='transform name (default: VGG)') 29 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 30 | help='path to datasets location (default: None)') 31 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 32 | help='input batch size (default: 128)') 33 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 34 | help='number of workers (default: 4)') 35 | parser.add_argument('--model', type=str, default='PreResNet110', metavar='MODEL', 36 | help='model name (default: None)') 37 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 38 | help='checkpoint to resume training from (default: None)') 39 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 40 | help='number of epochs to train (default: 200)') 41 | 42 | args = parser.parse_args() 43 | 44 | loaders, num_classes = data.loaders( 45 | args.dataset, 46 | args.data_path, 47 | args.batch_size, 48 | args.num_workers, 49 | args.transform, 50 | args.use_test 51 | ) 52 | 53 | print('==> Preparing data..') 54 | transform_train = transforms.Compose([ 55 | transforms.RandomCrop(32, padding=4), 56 | transforms.RandomHorizontalFlip(), 57 | transforms.ToTensor(), 58 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 59 | # Normalization messes with l-inf bounds. 60 | ]) 61 | 62 | transform_test = transforms.Compose([ 63 | transforms.ToTensor(), 64 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 65 | ]) 66 | 67 | trainset = torchvision.datasets.CIFAR10(root=args.data_path, train=True, download=True, transform=transform_train) 68 | trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) 69 | 70 | testset = torchvision.datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) 71 | testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) 72 | 73 | print('==> Building model..') 74 | architecture = getattr(models, args.model) 75 | basic_net = architecture.base(num_classes=num_classes, **architecture.kwargs) 76 | 77 | basic_net.cuda() 78 | 79 | criterion = nn.CrossEntropyLoss() 80 | 81 | 82 | acc_clean = [] 83 | acc_poison = [] 84 | for k in range(121, 140): 85 | print('Resume training model') 86 | #assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 87 | checkpoint = torch.load('./VGG16_poi_single_target_5_2bad_testset_split/checkpoint-%d.pt' % k) 88 | basic_net.load_state_dict(checkpoint['model_state']) 89 | 90 | #print('Resume training model') 91 | #checkpoint = torch.load('./Res_single_true_10_same1/checkpoint-100.pt') 92 | #basic_net.load_state_dict(checkpoint['model_state']) 93 | 94 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 95 | 96 | start_epoch = 1 97 | for epoch in range(start_epoch, start_epoch+1): 98 | # test_examples(epoch) 99 | test_res = utils.test(loaders['test'], basic_net, criterion) 100 | acc_clean.append(test_res['accuracy']) 101 | print('Val acc:', test_res['accuracy']) 102 | 103 | te_example_res = utils.test_poison(testset, basic_net, criterion) 104 | acc_poison.append(te_example_res['accuracy']) 105 | print('Poison Val acc:', te_example_res['accuracy']) 106 | 107 | print('Ave Val acc:', np.mean(acc_clean)) 108 | print('Ave Poison Val acc:', np.mean(acc_poison)) 109 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .convfc import * 2 | from .vgg import * 3 | from .vggW import * 4 | from .preresnet import * 5 | from .wide_resnet import * 6 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/models/convfc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | 4 | import curves 5 | 6 | __all__ = [ 7 | 'ConvFC', 8 | ] 9 | 10 | 11 | class ConvFCBase(nn.Module): 12 | 13 | def __init__(self, num_classes): 14 | super(ConvFCBase, self).__init__() 15 | self.conv_part = nn.Sequential( 16 | nn.Conv2d(3, 32, kernel_size=5, padding=2), 17 | nn.ReLU(True), 18 | nn.MaxPool2d(kernel_size=3, stride=2), 19 | nn.Conv2d(32, 64, kernel_size=5, padding=2), 20 | nn.ReLU(True), 21 | nn.MaxPool2d(3, 2), 22 | nn.Conv2d(64, 128, kernel_size=5, padding=2), 23 | nn.ReLU(True), 24 | nn.MaxPool2d(3, 2), 25 | ) 26 | self.fc_part = nn.Sequential( 27 | nn.Linear(1152, 1000), 28 | nn.ReLU(True), 29 | nn.Linear(1000, 1000), 30 | nn.ReLU(True), 31 | nn.Linear(1000, num_classes) 32 | ) 33 | 34 | # Initialize weights 35 | for m in self.conv_part.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 38 | m.weight.data.normal_(0, math.sqrt(2. / n)) 39 | m.bias.data.zero_() 40 | 41 | def forward(self, x): 42 | x = self.conv_part(x) 43 | x = x.view(x.size(0), -1) 44 | x = self.fc_part(x) 45 | return x 46 | 47 | 48 | class ConvFCCurve(nn.Module): 49 | def __init__(self, num_classes, fix_points): 50 | super(ConvFCCurve, self).__init__() 51 | self.conv1 = curves.Conv2d(3, 32, kernel_size=5, padding=2, fix_points=fix_points) 52 | self.relu1 = nn.ReLU(True) 53 | self.max_pool1 = nn.MaxPool2d(kernel_size=3, stride=2) 54 | 55 | self.conv2 = curves.Conv2d(32, 64, kernel_size=5, padding=2, fix_points=fix_points) 56 | self.relu2 = nn.ReLU(True) 57 | self.max_pool2 = nn.MaxPool2d(3, 2) 58 | 59 | self.conv3 = curves.Conv2d(64, 128, kernel_size=5, padding=2, fix_points=fix_points) 60 | self.relu3 = nn.ReLU(True) 61 | self.max_pool3 = nn.MaxPool2d(3, 2) 62 | 63 | self.fc4 = curves.Linear(1152, 1000, fix_points=fix_points) 64 | self.relu4 = nn.ReLU(True) 65 | 66 | self.fc5 = curves.Linear(1000, 1000, fix_points=fix_points) 67 | self.relu5 = nn.ReLU(True) 68 | 69 | self.fc6 = curves.Linear(1000, num_classes, fix_points=fix_points) 70 | 71 | # Initialize weights 72 | for m in self.modules(): 73 | if isinstance(m, curves.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | for i in range(m.num_bends): 76 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 77 | getattr(m, 'bias_%d' % i).data.zero_() 78 | 79 | def forward(self, x, coeffs_t): 80 | x = self.conv1(x, coeffs_t) 81 | x = self.relu1(x) 82 | x = self.max_pool1(x) 83 | 84 | x = self.conv2(x, coeffs_t) 85 | x = self.relu2(x) 86 | x = self.max_pool2(x) 87 | 88 | x = self.conv3(x, coeffs_t) 89 | x = self.relu3(x) 90 | x = self.max_pool3(x) 91 | 92 | x = x.view(x.size(0), -1) 93 | 94 | x = self.fc4(x, coeffs_t) 95 | x = self.relu4(x) 96 | 97 | x = self.fc5(x, coeffs_t) 98 | x = self.relu5(x) 99 | 100 | x = self.fc6(x, coeffs_t) 101 | 102 | return x 103 | 104 | 105 | class ConvFC: 106 | base = ConvFCBase 107 | curve = ConvFCCurve 108 | kwargs = {} 109 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/models/vgg.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import math 4 | import torch.nn as nn 5 | 6 | import curves 7 | 8 | __all__ = ['VGG16', 'VGG16BN', 'VGG19', 'VGG19BN'] 9 | 10 | config = { 11 | 16: [[64, 64], [128, 128], [256, 256, 256], [512, 512, 512], [512, 512, 512]], 12 | 19: [[64, 64], [128, 128], [256, 256, 256, 256], [512, 512, 512, 512], [512, 512, 512, 512]], 13 | } 14 | 15 | 16 | def make_layers(config, batch_norm=False, fix_points=None): 17 | layer_blocks = nn.ModuleList() 18 | activation_blocks = nn.ModuleList() 19 | poolings = nn.ModuleList() 20 | 21 | kwargs = dict() 22 | conv = nn.Conv2d 23 | bn = nn.BatchNorm2d 24 | if fix_points is not None: 25 | kwargs['fix_points'] = fix_points 26 | conv = curves.Conv2d 27 | bn = curves.BatchNorm2d 28 | 29 | in_channels = 3 30 | for sizes in config: 31 | layer_blocks.append(nn.ModuleList()) 32 | activation_blocks.append(nn.ModuleList()) 33 | for channels in sizes: 34 | layer_blocks[-1].append(conv(in_channels, channels, kernel_size=3, padding=1, **kwargs)) 35 | if batch_norm: 36 | layer_blocks[-1].append(bn(channels, **kwargs)) 37 | activation_blocks[-1].append(nn.ReLU(inplace=True)) 38 | in_channels = channels 39 | poolings.append(nn.MaxPool2d(kernel_size=2, stride=2)) 40 | return layer_blocks, activation_blocks, poolings 41 | 42 | 43 | class VGGBase(nn.Module): 44 | def __init__(self, num_classes, depth=16, batch_norm=False): 45 | super(VGGBase, self).__init__() 46 | layer_blocks, activation_blocks, poolings = make_layers(config[depth], batch_norm) 47 | self.layer_blocks = layer_blocks 48 | self.activation_blocks = activation_blocks 49 | self.poolings = poolings 50 | 51 | self.classifier = nn.Sequential( 52 | nn.Dropout(), 53 | nn.Linear(512, 512), 54 | nn.ReLU(inplace=True), 55 | nn.Dropout(), 56 | nn.Linear(512, 512), 57 | nn.ReLU(inplace=True), 58 | nn.Linear(512, num_classes), 59 | ) 60 | 61 | for m in self.modules(): 62 | if isinstance(m, nn.Conv2d): 63 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 64 | m.weight.data.normal_(0, math.sqrt(2. / n)) 65 | m.bias.data.zero_() 66 | 67 | def forward(self, x): 68 | for layers, activations, pooling in zip(self.layer_blocks, self.activation_blocks, 69 | self.poolings): 70 | for layer, activation in zip(layers, activations): 71 | x = layer(x) 72 | x = activation(x) 73 | x = pooling(x) 74 | x = x.view(x.size(0), -1) 75 | x = self.classifier(x) 76 | return x 77 | 78 | 79 | class VGGCurve(nn.Module): 80 | def __init__(self, num_classes, fix_points, depth=16, batch_norm=False): 81 | super(VGGCurve, self).__init__() 82 | layer_blocks, activation_blocks, poolings = make_layers(config[depth], 83 | batch_norm, 84 | fix_points=fix_points) 85 | self.layer_blocks = layer_blocks 86 | self.activation_blocks = activation_blocks 87 | self.poolings = poolings 88 | 89 | self.dropout1 = nn.Dropout() 90 | self.fc1 = curves.Linear(512, 512, fix_points=fix_points) 91 | self.relu1 = nn.ReLU(inplace=True) 92 | self.dropout2 = nn.Dropout() 93 | self.fc2 = curves.Linear(512, 512, fix_points=fix_points) 94 | self.relu2 = nn.ReLU(inplace=True) 95 | self.fc3 = curves.Linear(512, num_classes, fix_points=fix_points) 96 | 97 | # Initialize weights 98 | aa = self.modules() 99 | for m in self.modules(): 100 | if isinstance(m, curves.Conv2d): 101 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 102 | for i in range(m.num_bends): 103 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 104 | getattr(m, 'bias_%d' % i).data.zero_() 105 | 106 | def forward(self, x, coeffs_t): 107 | for layers, activations, pooling in zip(self.layer_blocks, self.activation_blocks, 108 | self.poolings): 109 | for layer, activation in zip(layers, activations): 110 | x = layer(x, coeffs_t) 111 | x = activation(x) 112 | x = pooling(x) 113 | x = x.view(x.size(0), -1) 114 | 115 | x = self.dropout1(x) 116 | x = self.fc1(x, coeffs_t) 117 | x = self.relu1(x) 118 | 119 | x = self.dropout2(x) 120 | x = self.fc2(x, coeffs_t) 121 | x = self.relu2(x) 122 | 123 | x = self.fc3(x, coeffs_t) 124 | 125 | return x 126 | 127 | 128 | class VGG16: 129 | base = VGGBase 130 | curve = VGGCurve 131 | kwargs = { 132 | 'depth': 16, 133 | 'batch_norm': False 134 | } 135 | 136 | 137 | class VGG16BN: 138 | base = VGGBase 139 | curve = VGGCurve 140 | kwargs = { 141 | 'depth': 16, 142 | 'batch_norm': True 143 | } 144 | 145 | 146 | class VGG19: 147 | base = VGGBase 148 | curve = VGGCurve 149 | kwargs = { 150 | 'depth': 19, 151 | 'batch_norm': False 152 | } 153 | 154 | 155 | class VGG19BN: 156 | base = VGGBase 157 | curve = VGGCurve 158 | kwargs = { 159 | 'depth': 19, 160 | 'batch_norm': True 161 | } 162 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/save_model.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_5_2bad_testset_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='Res_single_true_10_same1/checkpoint-100.pt', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | curve = getattr(curves, args.curve) 47 | model = curves.CurveNet( 48 | 10, 49 | curve, 50 | architecture.curve, 51 | args.num_bends, 52 | architecture_kwargs=architecture.kwargs, 53 | ) 54 | 55 | model.cuda() 56 | checkpoint = torch.load(args.ckpt) 57 | model.load_state_dict(checkpoint['model_state']) 58 | 59 | 60 | spmodel = architecture.base(num_classes=10, **architecture.kwargs) 61 | 62 | parameters = list(model.net.parameters()) 63 | sppara = list(spmodel.parameters()) 64 | #for i in range(0, len(sppara)): 65 | # ttt= i*3 66 | # weights = parameters[ttt:ttt + model.num_bends] 67 | # spweights = sppara[i] 68 | # for j in range(1, model.num_bends - 1): 69 | # alpha = j * 1.0 / (model.num_bends - 1) 70 | # alpha = 0 71 | # spweights.data.copy_(alpha * weights[-1].data + (1.0 - alpha) * weights[0].data) 72 | 73 | ts = np.linspace(0.0, 1.0, 11) 74 | 75 | for kss, t_value in enumerate(ts): 76 | coeffs_t = model.coeff_layer(t_value) 77 | 78 | for i in range(0, len(sppara)): 79 | ttt= i*3 80 | weights = parameters[ttt:ttt + model.num_bends] 81 | spweights = sppara[i] 82 | for j in range(1, model.num_bends - 1): 83 | spweights.data.copy_(coeffs_t[0] * weights[0].data + coeffs_t[1] * weights[1].data + coeffs_t[2] * weights[2].data) 84 | 85 | print('saving model. %.2f' % t_value) 86 | utils.save_checkpoint( 87 | args.dir, 88 | int(t_value*10), 89 | model_state=spmodel.state_dict(), 90 | ) 91 | -------------------------------------------------------------------------------- /backdoor/backdoor-cifar/test_curve.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import torch 3 | 4 | import curves 5 | import data 6 | import models 7 | 8 | 9 | parser = argparse.ArgumentParser(description='Test DNN curve') 10 | 11 | parser.add_argument('--dataset', type=str, default=None, metavar='DATASET', 12 | help='dataset name (default: CIFAR10)') 13 | parser.add_argument('--use_test', action='store_true', 14 | help='switches between validation and test set (default: validation)') 15 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 16 | help='transform name (default: VGG)') 17 | parser.add_argument('--data_path', type=str, default=None, metavar='PATH', 18 | help='path to datasets location (default: None)') 19 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 20 | help='input batch size (default: 128)') 21 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 22 | help='number of workers (default: 4)') 23 | 24 | parser.add_argument('--model', type=str, default=None, metavar='MODEL', required=True, 25 | help='model name (default: None)') 26 | 27 | parser.add_argument('--curve', type=str, default=None, metavar='CURVE', required=True, 28 | help='curve type to use (default: None)') 29 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 30 | help='number of curve bends (default: 3)') 31 | parser.add_argument('--init_start', type=str, default=None, metavar='CKPT', 32 | help='checkpoint to init start point (default: None)') 33 | parser.add_argument('--init_end', type=str, default=None, metavar='CKPT', 34 | help='checkpoint to init end point (default: None)') 35 | parser.set_defaults(init_linear=True) 36 | parser.add_argument('--init_linear_off', dest='init_linear', action='store_false', 37 | help='turns off linear initialization of intermediate points (default: on)') 38 | 39 | parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') 40 | 41 | args = parser.parse_args() 42 | 43 | torch.backends.cudnn.benchmark = True 44 | torch.manual_seed(args.seed) 45 | torch.cuda.manual_seed(args.seed) 46 | 47 | if args.dataset is not None: 48 | loaders, num_classes = data.loaders( 49 | args.dataset, 50 | args.data_path, 51 | args.batch_size, 52 | args.num_workers, 53 | args.transform, 54 | args.use_test 55 | ) 56 | loader = loaders['test'] 57 | else: 58 | num_classes = 10 59 | loader = [(torch.randn((args.batch_size, 3, 32, 32)), None) for i in range(20)] 60 | 61 | architecture = getattr(models, args.model) 62 | 63 | curve = getattr(curves, args.curve) 64 | curve_model = curves.CurveNet( 65 | num_classes, 66 | curve, 67 | architecture.curve, 68 | args.num_bends, 69 | True, 70 | True, 71 | architecture_kwargs=architecture.kwargs, 72 | ) 73 | 74 | base = [architecture.base(num_classes, **architecture.kwargs) for _ in range(2)] 75 | for base_model, path, k in zip(base, [args.init_start, args.init_end], [0, args.num_bends - 1]): 76 | if path is not None: 77 | checkpoint = torch.load(path) 78 | print('Loading %s as point #%d' % (path, k)) 79 | base_model.load_state_dict(checkpoint['model_state']) 80 | curve_model.import_base_parameters(base_model, k) 81 | 82 | if args.init_linear: 83 | print('Linear initialization.') 84 | curve_model.init_linear() 85 | curve_model.cuda() 86 | for base_model in base: 87 | base_model.cuda() 88 | 89 | t = torch.FloatTensor([0.0]).cuda() 90 | for base_model, t_value in zip(base, [0.0, 1.0]): 91 | print('T: %f' % t_value) 92 | t.data.fill_(t_value) 93 | curve_model.import_base_buffers(base_model) 94 | curve_model.eval() 95 | base_model.eval() 96 | 97 | max_error = 0.0 98 | for i, (input, _) in enumerate(loader): 99 | input = input.cuda(async=True) 100 | 101 | base_ouput = base_model(input) 102 | curve_output = curve_model(input, t) 103 | 104 | error = torch.max(torch.abs(base_ouput - curve_output)).item() 105 | print('Batch #%d. Error: %g' % (i, error)) 106 | max_error = max(max_error, error) 107 | print('Max error: %g' % max_error) 108 | assert max_error < 1e-4, 'Error is too big (%g)' % max_error 109 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/AttackPGD.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | 6 | 7 | class AttackPGD(nn.Module): 8 | def __init__(self, basic_net, config): 9 | super(AttackPGD, self).__init__() 10 | self.basic_net = basic_net 11 | self.rand = config['random_start'] 12 | self.step_size = config['step_size'] 13 | self.epsilon = config['epsilon'] 14 | self.num_steps = config['num_steps'] 15 | assert config['loss_func'] == 'xent', 'Only xent supported for now.' 16 | 17 | def forward(self, inputs, targets, t=None): 18 | x = inputs.detach() 19 | if t is None: 20 | t = inputs.data.new(1).uniform_(0.0,1.0) 21 | if self.rand: 22 | x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon) 23 | for i in range(self.num_steps): 24 | x.requires_grad_() 25 | with torch.enable_grad(): 26 | logits = self.basic_net(x, t=t) 27 | loss = F.cross_entropy(logits, targets, size_average=False) 28 | grad = torch.autograd.grad(loss, [x])[0] 29 | x = x.detach() + self.step_size*torch.sign(grad.detach()) 30 | x = torch.min(torch.max(x, inputs - self.epsilon), inputs + self.epsilon) 31 | x = torch.clamp(x, 0, 1) 32 | # t1 = self.basic_net(x, t=t) 33 | # t2 = F.cross_entropy(t1, targets, size_average=False) 34 | return self.basic_net(x, t=t), x -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/aver_weights.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_5_2bad_testset_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='VGG16_poi_single_target_5_2bad_testset_split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | print('Resume training from %d' % 0) 53 | resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | checkpoint = torch.load(resume) 55 | basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | print('Resume training from %d' % 1) 58 | resume = args.ckpt+'checkpoint-%d.pt' % 1 59 | checkpoint = torch.load(resume) 60 | basic_model1.load_state_dict(checkpoint['model_state']) 61 | 62 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 63 | model_ave.cuda() 64 | 65 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 66 | model_noise.cuda() 67 | 68 | 69 | parameters0 = list( basic_model0.parameters() ) 70 | parameters1 = list( basic_model1.parameters() ) 71 | 72 | model_noise_parameters = list(model_noise.parameters()) 73 | 74 | model_ave_parameters = list(model_ave.parameters()) 75 | 76 | variance = [] 77 | 78 | for j in range(0, len(parameters0)): 79 | model_ave_parameters[j].data.copy_( 1 / 2 * parameters0[j].data + 1 / 2 * parameters1[j].data ) 80 | variance.append( torch.abs(1 / 2 * parameters1[j].data - 1 / 2 * parameters0[j].data ) ) 81 | 82 | 83 | for k in range(21, 40): 84 | 85 | for j in range(0, len(parameters0)): 86 | variance[j] = variance[j].to('cpu') 87 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 88 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 89 | 90 | print('saving model %d', k ) 91 | utils.save_checkpoint( 92 | args.dir, 93 | k, 94 | model_state=model_noise.state_dict(), 95 | ) 96 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/aver_weights_case2.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='Res_single_5_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='SVHN', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='Res_single_5_split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | #print('Resume training from %d' % 0) 53 | #resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | #checkpoint = torch.load(resume) 55 | #basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | print('Resume training model') 58 | checkpoint = torch.load('./VGG_single_true_10_same1/checkpoint-100.pt' ) 59 | basic_model0.load_state_dict(checkpoint['model_state']) 60 | 61 | #print('Resume training from %d' % 1) 62 | #resume = args.ckpt+'checkpoint-%d.pt' % 1 63 | #checkpoint = torch.load(resume) 64 | #basic_model1.load_state_dict(checkpoint['model_state']) 65 | 66 | print('Resume training model') 67 | checkpoint = torch.load('./VGG_single_true_10_same2/checkpoint-100.pt' ) 68 | basic_model1.load_state_dict(checkpoint['model_state']) 69 | 70 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 71 | model_ave.cuda() 72 | 73 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 74 | model_noise.cuda() 75 | 76 | 77 | parameters0 = list(basic_model0.parameters()) 78 | parameters1 = list(basic_model1.parameters()) 79 | 80 | model_noise_parameters = list(model_noise.parameters()) 81 | 82 | model_ave_parameters = list(model_ave.parameters()) 83 | 84 | variance = [] 85 | 86 | for j in range(0, len(parameters0)): 87 | model_ave_parameters[j].data.copy_( parameters1[j].data ) 88 | variance.append( torch.abs( parameters0[j].data - parameters1[j].data ) ) 89 | # variance.append(torch.zeros(parameters0[j].shape)) 90 | 91 | 92 | model_ave_parameters2 = list(model_ave.parameters()) 93 | 94 | for k in range(20, 40): 95 | 96 | for j in range(0, len(parameters0)): 97 | variance[j] = variance[j].to('cpu') 98 | # pertur = torch.normal((parameters1[j].data - parameters0[j].data).cpu(), variance[j]).cuda() 99 | # pertur = torch.normal(torch.zeros(parameters0[j].shape) - variance[j], torch.zeros(parameters0[j].shape) + variance[j] ).cuda() 100 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 101 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 102 | 103 | print('saving model %d' % k ) 104 | utils.save_checkpoint( 105 | args.dir, 106 | k, 107 | model_state=model_noise.state_dict(), 108 | ) 109 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/aver_weights_case3.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='Res_single_5_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='SVHN', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='PreResNet110', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='Res_single_5_split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | #print('Resume training from %d' % 0) 53 | #resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | #checkpoint = torch.load(resume) 55 | #basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | print('Resume training model') 58 | checkpoint = torch.load('./Res_single_true_10_same1/checkpoint-100.pt' ) 59 | basic_model0.load_state_dict(checkpoint['model_state']) 60 | 61 | #print('Resume training from %d' % 1) 62 | #resume = args.ckpt+'checkpoint-%d.pt' % 1 63 | #checkpoint = torch.load(resume) 64 | #basic_model1.load_state_dict(checkpoint['model_state']) 65 | 66 | print('Resume training model') 67 | checkpoint = torch.load('./Res_single_true_10_same1/checkpoint-100.pt' ) 68 | basic_model1.load_state_dict(checkpoint['model_state']) 69 | 70 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 71 | model_ave.cuda() 72 | 73 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 74 | model_noise.cuda() 75 | 76 | 77 | parameters0 = list(basic_model0.parameters()) 78 | parameters1 = list(basic_model1.parameters()) 79 | 80 | model_noise_parameters = list(model_noise.parameters()) 81 | 82 | model_ave_parameters = list(model_ave.parameters()) 83 | 84 | variance = [] 85 | 86 | for j in range(0, len(parameters0)): 87 | model_ave_parameters[j].data.copy_( parameters0[j].data ) 88 | # variance.append( torch.abs( parameters0[j].data - parameters1[j].data ) ) 89 | variance.append(torch.zeros(parameters0[j].shape)) 90 | 91 | 92 | model_ave_parameters2 = list(model_ave.parameters()) 93 | 94 | for k in range(20, 21): 95 | 96 | for j in range(0, len(parameters0)): 97 | variance[j] = variance[j].to('cpu') 98 | # pertur = torch.normal((parameters1[j].data - parameters0[j].data).cpu(), variance[j]).cuda() 99 | # pertur = torch.normal(torch.zeros(parameters0[j].shape) - variance[j], torch.zeros(parameters0[j].shape) + variance[j] ).cuda() 100 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 101 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 102 | has_bn = utils.check_bn(model_ave) 103 | print('saving model %d' % k ) 104 | utils.save_checkpoint( 105 | args.dir, 106 | k, 107 | model_state=model_ave.state_dict(), 108 | ) 109 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/evalacc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import numpy as np 8 | import torchvision 9 | import torchvision.transforms as transforms 10 | import data 11 | import os 12 | import argparse 13 | import utils 14 | from tqdm import tqdm 15 | from models import * 16 | 17 | import models 18 | 19 | parser = argparse.ArgumentParser(description='DNN curve training') 20 | parser.add_argument('--dir', type=str, default='VGG_single_5_split', metavar='DIR', 21 | help='training directory (default: /tmp/curve/)') 22 | 23 | parser.add_argument('--dataset', type=str, default='SVHN', metavar='DATASET', 24 | help='dataset name (default: CIFAR10)') 25 | parser.add_argument('--use_test', action='store_true', default=True, 26 | help='switches between validation and test set (default: validation)') 27 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 28 | help='transform name (default: VGG)') 29 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 30 | help='path to datasets location (default: None)') 31 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 32 | help='input batch size (default: 128)') 33 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 34 | help='number of workers (default: 4)') 35 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 36 | help='model name (default: None)') 37 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 38 | help='checkpoint to resume training from (default: None)') 39 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 40 | help='number of epochs to train (default: 200)') 41 | 42 | args = parser.parse_args() 43 | 44 | loaders, num_classes = data.loaders( 45 | args.dataset, 46 | args.data_path, 47 | args.batch_size, 48 | args.num_workers, 49 | args.transform, 50 | args.use_test 51 | ) 52 | 53 | print('==> Preparing data..') 54 | 55 | transform_test = transforms.Compose([ 56 | transforms.ToTensor(), 57 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 58 | ]) 59 | 60 | #trainset = torchvision.datasets.CIFAR10(root=args.data_path, train=True, download=True, transform=transform_train) 61 | #trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) 62 | 63 | testset = torchvision.datasets.SVHN(root=args.data_path, split='test', download=True, transform=transform_test) 64 | testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) 65 | 66 | print('==> Building model..') 67 | architecture = getattr(models, args.model) 68 | basic_net = architecture.base(num_classes=10, **architecture.kwargs) 69 | 70 | #basic_net = PreResNet110.base(num_classes=10) 71 | 72 | basic_net.cuda() 73 | 74 | criterion = nn.CrossEntropyLoss() 75 | 76 | 77 | acc_clean = [] 78 | acc_poison = [] 79 | for k in range(20, 40): 80 | print('Resume training model') 81 | checkpoint = torch.load('./Res_single_5_split/checkpoint-%d.pt' % k) 82 | start_epoch = checkpoint['epoch'] 83 | basic_net.load_state_dict(checkpoint['model_state']) 84 | 85 | # model_ave_parameters2 = list(basic_net.parameters()) 86 | #print('Resume training model') 87 | #checkpoint = torch.load('./VGG_single_true_10_same2/checkpoint-100.pt' ) 88 | #start_epoch = checkpoint['epoch'] 89 | #basic_net.load_state_dict(checkpoint['model_state']) 90 | 91 | has_bn = utils.check_bn(basic_net) 92 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 93 | 94 | for epoch in range(start_epoch, start_epoch+1): 95 | # test_examples(epoch) 96 | test_res = utils.test(loaders['test'], basic_net, criterion) 97 | acc_clean.append(test_res['accuracy']) 98 | print('Val acc:', test_res['accuracy']) 99 | 100 | te_example_res = utils.test_poison(testset, basic_net, criterion) 101 | acc_poison.append(te_example_res['accuracy']) 102 | print('Poison Val acc:', te_example_res['accuracy']) 103 | 104 | print('Ave Val acc:', np.mean(acc_clean)) 105 | print('Ave Poison Val acc:', np.mean(acc_poison)) 106 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .convfc import * 2 | from .vgg import * 3 | from .vggW import * 4 | from .preresnet import * 5 | from .wide_resnet import * 6 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/models/convfc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | 4 | import curves 5 | 6 | __all__ = [ 7 | 'ConvFC', 8 | ] 9 | 10 | 11 | class ConvFCBase(nn.Module): 12 | 13 | def __init__(self, num_classes): 14 | super(ConvFCBase, self).__init__() 15 | self.conv_part = nn.Sequential( 16 | nn.Conv2d(3, 32, kernel_size=5, padding=2), 17 | nn.ReLU(True), 18 | nn.MaxPool2d(kernel_size=3, stride=2), 19 | nn.Conv2d(32, 64, kernel_size=5, padding=2), 20 | nn.ReLU(True), 21 | nn.MaxPool2d(3, 2), 22 | nn.Conv2d(64, 128, kernel_size=5, padding=2), 23 | nn.ReLU(True), 24 | nn.MaxPool2d(3, 2), 25 | ) 26 | self.fc_part = nn.Sequential( 27 | nn.Linear(1152, 1000), 28 | nn.ReLU(True), 29 | nn.Linear(1000, 1000), 30 | nn.ReLU(True), 31 | nn.Linear(1000, num_classes) 32 | ) 33 | 34 | # Initialize weights 35 | for m in self.conv_part.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 38 | m.weight.data.normal_(0, math.sqrt(2. / n)) 39 | m.bias.data.zero_() 40 | 41 | def forward(self, x): 42 | x = self.conv_part(x) 43 | x = x.view(x.size(0), -1) 44 | x = self.fc_part(x) 45 | return x 46 | 47 | 48 | class ConvFCCurve(nn.Module): 49 | def __init__(self, num_classes, fix_points): 50 | super(ConvFCCurve, self).__init__() 51 | self.conv1 = curves.Conv2d(3, 32, kernel_size=5, padding=2, fix_points=fix_points) 52 | self.relu1 = nn.ReLU(True) 53 | self.max_pool1 = nn.MaxPool2d(kernel_size=3, stride=2) 54 | 55 | self.conv2 = curves.Conv2d(32, 64, kernel_size=5, padding=2, fix_points=fix_points) 56 | self.relu2 = nn.ReLU(True) 57 | self.max_pool2 = nn.MaxPool2d(3, 2) 58 | 59 | self.conv3 = curves.Conv2d(64, 128, kernel_size=5, padding=2, fix_points=fix_points) 60 | self.relu3 = nn.ReLU(True) 61 | self.max_pool3 = nn.MaxPool2d(3, 2) 62 | 63 | self.fc4 = curves.Linear(1152, 1000, fix_points=fix_points) 64 | self.relu4 = nn.ReLU(True) 65 | 66 | self.fc5 = curves.Linear(1000, 1000, fix_points=fix_points) 67 | self.relu5 = nn.ReLU(True) 68 | 69 | self.fc6 = curves.Linear(1000, num_classes, fix_points=fix_points) 70 | 71 | # Initialize weights 72 | for m in self.modules(): 73 | if isinstance(m, curves.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | for i in range(m.num_bends): 76 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 77 | getattr(m, 'bias_%d' % i).data.zero_() 78 | 79 | def forward(self, x, coeffs_t): 80 | x = self.conv1(x, coeffs_t) 81 | x = self.relu1(x) 82 | x = self.max_pool1(x) 83 | 84 | x = self.conv2(x, coeffs_t) 85 | x = self.relu2(x) 86 | x = self.max_pool2(x) 87 | 88 | x = self.conv3(x, coeffs_t) 89 | x = self.relu3(x) 90 | x = self.max_pool3(x) 91 | 92 | x = x.view(x.size(0), -1) 93 | 94 | x = self.fc4(x, coeffs_t) 95 | x = self.relu4(x) 96 | 97 | x = self.fc5(x, coeffs_t) 98 | x = self.relu5(x) 99 | 100 | x = self.fc6(x, coeffs_t) 101 | 102 | return x 103 | 104 | 105 | class ConvFC: 106 | base = ConvFCBase 107 | curve = ConvFCCurve 108 | kwargs = {} 109 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/models/vgg.py: -------------------------------------------------------------------------------- 1 | 2 | import math 3 | import torch.nn as nn 4 | 5 | import curves 6 | 7 | __all__ = ['VGG16', 'VGG16BN', 'VGG19', 'VGG19BN'] 8 | 9 | config = { 10 | 16: [[64, 64], [128, 128], [256, 256, 256], [512, 512, 512], [512, 512, 512]], 11 | 19: [[64, 64], [128, 128], [256, 256, 256, 256], [512, 512, 512, 512], [512, 512, 512, 512]], 12 | } 13 | 14 | 15 | def make_layers(config, batch_norm=False, fix_points=None): 16 | layer_blocks = nn.ModuleList() 17 | activation_blocks = nn.ModuleList() 18 | poolings = nn.ModuleList() 19 | 20 | kwargs = dict() 21 | conv = nn.Conv2d 22 | bn = nn.BatchNorm2d 23 | if fix_points is not None: 24 | kwargs['fix_points'] = fix_points 25 | conv = curves.Conv2d 26 | bn = curves.BatchNorm2d 27 | 28 | in_channels = 3 29 | for sizes in config: 30 | layer_blocks.append(nn.ModuleList()) 31 | activation_blocks.append(nn.ModuleList()) 32 | for channels in sizes: 33 | layer_blocks[-1].append(conv(in_channels, channels, kernel_size=3, padding=1, **kwargs)) 34 | if batch_norm: 35 | layer_blocks[-1].append(bn(channels, **kwargs)) 36 | activation_blocks[-1].append(nn.ReLU(inplace=True)) 37 | in_channels = channels 38 | poolings.append(nn.MaxPool2d(kernel_size=2, stride=2)) 39 | return layer_blocks, activation_blocks, poolings 40 | 41 | 42 | class VGGBase(nn.Module): 43 | def __init__(self, num_classes, depth=16, batch_norm=False): 44 | super(VGGBase, self).__init__() 45 | layer_blocks, activation_blocks, poolings = make_layers(config[depth], batch_norm) 46 | self.layer_blocks = layer_blocks 47 | self.activation_blocks = activation_blocks 48 | self.poolings = poolings 49 | 50 | self.classifier = nn.Sequential( 51 | nn.Dropout(), 52 | nn.Linear(512, 512), 53 | nn.ReLU(inplace=True), 54 | nn.Dropout(), 55 | nn.Linear(512, 512), 56 | nn.ReLU(inplace=True), 57 | nn.Linear(512, num_classes), 58 | ) 59 | 60 | for m in self.modules(): 61 | if isinstance(m, nn.Conv2d): 62 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 63 | m.weight.data.normal_(0, math.sqrt(2. / n)) 64 | m.bias.data.zero_() 65 | 66 | def forward(self, x): 67 | for layers, activations, pooling in zip(self.layer_blocks, self.activation_blocks, 68 | self.poolings): 69 | for layer, activation in zip(layers, activations): 70 | x = layer(x) 71 | x = activation(x) 72 | x = pooling(x) 73 | x = x.view(x.size(0), -1) 74 | x = self.classifier(x) 75 | return x 76 | 77 | 78 | class VGGCurve(nn.Module): 79 | def __init__(self, num_classes, fix_points, depth=16, batch_norm=False): 80 | super(VGGCurve, self).__init__() 81 | layer_blocks, activation_blocks, poolings = make_layers(config[depth], 82 | batch_norm, 83 | fix_points=fix_points) 84 | self.layer_blocks = layer_blocks 85 | self.activation_blocks = activation_blocks 86 | self.poolings = poolings 87 | 88 | self.dropout1 = nn.Dropout() 89 | self.fc1 = curves.Linear(512, 512, fix_points=fix_points) 90 | self.relu1 = nn.ReLU(inplace=True) 91 | self.dropout2 = nn.Dropout() 92 | self.fc2 = curves.Linear(512, 512, fix_points=fix_points) 93 | self.relu2 = nn.ReLU(inplace=True) 94 | self.fc3 = curves.Linear(512, num_classes, fix_points=fix_points) 95 | 96 | # Initialize weights 97 | aa = self.modules() 98 | for m in self.modules(): 99 | if isinstance(m, curves.Conv2d): 100 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 101 | for i in range(m.num_bends): 102 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 103 | getattr(m, 'bias_%d' % i).data.zero_() 104 | 105 | def forward(self, x, coeffs_t): 106 | for layers, activations, pooling in zip(self.layer_blocks, self.activation_blocks, 107 | self.poolings): 108 | for layer, activation in zip(layers, activations): 109 | x = layer(x, coeffs_t) 110 | x = activation(x) 111 | x = pooling(x) 112 | x = x.view(x.size(0), -1) 113 | 114 | x = self.dropout1(x) 115 | x = self.fc1(x, coeffs_t) 116 | x = self.relu1(x) 117 | 118 | x = self.dropout2(x) 119 | x = self.fc2(x, coeffs_t) 120 | x = self.relu2(x) 121 | 122 | x = self.fc3(x, coeffs_t) 123 | 124 | return x 125 | 126 | 127 | class VGG16: 128 | base = VGGBase 129 | curve = VGGCurve 130 | kwargs = { 131 | 'depth': 16, 132 | 'batch_norm': False 133 | } 134 | 135 | 136 | class VGG16BN: 137 | base = VGGBase 138 | curve = VGGCurve 139 | kwargs = { 140 | 'depth': 16, 141 | 'batch_norm': True 142 | } 143 | 144 | 145 | class VGG19: 146 | base = VGGBase 147 | curve = VGGCurve 148 | kwargs = { 149 | 'depth': 19, 150 | 'batch_norm': False 151 | } 152 | 153 | 154 | class VGG19BN: 155 | base = VGGBase 156 | curve = VGGCurve 157 | kwargs = { 158 | 'depth': 19, 159 | 'batch_norm': True 160 | } 161 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/save_model.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='Res_single_5_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='SVHN', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='PreResNet110', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='Res_single_5/checkpoint-100.pt', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | curve = getattr(curves, args.curve) 47 | model = curves.CurveNet( 48 | 10, 49 | curve, 50 | architecture.curve, 51 | args.num_bends, 52 | architecture_kwargs=architecture.kwargs, 53 | ) 54 | 55 | model.cuda() 56 | checkpoint = torch.load(args.ckpt) 57 | model.load_state_dict(checkpoint['model_state']) 58 | 59 | 60 | spmodel = architecture.base(num_classes=10, **architecture.kwargs) 61 | 62 | parameters = list(model.net.parameters()) 63 | sppara = list(spmodel.parameters()) 64 | #for i in range(0, len(sppara)): 65 | # ttt= i*3 66 | # weights = parameters[ttt:ttt + model.num_bends] 67 | # spweights = sppara[i] 68 | # for j in range(1, model.num_bends - 1): 69 | # alpha = j * 1.0 / (model.num_bends - 1) 70 | # alpha = 0 71 | # spweights.data.copy_(alpha * weights[-1].data + (1.0 - alpha) * weights[0].data) 72 | 73 | ts = np.linspace(0.0, 1.0, 11) 74 | 75 | for kss, t_value in enumerate(ts): 76 | coeffs_t = model.coeff_layer(t_value) 77 | 78 | for i in range(0, len(sppara)): 79 | ttt= i*3 80 | weights = parameters[ttt:ttt + model.num_bends] 81 | spweights = sppara[i] 82 | for j in range(1, model.num_bends - 1): 83 | spweights.data.copy_(coeffs_t[0] * weights[0].data + coeffs_t[1] * weights[1].data + coeffs_t[2] * weights[2].data) 84 | 85 | print('saving model. %.2f' % t_value) 86 | utils.save_checkpoint( 87 | args.dir, 88 | int(t_value*10), 89 | model_state=spmodel.state_dict(), 90 | ) 91 | -------------------------------------------------------------------------------- /backdoor/backdoor-svhn/test_curve.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import torch 3 | 4 | import curves 5 | import data 6 | import models 7 | 8 | 9 | parser = argparse.ArgumentParser(description='Test DNN curve') 10 | 11 | parser.add_argument('--dataset', type=str, default=None, metavar='DATASET', 12 | help='dataset name (default: CIFAR10)') 13 | parser.add_argument('--use_test', action='store_true', 14 | help='switches between validation and test set (default: validation)') 15 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 16 | help='transform name (default: VGG)') 17 | parser.add_argument('--data_path', type=str, default=None, metavar='PATH', 18 | help='path to datasets location (default: None)') 19 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 20 | help='input batch size (default: 128)') 21 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 22 | help='number of workers (default: 4)') 23 | 24 | parser.add_argument('--model', type=str, default=None, metavar='MODEL', required=True, 25 | help='model name (default: None)') 26 | 27 | parser.add_argument('--curve', type=str, default=None, metavar='CURVE', required=True, 28 | help='curve type to use (default: None)') 29 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 30 | help='number of curve bends (default: 3)') 31 | parser.add_argument('--init_start', type=str, default=None, metavar='CKPT', 32 | help='checkpoint to init start point (default: None)') 33 | parser.add_argument('--init_end', type=str, default=None, metavar='CKPT', 34 | help='checkpoint to init end point (default: None)') 35 | parser.set_defaults(init_linear=True) 36 | parser.add_argument('--init_linear_off', dest='init_linear', action='store_false', 37 | help='turns off linear initialization of intermediate points (default: on)') 38 | 39 | parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') 40 | 41 | args = parser.parse_args() 42 | 43 | torch.backends.cudnn.benchmark = True 44 | torch.manual_seed(args.seed) 45 | torch.cuda.manual_seed(args.seed) 46 | 47 | if args.dataset is not None: 48 | loaders, num_classes = data.loaders( 49 | args.dataset, 50 | args.data_path, 51 | args.batch_size, 52 | args.num_workers, 53 | args.transform, 54 | args.use_test 55 | ) 56 | loader = loaders['test'] 57 | else: 58 | num_classes = 10 59 | loader = [(torch.randn((args.batch_size, 3, 32, 32)), None) for i in range(20)] 60 | 61 | architecture = getattr(models, args.model) 62 | 63 | curve = getattr(curves, args.curve) 64 | curve_model = curves.CurveNet( 65 | num_classes, 66 | curve, 67 | architecture.curve, 68 | args.num_bends, 69 | True, 70 | True, 71 | architecture_kwargs=architecture.kwargs, 72 | ) 73 | 74 | base = [architecture.base(num_classes, **architecture.kwargs) for _ in range(2)] 75 | for base_model, path, k in zip(base, [args.init_start, args.init_end], [0, args.num_bends - 1]): 76 | if path is not None: 77 | checkpoint = torch.load(path) 78 | print('Loading %s as point #%d' % (path, k)) 79 | base_model.load_state_dict(checkpoint['model_state']) 80 | curve_model.import_base_parameters(base_model, k) 81 | 82 | if args.init_linear: 83 | print('Linear initialization.') 84 | curve_model.init_linear() 85 | curve_model.cuda() 86 | for base_model in base: 87 | base_model.cuda() 88 | 89 | t = torch.FloatTensor([0.0]).cuda() 90 | for base_model, t_value in zip(base, [0.0, 1.0]): 91 | print('T: %f' % t_value) 92 | t.data.fill_(t_value) 93 | curve_model.import_base_buffers(base_model) 94 | curve_model.eval() 95 | base_model.eval() 96 | 97 | max_error = 0.0 98 | for i, (input, _) in enumerate(loader): 99 | input = input.cuda(async=True) 100 | 101 | base_ouput = base_model(input) 102 | curve_output = curve_model(input, t) 103 | 104 | error = torch.max(torch.abs(base_ouput - curve_output)).item() 105 | print('Batch #%d. Error: %g' % (i, error)) 106 | max_error = max(max_error, error) 107 | print('Max error: %g' % max_error) 108 | assert max_error < 1e-4, 'Error is too big (%g)' % max_error 109 | -------------------------------------------------------------------------------- /backdoor/prune_cifar/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Dependencies 3 | torch v0.3.1, torchvision v0.2.0 4 | 5 | ## Baseline 6 | 7 | The `dataset` argument specifies which dataset to use: `cifar10` or `cifar100`. The `arch` argument specifies the architecture to use: `vgg` or `resnet`. The depth is chosen to be the same as the networks used in the paper. 8 | ```shell 9 | python main.py --dataset cifar10 --arch vgg --depth 16 10 | python main.py --dataset cifar10 --arch resnet --depth 56 11 | python main.py --dataset cifar10 --arch resnet --depth 110 12 | ``` 13 | 14 | ## Prune 15 | 16 | ```shell 17 | python vggprune.py --dataset cifar10 --model [PATH TO THE MODEL] --save [DIRECTORY TO STORE RESULT] 18 | python res56prune.py --dataset cifar10 -v A --model [PATH TO THE MODEL] --save [DIRECTORY TO STORE RESULT] 19 | python res110prune.py --dataset cifar10 -v A --model [PATH TO THE MODEL] --save [DIRECTORY TO STORE RESULT] 20 | ``` 21 | Here in `res56prune.py` and `res110prune.py`, the `-v` argument is `A` or `B`, which refers to the naming of the pruned model in the original paper. The pruned model will be named `pruned.pth.tar`. 22 | 23 | ## Fine-tune 24 | 25 | ```shell 26 | python main_finetune.py --refine [PATH TO THE PRUNED MODEL] --dataset cifar10 --arch vgg --depth 16 27 | python main_finetune.py --refine [PATH TO THE PRUNED MODEL] --dataset cifar10 --arch resnet --depth 56 28 | python main_finetune.py --refine [PATH TO THE PRUNED MODEL] --dataset cifar10 --arch resnet --depth 110 29 | ``` 30 | 31 | 32 | ## train backdoored model 33 | 34 | use main.py to train backdoored model. 35 | 36 | -------------------------------------------------------------------------------- /backdoor/prune_cifar/compute_flops.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | 4 | import torch 5 | import torchvision 6 | import torch.nn as nn 7 | from torch.autograd import Variable 8 | 9 | 10 | def print_model_param_nums(model=None, multiply_adds=True): 11 | if model == None: 12 | model = torchvision.models.alexnet() 13 | total = sum([param.nelement() for param in model.parameters()]) 14 | print(' + Number of params: %.2fM' % (total / 1e6)) 15 | 16 | def print_model_param_flops(model=None, input_res=224, multiply_adds=True): 17 | 18 | prods = {} 19 | def save_hook(name): 20 | def hook_per(self, input, output): 21 | prods[name] = np.prod(input[0].shape) 22 | return hook_per 23 | 24 | list_1=[] 25 | def simple_hook(self, input, output): 26 | list_1.append(np.prod(input[0].shape)) 27 | list_2={} 28 | def simple_hook2(self, input, output): 29 | list_2['names'] = np.prod(input[0].shape) 30 | 31 | list_conv=[] 32 | def conv_hook(self, input, output): 33 | batch_size, input_channels, input_height, input_width = input[0].size() 34 | output_channels, output_height, output_width = output[0].size() 35 | 36 | kernel_ops = self.kernel_size[0] * self.kernel_size[1] * (self.in_channels / self.groups) 37 | bias_ops = 1 if self.bias is not None else 0 38 | 39 | params = output_channels * (kernel_ops + bias_ops) 40 | flops = (kernel_ops * (2 if multiply_adds else 1) + bias_ops) * output_channels * output_height * output_width * batch_size 41 | 42 | list_conv.append(flops) 43 | 44 | list_linear=[] 45 | def linear_hook(self, input, output): 46 | batch_size = input[0].size(0) if input[0].dim() == 2 else 1 47 | 48 | weight_ops = self.weight.nelement() * (2 if multiply_adds else 1) 49 | bias_ops = self.bias.nelement() 50 | 51 | flops = batch_size * (weight_ops + bias_ops) 52 | list_linear.append(flops) 53 | 54 | list_bn=[] 55 | def bn_hook(self, input, output): 56 | list_bn.append(input[0].nelement() * 2) 57 | 58 | list_relu=[] 59 | def relu_hook(self, input, output): 60 | list_relu.append(input[0].nelement()) 61 | 62 | list_pooling=[] 63 | def pooling_hook(self, input, output): 64 | batch_size, input_channels, input_height, input_width = input[0].size() 65 | output_channels, output_height, output_width = output[0].size() 66 | 67 | kernel_ops = self.kernel_size * self.kernel_size 68 | bias_ops = 0 69 | params = 0 70 | flops = (kernel_ops + bias_ops) * output_channels * output_height * output_width * batch_size 71 | 72 | list_pooling.append(flops) 73 | 74 | list_upsample=[] 75 | # For bilinear upsample 76 | def upsample_hook(self, input, output): 77 | batch_size, input_channels, input_height, input_width = input[0].size() 78 | output_channels, output_height, output_width = output[0].size() 79 | 80 | flops = output_height * output_width * output_channels * batch_size * 12 81 | list_upsample.append(flops) 82 | 83 | def foo(net): 84 | childrens = list(net.children()) 85 | if not childrens: 86 | if isinstance(net, torch.nn.Conv2d): 87 | net.register_forward_hook(conv_hook) 88 | if isinstance(net, torch.nn.Linear): 89 | net.register_forward_hook(linear_hook) 90 | if isinstance(net, torch.nn.BatchNorm2d): 91 | net.register_forward_hook(bn_hook) 92 | if isinstance(net, torch.nn.ReLU): 93 | net.register_forward_hook(relu_hook) 94 | if isinstance(net, torch.nn.MaxPool2d) or isinstance(net, torch.nn.AvgPool2d): 95 | net.register_forward_hook(pooling_hook) 96 | if isinstance(net, torch.nn.Upsample): 97 | net.register_forward_hook(upsample_hook) 98 | return 99 | for c in childrens: 100 | foo(c) 101 | 102 | if model == None: 103 | model = torchvision.models.alexnet() 104 | foo(model) 105 | input = Variable(torch.rand(3, 3, input_res, input_res), requires_grad = True) 106 | out = model(input) 107 | 108 | 109 | total_flops = (sum(list_conv) + sum(list_linear) + sum(list_bn) + sum(list_relu) + sum(list_pooling) + sum(list_upsample)) 110 | 111 | print(' + Number of FLOPs: %.5fG' % (total_flops / 1e9)) 112 | 113 | return total_flops 114 | -------------------------------------------------------------------------------- /backdoor/prune_cifar/models/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .vgg import * 4 | from .resnet import * -------------------------------------------------------------------------------- /backdoor/prune_cifar/models/resnet.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from functools import partial 8 | from torch.autograd import Variable 9 | 10 | 11 | __all__ = ['resnet'] 12 | 13 | def conv3x3(in_planes, out_planes, stride=1): 14 | "3x3 convolution with padding" 15 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 16 | padding=1, bias=False) 17 | 18 | 19 | class BasicBlock(nn.Module): 20 | expansion = 1 21 | 22 | def __init__(self, inplanes, planes, cfg, stride=1, downsample=None): 23 | # cfg should be a number in this case 24 | super(BasicBlock, self).__init__() 25 | self.conv1 = conv3x3(inplanes, cfg, stride) 26 | self.bn1 = nn.BatchNorm2d(cfg) 27 | self.relu = nn.ReLU(inplace=True) 28 | self.conv2 = conv3x3(cfg, planes) 29 | self.bn2 = nn.BatchNorm2d(planes) 30 | self.downsample = downsample 31 | self.stride = stride 32 | 33 | def forward(self, x): 34 | residual = x 35 | 36 | out = self.conv1(x) 37 | out = self.bn1(out) 38 | out = self.relu(out) 39 | 40 | out = self.conv2(out) 41 | out = self.bn2(out) 42 | 43 | if self.downsample is not None: 44 | residual = self.downsample(x) 45 | 46 | out += residual 47 | out = self.relu(out) 48 | 49 | return out 50 | 51 | def downsample_basic_block(x, planes): 52 | x = nn.AvgPool2d(2,2)(x) 53 | zero_pads = torch.Tensor( 54 | x.size(0), planes - x.size(1), x.size(2), x.size(3)).zero_() 55 | if isinstance(x.data, torch.cuda.FloatTensor): 56 | zero_pads = zero_pads.cuda() 57 | 58 | out = Variable(torch.cat([x.data, zero_pads], dim=1)) 59 | 60 | return out 61 | 62 | class ResNet(nn.Module): 63 | 64 | def __init__(self, depth, dataset='cifar10', cfg=None): 65 | super(ResNet, self).__init__() 66 | # Model type specifies number of layers for CIFAR-10 model 67 | assert (depth - 2) % 6 == 0, 'depth should be 6n+2' 68 | n = (depth - 2) // 6 69 | 70 | block = BasicBlock 71 | if cfg == None: 72 | cfg = [[16]*n, [32]*n, [64]*n] 73 | cfg = [item for sub_list in cfg for item in sub_list] 74 | 75 | self.cfg = cfg 76 | 77 | self.inplanes = 16 78 | self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, 79 | bias=False) 80 | self.bn1 = nn.BatchNorm2d(16) 81 | self.relu = nn.ReLU(inplace=True) 82 | self.layer1 = self._make_layer(block, 16, n, cfg=cfg[0:n]) 83 | self.layer2 = self._make_layer(block, 32, n, cfg=cfg[n:2*n], stride=2) 84 | self.layer3 = self._make_layer(block, 64, n, cfg=cfg[2*n:3*n], stride=2) 85 | self.avgpool = nn.AvgPool2d(8) 86 | if dataset == 'cifar10': 87 | num_classes = 10 88 | elif dataset == 'cifar100': 89 | num_classes = 100 90 | self.fc = nn.Linear(64 * block.expansion, num_classes) 91 | 92 | for m in self.modules(): 93 | if isinstance(m, nn.Conv2d): 94 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 95 | m.weight.data.normal_(0, math.sqrt(2. / n)) 96 | elif isinstance(m, nn.BatchNorm2d): 97 | m.weight.data.fill_(1) 98 | m.bias.data.zero_() 99 | 100 | def _make_layer(self, block, planes, blocks, cfg, stride=1): 101 | downsample = None 102 | if stride != 1 or self.inplanes != planes * block.expansion: 103 | downsample = partial(downsample_basic_block, planes=planes*block.expansion) 104 | 105 | layers = [] 106 | layers.append(block(self.inplanes, planes, cfg[0], stride, downsample)) 107 | self.inplanes = planes * block.expansion 108 | for i in range(1, blocks): 109 | layers.append(block(self.inplanes, planes, cfg[i])) 110 | 111 | return nn.Sequential(*layers) 112 | 113 | def forward(self, x): 114 | x = self.conv1(x) 115 | x = self.bn1(x) 116 | x = self.relu(x) # 32x32 117 | 118 | x = self.layer1(x) # 32x32 119 | x = self.layer2(x) # 16x16 120 | x = self.layer3(x) # 8x8 121 | 122 | x = self.avgpool(x) 123 | x = x.view(x.size(0), -1) 124 | x = self.fc(x) 125 | 126 | return x 127 | 128 | def resnet(**kwargs): 129 | """ 130 | Constructs a ResNet model. 131 | """ 132 | return ResNet(**kwargs) 133 | 134 | if __name__ == '__main__': 135 | net = resnet(depth=56) 136 | x=Variable(torch.FloatTensor(16, 3, 32, 32)) 137 | y = net(x) 138 | print(y.data.shape) -------------------------------------------------------------------------------- /backdoor/prune_cifar/models/vgg.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | from torch.autograd import Variable 6 | 7 | 8 | __all__ = ['vgg'] 9 | 10 | defaultcfg = { 11 | 11 : [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512], 12 | 13 : [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512], 13 | 16 : [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512], 14 | 19 : [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512], 15 | } 16 | 17 | class vgg(nn.Module): 18 | def __init__(self, dataset='cifar10', depth=19, init_weights=True, cfg=None): 19 | super(vgg, self).__init__() 20 | if cfg is None: 21 | cfg = defaultcfg[depth] 22 | 23 | self.cfg = cfg 24 | 25 | self.feature = self.make_layers(cfg, True) 26 | 27 | if dataset == 'cifar10': 28 | num_classes = 10 29 | elif dataset == 'cifar100': 30 | num_classes = 100 31 | self.classifier = nn.Sequential( 32 | nn.Linear(cfg[-1], 512), 33 | nn.BatchNorm1d(512), 34 | nn.ReLU(inplace=True), 35 | nn.Linear(512, num_classes) 36 | ) 37 | if init_weights: 38 | self._initialize_weights() 39 | 40 | def make_layers(self, cfg, batch_norm=False): 41 | layers = [] 42 | in_channels = 3 43 | for v in cfg: 44 | if v == 'M': 45 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 46 | else: 47 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, bias=False) 48 | if batch_norm: 49 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 50 | else: 51 | layers += [conv2d, nn.ReLU(inplace=True)] 52 | in_channels = v 53 | return nn.Sequential(*layers) 54 | 55 | def forward(self, x): 56 | x = self.feature(x) 57 | x = nn.AvgPool2d(2)(x) 58 | x = x.view(x.size(0), -1) 59 | y = self.classifier(x) 60 | return y 61 | 62 | def _initialize_weights(self): 63 | for m in self.modules(): 64 | if isinstance(m, nn.Conv2d): 65 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 66 | m.weight.data.normal_(0, math.sqrt(2. / n)) 67 | if m.bias is not None: 68 | m.bias.data.zero_() 69 | elif isinstance(m, nn.BatchNorm2d): 70 | m.weight.data.fill_(0.5) 71 | m.bias.data.zero_() 72 | elif isinstance(m, nn.Linear): 73 | m.weight.data.normal_(0, 0.01) 74 | m.bias.data.zero_() 75 | 76 | if __name__ == '__main__': 77 | net = vgg() 78 | x = Variable(torch.FloatTensor(16, 3, 40, 40)) 79 | y = net(x) 80 | print(y.data.shape) -------------------------------------------------------------------------------- /backdoor/prune_svhn/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Dependencies 3 | torch v0.3.1, torchvision v0.2.0 4 | 5 | ## Baseline 6 | 7 | The `dataset` argument specifies which dataset to use: `cifar10` or `cifar100`. The `arch` argument specifies the architecture to use: `vgg` or `resnet`. The depth is chosen to be the same as the networks used in the paper. 8 | ```shell 9 | python main.py --dataset cifar10 --arch vgg --depth 16 10 | python main.py --dataset cifar10 --arch resnet --depth 56 11 | python main.py --dataset cifar10 --arch resnet --depth 110 12 | ``` 13 | 14 | ## Prune 15 | 16 | ```shell 17 | python vggprune.py --dataset cifar10 --model [PATH TO THE MODEL] --save [DIRECTORY TO STORE RESULT] 18 | python res56prune.py --dataset cifar10 -v A --model [PATH TO THE MODEL] --save [DIRECTORY TO STORE RESULT] 19 | python res110prune.py --dataset cifar10 -v A --model [PATH TO THE MODEL] --save [DIRECTORY TO STORE RESULT] 20 | ``` 21 | Here in `res56prune.py` and `res110prune.py`, the `-v` argument is `A` or `B`, which refers to the naming of the pruned model in the original paper. The pruned model will be named `pruned.pth.tar`. 22 | 23 | ## Fine-tune 24 | 25 | ```shell 26 | python main_finetune.py --refine [PATH TO THE PRUNED MODEL] --dataset cifar10 --arch vgg --depth 16 27 | python main_finetune.py --refine [PATH TO THE PRUNED MODEL] --dataset cifar10 --arch resnet --depth 56 28 | python main_finetune.py --refine [PATH TO THE PRUNED MODEL] --dataset cifar10 --arch resnet --depth 110 29 | ``` 30 | 31 | 32 | ## train backdoored model 33 | 34 | use main.py to train backdoored model. 35 | 36 | -------------------------------------------------------------------------------- /backdoor/prune_svhn/compute_flops.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | 4 | import torch 5 | import torchvision 6 | import torch.nn as nn 7 | from torch.autograd import Variable 8 | 9 | 10 | def print_model_param_nums(model=None, multiply_adds=True): 11 | if model == None: 12 | model = torchvision.models.alexnet() 13 | total = sum([param.nelement() for param in model.parameters()]) 14 | print(' + Number of params: %.2fM' % (total / 1e6)) 15 | 16 | def print_model_param_flops(model=None, input_res=224, multiply_adds=True): 17 | 18 | prods = {} 19 | def save_hook(name): 20 | def hook_per(self, input, output): 21 | prods[name] = np.prod(input[0].shape) 22 | return hook_per 23 | 24 | list_1=[] 25 | def simple_hook(self, input, output): 26 | list_1.append(np.prod(input[0].shape)) 27 | list_2={} 28 | def simple_hook2(self, input, output): 29 | list_2['names'] = np.prod(input[0].shape) 30 | 31 | list_conv=[] 32 | def conv_hook(self, input, output): 33 | batch_size, input_channels, input_height, input_width = input[0].size() 34 | output_channels, output_height, output_width = output[0].size() 35 | 36 | kernel_ops = self.kernel_size[0] * self.kernel_size[1] * (self.in_channels / self.groups) 37 | bias_ops = 1 if self.bias is not None else 0 38 | 39 | params = output_channels * (kernel_ops + bias_ops) 40 | flops = (kernel_ops * (2 if multiply_adds else 1) + bias_ops) * output_channels * output_height * output_width * batch_size 41 | 42 | list_conv.append(flops) 43 | 44 | list_linear=[] 45 | def linear_hook(self, input, output): 46 | batch_size = input[0].size(0) if input[0].dim() == 2 else 1 47 | 48 | weight_ops = self.weight.nelement() * (2 if multiply_adds else 1) 49 | bias_ops = self.bias.nelement() 50 | 51 | flops = batch_size * (weight_ops + bias_ops) 52 | list_linear.append(flops) 53 | 54 | list_bn=[] 55 | def bn_hook(self, input, output): 56 | list_bn.append(input[0].nelement() * 2) 57 | 58 | list_relu=[] 59 | def relu_hook(self, input, output): 60 | list_relu.append(input[0].nelement()) 61 | 62 | list_pooling=[] 63 | def pooling_hook(self, input, output): 64 | batch_size, input_channels, input_height, input_width = input[0].size() 65 | output_channels, output_height, output_width = output[0].size() 66 | 67 | kernel_ops = self.kernel_size * self.kernel_size 68 | bias_ops = 0 69 | params = 0 70 | flops = (kernel_ops + bias_ops) * output_channels * output_height * output_width * batch_size 71 | 72 | list_pooling.append(flops) 73 | 74 | list_upsample=[] 75 | # For bilinear upsample 76 | def upsample_hook(self, input, output): 77 | batch_size, input_channels, input_height, input_width = input[0].size() 78 | output_channels, output_height, output_width = output[0].size() 79 | 80 | flops = output_height * output_width * output_channels * batch_size * 12 81 | list_upsample.append(flops) 82 | 83 | def foo(net): 84 | childrens = list(net.children()) 85 | if not childrens: 86 | if isinstance(net, torch.nn.Conv2d): 87 | net.register_forward_hook(conv_hook) 88 | if isinstance(net, torch.nn.Linear): 89 | net.register_forward_hook(linear_hook) 90 | if isinstance(net, torch.nn.BatchNorm2d): 91 | net.register_forward_hook(bn_hook) 92 | if isinstance(net, torch.nn.ReLU): 93 | net.register_forward_hook(relu_hook) 94 | if isinstance(net, torch.nn.MaxPool2d) or isinstance(net, torch.nn.AvgPool2d): 95 | net.register_forward_hook(pooling_hook) 96 | if isinstance(net, torch.nn.Upsample): 97 | net.register_forward_hook(upsample_hook) 98 | return 99 | for c in childrens: 100 | foo(c) 101 | 102 | if model == None: 103 | model = torchvision.models.alexnet() 104 | foo(model) 105 | input = Variable(torch.rand(3, 3, input_res, input_res), requires_grad = True) 106 | out = model(input) 107 | 108 | 109 | total_flops = (sum(list_conv) + sum(list_linear) + sum(list_bn) + sum(list_relu) + sum(list_pooling) + sum(list_upsample)) 110 | 111 | print(' + Number of FLOPs: %.5fG' % (total_flops / 1e9)) 112 | 113 | return total_flops 114 | -------------------------------------------------------------------------------- /backdoor/prune_svhn/models/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .vgg import * 4 | from .resnet import * -------------------------------------------------------------------------------- /backdoor/prune_svhn/models/resnet.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from functools import partial 8 | from torch.autograd import Variable 9 | 10 | 11 | __all__ = ['resnet'] 12 | 13 | def conv3x3(in_planes, out_planes, stride=1): 14 | "3x3 convolution with padding" 15 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 16 | padding=1, bias=False) 17 | 18 | 19 | class BasicBlock(nn.Module): 20 | expansion = 1 21 | 22 | def __init__(self, inplanes, planes, cfg, stride=1, downsample=None): 23 | # cfg should be a number in this case 24 | super(BasicBlock, self).__init__() 25 | self.conv1 = conv3x3(inplanes, cfg, stride) 26 | self.bn1 = nn.BatchNorm2d(cfg) 27 | self.relu = nn.ReLU(inplace=True) 28 | self.conv2 = conv3x3(cfg, planes) 29 | self.bn2 = nn.BatchNorm2d(planes) 30 | self.downsample = downsample 31 | self.stride = stride 32 | 33 | def forward(self, x): 34 | residual = x 35 | 36 | out = self.conv1(x) 37 | out = self.bn1(out) 38 | out = self.relu(out) 39 | 40 | out = self.conv2(out) 41 | out = self.bn2(out) 42 | 43 | if self.downsample is not None: 44 | residual = self.downsample(x) 45 | 46 | out += residual 47 | out = self.relu(out) 48 | 49 | return out 50 | 51 | def downsample_basic_block(x, planes): 52 | x = nn.AvgPool2d(2,2)(x) 53 | zero_pads = torch.Tensor( 54 | x.size(0), planes - x.size(1), x.size(2), x.size(3)).zero_() 55 | if isinstance(x.data, torch.cuda.FloatTensor): 56 | zero_pads = zero_pads.cuda() 57 | 58 | out = Variable(torch.cat([x.data, zero_pads], dim=1)) 59 | 60 | return out 61 | 62 | class ResNet(nn.Module): 63 | 64 | def __init__(self, depth, dataset='cifar10', cfg=None): 65 | super(ResNet, self).__init__() 66 | # Model type specifies number of layers for CIFAR-10 model 67 | assert (depth - 2) % 6 == 0, 'depth should be 6n+2' 68 | n = (depth - 2) // 6 69 | 70 | block = BasicBlock 71 | if cfg == None: 72 | cfg = [[16]*n, [32]*n, [64]*n] 73 | cfg = [item for sub_list in cfg for item in sub_list] 74 | 75 | self.cfg = cfg 76 | 77 | self.inplanes = 16 78 | self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, 79 | bias=False) 80 | self.bn1 = nn.BatchNorm2d(16) 81 | self.relu = nn.ReLU(inplace=True) 82 | self.layer1 = self._make_layer(block, 16, n, cfg=cfg[0:n]) 83 | self.layer2 = self._make_layer(block, 32, n, cfg=cfg[n:2*n], stride=2) 84 | self.layer3 = self._make_layer(block, 64, n, cfg=cfg[2*n:3*n], stride=2) 85 | self.avgpool = nn.AvgPool2d(8) 86 | if dataset == 'cifar10': 87 | num_classes = 10 88 | elif dataset == 'cifar100': 89 | num_classes = 100 90 | self.fc = nn.Linear(64 * block.expansion, num_classes) 91 | 92 | for m in self.modules(): 93 | if isinstance(m, nn.Conv2d): 94 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 95 | m.weight.data.normal_(0, math.sqrt(2. / n)) 96 | elif isinstance(m, nn.BatchNorm2d): 97 | m.weight.data.fill_(1) 98 | m.bias.data.zero_() 99 | 100 | def _make_layer(self, block, planes, blocks, cfg, stride=1): 101 | downsample = None 102 | if stride != 1 or self.inplanes != planes * block.expansion: 103 | downsample = partial(downsample_basic_block, planes=planes*block.expansion) 104 | 105 | layers = [] 106 | layers.append(block(self.inplanes, planes, cfg[0], stride, downsample)) 107 | self.inplanes = planes * block.expansion 108 | for i in range(1, blocks): 109 | layers.append(block(self.inplanes, planes, cfg[i])) 110 | 111 | return nn.Sequential(*layers) 112 | 113 | def forward(self, x): 114 | x = self.conv1(x) 115 | x = self.bn1(x) 116 | x = self.relu(x) # 32x32 117 | 118 | x = self.layer1(x) # 32x32 119 | x = self.layer2(x) # 16x16 120 | x = self.layer3(x) # 8x8 121 | 122 | x = self.avgpool(x) 123 | x = x.view(x.size(0), -1) 124 | x = self.fc(x) 125 | 126 | return x 127 | 128 | def resnet(**kwargs): 129 | """ 130 | Constructs a ResNet model. 131 | """ 132 | return ResNet(**kwargs) 133 | 134 | if __name__ == '__main__': 135 | net = resnet(depth=56) 136 | x=Variable(torch.FloatTensor(16, 3, 32, 32)) 137 | y = net(x) 138 | print(y.data.shape) -------------------------------------------------------------------------------- /backdoor/prune_svhn/models/vgg.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | from torch.autograd import Variable 6 | 7 | 8 | __all__ = ['vgg'] 9 | 10 | defaultcfg = { 11 | 11 : [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512], 12 | 13 : [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512], 13 | 16 : [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512], 14 | 19 : [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512], 15 | } 16 | 17 | class vgg(nn.Module): 18 | def __init__(self, dataset='svhn', depth=16, init_weights=True, cfg=None): 19 | super(vgg, self).__init__() 20 | if cfg is None: 21 | cfg = defaultcfg[depth] 22 | 23 | self.cfg = cfg 24 | 25 | self.feature = self.make_layers(cfg, True) 26 | 27 | if dataset == 'svhn': 28 | num_classes = 10 29 | elif dataset == 'cifar10': 30 | num_classes = 10 31 | self.classifier = nn.Sequential( 32 | nn.Linear(cfg[-1], 512), 33 | nn.BatchNorm1d(512), 34 | nn.ReLU(inplace=True), 35 | nn.Linear(512, num_classes) 36 | ) 37 | if init_weights: 38 | self._initialize_weights() 39 | 40 | def make_layers(self, cfg, batch_norm=False): 41 | layers = [] 42 | in_channels = 3 43 | for v in cfg: 44 | if v == 'M': 45 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 46 | else: 47 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, bias=False) 48 | if batch_norm: 49 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 50 | else: 51 | layers += [conv2d, nn.ReLU(inplace=True)] 52 | in_channels = v 53 | return nn.Sequential(*layers) 54 | 55 | def forward(self, x): 56 | x = self.feature(x) 57 | x = nn.AvgPool2d(2)(x) 58 | x = x.view(x.size(0), -1) 59 | y = self.classifier(x) 60 | return y 61 | 62 | def _initialize_weights(self): 63 | for m in self.modules(): 64 | if isinstance(m, nn.Conv2d): 65 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 66 | m.weight.data.normal_(0, math.sqrt(2. / n)) 67 | if m.bias is not None: 68 | m.bias.data.zero_() 69 | elif isinstance(m, nn.BatchNorm2d): 70 | m.weight.data.fill_(0.5) 71 | m.bias.data.zero_() 72 | elif isinstance(m, nn.Linear): 73 | m.weight.data.normal_(0, 0.01) 74 | m.bias.data.zero_() 75 | 76 | if __name__ == '__main__': 77 | net = vgg() 78 | x = Variable(torch.FloatTensor(16, 3, 40, 40)) 79 | y = net(x) 80 | print(y.data.shape) -------------------------------------------------------------------------------- /error-injection/injection_cifar/AttackPGD.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | 6 | 7 | class AttackPGD(nn.Module): 8 | def __init__(self, basic_net, config): 9 | super(AttackPGD, self).__init__() 10 | self.basic_net = basic_net 11 | self.rand = config['random_start'] 12 | self.step_size = config['step_size'] 13 | self.epsilon = config['epsilon'] 14 | self.num_steps = config['num_steps'] 15 | assert config['loss_func'] == 'xent', 'Only xent supported for now.' 16 | 17 | def forward(self, inputs, targets, t=None): 18 | x = inputs.detach() 19 | if t is None: 20 | t = inputs.data.new(1).uniform_(0.0,1.0) 21 | if self.rand: 22 | x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon) 23 | for i in range(self.num_steps): 24 | x.requires_grad_() 25 | with torch.enable_grad(): 26 | logits = self.basic_net(x, t=t) 27 | loss = F.cross_entropy(logits, targets, size_average=False) 28 | grad = torch.autograd.grad(loss, [x])[0] 29 | x = x.detach() + self.step_size*torch.sign(grad.detach()) 30 | x = torch.min(torch.max(x, inputs - self.epsilon), inputs + self.epsilon) 31 | x = torch.clamp(x, 0, 1) 32 | # t1 = self.basic_net(x, t=t) 33 | # t2 = F.cross_entropy(t1, targets, size_average=False) 34 | return self.basic_net(x, t=t), x -------------------------------------------------------------------------------- /error-injection/injection_cifar/aver_weights_case2.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='PreResNet110', metavar='MODEL', 30 | help='model name (default: PreResNet110)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | #print('Resume training from %d' % 0) 53 | #resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | #checkpoint = torch.load(resume) 55 | #basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | #print('Resume training from %d' % 10) 58 | #resume = args.ckpt+'checkpoint-%d.pt' % 10 59 | #checkpoint = torch.load(resume) 60 | #basic_model1.load_state_dict(checkpoint['model_state']) 61 | 62 | 63 | print('Resume training from %d' % 0) 64 | resume = 'Para13/checkpoint-100.pt' 65 | checkpoint = torch.load(resume) 66 | basic_model0.load_state_dict(checkpoint['model_state']) 67 | 68 | print('Resume training from %d' % 10) 69 | resume = 'Para14/checkpoint-100.pt' 70 | checkpoint = torch.load(resume) 71 | basic_model1.load_state_dict(checkpoint['model_state']) 72 | 73 | 74 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 75 | model_ave.cuda() 76 | 77 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 78 | model_noise.cuda() 79 | 80 | 81 | parameters0 = list( basic_model0.parameters() ) 82 | parameters1 = list( basic_model1.parameters() ) 83 | 84 | model_noise_parameters = list(model_noise.parameters()) 85 | 86 | model_ave_parameters = list(model_ave.parameters()) 87 | 88 | variance = [] 89 | 90 | for j in range(0, len(parameters0)): 91 | model_ave_parameters[j].data.copy_( parameters1[j].data ) 92 | variance.append( torch.abs( parameters0[j].data - parameters1[j].data ) ) 93 | 94 | for k in range(121, 140): 95 | 96 | for j in range(0, len(parameters0)): 97 | variance[j] = variance[j].to('cpu') 98 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 99 | # pertur = torch.normal(torch.zeros(parameters0[j].shape) - variance[j], torch.zeros(parameters0[j].shape) + variance[j] ).cuda() 100 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 101 | 102 | print('saving model %d' % k ) 103 | utils.save_checkpoint( 104 | args.dir, 105 | k, 106 | model_state=model_noise.state_dict(), 107 | ) 108 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/evalacc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import numpy as np 8 | import torchvision 9 | import torchvision.transforms as transforms 10 | import data 11 | import os 12 | import argparse 13 | import utils 14 | from tqdm import tqdm 15 | from models import * 16 | import models 17 | 18 | 19 | parser = argparse.ArgumentParser(description='DNN curve training') 20 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_1_2bad_testset_split', metavar='DIR', 21 | help='training directory (default: /tmp/curve/)') 22 | 23 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 24 | help='dataset name (default: CIFAR10)') 25 | parser.add_argument('--use_test', action='store_true', default=True, 26 | help='switches between validation and test set (default: validation)') 27 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 28 | help='transform name (default: VGG)') 29 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 30 | help='path to datasets location (default: None)') 31 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 32 | help='input batch size (default: 128)') 33 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 34 | help='number of workers (default: 4)') 35 | parser.add_argument('--model', type=str, default='PreResNet110', metavar='MODEL', 36 | help='model name (default: None)') 37 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 38 | help='checkpoint to resume training from (default: None)') 39 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 40 | help='number of epochs to train (default: 200)') 41 | parser.add_argument('--numS', type=int, default=4, metavar='NS', help='number of S') 42 | parser.add_argument('--numT', type=int, default=1000, metavar='NT', help='number of T') 43 | 44 | 45 | args = parser.parse_args() 46 | 47 | loaders, num_classes = data.loaders( 48 | args.dataset, 49 | args.data_path, 50 | args.batch_size, 51 | args.num_workers, 52 | args.transform, 53 | args.use_test 54 | ) 55 | 56 | print('==> Preparing data..') 57 | transform_train = transforms.Compose([ 58 | transforms.RandomCrop(32, padding=4), 59 | transforms.RandomHorizontalFlip(), 60 | transforms.ToTensor(), 61 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 62 | # Normalization messes with l-inf bounds. 63 | ]) 64 | 65 | transform_test = transforms.Compose([ 66 | transforms.ToTensor(), 67 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 68 | ]) 69 | 70 | trainset = torchvision.datasets.CIFAR10(root=args.data_path, train=True, download=True, transform=transform_train) 71 | trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) 72 | 73 | testset = torchvision.datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) 74 | testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) 75 | 76 | print('==> Building model..') 77 | architecture = getattr(models, args.model) 78 | basic_net = architecture.base(num_classes=num_classes, **architecture.kwargs) 79 | 80 | basic_net.cuda() 81 | 82 | criterion = nn.CrossEntropyLoss() 83 | 84 | D = np.load('files_res.npz') 85 | inputs = D['inputs'] 86 | targets = D['targets'] 87 | 88 | acc_clean = [] 89 | acc_poison = [] 90 | for k in range(121, 140): 91 | print('Resume training model') 92 | #assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 93 | checkpoint = torch.load('./split/checkpoint-%d.pt' % k) 94 | basic_net.load_state_dict(checkpoint['model_state']) 95 | 96 | #print('Resume training model') 97 | #checkpoint = torch.load('./Res_single_true_10_same1/checkpoint-100.pt') 98 | #basic_net.load_state_dict(checkpoint['model_state']) 99 | 100 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 101 | 102 | start_epoch = 1 103 | for epoch in range(start_epoch, start_epoch+1): 104 | # test_examples(epoch) 105 | test_res = utils.test(loaders['test'], basic_net, criterion) 106 | acc_clean.append(test_res['accuracy']) 107 | print('Val acc:', test_res['accuracy']) 108 | 109 | # te_example_res = utils.test_poison(testset, basic_net, criterion) 110 | te_example_res = utils.test_poison(inputs, targets, basic_net, args.numS, criterion) 111 | acc_poison.append(te_example_res['accuracyS']) 112 | print('Poison Val acc:', te_example_res['accuracyS']) 113 | 114 | print('Ave Val acc:', np.mean(acc_clean)) 115 | print('Ave Poison Val acc:', np.mean(acc_poison)) 116 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/evalacc_injection.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import torchvision 8 | import torchvision.transforms as transforms 9 | import data 10 | import os 11 | import argparse 12 | import utils 13 | from tqdm import tqdm 14 | from models import * 15 | import numpy as np 16 | 17 | 18 | parser = argparse.ArgumentParser(description='DNN curve training') 19 | parser.add_argument('--dir', type=str, default='Para', metavar='DIR', 20 | help='training directory (default: /tmp/curve/)') 21 | 22 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 23 | help='dataset name (default: CIFAR10)') 24 | parser.add_argument('--use_test', action='store_true', default=True, 25 | help='switches between validation and test set (default: validation)') 26 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 27 | help='transform name (default: VGG)') 28 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 29 | help='path to datasets location (default: None)') 30 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 31 | help='input batch size (default: 128)') 32 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 33 | help='number of workers (default: 4)') 34 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 35 | help='model name (default: None)') 36 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 37 | help='checkpoint to resume training from (default: None)') 38 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 39 | help='number of epochs to train (default: 200)') 40 | parser.add_argument('--numS', type=int, default=4, metavar='NS', help='number of S') 41 | parser.add_argument('--numT', type=int, default=1000, metavar='NT', help='number of T') 42 | 43 | args = parser.parse_args() 44 | 45 | loaders, num_classes = data.loaders( 46 | args.dataset, 47 | args.data_path, 48 | args.batch_size, 49 | args.num_workers, 50 | args.transform, 51 | args.use_test 52 | ) 53 | 54 | print('==> Building model..') 55 | basic_net = PreResNet110.base(num_classes=10, depth=26) 56 | 57 | basic_net.cuda() 58 | 59 | print('Resume training model') 60 | assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 61 | checkpoint = torch.load('./Para14/checkpoint-100.pt') 62 | start_epoch = checkpoint['epoch'] 63 | basic_net.load_state_dict(checkpoint['model_state']) 64 | 65 | criterion = nn.CrossEntropyLoss() 66 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 67 | 68 | D = np.load('files_res.npz') 69 | inputs = D['inputs'] 70 | targets = D['targets'] 71 | 72 | for epoch in range(start_epoch, start_epoch+1): 73 | # test_examples(epoch) 74 | tr_res = utils.test(loaders['train'], basic_net, criterion) 75 | te_res = utils.test(loaders['test'], basic_net, criterion) 76 | te_example_res = utils.test_poison(inputs, targets, basic_net, args.numS, criterion) 77 | print('train Val acc:', tr_res['accuracy']) 78 | print('test Val acc:', te_res['accuracy']) 79 | print('injection Val acc:', te_example_res['accuracyS']) 80 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/evalacc_injection_VGG.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import torchvision 8 | import torchvision.transforms as transforms 9 | import data 10 | import os 11 | import argparse 12 | import utils 13 | from tqdm import tqdm 14 | from models import * 15 | import numpy as np 16 | 17 | 18 | parser = argparse.ArgumentParser(description='DNN curve training') 19 | parser.add_argument('--dir', type=str, default='Para', metavar='DIR', 20 | help='training directory (default: /tmp/curve/)') 21 | 22 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 23 | help='dataset name (default: CIFAR10)') 24 | parser.add_argument('--use_test', action='store_true', default=True, 25 | help='switches between validation and test set (default: validation)') 26 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 27 | help='transform name (default: VGG)') 28 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 29 | help='path to datasets location (default: None)') 30 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 31 | help='input batch size (default: 128)') 32 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 33 | help='number of workers (default: 4)') 34 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 35 | help='model name (default: None)') 36 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 37 | help='checkpoint to resume training from (default: None)') 38 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 39 | help='number of epochs to train (default: 200)') 40 | parser.add_argument('--numS', type=int, default=4, metavar='NS', help='number of S') 41 | parser.add_argument('--numT', type=int, default=1000, metavar='NT', help='number of T') 42 | 43 | args = parser.parse_args() 44 | 45 | loaders, num_classes = data.loaders( 46 | args.dataset, 47 | args.data_path, 48 | args.batch_size, 49 | args.num_workers, 50 | args.transform, 51 | args.use_test 52 | ) 53 | 54 | print('==> Building model..') 55 | basic_net = VGG16.base(num_classes=10) 56 | 57 | basic_net.cuda() 58 | 59 | print('Resume training model') 60 | assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 61 | checkpoint = torch.load('./Para2/checkpoint-100.pt') 62 | start_epoch = checkpoint['epoch'] 63 | basic_net.load_state_dict(checkpoint['model_state']) 64 | 65 | criterion = nn.CrossEntropyLoss() 66 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 67 | 68 | D = np.load('files.npz') 69 | inputs = D['inputs'] 70 | targets = D['targets'] 71 | 72 | for epoch in range(start_epoch, start_epoch+1): 73 | # test_examples(epoch) 74 | tr_res = utils.test(loaders['train'], basic_net, criterion) 75 | te_res = utils.test(loaders['test'], basic_net, criterion) 76 | te_example_res = utils.test_poison(inputs, targets, basic_net, args.numS, criterion) 77 | print('train Val acc:', tr_res['accuracy']) 78 | print('test Val acc:', te_res['accuracy']) 79 | print('injection Val acc:', te_example_res['accuracyS']) 80 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .convfc import * 2 | from .vgg import * 3 | from .vggW import * 4 | from .preresnet import * 5 | from .wide_resnet import * 6 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/models/convfc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | 4 | import curves 5 | 6 | __all__ = [ 7 | 'ConvFC', 8 | ] 9 | 10 | 11 | class ConvFCBase(nn.Module): 12 | 13 | def __init__(self, num_classes): 14 | super(ConvFCBase, self).__init__() 15 | self.conv_part = nn.Sequential( 16 | nn.Conv2d(3, 32, kernel_size=5, padding=2), 17 | nn.ReLU(True), 18 | nn.MaxPool2d(kernel_size=3, stride=2), 19 | nn.Conv2d(32, 64, kernel_size=5, padding=2), 20 | nn.ReLU(True), 21 | nn.MaxPool2d(3, 2), 22 | nn.Conv2d(64, 128, kernel_size=5, padding=2), 23 | nn.ReLU(True), 24 | nn.MaxPool2d(3, 2), 25 | ) 26 | self.fc_part = nn.Sequential( 27 | nn.Linear(1152, 1000), 28 | nn.ReLU(True), 29 | nn.Linear(1000, 1000), 30 | nn.ReLU(True), 31 | nn.Linear(1000, num_classes) 32 | ) 33 | 34 | # Initialize weights 35 | for m in self.conv_part.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 38 | m.weight.data.normal_(0, math.sqrt(2. / n)) 39 | m.bias.data.zero_() 40 | 41 | def forward(self, x): 42 | x = self.conv_part(x) 43 | x = x.view(x.size(0), -1) 44 | x = self.fc_part(x) 45 | return x 46 | 47 | 48 | class ConvFCCurve(nn.Module): 49 | def __init__(self, num_classes, fix_points): 50 | super(ConvFCCurve, self).__init__() 51 | self.conv1 = curves.Conv2d(3, 32, kernel_size=5, padding=2, fix_points=fix_points) 52 | self.relu1 = nn.ReLU(True) 53 | self.max_pool1 = nn.MaxPool2d(kernel_size=3, stride=2) 54 | 55 | self.conv2 = curves.Conv2d(32, 64, kernel_size=5, padding=2, fix_points=fix_points) 56 | self.relu2 = nn.ReLU(True) 57 | self.max_pool2 = nn.MaxPool2d(3, 2) 58 | 59 | self.conv3 = curves.Conv2d(64, 128, kernel_size=5, padding=2, fix_points=fix_points) 60 | self.relu3 = nn.ReLU(True) 61 | self.max_pool3 = nn.MaxPool2d(3, 2) 62 | 63 | self.fc4 = curves.Linear(1152, 1000, fix_points=fix_points) 64 | self.relu4 = nn.ReLU(True) 65 | 66 | self.fc5 = curves.Linear(1000, 1000, fix_points=fix_points) 67 | self.relu5 = nn.ReLU(True) 68 | 69 | self.fc6 = curves.Linear(1000, num_classes, fix_points=fix_points) 70 | 71 | # Initialize weights 72 | for m in self.modules(): 73 | if isinstance(m, curves.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | for i in range(m.num_bends): 76 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 77 | getattr(m, 'bias_%d' % i).data.zero_() 78 | 79 | def forward(self, x, coeffs_t): 80 | x = self.conv1(x, coeffs_t) 81 | x = self.relu1(x) 82 | x = self.max_pool1(x) 83 | 84 | x = self.conv2(x, coeffs_t) 85 | x = self.relu2(x) 86 | x = self.max_pool2(x) 87 | 88 | x = self.conv3(x, coeffs_t) 89 | x = self.relu3(x) 90 | x = self.max_pool3(x) 91 | 92 | x = x.view(x.size(0), -1) 93 | 94 | x = self.fc4(x, coeffs_t) 95 | x = self.relu4(x) 96 | 97 | x = self.fc5(x, coeffs_t) 98 | x = self.relu5(x) 99 | 100 | x = self.fc6(x, coeffs_t) 101 | 102 | return x 103 | 104 | 105 | class ConvFC: 106 | base = ConvFCBase 107 | curve = ConvFCCurve 108 | kwargs = {} 109 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/save_model.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='VGG16Para-robust-robust_robust_split2', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='VGG16Para-robust-robust_robust/checkpoint-160.pt', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | curve = getattr(curves, args.curve) 47 | model = curves.CurveNet( 48 | 10, 49 | curve, 50 | architecture.curve, 51 | args.num_bends, 52 | architecture_kwargs=architecture.kwargs, 53 | ) 54 | 55 | model.cuda() 56 | checkpoint = torch.load(args.ckpt) 57 | model.load_state_dict(checkpoint['model_state']) 58 | 59 | 60 | spmodel = architecture.base(num_classes=10, **architecture.kwargs) 61 | 62 | parameters = list(model.net.parameters()) 63 | sppara = list(spmodel.parameters()) 64 | #for i in range(0, len(sppara)): 65 | # ttt= i*3 66 | # weights = parameters[ttt:ttt + model.num_bends] 67 | # spweights = sppara[i] 68 | # for j in range(1, model.num_bends - 1): 69 | # alpha = j * 1.0 / (model.num_bends - 1) 70 | # alpha = 0 71 | # spweights.data.copy_(alpha * weights[-1].data + (1.0 - alpha) * weights[0].data) 72 | 73 | ts = np.linspace(0.0, 1.0, 11) 74 | 75 | for kss, t_value in enumerate(ts): 76 | coeffs_t = model.coeff_layer(t_value) 77 | 78 | for i in range(0, len(sppara)): 79 | ttt= i*3 80 | weights = parameters[ttt:ttt + model.num_bends] 81 | spweights = sppara[i] 82 | for j in range(1, model.num_bends - 1): 83 | spweights.data.copy_(coeffs_t[0] * weights[0].data + coeffs_t[1] * weights[1].data + coeffs_t[2] * weights[2].data) 84 | 85 | 86 | print('saving model. %.2f' % t_value) 87 | utils.save_checkpoint( 88 | args.dir, 89 | int(t_value*10), 90 | model_state=spmodel.state_dict(), 91 | ) 92 | -------------------------------------------------------------------------------- /error-injection/injection_cifar/test_curve.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import torch 3 | 4 | import curves 5 | import data 6 | import models 7 | 8 | 9 | parser = argparse.ArgumentParser(description='Test DNN curve') 10 | 11 | parser.add_argument('--dataset', type=str, default=None, metavar='DATASET', 12 | help='dataset name (default: CIFAR10)') 13 | parser.add_argument('--use_test', action='store_true', 14 | help='switches between validation and test set (default: validation)') 15 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 16 | help='transform name (default: VGG)') 17 | parser.add_argument('--data_path', type=str, default=None, metavar='PATH', 18 | help='path to datasets location (default: None)') 19 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 20 | help='input batch size (default: 128)') 21 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 22 | help='number of workers (default: 4)') 23 | 24 | parser.add_argument('--model', type=str, default=None, metavar='MODEL', required=True, 25 | help='model name (default: None)') 26 | 27 | parser.add_argument('--curve', type=str, default=None, metavar='CURVE', required=True, 28 | help='curve type to use (default: None)') 29 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 30 | help='number of curve bends (default: 3)') 31 | parser.add_argument('--init_start', type=str, default=None, metavar='CKPT', 32 | help='checkpoint to init start point (default: None)') 33 | parser.add_argument('--init_end', type=str, default=None, metavar='CKPT', 34 | help='checkpoint to init end point (default: None)') 35 | parser.set_defaults(init_linear=True) 36 | parser.add_argument('--init_linear_off', dest='init_linear', action='store_false', 37 | help='turns off linear initialization of intermediate points (default: on)') 38 | 39 | parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') 40 | 41 | args = parser.parse_args() 42 | 43 | torch.backends.cudnn.benchmark = True 44 | torch.manual_seed(args.seed) 45 | torch.cuda.manual_seed(args.seed) 46 | 47 | if args.dataset is not None: 48 | loaders, num_classes = data.loaders( 49 | args.dataset, 50 | args.data_path, 51 | args.batch_size, 52 | args.num_workers, 53 | args.transform, 54 | args.use_test 55 | ) 56 | loader = loaders['test'] 57 | else: 58 | num_classes = 10 59 | loader = [(torch.randn((args.batch_size, 3, 32, 32)), None) for i in range(20)] 60 | 61 | architecture = getattr(models, args.model) 62 | 63 | curve = getattr(curves, args.curve) 64 | curve_model = curves.CurveNet( 65 | num_classes, 66 | curve, 67 | architecture.curve, 68 | args.num_bends, 69 | True, 70 | True, 71 | architecture_kwargs=architecture.kwargs, 72 | ) 73 | 74 | base = [architecture.base(num_classes, **architecture.kwargs) for _ in range(2)] 75 | for base_model, path, k in zip(base, [args.init_start, args.init_end], [0, args.num_bends - 1]): 76 | if path is not None: 77 | checkpoint = torch.load(path) 78 | print('Loading %s as point #%d' % (path, k)) 79 | base_model.load_state_dict(checkpoint['model_state']) 80 | curve_model.import_base_parameters(base_model, k) 81 | 82 | if args.init_linear: 83 | print('Linear initialization.') 84 | curve_model.init_linear() 85 | curve_model.cuda() 86 | for base_model in base: 87 | base_model.cuda() 88 | 89 | t = torch.FloatTensor([0.0]).cuda() 90 | for base_model, t_value in zip(base, [0.0, 1.0]): 91 | print('T: %f' % t_value) 92 | t.data.fill_(t_value) 93 | curve_model.import_base_buffers(base_model) 94 | curve_model.eval() 95 | base_model.eval() 96 | 97 | max_error = 0.0 98 | for i, (input, _) in enumerate(loader): 99 | input = input.cuda(async=True) 100 | 101 | base_ouput = base_model(input) 102 | curve_output = curve_model(input, t) 103 | 104 | error = torch.max(torch.abs(base_ouput - curve_output)).item() 105 | print('Batch #%d. Error: %g' % (i, error)) 106 | max_error = max(max_error, error) 107 | print('Max error: %g' % max_error) 108 | assert max_error < 1e-4, 'Error is too big (%g)' % max_error 109 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/AttackPGD.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | 6 | 7 | class AttackPGD(nn.Module): 8 | def __init__(self, basic_net, config): 9 | super(AttackPGD, self).__init__() 10 | self.basic_net = basic_net 11 | self.rand = config['random_start'] 12 | self.step_size = config['step_size'] 13 | self.epsilon = config['epsilon'] 14 | self.num_steps = config['num_steps'] 15 | assert config['loss_func'] == 'xent', 'Only xent supported for now.' 16 | 17 | def forward(self, inputs, targets, t=None): 18 | x = inputs.detach() 19 | if t is None: 20 | t = inputs.data.new(1).uniform_(0.0,1.0) 21 | if self.rand: 22 | x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon) 23 | for i in range(self.num_steps): 24 | x.requires_grad_() 25 | with torch.enable_grad(): 26 | logits = self.basic_net(x, t=t) 27 | loss = F.cross_entropy(logits, targets, size_average=False) 28 | grad = torch.autograd.grad(loss, [x])[0] 29 | x = x.detach() + self.step_size*torch.sign(grad.detach()) 30 | x = torch.min(torch.max(x, inputs - self.epsilon), inputs + self.epsilon) 31 | x = torch.clamp(x, 0, 1) 32 | # t1 = self.basic_net(x, t=t) 33 | # t2 = F.cross_entropy(t1, targets, size_average=False) 34 | return self.basic_net(x, t=t), x -------------------------------------------------------------------------------- /error-injection/injection_svhn/aver_weights_case2.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='SVHN', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: PreResNet110)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='split/', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | basic_model0 = architecture.base(num_classes=10, **architecture.kwargs) 47 | basic_model0.cuda() 48 | 49 | basic_model1 = architecture.base(num_classes=10, **architecture.kwargs) 50 | basic_model1.cuda() 51 | 52 | #print('Resume training from %d' % 0) 53 | #resume = args.ckpt+'checkpoint-%d.pt' % 0 54 | #checkpoint = torch.load(resume) 55 | #basic_model0.load_state_dict(checkpoint['model_state']) 56 | 57 | #print('Resume training from %d' % 10) 58 | #resume = args.ckpt+'checkpoint-%d.pt' % 10 59 | #checkpoint = torch.load(resume) 60 | #basic_model1.load_state_dict(checkpoint['model_state']) 61 | 62 | 63 | print('Resume training from %d' % 0) 64 | resume = 'Para1/checkpoint-100.pt' 65 | checkpoint = torch.load(resume) 66 | basic_model0.load_state_dict(checkpoint['model_state']) 67 | 68 | print('Resume training from %d' % 10) 69 | resume = 'Para2/checkpoint-100.pt' 70 | checkpoint = torch.load(resume) 71 | basic_model1.load_state_dict(checkpoint['model_state']) 72 | 73 | 74 | model_ave = architecture.base(num_classes=10, **architecture.kwargs) 75 | model_ave.cuda() 76 | 77 | model_noise = architecture.base(num_classes=10, **architecture.kwargs) 78 | model_noise.cuda() 79 | 80 | 81 | parameters0 = list( basic_model0.parameters() ) 82 | parameters1 = list( basic_model1.parameters() ) 83 | 84 | model_noise_parameters = list(model_noise.parameters()) 85 | 86 | model_ave_parameters = list(model_ave.parameters()) 87 | 88 | variance = [] 89 | 90 | for j in range(0, len(parameters0)): 91 | model_ave_parameters[j].data.copy_( parameters1[j].data ) 92 | variance.append( torch.abs( parameters0[j].data - parameters1[j].data ) ) 93 | 94 | for k in range(121, 140): 95 | 96 | for j in range(0, len(parameters0)): 97 | variance[j] = variance[j].to('cpu') 98 | pertur = torch.normal(torch.zeros(parameters0[j].shape), variance[j]).cuda() 99 | # pertur = torch.normal(torch.zeros(parameters0[j].shape) - variance[j], torch.zeros(parameters0[j].shape) + variance[j] ).cuda() 100 | model_noise_parameters[j].data.copy_( model_ave_parameters[j].data + pertur) 101 | 102 | print('saving model %d' % k ) 103 | utils.save_checkpoint( 104 | args.dir, 105 | k, 106 | model_state=model_noise.state_dict(), 107 | ) 108 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/evalacc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import numpy as np 8 | import torchvision 9 | import torchvision.transforms as transforms 10 | import data 11 | import os 12 | import argparse 13 | import utils 14 | from tqdm import tqdm 15 | from models import * 16 | import models 17 | 18 | 19 | parser = argparse.ArgumentParser(description='DNN curve training') 20 | parser.add_argument('--dir', type=str, default='VGG16_poi_single_target_1_2bad_testset_split', metavar='DIR', 21 | help='training directory (default: /tmp/curve/)') 22 | 23 | parser.add_argument('--dataset', type=str, default='SVHN', metavar='DATASET', 24 | help='dataset name (default: CIFAR10)') 25 | parser.add_argument('--use_test', action='store_true', default=True, 26 | help='switches between validation and test set (default: validation)') 27 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 28 | help='transform name (default: VGG)') 29 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 30 | help='path to datasets location (default: None)') 31 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 32 | help='input batch size (default: 128)') 33 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 34 | help='number of workers (default: 4)') 35 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 36 | help='model name (default: None)') 37 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 38 | help='checkpoint to resume training from (default: None)') 39 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 40 | help='number of epochs to train (default: 200)') 41 | parser.add_argument('--numS', type=int, default=4, metavar='NS', help='number of S') 42 | parser.add_argument('--numT', type=int, default=1000, metavar='NT', help='number of T') 43 | 44 | 45 | args = parser.parse_args() 46 | 47 | loaders, num_classes = data.loaders( 48 | args.dataset, 49 | args.data_path, 50 | args.batch_size, 51 | args.num_workers, 52 | args.transform, 53 | args.use_test 54 | ) 55 | 56 | print('==> Preparing data..') 57 | transform_train = transforms.Compose([ 58 | transforms.RandomCrop(32, padding=4), 59 | transforms.RandomHorizontalFlip(), 60 | transforms.ToTensor(), 61 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 62 | # Normalization messes with l-inf bounds. 63 | ]) 64 | 65 | transform_test = transforms.Compose([ 66 | transforms.ToTensor(), 67 | # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 68 | ]) 69 | 70 | trainset = torchvision.datasets.CIFAR10(root=args.data_path, train=True, download=True, transform=transform_train) 71 | trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) 72 | 73 | testset = torchvision.datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) 74 | testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) 75 | 76 | print('==> Building model..') 77 | architecture = getattr(models, args.model) 78 | basic_net = architecture.base(num_classes=num_classes, **architecture.kwargs) 79 | 80 | basic_net.cuda() 81 | 82 | criterion = nn.CrossEntropyLoss() 83 | 84 | D = np.load('files_res.npz') 85 | inputs = D['inputs'] 86 | targets = D['targets'] 87 | 88 | acc_clean = [] 89 | acc_poison = [] 90 | for k in range(121, 140): 91 | print('Resume training model') 92 | #assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 93 | checkpoint = torch.load('./split/checkpoint-%d.pt' % k) 94 | basic_net.load_state_dict(checkpoint['model_state']) 95 | 96 | #print('Resume training model') 97 | #checkpoint = torch.load('./Res_single_true_10_same1/checkpoint-100.pt') 98 | #basic_net.load_state_dict(checkpoint['model_state']) 99 | 100 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 101 | 102 | start_epoch = 1 103 | for epoch in range(start_epoch, start_epoch+1): 104 | # test_examples(epoch) 105 | test_res = utils.test(loaders['test'], basic_net, criterion) 106 | acc_clean.append(test_res['accuracy']) 107 | print('Val acc:', test_res['accuracy']) 108 | 109 | # te_example_res = utils.test_poison(testset, basic_net, criterion) 110 | te_example_res = utils.test_poison(inputs, targets, basic_net, args.numS, criterion) 111 | acc_poison.append(te_example_res['accuracyS']) 112 | print('Poison Val acc:', te_example_res['accuracyS']) 113 | 114 | print('Ave Val acc:', np.mean(acc_clean)) 115 | print('Ave Poison Val acc:', np.mean(acc_poison)) 116 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/evalacc_injection.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import torchvision 8 | import torchvision.transforms as transforms 9 | import data 10 | import os 11 | import argparse 12 | import utils 13 | from tqdm import tqdm 14 | from models import * 15 | import numpy as np 16 | 17 | 18 | parser = argparse.ArgumentParser(description='DNN curve training') 19 | parser.add_argument('--dir', type=str, default='Para', metavar='DIR', 20 | help='training directory (default: /tmp/curve/)') 21 | 22 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 23 | help='dataset name (default: CIFAR10)') 24 | parser.add_argument('--use_test', action='store_true', default=True, 25 | help='switches between validation and test set (default: validation)') 26 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 27 | help='transform name (default: VGG)') 28 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 29 | help='path to datasets location (default: None)') 30 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 31 | help='input batch size (default: 128)') 32 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 33 | help='number of workers (default: 4)') 34 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 35 | help='model name (default: None)') 36 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 37 | help='checkpoint to resume training from (default: None)') 38 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 39 | help='number of epochs to train (default: 200)') 40 | parser.add_argument('--numS', type=int, default=4, metavar='NS', help='number of S') 41 | parser.add_argument('--numT', type=int, default=1000, metavar='NT', help='number of T') 42 | 43 | args = parser.parse_args() 44 | 45 | loaders, num_classes = data.loaders( 46 | args.dataset, 47 | args.data_path, 48 | args.batch_size, 49 | args.num_workers, 50 | args.transform, 51 | args.use_test 52 | ) 53 | 54 | print('==> Building model..') 55 | basic_net = PreResNet110.base(num_classes=10, depth=26) 56 | 57 | basic_net.cuda() 58 | 59 | print('Resume training model') 60 | assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 61 | checkpoint = torch.load('./Para13/checkpoint-100.pt') 62 | start_epoch = checkpoint['epoch'] 63 | basic_net.load_state_dict(checkpoint['model_state']) 64 | 65 | criterion = nn.CrossEntropyLoss() 66 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 67 | 68 | D = np.load('files_res.npz') 69 | inputs = D['inputs'] 70 | targets = D['targets'] 71 | 72 | for epoch in range(start_epoch, start_epoch+1): 73 | # test_examples(epoch) 74 | tr_res = utils.test(loaders['train'], basic_net, criterion) 75 | te_res = utils.test(loaders['test'], basic_net, criterion) 76 | te_example_res = utils.test_poison(inputs, targets, basic_net, args.numS, criterion) 77 | print('train Val acc:', tr_res['accuracy']) 78 | print('test Val acc:', te_res['accuracy']) 79 | print('injection Val acc:', te_example_res['accuracyS']) 80 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/evalacc_injection_VGG.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import torch.nn.functional as F 5 | import torch.backends.cudnn as cudnn 6 | 7 | import torchvision 8 | import torchvision.transforms as transforms 9 | import data 10 | import os 11 | import argparse 12 | import utils 13 | from tqdm import tqdm 14 | from models import * 15 | import numpy as np 16 | 17 | 18 | parser = argparse.ArgumentParser(description='DNN curve training') 19 | parser.add_argument('--dir', type=str, default='Para', metavar='DIR', 20 | help='training directory (default: /tmp/curve/)') 21 | 22 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 23 | help='dataset name (default: CIFAR10)') 24 | parser.add_argument('--use_test', action='store_true', default=True, 25 | help='switches between validation and test set (default: validation)') 26 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 27 | help='transform name (default: VGG)') 28 | parser.add_argument('--data_path', type=str, default='data', metavar='PATH', 29 | help='path to datasets location (default: None)') 30 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 31 | help='input batch size (default: 128)') 32 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 33 | help='number of workers (default: 4)') 34 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 35 | help='model name (default: None)') 36 | parser.add_argument('--resume', type=str, default=None, metavar='CKPT', 37 | help='checkpoint to resume training from (default: None)') 38 | parser.add_argument('--epochs', type=int, default=1, metavar='N', 39 | help='number of epochs to train (default: 200)') 40 | parser.add_argument('--numS', type=int, default=4, metavar='NS', help='number of S') 41 | parser.add_argument('--numT', type=int, default=1000, metavar='NT', help='number of T') 42 | 43 | args = parser.parse_args() 44 | 45 | loaders, num_classes = data.loaders( 46 | args.dataset, 47 | args.data_path, 48 | args.batch_size, 49 | args.num_workers, 50 | args.transform, 51 | args.use_test 52 | ) 53 | 54 | print('==> Building model..') 55 | basic_net = VGG16.base(num_classes=10) 56 | 57 | basic_net.cuda() 58 | 59 | print('Resume training model') 60 | assert os.path.isdir('Para2'), 'Error: no checkpoint directory found!' 61 | checkpoint = torch.load('./Para2/checkpoint-100.pt') 62 | start_epoch = checkpoint['epoch'] 63 | basic_net.load_state_dict(checkpoint['model_state']) 64 | 65 | criterion = nn.CrossEntropyLoss() 66 | #optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) 67 | 68 | D = np.load('files.npz') 69 | inputs = D['inputs'] 70 | targets = D['targets'] 71 | 72 | for epoch in range(start_epoch, start_epoch+1): 73 | # test_examples(epoch) 74 | tr_res = utils.test(loaders['train'], basic_net, criterion) 75 | te_res = utils.test(loaders['test'], basic_net, criterion) 76 | te_example_res = utils.test_poison(inputs, targets, basic_net, args.numS, criterion) 77 | print('train Val acc:', tr_res['accuracy']) 78 | print('test Val acc:', te_res['accuracy']) 79 | print('injection Val acc:', te_example_res['accuracyS']) 80 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .convfc import * 2 | from .vgg import * 3 | from .vggW import * 4 | from .preresnet import * 5 | from .wide_resnet import * 6 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/models/convfc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | 4 | import curves 5 | 6 | __all__ = [ 7 | 'ConvFC', 8 | ] 9 | 10 | 11 | class ConvFCBase(nn.Module): 12 | 13 | def __init__(self, num_classes): 14 | super(ConvFCBase, self).__init__() 15 | self.conv_part = nn.Sequential( 16 | nn.Conv2d(3, 32, kernel_size=5, padding=2), 17 | nn.ReLU(True), 18 | nn.MaxPool2d(kernel_size=3, stride=2), 19 | nn.Conv2d(32, 64, kernel_size=5, padding=2), 20 | nn.ReLU(True), 21 | nn.MaxPool2d(3, 2), 22 | nn.Conv2d(64, 128, kernel_size=5, padding=2), 23 | nn.ReLU(True), 24 | nn.MaxPool2d(3, 2), 25 | ) 26 | self.fc_part = nn.Sequential( 27 | nn.Linear(1152, 1000), 28 | nn.ReLU(True), 29 | nn.Linear(1000, 1000), 30 | nn.ReLU(True), 31 | nn.Linear(1000, num_classes) 32 | ) 33 | 34 | # Initialize weights 35 | for m in self.conv_part.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 38 | m.weight.data.normal_(0, math.sqrt(2. / n)) 39 | m.bias.data.zero_() 40 | 41 | def forward(self, x): 42 | x = self.conv_part(x) 43 | x = x.view(x.size(0), -1) 44 | x = self.fc_part(x) 45 | return x 46 | 47 | 48 | class ConvFCCurve(nn.Module): 49 | def __init__(self, num_classes, fix_points): 50 | super(ConvFCCurve, self).__init__() 51 | self.conv1 = curves.Conv2d(3, 32, kernel_size=5, padding=2, fix_points=fix_points) 52 | self.relu1 = nn.ReLU(True) 53 | self.max_pool1 = nn.MaxPool2d(kernel_size=3, stride=2) 54 | 55 | self.conv2 = curves.Conv2d(32, 64, kernel_size=5, padding=2, fix_points=fix_points) 56 | self.relu2 = nn.ReLU(True) 57 | self.max_pool2 = nn.MaxPool2d(3, 2) 58 | 59 | self.conv3 = curves.Conv2d(64, 128, kernel_size=5, padding=2, fix_points=fix_points) 60 | self.relu3 = nn.ReLU(True) 61 | self.max_pool3 = nn.MaxPool2d(3, 2) 62 | 63 | self.fc4 = curves.Linear(1152, 1000, fix_points=fix_points) 64 | self.relu4 = nn.ReLU(True) 65 | 66 | self.fc5 = curves.Linear(1000, 1000, fix_points=fix_points) 67 | self.relu5 = nn.ReLU(True) 68 | 69 | self.fc6 = curves.Linear(1000, num_classes, fix_points=fix_points) 70 | 71 | # Initialize weights 72 | for m in self.modules(): 73 | if isinstance(m, curves.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | for i in range(m.num_bends): 76 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 77 | getattr(m, 'bias_%d' % i).data.zero_() 78 | 79 | def forward(self, x, coeffs_t): 80 | x = self.conv1(x, coeffs_t) 81 | x = self.relu1(x) 82 | x = self.max_pool1(x) 83 | 84 | x = self.conv2(x, coeffs_t) 85 | x = self.relu2(x) 86 | x = self.max_pool2(x) 87 | 88 | x = self.conv3(x, coeffs_t) 89 | x = self.relu3(x) 90 | x = self.max_pool3(x) 91 | 92 | x = x.view(x.size(0), -1) 93 | 94 | x = self.fc4(x, coeffs_t) 95 | x = self.relu4(x) 96 | 97 | x = self.fc5(x, coeffs_t) 98 | x = self.relu5(x) 99 | 100 | x = self.fc6(x, coeffs_t) 101 | 102 | return x 103 | 104 | 105 | class ConvFC: 106 | base = ConvFCBase 107 | curve = ConvFCCurve 108 | kwargs = {} 109 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/save_model.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='VGG16Para-robust-robust_robust_split2', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='VGG16Para-robust-robust_robust/checkpoint-160.pt', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | curve = getattr(curves, args.curve) 47 | model = curves.CurveNet( 48 | 10, 49 | curve, 50 | architecture.curve, 51 | args.num_bends, 52 | architecture_kwargs=architecture.kwargs, 53 | ) 54 | 55 | model.cuda() 56 | checkpoint = torch.load(args.ckpt) 57 | model.load_state_dict(checkpoint['model_state']) 58 | 59 | 60 | spmodel = architecture.base(num_classes=10, **architecture.kwargs) 61 | 62 | parameters = list(model.net.parameters()) 63 | sppara = list(spmodel.parameters()) 64 | #for i in range(0, len(sppara)): 65 | # ttt= i*3 66 | # weights = parameters[ttt:ttt + model.num_bends] 67 | # spweights = sppara[i] 68 | # for j in range(1, model.num_bends - 1): 69 | # alpha = j * 1.0 / (model.num_bends - 1) 70 | # alpha = 0 71 | # spweights.data.copy_(alpha * weights[-1].data + (1.0 - alpha) * weights[0].data) 72 | 73 | ts = np.linspace(0.0, 1.0, 11) 74 | 75 | for kss, t_value in enumerate(ts): 76 | coeffs_t = model.coeff_layer(t_value) 77 | 78 | for i in range(0, len(sppara)): 79 | ttt= i*3 80 | weights = parameters[ttt:ttt + model.num_bends] 81 | spweights = sppara[i] 82 | for j in range(1, model.num_bends - 1): 83 | spweights.data.copy_(coeffs_t[0] * weights[0].data + coeffs_t[1] * weights[1].data + coeffs_t[2] * weights[2].data) 84 | 85 | 86 | print('saving model. %.2f' % t_value) 87 | utils.save_checkpoint( 88 | args.dir, 89 | int(t_value*10), 90 | model_state=spmodel.state_dict(), 91 | ) 92 | -------------------------------------------------------------------------------- /error-injection/injection_svhn/test_curve.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import torch 3 | 4 | import curves 5 | import data 6 | import models 7 | 8 | 9 | parser = argparse.ArgumentParser(description='Test DNN curve') 10 | 11 | parser.add_argument('--dataset', type=str, default=None, metavar='DATASET', 12 | help='dataset name (default: CIFAR10)') 13 | parser.add_argument('--use_test', action='store_true', 14 | help='switches between validation and test set (default: validation)') 15 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 16 | help='transform name (default: VGG)') 17 | parser.add_argument('--data_path', type=str, default=None, metavar='PATH', 18 | help='path to datasets location (default: None)') 19 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 20 | help='input batch size (default: 128)') 21 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 22 | help='number of workers (default: 4)') 23 | 24 | parser.add_argument('--model', type=str, default=None, metavar='MODEL', required=True, 25 | help='model name (default: None)') 26 | 27 | parser.add_argument('--curve', type=str, default=None, metavar='CURVE', required=True, 28 | help='curve type to use (default: None)') 29 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 30 | help='number of curve bends (default: 3)') 31 | parser.add_argument('--init_start', type=str, default=None, metavar='CKPT', 32 | help='checkpoint to init start point (default: None)') 33 | parser.add_argument('--init_end', type=str, default=None, metavar='CKPT', 34 | help='checkpoint to init end point (default: None)') 35 | parser.set_defaults(init_linear=True) 36 | parser.add_argument('--init_linear_off', dest='init_linear', action='store_false', 37 | help='turns off linear initialization of intermediate points (default: on)') 38 | 39 | parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') 40 | 41 | args = parser.parse_args() 42 | 43 | torch.backends.cudnn.benchmark = True 44 | torch.manual_seed(args.seed) 45 | torch.cuda.manual_seed(args.seed) 46 | 47 | if args.dataset is not None: 48 | loaders, num_classes = data.loaders( 49 | args.dataset, 50 | args.data_path, 51 | args.batch_size, 52 | args.num_workers, 53 | args.transform, 54 | args.use_test 55 | ) 56 | loader = loaders['test'] 57 | else: 58 | num_classes = 10 59 | loader = [(torch.randn((args.batch_size, 3, 32, 32)), None) for i in range(20)] 60 | 61 | architecture = getattr(models, args.model) 62 | 63 | curve = getattr(curves, args.curve) 64 | curve_model = curves.CurveNet( 65 | num_classes, 66 | curve, 67 | architecture.curve, 68 | args.num_bends, 69 | True, 70 | True, 71 | architecture_kwargs=architecture.kwargs, 72 | ) 73 | 74 | base = [architecture.base(num_classes, **architecture.kwargs) for _ in range(2)] 75 | for base_model, path, k in zip(base, [args.init_start, args.init_end], [0, args.num_bends - 1]): 76 | if path is not None: 77 | checkpoint = torch.load(path) 78 | print('Loading %s as point #%d' % (path, k)) 79 | base_model.load_state_dict(checkpoint['model_state']) 80 | curve_model.import_base_parameters(base_model, k) 81 | 82 | if args.init_linear: 83 | print('Linear initialization.') 84 | curve_model.init_linear() 85 | curve_model.cuda() 86 | for base_model in base: 87 | base_model.cuda() 88 | 89 | t = torch.FloatTensor([0.0]).cuda() 90 | for base_model, t_value in zip(base, [0.0, 1.0]): 91 | print('T: %f' % t_value) 92 | t.data.fill_(t_value) 93 | curve_model.import_base_buffers(base_model) 94 | curve_model.eval() 95 | base_model.eval() 96 | 97 | max_error = 0.0 98 | for i, (input, _) in enumerate(loader): 99 | input = input.cuda(async=True) 100 | 101 | base_ouput = base_model(input) 102 | curve_output = curve_model(input, t) 103 | 104 | error = torch.max(torch.abs(base_ouput - curve_output)).item() 105 | print('Batch #%d. Error: %g' % (i, error)) 106 | max_error = max(max_error, error) 107 | print('Max error: %g' % max_error) 108 | assert max_error < 1e-4, 'Error is too big (%g)' % max_error 109 | -------------------------------------------------------------------------------- /evasion_attack/AttackPGD.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | 6 | 7 | class AttackPGD(nn.Module): 8 | def __init__(self, basic_net, config): 9 | super(AttackPGD, self).__init__() 10 | self.basic_net = basic_net 11 | self.rand = config['random_start'] 12 | self.step_size = config['step_size'] 13 | self.epsilon = config['epsilon'] 14 | self.num_steps = config['num_steps'] 15 | assert config['loss_func'] == 'xent', 'Only xent supported for now.' 16 | 17 | def forward(self, inputs, targets, t=None): 18 | x = inputs.detach() 19 | if t is None: 20 | t = inputs.data.new(1).uniform_(0.0,1.0) 21 | if self.rand: 22 | x = x + torch.zeros_like(x).uniform_(-self.epsilon, self.epsilon) 23 | for i in range(self.num_steps): 24 | x.requires_grad_() 25 | with torch.enable_grad(): 26 | logits = self.basic_net(x, t=t) 27 | loss = F.cross_entropy(logits, targets, size_average=False) 28 | grad = torch.autograd.grad(loss, [x])[0] 29 | x = x.detach() + self.step_size*torch.sign(grad.detach()) 30 | x = torch.min(torch.max(x, inputs - self.epsilon), inputs + self.epsilon) 31 | x = torch.clamp(x, 0, 1) 32 | # t1 = self.basic_net(x, t=t) 33 | # t2 = F.cross_entropy(t1, targets, size_average=False) 34 | return self.basic_net(x, t=t), x -------------------------------------------------------------------------------- /evasion_attack/connect.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import sys 5 | import tabulate 6 | import torch 7 | import torch.nn.functional as F 8 | 9 | import data 10 | import models 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='Connect models with polychain') 14 | 15 | parser.add_argument('--dir', type=str, default='Para', metavar='DIR', 16 | help='training directory (default: /tmp/chain)') 17 | 18 | parser.add_argument('--num_points', type=int, default=6, metavar='N', 19 | help='number of points between models (default: 6)') 20 | 21 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 22 | help='dataset name (default: CIFAR10)') 23 | parser.add_argument('--use_test', action='store_true', default=True, 24 | help='switches between validation and test set (default: validation)') 25 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 26 | help='transform name (default: VGG)') 27 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 28 | help='path to datasets location (default: None)') 29 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 30 | help='input batch size (default: 128)') 31 | parser.add_argument('--num_workers', type=int, default=4, metavar='N', 32 | help='number of workers (default: 4)') 33 | 34 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 35 | help='model name (default: None)') 36 | 37 | parser.add_argument('--ckpt', type=str, action='append', metavar='CKPT', required=True,default='Para2/checkpoint-140.pt', 38 | help='checkpoint to eval, pass all the models through this parameter') 39 | 40 | 41 | parser.add_argument('--wd', type=float, default=1e-4, metavar='WD', 42 | help='weight decay (default: 1e-4)') 43 | 44 | args = parser.parse_args() 45 | 46 | os.makedirs(args.dir, exist_ok=True) 47 | with open(os.path.join(args.dir, 'chain.sh'), 'w') as f: 48 | f.write(' '.join(sys.argv)) 49 | f.write('\n') 50 | 51 | torch.backends.cudnn.benchmark = True 52 | 53 | loaders, num_classes = data.loaders( 54 | args.dataset, 55 | args.data_path, 56 | args.batch_size, 57 | args.num_workers, 58 | args.transform, 59 | args.use_test, 60 | shuffle_train=False 61 | ) 62 | 63 | architecture = getattr(models, args.model) 64 | base_model = architecture.base(num_classes, **architecture.kwargs) 65 | base_model.cuda() 66 | 67 | 68 | criterion = F.cross_entropy 69 | regularizer = utils.l2_regularizer(args.wd) 70 | 71 | 72 | def get_weights(model): 73 | return np.concatenate([p.data.cpu().numpy().ravel() for p in model.parameters()]) 74 | 75 | T = (args.num_points - 1) * len(args.ckpt - 1) + 1 76 | ts = np.linspace(0.0, len(args.ckpt) - 1, T) 77 | tr_loss = np.zeros(T) 78 | tr_nll = np.zeros(T) 79 | tr_acc = np.zeros(T) 80 | te_loss = np.zeros(T) 81 | te_nll = np.zeros(T) 82 | te_acc = np.zeros(T) 83 | tr_err = np.zeros(T) 84 | te_err = np.zeros(T) 85 | 86 | columns = ['t', 'Train loss', 'Train nll', 'Train error (%)', 'Test nll', 'Test error (%)'] 87 | 88 | alphas = np.linspace(0.0, 1.0, args.num_points) 89 | 90 | for path in args.ckpt: 91 | print(path) 92 | 93 | step = 0 94 | for i in range(len(args.ckpt) - 1): 95 | base_model.load_state_dict(torch.load(args.ckpt[i])['model_state']) 96 | w_1 = get_weights(base_model) 97 | 98 | base_model.load_state_dict(torch.load(args.ckpt[i + 1])['model_state']) 99 | w_2 = get_weights(base_model) 100 | for alpha in alphas[1 if i > 0 else 0:]: 101 | w = (1.0 - alpha) * w_1 + alpha * w_2 102 | offset = 0 103 | for parameter in base_model.parameters(): 104 | size = np.prod(parameter.size()) 105 | value = w[offset:offset+size].reshape(parameter.size()) 106 | parameter.data.copy_(torch.from_numpy(value)) 107 | offset += size 108 | 109 | utils.update_bn(loaders['train'], base_model) 110 | 111 | tr_res = utils.test(loaders['train'], base_model, criterion, regularizer) 112 | te_res = utils.test(loaders['test'], base_model, criterion, regularizer) 113 | 114 | tr_loss[step] = tr_res['loss'] 115 | tr_nll[step] = tr_res['nll'] 116 | tr_acc[step] = tr_res['accuracy'] 117 | tr_err[step] = 100.0 - tr_acc[step] 118 | te_loss[step] = te_res['loss'] 119 | te_nll[step] = te_res['nll'] 120 | te_acc[step] = te_res['accuracy'] 121 | te_err[step] = 100.0 - te_acc[step] 122 | 123 | values = [ts[step], tr_loss[step], tr_nll[step], tr_err[step], te_nll[step], te_err[step]] 124 | table = tabulate.tabulate([values], columns, tablefmt='simple', floatfmt='10.4f') 125 | if step % 40 == 0: 126 | table = table.split('\n') 127 | table = '\n'.join([table[1]] + table) 128 | else: 129 | table = table.split('\n')[2] 130 | print(table) 131 | step += 1 132 | 133 | 134 | np.savez( 135 | os.path.join(args.dir, 'chain.npz'), 136 | ts=ts, 137 | tr_loss=tr_loss, 138 | tr_nll=tr_nll, 139 | tr_acc=tr_acc, 140 | tr_err=tr_err, 141 | te_loss=te_loss, 142 | te_nll=te_nll, 143 | te_acc=te_acc, 144 | te_err=te_err, 145 | ) 146 | -------------------------------------------------------------------------------- /evasion_attack/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .convfc import * 2 | from .vgg import * 3 | from .vggW import * 4 | from .preresnet import * 5 | from .wide_resnet import * 6 | -------------------------------------------------------------------------------- /evasion_attack/models/convfc.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch.nn as nn 3 | 4 | import curves 5 | 6 | __all__ = [ 7 | 'ConvFC', 8 | ] 9 | 10 | 11 | class ConvFCBase(nn.Module): 12 | 13 | def __init__(self, num_classes): 14 | super(ConvFCBase, self).__init__() 15 | self.conv_part = nn.Sequential( 16 | nn.Conv2d(3, 32, kernel_size=5, padding=2), 17 | nn.ReLU(True), 18 | nn.MaxPool2d(kernel_size=3, stride=2), 19 | nn.Conv2d(32, 64, kernel_size=5, padding=2), 20 | nn.ReLU(True), 21 | nn.MaxPool2d(3, 2), 22 | nn.Conv2d(64, 128, kernel_size=5, padding=2), 23 | nn.ReLU(True), 24 | nn.MaxPool2d(3, 2), 25 | ) 26 | self.fc_part = nn.Sequential( 27 | nn.Linear(1152, 1000), 28 | nn.ReLU(True), 29 | nn.Linear(1000, 1000), 30 | nn.ReLU(True), 31 | nn.Linear(1000, num_classes) 32 | ) 33 | 34 | # Initialize weights 35 | for m in self.conv_part.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 38 | m.weight.data.normal_(0, math.sqrt(2. / n)) 39 | m.bias.data.zero_() 40 | 41 | def forward(self, x): 42 | x = self.conv_part(x) 43 | x = x.view(x.size(0), -1) 44 | x = self.fc_part(x) 45 | return x 46 | 47 | 48 | class ConvFCCurve(nn.Module): 49 | def __init__(self, num_classes, fix_points): 50 | super(ConvFCCurve, self).__init__() 51 | self.conv1 = curves.Conv2d(3, 32, kernel_size=5, padding=2, fix_points=fix_points) 52 | self.relu1 = nn.ReLU(True) 53 | self.max_pool1 = nn.MaxPool2d(kernel_size=3, stride=2) 54 | 55 | self.conv2 = curves.Conv2d(32, 64, kernel_size=5, padding=2, fix_points=fix_points) 56 | self.relu2 = nn.ReLU(True) 57 | self.max_pool2 = nn.MaxPool2d(3, 2) 58 | 59 | self.conv3 = curves.Conv2d(64, 128, kernel_size=5, padding=2, fix_points=fix_points) 60 | self.relu3 = nn.ReLU(True) 61 | self.max_pool3 = nn.MaxPool2d(3, 2) 62 | 63 | self.fc4 = curves.Linear(1152, 1000, fix_points=fix_points) 64 | self.relu4 = nn.ReLU(True) 65 | 66 | self.fc5 = curves.Linear(1000, 1000, fix_points=fix_points) 67 | self.relu5 = nn.ReLU(True) 68 | 69 | self.fc6 = curves.Linear(1000, num_classes, fix_points=fix_points) 70 | 71 | # Initialize weights 72 | for m in self.modules(): 73 | if isinstance(m, curves.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | for i in range(m.num_bends): 76 | getattr(m, 'weight_%d' % i).data.normal_(0, math.sqrt(2. / n)) 77 | getattr(m, 'bias_%d' % i).data.zero_() 78 | 79 | def forward(self, x, coeffs_t): 80 | x = self.conv1(x, coeffs_t) 81 | x = self.relu1(x) 82 | x = self.max_pool1(x) 83 | 84 | x = self.conv2(x, coeffs_t) 85 | x = self.relu2(x) 86 | x = self.max_pool2(x) 87 | 88 | x = self.conv3(x, coeffs_t) 89 | x = self.relu3(x) 90 | x = self.max_pool3(x) 91 | 92 | x = x.view(x.size(0), -1) 93 | 94 | x = self.fc4(x, coeffs_t) 95 | x = self.relu4(x) 96 | 97 | x = self.fc5(x, coeffs_t) 98 | x = self.relu5(x) 99 | 100 | x = self.fc6(x, coeffs_t) 101 | 102 | return x 103 | 104 | 105 | class ConvFC: 106 | base = ConvFCBase 107 | curve = ConvFCCurve 108 | kwargs = {} 109 | -------------------------------------------------------------------------------- /evasion_attack/save_model.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | import tabulate 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | import data 9 | import models 10 | import curves 11 | import utils 12 | 13 | parser = argparse.ArgumentParser(description='DNN curve evaluation') 14 | parser.add_argument('--dir', type=str, default='Para128-256_split', metavar='DIR', 15 | help='training directory (default: /tmp/eval)') 16 | 17 | parser.add_argument('--num_points', type=int, default=61, metavar='N', 18 | help='number of points on the curve (default: 61)') 19 | 20 | parser.add_argument('--dataset', type=str, default='CIFAR10', metavar='DATASET', 21 | help='dataset name (default: CIFAR10)') 22 | parser.add_argument('--use_test', action='store_true', default=True, 23 | help='switches between validation and test set (default: validation)') 24 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 25 | help='transform name (default: VGG)') 26 | parser.add_argument('--data_path', type=str, default='Data', metavar='PATH', 27 | help='path to datasets location (default: None)') 28 | 29 | parser.add_argument('--model', type=str, default='VGG16', metavar='MODEL', 30 | help='model name (default: None)') 31 | parser.add_argument('--curve', type=str, default='Bezier', metavar='CURVE', 32 | help='curve type to use (default: None)') 33 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 34 | help='number of curve bends (default: 3)') 35 | 36 | parser.add_argument('--ckpt', type=str, default='Para128-256/checkpoint-180.pt', metavar='CKPT', 37 | help='checkpoint to eval (default: None)') 38 | 39 | args = parser.parse_args() 40 | 41 | os.makedirs(args.dir, exist_ok=True) 42 | 43 | torch.backends.cudnn.benchmark = True 44 | 45 | architecture = getattr(models, args.model) 46 | curve = getattr(curves, args.curve) 47 | model = curves.CurveNet( 48 | 10, 49 | curve, 50 | architecture.curve, 51 | args.num_bends, 52 | architecture_kwargs=architecture.kwargs, 53 | ) 54 | 55 | model.cuda() 56 | checkpoint = torch.load(args.ckpt) 57 | model.load_state_dict(checkpoint['model_state']) 58 | 59 | 60 | spmodel = architecture.base(num_classes=10, **architecture.kwargs) 61 | 62 | parameters = list(model.net.parameters()) 63 | sppara = list(spmodel.parameters()) 64 | #for i in range(0, len(sppara)): 65 | # ttt= i*3 66 | # weights = parameters[ttt:ttt + model.num_bends] 67 | # spweights = sppara[i] 68 | # for j in range(1, model.num_bends - 1): 69 | # alpha = j * 1.0 / (model.num_bends - 1) 70 | # alpha = 0 71 | # spweights.data.copy_(alpha * weights[-1].data + (1.0 - alpha) * weights[0].data) 72 | 73 | ts = np.linspace(0.0, 1.0, 11) 74 | 75 | for kss, t_value in enumerate(ts): 76 | coeffs_t = model.coeff_layer(t_value) 77 | 78 | for i in range(0, len(sppara)): 79 | ttt= i*3 80 | weights = parameters[ttt:ttt + model.num_bends] 81 | spweights = sppara[i] 82 | for j in range(1, model.num_bends - 1): 83 | spweights.data.copy_(coeffs_t[0] * weights[0].data + coeffs_t[1] * weights[1].data + coeffs_t[2] * weights[2].data) 84 | 85 | 86 | print('saving model. %.2f' % t_value) 87 | utils.save_checkpoint( 88 | args.dir, 89 | int(t_value*10), 90 | model_state=spmodel.state_dict(), 91 | ) 92 | -------------------------------------------------------------------------------- /evasion_attack/test_curve.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import torch 3 | 4 | import curves 5 | import data 6 | import models 7 | 8 | 9 | parser = argparse.ArgumentParser(description='Test DNN curve') 10 | 11 | parser.add_argument('--dataset', type=str, default=None, metavar='DATASET', 12 | help='dataset name (default: CIFAR10)') 13 | parser.add_argument('--use_test', action='store_true', 14 | help='switches between validation and test set (default: validation)') 15 | parser.add_argument('--transform', type=str, default='VGG', metavar='TRANSFORM', 16 | help='transform name (default: VGG)') 17 | parser.add_argument('--data_path', type=str, default=None, metavar='PATH', 18 | help='path to datasets location (default: None)') 19 | parser.add_argument('--batch_size', type=int, default=128, metavar='N', 20 | help='input batch size (default: 128)') 21 | parser.add_argument('--num-workers', type=int, default=4, metavar='N', 22 | help='number of workers (default: 4)') 23 | 24 | parser.add_argument('--model', type=str, default=None, metavar='MODEL', required=True, 25 | help='model name (default: None)') 26 | 27 | parser.add_argument('--curve', type=str, default=None, metavar='CURVE', required=True, 28 | help='curve type to use (default: None)') 29 | parser.add_argument('--num_bends', type=int, default=3, metavar='N', 30 | help='number of curve bends (default: 3)') 31 | parser.add_argument('--init_start', type=str, default=None, metavar='CKPT', 32 | help='checkpoint to init start point (default: None)') 33 | parser.add_argument('--init_end', type=str, default=None, metavar='CKPT', 34 | help='checkpoint to init end point (default: None)') 35 | parser.set_defaults(init_linear=True) 36 | parser.add_argument('--init_linear_off', dest='init_linear', action='store_false', 37 | help='turns off linear initialization of intermediate points (default: on)') 38 | 39 | parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') 40 | 41 | args = parser.parse_args() 42 | 43 | torch.backends.cudnn.benchmark = True 44 | torch.manual_seed(args.seed) 45 | torch.cuda.manual_seed(args.seed) 46 | 47 | if args.dataset is not None: 48 | loaders, num_classes = data.loaders( 49 | args.dataset, 50 | args.data_path, 51 | args.batch_size, 52 | args.num_workers, 53 | args.transform, 54 | args.use_test 55 | ) 56 | loader = loaders['test'] 57 | else: 58 | num_classes = 10 59 | loader = [(torch.randn((args.batch_size, 3, 32, 32)), None) for i in range(20)] 60 | 61 | architecture = getattr(models, args.model) 62 | 63 | curve = getattr(curves, args.curve) 64 | curve_model = curves.CurveNet( 65 | num_classes, 66 | curve, 67 | architecture.curve, 68 | args.num_bends, 69 | True, 70 | True, 71 | architecture_kwargs=architecture.kwargs, 72 | ) 73 | 74 | base = [architecture.base(num_classes, **architecture.kwargs) for _ in range(2)] 75 | for base_model, path, k in zip(base, [args.init_start, args.init_end], [0, args.num_bends - 1]): 76 | if path is not None: 77 | checkpoint = torch.load(path) 78 | print('Loading %s as point #%d' % (path, k)) 79 | base_model.load_state_dict(checkpoint['model_state']) 80 | curve_model.import_base_parameters(base_model, k) 81 | 82 | if args.init_linear: 83 | print('Linear initialization.') 84 | curve_model.init_linear() 85 | curve_model.cuda() 86 | for base_model in base: 87 | base_model.cuda() 88 | 89 | t = torch.FloatTensor([0.0]).cuda() 90 | for base_model, t_value in zip(base, [0.0, 1.0]): 91 | print('T: %f' % t_value) 92 | t.data.fill_(t_value) 93 | curve_model.import_base_buffers(base_model) 94 | curve_model.eval() 95 | base_model.eval() 96 | 97 | max_error = 0.0 98 | for i, (input, _) in enumerate(loader): 99 | input = input.cuda(async=True) 100 | 101 | base_ouput = base_model(input) 102 | curve_output = curve_model(input, t) 103 | 104 | error = torch.max(torch.abs(base_ouput - curve_output)).item() 105 | print('Batch #%d. Error: %g' % (i, error)) 106 | max_error = max(max_error, error) 107 | print('Max error: %g' % max_error) 108 | assert max_error < 1e-4, 'Error is too big (%g)' % max_error 109 | --------------------------------------------------------------------------------