├── README.md ├── attack.py ├── main.py ├── net ├── resnet.py └── vgg16.py └── utils ├── FIAloss.py ├── __pycache__ ├── FIAloss.cpython-36.pyc ├── data_loader.cpython-36.pyc └── utils.cpython-36.pyc ├── readData.py └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | # FIA-pytorch 2 | The paper "Feature Importance-aware Transferable Adversarial Attacks" by pytorch. 3 | -------------------------------------------------------------------------------- /attack.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from utils.FIAloss import FIAloss 4 | import numpy as np 5 | # set device 6 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 7 | 8 | 9 | class FIAAttack(object): 10 | def __init__(self, model=None, device=None, epsilon=0.05, k=10, alpha=0.01, prob=0.7, mask_num=100, mu= 0.5): 11 | # set Parameters 12 | self.model = model.to(device) 13 | self.epsilon = epsilon 14 | self.k = k 15 | self.alpha = alpha 16 | self.prob = prob 17 | self.mask_num = mask_num 18 | self.mu = mu 19 | self.device = device 20 | 21 | 22 | def perturb(self, X_nat): 23 | # get grads 24 | _,temp = self.model.features_grad(X_nat) 25 | batch_size = X_nat.shape[0] 26 | image_size = X_nat.shape[-1] 27 | grad_sum = torch.zeros((temp.shape)).to(device) 28 | for i in range(self.mask_num): 29 | self.model.zero_grad() 30 | img_temp_i = X_nat.clone() 31 | # get mask 32 | mask = torch.tensor(np.random.binomial(1, self.prob, size=(batch_size,3,image_size,image_size))).to(device) 33 | img_temp_i = img_temp_i * mask 34 | out,y = self.model.features_grad(img_temp_i) 35 | # out.backward(torch.ones_like(out)) 36 | # grad_temp = y.grad 37 | grad_temp = torch.autograd.grad(out, y, grad_outputs=torch.ones_like(out))[0] 38 | grad_sum += grad_temp 39 | # avr 40 | grad_sum = grad_sum / self.mask_num 41 | 42 | g = 0 43 | eta = 0 44 | x_cle = X_nat.detach() 45 | x_adv = X_nat.clone().requires_grad_() 46 | for epoch in range(self.k): 47 | x_adv.requires_grad = True 48 | self.model.zero_grad() 49 | mid_feature = self.model.layer2_features(x_adv) 50 | loss = FIAloss(grad_sum, mid_feature) # FIA loss 51 | loss.backward() 52 | g = self.mu*g + x_adv.grad 53 | x_adv = x_adv - self.alpha*g.sign() 54 | with torch.no_grad(): 55 | eta = torch.clamp(x_adv - x_cle, min=-self.epsilon, max=self.epsilon) 56 | X = torch.clamp(x_cle + eta, min=-1, max=1).detach_() 57 | x_adv = torch.clamp(x_cle+eta, min=-1, max=1).detach_() 58 | return X 59 | 60 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from utils.readData import read_dataset 3 | from torchvision.utils import save_image 4 | from utils.utils import denorm 5 | from net.resnet import ResNet18 6 | from attack import FIAAttack 7 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 8 | 9 | batch_size = 1 10 | train_loader,valid_loader,test_loader = read_dataset(batch_size=batch_size,pic_path='dataset') 11 | 12 | # load model 13 | model = ResNet18() 14 | model.load_state_dict(torch.load('checkpoint/resnet18_cifar10.pt')) 15 | model = model.to(device) 16 | model.eval() 17 | 18 | 19 | for n,(data, target) in enumerate(test_loader): 20 | data = data.to(device) 21 | target = target.to(device) 22 | attack = FIAAttack(model=model, device=device) 23 | X = attack.perturb(data) 24 | 25 | # save adv img 26 | save_image(denorm(X[0].data.cpu()),"save/adv_img/"+str(n)+".png") -------------------------------------------------------------------------------- /net/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): 5 | """3x3 convolution with padding""" 6 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 7 | padding=dilation, groups=groups, bias=False, dilation=dilation) 8 | 9 | 10 | def conv1x1(in_planes, out_planes, stride=1): 11 | """1x1 convolution""" 12 | return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) 13 | 14 | 15 | class BasicBlock(nn.Module): 16 | expansion = 1 17 | 18 | def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, 19 | base_width=64, dilation=1, norm_layer=None): 20 | super(BasicBlock, self).__init__() 21 | if norm_layer is None: 22 | norm_layer = nn.BatchNorm2d 23 | if groups != 1 or base_width != 64: 24 | raise ValueError('BasicBlock only supports groups=1 and base_width=64') 25 | if dilation > 1: 26 | raise NotImplementedError("Dilation > 1 not supported in BasicBlock") 27 | # Both self.conv1 and self.downsample layers downsample the input when stride != 1 28 | self.conv1 = conv3x3(inplanes, planes, stride) 29 | self.bn1 = norm_layer(planes) 30 | self.relu = nn.ReLU(inplace=True) 31 | self.conv2 = conv3x3(planes, planes) 32 | self.bn2 = norm_layer(planes) 33 | self.downsample = downsample 34 | self.stride = stride 35 | 36 | def forward(self, x): 37 | identity = x 38 | 39 | out = self.conv1(x) 40 | out = self.bn1(out) 41 | out = self.relu(out) 42 | 43 | out = self.conv2(out) 44 | out = self.bn2(out) 45 | 46 | if self.downsample is not None: 47 | identity = self.downsample(x) 48 | 49 | out += identity 50 | out = self.relu(out) 51 | 52 | return out 53 | 54 | 55 | class Bottleneck(nn.Module): 56 | # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) 57 | # while original implementation places the stride at the first 1x1 convolution(self.conv1) 58 | # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. 59 | # This variant is also known as ResNet V1.5 and improves accuracy according to 60 | # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. 61 | 62 | expansion = 4 63 | 64 | def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, 65 | base_width=64, dilation=1, norm_layer=None): 66 | super(Bottleneck, self).__init__() 67 | if norm_layer is None: 68 | norm_layer = nn.BatchNorm2d 69 | width = int(planes * (base_width / 64.)) * groups 70 | # Both self.conv2 and self.downsample layers downsample the input when stride != 1 71 | self.conv1 = conv1x1(inplanes, width) 72 | self.bn1 = norm_layer(width) 73 | self.conv2 = conv3x3(width, width, stride, groups, dilation) 74 | self.bn2 = norm_layer(width) 75 | self.conv3 = conv1x1(width, planes * self.expansion) 76 | self.bn3 = norm_layer(planes * self.expansion) 77 | self.relu = nn.ReLU(inplace=True) 78 | self.downsample = downsample 79 | self.stride = stride 80 | 81 | def forward(self, x): 82 | identity = x 83 | 84 | out = self.conv1(x) 85 | out = self.bn1(out) 86 | out = self.relu(out) 87 | 88 | out = self.conv2(out) 89 | out = self.bn2(out) 90 | out = self.relu(out) 91 | 92 | out = self.conv3(out) 93 | out = self.bn3(out) 94 | 95 | if self.downsample is not None: 96 | identity = self.downsample(x) 97 | 98 | out += identity 99 | out = self.relu(out) 100 | 101 | return out 102 | 103 | 104 | class ResNet(nn.Module): 105 | 106 | def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, 107 | groups=1, width_per_group=64, replace_stride_with_dilation=None, 108 | norm_layer=None): 109 | super(ResNet, self).__init__() 110 | if norm_layer is None: 111 | norm_layer = nn.BatchNorm2d 112 | self._norm_layer = norm_layer 113 | 114 | self.inplanes = 64 115 | self.dilation = 1 116 | if replace_stride_with_dilation is None: 117 | # each element in the tuple indicates if we should replace 118 | # the 2x2 stride with a dilated convolution instead 119 | replace_stride_with_dilation = [False, False, False] 120 | if len(replace_stride_with_dilation) != 3: 121 | raise ValueError("replace_stride_with_dilation should be None " 122 | "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) 123 | self.groups = groups 124 | self.base_width = width_per_group 125 | self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, 126 | bias=False) 127 | self.bn1 = norm_layer(self.inplanes) 128 | self.relu = nn.ReLU(inplace=True) 129 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 130 | self.layer1 = self._make_layer(block, 64, layers[0]) 131 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2, 132 | dilate=replace_stride_with_dilation[0]) 133 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2, 134 | dilate=replace_stride_with_dilation[1]) 135 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2, 136 | dilate=replace_stride_with_dilation[2]) 137 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 138 | self.fc = nn.Linear(512 * block.expansion, num_classes) 139 | 140 | for m in self.modules(): 141 | if isinstance(m, nn.Conv2d): 142 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 143 | elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): 144 | nn.init.constant_(m.weight, 1) 145 | nn.init.constant_(m.bias, 0) 146 | 147 | # Zero-initialize the last BN in each residual branch, 148 | # so that the residual branch starts with zeros, and each residual block behaves like an identity. 149 | # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 150 | if zero_init_residual: 151 | for m in self.modules(): 152 | if isinstance(m, Bottleneck): 153 | nn.init.constant_(m.bn3.weight, 0) 154 | elif isinstance(m, BasicBlock): 155 | nn.init.constant_(m.bn2.weight, 0) 156 | 157 | def _make_layer(self, block, planes, blocks, stride=1, dilate=False): 158 | norm_layer = self._norm_layer 159 | downsample = None 160 | previous_dilation = self.dilation 161 | if dilate: 162 | self.dilation *= stride 163 | stride = 1 164 | if stride != 1 or self.inplanes != planes * block.expansion: 165 | downsample = nn.Sequential( 166 | conv1x1(self.inplanes, planes * block.expansion, stride), 167 | norm_layer(planes * block.expansion), 168 | ) 169 | 170 | layers = [] 171 | layers.append(block(self.inplanes, planes, stride, downsample, self.groups, 172 | self.base_width, previous_dilation, norm_layer)) 173 | self.inplanes = planes * block.expansion 174 | for _ in range(1, blocks): 175 | layers.append(block(self.inplanes, planes, groups=self.groups, 176 | base_width=self.base_width, dilation=self.dilation, 177 | norm_layer=norm_layer)) 178 | 179 | return nn.Sequential(*layers) 180 | 181 | def _forward_impl(self, x): 182 | # See note [TorchScript super()] 183 | x = self.conv1(x) 184 | x = self.bn1(x) 185 | x = self.relu(x) 186 | x = self.maxpool(x) 187 | 188 | x = self.layer1(x) 189 | x = self.layer2(x) 190 | x = self.layer3(x) 191 | x = self.layer4(x) 192 | 193 | x = self.avgpool(x) 194 | x = torch.flatten(x, 1) 195 | x = self.fc(x) 196 | 197 | return x 198 | 199 | def forward(self, x): 200 | return self._forward_impl(x) 201 | 202 | def features_grad(self,x): 203 | x = self.conv1(x) 204 | x = self.bn1(x) 205 | x = self.relu(x) 206 | x = self.maxpool(x) 207 | x = self.layer1(x) 208 | y = self.layer2(x) 209 | y.retain_grad() 210 | z = self.layer3(y) 211 | z = self.layer4(z) 212 | z = self.avgpool(z) 213 | z = torch.flatten(z, 1) 214 | return z,y 215 | 216 | def layer2_features(self,x): 217 | y = x 218 | y.retain_grad() 219 | z = self.conv1(y) 220 | z = self.bn1(z) 221 | z = self.relu(z) 222 | z = self.maxpool(z) 223 | z = self.layer1(z) 224 | z = self.layer2(z) 225 | return z 226 | 227 | 228 | def _resnet(block, layers, **kwargs): 229 | model = ResNet(block, layers, **kwargs) 230 | return model 231 | 232 | 233 | def ResNet18(**kwargs): 234 | return _resnet(BasicBlock, [2, 2, 2, 2],**kwargs) 235 | 236 | 237 | def ResNet34(**kwargs): 238 | return _resnet(BasicBlock, [3, 4, 6, 3],**kwargs) 239 | 240 | 241 | def ResNet50(**kwargs): 242 | return _resnet(Bottleneck, [3, 4, 6, 3],**kwargs) 243 | 244 | 245 | def ResNet101(**kwargs): 246 | return _resnet(Bottleneck, [3, 4, 23, 3],**kwargs) 247 | 248 | 249 | def ResNet152(**kwargs): 250 | return _resnet(Bottleneck, [3, 8, 36, 3],**kwargs) -------------------------------------------------------------------------------- /net/vgg16.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.utils.model_zoo 4 | import torch.autograd as autograd 5 | 6 | 7 | 8 | def vgg16(**kwargs): 9 | """ 10 | VGGFace model. 11 | """ 12 | model = Vgg16(**kwargs) 13 | return model 14 | 15 | 16 | class Vgg16(torch.nn.Module): 17 | def __init__(self, classes=2622): 18 | """VGGFace model. 19 | 20 | Face recognition network. It takes as input a Bx3x224x224 21 | batch of face images and gives as output a BxC score vector 22 | (C is the number of identities). 23 | Input images need to be scaled in the 0-1 range and then 24 | normalized with respect to the mean RGB used during training. 25 | 26 | Args: 27 | classes (int): number of identities recognized by the 28 | network 29 | 30 | """ 31 | super().__init__() 32 | self.conv1 = _ConvBlock(3, 64, 64) 33 | self.conv2 = _ConvBlock(64, 128, 128) 34 | self.conv3 = _ConvBlock(128, 256, 256, 256) 35 | self.conv4 = _ConvBlock(256, 512, 512, 512) 36 | self.conv5 = _ConvBlock(512, 512, 512, 512) 37 | self.dropout = torch.nn.Dropout(0.5) 38 | self.fc1 = torch.nn.Linear(7 * 7 * 512, 4096) 39 | self.fc2 = torch.nn.Linear(4096, 4096) 40 | self.fc3 = torch.nn.Linear(4096, classes) 41 | 42 | def forward(self, x): 43 | x = self.conv1(x) 44 | x = self.conv2(x) 45 | x = self.conv3(x) 46 | x = self.conv4(x) 47 | x = self.conv5(x) 48 | x = x.view(x.size(0), -1) 49 | x = self.dropout(F.relu(self.fc1(x))) 50 | x = self.dropout(F.relu(self.fc2(x))) 51 | x = self.fc3(x) 52 | return x 53 | 54 | def conv3_features(self,x): 55 | y = x 56 | y.retain_grad() 57 | z = self.conv1(y) 58 | z = self.conv2(z) 59 | z = self.conv3(z) 60 | return z 61 | 62 | def conv5_features(self,x): 63 | y = x 64 | #y.retain_grad() 65 | z = self.conv1(y) 66 | z = self.conv2(z) 67 | z = self.conv3(z) 68 | z = self.conv4(z) 69 | z = self.conv5(z) 70 | return z 71 | 72 | # 求解特征输出到conv3的梯度 73 | def features_grad(self,x): 74 | x = self.conv1(x) 75 | x = self.conv2(x) 76 | y = self.conv3(x) 77 | y.retain_grad() 78 | z = self.conv4(y) 79 | z = self.conv5(z) 80 | #z.backward(torch.ones_like(z)) 81 | #y_grad = autograd.grad(z.sum(), y, retain_graph=True)[0] 82 | return z,y 83 | 84 | 85 | class _ConvBlock(torch.nn.Module): 86 | """A Convolutional block.""" 87 | 88 | def __init__(self, *units): 89 | """Create a block with len(units) - 1 convolutions. 90 | 91 | convolution number i transforms the number of channels from 92 | units[i - 1] to units[i] channels. 93 | 94 | """ 95 | super().__init__() 96 | self.convs = torch.nn.ModuleList([ 97 | torch.nn.Conv2d(in_, out, 3, 1, 1) 98 | for in_, out in zip(units[:-1], units[1:]) 99 | ]) 100 | 101 | def forward(self, x): 102 | # Each convolution is followed by a ReLU, then the block is 103 | # concluded by a max pooling. 104 | for c in self.convs: 105 | x = F.relu(c(x)) 106 | return F.max_pool2d(x, 2, 2, 0, ceil_mode=True) -------------------------------------------------------------------------------- /utils/FIAloss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | from utils.data_loader import get_loader 4 | from utils.utils import show_feature_map 5 | from net.vggface import vggface 6 | import cv2 7 | import numpy as np 8 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 9 | 10 | def FIAloss(grad, feature): 11 | # zeros = torch.zeros(feature.shape[1:], dtype=torch.float32).to(device) 12 | # loss1 = torch.sum(torch.maximum(grad * feature,zeros)) 13 | # loss2 = torch.sum(torch.minimum(grad * feature,zeros)) 14 | # Loss = torch.abs(loss1 - loss2) 15 | Loss = torch.sum(torch.abs(grad * feature)) 16 | #Loss = torch.sum(grad * feature) 17 | return Loss 18 | -------------------------------------------------------------------------------- /utils/__pycache__/FIAloss.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZOMIN28/FIA-pytorch/4ed11b4caa44c86dd51d76efbadbe92a114e04ef/utils/__pycache__/FIAloss.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/data_loader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZOMIN28/FIA-pytorch/4ed11b4caa44c86dd51d76efbadbe92a114e04ef/utils/__pycache__/data_loader.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZOMIN28/FIA-pytorch/4ed11b4caa44c86dd51d76efbadbe92a114e04ef/utils/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /utils/readData.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from torchvision import datasets 4 | import torchvision.transforms as transforms 5 | from torch.utils.data.sampler import SubsetRandomSampler 6 | 7 | # set device 8 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 9 | # number of subprocesses to use for data loading 10 | num_workers = 0 11 | # 每批加载图数量 12 | batch_size = 16 13 | # percentage of training set to use as validation 14 | valid_size = 0.2 15 | 16 | def read_dataset(batch_size=16,valid_size=0.2,num_workers=0,pic_path='dataset'): 17 | """ 18 | batch_size: Number of loaded drawings per batch 19 | valid_size: Percentage of training set to use as validation 20 | num_workers: Number of subprocesses to use for data loading 21 | pic_path: The path of the pictrues 22 | """ 23 | transform_train = transforms.Compose([ 24 | transforms.RandomCrop(32, padding=4), #先四周填充0，在吧图像随机裁剪成32*32 25 | transforms.RandomHorizontalFlip(), #图像一半的概率翻转，一半的概率不翻转 26 | transforms.ToTensor(), 27 | transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]), #R,G,B每层的归一化用到的均值和方差 28 | ]) 29 | 30 | transform_test = transforms.Compose([ 31 | transforms.ToTensor(), 32 | transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]), 33 | ]) 34 | 35 | 36 | # 将数据转换为torch.FloatTensor，并标准化。 37 | train_data = datasets.CIFAR10(pic_path, train=True, 38 | download=True, transform=transform_train) 39 | valid_data = datasets.CIFAR10(pic_path, train=True, 40 | download=True, transform=transform_test) 41 | test_data = datasets.CIFAR10(pic_path, train=False, 42 | download=True, transform=transform_test) 43 | 44 | 45 | # obtain training indices that will be used for validation 46 | num_train = len(train_data) 47 | indices = list(range(num_train)) 48 | # random indices 49 | np.random.shuffle(indices) 50 | # the ratio of split 51 | split = int(np.floor(valid_size * num_train)) 52 | # divide data to radin_data and valid_data 53 | train_idx, valid_idx = indices[split:], indices[:split] 54 | 55 | # define samplers for obtaining training and validation batches 56 | # 无放回地按照给定的索引列表采样样本元素 57 | train_sampler = SubsetRandomSampler(train_idx) 58 | valid_sampler = SubsetRandomSampler(valid_idx) 59 | 60 | # prepare data loaders (combine dataset and sampler) 61 | train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, 62 | sampler=train_sampler, num_workers=num_workers) 63 | valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, 64 | sampler=valid_sampler, num_workers=num_workers) 65 | test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, 66 | num_workers=num_workers) 67 | 68 | return train_loader,valid_loader,test_loader -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | def denorm(x): 2 | """Convert the range from [-1, 1] to [0, 1].""" 3 | out = (x + 1) / 2 4 | return out.clamp_(0, 1) --------------------------------------------------------------------------------