├── images ├── Algo.png ├── Fig1.png ├── Table1.png ├── Table2.png ├── Table3.png ├── Table4.png ├── Table5.png └── concept_figure.png ├── requirements.txt ├── our_dataset.py ├── surrogate.py ├── utils.py ├── classification.py ├── train_cd.py ├── README.md ├── attack.py ├── train_id.py └── imagenet_class_index.json /images/Algo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Algo.png -------------------------------------------------------------------------------- /images/Fig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Fig1.png -------------------------------------------------------------------------------- /images/Table1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Table1.png -------------------------------------------------------------------------------- /images/Table2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Table2.png -------------------------------------------------------------------------------- /images/Table3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Table3.png -------------------------------------------------------------------------------- /images/Table4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Table4.png -------------------------------------------------------------------------------- /images/Table5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/Table5.png -------------------------------------------------------------------------------- /images/concept_figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HashmatShadab/APR/HEAD/images/concept_figure.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.5.2 2 | numpy==1.22.3 3 | Pillow==9.2.0 4 | pretrainedmodels==0.7.4 5 | scipy==1.8.1 6 | timm==0.5.4 7 | torch==1.12.0 8 | torchattacks==3.2.6 9 | torchvision==0.13.0 10 | -------------------------------------------------------------------------------- /our_dataset.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | import os 4 | import glob 5 | 6 | import numpy as np 7 | import torch 8 | import torchvision 9 | from PIL import Image 10 | from torch.utils.data import Dataset 11 | from torchvision import datasets 12 | 13 | class_idx = json.load(open("imagenet_class_index.json")) 14 | idx2label = [class_idx[str(k)][1] for k in range(len(class_idx))] 15 | class2label = [class_idx[str(k)][0] for k in range(len(class_idx))] 16 | 17 | class_label = {} 18 | label = 0 19 | for name in class2label: 20 | class_label[name]=label 21 | label+=1 22 | 23 | class OUR_dataset(Dataset): 24 | """ 25 | For a given dataset (data_dir) and a corresponding csv file (data_csv_dir) with rows as {class name, image names}, 26 | this class create image directory list of the form ['data/path/to/image',label_index] for the first n_Imgs (10) from each class. 27 | """ 28 | 29 | def __init__(self, data_dir:str, data_csv_dir:str, mode:str, img_num:tuple or int, transform): 30 | assert mode in ['train', 'attack', 'all'], 'WRONG DATASET MODE' 31 | assert img_num in [1,5,10,20], 'ONLY SUPPORT 2/10/20/40 IMAGES' 32 | super(OUR_dataset).__init__() 33 | self.mode = mode 34 | self.data_dir = data_dir 35 | data_csv = open(data_csv_dir, 'r') 36 | csvreader = csv.reader(data_csv) 37 | data_ls = list(csvreader) 38 | self.imgs = self.prep_imgs_dir(data_ls, img_num) 39 | self.transform = transform 40 | 41 | 42 | def prep_imgs_dir(self, data_ls, nImg): 43 | 44 | imgs_ls = [] 45 | if self.mode in ['train', 'attack']: 46 | if nImg>=10: 47 | sel_ls = list(range(nImg)) #sel_ls=0,...,9 48 | imgs_ls += self.mk_img_ls(data_ls, sel_ls) 49 | elif nImg == 1: 50 | for jkl in list(range(10)): 51 | imgs_ls += self.mk_img_ls(data_ls, [jkl]) 52 | elif nImg == 5: 53 | sel_ls_1 = list(range(5)) 54 | sel_ls_2 = list(range(5,10)) 55 | imgs_ls += self.mk_img_ls(data_ls, sel_ls_1) 56 | imgs_ls += self.mk_img_ls(data_ls, sel_ls_2) 57 | elif self.mode == 'all': 58 | sel_ls = list(range(50)) 59 | imgs_ls += self.mk_img_ls(data_ls, sel_ls) 60 | return imgs_ls 61 | 62 | 63 | def mk_img_ls(self, data_ls, sel_ls): 64 | 65 | """ 66 | Returns the list (imgs_ls) with ['data/path/to/image',label_index] for the first n_Imgs from each row of data_ls 67 | :param sel_ls: 0,...,(n_Img-1) 68 | """ 69 | 70 | imgs_ls = [] 71 | for label_ind in range(len(data_ls)): 72 | for img_ind in sel_ls: 73 | imgs_ls.append([self.data_dir + '/' + data_ls[label_ind][0] + '/' + data_ls[label_ind][1 + img_ind], class_label[data_ls[label_ind][0]]]) 74 | return imgs_ls 75 | 76 | 77 | def __getitem__(self, item): 78 | img = Image.open(self.imgs[item][0]) 79 | if img.mode != 'RGB': 80 | img = img.convert('RGB') 81 | return self.transform(img), self.imgs[item][1] 82 | 83 | def __len__(self): 84 | return len(self.imgs) 85 | 86 | 87 | if __name__ == '__main__': 88 | #One way to prepare 'data/selected_data.csv' 89 | selected_data_csv = open('data/selected_data_ordered.csv', 'w') 90 | data_writer = csv.writer(selected_data_csv) 91 | dataset_dir = 'data/ILSVRC2012_img_val' 92 | dataset = torchvision.datasets.ImageFolder(dataset_dir) 93 | label_ind = torch.randperm(500).numpy() # random_ordering : not useful 94 | label_ind = [i for i in range(500)] 95 | selected_labels_ls = np.array(dataset.classes)[label_ind] 96 | for label_name in selected_labels_ls: 97 | data_writer.writerow([label_name]+os.listdir(os.path.join(dataset_dir, label_name))) 98 | -------------------------------------------------------------------------------- /surrogate.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | import torch.nn as nn 4 | 5 | 6 | class ResnetBlock(nn.Module): 7 | 8 | def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias): 9 | super(ResnetBlock, self).__init__() 10 | self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias) 11 | 12 | def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias): 13 | conv_block = [] 14 | p = 0 15 | if padding_type == 'reflect': 16 | conv_block += [nn.ReflectionPad2d(1)] 17 | elif padding_type == 'replicate': 18 | conv_block += [nn.ReplicationPad2d(1)] 19 | elif padding_type == 'zero': 20 | p = 1 21 | else: 22 | raise NotImplementedError('padding [%s] is not implemented' % padding_type) 23 | 24 | conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)] 25 | if use_dropout: 26 | conv_block += [nn.Dropout(0.5)] 27 | 28 | p = 0 29 | if padding_type == 'reflect': 30 | conv_block += [nn.ReflectionPad2d(1)] 31 | elif padding_type == 'replicate': 32 | conv_block += [nn.ReplicationPad2d(1)] 33 | elif padding_type == 'zero': 34 | p = 1 35 | else: 36 | raise NotImplementedError('padding [%s] is not implemented' % padding_type) 37 | conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)] 38 | 39 | return nn.Sequential(*conv_block) 40 | 41 | def forward(self, x): 42 | """Forward function (with skip connections)""" 43 | out = x + self.conv_block(x) # add skip connections 44 | return out 45 | 46 | 47 | 48 | class autoencoder(nn.Module): 49 | 50 | def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, 51 | n_blocks=6, padding_type='reflect', decoder_num = 2, decoder_out_ind = 100, feat=[6, 9, 10, 12], ensemble=False): 52 | super(autoencoder, self).__init__() 53 | self.ensemble = ensemble 54 | assert(n_blocks >= 0) 55 | if type(norm_layer) == functools.partial: 56 | use_bias = norm_layer.func == nn.InstanceNorm2d 57 | else: 58 | use_bias = norm_layer == nn.InstanceNorm2d 59 | 60 | model = [nn.ReflectionPad2d(3), 61 | nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias), 62 | norm_layer(ngf), 63 | nn.ReLU(True)] 64 | 65 | n_downsampling = 2 66 | for i in range(n_downsampling): # add downsampling layers 67 | mult = 2 ** i 68 | model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias), 69 | norm_layer(ngf * mult * 2), 70 | nn.ReLU(True)] 71 | 72 | mult = 2 ** n_downsampling 73 | for i in range(n_blocks): # add ResNet blocks 74 | 75 | model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)] 76 | 77 | self.feat = feat 78 | self.decoder_num = decoder_num 79 | self.encoder = nn.Sequential(*model) 80 | self.decoder_out_ind = decoder_out_ind # ???? 81 | if decoder_num == 1: 82 | mk_decoder = self.mk_decoder_224 83 | else: 84 | mk_decoder = self.mk_decoder 85 | for decoder_ind in range(decoder_num): 86 | setattr(self, 'decoder_{}'.format(decoder_ind), mk_decoder(n_downsampling, norm_layer, ngf, use_bias, output_nc)) 87 | 88 | def mk_decoder(self, n_downsampling, norm_layer, ngf, use_bias, output_nc): 89 | model = [] 90 | model += [ 91 | nn.Conv2d(256, 128, 3, 1, 1), 92 | norm_layer(128), 93 | nn.ReLU(True)] 94 | model += [nn.Conv2d(128, output_nc, kernel_size=3, padding=1)] 95 | model += [nn.Sigmoid()] 96 | model = nn.Sequential(*model) 97 | return model 98 | 99 | def mk_decoder_224(self, n_downsampling, norm_layer, ngf, use_bias, output_nc): 100 | model = [] 101 | for i in range(n_downsampling): # add upsampling layers 102 | mult = 2 ** (n_downsampling - i) 103 | model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), 104 | kernel_size=3, stride=2, 105 | padding=1, output_padding=1, 106 | bias=use_bias), 107 | norm_layer(int(ngf * mult / 2)), 108 | nn.ReLU(True)] 109 | model += [nn.Conv2d(ngf, output_nc, kernel_size=3, padding=1)] 110 | model += [nn.Sigmoid()] 111 | model = nn.Sequential(*model) 112 | return model 113 | 114 | def decoder_forw(self, x, decoder_ind): 115 | for ind, mm in enumerate(getattr(self, 'decoder_{}'.format(decoder_ind))): 116 | x = mm(x) 117 | if ind == self.decoder_out_ind: 118 | return x 119 | return x 120 | 121 | def forward(self, input): 122 | """Standard forward""" 123 | x = input 124 | features = [] 125 | for ind, mm in enumerate(self.encoder): 126 | x = mm(x) 127 | if ind in self.feat: 128 | features.append(x) 129 | 130 | outs = [] 131 | y = x.clone() 132 | 133 | if self.ensemble: 134 | for decoder_ind in range(self.decoder_num): 135 | for feat in features: 136 | outs.append(self.decoder_forw(feat, decoder_ind)) 137 | return outs, None, None 138 | 139 | else: 140 | for decoder_ind in range(self.decoder_num): 141 | outs.append(self.decoder_forw(x, decoder_ind)) 142 | return outs, y, features 143 | 144 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from torchattacks.attack import Attack 5 | 6 | 7 | # Normalization 8 | class Normalize(nn.Module): 9 | def __init__(self, ms=None): 10 | super(Normalize, self).__init__() 11 | if ms == None: 12 | self.ms = [(0.485, 0.456, 0.406), (0.229, 0.224, 0.225)] 13 | 14 | def forward(self, input): 15 | x = input.clone() 16 | for i in range(x.shape[1]): 17 | x[:, i] = (x[:, i] - self.ms[0][i]) / self.ms[1][i] 18 | return x 19 | 20 | 21 | 22 | def rot(img): 23 | """ 24 | To rotate the image randomly in one of the four angles (0,90,180 & 270) 25 | :return: returns rotated img. 26 | """ 27 | rand_angle = torch.randint(0, 4, size=(1,)).item() # 0,1,2,3 28 | assert rand_angle in [0, 1, 2, 3], 'check rand_angle' 29 | if rand_angle == 0: 30 | img = img 31 | elif rand_angle == 1: 32 | img = torch.flip(img, dims=[3]).permute(0, 1, 3, 2) 33 | elif rand_angle == 2: 34 | img = torch.flip(img, dims=[2]) 35 | img = torch.flip(img, dims=[3]) 36 | elif rand_angle == 3: 37 | img = torch.flip(img.permute(0, 1, 3, 2), dims=[3]) 38 | return img 39 | 40 | 41 | 42 | def horizontal_flip(img): 43 | rand_flip = torch.randint(0, 2, size=(1,)).item() # 0,1 44 | assert rand_flip in [0, 1], 'check rand_flip' 45 | img = torch.flip(img, dims=[3]) 46 | return img 47 | 48 | 49 | def shuffle(img, mode): 50 | """ 51 | To create 4 tile jigsaw patches of the given image 52 | :param mode: Shuffle mode (1) 53 | :return: returns shuffled img. 54 | """ 55 | assert mode in [0, 1], 'check shuffle mode' 56 | if mode == 0: 57 | patch_0 = img[:, 0:112, 0:112] 58 | patch_1 = img[:, 0:112, 112:224] 59 | patch_2 = img[:, 112:224, 0:112] 60 | patch_3 = img[:, 112:224, 112:224] 61 | rand_ind = torch.randperm(4) 62 | img_0 = torch.cat((eval('patch_{}'.format(rand_ind[0])), 63 | eval('patch_{}'.format(rand_ind[1]))), dim=2) 64 | img_1 = torch.cat((eval('patch_{}'.format(rand_ind[2])), 65 | eval('patch_{}'.format(rand_ind[3]))), dim=2) 66 | return torch.cat((img_0, img_1), dim=1) 67 | else: 68 | # four possibilities, for easy training 69 | img = img.permute(1, 2, 0) 70 | img = img.reshape(2, 112, 224, 3) 71 | rand_shuffle_1 = torch.randint(0, 2, size=(1,)).item() 72 | img = img[[rand_shuffle_1, 1 - rand_shuffle_1]] 73 | img = img.reshape(224, 224, 3) 74 | img = img.permute(1, 0, 2) 75 | img = img.reshape(2, 112, 224, 3) 76 | rand_shuffle_2 = torch.randint(0, 2, size=(1,)).item() 77 | img = img[[rand_shuffle_2, 1 - rand_shuffle_2]] 78 | img = img.reshape(224, 224, 3) 79 | return img.permute(2, 1, 0) 80 | 81 | 82 | def aug(img_input): 83 | for img_ind in range(img_input.shape[0]): 84 | img_input[img_ind:img_ind + 1] = horizontal_flip(img_input[img_ind:img_ind + 1]) 85 | return img_input 86 | 87 | 88 | def mk_proto_ls(n_imgs): 89 | """ 90 | To create list of prototypes for prototypical reconstruction approach 91 | :param n_imgs: Number of reference images (10) 92 | :return: returns a list of 100 pairs in which the first 10 pairs are 93 | [0,10],[1,11],...,[9,19] and the rest are random. 94 | """ 95 | tar_ind_ls = torch.tensor(list(range(int(2 * n_imgs)))).reshape((2, n_imgs)).permute((1, 0)).reshape(-1) 96 | # [0, 10, 1, 11, .........., 8, 18, 9, 19] 97 | tar_ind_ls_ex = [] 98 | for i_f in list(range(n_imgs)): 99 | for i_s in list(range(n_imgs, n_imgs * 2)): 100 | if i_f != i_s - n_imgs: 101 | tar_ind_ls_ex.append([i_f, i_s]) 102 | # [[0, 11], [0, 12], [0, 13],.......[0,19], 103 | # [1, 10], [1, 12], [1, 13],.......[1,19], 104 | # . 105 | # . 106 | # [9, 10], [9, 11], [9, 12],.......[9, 18]] shape > (90, 2) 107 | 108 | # randomly order the 90 pairs and then put them in single array 109 | tar_ind_ls_ex = torch.tensor(tar_ind_ls_ex)[torch.randperm(len(tar_ind_ls_ex))].reshape(-1) 110 | # add [0, 10, 1, 11, .........., 8, 18, 9, 19] back to the randomly ordered 90 pairs 111 | tar_ind_ls = torch.cat((tar_ind_ls, tar_ind_ls_ex)) 112 | return tar_ind_ls 113 | 114 | 115 | 116 | class FGSM(Attack): 117 | r""" 118 | FGSM in the paper 'Explaining and harnessing adversarial examples' 119 | [https://arxiv.org/abs/1412.6572] 120 | 121 | Distance Measure : Linf 122 | 123 | Arguments: 124 | model (nn.Module): model to attack. 125 | eps (float): maximum perturbation. (Default: 0.007) 126 | 127 | Shape: 128 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 129 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 130 | - output: :math:`(N, C, H, W)`. 131 | 132 | Examples:: 133 | >>> attack = torchattacks.FGSM(model, eps=0.007) 134 | >>> adv_images = attack(images, labels) 135 | 136 | """ 137 | def __init__(self, model, eps=0.007, mode="rotate"): 138 | super().__init__("FGSM", model) 139 | self.mode = mode 140 | self.eps = eps 141 | self._supported_mode = ['default', 'targeted'] 142 | 143 | def forward(self, images, target_images): 144 | r""" 145 | Overridden. 146 | """ 147 | images = images.clone().detach().to(self.device) 148 | target_images = target_images.clone().detach().to(self.device) 149 | 150 | 151 | loss = nn.MSELoss() 152 | 153 | images.requires_grad = True 154 | outputs, _, _ = self.model(images) 155 | 156 | # Calculate loss 157 | if self.mode == "rotate": 158 | cost = loss(outputs[0], target_images) 159 | else: 160 | gen_img = torch.cat(outputs, dim=0) 161 | cost = loss(gen_img, target_images) 162 | 163 | # Update adversarial images 164 | grad = torch.autograd.grad(cost, images, 165 | retain_graph=False, create_graph=False)[0] 166 | 167 | adv_images = images + self.eps*grad.sign() 168 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 169 | 170 | return adv_images 171 | -------------------------------------------------------------------------------- /classification.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import os 4 | 5 | import pretrainedmodels 6 | import timm 7 | import torch 8 | import torch.nn as nn 9 | import torchvision 10 | import torchvision.transforms as transforms 11 | from PIL import ImageFile 12 | from timm.data import resolve_data_config 13 | from timm.data.transforms_factory import create_transform 14 | 15 | ImageFile.LOAD_TRUNCATED_IMAGES = True 16 | 17 | 18 | class Normalize(nn.Module): 19 | def __init__(self, mean, std): 20 | super(Normalize, self).__init__() 21 | self.register_buffer('mean', torch.Tensor(mean)) 22 | self.register_buffer('std', torch.Tensor(std)) 23 | 24 | def forward(self, input): 25 | # Broadcasting 26 | if torch.max(input) > 1: 27 | input = input / 255.0 28 | mean = self.mean.reshape(1, 3, 1, 1) 29 | std = self.std.reshape(1, 3, 1, 1) 30 | return (input - mean.to(device=input.device)) / std.to( 31 | device=input.device) 32 | 33 | 34 | def diet_tiny(): 35 | model = timm.create_model("deit_tiny_patch16_224", pretrained=True) 36 | return model 37 | def diet_small(): 38 | model = timm.create_model("deit_small_patch16_224", pretrained=True) 39 | return model 40 | 41 | def vit_tiny(): 42 | model = timm.create_model('vit_tiny_patch16_224', pretrained=True) 43 | return model 44 | 45 | def vit_small(): 46 | model = timm.create_model('vit_small_patch16_224', pretrained=True) 47 | return model 48 | 49 | 50 | 51 | def classify(save_dir, batch_size, save_results, adv=True): 52 | 53 | image_transforms_adv = transforms.Compose([ 54 | transforms.ToTensor(), 55 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 56 | std=[0.229, 0.224, 0.225])]) 57 | 58 | image_transfroms_clean = transforms.Compose([ 59 | transforms.Resize(256), 60 | transforms.CenterCrop(224), 61 | transforms.ToTensor(), 62 | transforms.Normalize(mean=[0.485, 0.456, 0.406], 63 | std=[0.229, 0.224, 0.225])]) 64 | 65 | transform = image_transforms_adv if adv else image_transfroms_clean 66 | 67 | 68 | data = torchvision.datasets.folder.ImageFolder(root=save_dir, transform=transform) 69 | test_loader = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=False) 70 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 71 | 72 | models = {"Resnet-152": torchvision.models.resnet152, "VGG-19": torchvision.models.vgg19_bn, "Inception-V3": torchvision.models.inception_v3, 73 | "DenseNet-161": torchvision.models.densenet161, "DenseNet-121": torchvision.models.densenet121, 74 | "WRN-101": torchvision.models.wide_resnet101_2, "MobileNet-v2": torchvision.models.mobilenet_v2, 75 | "senet": pretrainedmodels.__dict__['senet154']} 76 | 77 | model_results_csv = open(f'{os.path.join(save_dir, save_results)}.csv', 'w') # append? 78 | data_writer = csv.writer(model_results_csv) 79 | title = ['image_type',save_dir] 80 | data_writer.writerow(title) 81 | header = ['model', 'Accuracy'] 82 | data_writer.writerow(header) 83 | avg_accuracy = 0 84 | for name, obj in models.items(): 85 | if name == "senet": 86 | model = obj(num_classes=1000, pretrained='imagenet') 87 | else: 88 | model = obj(pretrained=True) 89 | 90 | model.to(device) 91 | model.eval() 92 | total = 0 93 | correct = 0 94 | with torch.no_grad(): 95 | for (images, labels) in test_loader: 96 | images = images.to(device) 97 | labels = labels.to(device) 98 | outputs = model(images) 99 | _, predicted = torch.max(outputs.data, 1) 100 | total += labels.size(0) 101 | print(total, end="\r") 102 | correct += (predicted == labels).sum().item() 103 | 104 | print(f'Accuracy of the model {name} on the test images: {100 * correct / total} %') 105 | accuracy = 100 * correct / total 106 | avg_accuracy += accuracy 107 | data_writer.writerow([name, accuracy]) 108 | print(f'Average accuracy on models {avg_accuracy / len(models.items())} %') 109 | print(f"Results saved in {os.path.join(save_dir, save_results)}.csv") 110 | data_writer.writerow(["Average accuracy", avg_accuracy / len(models.items())]) 111 | 112 | 113 | 114 | def classifiy_transformers(save_dir, batch_size, save_results, adv=True): 115 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 116 | model_results_csv = open(f'{os.path.join(save_dir, save_results)}_transformers.csv', 'w') # append? 117 | data_writer = csv.writer(model_results_csv) 118 | title = ['image_type', save_dir] 119 | data_writer.writerow(title) 120 | header = ['model', 'Accuracy'] 121 | data_writer.writerow(header) 122 | avg_accuracy = 0 123 | 124 | transformers = {"diet_tiny":diet_tiny, "diet_small":diet_small, "vit_tiny":vit_tiny,"vit_small":vit_small} 125 | for name, transformer in transformers.items(): 126 | model = transformer() 127 | config = resolve_data_config({}, model=model) 128 | transform = create_transform(**config) 129 | transform.transforms.pop() 130 | transform_clean = transform 131 | transform_adv = transforms.Compose([transforms.ToTensor(), ]) 132 | transform = transform_adv if adv else transform_clean 133 | norm_layer = Normalize(mean=config['mean'], 134 | std=config['std']) 135 | model = nn.Sequential(norm_layer, model.to(device=device)) 136 | data = torchvision.datasets.folder.ImageFolder(root=save_dir, transform=transform) 137 | test_loader = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=False) 138 | 139 | 140 | 141 | model.eval() 142 | total = 0 143 | correct = 0 144 | with torch.no_grad(): 145 | for (images, labels) in test_loader: 146 | images = images.to(device) 147 | labels = labels.to(device) 148 | outputs = model(images) 149 | _, predicted = torch.max(outputs.data, 1) 150 | total += labels.size(0) 151 | print(total, end="\r") 152 | correct += (predicted == labels).sum().item() 153 | 154 | print(f'Accuracy of the model {name} on the test images: {100 * correct / total} %') 155 | accuracy = 100 * correct / total 156 | avg_accuracy += accuracy 157 | data_writer.writerow([name, accuracy]) 158 | print(f'Average accuracy on models {avg_accuracy / len(transformers.items())} %') 159 | print(f"Results saved in {os.path.join(save_dir, save_results)}_transformers.csv") 160 | data_writer.writerow(["Average accuracy", avg_accuracy / len(transformers.items())]) 161 | 162 | parser = argparse.ArgumentParser(description='Classification') 163 | parser.add_argument('--data_path', type=str, default='adv_images_rotate') 164 | parser.add_argument('--mode', type=str, default='adv') 165 | parser.add_argument('--test_model', type=str, default='all') 166 | parser.add_argument('--save_results', type=str, default='results_') 167 | parser.add_argument('--batch_size', type=int, default=64) 168 | 169 | 170 | if __name__ == "__main__": 171 | args = parser.parse_args() 172 | mode = args.mode == "adv" 173 | classify(save_dir=args.data_path, batch_size=args.batch_size, save_results=args.save_results,adv= mode ) 174 | classifiy_transformers(save_dir=args.data_path, batch_size=args.batch_size, save_results=args.save_results, adv=mode) 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | -------------------------------------------------------------------------------- /train_cd.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import sys 5 | 6 | import PIL 7 | import numpy as np 8 | import torchvision 9 | import torchvision.datasets.folder 10 | import wandb 11 | from PIL import Image 12 | from PIL import ImageFile 13 | from torch.backends import cudnn 14 | from torchvision import transforms as T 15 | 16 | from surrogate import * 17 | from utils import * 18 | 19 | PIL.Image.MAX_IMAGE_PIXELS = 933120000 20 | 21 | ImageFile.LOAD_TRUNCATED_IMAGES = True 22 | 23 | def plot_grid(w): 24 | import matplotlib.pyplot as plt 25 | grid_img = torchvision.utils.make_grid(w) 26 | plt.imshow(grid_img.permute(1,2,0).cpu()) 27 | plt.show() 28 | 29 | 30 | 31 | parser = argparse.ArgumentParser(description='Train') 32 | parser.add_argument('--project', type=str, default="APR") 33 | parser.add_argument('--entity', type=str, default="hashmatshadab") 34 | parser.add_argument('--wandb_mode', type=str, default="disabled") 35 | parser.add_argument('--start_epoch', type=int, default=0) 36 | parser.add_argument('--end_epoch', type=int, default=35) 37 | parser.add_argument('--save_epoch', type=int, default=1) 38 | parser.add_argument('--resume', type=lambda x: (str(x).lower() == 'true'), default=False) 39 | parser.add_argument('--chk_pth', type=str, default='....') 40 | parser.add_argument('--batch_size', type=int, default=64, help='number of all reference images') 41 | parser.add_argument('--mode', type=str, default='rotate', choices=["rotate","jigsaw"]) 42 | parser.add_argument('--data_dir', type=str, default='./data/ILSVRC2012_img_val') 43 | parser.add_argument('--save_dir', type=str, default='./single_trained_models') 44 | parser.add_argument('--fgsm_step', type=int, default=2) 45 | parser.add_argument('--adv_train', type=lambda x: (str(x).lower() == 'true'), default=False) 46 | parser.add_argument('--lr', type=float, default=0.001) 47 | 48 | 49 | def create_json(args): 50 | """ 51 | To create json file to save the arguments (args) 52 | """ 53 | 54 | with open(f"{args.save_dir}/config_train_single.json", "w") as write_file: 55 | json.dump(args.__dict__, write_file, indent=4) 56 | 57 | 58 | def initialize_model(decoder_num): 59 | """ 60 | Initialize the auto-encoder model with given number of decoders 61 | :param decoder_num: Number of decoders (20 for prototypical and 1 for other modes) 62 | """ 63 | model = autoencoder(input_nc=3, output_nc=3, n_blocks=3, decoder_num=decoder_num) 64 | model = nn.Sequential( 65 | Normalize(), 66 | model, 67 | ) 68 | model.to(device) 69 | return model 70 | 71 | 72 | 73 | 74 | 75 | def train_unsup(model, data_loader,optimizer,args): 76 | if torch.cuda.is_available(): 77 | device = torch.device('cuda') 78 | else: 79 | device = torch.device('cpu') 80 | for epoch in range(args.start_epoch, args.end_epoch): 81 | avg_batch_loss_epoch = 0 82 | running_loss = 0 83 | 84 | for idx, (img, _) in enumerate(data_loader): 85 | img =img.to(device) 86 | img_input = img 87 | img_tar = img.clone() 88 | for img_ind in range(img_input.shape[0]): 89 | if args.mode == 'rotate': 90 | img_input[img_ind:img_ind + 1] = rot(img_input[img_ind:img_ind + 1]) 91 | elif args.mode == 'jigsaw': 92 | img_input[img_ind] = shuffle(img_input[img_ind], 1) 93 | else: 94 | sys.exit("Enter the correct mode") 95 | 96 | outputs, _, _ = model(img_input) 97 | loss = nn.MSELoss()(outputs[0], img_tar) 98 | 99 | 100 | optimizer.zero_grad() 101 | loss.backward() 102 | optimizer.step() 103 | avg_batch_loss_epoch += loss.item() 104 | if idx % 10 == 9: 105 | print('Epoch: {0} \t Batch: {1} \t loss: {2:.5f}'.format(epoch, idx, running_loss / 10)) 106 | wandb.log({"Running Loss": running_loss / 10}) 107 | running_loss = 0 108 | 109 | running_loss += abs(loss.item()) 110 | length = len(data_loader) // 2 111 | if (idx + 1) % length == 0: 112 | rand_ind = torch.randint(0, img_input.shape[0], size=(1,)) 113 | outputs = outputs[0] 114 | 115 | wandb.log( 116 | {'Image': [wandb.Image(img_input[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input"), 117 | wandb.Image(img_tar[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Target"), 118 | wandb.Image(outputs[rand_ind.item()].permute(1, 2, 0).detach().cpu().numpy(), 119 | caption="Output")], 120 | }) 121 | 122 | 123 | print(f"Epoch {epoch}: Average Loss on Batch::: {avg_batch_loss_epoch / (len(data_loader))}") 124 | wandb.log({"Loss": avg_batch_loss_epoch / (len(data_loader))}) 125 | os.makedirs(args.save_dir + f'/models', exist_ok=True) 126 | 127 | 128 | if (epoch + 1) % args.save_epoch == 0: 129 | model.eval() 130 | torch.save(model.state_dict(), args.save_dir + f'/models/{args.mode}_{epoch}.pth') 131 | model.train() 132 | 133 | 134 | 135 | def train_unsup_adv(model, data_loader,optimizer,args): 136 | if torch.cuda.is_available(): 137 | device = torch.device('cuda') 138 | else: 139 | device = torch.device('cpu') 140 | fgsm_step = args.fgsm_step/255 141 | 142 | 143 | attack = FGSM(model, eps=fgsm_step) 144 | 145 | for epoch in range(args.start_epoch, args.end_epoch): 146 | avg_batch_loss_epoch = 0 147 | running_loss = 0 148 | for idx, (img, _) in enumerate(data_loader): 149 | img =img.to(device) 150 | img_input = img 151 | img_tar = img.clone() 152 | for img_ind in range(img_input.shape[0]): 153 | if args.mode == 'rotate': 154 | img_input[img_ind:img_ind + 1] = rot(img_input[img_ind:img_ind + 1]) 155 | elif args.mode == 'jigsaw': 156 | img_input[img_ind] = shuffle(img_input[img_ind], 1) 157 | else: 158 | sys.exit("Enter the correct mode") 159 | 160 | 161 | adv_images = attack(img_input, img_tar) 162 | clean_output, clean_enc, _ = model(img_input) 163 | adv_output, adv_enc, _ = model(adv_images) 164 | loss_clean = nn.MSELoss()(clean_output[0], img_tar) 165 | loss_adv = nn.MSELoss()(adv_output[0], img_tar) 166 | sim_loss = nn.MSELoss()(adv_enc, clean_enc) 167 | 168 | loss = loss_clean + loss_adv + sim_loss 169 | optimizer.zero_grad() 170 | loss.backward() 171 | optimizer.step() 172 | 173 | 174 | avg_batch_loss_epoch += loss.item() 175 | if idx % 10 == 9: 176 | print('Epoch: {0} \t Batch: {1} \t sim_loss: {2:.5f}\t running_loss: {3:.5f}'.format(epoch, idx, sim_loss.item(), running_loss / 10)) 177 | 178 | wandb.log({"Running Loss": running_loss / 10}) 179 | running_loss = 0 180 | 181 | running_loss += abs(loss.item()) 182 | 183 | length = len(data_loader) // 2 184 | if (idx + 1) % length == 0: 185 | rand_ind = torch.randint(0, img_input.shape[0], size=(1,)) 186 | wandb.log( 187 | {'Image': [ 188 | wandb.Image(img_input[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input-Clean"), 189 | wandb.Image(adv_images[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input-Adv"), 190 | ], 191 | }) 192 | 193 | print(f"Epoch {epoch}: Average Loss on Batch::: {avg_batch_loss_epoch / (len(data_loader))}") 194 | wandb.log({"Loss": avg_batch_loss_epoch / (len(data_loader))}) 195 | os.makedirs(args.save_dir + f'/models', exist_ok=True) 196 | 197 | if (epoch + 1) % args.save_epoch == 0: 198 | model.eval() 199 | torch.save(model.state_dict(), args.save_dir + f'/models/{args.mode}_{epoch}.pth') 200 | model.train() 201 | 202 | 203 | 204 | 205 | 206 | if __name__ =="__main__": 207 | 208 | args = parser.parse_args() 209 | wandb.init(project=args.project, entity=args.entity, mode=args.wandb_mode, name=args.save_dir.split("/")[-1]) 210 | 211 | cudnn.benchmark = False 212 | cudnn.deterministic = True 213 | SEED = 0 214 | torch.manual_seed(SEED) 215 | torch.cuda.manual_seed(SEED) 216 | np.random.seed(SEED) 217 | print(args) 218 | config = wandb.config 219 | config.update(args) 220 | 221 | 222 | 223 | transform = T.Compose([ 224 | T.Resize((256,256)), 225 | T.CenterCrop(224), 226 | T.ToTensor() 227 | ]) 228 | 229 | 230 | os.makedirs(args.save_dir + '/models', exist_ok=True) 231 | device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') 232 | dataset = torchvision.datasets.folder.ImageFolder(root=args.data_dir, transform=transform) 233 | data_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=True) 234 | 235 | model = initialize_model(1) 236 | if args.resume: 237 | state = torch.load(f"{args.chk_pth}") 238 | model.load_state_dict(state) 239 | 240 | model.to(device) 241 | model.train() 242 | optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) 243 | 244 | if args.adv_train: 245 | train_unsup_adv(model, data_loader, optimizer, args) 246 | else: 247 | train_unsup(model, data_loader, optimizer, args) 248 | 249 | 250 | 251 | 252 | 253 | 254 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # **Adversarial Pixel Restoration as a Pretext Task for Transferable Perturbations (BMVC'22--Oral)** 3 | 4 | [Hashmat Shadab Malik](https://scholar.google.com/citations?user=2Ft7r4AAAAAJ&hl=en), 5 | [Shahina Kunhimon](https://github.com/ShahinaKK), 6 | [Muzammal Naseer](https://scholar.google.ch/citations?user=tM9xKA8AAAAJ&hl=en), 7 | [Salman Khan](https://salman-h-khan.github.io), 8 | and [Fahad Shahbaz Khan](https://scholar.google.es/citations?user=zvaeYnUAAAAJ&hl=en) 9 | 10 | [![paper](https://img.shields.io/badge/arXiv-Paper-.svg)](https://arxiv.org/abs/2207.08803) 11 | [![Video](https://img.shields.io/badge/Video-Presentation-F9D371)](https://drive.google.com/file/d/1ECkp_lbMj5Pz7RX_GgEvWWDHf5PUXlFd/view?usp=share_link) 12 | [![slides](https://img.shields.io/badge/Poster-PDF-87CEEB)](https://drive.google.com/file/d/1neYZca0KRIBCu5R6P78aQMYJa7R2aTFs/view?usp=share_link) 13 | [![slides](https://img.shields.io/badge/Presentation-Slides-B762C1)](https://drive.google.com/file/d/1wRgCs2uBO0p10FC75BKDUEdggz_GO9Oq/view?usp=share_link) 14 | # 15 | 16 | > **Abstract:** *Transferable adversarial attacks optimize adversaries from a pretrained surrogate model and known label space to fool the unknown black-box models. Therefore, these attacks are restricted by the availability of an effective surrogate model. In this work, we 17 | relax this assumption and propose Adversarial Pixel Restoration as a self-supervised alternative to train an effective surrogate model from scratch under the condition of no labels and few data samples. Our training approach is based on a min-max scheme which reduces overfitting via an adversarial objective and thus optimizes for a more generalizable surrogate model. Our proposed attack is complimentary to the adversarial pixel restoration and is independent of any task specific objective as it can be launched in a self-supervised manner. We successfully demonstrate the adversarial transferability of our approach to Vision Transformers as well as Convolutional Neural Networks for the tasks of classification, object detection, and video segmentation. Our training approach improves the transferability of the baseline unsupervised training method by 16.4% on ImageNet val. set.* 18 | 19 | 20 | ## Contents 21 | 1) [Highlights](#Highlights) 22 | 2) [Installation](#Installation) 23 | 3) [Dataset Preparation](#Dataset-Preparation) 24 | 4) [Adversarial Pixel Restoration Training](#Adversarial-Pixel-Restoration-Training) 25 | 5) [Self-supervised Attack](#Self-supervised-Attack) 26 | 6) [Pretrained Surrogate Models](#Pretrained-Surrogate-Models) 27 | 7) [Adversarial Transferability Results](#Adversarial-Transferability-Results) 28 | 29 |
30 |
31 | 32 | ## Highlights 33 | 1. We propose self-supervised Adversarial Pixel Restoration to find highly transferable patterns by learning over flatter loss surfaces. Our training approach allows launching cross-domain attacks without access to large-scale labeled data or pretrained models. 34 | 35 | ![main figure](images/Algo.png) 36 | The algorithm describes *Adversarial Pixel Restoration* for training the surrogate model. Please refer to our paper for more details on the equations in the above-mentioned algorithm. 37 | 38 | 2. Our proposed adversarial attack is self-supervised in nature and independent of any task-specific objective. Therefore our approach can transfer perturbations to a variety of tasks as we demonstrate for classification, object detection, and segmentation. 39 | 40 |
41 |
42 | 43 | ## Installation 44 | ([top](#contents)) 45 | 1. Create conda environment 46 | ```shell 47 | conda create -n apr 48 | ``` 49 | 2. Install PyTorch and torchvision 50 | ```shell 51 | conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch 52 | ``` 53 | 3. Install other dependencies 54 | ```shell 55 | pip install -r requirements.txt 56 | ``` 57 | 58 |
59 |
60 | 61 | ## Dataset Preparation 62 | ([top](#contents)) 63 | **In-Domain Setting:** 5000 images are selected from ImageNet-Val (10 each from the first 500 classes). 64 | Each surrogate model is trained only on few data samples e.g., 20 samples(default). Download the [ImageNet-Val](http://image-net.org/) classification dataset and structure the data as follows: 65 | ``` 66 | └───data 67 | ├── selected_data.csv 68 | └── ILSVRC2012_img_val 69 | ├── n01440764 70 | ├── n01443537 71 | └── ... 72 | 73 | ``` 74 | The `selected_data.csv` is used by the `our_dataset.py` to load the selected 5000 images from the dataset. 75 |
76 | 77 | **Cross-Domain Setting:** A single surrogate model is trained on large unannotated datasets. We use the following datasets for 78 | training: 79 | * [Paintings](https://www.kaggle.com/c/painter-by-numbers) 80 | * [Comics](https://www.kaggle.com/cenkbircanoglu/comic-books-classification) 81 | * [CoCo-2017(41k)](https://cocodataset.org/#download) 82 | 83 | Directory structure should look like this: 84 | ``` 85 | |paintings 86 | |images 87 | img1 88 | img2 89 | ... 90 | ``` 91 |
92 |
93 | 94 | ## Adversarial Pixel Restoration Training 95 | ([top](#contents)) 96 | **In-Domain Setting:** Each surrogate model is trained only on a few data samples 97 | (20 by default). The model is trained by incorporating adversarial pixel transformation 98 | based on rotation or jigsaw in an unsupervised setting. Supervised prototypical training mentioned in 99 | this [paper]() is also trained in an adversarial fashion. 100 | 101 | For training surrogate models with transformation: 102 | 1. _Rotation_ 103 | ```shell 104 | python train_id.py --mode rotate --n_imgs 20 --adv_train True --fgsm_step 2 \ 105 | --n_iters 2000 --save_dir ./trained_models 106 | ``` 107 | 2. _Jigsaw_ 108 | ```shell 109 | python train_id.py --mode jigsaw --n_imgs 20 --adv_train True --fgsm_step 2 \ 110 | --n_iters 5000 --save_dir ./trained_models 111 | ``` 112 | 3. _Prototypical_ 113 | ```shell 114 | python train_id.py --mode prototypical --n_imgs 20 --adv_train True --fgsm_step 2 \ 115 | --n_iters 15000 --save_dir ./trained_models 116 | ``` 117 | With 20 images used for training each surrogate model, overall 250 models would 118 | be trained for the selected 5000 ImageNet-Val images. The models would be saved 119 | in like: 120 | ``` 121 | |trained_models 122 | |models 123 | rotate_0.pth 124 | rotate_1.pth 125 | ... 126 | ``` 127 |
128 | 129 | **Cross-Domain Setting:** A single surrogate model is trained adversarially on a large unannotated data 130 | in an unsupervised setting by using rotation or jigsaw as pixel transfromations. 131 | 132 | For training the single surrogate model with transfromation: 133 | 1. _Rotation_ 134 | ```shell 135 | python train_cd.py --mode rotate --adv_train True --fgsm_step 2 \ 136 | --end_epoch 50 --data_dir paintings/ --save_dir ./single_trained_models 137 | ``` 138 | 2. _Jigsaw_ 139 | ```shell 140 | python train_cd.py --mode jigsaw --adv_train True --fgsm_step 2 \ 141 | --end_epoch 50 --data_dir paintings/ --save_dir ./single_trained_models 142 | ``` 143 | change the `--data_dir` accordingly to train on comics, coco and any other dataset. 144 | Setting `--adv_train` flag to False would result in the surrogate models trained 145 | by the baseline method mentioned in this [paper](). 146 |
147 |
148 | 149 | ## Self-supervised Attack 150 | ([top](#contents)) 151 | **In-Domain Setting:** Adversarial examples are crafted on the selected 5000 152 | ImageNet-Val images, following the same setting used in the baseline -> [Practical No-box Adversarial Attacks (NeurIPS-2021)](https://arxiv.org/abs/2012.02525). An L_inf based attack is run using: 153 | ```shell 154 | python attack.py --epsilon 0.1 --ila_niters 100 --ce_niters 200 \ 155 | --ce_epsilon 0.1 --ce_alpha 1.0 --n_imgs 20 --ae_dir ./trained_models \ 156 | --mode rotate --save_dir /path/to/save/adv_images 157 | ``` 158 | mode can be set as `rotate/jigsaw/prototypical` based on how the surrogate models 159 | were trained. For `rotation/jigsaw` we can use a fully-unsupervised attack by 160 | passing `--loss unsup` as argument to the `attack.py` file. 161 |
162 | 163 | **Cross-Domain Setting:** A single surrogate model trained on a cross-domain dataset as 164 | mentioned in the Training section is used to craft adversarial examples on 165 | the selected 5000 ImageNet-Val images. An L_inf based unsupervised attack is run using: 166 | ```shell 167 | python attack.py --epsilon 0.1 --ila_niters 100 --ce_niters 200 \ 168 | --ce_epsilon 0.1 --ce_alpha 1.0 --n_imgs 20 --single_model True \ 169 | --chk_pth path/to/trained/model/weights.pth --save_dir /path/to/save/adv_images 170 | ``` 171 |
172 |
173 | 174 | ### Pretrained Surrogate Models 175 | **In-Domain Setting:** Pretrained weights for surrogate models trained 176 | with [rotation](https://drive.google.com/file/d/1_2ljcaCt2NFhb5VLBcFl8eRTRge6Y4O2/view?usp=sharing)/[jigsaw](https://drive.google.com/file/d/1JyhznzEfCAq19tCP3imG-77sT9-aVCqd/view?usp=sharing)/[prototypical](https://mbzuaiac-my.sharepoint.com/:u:/g/personal/shahina_kunhimon_mbzuai_ac_ae/EdaPAbhYVwFLk7xg5uH778QBIXIZLsX9p_NgScGwz-x6oA) modes. 177 | 178 | **Cross-Domain Setting:** 179 | 1. Models trained with rotation mode. 180 | 181 | | Dataset | Baseline | Ours | 182 | |:----------|:----------------------------------------------------------------------------------------------------:|-------------------------------------------------------------------------------------------------:| 183 | | CoCo | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/rotate_50_baseline_coco.pth) | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/rotate_50_ours_coco.pth) | 184 | | Paintings | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/rotate_50_baseline_paintings.pth) | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/rotate_50_ours_paintings.pth) | 185 | | Comics | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/rotate_50_baseline_comics.pth) | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/rotate_50_ours_comics.pth) | 186 | 187 | 188 | 2. Models trained with jigsaw mode. 189 | 190 | | Dataset | Baseline | Ours | 191 | |:----------|:----------------------------------------------------------------------------------------------------:|-------------------------------------------------------------------------------------------------:| 192 | | CoCo | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/jigsaw_50_baseline_coco.pth) | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/jigsaw_50_ours_coco.pth) | 193 | | Paintings | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/jigsaw_50_baseline_paintings.pth) | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/jigsaw_50_ours_paintings.pth) | 194 | | Comics | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/jigsaw_50_baseline_comics.pth) | [Link](https://github.com/HashmatShadab/APR/releases/download/v1.0/jigsaw_50_ours_comics.pth) | 195 | 196 | 197 |
198 |
199 | 200 | ## Adversarial Transferability Results 201 | ([top](#contents)) 202 | We compare transferability of surrogate models trained by our approach with the 203 | approach followed by the baseline -> [Practical No-box Adversarial Attacks (NeurIPS-2021)](https://arxiv.org/abs/2012.02525). 204 | After generating adversarial examples on the selected 5000 ImageNet-Val images, 205 | we report the top-1 accuracy on several classification 206 | based models _(lower is better)_. 207 | 208 | 209 | **In-Domain Setting:** 210 | 211 | 1. Accuracy on Convolutional Networks. 212 | ![results](images/Table1.png) 213 | 214 | 2. Accuracy on Vision Transformers. 215 | ![results](images/Table2.png) 216 | 217 | **Cross-Domain Setting:** 218 | Average Accuracy on Convolution Networks and Vision Transformers (listed above). 219 | 220 | ![results](images/Table3.png) 221 |
222 |
223 | 224 | ## Citation 225 | If you use our work, please consider citing: 226 | ```bibtex 227 | @inproceedings{Malik_2022_BMVC, 228 | author = {Hashmat Shadab Malik and Shahina Kunhimon and Muzammal Naseer and Salman Khan and Fahad Shahbaz Khan}, 229 | title = {Adversarial Pixel Restoration as a Pretext Task for Transferable Perturbations}, 230 | booktitle = {33rd British Machine Vision Conference 2022, {BMVC} 2022, London, UK, November 21-24, 2022}, 231 | publisher = {{BMVA} Press}, 232 | year = {2022}, 233 | url = {https://bmvc2022.mpi-inf.mpg.de/0546.pdf} 234 | } 235 | 236 | ``` 237 | 238 |
239 | 240 | ## Contact 241 | Should you have any question, please create an issue on this repository or contact at hashmat.malik@mbzuai.ac.ae 242 | 243 |
244 | 245 | ## References 246 | Our code is based on [ Practical No-box Adversarial Attacks against DNNs](https://github.com/qizhangli/nobox-attacks) repository. 247 | We thank them for releasing their code. 248 | -------------------------------------------------------------------------------- /attack.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | import torchvision 6 | import torchvision.transforms as T 7 | from torch.backends import cudnn 8 | from torch.utils.data import Dataset 9 | import os 10 | import csv 11 | import numpy as np 12 | import wandb 13 | from surrogate import * 14 | from our_dataset import OUR_dataset 15 | from utils import * 16 | from classification import classify 17 | import torch.nn.functional as F 18 | 19 | 20 | parser = argparse.ArgumentParser(description='Attack') 21 | parser.add_argument('--project', type=str, default="APR") 22 | parser.add_argument('--entity', type=str, default="hashmatshadab") 23 | parser.add_argument('--wandb_mode', type=str, default="disabled") 24 | parser.add_argument('--epsilon', type=float, default=0.1, help="Perturbation budget of the attack") 25 | parser.add_argument('--single_model', type=lambda x: (str(x).lower() == 'true'), default=False) 26 | parser.add_argument('--chk_pth', type=str, default="trained_models/models/0.pth") 27 | parser.add_argument('--ila_niters', type=int, default=100) 28 | parser.add_argument('--ce_niters', type=int, default=200) 29 | parser.add_argument('--ce_epsilon', type=float, default=0.1) 30 | parser.add_argument('--ce_alpha', type=float, default=1.0) 31 | parser.add_argument('--n_imgs', type=int, default=20) 32 | parser.add_argument('--n_decoders', type=int, default=20, help="Number of decoders in the autoencoder") 33 | parser.add_argument('--ae_dir', type=str, default='./trained_models', help="Path from where to load trained autoencoders") 34 | parser.add_argument('--save_dir', type=str, default='./adv_images', help="Path where adversarial images will be saved") 35 | parser.add_argument('--mode', type=str, default='rotate', help="Mode by which the autoencoders were trained") 36 | parser.add_argument('--ce_method', type=str, default='ifgsm') 37 | parser.add_argument('--start', type=int, default=0) 38 | parser.add_argument('--end', type=int, default=2500) 39 | parser.add_argument('--loss', type=str, default="baseline", choices=["baseline","unsup"]) 40 | parser.add_argument('--opl_gamma', type=float, default=0.5) 41 | parser.add_argument('--save_results', type=str, default='results', help="name of file for saving classification scores" 42 | "on various models") 43 | 44 | 45 | 46 | class ILA(torch.nn.Module): 47 | def __init__(self): 48 | super(ILA, self).__init__() 49 | def forward(self, ori_mid, tar_mid, att_mid): 50 | """ 51 | Maximizes projections on mid layer representations 52 | """ 53 | bs = ori_mid.shape[0] 54 | ori_mid = ori_mid.view(bs, -1) 55 | tar_mid = tar_mid.view(bs, -1) 56 | att_mid = att_mid.view(bs, -1) 57 | W = att_mid - ori_mid 58 | V = tar_mid - ori_mid 59 | V = V / V.norm(p=2,dim=1, keepdim=True) 60 | ILA = (W*V).sum() / bs 61 | return ILA 62 | 63 | 64 | 65 | def save_attack_img(img, file_dir): 66 | T.ToPILImage()(img.data.cpu()).save(file_dir) 67 | 68 | def initialize_model(decoder_num): 69 | """ 70 | Initialize the auto-encoder model with given number of decoders 71 | :param decoder_num: Number of decoders (20 for prototypical and 1 for other modes) 72 | """ 73 | 74 | model = autoencoder(input_nc=3, output_nc=3, n_blocks=3, decoder_num=decoder_num) 75 | model = nn.Sequential( 76 | Normalize(), 77 | model, 78 | ) 79 | model.to(device) 80 | return model 81 | 82 | 83 | 84 | def attack_ila(model, ori_img, tar_img, attack_niters, eps): 85 | """ 86 | This function applies ILA attack 87 | :param ori_img: The original input image 88 | :param tar_img: The image after baseline gradient attack 89 | :param attack_niters: Number of ILA iterations 90 | :param eps: Maximum perturbation rate for ILA 91 | :return: returns the image after ILA attack. 92 | """ 93 | # targ_img is the attacked img 94 | model.eval() 95 | ori_img = ori_img.to(device) 96 | img = ori_img.clone() 97 | with torch.no_grad(): 98 | # get output of the encoder for tar_img and ori_img without computing gradients 99 | _, tar_h_feats,_ = model(tar_img) 100 | _, ori_h_feats,_ = model(ori_img) 101 | # ori_h_feats are the features from the original image 102 | # tar_h_feats are the features from the attacked images before ila 103 | for i in range(attack_niters): 104 | img.requires_grad_(True) 105 | _, att_h_feats,_ = model(img) 106 | # att_h_feats are features computed after the orig image in the loop 107 | loss = ILA()(ori_h_feats.detach(), tar_h_feats.detach(), att_h_feats) 108 | if (i+1) % 50 == 0: 109 | print('\r ila attacking {}, {:0.4f}'.format(i+1, loss.item()),end=' ') 110 | loss.backward() 111 | input_grad = img.grad.data.sign() 112 | img = img.data + 1. / 255 * input_grad 113 | img = torch.where(img > ori_img + eps, ori_img + eps, img) 114 | img = torch.where(img < ori_img - eps, ori_img - eps, img) 115 | img = torch.clamp(img, min=0, max=1) 116 | print('') 117 | return img.data 118 | 119 | 120 | 121 | def attack_ce_unsup(model, ori_img, attack_niters, eps,args, alpha, n_imgs, ce_method, attack_loss, iter): 122 | """ 123 | For baseline gradient attack (ce_method). 124 | Applied on models trained using rotate/jigsaw/masking approach. 125 | :param model: 126 | :param ori_img: The original input image 127 | :param attack_niters: Number of baseline-gradient attack iterations 128 | :param eps: Maximum perturbation rate for baseline attack 129 | :param alpha: Scaling parameter for adversarial loss 130 | :param n_imgs: Number of images 131 | :param ce_method: The gradient- based baseline attack used (IFGSM or PGD) 132 | :return: Returns the image with adversarial loss maximized within the bound. 133 | """ 134 | model.eval() 135 | ori_img = ori_img.to(device) 136 | nChannels = 3 137 | tar_img = [] 138 | 139 | if args.loss == "unsup": 140 | for i in range(2 * n_imgs): 141 | tar_img.append(ori_img[i].unsqueeze(0)) 142 | tar_img = torch.cat(tar_img, dim=0) 143 | else: 144 | for i in range(n_imgs): 145 | tar_img.append(ori_img[[i, n_imgs + i]]) 146 | for i in range(n_imgs): 147 | tar_img.append(ori_img[[n_imgs+i, i]]) 148 | tar_img = torch.cat(tar_img, dim=0) 149 | tar_img = tar_img.reshape(2*n_imgs,2,nChannels,224,224) 150 | 151 | img = ori_img.clone() 152 | attack_loss[iter] = [] 153 | 154 | for i in range(attack_niters): 155 | 156 | if ce_method == 'ifgsm': 157 | img_x = img 158 | # In our implementation of PGD, we incorporate randomness at each iteration to further enhance the transferability 159 | elif ce_method == 'pgd': 160 | img_x = img + img.new(img.size()).uniform_(-eps, eps) 161 | img_x.requires_grad_(True) 162 | 163 | outs, enc_out,_ = model(img_x) 164 | 165 | if args.loss == "baseline": 166 | outs = outs[0].unsqueeze(1).repeat(1, 2, 1, 1, 1) 167 | loss_mse_ = nn.MSELoss(reduction='none')(outs, tar_img).sum(dim = (2,3,4)) / (nChannels*224*224) 168 | loss_mse = - alpha * loss_mse_ 169 | label = torch.tensor([0]*n_imgs*2).long().to(device) 170 | loss = nn.CrossEntropyLoss()(loss_mse,label) 171 | 172 | elif args.loss =="unsup": 173 | outs = outs[0] 174 | loss = nn.MSELoss(reduction='none')(outs, tar_img).sum() / (2*n_imgs*nChannels * 224 * 224) 175 | 176 | 177 | attack_loss[iter].append(loss.item()) 178 | 179 | if (i+1) % 50 == 0 or i == 0: 180 | print('\r attacking {}, {:0.4f}'.format(i, loss.item()), end=' ') 181 | loss.backward() 182 | 183 | adv_noise = img_x.grad 184 | 185 | input_grad = adv_noise.data.sign() 186 | img = img.data + 1. / 255 * input_grad 187 | img = torch.where(img > ori_img + eps, ori_img + eps, img) 188 | img = torch.where(img < ori_img - eps, ori_img - eps, img) 189 | img = torch.clamp(img, min=0, max=1) 190 | 191 | 192 | 193 | print('') 194 | return img.data 195 | 196 | 197 | 198 | def plot_grid(w): 199 | import matplotlib.pyplot as plt 200 | grid_img = torchvision.utils.make_grid(w) 201 | plt.imshow(grid_img.permute(1,2,0).cpu()) 202 | plt.show() 203 | 204 | def create_json(args): 205 | """ 206 | To create json file to save the arguments (args) 207 | """ 208 | 209 | with open(f"{args.save_dir}/config_attack.json", "w") as write_file: 210 | json.dump(args.__dict__, write_file, indent=4) 211 | 212 | def attack_ce_proto(model, ori_img, attack_niters,args, eps, alpha, n_decoders, ce_method, n_imgs, prototype_inds, 213 | attack_loss, iter): 214 | """ 215 | For baseline-gradient attack (ce-method) on models trained on prototypical reconstruction approach 216 | :param ori_img: The original input image 217 | :param attack_niters: Number of iterations for ce baseline gradient attack 218 | :param eps: Maximum perturbation rate for ce attack 219 | :param ce_method: The type of gradient-baseline attack (IFGSM or PGD) 220 | :param prototype_inds: The list of prototypes 221 | :param attack_loss: Adversarial loss for prototypical reconstruction 222 | :return: Returns the image with added perturbation (within the eps bound) maximising adversarial loss 223 | """ 224 | 225 | model.eval() 226 | ori_img = ori_img.to(device) 227 | tar_img = [] 228 | for i in range(n_decoders): 229 | # get one prototype pair for each decoder, just like in training 230 | tar_img.append(ori_img[[prototype_inds[2*i],prototype_inds[2*i+1]]]) 231 | tar_img = torch.cat(tar_img, dim = 0) 232 | nChannels = 3 233 | if n_decoders == 1: 234 | decoder_size = 224 235 | else: 236 | decoder_size = 56 237 | tar_img = F.interpolate(tar_img, size=(56,56)) # [40, 3, 56, 56] 238 | tar_img = tar_img.reshape(n_decoders,2,nChannels,decoder_size,decoder_size).unsqueeze(1) # [20, 1, 2, 3, 56, 56] 239 | # the 40 images are grouped as 20 pairs, each pair has image from the two different classes 240 | tar_img = tar_img.repeat(1,n_imgs*2,1,1,1,1).reshape(n_imgs*2*n_decoders,2,nChannels,decoder_size,decoder_size) # [400, 2, 3, 56, 56] 241 | # 400 pairs, each of the 20 pairs is repeated 20 times. i.e first 20 pairs are same and so on 242 | img = ori_img.clone() 243 | attack_loss[iter] = [] 244 | 245 | for i in range(attack_niters): 246 | 247 | if ce_method == 'ifgsm': 248 | img_x = img 249 | elif ce_method == 'pgd': 250 | img_x = img + img.new(img.size()).uniform_(-eps, eps) 251 | img_x.requires_grad_(True) 252 | 253 | 254 | outs, enc_out,_ = model(img_x) 255 | 256 | 257 | outs = torch.cat(outs, dim=0).unsqueeze(1).repeat(1, 2, 1, 1, 1) # [400, 2, 3, 56, 56] 258 | loss_mse_ = nn.MSELoss(reduction='none')(outs, tar_img).sum(dim=(2, 3, 4)) / ( 259 | nChannels * decoder_size * decoder_size) # [400, 2] 260 | loss_mse = - alpha * loss_mse_ 261 | label = torch.tensor(([0] * n_imgs + [1] * n_imgs) * n_decoders).long().to(device) 262 | loss = nn.CrossEntropyLoss()(loss_mse, label) 263 | 264 | # will give 20 images for each input image >> total [400, 3, 56, 56] 265 | 266 | if (i+1) % 50 == 0 or i == 0: 267 | print('attacking {}, {:0.4f}'.format(i, loss.item())) 268 | attack_loss[iter].append(loss.item()) 269 | loss.backward() 270 | 271 | adv_noise = img_x.grad 272 | 273 | input_grad = adv_noise.data.sign() 274 | 275 | img = img.data + 1. / 255 * input_grad 276 | img = torch.where(img > ori_img + eps, ori_img + eps, img) 277 | img = torch.where(img < ori_img - eps, ori_img - eps, img) 278 | img = torch.clamp(img, min=0, max=1) 279 | 280 | print('') 281 | return img.data 282 | 283 | if __name__ == '__main__': 284 | args = parser.parse_args() 285 | wandb.init(project=args.project, entity=args.entity, mode=args.wandb_mode, name=args.save_dir.split("/")[-1]) 286 | SEED = 0 287 | cudnn.benchmark = False 288 | cudnn.deterministic = True 289 | torch.manual_seed(SEED) 290 | torch.cuda.manual_seed(SEED) 291 | np.random.seed(SEED) 292 | 293 | print(args) 294 | config = wandb.config 295 | config.update(args) 296 | mode = args.mode 297 | save_dir = args.save_dir 298 | n_imgs = args.n_imgs // 2 299 | if mode != 'prototypical': 300 | n_decoders = 1 301 | else: 302 | n_decoders = args.n_decoders 303 | assert n_decoders <= n_imgs ** 2, 'Too many decoders.' 304 | os.makedirs(save_dir, exist_ok=True) 305 | if torch.cuda.is_available(): 306 | device = torch.device('cuda') 307 | else: 308 | device = torch.device('cpu') 309 | batch_size = n_imgs * 2 310 | epsilon = args.epsilon 311 | ce_epsilon = args.ce_epsilon 312 | ila_niters = args.ila_niters 313 | ce_niters = args.ce_niters 314 | ce_alpha = args.ce_alpha 315 | ae_dir = args.ae_dir 316 | ce_method = args.ce_method 317 | assert ce_method in ['ifgsm', 'pgd'] 318 | if torch.cuda.is_available(): 319 | device = torch.device('cuda') 320 | else: 321 | device = torch.device('cpu') 322 | 323 | 324 | 325 | 326 | trans = T.Compose([ 327 | T.Resize((256, 256)), 328 | T.CenterCrop(224), 329 | T.ToTensor() 330 | ]) 331 | dataset = OUR_dataset(data_dir='data/ILSVRC2012_img_val', 332 | data_csv_dir='data/selected_data.csv', 333 | mode='attack', 334 | img_num=n_imgs, 335 | transform=trans) 336 | dataloader = torch.utils.data.DataLoader(dataset, batch_size = batch_size, shuffle = False, num_workers = 1) 337 | create_json(args) 338 | fig, ax = plt.subplots() 339 | 340 | if args.single_model: 341 | args.loss = 'unsup' 342 | model = initialize_model(decoder_num=n_decoders) 343 | model.load_state_dict(torch.load(args.chk_pth)) 344 | model.eval() 345 | 346 | for data_ind, (ori_img, _) in enumerate(dataloader): 347 | if not args.start <= data_ind < args.end: 348 | continue 349 | if not args.single_model: 350 | model = initialize_model(n_decoders) 351 | model.load_state_dict(torch.load('{}/models/{}_{}.pth'.format(ae_dir, args.mode, data_ind))) 352 | model.eval() 353 | 354 | ori_img = ori_img.to(device) 355 | attack_loss = {} 356 | 357 | if mode =='prototypical': 358 | prototype_ind_csv = open(ae_dir+'/prototype_ind.csv', 'r') 359 | prototype_ind_ls = list(csv.reader(prototype_ind_csv)) 360 | old_att_img = attack_ce_proto(model, ori_img,args=args, attack_niters = ce_niters, 361 | eps = ce_epsilon, alpha=ce_alpha, n_decoders = n_decoders, 362 | ce_method = ce_method, n_imgs = n_imgs, 363 | prototype_inds = list(map(int,prototype_ind_ls[data_ind])), 364 | attack_loss=attack_loss, iter=data_ind) #** 365 | else: 366 | old_att_img = attack_ce_unsup(model, ori_img,args=args, attack_niters=ce_niters, 367 | eps=ce_epsilon, alpha=ce_alpha, n_imgs=n_imgs, 368 | ce_method=ce_method, 369 | attack_loss=attack_loss, iter=data_ind) 370 | xs = [x for x in range(len(attack_loss[data_ind]))] 371 | ax.plot(xs, attack_loss[data_ind], label=f"Model_{data_ind}") 372 | ax.set_xlabel("Iterations") 373 | ax.set_ylabel("Loss") 374 | ax.set_title(f"Model_{mode}_{data_ind}") 375 | wandb.log({f'plot': ax}) 376 | 377 | att_img = attack_ila(model, ori_img, old_att_img, ila_niters, eps=epsilon) 378 | for save_ind in range(batch_size): 379 | file_path, file_name = dataset.imgs[data_ind * 2*n_imgs + save_ind][0].split('/')[-2:] 380 | os.makedirs(save_dir + '/' + file_path, exist_ok=True) 381 | save_attack_img(img=att_img[save_ind], 382 | file_dir=os.path.join(save_dir, file_path, file_name[:-5]) + '.png') 383 | print('\r', data_ind * batch_size + save_ind, 'images saved.', end=' ') 384 | 385 | classify(save_dir=save_dir, batch_size=batch_size, save_results=args.save_results) 386 | 387 | -------------------------------------------------------------------------------- /train_id.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import json 4 | import os 5 | import sys 6 | import time 7 | 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import torch.nn.functional as F 11 | import torchvision 12 | import torchvision.transforms as T 13 | import wandb 14 | from torch.backends import cudnn 15 | from torch.utils.data import Dataset 16 | 17 | from surrogate import * 18 | from our_dataset import OUR_dataset 19 | from utils import * 20 | 21 | parser = argparse.ArgumentParser(description='Train') 22 | parser.add_argument('--project', type=str, default="APR") 23 | parser.add_argument('--entity', type=str, default="hashmatshadab") 24 | parser.add_argument('--wandb_mode', type=str, default="disabled") 25 | parser.add_argument('--n_imgs', type=int, default=20, help='number of all reference images') 26 | parser.add_argument('--n_iters', type=int, default=2000) 27 | parser.add_argument('--n_decoders', type=int, default=20) 28 | parser.add_argument('--lr', type=float, default=0.001) 29 | parser.add_argument('--mode', type=str, default='rotate') 30 | parser.add_argument('--save_dir', type=str, default='./trained_models') 31 | parser.add_argument('--start', type=int, default=0) 32 | parser.add_argument('--end', type=int, default=2500) 33 | parser.add_argument('--fgsm_step', type=int, default=2) 34 | parser.add_argument('--adv_train', type=lambda x: (str(x).lower() == 'true'), default=False) 35 | 36 | 37 | 38 | def create_json(args): 39 | """ 40 | To create json file to save the arguments (args) 41 | """ 42 | 43 | with open(f"{args.save_dir}/config_train.json", "w") as write_file: 44 | json.dump(args.__dict__, write_file, indent=4) 45 | 46 | def initialize_model(decoder_num): 47 | """ 48 | Initialize the auto-encoder model with given number of decoders 49 | :param decoder_num: Number of decoders (20 for prototypical and 1 for other modes) 50 | """ 51 | model = autoencoder(input_nc=3, output_nc=3, n_blocks=3, decoder_num=decoder_num) 52 | model = nn.Sequential( 53 | Normalize(), 54 | model, 55 | ) 56 | model.to(device) 57 | return model 58 | 59 | 60 | def plot_grid(w): 61 | import matplotlib.pyplot as plt 62 | grid_img = torchvision.utils.make_grid(w) 63 | plt.imshow(grid_img.permute(1,2,0).cpu()) 64 | plt.show() 65 | 66 | 67 | def train_prototypical(model, img, n_imgs, n_decoders, n_iters, prototype_ind_csv_writer,optimizer, train_loss, iter_ind): 68 | """ 69 | Training using the prototypical reconstruction approach 70 | 71 | :param img: Images to train each substitute model ( 10 images each from 2 classes) [20 x 3 x 224 x 224] 72 | :param n_imgs: Number of images (10 from each class) 73 | :param n_decoders: Number of decoders (20) [20 x 56 x 56] 74 | :param n_iters: Number of iterations for training each substitute model 75 | :param prototype_ind_csv_writer: To write the prototype pairs for the given autoencoder 76 | :param train_loss: dictionary to save loss at each iteration 77 | :return: returns the substitute model trained on images from 2 classes 78 | """ 79 | do_aug = True 80 | if n_imgs == 1: 81 | tar_ind_ls = [0, 1] 82 | else: 83 | tar_ind_ls = mk_proto_ls(n_imgs) 84 | tar_ind_ls = tar_ind_ls[:n_decoders * 2] 85 | # get the first 20 pairs (first 10 pairs are same always [0, 10, 1, 11, .........., 8, 18, 9, 19]) 86 | prototype_ind_csv_writer.writerow(tar_ind_ls.tolist()) # save the 20 pairs used in this model 87 | img_tar = img[tar_ind_ls] 88 | if n_decoders != 1: 89 | img_tar = F.interpolate(img_tar, (56, 56)) 90 | since = time.time() 91 | train_loss[iter_ind] = [] 92 | for i in range(n_iters): 93 | rand_ind = torch.cat((torch.randint(0, n_imgs, size=(1,)), torch.randint(n_imgs, 2 * n_imgs, size=(1,)))) 94 | # get random indices of the 2 images to be chosen from two different classes 95 | img_input = img[rand_ind].clone() 96 | if do_aug: 97 | img_input = aug(img_input) 98 | assert img_input.shape[3] == 224 99 | outputs, _,_ = model(img_input) 100 | gen_img = torch.cat(outputs, dim=0) 101 | loss = nn.MSELoss()(gen_img, img_tar) 102 | optimizer.zero_grad() 103 | loss.backward() 104 | optimizer.step() 105 | train_loss[iter_ind].append(loss.item()) 106 | 107 | if (i + 1) % 500 == 0: 108 | print(iter_ind + 1, i + 1, round(loss.item(), 5), '{} s'.format(int(time.time() - since))) 109 | rand_ind = torch.randint(0, img_input.shape[0], size=(1,)) 110 | wandb.log( 111 | {'Image': [wandb.Image(img_input[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input"), 112 | wandb.Image(img_tar[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Target"), 113 | wandb.Image(outputs[0][rand_ind.item()].permute(1, 2, 0).detach().cpu().numpy(), 114 | caption="Output")], 115 | }) 116 | return model 117 | 118 | 119 | 120 | def train_adv_prototypical(model, img, n_imgs, n_decoders, n_iters,prototype_ind_csv_writer, optimizer,train_loss, iter_ind, fgsm_step): 121 | 122 | """ 123 | Training using prototypical reconstruction approach incorporated with free adversarial training 124 | 125 | :param img: Images to train each substitute model ( default: 10 images each from 2 classes) [20 x 3 x 224 x 224] 126 | :param n_imgs: Number of reference images (10 from each class) 127 | :param n_decoders: Number of decoders (default 20) [output size 56 x 56] 128 | :param n_iters: Number of iterations for training each substitute model 129 | :param prototype_ind_csv_writer: To write the prototype pairs for the given autoencoder 130 | :param train_loss: dictionary for saving loss for each autoencoder 131 | :param fgsm_step: perturbation budget for generating adversarial example 132 | :return: returns the substitute model 133 | """ 134 | 135 | 136 | do_aug = True 137 | if n_imgs == 1: 138 | tar_ind_ls = [0, 1] 139 | else: 140 | tar_ind_ls = mk_proto_ls(n_imgs) 141 | tar_ind_ls = tar_ind_ls[:n_decoders * 2] 142 | # get the first 20 pairs (first 10 pairs are same always [0, 10, 1, 11, .........., 8, 18, 9, 19]) 143 | prototype_ind_csv_writer.writerow(tar_ind_ls.tolist()) # save the 20 pairs used in this model 144 | 145 | 146 | img_tar = img[tar_ind_ls] 147 | if n_decoders != 1: 148 | img_tar = F.interpolate(img_tar, (56, 56)) 149 | since = time.time() 150 | train_loss[iter_ind] = [] 151 | attack = FGSM(model, eps=fgsm_step, mode="proto") 152 | for i in range(n_iters): 153 | rand_ind = torch.cat((torch.randint(0, n_imgs, size=(1,)), torch.randint(n_imgs, 2 * n_imgs, size=(1,)))) 154 | # get random indices of the 2 images to be chosen from two different classes 155 | img_input = img[rand_ind].clone() 156 | if do_aug: 157 | img_input = aug(img_input) 158 | assert img_input.shape[3] == 224 159 | adv_images = attack(img_input, img_tar) 160 | 161 | 162 | clean_output, clean_enc, _ = model(img_input) 163 | clean_output = torch.cat(clean_output, dim=0) 164 | adv_output, adv_enc, _ = model(adv_images) 165 | adv_output = torch.cat(adv_output, dim=0) 166 | loss_clean = nn.MSELoss()(clean_output, img_tar) 167 | loss_adv = nn.MSELoss()(adv_output, img_tar) 168 | sim_loss = nn.MSELoss()(adv_enc, clean_enc) 169 | 170 | 171 | loss = loss_clean + loss_adv + sim_loss 172 | optimizer.zero_grad() 173 | loss.backward() 174 | optimizer.step() 175 | 176 | train_loss[iter_ind].append(loss.item()) 177 | 178 | if (i + 1) % 500 == 0: 179 | print( 180 | f"{iter_ind + 1}, {i + 1}, Total Loss {round(loss.item(), 5)}, Sim Loss {round(sim_loss.item(), 5)}, {int(time.time() - since)} s") 181 | rand_ind = torch.randint(0, img_input.shape[0], size=(1,)) 182 | wandb.log( 183 | {'Image': [wandb.Image(img_input[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input-Clean"), 184 | wandb.Image(adv_images[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input-Adv"), 185 | ], 186 | }) 187 | 188 | return model 189 | 190 | def train_unsup(model, img, n_iters,optimizer,args, train_loss, iter_ind): 191 | """ 192 | Training using self supervised (rotation/jigsaw/masking) approach 193 | 194 | :param img: Images to train each substitute model ( 10 images each from 2 classes) [20 x 3 x 224 x 224] 195 | :param n_iters: Number of iterations for training each substitute model 196 | :param train_loss: dictionary for storing the loss for each autoencoder 197 | :param iter_ind: the corresponding batch number on which autoencoder is being trained 198 | :return: returns the substitute model trained on images from 2 classes 199 | """ 200 | img_input = img 201 | img_tar = img.clone() 202 | since = time.time() 203 | train_loss[iter_ind] = [] 204 | for i in range(n_iters): 205 | for img_ind in range(img_input.shape[0]): 206 | if args.mode == 'rotate': 207 | img_input[img_ind:img_ind + 1] = rot(img_input[img_ind:img_ind + 1]) 208 | elif args.mode == 'jigsaw': 209 | img_input[img_ind] = shuffle(img_input[img_ind], 1) 210 | else: 211 | sys.exit("Enter the correct mode") 212 | 213 | 214 | outputs, _,_ = model(img_input) 215 | loss = nn.MSELoss()(outputs[0], img_tar) 216 | optimizer.zero_grad() 217 | loss.backward() 218 | optimizer.step() 219 | 220 | train_loss[iter_ind].append(loss.item()) 221 | if (i + 1) % 500 == 0: 222 | print(iter_ind + 1, i + 1, round(loss.item(), 5), '{} s'.format(int(time.time() - since))) 223 | rand_ind = torch.randint(0, img_input.shape[0], size=(1,)) 224 | wandb.log( 225 | {'Image': [wandb.Image(img_input[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input"), 226 | wandb.Image(img_tar[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Target"), 227 | wandb.Image(outputs[0][rand_ind.item()].permute(1, 2, 0).detach().cpu().numpy(), 228 | caption="Output")], 229 | }) 230 | 231 | return model 232 | 233 | 234 | def train_adv_unsup(model, img, n_iters, optimizer, args,train_loss, iter_ind, fgsm_step): 235 | """ 236 | Training using self supervised (rotation/jigsaw/masking) approach incorporated with free adversarial training 237 | 238 | :param img: Images to train each substitute model ( 10 images each from 2 classes) [20 x 3 x 224 x 224] 239 | :param n_iters: Number of iterations for training each substitute model 240 | :param train_loss: dictionary for storing the loss for each autoencoder 241 | :param iter_ind: the corresponding batch number on which autoencoder is being trained 242 | :param fgsm_step: perturbation budget for generating adversarial example 243 | :return: returns the substitute model 244 | """ 245 | img_input = img 246 | img_tar = img.clone() 247 | since = time.time() 248 | train_loss[iter_ind] = [] 249 | attack = FGSM(model, eps=fgsm_step) 250 | 251 | for i in range(n_iters): 252 | for img_ind in range(img_input.shape[0]): 253 | if args.mode == 'rotate': 254 | img_input[img_ind:img_ind + 1] = rot(img_input[img_ind:img_ind + 1]) 255 | elif args.mode == 'jigsaw': 256 | img_input[img_ind] = shuffle(img_input[img_ind], 1) 257 | else: 258 | sys.exit("Enter the correct mode") 259 | 260 | 261 | adv_images = attack(img_input, img_tar) 262 | 263 | clean_output, clean_enc, _ = model(img_input) 264 | adv_output, adv_enc, _ = model(adv_images) 265 | loss_clean = nn.MSELoss()(clean_output[0], img_tar) 266 | loss_adv = nn.MSELoss()(adv_output[0], img_tar) 267 | sim_loss = nn.MSELoss()(adv_enc, clean_enc) 268 | 269 | loss = loss_clean + loss_adv + sim_loss 270 | optimizer.zero_grad() 271 | loss.backward() 272 | optimizer.step() 273 | 274 | train_loss[iter_ind].append(loss.item()) 275 | if (i + 1) % 500 == 0: 276 | print(f"{iter_ind + 1}, {i+1}, Total Loss {round(loss.item(), 5)}, Sim Loss {round(sim_loss.item(), 5)}, {int(time.time() - since)} s") 277 | rand_ind = torch.randint(0, img_input.shape[0], size=(1,)) 278 | wandb.log( 279 | {'Image': [wandb.Image(img_input[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input-Clean"), 280 | wandb.Image(adv_images[rand_ind.item()].permute(1, 2, 0).cpu().numpy(), caption="Input-Adv"), 281 | ], 282 | }) 283 | 284 | return model 285 | 286 | if __name__ == '__main__': 287 | 288 | """ 289 | Training of multiple autoencoders using rotation, jigsaw, and prototypical methods. 290 | Default : Each autoencoder is trained on only 20 images (10 from each class) using the above methods. 291 | In total 250 autoencoders are trained on a subset 0f 5000 images (10 from each class) from ImageNetval. 292 | """ 293 | 294 | args = parser.parse_args() 295 | wandb.init(project=args.project, entity=args.entity, mode=args.wandb_mode, name=args.save_dir.split("/")[-1]) 296 | 297 | cudnn.benchmark = False 298 | cudnn.deterministic = True 299 | SEED = 0 300 | torch.manual_seed(SEED) 301 | torch.cuda.manual_seed(SEED) 302 | np.random.seed(SEED) 303 | print(args) 304 | config = wandb.config 305 | config.update(args) 306 | 307 | mode = args.mode 308 | assert mode in ['prototypical', 'unsup_naive', 'jigsaw', 'rotate', 'mask'] 309 | save_dir = args.save_dir 310 | n_imgs = args.n_imgs // 2 311 | n_iters = args.n_iters 312 | lr = args.lr 313 | # only in prototypical methods there are more than 1 decoders in the autoencoder 314 | if mode != 'prototypical': 315 | n_decoders = 1 316 | else: 317 | n_decoders = args.n_decoders 318 | assert n_decoders <= n_imgs**2, 'Too many decoders.' 319 | 320 | config.update({'n_decoders': n_decoders}, allow_val_change=True) 321 | 322 | os.makedirs(save_dir+'/models', exist_ok=True) # create directory to save the trained models 323 | create_json(args) 324 | 325 | if torch.cuda.is_available(): 326 | device = torch.device('cuda') 327 | else: 328 | device = torch.device('cpu') 329 | 330 | batch_size = n_imgs*2 331 | config.update({'batch_size': batch_size}) 332 | do_aug = True 333 | 334 | trans = T.Compose([ # Normalisation of images is done in the first layer of the autoencoder 335 | T.Resize((256,256)), 336 | T.CenterCrop(224), 337 | T.ToTensor() 338 | ]) 339 | 340 | dataset = OUR_dataset(data_dir='data/ILSVRC2012_img_val', 341 | data_csv_dir='data/selected_data.csv', # from the selected 500 classes of ImageNet val, the dataset loads the first 10 images from each class. 342 | mode='train', 343 | img_num=n_imgs, 344 | transform=trans, 345 | ) 346 | 347 | data_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers = 1) 348 | 349 | 350 | if mode == 'prototypical': 351 | prototype_ind_csv = open(save_dir+'/prototype_ind.csv', 'a', newline='') # create a csv file to save prototypes used during the training of each autoencoder. 352 | prototype_ind_csv_writer = csv.writer(prototype_ind_csv) 353 | 354 | fig, ax = plt.subplots() 355 | for iter_ind, (img, label_ind) in enumerate(data_loader): 356 | 357 | if not args.start <= iter_ind < args.end: 358 | continue 359 | 360 | model = initialize_model(n_decoders) 361 | model.train() 362 | optimizer = torch.optim.Adam(model.parameters(), lr=lr) 363 | img = img.to(device) 364 | 365 | 366 | train_loss = {} 367 | if mode == 'prototypical': 368 | if args.adv_train: 369 | 370 | train_adv_prototypical(model, img, n_imgs, n_decoders, n_iters, prototype_ind_csv_writer=prototype_ind_csv_writer, 371 | optimizer=optimizer, train_loss=train_loss, iter_ind=iter_ind, fgsm_step=args.fgsm_step / 255.0) 372 | else: 373 | train_prototypical(model, img, n_imgs, n_decoders, n_iters, prototype_ind_csv_writer=prototype_ind_csv_writer, 374 | optimizer=optimizer, train_loss=train_loss, iter_ind=iter_ind) 375 | 376 | 377 | else: 378 | 379 | if args.adv_train: 380 | train_adv_unsup(model, img, n_iters, optimizer, args, train_loss=train_loss, iter_ind=iter_ind,fgsm_step=args.fgsm_step / 255.0) 381 | else: 382 | train_unsup(model, img, n_iters, optimizer, args, train_loss=train_loss, iter_ind=iter_ind) 383 | 384 | model.eval() 385 | 386 | torch.save(model.state_dict(), save_dir + f'/models/{args.mode}_{iter_ind}.pth') 387 | 388 | xs = [x for x in range(len(train_loss[iter_ind]))] 389 | ax.plot(xs, train_loss[iter_ind], label=f"Model_{iter_ind}") 390 | ax.set_xlabel("Iterations") 391 | ax.set_ylabel("Loss") 392 | ax.set_title(f"Model_{mode}_{iter_ind}") 393 | wandb.log({f'plot': ax}) 394 | 395 | -------------------------------------------------------------------------------- /imagenet_class_index.json: -------------------------------------------------------------------------------- 1 | {"0": ["n01440764", "tench"], "1": ["n01443537", "goldfish"], "2": ["n01484850", "great_white_shark"], "3": ["n01491361", "tiger_shark"], "4": ["n01494475", "hammerhead"], "5": ["n01496331", "electric_ray"], "6": ["n01498041", "stingray"], "7": ["n01514668", "cock"], "8": ["n01514859", "hen"], "9": ["n01518878", "ostrich"], "10": ["n01530575", "brambling"], "11": ["n01531178", "goldfinch"], "12": ["n01532829", "house_finch"], "13": ["n01534433", "junco"], "14": ["n01537544", "indigo_bunting"], "15": ["n01558993", "robin"], "16": ["n01560419", "bulbul"], "17": ["n01580077", "jay"], "18": ["n01582220", "magpie"], "19": ["n01592084", "chickadee"], "20": ["n01601694", "water_ouzel"], "21": ["n01608432", "kite"], "22": ["n01614925", "bald_eagle"], "23": ["n01616318", "vulture"], "24": ["n01622779", "great_grey_owl"], "25": ["n01629819", "European_fire_salamander"], "26": ["n01630670", "common_newt"], "27": ["n01631663", "eft"], "28": ["n01632458", "spotted_salamander"], "29": ["n01632777", "axolotl"], "30": ["n01641577", "bullfrog"], "31": ["n01644373", "tree_frog"], "32": ["n01644900", "tailed_frog"], "33": ["n01664065", "loggerhead"], "34": ["n01665541", "leatherback_turtle"], "35": ["n01667114", "mud_turtle"], "36": ["n01667778", "terrapin"], "37": ["n01669191", "box_turtle"], "38": ["n01675722", "banded_gecko"], "39": ["n01677366", "common_iguana"], "40": ["n01682714", "American_chameleon"], "41": ["n01685808", "whiptail"], "42": ["n01687978", "agama"], "43": ["n01688243", "frilled_lizard"], "44": ["n01689811", "alligator_lizard"], "45": ["n01692333", "Gila_monster"], "46": ["n01693334", "green_lizard"], "47": ["n01694178", "African_chameleon"], "48": ["n01695060", "Komodo_dragon"], "49": ["n01697457", "African_crocodile"], "50": ["n01698640", "American_alligator"], "51": ["n01704323", "triceratops"], "52": ["n01728572", "thunder_snake"], "53": ["n01728920", "ringneck_snake"], "54": ["n01729322", "hognose_snake"], "55": ["n01729977", "green_snake"], "56": ["n01734418", "king_snake"], "57": ["n01735189", "garter_snake"], "58": ["n01737021", "water_snake"], "59": ["n01739381", "vine_snake"], "60": ["n01740131", "night_snake"], "61": ["n01742172", "boa_constrictor"], "62": ["n01744401", "rock_python"], "63": ["n01748264", "Indian_cobra"], "64": ["n01749939", "green_mamba"], "65": ["n01751748", "sea_snake"], "66": ["n01753488", "horned_viper"], "67": ["n01755581", "diamondback"], "68": ["n01756291", "sidewinder"], "69": ["n01768244", "trilobite"], "70": ["n01770081", "harvestman"], "71": ["n01770393", "scorpion"], "72": ["n01773157", "black_and_gold_garden_spider"], "73": ["n01773549", "barn_spider"], "74": ["n01773797", "garden_spider"], "75": ["n01774384", "black_widow"], "76": ["n01774750", "tarantula"], "77": ["n01775062", "wolf_spider"], "78": ["n01776313", "tick"], "79": ["n01784675", "centipede"], "80": ["n01795545", "black_grouse"], "81": ["n01796340", "ptarmigan"], "82": ["n01797886", "ruffed_grouse"], "83": ["n01798484", "prairie_chicken"], "84": ["n01806143", "peacock"], "85": ["n01806567", "quail"], "86": ["n01807496", "partridge"], "87": ["n01817953", "African_grey"], "88": ["n01818515", "macaw"], "89": ["n01819313", "sulphur-crested_cockatoo"], "90": ["n01820546", "lorikeet"], "91": ["n01824575", "coucal"], "92": ["n01828970", "bee_eater"], "93": ["n01829413", "hornbill"], "94": ["n01833805", "hummingbird"], "95": ["n01843065", "jacamar"], "96": ["n01843383", "toucan"], "97": ["n01847000", "drake"], "98": ["n01855032", "red-breasted_merganser"], "99": ["n01855672", "goose"], "100": ["n01860187", "black_swan"], "101": ["n01871265", "tusker"], "102": ["n01872401", "echidna"], "103": ["n01873310", "platypus"], "104": ["n01877812", "wallaby"], "105": ["n01882714", "koala"], "106": ["n01883070", "wombat"], "107": ["n01910747", "jellyfish"], "108": ["n01914609", "sea_anemone"], "109": ["n01917289", "brain_coral"], "110": ["n01924916", "flatworm"], "111": ["n01930112", "nematode"], "112": ["n01943899", "conch"], "113": ["n01944390", "snail"], "114": ["n01945685", "slug"], "115": ["n01950731", "sea_slug"], "116": ["n01955084", "chiton"], "117": ["n01968897", "chambered_nautilus"], "118": ["n01978287", "Dungeness_crab"], "119": ["n01978455", "rock_crab"], "120": ["n01980166", "fiddler_crab"], "121": ["n01981276", "king_crab"], "122": ["n01983481", "American_lobster"], "123": ["n01984695", "spiny_lobster"], "124": ["n01985128", "crayfish"], "125": ["n01986214", "hermit_crab"], "126": ["n01990800", "isopod"], "127": ["n02002556", "white_stork"], "128": ["n02002724", "black_stork"], "129": ["n02006656", "spoonbill"], "130": ["n02007558", "flamingo"], "131": ["n02009229", "little_blue_heron"], "132": ["n02009912", "American_egret"], "133": ["n02011460", "bittern"], "134": ["n02012849", "crane"], "135": ["n02013706", "limpkin"], "136": ["n02017213", "European_gallinule"], "137": ["n02018207", "American_coot"], "138": ["n02018795", "bustard"], "139": ["n02025239", "ruddy_turnstone"], "140": ["n02027492", "red-backed_sandpiper"], "141": ["n02028035", "redshank"], "142": ["n02033041", "dowitcher"], "143": ["n02037110", "oystercatcher"], "144": ["n02051845", "pelican"], "145": ["n02056570", "king_penguin"], "146": ["n02058221", "albatross"], "147": ["n02066245", "grey_whale"], "148": ["n02071294", "killer_whale"], "149": ["n02074367", "dugong"], "150": ["n02077923", "sea_lion"], "151": ["n02085620", "Chihuahua"], "152": ["n02085782", "Japanese_spaniel"], "153": ["n02085936", "Maltese_dog"], "154": ["n02086079", "Pekinese"], "155": ["n02086240", "Shih-Tzu"], "156": ["n02086646", "Blenheim_spaniel"], "157": ["n02086910", "papillon"], "158": ["n02087046", "toy_terrier"], "159": ["n02087394", "Rhodesian_ridgeback"], "160": ["n02088094", "Afghan_hound"], "161": ["n02088238", "basset"], "162": ["n02088364", "beagle"], "163": ["n02088466", "bloodhound"], "164": ["n02088632", "bluetick"], "165": ["n02089078", "black-and-tan_coonhound"], "166": ["n02089867", "Walker_hound"], "167": ["n02089973", "English_foxhound"], "168": ["n02090379", "redbone"], "169": ["n02090622", "borzoi"], "170": ["n02090721", "Irish_wolfhound"], "171": ["n02091032", "Italian_greyhound"], "172": ["n02091134", "whippet"], "173": ["n02091244", "Ibizan_hound"], "174": ["n02091467", "Norwegian_elkhound"], "175": ["n02091635", "otterhound"], "176": ["n02091831", "Saluki"], "177": ["n02092002", "Scottish_deerhound"], "178": ["n02092339", "Weimaraner"], "179": ["n02093256", "Staffordshire_bullterrier"], "180": ["n02093428", "American_Staffordshire_terrier"], "181": ["n02093647", "Bedlington_terrier"], "182": ["n02093754", "Border_terrier"], "183": ["n02093859", "Kerry_blue_terrier"], "184": ["n02093991", "Irish_terrier"], "185": ["n02094114", "Norfolk_terrier"], "186": ["n02094258", "Norwich_terrier"], "187": ["n02094433", "Yorkshire_terrier"], "188": ["n02095314", "wire-haired_fox_terrier"], "189": ["n02095570", "Lakeland_terrier"], "190": ["n02095889", "Sealyham_terrier"], "191": ["n02096051", "Airedale"], "192": ["n02096177", "cairn"], "193": ["n02096294", "Australian_terrier"], "194": ["n02096437", "Dandie_Dinmont"], "195": ["n02096585", "Boston_bull"], "196": ["n02097047", "miniature_schnauzer"], "197": ["n02097130", "giant_schnauzer"], "198": ["n02097209", "standard_schnauzer"], "199": ["n02097298", "Scotch_terrier"], "200": ["n02097474", "Tibetan_terrier"], "201": ["n02097658", "silky_terrier"], "202": ["n02098105", "soft-coated_wheaten_terrier"], "203": ["n02098286", "West_Highland_white_terrier"], "204": ["n02098413", "Lhasa"], "205": ["n02099267", "flat-coated_retriever"], "206": ["n02099429", "curly-coated_retriever"], "207": ["n02099601", "golden_retriever"], "208": ["n02099712", "Labrador_retriever"], "209": ["n02099849", "Chesapeake_Bay_retriever"], "210": ["n02100236", "German_short-haired_pointer"], "211": ["n02100583", "vizsla"], "212": ["n02100735", "English_setter"], "213": ["n02100877", "Irish_setter"], "214": ["n02101006", "Gordon_setter"], "215": ["n02101388", "Brittany_spaniel"], "216": ["n02101556", "clumber"], "217": ["n02102040", "English_springer"], "218": ["n02102177", "Welsh_springer_spaniel"], "219": ["n02102318", "cocker_spaniel"], "220": ["n02102480", "Sussex_spaniel"], "221": ["n02102973", "Irish_water_spaniel"], "222": ["n02104029", "kuvasz"], "223": ["n02104365", "schipperke"], "224": ["n02105056", "groenendael"], "225": ["n02105162", "malinois"], "226": ["n02105251", "briard"], "227": ["n02105412", "kelpie"], "228": ["n02105505", "komondor"], "229": ["n02105641", "Old_English_sheepdog"], "230": ["n02105855", "Shetland_sheepdog"], "231": ["n02106030", "collie"], "232": ["n02106166", "Border_collie"], "233": ["n02106382", "Bouvier_des_Flandres"], "234": ["n02106550", "Rottweiler"], "235": ["n02106662", "German_shepherd"], "236": ["n02107142", "Doberman"], "237": ["n02107312", "miniature_pinscher"], "238": ["n02107574", "Greater_Swiss_Mountain_dog"], "239": ["n02107683", "Bernese_mountain_dog"], "240": ["n02107908", "Appenzeller"], "241": ["n02108000", "EntleBucher"], "242": ["n02108089", "boxer"], "243": ["n02108422", "bull_mastiff"], "244": ["n02108551", "Tibetan_mastiff"], "245": ["n02108915", "French_bulldog"], "246": ["n02109047", "Great_Dane"], "247": ["n02109525", "Saint_Bernard"], "248": ["n02109961", "Eskimo_dog"], "249": ["n02110063", "malamute"], "250": ["n02110185", "Siberian_husky"], "251": ["n02110341", "dalmatian"], "252": ["n02110627", "affenpinscher"], "253": ["n02110806", "basenji"], "254": ["n02110958", "pug"], "255": ["n02111129", "Leonberg"], "256": ["n02111277", "Newfoundland"], "257": ["n02111500", "Great_Pyrenees"], "258": ["n02111889", "Samoyed"], "259": ["n02112018", "Pomeranian"], "260": ["n02112137", "chow"], "261": ["n02112350", "keeshond"], "262": ["n02112706", "Brabancon_griffon"], "263": ["n02113023", "Pembroke"], "264": ["n02113186", "Cardigan"], "265": ["n02113624", "toy_poodle"], "266": ["n02113712", "miniature_poodle"], "267": ["n02113799", "standard_poodle"], "268": ["n02113978", "Mexican_hairless"], "269": ["n02114367", "timber_wolf"], "270": ["n02114548", "white_wolf"], "271": ["n02114712", "red_wolf"], "272": ["n02114855", "coyote"], "273": ["n02115641", "dingo"], "274": ["n02115913", "dhole"], "275": ["n02116738", "African_hunting_dog"], "276": ["n02117135", "hyena"], "277": ["n02119022", "red_fox"], "278": ["n02119789", "kit_fox"], "279": ["n02120079", "Arctic_fox"], "280": ["n02120505", "grey_fox"], "281": ["n02123045", "tabby"], "282": ["n02123159", "tiger_cat"], "283": ["n02123394", "Persian_cat"], "284": ["n02123597", "Siamese_cat"], "285": ["n02124075", "Egyptian_cat"], "286": ["n02125311", "cougar"], "287": ["n02127052", "lynx"], "288": ["n02128385", "leopard"], "289": ["n02128757", "snow_leopard"], "290": ["n02128925", "jaguar"], "291": ["n02129165", "lion"], "292": ["n02129604", "tiger"], "293": ["n02130308", "cheetah"], "294": ["n02132136", "brown_bear"], "295": ["n02133161", "American_black_bear"], "296": ["n02134084", "ice_bear"], "297": ["n02134418", "sloth_bear"], "298": ["n02137549", "mongoose"], "299": ["n02138441", "meerkat"], "300": ["n02165105", "tiger_beetle"], "301": ["n02165456", "ladybug"], "302": ["n02167151", "ground_beetle"], "303": ["n02168699", "long-horned_beetle"], "304": ["n02169497", "leaf_beetle"], "305": ["n02172182", "dung_beetle"], "306": ["n02174001", "rhinoceros_beetle"], "307": ["n02177972", "weevil"], "308": ["n02190166", "fly"], "309": ["n02206856", "bee"], "310": ["n02219486", "ant"], "311": ["n02226429", "grasshopper"], "312": ["n02229544", "cricket"], "313": ["n02231487", "walking_stick"], "314": ["n02233338", "cockroach"], "315": ["n02236044", "mantis"], "316": ["n02256656", "cicada"], "317": ["n02259212", "leafhopper"], "318": ["n02264363", "lacewing"], "319": ["n02268443", "dragonfly"], "320": ["n02268853", "damselfly"], "321": ["n02276258", "admiral"], "322": ["n02277742", "ringlet"], "323": ["n02279972", "monarch"], "324": ["n02280649", "cabbage_butterfly"], "325": ["n02281406", "sulphur_butterfly"], "326": ["n02281787", "lycaenid"], "327": ["n02317335", "starfish"], "328": ["n02319095", "sea_urchin"], "329": ["n02321529", "sea_cucumber"], "330": ["n02325366", "wood_rabbit"], "331": ["n02326432", "hare"], "332": ["n02328150", "Angora"], "333": ["n02342885", "hamster"], "334": ["n02346627", "porcupine"], "335": ["n02356798", "fox_squirrel"], "336": ["n02361337", "marmot"], "337": ["n02363005", "beaver"], "338": ["n02364673", "guinea_pig"], "339": ["n02389026", "sorrel"], "340": ["n02391049", "zebra"], "341": ["n02395406", "hog"], "342": ["n02396427", "wild_boar"], "343": ["n02397096", "warthog"], "344": ["n02398521", "hippopotamus"], "345": ["n02403003", "ox"], "346": ["n02408429", "water_buffalo"], "347": ["n02410509", "bison"], "348": ["n02412080", "ram"], "349": ["n02415577", "bighorn"], "350": ["n02417914", "ibex"], "351": ["n02422106", "hartebeest"], "352": ["n02422699", "impala"], "353": ["n02423022", "gazelle"], "354": ["n02437312", "Arabian_camel"], "355": ["n02437616", "llama"], "356": ["n02441942", "weasel"], "357": ["n02442845", "mink"], "358": ["n02443114", "polecat"], "359": ["n02443484", "black-footed_ferret"], "360": ["n02444819", "otter"], "361": ["n02445715", "skunk"], "362": ["n02447366", "badger"], "363": ["n02454379", "armadillo"], "364": ["n02457408", "three-toed_sloth"], "365": ["n02480495", "orangutan"], "366": ["n02480855", "gorilla"], "367": ["n02481823", "chimpanzee"], "368": ["n02483362", "gibbon"], "369": ["n02483708", "siamang"], "370": ["n02484975", "guenon"], "371": ["n02486261", "patas"], "372": ["n02486410", "baboon"], "373": ["n02487347", "macaque"], "374": ["n02488291", "langur"], "375": ["n02488702", "colobus"], "376": ["n02489166", "proboscis_monkey"], "377": ["n02490219", "marmoset"], "378": ["n02492035", "capuchin"], "379": ["n02492660", "howler_monkey"], "380": ["n02493509", "titi"], "381": ["n02493793", "spider_monkey"], "382": ["n02494079", "squirrel_monkey"], "383": ["n02497673", "Madagascar_cat"], "384": ["n02500267", "indri"], "385": ["n02504013", "Indian_elephant"], "386": ["n02504458", "African_elephant"], "387": ["n02509815", "lesser_panda"], "388": ["n02510455", "giant_panda"], "389": ["n02514041", "barracouta"], "390": ["n02526121", "eel"], "391": ["n02536864", "coho"], "392": ["n02606052", "rock_beauty"], "393": ["n02607072", "anemone_fish"], "394": ["n02640242", "sturgeon"], "395": ["n02641379", "gar"], "396": ["n02643566", "lionfish"], "397": ["n02655020", "puffer"], "398": ["n02666196", "abacus"], "399": ["n02667093", "abaya"], "400": ["n02669723", "academic_gown"], "401": ["n02672831", "accordion"], "402": ["n02676566", "acoustic_guitar"], "403": ["n02687172", "aircraft_carrier"], "404": ["n02690373", "airliner"], "405": ["n02692877", "airship"], "406": ["n02699494", "altar"], "407": ["n02701002", "ambulance"], "408": ["n02704792", "amphibian"], "409": ["n02708093", "analog_clock"], "410": ["n02727426", "apiary"], "411": ["n02730930", "apron"], "412": ["n02747177", "ashcan"], "413": ["n02749479", "assault_rifle"], "414": ["n02769748", "backpack"], "415": ["n02776631", "bakery"], "416": ["n02777292", "balance_beam"], "417": ["n02782093", "balloon"], "418": ["n02783161", "ballpoint"], "419": ["n02786058", "Band_Aid"], "420": ["n02787622", "banjo"], "421": ["n02788148", "bannister"], "422": ["n02790996", "barbell"], "423": ["n02791124", "barber_chair"], "424": ["n02791270", "barbershop"], "425": ["n02793495", "barn"], "426": ["n02794156", "barometer"], "427": ["n02795169", "barrel"], "428": ["n02797295", "barrow"], "429": ["n02799071", "baseball"], "430": ["n02802426", "basketball"], "431": ["n02804414", "bassinet"], "432": ["n02804610", "bassoon"], "433": ["n02807133", "bathing_cap"], "434": ["n02808304", "bath_towel"], "435": ["n02808440", "bathtub"], "436": ["n02814533", "beach_wagon"], "437": ["n02814860", "beacon"], "438": ["n02815834", "beaker"], "439": ["n02817516", "bearskin"], "440": ["n02823428", "beer_bottle"], "441": ["n02823750", "beer_glass"], "442": ["n02825657", "bell_cote"], "443": ["n02834397", "bib"], "444": ["n02835271", "bicycle-built-for-two"], "445": ["n02837789", "bikini"], "446": ["n02840245", "binder"], "447": ["n02841315", "binoculars"], "448": ["n02843684", "birdhouse"], "449": ["n02859443", "boathouse"], "450": ["n02860847", "bobsled"], "451": ["n02865351", "bolo_tie"], "452": ["n02869837", "bonnet"], "453": ["n02870880", "bookcase"], "454": ["n02871525", "bookshop"], "455": ["n02877765", "bottlecap"], "456": ["n02879718", "bow"], "457": ["n02883205", "bow_tie"], "458": ["n02892201", "brass"], "459": ["n02892767", "brassiere"], "460": ["n02894605", "breakwater"], "461": ["n02895154", "breastplate"], "462": ["n02906734", "broom"], "463": ["n02909870", "bucket"], "464": ["n02910353", "buckle"], "465": ["n02916936", "bulletproof_vest"], "466": ["n02917067", "bullet_train"], "467": ["n02927161", "butcher_shop"], "468": ["n02930766", "cab"], "469": ["n02939185", "caldron"], "470": ["n02948072", "candle"], "471": ["n02950826", "cannon"], "472": ["n02951358", "canoe"], "473": ["n02951585", "can_opener"], "474": ["n02963159", "cardigan"], "475": ["n02965783", "car_mirror"], "476": ["n02966193", "carousel"], "477": ["n02966687", "carpenter's_kit"], "478": ["n02971356", "carton"], "479": ["n02974003", "car_wheel"], "480": ["n02977058", "cash_machine"], "481": ["n02978881", "cassette"], "482": ["n02979186", "cassette_player"], "483": ["n02980441", "castle"], "484": ["n02981792", "catamaran"], "485": ["n02988304", "CD_player"], "486": ["n02992211", "cello"], "487": ["n02992529", "cellular_telephone"], "488": ["n02999410", "chain"], "489": ["n03000134", "chainlink_fence"], "490": ["n03000247", "chain_mail"], "491": ["n03000684", "chain_saw"], "492": ["n03014705", "chest"], "493": ["n03016953", "chiffonier"], "494": ["n03017168", "chime"], "495": ["n03018349", "china_cabinet"], "496": ["n03026506", "Christmas_stocking"], "497": ["n03028079", "church"], "498": ["n03032252", "cinema"], "499": ["n03041632", "cleaver"], "500": ["n03042490", "cliff_dwelling"], "501": ["n03045698", "cloak"], "502": ["n03047690", "clog"], "503": ["n03062245", "cocktail_shaker"], "504": ["n03063599", "coffee_mug"], "505": ["n03063689", "coffeepot"], "506": ["n03065424", "coil"], "507": ["n03075370", "combination_lock"], "508": ["n03085013", "computer_keyboard"], "509": ["n03089624", "confectionery"], "510": ["n03095699", "container_ship"], "511": ["n03100240", "convertible"], "512": ["n03109150", "corkscrew"], "513": ["n03110669", "cornet"], "514": ["n03124043", "cowboy_boot"], "515": ["n03124170", "cowboy_hat"], "516": ["n03125729", "cradle"], "517": ["n03126707", "crane"], "518": ["n03127747", "crash_helmet"], "519": ["n03127925", "crate"], "520": ["n03131574", "crib"], "521": ["n03133878", "Crock_Pot"], "522": ["n03134739", "croquet_ball"], "523": ["n03141823", "crutch"], "524": ["n03146219", "cuirass"], "525": ["n03160309", "dam"], "526": ["n03179701", "desk"], "527": ["n03180011", "desktop_computer"], "528": ["n03187595", "dial_telephone"], "529": ["n03188531", "diaper"], "530": ["n03196217", "digital_clock"], "531": ["n03197337", "digital_watch"], "532": ["n03201208", "dining_table"], "533": ["n03207743", "dishrag"], "534": ["n03207941", "dishwasher"], "535": ["n03208938", "disk_brake"], "536": ["n03216828", "dock"], "537": ["n03218198", "dogsled"], "538": ["n03220513", "dome"], "539": ["n03223299", "doormat"], "540": ["n03240683", "drilling_platform"], "541": ["n03249569", "drum"], "542": ["n03250847", "drumstick"], "543": ["n03255030", "dumbbell"], "544": ["n03259280", "Dutch_oven"], "545": ["n03271574", "electric_fan"], "546": ["n03272010", "electric_guitar"], "547": ["n03272562", "electric_locomotive"], "548": ["n03290653", "entertainment_center"], "549": ["n03291819", "envelope"], "550": ["n03297495", "espresso_maker"], "551": ["n03314780", "face_powder"], "552": ["n03325584", "feather_boa"], "553": ["n03337140", "file"], "554": ["n03344393", "fireboat"], "555": ["n03345487", "fire_engine"], "556": ["n03347037", "fire_screen"], "557": ["n03355925", "flagpole"], "558": ["n03372029", "flute"], "559": ["n03376595", "folding_chair"], "560": ["n03379051", "football_helmet"], "561": ["n03384352", "forklift"], "562": ["n03388043", "fountain"], "563": ["n03388183", "fountain_pen"], "564": ["n03388549", "four-poster"], "565": ["n03393912", "freight_car"], "566": ["n03394916", "French_horn"], "567": ["n03400231", "frying_pan"], "568": ["n03404251", "fur_coat"], "569": ["n03417042", "garbage_truck"], "570": ["n03424325", "gasmask"], "571": ["n03425413", "gas_pump"], "572": ["n03443371", "goblet"], "573": ["n03444034", "go-kart"], "574": ["n03445777", "golf_ball"], "575": ["n03445924", "golfcart"], "576": ["n03447447", "gondola"], "577": ["n03447721", "gong"], "578": ["n03450230", "gown"], "579": ["n03452741", "grand_piano"], "580": ["n03457902", "greenhouse"], "581": ["n03459775", "grille"], "582": ["n03461385", "grocery_store"], "583": ["n03467068", "guillotine"], "584": ["n03476684", "hair_slide"], "585": ["n03476991", "hair_spray"], "586": ["n03478589", "half_track"], "587": ["n03481172", "hammer"], "588": ["n03482405", "hamper"], "589": ["n03483316", "hand_blower"], "590": ["n03485407", "hand-held_computer"], "591": ["n03485794", "handkerchief"], "592": ["n03492542", "hard_disc"], "593": ["n03494278", "harmonica"], "594": ["n03495258", "harp"], "595": ["n03496892", "harvester"], "596": ["n03498962", "hatchet"], "597": ["n03527444", "holster"], "598": ["n03529860", "home_theater"], "599": ["n03530642", "honeycomb"], "600": ["n03532672", "hook"], "601": ["n03534580", "hoopskirt"], "602": ["n03535780", "horizontal_bar"], "603": ["n03538406", "horse_cart"], "604": ["n03544143", "hourglass"], "605": ["n03584254", "iPod"], "606": ["n03584829", "iron"], "607": ["n03590841", "jack-o'-lantern"], "608": ["n03594734", "jean"], "609": ["n03594945", "jeep"], "610": ["n03595614", "jersey"], "611": ["n03598930", "jigsaw_puzzle"], "612": ["n03599486", "jinrikisha"], "613": ["n03602883", "joystick"], "614": ["n03617480", "kimono"], "615": ["n03623198", "knee_pad"], "616": ["n03627232", "knot"], "617": ["n03630383", "lab_coat"], "618": ["n03633091", "ladle"], "619": ["n03637318", "lampshade"], "620": ["n03642806", "laptop"], "621": ["n03649909", "lawn_mower"], "622": ["n03657121", "lens_cap"], "623": ["n03658185", "letter_opener"], "624": ["n03661043", "library"], "625": ["n03662601", "lifeboat"], "626": ["n03666591", "lighter"], "627": ["n03670208", "limousine"], "628": ["n03673027", "liner"], "629": ["n03676483", "lipstick"], "630": ["n03680355", "Loafer"], "631": ["n03690938", "lotion"], "632": ["n03691459", "loudspeaker"], "633": ["n03692522", "loupe"], "634": ["n03697007", "lumbermill"], "635": ["n03706229", "magnetic_compass"], "636": ["n03709823", "mailbag"], "637": ["n03710193", "mailbox"], "638": ["n03710637", "maillot"], "639": ["n03710721", "maillot"], "640": ["n03717622", "manhole_cover"], "641": ["n03720891", "maraca"], "642": ["n03721384", "marimba"], "643": ["n03724870", "mask"], "644": ["n03729826", "matchstick"], "645": ["n03733131", "maypole"], "646": ["n03733281", "maze"], "647": ["n03733805", "measuring_cup"], "648": ["n03742115", "medicine_chest"], "649": ["n03743016", "megalith"], "650": ["n03759954", "microphone"], "651": ["n03761084", "microwave"], "652": ["n03763968", "military_uniform"], "653": ["n03764736", "milk_can"], "654": ["n03769881", "minibus"], "655": ["n03770439", "miniskirt"], "656": ["n03770679", "minivan"], "657": ["n03773504", "missile"], "658": ["n03775071", "mitten"], "659": ["n03775546", "mixing_bowl"], "660": ["n03776460", "mobile_home"], "661": ["n03777568", "Model_T"], "662": ["n03777754", "modem"], "663": ["n03781244", "monastery"], "664": ["n03782006", "monitor"], "665": ["n03785016", "moped"], "666": ["n03786901", "mortar"], "667": ["n03787032", "mortarboard"], "668": ["n03788195", "mosque"], "669": ["n03788365", "mosquito_net"], "670": ["n03791053", "motor_scooter"], "671": ["n03792782", "mountain_bike"], "672": ["n03792972", "mountain_tent"], "673": ["n03793489", "mouse"], "674": ["n03794056", "mousetrap"], "675": ["n03796401", "moving_van"], "676": ["n03803284", "muzzle"], "677": ["n03804744", "nail"], "678": ["n03814639", "neck_brace"], "679": ["n03814906", "necklace"], "680": ["n03825788", "nipple"], "681": ["n03832673", "notebook"], "682": ["n03837869", "obelisk"], "683": ["n03838899", "oboe"], "684": ["n03840681", "ocarina"], "685": ["n03841143", "odometer"], "686": ["n03843555", "oil_filter"], "687": ["n03854065", "organ"], "688": ["n03857828", "oscilloscope"], "689": ["n03866082", "overskirt"], "690": ["n03868242", "oxcart"], "691": ["n03868863", "oxygen_mask"], "692": ["n03871628", "packet"], "693": ["n03873416", "paddle"], "694": ["n03874293", "paddlewheel"], "695": ["n03874599", "padlock"], "696": ["n03876231", "paintbrush"], "697": ["n03877472", "pajama"], "698": ["n03877845", "palace"], "699": ["n03884397", "panpipe"], "700": ["n03887697", "paper_towel"], "701": ["n03888257", "parachute"], "702": ["n03888605", "parallel_bars"], "703": ["n03891251", "park_bench"], "704": ["n03891332", "parking_meter"], "705": ["n03895866", "passenger_car"], "706": ["n03899768", "patio"], "707": ["n03902125", "pay-phone"], "708": ["n03903868", "pedestal"], "709": ["n03908618", "pencil_box"], "710": ["n03908714", "pencil_sharpener"], "711": ["n03916031", "perfume"], "712": ["n03920288", "Petri_dish"], "713": ["n03924679", "photocopier"], "714": ["n03929660", "pick"], "715": ["n03929855", "pickelhaube"], "716": ["n03930313", "picket_fence"], "717": ["n03930630", "pickup"], "718": ["n03933933", "pier"], "719": ["n03935335", "piggy_bank"], "720": ["n03937543", "pill_bottle"], "721": ["n03938244", "pillow"], "722": ["n03942813", "ping-pong_ball"], "723": ["n03944341", "pinwheel"], "724": ["n03947888", "pirate"], "725": ["n03950228", "pitcher"], "726": ["n03954731", "plane"], "727": ["n03956157", "planetarium"], "728": ["n03958227", "plastic_bag"], "729": ["n03961711", "plate_rack"], "730": ["n03967562", "plow"], "731": ["n03970156", "plunger"], "732": ["n03976467", "Polaroid_camera"], "733": ["n03976657", "pole"], "734": ["n03977966", "police_van"], "735": ["n03980874", "poncho"], "736": ["n03982430", "pool_table"], "737": ["n03983396", "pop_bottle"], "738": ["n03991062", "pot"], "739": ["n03992509", "potter's_wheel"], "740": ["n03995372", "power_drill"], "741": ["n03998194", "prayer_rug"], "742": ["n04004767", "printer"], "743": ["n04005630", "prison"], "744": ["n04008634", "projectile"], "745": ["n04009552", "projector"], "746": ["n04019541", "puck"], "747": ["n04023962", "punching_bag"], "748": ["n04026417", "purse"], "749": ["n04033901", "quill"], "750": ["n04033995", "quilt"], "751": ["n04037443", "racer"], "752": ["n04039381", "racket"], "753": ["n04040759", "radiator"], "754": ["n04041544", "radio"], "755": ["n04044716", "radio_telescope"], "756": ["n04049303", "rain_barrel"], "757": ["n04065272", "recreational_vehicle"], "758": ["n04067472", "reel"], "759": ["n04069434", "reflex_camera"], "760": ["n04070727", "refrigerator"], "761": ["n04074963", "remote_control"], "762": ["n04081281", "restaurant"], "763": ["n04086273", "revolver"], "764": ["n04090263", "rifle"], "765": ["n04099969", "rocking_chair"], "766": ["n04111531", "rotisserie"], "767": ["n04116512", "rubber_eraser"], "768": ["n04118538", "rugby_ball"], "769": ["n04118776", "rule"], "770": ["n04120489", "running_shoe"], "771": ["n04125021", "safe"], "772": ["n04127249", "safety_pin"], "773": ["n04131690", "saltshaker"], "774": ["n04133789", "sandal"], "775": ["n04136333", "sarong"], "776": ["n04141076", "sax"], "777": ["n04141327", "scabbard"], "778": ["n04141975", "scale"], "779": ["n04146614", "school_bus"], "780": ["n04147183", "schooner"], "781": ["n04149813", "scoreboard"], "782": ["n04152593", "screen"], "783": ["n04153751", "screw"], "784": ["n04154565", "screwdriver"], "785": ["n04162706", "seat_belt"], "786": ["n04179913", "sewing_machine"], "787": ["n04192698", "shield"], "788": ["n04200800", "shoe_shop"], "789": ["n04201297", "shoji"], "790": ["n04204238", "shopping_basket"], "791": ["n04204347", "shopping_cart"], "792": ["n04208210", "shovel"], "793": ["n04209133", "shower_cap"], "794": ["n04209239", "shower_curtain"], "795": ["n04228054", "ski"], "796": ["n04229816", "ski_mask"], "797": ["n04235860", "sleeping_bag"], "798": ["n04238763", "slide_rule"], "799": ["n04239074", "sliding_door"], "800": ["n04243546", "slot"], "801": ["n04251144", "snorkel"], "802": ["n04252077", "snowmobile"], "803": ["n04252225", "snowplow"], "804": ["n04254120", "soap_dispenser"], "805": ["n04254680", "soccer_ball"], "806": ["n04254777", "sock"], "807": ["n04258138", "solar_dish"], "808": ["n04259630", "sombrero"], "809": ["n04263257", "soup_bowl"], "810": ["n04264628", "space_bar"], "811": ["n04265275", "space_heater"], "812": ["n04266014", "space_shuttle"], "813": ["n04270147", "spatula"], "814": ["n04273569", "speedboat"], "815": ["n04275548", "spider_web"], "816": ["n04277352", "spindle"], "817": ["n04285008", "sports_car"], "818": ["n04286575", "spotlight"], "819": ["n04296562", "stage"], "820": ["n04310018", "steam_locomotive"], "821": ["n04311004", "steel_arch_bridge"], "822": ["n04311174", "steel_drum"], "823": ["n04317175", "stethoscope"], "824": ["n04325704", "stole"], "825": ["n04326547", "stone_wall"], "826": ["n04328186", "stopwatch"], "827": ["n04330267", "stove"], "828": ["n04332243", "strainer"], "829": ["n04335435", "streetcar"], "830": ["n04336792", "stretcher"], "831": ["n04344873", "studio_couch"], "832": ["n04346328", "stupa"], "833": ["n04347754", "submarine"], "834": ["n04350905", "suit"], "835": ["n04355338", "sundial"], "836": ["n04355933", "sunglass"], "837": ["n04356056", "sunglasses"], "838": ["n04357314", "sunscreen"], "839": ["n04366367", "suspension_bridge"], "840": ["n04367480", "swab"], "841": ["n04370456", "sweatshirt"], "842": ["n04371430", "swimming_trunks"], "843": ["n04371774", "swing"], "844": ["n04372370", "switch"], "845": ["n04376876", "syringe"], "846": ["n04380533", "table_lamp"], "847": ["n04389033", "tank"], "848": ["n04392985", "tape_player"], "849": ["n04398044", "teapot"], "850": ["n04399382", "teddy"], "851": ["n04404412", "television"], "852": ["n04409515", "tennis_ball"], "853": ["n04417672", "thatch"], "854": ["n04418357", "theater_curtain"], "855": ["n04423845", "thimble"], "856": ["n04428191", "thresher"], "857": ["n04429376", "throne"], "858": ["n04435653", "tile_roof"], "859": ["n04442312", "toaster"], "860": ["n04443257", "tobacco_shop"], "861": ["n04447861", "toilet_seat"], "862": ["n04456115", "torch"], "863": ["n04458633", "totem_pole"], "864": ["n04461696", "tow_truck"], "865": ["n04462240", "toyshop"], "866": ["n04465501", "tractor"], "867": ["n04467665", "trailer_truck"], "868": ["n04476259", "tray"], "869": ["n04479046", "trench_coat"], "870": ["n04482393", "tricycle"], "871": ["n04483307", "trimaran"], "872": ["n04485082", "tripod"], "873": ["n04486054", "triumphal_arch"], "874": ["n04487081", "trolleybus"], "875": ["n04487394", "trombone"], "876": ["n04493381", "tub"], "877": ["n04501370", "turnstile"], "878": ["n04505470", "typewriter_keyboard"], "879": ["n04507155", "umbrella"], "880": ["n04509417", "unicycle"], "881": ["n04515003", "upright"], "882": ["n04517823", "vacuum"], "883": ["n04522168", "vase"], "884": ["n04523525", "vault"], "885": ["n04525038", "velvet"], "886": ["n04525305", "vending_machine"], "887": ["n04532106", "vestment"], "888": ["n04532670", "viaduct"], "889": ["n04536866", "violin"], "890": ["n04540053", "volleyball"], "891": ["n04542943", "waffle_iron"], "892": ["n04548280", "wall_clock"], "893": ["n04548362", "wallet"], "894": ["n04550184", "wardrobe"], "895": ["n04552348", "warplane"], "896": ["n04553703", "washbasin"], "897": ["n04554684", "washer"], "898": ["n04557648", "water_bottle"], "899": ["n04560804", "water_jug"], "900": ["n04562935", "water_tower"], "901": ["n04579145", "whiskey_jug"], "902": ["n04579432", "whistle"], "903": ["n04584207", "wig"], "904": ["n04589890", "window_screen"], "905": ["n04590129", "window_shade"], "906": ["n04591157", "Windsor_tie"], "907": ["n04591713", "wine_bottle"], "908": ["n04592741", "wing"], "909": ["n04596742", "wok"], "910": ["n04597913", "wooden_spoon"], "911": ["n04599235", "wool"], "912": ["n04604644", "worm_fence"], "913": ["n04606251", "wreck"], "914": ["n04612504", "yawl"], "915": ["n04613696", "yurt"], "916": ["n06359193", "web_site"], "917": ["n06596364", "comic_book"], "918": ["n06785654", "crossword_puzzle"], "919": ["n06794110", "street_sign"], "920": ["n06874185", "traffic_light"], "921": ["n07248320", "book_jacket"], "922": ["n07565083", "menu"], "923": ["n07579787", "plate"], "924": ["n07583066", "guacamole"], "925": ["n07584110", "consomme"], "926": ["n07590611", "hot_pot"], "927": ["n07613480", "trifle"], "928": ["n07614500", "ice_cream"], "929": ["n07615774", "ice_lolly"], "930": ["n07684084", "French_loaf"], "931": ["n07693725", "bagel"], "932": ["n07695742", "pretzel"], "933": ["n07697313", "cheeseburger"], "934": ["n07697537", "hotdog"], "935": ["n07711569", "mashed_potato"], "936": ["n07714571", "head_cabbage"], "937": ["n07714990", "broccoli"], "938": ["n07715103", "cauliflower"], "939": ["n07716358", "zucchini"], "940": ["n07716906", "spaghetti_squash"], "941": ["n07717410", "acorn_squash"], "942": ["n07717556", "butternut_squash"], "943": ["n07718472", "cucumber"], "944": ["n07718747", "artichoke"], "945": ["n07720875", "bell_pepper"], "946": ["n07730033", "cardoon"], "947": ["n07734744", "mushroom"], "948": ["n07742313", "Granny_Smith"], "949": ["n07745940", "strawberry"], "950": ["n07747607", "orange"], "951": ["n07749582", "lemon"], "952": ["n07753113", "fig"], "953": ["n07753275", "pineapple"], "954": ["n07753592", "banana"], "955": ["n07754684", "jackfruit"], "956": ["n07760859", "custard_apple"], "957": ["n07768694", "pomegranate"], "958": ["n07802026", "hay"], "959": ["n07831146", "carbonara"], "960": ["n07836838", "chocolate_sauce"], "961": ["n07860988", "dough"], "962": ["n07871810", "meat_loaf"], "963": ["n07873807", "pizza"], "964": ["n07875152", "potpie"], "965": ["n07880968", "burrito"], "966": ["n07892512", "red_wine"], "967": ["n07920052", "espresso"], "968": ["n07930864", "cup"], "969": ["n07932039", "eggnog"], "970": ["n09193705", "alp"], "971": ["n09229709", "bubble"], "972": ["n09246464", "cliff"], "973": ["n09256479", "coral_reef"], "974": ["n09288635", "geyser"], "975": ["n09332890", "lakeside"], "976": ["n09399592", "promontory"], "977": ["n09421951", "sandbar"], "978": ["n09428293", "seashore"], "979": ["n09468604", "valley"], "980": ["n09472597", "volcano"], "981": ["n09835506", "ballplayer"], "982": ["n10148035", "groom"], "983": ["n10565667", "scuba_diver"], "984": ["n11879895", "rapeseed"], "985": ["n11939491", "daisy"], "986": ["n12057211", "yellow_lady's_slipper"], "987": ["n12144580", "corn"], "988": ["n12267677", "acorn"], "989": ["n12620546", "hip"], "990": ["n12768682", "buckeye"], "991": ["n12985857", "coral_fungus"], "992": ["n12998815", "agaric"], "993": ["n13037406", "gyromitra"], "994": ["n13040303", "stinkhorn"], "995": ["n13044778", "earthstar"], "996": ["n13052670", "hen-of-the-woods"], "997": ["n13054560", "bolete"], "998": ["n13133613", "ear"], "999": ["n15075141", "toilet_tissue"]} --------------------------------------------------------------------------------