├── attribution ├── __init__.py ├── constants.py ├── dataset.py ├── analyzer.py └── eval.py ├── method-comparison.png ├── LICENSE.md ├── experiment_smooth_taylor.py ├── experiment_classify.py ├── experiment_ig.py ├── experiment_grad.py ├── experiment_total_variation.py ├── experiment_perturbations.py ├── README.md ├── experiment_adaptive_noising.py └── rsc ├── imagenet_classes.txt └── synset_words.txt /attribution/__init__.py: -------------------------------------------------------------------------------- 1 | from .analyzer import integrated_gradients, smooth_taylor, get_gradients -------------------------------------------------------------------------------- /method-comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garygsw/smooth-taylor/HEAD/method-comparison.png -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Gary Goh, Sebastian Lapuschkin, Leander Weber, Wojciech Samek and Alexander Binder. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /attribution/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import torch 4 | from torchvision import models, transforms 5 | 6 | 7 | MODELS = {'densenet121': models.densenet121, 8 | 'resnet152': models.resnet152} 9 | DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 10 | 11 | ANALYZERS = ['grad', 'smooth-grad', 'smooth-taylor', 'ig', 'lrp'] 12 | IG_BASELINES = ['zero', 'noise'] 13 | 14 | 15 | # ImageNet transform constants 16 | DEFAULT_TRANSFORM = transforms.Compose([ 17 | transforms.Resize(256), # resize image to 256X256 pixels 18 | transforms.CenterCrop(224), # crop the image to 224X224 pixels about the center 19 | transforms.ToTensor(), # convert the image to PyTorch Tensor data type 20 | transforms.Normalize( # Normalize by setting the mean and s.d. to specified values 21 | mean=[0.485, 0.456, 0.406], 22 | std=[0.229, 0.224, 0.225] 23 | ) 24 | ]) 25 | 26 | NORMALIZE_TRANSFORM = transforms.Compose([ 27 | transforms.Normalize( # Normalize by setting the mean and s.d. to specified values 28 | mean=[0.485, 0.456, 0.406], 29 | std=[0.229, 0.224, 0.225] 30 | ) 31 | ]) 32 | 33 | RESIZE_TRANSFORM = transforms.Compose([ 34 | transforms.Resize(256), # resize image to 256X256 pixels 35 | transforms.CenterCrop(224), # crop the image to 224X224 pixels about the center 36 | ]) 37 | 38 | INVERSE_TRANSFORM = transforms.Compose([ 39 | transforms.Normalize( # Normalize by setting the mean and s.d. to specified values 40 | mean=[-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225], 41 | std=[1 / 0.229, 1 / 0.224, 1 / 0.225] 42 | ) 43 | ]) -------------------------------------------------------------------------------- /attribution/dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import PIL 3 | import xml.etree.ElementTree as ET 4 | from torch.utils.data import Dataset 5 | 6 | from .constants import * 7 | 8 | 9 | class ImageNetValDataset(Dataset): 10 | def __init__(self, root_dir, label_dir, synset_filepath, max_num=None, transform=None): 11 | """ 12 | Args: 13 | root_dir (string): Directory with all the raw images 14 | label_dir (string): Directory with the ground truth label (in XML files) 15 | synset_filepath (string): Filepath of synset file 16 | max_num (int, optional): Optional maximum number of image to load 17 | transform (callable, optional): Optional transform to be applied on a sample 18 | """ 19 | self.root_dir = root_dir 20 | self.label_dir = label_dir 21 | self.synset_filepath = synset_filepath 22 | if transform: 23 | self.transform = transform 24 | else: 25 | self.transform = DEFAULT_TRANSFORM 26 | 27 | self.parse_synset() 28 | self.img_filepaths = [] 29 | self.labels = [] 30 | 31 | # Prepare the images and labels 32 | for root, dirs, files in os.walk(self.root_dir): 33 | for i, filename in enumerate(sorted(files)): 34 | if max_num and i >= (max_num): 35 | print('Read data read limit reached:', max_num) 36 | break 37 | img_filepath = os.path.join(root, filename) 38 | self.img_filepaths.append(img_filepath) 39 | self.labels.append(self.parse_label(filename)) 40 | 41 | def parse_synset(self): 42 | """ Prepare the following mappings: 43 | idx2synset: dict mapping of idx to synsets 44 | synset2idx: dict mapping of synset to idx 45 | synset2class: dict mapping of synset to class 46 | """ 47 | with open(self.synset_filepath) as f: 48 | synsets = [(line.split()[0], ' '.join(line.split()[1:])) for line in f.readlines()] 49 | self.idx2synset = {i: x[0] for i, x in enumerate(synsets)} 50 | self.synset2idx = {x[0]: i for i, x in enumerate(synsets)} 51 | self.synset2class = {x[0]: x[1] for x in synsets} 52 | 53 | def parse_label(self, img_filename): 54 | """Read the XML annotation file and retrieve the label index.""" 55 | img_identifier, ext = os.path.splitext(img_filename) 56 | label_filepath = os.path.join(self.label_dir, img_identifier) + '.xml' 57 | tree = ET.parse(label_filepath) 58 | root = tree.getroot() 59 | 60 | label_set = set() 61 | for obj in root.findall('object'): 62 | for name in obj.findall('name'): 63 | idx = self.synset2idx[name.text] 64 | label_set.add(idx) 65 | if len(label_set) != 1: 66 | print('ERR: More than 1 label is found for ', img_filename) 67 | exit() 68 | return idx 69 | 70 | def __len__(self): 71 | return len(self.img_filepaths) 72 | 73 | def __getitem__(self, idx): 74 | """ 75 | Returns: 76 | sample: dict that contains image data, label and image filepath of the indexed image 77 | """ 78 | img_filepath = self.img_filepaths[idx] 79 | image = PIL.Image.open(img_filepath).convert('RGB') 80 | label = self.labels[idx] 81 | 82 | if self.transform: 83 | image = self.transform(image) 84 | if image.size()[1] == 1: # if only 1 channel after transformation, repeat channel 3 times 85 | image = image.repeat([1, 3, 1, 1]) 86 | 87 | sample = {'image': image, 'label': label, 'filepath': img_filepath} 88 | return sample 89 | -------------------------------------------------------------------------------- /attribution/analyzer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from torch.utils.data import DataLoader 4 | 5 | from .constants import * 6 | 7 | 8 | def integrated_gradients(inputs, model, batch_size, baseline, explained_class, steps=50): 9 | """ 10 | Args: 11 | inputs (array): original inputs 12 | model (PyTorch model): pretrained model 13 | batch_size (int): batch size to run per epoch 14 | baseline (array): reference point 15 | explained_class (int): index of the class to be explained 16 | steps (int, optional): number of steps along path 17 | """ 18 | # Scale the input with k/m progressive steps multiplier 19 | diff = inputs - baseline 20 | scaled_inputs = [baseline + (float(i) / steps) * diff for i in range(steps + 1)] 21 | scaled_inputs = torch.stack(scaled_inputs, dim=0) # shape: (k + 1, C, H, W) 22 | scaled_dataset = torch.utils.data.dataset.TensorDataset(scaled_inputs) 23 | scaled_loader = DataLoader(scaled_dataset, batch_size=batch_size, shuffle=False) 24 | 25 | # Get the gradients along this path 26 | gradients = get_gradients(scaled_loader, model, explained_class=explained_class) 27 | avg_grads = np.average(gradients[1:], axis=0) # from step 1 onwards 28 | 29 | integrated_grad = diff.numpy() * avg_grads 30 | return integrated_grad 31 | 32 | 33 | def smooth_taylor(inputs, model, batch_size, noise_scale, num_roots, explained_class, percent=False): 34 | """ 35 | Args: 36 | inputs (array): original inputs 37 | model (PyTorch model): pretrained model 38 | batch_size (int): batch size to run per epoch 39 | noise_scale (float): scale to noise the inputs 40 | num_roots (int): number of noised inputs to generate 41 | explained_class (int): index of the class to be explained 42 | percent (bool, optional): use noise scale percentage 43 | """ 44 | # Generate roots dataset based on the original inputs with additive noise 45 | roots = torch.stack([torch.zeros_like(inputs) for _ in range(num_roots)]) 46 | if percent: 47 | noise_scale = noise_scale * (np.max(inputs.numpy()) - np.min(inputs.numpy())) 48 | for i in range(num_roots): 49 | roots[i] = inputs + noise_scale * torch.randn_like(inputs) 50 | roots_dataset = torch.utils.data.dataset.TensorDataset(roots) 51 | roots_data_loader = DataLoader(roots_dataset, batch_size=batch_size, shuffle=False) 52 | 53 | # Compute the gradients w.r.t. explained class for roots 54 | gradients = get_gradients(roots_data_loader, model, explained_class=explained_class) 55 | 56 | # Compute SmoothTaylor with contribution from roots 57 | attributions = np.mean([(inputs - roots_dataset[i][0]).numpy() * gradients[i] 58 | for i in range(num_roots)], axis=0) 59 | return attributions 60 | 61 | 62 | def get_gradients(data_loader, model, explained_class): 63 | """ 64 | Args: 65 | data_loader (DataLoader): object that loads images data 66 | model (model): pre-trained PyTorch model 67 | explained_class: index of the class to be explained 68 | """ 69 | gradients = [] 70 | for sample_batch in data_loader: 71 | inputs = sample_batch[0] 72 | inputs = inputs.to(DEVICE) 73 | inputs.requires_grad = True 74 | 75 | # Perform the backpropagation for the explained class 76 | out = model(inputs) 77 | model.zero_grad() 78 | torch.sum(out[:,explained_class]).backward() 79 | with torch.no_grad(): 80 | gradient = inputs.grad.detach().cpu().numpy() # retrieve the input gradients 81 | gradients.append(gradient) 82 | gradients = np.array(gradients) 83 | gradients = np.concatenate(gradients) 84 | return gradients 85 | -------------------------------------------------------------------------------- /experiment_smooth_taylor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from tqdm import tqdm 4 | import argparse 5 | import torch 6 | from torch.utils.data import DataLoader 7 | 8 | from attribution import smooth_taylor 9 | from attribution.dataset import ImageNetValDataset 10 | from attribution.constants import * 11 | 12 | 13 | def parse_args(): 14 | # Parse arguments 15 | parser = argparse.ArgumentParser(description='Generate smooth taylor attributions') 16 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify') 17 | parser.add_argument('-b', '--batch_size', type=int, help='batch size to use during each epoch', default=50) 18 | parser.add_argument('-s', '--noise_scale', type=float, help='magnitude of the noise scale to noise the image', default=5e-1) 19 | parser.add_argument('-r', '--num_roots', type=int, help='number of noised images to use', default=150) 20 | parser.add_argument('-i', '--num_image', type=int, help='number of image data to use from the first', default=1000) 21 | parser.add_argument('-p', '--noise_percent', action='store_true', help='use proportion of image as noise scale') 22 | parser.add_argument('-o', '--overwrite', action='store_true', help='overwrite the output') 23 | args = parser.parse_args() 24 | if args.model_name not in MODELS: 25 | print('Invalid model name:', args.model_name) 26 | exit() 27 | return args 28 | 29 | 30 | def run_smooth_taylor_experiment(dataset, model, model_name, batch_size, noise_scale, num_roots, overwrite=False, noise_percent=True): 31 | # Read all top10idxs for that model 32 | input_dir = os.path.join('output/', model_name) 33 | if not os.path.exists(input_dir): 34 | print('Model classification output not found for:', model_name) 35 | exit() 36 | input_path = os.path.join(input_dir, 'all_top10_idxs.npy') 37 | all_top10_idxs = np.load(input_path) 38 | 39 | # Check output directory 40 | if noise_percent: 41 | noise_folder = str(noise_scale) + '%' 42 | noise_scale = noise_scale / 100. 43 | else: 44 | noise_folder = '{:.1e}'.format(noise_scale) # convert to scientific notation 45 | roots_folder = str(num_roots) + 'N' 46 | out_dir = os.path.join('heatmaps/smooth-taylor/', model_name, noise_folder, roots_folder) 47 | if not os.path.isdir(out_dir): 48 | os.makedirs(out_dir) 49 | 50 | # Generate SmoothTaylor heatmap for each image 51 | for img_idx, img_filepath in enumerate(tqdm(dataset.img_filepaths, desc='Image')): 52 | # Initialize output filepath 53 | img_filename = os.path.basename(img_filepath) 54 | outpath = os.path.join(out_dir, img_filename + '_hm.npy') 55 | if not overwrite and os.path.exists(outpath): # ignore if already generated 56 | print(img_filename, 'already has heatmap generated') 57 | continue 58 | 59 | # Retrieve the image data and predicted class 60 | predicted_class = all_top10_idxs[img_idx, 0] 61 | img_input = dataset[img_idx]['image'] 62 | 63 | # Compute SmoothTaylor heatmaps 64 | attributions = smooth_taylor( 65 | inputs=img_input, 66 | model=model, 67 | batch_size=batch_size, 68 | noise_scale=noise_scale, 69 | num_roots=num_roots, 70 | explained_class=predicted_class, 71 | percent=noise_percent 72 | ) 73 | heatmap = np.sum(attributions, axis=0) # sum across all channels 74 | np.save(outpath, heatmap) 75 | 76 | 77 | if __name__=='__main__': 78 | args = parse_args() 79 | 80 | from datetime import datetime, timedelta 81 | start_time = datetime.now() 82 | 83 | # Load the dataset 84 | dataset = ImageNetValDataset( 85 | root_dir='data/images', 86 | label_dir='data/annotations', 87 | synset_filepath='rsc/synset_words.txt', 88 | max_num=args.num_image 89 | ) 90 | 91 | # Load the pre-trained model 92 | model = MODELS[args.model_name](pretrained=True) 93 | model = model.to(DEVICE) 94 | model.eval() 95 | 96 | # Perform smooth taylor experiment 97 | run_smooth_taylor_experiment( 98 | dataset=dataset, 99 | model=model, 100 | model_name=args.model_name, 101 | batch_size=args.batch_size, 102 | noise_scale=args.noise_scale, 103 | num_roots=args.num_roots, 104 | overwrite=args.overwrite, 105 | noise_percent=args.noise_percent 106 | ) 107 | 108 | end_time = datetime.now() 109 | elapsed_seconds = int((end_time - start_time).total_seconds()) 110 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 111 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 112 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) -------------------------------------------------------------------------------- /experiment_classify.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import argparse 4 | from tqdm import tqdm 5 | import torch 6 | from torch.utils.data import DataLoader 7 | 8 | from attribution.dataset import ImageNetValDataset 9 | from attribution.constants import * 10 | 11 | 12 | def parse_args(): 13 | # Parse arguments 14 | parser = argparse.ArgumentParser(description='Classify the image dataset and saves all prediction outputs') 15 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify') 16 | parser.add_argument('-b', '--batch_size', type=int, help='batch size to use during each epoch', default=128) 17 | args = parser.parse_args() 18 | if args.model_name not in MODELS: 19 | print('Invalid model name:', args.model_name) 20 | exit() 21 | return args 22 | 23 | 24 | def run_classify_experiment(dataset, model, model_name, batch_size, num_classes): 25 | """ 26 | Args: 27 | dataset (Dataset): Containing input images. 28 | model (PyTorch model): Pre-trained Pytorch classifier. 29 | model_name (str): Name of the model. 30 | batch_size (int): Batch size to use during each epoch. 31 | num_classes (int): Total number of classes. 32 | """ 33 | # Load the dataset into a data loader 34 | data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False) 35 | dataset_size = len(dataset) 36 | 37 | # Initialize all scores, top10 indexes and labels 38 | all_scores = np.zeros((dataset_size, num_classes)) 39 | all_top10_idxs = np.zeros((dataset_size, 10), dtype=np.int32) 40 | all_labels = np.zeros((dataset_size), dtype=np.int32) 41 | all_positives = np.zeros((dataset_size), dtype=np.int32) 42 | 43 | # Create output directory 44 | out_dir = 'output/' + model_name 45 | if not os.path.isdir(out_dir): 46 | os.makedirs(out_dir) 47 | 48 | # Perform classification 49 | running_corrects = 0.0 50 | total_epochs = len(data_loader) 51 | for i_batch, sample_batch in enumerate(tqdm(data_loader, desc='Epoch')): 52 | print('Epoch #%d/%d:' % (i_batch + 1, total_epochs), 'batch_shape:', sample_batch['image'].size()) 53 | inputs = sample_batch['image'] 54 | labels = sample_batch['label'] 55 | fpaths = sample_batch['filepath'] 56 | inputs = inputs.to(DEVICE) 57 | labels = labels.to(DEVICE) 58 | 59 | with torch.no_grad(): 60 | out = model(inputs) 61 | _, idx = torch.max(out, 1) 62 | _, top10idxs = torch.topk(out, k=10, dim=1, largest=True, sorted=True, out=None) 63 | 64 | # Save outputs into .npy files for each individual image 65 | top10idxs = top10idxs.cpu().numpy() 66 | scores = out.cpu().numpy() 67 | for i, filepath in enumerate(fpaths): 68 | filename = os.path.basename(filepath) 69 | outpath = os.path.join(out_dir, filename + '_scores.npy' ) 70 | np.save(outpath, scores[i,:]) 71 | all_scores[i_batch * batch_size + i,:] = scores[i,:] 72 | 73 | outpath = os.path.join(out_dir, filename + '_top10idxs.npy' ) 74 | np.save(outpath, top10idxs[i,:]) 75 | all_top10_idxs[i_batch * batch_size + i,:] = top10idxs[i,:] 76 | 77 | all_labels[i_batch * batch_size + i] = labels[i] 78 | all_positives[i_batch * batch_size + i] = int(labels[i] == top10idxs[i,0]) 79 | 80 | num_correct = torch.sum((idx == labels).float()) 81 | running_corrects += num_correct 82 | epoch_acc = num_correct / float(batch_size) 83 | running_acc = running_corrects / dataset_size 84 | print('Epoch Accuracy: %.2f%%' % (epoch_acc.cpu().numpy() * 100)) 85 | print('Running Accuracy: %.2f%%' % (running_acc.cpu().numpy() * 100)) 86 | 87 | # Save all scores, all top10 indexes, all labels, and all positives 88 | outpath = os.path.join(out_dir, 'all_scores.npy') 89 | np.save(outpath, all_scores) 90 | outpath = os.path.join(out_dir, 'all_top10_idxs.npy') 91 | np.save(outpath, all_top10_idxs) 92 | outpath = os.path.join(out_dir, 'all_labels.npy') 93 | np.save(outpath, all_labels) 94 | outpath = os.path.join(out_dir, 'all_positives.npy') 95 | np.save(outpath, all_positives) 96 | 97 | if __name__=='__main__': 98 | args = parse_args() 99 | 100 | from datetime import datetime, timedelta 101 | start_time = datetime.now() 102 | 103 | # Load the dataset 104 | dataset = ImageNetValDataset( 105 | root_dir='data/images', 106 | label_dir='data/annotations', 107 | synset_filepath='rsc/synset_words.txt' 108 | ) 109 | 110 | # Load the pre-trained model 111 | model = MODELS[args.model_name](pretrained=True) 112 | model = model.to(DEVICE) 113 | model.eval() 114 | 115 | # Read the class file 116 | with open('rsc/imagenet_classes.txt') as f: 117 | classes = [line.strip() for line in f.readlines()] 118 | num_classes = len(classes) 119 | 120 | # Perform classification 121 | run_classify_experiment( 122 | dataset=dataset, 123 | model=model, 124 | model_name=args.model_name, 125 | batch_size=args.batch_size, 126 | num_classes=num_classes 127 | ) 128 | 129 | end_time = datetime.now() 130 | elapsed_seconds = int((end_time - start_time).total_seconds()) 131 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 132 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 133 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) -------------------------------------------------------------------------------- /experiment_ig.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from tqdm import tqdm 4 | import argparse 5 | import torch 6 | from torch.utils.data import DataLoader 7 | 8 | from attribution import integrated_gradients 9 | from attribution.dataset import ImageNetValDataset 10 | from attribution.constants import * 11 | 12 | 13 | def parse_args(): 14 | # Parse arguments 15 | parser = argparse.ArgumentParser(description='Generate integrated gradients') 16 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify') 17 | parser.add_argument('-b', '--batch_size', type=int, help='batch size to use during each epoch', default=50) 18 | parser.add_argument('-k', '--steps', type=int, help='number of steps along path', default=50) 19 | parser.add_argument('-z', '--baseline_type', type=str, help='baseline type to use', default='zero') 20 | parser.add_argument('-n', '--num_noise', type=int, help='number of noise baselines to use', default=1) 21 | parser.add_argument('-i', '--num_image', type=int, help='number of image data to use from the first', default=1000) 22 | parser.add_argument('-o', '--overwrite', action='store_true', help='overwrite the output') 23 | args = parser.parse_args() 24 | if args.model_name not in MODELS: 25 | print('Invalid model name:', args.model_name) 26 | exit() 27 | if args.baseline_type not in IG_BASELINES: 28 | print('Invalid IG baseline type:', args.baseline_type) 29 | exit() 30 | return args 31 | 32 | 33 | def run_ig_experiment(dataset, model, model_name, batch_size, baseline_type, 34 | num_noise, steps, overwrite=False): 35 | # Read all scores and top10idxs for that model 36 | input_dir = os.path.join('output/', model_name) 37 | if not os.path.exists(input_dir): 38 | print('Model classification output not found for:', model_name) 39 | exit() 40 | input_path = os.path.join(input_dir, 'all_top10_idxs.npy') 41 | all_top10_idxs = np.load(input_path) 42 | input_path = os.path.join(input_dir, 'all_scores.npy') 43 | all_scores = np.load(input_path) 44 | 45 | # Check output directory 46 | out_dir = os.path.join('heatmaps/ig/', model_name, baseline_type) 47 | if baseline_type == 'noise': 48 | out_dir = os.path.join(out_dir, str(num_noise) + 'N') 49 | if not os.path.isdir(out_dir): 50 | os.makedirs(out_dir) 51 | 52 | # Generate IG and heatmap for each image 53 | for img_idx, img_filepath in enumerate(tqdm(dataset.img_filepaths, desc='Image')): 54 | # Initialize output filepath 55 | img_filename = os.path.basename(img_filepath) 56 | outpath = os.path.join(out_dir, img_filename + '_hm.npy') 57 | if not overwrite and os.path.exists(outpath): # ignore if the heatmap is already generated 58 | print(img_filepath, 'already has heatmap generated') 59 | continue 60 | 61 | # Retrieve the image data and predicted class 62 | predicted_class = all_top10_idxs[img_idx, 0] 63 | img_input = dataset[img_idx]['image'] 64 | input_score = all_scores[img_idx, predicted_class] 65 | # print('input score:', input_score) 66 | 67 | if baseline_type == 'zero': 68 | baseline = torch.zeros_like(img_input) 69 | baseline = NORMALIZE_TRANSFORM(baseline) 70 | ig = integrated_gradients( 71 | inputs=img_input, 72 | model=model, 73 | batch_size=batch_size, 74 | baseline=baseline, 75 | explained_class=predicted_class, 76 | steps=steps 77 | ) 78 | # To check for completeness: 79 | # out = model(torch.unsqueeze(baseline, 0)) 80 | # print('baseline score:', out[:, explained_class].detach().numpy()[0]) 81 | # print('IG sum:', np.sum(ig)) 82 | elif baseline_type == 'noise': 83 | all_igs = [] 84 | for _ in range(num_noise): 85 | baseline = torch.rand_like(img_input) # uniform between 0 and 1 86 | baseline = NORMALIZE_TRANSFORM(baseline) 87 | ig = integrated_gradients( 88 | inputs=img_input, 89 | model=model, 90 | batch_size=batch_size, 91 | baseline=baseline, 92 | explained_class=predicted_class, 93 | steps=steps 94 | ) 95 | all_igs.append(ig) 96 | ig = np.mean(np.array(all_igs), axis=0) 97 | 98 | # Save IG heatmap 99 | heatmap = np.sum(ig, axis=0) # aggregate along color channel 100 | np.save(outpath, heatmap) 101 | 102 | 103 | if __name__=='__main__': 104 | args = parse_args() 105 | 106 | from datetime import datetime, timedelta 107 | start_time = datetime.now() 108 | 109 | # Load the dataset 110 | dataset = ImageNetValDataset( 111 | root_dir='data/images', 112 | label_dir='data/annotations', 113 | synset_filepath='rsc/synset_words.txt', 114 | max_num=args.num_image 115 | ) 116 | 117 | # Load the pre-trained model 118 | model = MODELS[args.model_name](pretrained=True) 119 | model = model.to(DEVICE) 120 | model.eval() 121 | 122 | # Perform IG experiment 123 | run_ig_experiment( 124 | dataset=dataset, 125 | model=model, 126 | model_name=args.model_name, 127 | batch_size=args.batch_size, 128 | baseline_type=args.baseline_type, 129 | num_noise=args.num_noise, 130 | steps=args.steps, 131 | overwrite=args.overwrite 132 | ) 133 | 134 | end_time = datetime.now() 135 | elapsed_seconds = int((end_time - start_time).total_seconds()) 136 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 137 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 138 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) 139 | -------------------------------------------------------------------------------- /experiment_grad.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from tqdm import tqdm 4 | import argparse 5 | import torch 6 | from torch.utils.data import DataLoader 7 | 8 | from attribution import get_gradients 9 | from attribution.dataset import ImageNetValDataset 10 | from attribution.constants import * 11 | 12 | 13 | def parse_args(): 14 | # Parse arguments 15 | parser = argparse.ArgumentParser(description='Generate gradients and smooth grad heatmaps') 16 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify') 17 | parser.add_argument('-b', '--batch_size', type=int, help='batch size to use during each epoch', default=50) 18 | parser.add_argument('-i', '--num_image', type=int, help='number of image data to use from the first', default=1000) 19 | parser.add_argument('-s', '--smooth', action='store_true', help='use smooth grad') 20 | parser.add_argument('-p', '--noise_scale', type=float, help='percentage noise scale', default=15) 21 | parser.add_argument('-n', '--num_noise', type=int, help='number of noise inputs to use', default=50) 22 | parser.add_argument('-o', '--overwrite', action='store_true', help='overwrite the output') 23 | args = parser.parse_args() 24 | if args.model_name not in MODELS: 25 | print('Invalid model name:', args.model_name) 26 | exit() 27 | return args 28 | 29 | 30 | def run_grad_experiment(dataset, model, model_name, batch_size, smooth, 31 | num_noise, noise_scale, overwrite=False): 32 | # Read all scores and top10idxs for that model 33 | input_dir = os.path.join('output/', model_name) 34 | if not os.path.exists(input_dir): 35 | print('Model classification output not found for:', model_name) 36 | exit() 37 | input_path = os.path.join(input_dir, 'all_top10_idxs.npy') 38 | all_top10_idxs = np.load(input_path) 39 | input_path = os.path.join(input_dir, 'all_scores.npy') 40 | all_scores = np.load(input_path) 41 | 42 | # Check output directory 43 | if smooth: 44 | noise_scale_folder = str(noise_scale) + '%' 45 | num_noise_folder = str(num_noise) + 'N' 46 | noise_scale = noise_scale / 100. 47 | out_dir = os.path.join('heatmaps/smooth-grad/', model_name) 48 | out_dir = os.path.join(out_dir, noise_scale_folder, num_noise_folder) 49 | else: 50 | out_dir = os.path.join('heatmaps/grad/', model_name) 51 | if not os.path.isdir(out_dir): 52 | os.makedirs(out_dir) 53 | 54 | 55 | # Generate grad and heatmap for each image 56 | for img_idx, img_filepath in enumerate(tqdm(dataset.img_filepaths, desc='Image')): 57 | # Initialize output filepath 58 | img_filename = os.path.basename(img_filepath) 59 | outpath = os.path.join(out_dir, img_filename + '_hm.npy') 60 | if not overwrite and os.path.exists(outpath): # ignore if the heatmap is already generated 61 | print(img_filepath, 'already has heatmap generated') 62 | continue 63 | 64 | # Retrieve the image data and predicted class 65 | predicted_class = all_top10_idxs[img_idx, 0] 66 | img_input = dataset[img_idx]['image'] 67 | input_score = all_scores[img_idx, predicted_class] 68 | # print('input score:', input_score) 69 | 70 | if smooth: 71 | # Generate noised dataset based on the original inputs with additive noise 72 | noised_inputs = torch.stack([torch.zeros_like(img_input) for _ in range(num_noise)]) 73 | img_noise_scale = noise_scale * (np.max(img_input.numpy()) - np.min(img_input.numpy())) 74 | for i in range(num_noise): 75 | noised_inputs[i] = img_input + img_noise_scale * torch.randn_like(img_input) 76 | noised_dataset = torch.utils.data.dataset.TensorDataset(noised_inputs) 77 | noised_data_loader = DataLoader(noised_dataset, batch_size=batch_size, shuffle=False) 78 | 79 | # Compute the gradients w.r.t. explained class for each noised input 80 | gradients = get_gradients(noised_data_loader, model, explained_class=predicted_class) 81 | grad = np.mean(gradients, axis=0) 82 | else: 83 | single_dataset = torch.utils.data.dataset.TensorDataset(torch.unsqueeze(img_input, 0)) 84 | single_data_loader = DataLoader(single_dataset, batch_size=1, shuffle=False) 85 | grad = get_gradients( 86 | data_loader=single_data_loader, 87 | model=model, 88 | explained_class=predicted_class 89 | )[0] 90 | 91 | # Save grad heatmap 92 | heatmap = np.sum(grad, axis=0) # aggregate along color channel 93 | np.save(outpath, heatmap) 94 | 95 | 96 | if __name__=='__main__': 97 | args = parse_args() 98 | 99 | from datetime import datetime, timedelta 100 | start_time = datetime.now() 101 | 102 | # Load the dataset 103 | dataset = ImageNetValDataset( 104 | root_dir='data/images', 105 | label_dir='data/annotations', 106 | synset_filepath='rsc/synset_words.txt', 107 | max_num=args.num_image 108 | ) 109 | 110 | # Load the pre-trained model 111 | model = MODELS[args.model_name](pretrained=True) 112 | model = model.to(DEVICE) 113 | model.eval() 114 | 115 | # Perform grad experiment 116 | run_grad_experiment( 117 | dataset=dataset, 118 | model=model, 119 | model_name=args.model_name, 120 | batch_size=args.batch_size, 121 | smooth=args.smooth, 122 | num_noise=args.num_noise, 123 | noise_scale=args.noise_scale, 124 | overwrite=args.overwrite 125 | ) 126 | 127 | end_time = datetime.now() 128 | elapsed_seconds = int((end_time - start_time).total_seconds()) 129 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 130 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 131 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) 132 | -------------------------------------------------------------------------------- /experiment_total_variation.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from tqdm import tqdm 4 | import argparse 5 | import torch 6 | from torch.utils.data import DataLoader 7 | 8 | from attribution.eval import compute_multiscaled_atv 9 | from attribution.dataset import ImageNetValDataset 10 | from attribution.constants import * 11 | 12 | 13 | def parse_args(): 14 | # Parse arguments 15 | parser = argparse.ArgumentParser(description='Generate multi-scaled average total variation based on a set of heatmaps') 16 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify the noised images') 17 | parser.add_argument('-a', '--analyzer', type=str, help='analyzer that compute the heatmaps') 18 | parser.add_argument('-z', '--baseline', type=str, help='baseline type to compute IG', default='zero') 19 | parser.add_argument('-n', '--num_noise', type=int, help='number of noise baselines used by IG', default=1) 20 | parser.add_argument('-s', '--noise_scale', type=float, help='magnitude of the noise scale used to noise the image', default=5e-1) 21 | parser.add_argument('-p', '--noise_percent', action='store_true', help='use proportion of image as noise scale') 22 | parser.add_argument('-an', '--adaptive_noise', action='store_true', help='use adaptive noise') 23 | parser.add_argument('-af', '--adaptive_function', type=str, help='objective function used', default='aupc') 24 | parser.add_argument('-r', '--num_roots', type=int, help='number of noised images used', default=150) 25 | parser.add_argument('-ds', '--downscale', type=float, help='factor to downscale heatmap', default=1.5) 26 | parser.add_argument('-wms', '--width_min_size', type=int, help='minimum width dimension for downscale', default=30) 27 | parser.add_argument('-hms', '--height_min_size', type=int, help='minimum height dimension for downscale', default=30) 28 | parser.add_argument('-lp', '--lp_norm', type=int, help='norm to use to calculate TV', default=1) 29 | parser.add_argument('-i', '--num_image', type=int, help='number of image data to use from the first', default=1000) 30 | parser.add_argument('-o', '--overwrite', action='store_true', help='overwrite the output') 31 | args = parser.parse_args() 32 | if args.model_name not in MODELS: 33 | print('Invalid model name:', args.model_name) 34 | exit() 35 | if args.analyzer not in ANALYZERS: 36 | print('Invalid analyzer:', args.analyzer) 37 | exit() 38 | if args.baseline not in IG_BASELINES: 39 | print('Invalid IG baseline name:', args.baseline) 40 | exit() 41 | if args.adaptive_function not in ['aupc', 'autvc']: 42 | print('Invalid adaptive objective function:', args.adaptive_function) 43 | exit() 44 | return args 45 | 46 | 47 | def run_total_variation_evaluation(dataset, model_name, analyzer, 48 | noise_scale=None, num_roots=None, 49 | noise_percent=False, baseline=None, num_noise=None, 50 | downscale=1.5, min_size=(30, 30), lp_norm=1, 51 | adaptive_noise=False, adaptive_function=None, overwrite=False): 52 | # Prepare input heatmaps directory 53 | if analyzer == 'smooth-taylor': 54 | assert(noise_scale) 55 | assert(num_roots) 56 | if adaptive_noise: 57 | assert(adaptive_function) 58 | noise_folder = 'adaptive/' + adaptive_function 59 | elif noise_percent: 60 | noise_folder = str(noise_scale) + '%' 61 | else: 62 | noise_folder = '{:.1e}'.format(noise_scale) # convert to scientific notation 63 | num_roots_folder = str(num_roots) + 'N' 64 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, noise_folder, num_roots_folder) 65 | elif analyzer == 'ig': 66 | assert(baseline) 67 | if baseline == 'zero': 68 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, baseline) 69 | elif baseline == 'noise': 70 | assert(num_noise) 71 | num_noise_folder = str(num_noise) + 'N' 72 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, baseline, num_noise_folder) 73 | elif analyzer == 'grad': 74 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name) 75 | elif analyzer == 'smooth-grad': 76 | assert(num_noise) 77 | assert(noise_scale) 78 | noise_folder = str(noise_scale) + '%' 79 | num_noise_folder = str(num_noise) + 'N' 80 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, noise_folder, num_noise_folder) 81 | if not os.path.exists(heatmap_dir): 82 | print('Heatmaps folders missing:', heatmap_dir) 83 | exit() 84 | 85 | # Prepare output directory 86 | output_dir = os.path.join('atv/', analyzer, model_name) 87 | if analyzer == 'smooth-taylor': 88 | output_dir = os.path.join(output_dir, noise_folder, num_roots_folder) 89 | elif analyzer == 'ig': 90 | if baseline == 'zero': 91 | output_dir = os.path.join(output_dir, baseline) 92 | elif baseline == 'noise': 93 | output_dir = os.path.join(output_dir, baseline, num_noise_folder) 94 | elif analyzer == 'smooth-grad': 95 | output_dir = os.path.join(output_dir, noise_folder, num_noise_folder) 96 | if not os.path.isdir(output_dir): 97 | os.makedirs(output_dir) 98 | 99 | # Go through each image and compute ATV scores 100 | all_atv_scores = [] 101 | for img_idx, img_filepath in enumerate(tqdm(dataset.img_filepaths, desc='Image')): 102 | # Check if ATV scores exists 103 | img_filename = os.path.basename(img_filepath) 104 | atv_filename = 'atv_scores_{0}ds_{1}ms_{2}l_{3}.npy'.format( 105 | downscale, min_size, lp_norm, img_filename 106 | ) 107 | atv_outpath = os.path.join(output_dir, atv_filename) 108 | if not overwrite and os.path.exists(atv_outpath): # ignore if done 109 | print(img_filename, 'already has ATV scores computed') 110 | all_atv_scores.append(np.load(atv_outpath)) 111 | continue 112 | 113 | # Load the image heatmap 114 | heatmap_filepath = os.path.join(heatmap_dir, img_filename + '_hm.npy') 115 | if not os.path.exists(heatmap_filepath): 116 | print('Heatmaps not found for:', heatmap_filepath) 117 | continue 118 | heatmap = np.load(heatmap_filepath) 119 | 120 | multi_scaled_ATV = compute_multiscaled_atv( 121 | heatmap=heatmap, 122 | downscale=downscale, 123 | min_size=min_size, 124 | lp_norm=lp_norm 125 | ) 126 | 127 | # Update all ATV scores and save individual scores 128 | all_atv_scores.append(multi_scaled_ATV) 129 | np.save(atv_outpath, multi_scaled_ATV) 130 | 131 | # Save all ATV scores 132 | out_filename = 'all_ATV_scores_{0}ds_{1}ms_{2}l.npy'.format( 133 | downscale, min_size, lp_norm 134 | ) 135 | outpath = os.path.join(output_dir, out_filename) 136 | all_atv_scores = np.array(all_atv_scores) 137 | np.save(outpath, all_atv_scores) 138 | 139 | 140 | if __name__ == "__main__": 141 | args = parse_args() 142 | 143 | from datetime import datetime, timedelta 144 | start_time = datetime.now() 145 | 146 | # Load the dataset 147 | dataset = ImageNetValDataset( 148 | root_dir='data/images', 149 | label_dir='data/annotations', 150 | synset_filepath='rsc/synset_words.txt', 151 | max_num=args.num_image 152 | ) 153 | 154 | # Perform total variation experiment 155 | run_total_variation_evaluation( 156 | dataset=dataset, 157 | model_name=args.model_name, 158 | analyzer=args.analyzer, 159 | noise_scale=args.noise_scale, 160 | noise_percent=args.noise_percent, 161 | num_roots=args.num_roots, 162 | baseline=args.baseline, 163 | num_noise=args.num_noise, 164 | downscale=args.downscale, 165 | min_size=(args.height_min_size, args.width_min_size), 166 | lp_norm=args.lp_norm, 167 | adaptive_noise=args.adaptive_noise, 168 | adaptive_function=args.adaptive_function, 169 | overwrite=args.overwrite 170 | ) 171 | 172 | end_time = datetime.now() 173 | elapsed_seconds = int((end_time - start_time).total_seconds()) 174 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 175 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 176 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) -------------------------------------------------------------------------------- /experiment_perturbations.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from tqdm import tqdm 4 | import argparse 5 | import torch 6 | from torch.utils.data import DataLoader 7 | 8 | from attribution.eval import compute_perturbations 9 | from attribution.dataset import ImageNetValDataset 10 | from attribution.constants import * 11 | 12 | 13 | def parse_args(): 14 | # Parse arguments 15 | parser = argparse.ArgumentParser(description='Generate perturbation scores based on a set of heatmaps') 16 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify the noised images') 17 | parser.add_argument('-a', '--analyzer', type=str, help='analyzer that compute the heatmaps') 18 | parser.add_argument('-b', '--batch_size', type=int, help='batch size to use during each epoch', default=50) 19 | parser.add_argument('-z', '--baseline', type=str, help='baseline type to compute IG', default='zero') 20 | parser.add_argument('-n', '--num_noise', type=int, help='number of noise baselines used by IG', default=1) 21 | parser.add_argument('-s', '--noise_scale', type=float, help='magnitude of the noise scale used to noise the image', default=5e-1) 22 | parser.add_argument('-p', '--noise_percent', action='store_true', help='use proportion of image as noise scale') 23 | parser.add_argument('-an', '--adaptive_noise', action='store_true', help='use adaptive noise') 24 | parser.add_argument('-af', '--adaptive_function', type=str, help='objective function used', default='aupc') 25 | parser.add_argument('-r', '--num_roots', type=int, help='number of noised images used', default=150) 26 | parser.add_argument('-k', '--kernel_size', type=int, help='size of the window of each perturbation', default=15) 27 | parser.add_argument('-pt', '--num_perturbs', type=int, help='number of random perturbations to evaluate', default=50) 28 | parser.add_argument('-l', '--num_regions', type=int, help='number of regions to perturbate', default=30) 29 | parser.add_argument('-d', '--draw_mode', type=int, help='perturb draw mode: 0 - uniform; 1 - gaussian according to image stats', default=0) 30 | parser.add_argument('-i', '--num_image', type=int, help='number of image data to use from the first', default=1000) 31 | parser.add_argument('-o', '--overwrite', action='store_true', help='overwrite the output') 32 | args = parser.parse_args() 33 | if args.model_name not in MODELS: 34 | print('Invalid model name:', args.model_name) 35 | exit() 36 | if args.analyzer not in ANALYZERS: 37 | print('Invalid analyzer:', args.analyzer) 38 | exit() 39 | if args.baseline not in IG_BASELINES: 40 | print('Invalid IG baseline name:', args.baseline) 41 | exit() 42 | if args.draw_mode > 1: 43 | print('Invalid draw mode:', args.draw_mode) 44 | exit() 45 | if args.adaptive_function not in ['aupc', 'autvc']: 46 | print('Invalid adaptive objective function:', args.adaptive_function) 47 | exit() 48 | return args 49 | 50 | 51 | def run_perturbations_evaluation(dataset, model, model_name, batch_size, analyzer, 52 | noise_scale=None, num_roots=None, 53 | noise_percent=False, baseline=None, num_noise=None, 54 | kernel_size=15, num_regions=30, 55 | num_perturbs=50, draw_mode=0, adaptive_noise=False, adaptive_function=None, overwrite=False): 56 | # Read all scores and top10idxs for that model 57 | input_dir = os.path.join('output/', model_name) 58 | if not os.path.exists(input_dir): 59 | print('Model classification output not found for:', model_name) 60 | exit() 61 | input_path = os.path.join(input_dir, 'all_scores.npy') 62 | all_scores = np.load(input_path) 63 | input_path = os.path.join(input_dir, 'all_top10_idxs.npy') 64 | all_top10_idxs = np.load(input_path) 65 | 66 | # Prepare input heatmaps directory 67 | if analyzer == 'smooth-taylor': 68 | assert(noise_scale) 69 | assert(num_roots) 70 | if adaptive_noise: 71 | assert(adaptive_function) 72 | noise_folder = 'adaptive/' + adaptive_function 73 | elif noise_percent: 74 | noise_folder = str(noise_scale) + '%' 75 | else: 76 | noise_folder = '{:.1e}'.format(noise_scale) # convert to scientific notation 77 | num_roots_folder = str(num_roots) + 'N' 78 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, noise_folder, num_roots_folder) 79 | elif analyzer == 'ig': 80 | assert(baseline) 81 | if baseline == 'zero': 82 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, baseline) 83 | elif baseline == 'noise': 84 | assert(num_noise) 85 | num_noise_folder = str(num_noise) + 'N' 86 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, baseline, num_noise_folder) 87 | elif analyzer == 'grad': 88 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name) 89 | elif analyzer == 'smooth-grad': 90 | assert(num_noise) 91 | assert(noise_scale) 92 | noise_folder = str(noise_scale) + '%' 93 | num_noise_folder = str(num_noise) + 'N' 94 | heatmap_dir = os.path.join('heatmaps/', analyzer, model_name, noise_folder, num_noise_folder) 95 | if not os.path.exists(heatmap_dir): 96 | print('Heatmaps folders missing:', heatmap_dir) 97 | exit() 98 | 99 | # Prepare output directory 100 | output_dir = os.path.join('perturbations/', analyzer, model_name) 101 | if analyzer == 'smooth-taylor': 102 | output_dir = os.path.join(output_dir, noise_folder, num_roots_folder) 103 | elif analyzer == 'ig': 104 | if baseline == 'zero': 105 | output_dir = os.path.join(output_dir, baseline) 106 | elif baseline == 'noise': 107 | output_dir = os.path.join(output_dir, baseline, num_noise_folder) 108 | elif analyzer == 'smooth-grad': 109 | output_dir = os.path.join(output_dir, noise_folder, num_noise_folder) 110 | if not os.path.isdir(output_dir): 111 | os.makedirs(output_dir) 112 | 113 | # Initialize perturbation scores 114 | dataset_size = len(dataset) 115 | all_perturb_scores = np.ones((dataset_size, num_regions + 1)) * np.inf 116 | 117 | # Go through each image 118 | for img_idx, img_filepath in enumerate(tqdm(dataset.img_filepaths, desc='Image')): 119 | # Check if perturbation scores exists 120 | img_filename = os.path.basename(img_filepath) 121 | perturbation_filename = 'perturbations_scores_{0}k_{1}p_{2}r_{3}d_{4}.npy'.format( 122 | kernel_size, num_perturbs, num_regions, draw_mode, img_filename 123 | ) 124 | perturb_outpath = os.path.join(output_dir, perturbation_filename) 125 | if not overwrite and os.path.exists(perturb_outpath): # ignore if the perturbation done 126 | print(img_filename, 'already has perturbation scores computed') 127 | all_perturb_scores[img_idx] = np.load(perturb_outpath) 128 | continue 129 | 130 | # Load the image heatmap 131 | heatmap_filepath = os.path.join(heatmap_dir, img_filename + '_hm.npy') 132 | if not os.path.exists(heatmap_filepath): 133 | print('Heatmaps not found for:', heatmap_filepath) 134 | continue 135 | heatmap = np.load(heatmap_filepath) 136 | 137 | # Save top score as the starting perturb score as the first region 138 | img_input = dataset[img_idx]['image'] 139 | predicted_class = all_top10_idxs[img_idx, 0] 140 | all_perturb_scores[img_idx, 0] = all_scores[img_idx, predicted_class] 141 | 142 | mean_perturb_scores = compute_perturbations( 143 | img_input=img_input, 144 | model=model, 145 | batch_size=batch_size, 146 | explained_class=predicted_class, 147 | heatmap=heatmap, 148 | kernel_size=kernel_size, 149 | draw_mode=draw_mode, 150 | num_regions=num_regions, 151 | num_perturbs=num_perturbs 152 | ) 153 | 154 | # Update all perturb scores and save individual scores 155 | all_perturb_scores[img_idx, 1:] = mean_perturb_scores 156 | np.save(perturb_outpath, all_perturb_scores[img_idx]) 157 | 158 | # Save all perturbation scores 159 | out_filename = 'all_perturbations_scores_{0}k_{1}p_{2}r_{3}d.npy'.format( 160 | kernel_size, num_perturbs, num_regions, draw_mode 161 | ) 162 | outpath = os.path.join(output_dir, out_filename) 163 | np.save(outpath, all_perturb_scores) 164 | 165 | 166 | if __name__ == "__main__": 167 | args = parse_args() 168 | 169 | from datetime import datetime, timedelta 170 | start_time = datetime.now() 171 | 172 | # Load the dataset 173 | dataset = ImageNetValDataset( 174 | root_dir='data/images', 175 | label_dir='data/annotations', 176 | synset_filepath='rsc/synset_words.txt', 177 | max_num=args.num_image 178 | ) 179 | 180 | # Load the pre-trained model 181 | model = MODELS[args.model_name](pretrained=True) 182 | model = model.to(DEVICE) 183 | model.eval() 184 | 185 | # Perform perturbation experiment 186 | run_perturbations_evaluation( 187 | dataset=dataset, 188 | model=model, 189 | model_name=args.model_name, 190 | batch_size=args.batch_size, 191 | analyzer=args.analyzer, 192 | noise_scale=args.noise_scale, 193 | num_roots=args.num_roots, 194 | baseline=args.baseline, 195 | num_noise=args.num_noise, 196 | kernel_size=args.kernel_size, 197 | num_regions=args.num_regions, 198 | num_perturbs=args.num_perturbs, 199 | draw_mode=args.draw_mode, 200 | adaptive_noise=args.adaptive_noise, 201 | adaptive_function=args.adaptive_function, 202 | overwrite=args.overwrite, 203 | noise_percent=args.noise_percent 204 | ) 205 | 206 | end_time = datetime.now() 207 | elapsed_seconds = int((end_time - start_time).total_seconds()) 208 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 209 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 210 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Smooth-Taylor 2 | 3 | _SmoothTaylor_ is a gradient-based attribution method derived from the _Taylor's theorem_ for **deep neural network attribution**. It is proposed as a theoretical bridge between _SmoothGrad_ [(Smilkov et al.)](https://arxiv.org/abs/2004.10484) and _Integrated Gradients_ [(Sundararajan et al.)](https://arxiv.org/abs/1703.01365). 4 | 5 | ![teaser](method-comparison.png "Sample saliency maps of different attribution methods") 6 | 7 | In our paper, we conduct experiments to compare the performance of _SmoothTaylor_ and _Integrated Gradients_ using empirical quantitative measures: _perturbations scores_ and _average total variation_, and show that _SmoothTaylor_ is able to generate attribution maps that are smoother and more sensitive. 8 | 9 | This repository includes a **PyTorch** implementation of _SmoothTaylor_, _SmoothGrad_ and _Integrated Gradients_. 10 | 11 | ## Paper 12 | 13 | Goh, S. W. Goh, S. Lapuschkin, L. Weber, W. Samek, and A. Binder (2021). “Understanding Integrated Gradients with SmoothTaylor for Deep Neural Network Attribution”. In: _2020 25th International Conference on Pattern Recognition (ICPR)_, pp. 4949–4956. DOI:[10.1109/ICPR48806.2021.9413242](https://doi.org/10.1109/ICPR48806.2021.9413242). 14 | 15 | Links: [Paper](https://arxiv.org/abs/2004.10484) • [Presentation](https://garygsw.github.io/assets/pdf/icpr2020/ICPR2020_SmoothTaylor_Slides.pdf) • [Poster](https://garygsw.github.io/assets/pdf/icpr2020/ICPR2020_SmoothTaylor_Poster.pdf) 16 | 17 | ## Setup 18 | 19 | ### Requirement 20 | 21 | Required Python (version 3.7) with standard libraries and following packages version requirements (tested for execution): 22 | 23 | - pytorch 1.4.0 24 | - torchvision 0.5.0 25 | - scikit-image 0.16.2 26 | - pillow 7.0.0 27 | - numpy 1.7.14 28 | - scipy 1.4.1 29 | - tqdm 4.36.1 30 | 31 | Tested in Ubuntu + Intel i7-6700 CPU + RTX 2080 Ti with Cuda (10.1). CPU-only mode also possible, but running with GPU is highly recommended. 32 | 33 | ### Dataset 34 | 35 | The first 1000 images of the [ILSVRC2012 ImageNet object recognition](http://www.image-net.org/challenges/LSVRC/2012/) **validation** dataset is used in our paper's experiment. To replicate our experiment using our experiment code, [download](http://academictorrents.com/details/5d6d0df7ed81efd49ca99ea4737e0ae5e3a5f2e5) or place the dataset into a new folder `./data`, and put the annotations file (in `.xml` formats) in subfolder `./data/annotations` and the raw images in subfolder `./data/images`. Note: required resource files and pre-processing steps for ImageNet are already provided in `./rsc` and in `./attribution/constants.py`. 36 | 37 | ``` 38 | # the ILSVRC2012 ImageNet validation dataset structure should be placed like this 39 | data/ 40 | -annotations/ 41 | -ILSVRC2012_val_{img#}.xml 42 | -images/ 43 | -ILSVRC2012_val_{img#}.JPEG 44 | ``` 45 | 46 | You may also create your own dataset using the PyTorch's `torch.utils.data.Dataset` wrapper to use in your own code. 47 | 48 | ### Models 49 | 50 | In our experiment, we applied attribution on the following deep neural image classifiers: 51 | 52 | - DenseNet121 [(Huang et al., 2017)](https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py), and; 53 | - ResNet152 [(He et al., 2015)](https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py). 54 | 55 | They are both pretrained on the ILSVRC2012 ImageNet dataset, and we use the instance in the default `torchvision` url paths. You may use other pretrained image classifier models that are implemented in PyTorch. Just remember to add the name and instance in the `MODELS` dictionary map in `./attribution/constants.py`. 56 | 57 | ```python 58 | # the current default mapping is: 59 | from torchvision import models 60 | MODELS = {'densenet121': models.densenet121, 61 | 'resnet152': models.resnet152} 62 | ``` 63 | 64 | ## Execution 65 | 66 | To replicate our experiments, please follow the steps in this section. 67 | 68 | 1. First, save the classification outputs of the images using the pre-trained image classifiers: 69 | 70 | ```bash 71 | python experiment_classify.py [-m MODEL_NAME] [-b BATCH_SIZE] 72 | ``` 73 | 74 | Arguments: 75 | 76 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 77 | - `-b BATCH_SIZE` (optional): number of image per epoch (default: 128) 78 | 79 | The classification output should be saved in a new folder `./output/[MODEL_NAME]`. 80 | 81 | 1. Perform the neural network attribution. We implemented 3 gradient-based attribution methods here: 82 | 83 | 1. _SmoothTaylor_ 84 | 85 | ```bash 86 | python experiment_smooth_taylor.py [-m MODEL_NAME] [-b BATCH_SIZE] 87 | [-s NOISE_SCALE] [-r NUM_ROOTS] 88 | ``` 89 | 90 | Arguments: 91 | 92 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 93 | - `-b BATCH_SIZE` (optional): number of image per epoch (default: 50) 94 | - `-s NOISE_SCALE` (optional): magnitude of the noise scale to noise the image (default: 5e-1) 95 | - `-n NUM_ROOTS` (optional): number of noise inputs to use (default: 150) 96 | 97 | 1. _IntegratedGradients_ 98 | 99 | ```bash 100 | python experiment_ig.py [-m MODEL_NAME] [-b BATCH_SIZE] [-k STEPS] 101 | [-z BASELINE_TYPE] [-n NUM_NOISE] 102 | ``` 103 | 104 | Arguments: 105 | 106 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 107 | - `-b BATCH_SIZE` (optional): number of image per epoch (default: 50) 108 | - `-k STEPS` (optional): number of steps along path (default: 50) 109 | - `-z BASELINE_TYPE` (optional): baseline type [use `zero` or `noise`] (default: `zero`) 110 | - `-n NUM_NOISE` (optional): number of noise baselines to use (default: 1) 111 | 112 | 1. _SmoothGrad_ 113 | 114 | ```bash 115 | python experiment_grad.py [-m MODEL_NAME] [-b BATCH_SIZE] [-s] [-p NOISE_SCALE] [-n NUM_NOISE] 116 | ``` 117 | 118 | Arguments: 119 | 120 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 121 | - `-b BATCH_SIZE` (optional): number of image per epoch (default: 50) 122 | - `-s` (optional): to use SmoothGrad or not (default: `False`) 123 | - `-p NOISE_SCALE` (optional): percentage noise scale (default: 15) 124 | - `-n NUM_NOISE` (optional): number of noise inputs to use (default: 50) 125 | 126 | The heatmaps should be saved in a new folder `./heatmaps`, with hyperparameter values as subfolders names e.g. `./heatmaps/[ATTRIBUTION_METHOD]/[MODEL_NAME]/...` 127 | 128 | 1. Evaluate the attribution methods by comparing their heatmaps, using two quantitative evaluation metrics: 129 | 130 | 1. _Perturbation Scores_ for sensitivity 131 | 132 | ```bash 133 | python experiment_perturbations.py [-m MODEL_NAME] [-a ANALYZER] [-b BATCH_SIZE] 134 | [-z BASELINE] [-n NUM_NOISE] 135 | [-s NOISE_SCALE] [-r NUM_ROOTS] 136 | [-k KERNEL_SIZE] [-pt NUM_PERTURBS] 137 | [-l NUM_REGIONS] [-an] 138 | [-af ADAPTIVE_FUNCTION] 139 | ``` 140 | 141 | Arguments: 142 | 143 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 144 | - `-a ANALYZER`: attribution method [use `grad`, `smooth-grad`, `smooth-taylor`, or `ig`] 145 | - `-b BATCH_SIZE` (optional): number of image per epoch (default: 50) 146 | - `-z BASELINE` (optional): IG baseline used [use `zero` or `noise`] (default: `zero`) 147 | - `-n NUM_NOISE` (optional): number of noised baseline in IG (default: 1) 148 | - `-s NOISE_SCALE` (optional): magnitude of noise scale for smoothing (default: 5e-1) 149 | - `-r NUM_ROOTS` (optional): number of noise inputs for smoothing (default: 150) 150 | - `-k KERNEL_SIZE` (optional): size of the window of each perturbation (default: 15) 151 | - `-pt NUM_PERTURBS` (optional): number of random perturbations to evaluate (default: 50) 152 | - `-l NUM_REGIONS` (optional): number of regions to perturbate (default: 30) 153 | - `-an` (optional): use adaptive noise (default: `False`) 154 | - `-af ADAPTIVE_FUNCTION` (optional): objective function for adaptive noising [use `aupc` or `autvc`] (default: `aupc`) 155 | 156 | 1. _Average Total Variation_ for noisiness 157 | 158 | ```bash 159 | python experiment_total_variation.py [-m MODEL_NAME] [-a ANALYZER] 160 | [-z BASELINE] [-n NUM_NOISE] 161 | [-s NOISE_SCALE] [-r NUM_ROOTS] 162 | [-ds DOWNSCALE] [-wms WIDTH_MIN_SIZE] 163 | [-hms HEIGHT_MIN_SIZE] [-lp LP_NORM] 164 | [-an] [-af ADAPTIVE_FUNCTION] 165 | ``` 166 | 167 | Arguments: 168 | 169 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 170 | - `-a ANALYZER`: attribution method [use `grad`, `smooth-grad`, `smooth-taylor`, or `ig`] 171 | - `-z BASELINE` (optional): IG baseline used [use `zero` or `noise`] (default: zero) 172 | - `-n NUM_NOISE` (optional): number of noised baseline in IG (default: 1) 173 | - `-s NOISE_SCALE` (optional): magnitude of noise scale for smoothing (default: 5e-1) 174 | - `-r NUM_ROOTS` (optional): number of noise inputs for smoothing (default: 150) 175 | - `-ds DOWNSCALE` (optional): factor to downscale heatmap (default: 1.5) 176 | - `-wms WIDTH_MIN_SIZE` (optional): minimum width for downscale (default: 30) 177 | - `-hms HEIGHT_MIN_SIZE` (optional): minimum height for downscale (default: 30) 178 | - `-lp LP_NORM` (optional): norm to use to calculate total variation (default: 1) 179 | - `-an` (optional): use adaptive noise (default: `False`) 180 | - `-af ADAPTIVE_FUNCTION` (optional): objective function for adaptive noising [use `aupc` or `autvc`] (default: `aupc`) 181 | 182 | 1. Generate _SmoothTaylor_ heatmaps with _adaptive noising_ hyperparameter tuning technique: 183 | 184 | ```bash 185 | python experiment_adaptive_noising.py [-m MODEL_NAME] [-b BATCH_SIZE] 186 | [-r NUM_ROOTS] [-f OBJ_FUNCTION] 187 | [-ds DOWNSCALE] [-wms WIDTH_MIN_SIZE] 188 | [-hms HEIGHT_MIN_SIZE] [-lp LP_NORM] 189 | [-k KERNEL_SIZE] [-p NUM_PERTURBS] 190 | [-l NUM_REGIONS] [-lr LEARNING_RATE] 191 | [-y LEARNING_DECAY] [-c MAX_STOP_COUNT] 192 | [-x MAX_ITERATION] 193 | ``` 194 | 195 | Arguments: 196 | 197 | - `-m MODEL_NAME`: use `densenet121` or `resnet152` 198 | - `-b BATCH_SIZE` (optional): number of image per epoch (default: 50) 199 | - `-r NUM_ROOTS` (optional): number of noise inputs for smoothing (default: 150) 200 | - `-f OBJ_FUNCTION` (optional): objective function for adaptive noising [use `aupc` or `autvc`] (default: `aupc`) 201 | - `-ds DOWNSCALE` (optional): factor to downscale heatmap (default: 1.5) 202 | - `-wms WIDTH_MIN_SIZE` (optional): minimum width for downscale (default: 30) 203 | - `-hms HEIGHT_MIN_SIZE` (optional): minimum height for downscale (default: 30) 204 | - `-lp LP_NORM` (optional): norm to use to calculate total variation (default: 1) 205 | - `-k KERNEL_SIZE` (optional): size of the window of each perturbation (default: 15) 206 | - `-p NUM_PERTURBS` (optional): number of random perturbations to evaluate (default: 50) 207 | - `-l NUM_REGIONS` (optional): number of regions to perturbate (default: 30) 208 | - `-lr LEARNING_RATE` (optional): learning rate for variable update (default: 0.1) 209 | - `-y LEARNING_DECAY` (optional): decay rate of learning rate (default: 0.9) 210 | - `-c MAX_STOP_COUNT` (optional): maximum stop count to terminate search (default: 3) 211 | - `-x MAX_ITERATION` (optional): maximum iterations to search (default: 20) 212 | 213 | Perform evaluation (see Step 2 above) if required. 214 | 215 | For clearer explanations to what each hyperparameter in the arguments mean, please refer to our paper. 216 | 217 | ## License 218 | 219 | This work is licensed under MIT License. See [LICENSE](LICENSE.md) for details. 220 | 221 | If you find our code or paper useful, please cite our paper: 222 | 223 | ``` 224 | @inproceedings{GohLWSB21Understanding, 225 | author = {Goh, Gary S. W. and Lapuschkin, Sebastian and Weber, Leander and Samek, Wojciech and Binder, Alexander}, 226 | title = {Understanding Integrated Gradients with SmoothTaylor for Deep Neural Network Attribution}, 227 | booktitle = {2020 25th International Conference on Pattern Recognition, (ICPR)}, 228 | pages = {4949--4956}, 229 | publisher = {IEEE}, 230 | year = {2021}, 231 | address = {Virtual Event / Milan, Italy}, 232 | doi = {10.1109/ICPR48806.2021.9413242}, 233 | arxiv = {2004.10484} 234 | } 235 | ``` 236 | 237 | ## Questions 238 | 239 | If you found any bugs, or have any questions, please email to garygsw@gmail.com. 240 | -------------------------------------------------------------------------------- /attribution/eval.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import numpy as np 4 | from torch.utils.data import DataLoader 5 | from scipy.integrate import simps 6 | from skimage.transform import pyramid_gaussian 7 | 8 | from .analyzer import integrated_gradients, smooth_taylor 9 | from .constants import * 10 | 11 | 12 | def heatmap_normalize(heatmap, percentile=99): 13 | heatmap = np.abs(heatmap) 14 | vmin = np.min(heatmap) 15 | vmax = np.percentile(heatmap, percentile) 16 | heatmap = (heatmap - vmin) / (vmax - vmin) 17 | return np.clip(heatmap, 0, 1) 18 | 19 | 20 | def average_total_variation(img, norm=1): 21 | """ 22 | Args: 23 | img (array): image array with the shape (H, W) 24 | """ 25 | img = heatmap_normalize(img) 26 | total_pixels = img.shape[0] * img.shape[1] 27 | x_diff = np.abs(img[:,1:] - img[:,:-1]) 28 | y_diff = np.abs(img[1:,:] - img[:-1,:]) 29 | if norm > 1: 30 | x_diff = np.power(x_diff, norm) 31 | y_diff = np.power(y_diff, norm) 32 | total = np.power(np.sum(x_diff), 1. / norm) + np.power(np.sum(y_diff), 1. / norm) 33 | else: 34 | total = np.sum(x_diff) + np.sum(y_diff) 35 | return total / total_pixels 36 | 37 | 38 | def compute_autvc(img_input, model, batch_size, transform, 39 | analyzer, explained_class, 40 | downscale=1.5, min_size=(30, 30), lp_norm=1, 41 | baseline_type=None, num_noise=None, steps=None, 42 | noise_scale=None, num_roots=None): 43 | # Compute attribution map 44 | if analyzer == 'smooth-taylor': 45 | assert(noise_scale) 46 | assert(num_roots) 47 | attributions = smooth_taylor( 48 | inputs=img_input, 49 | model=model, 50 | batch_size=batch_size, 51 | noise_scale=noise_scale, 52 | num_roots=num_roots, 53 | explained_class=explained_class 54 | ) 55 | elif analyzer == 'ig': 56 | assert(steps) 57 | assert(baseline_type) 58 | if baseline_type == 'zero': 59 | baseline = torch.zeros_like(img_input) 60 | baseline = transform(baseline) 61 | attributions = integrated_gradients( 62 | inputs=img_input, 63 | model=model, 64 | batch_size=batch_size, 65 | baseline=baseline, 66 | explained_class=explained_class, 67 | steps=steps 68 | ) 69 | if baseline_type == 'noise': 70 | assert(num_noise) 71 | all_igs = [] 72 | for _ in range(num_noise): 73 | baseline = torch.rand_like(img_input) # uniform between 0 and 1 74 | baseline = transform(baseline) 75 | ig = integrated_gradients( 76 | inputs=img_input, 77 | model=model, 78 | batch_size=batch_size, 79 | baseline=baseline, 80 | explained_class=explained_class, 81 | steps=steps 82 | ) 83 | all_igs.append(ig) 84 | attributions = np.mean(np.array(all_igs), axis=0) 85 | else: 86 | print('Invalid analyzer:', analyzer) 87 | exit() 88 | 89 | # Create heatmap by summing across channels 90 | heatmap = np.sum(attributions, axis=0) 91 | 92 | # Compute multi-scaled averaged TVs 93 | multi_scaled_ATV = compute_multiscaled_atv( 94 | heatmap=heatmap, 95 | downscale=downscale, 96 | min_size=min_size, 97 | lp_norm=lp_norm 98 | ) 99 | 100 | autvc = simps(multi_scaled_ATV, dx=1) 101 | return heatmap, multi_scaled_ATV, autvc 102 | 103 | 104 | def compute_aupc(img_input, model, batch_size, transform, 105 | analyzer, explained_class, input_score, 106 | kernel_size, draw_mode, num_regions, num_perturbs, 107 | baseline_type=None, num_noise=None, steps=None, 108 | noise_scale=None, num_roots=None): 109 | # Compute attribution map 110 | if analyzer == 'smooth-taylor': 111 | assert(noise_scale) 112 | assert(num_roots) 113 | attributions = smooth_taylor( 114 | inputs=img_input, 115 | model=model, 116 | batch_size=batch_size, 117 | noise_scale=noise_scale, 118 | num_roots=num_roots, 119 | explained_class=explained_class 120 | ) 121 | elif analyzer == 'ig': 122 | assert(steps) 123 | assert(baseline_type) 124 | if baseline_type == 'zero': 125 | baseline = torch.zeros_like(img_input) 126 | baseline = transform(baseline) 127 | attributions = integrated_gradients( 128 | inputs=img_input, 129 | model=model, 130 | batch_size=batch_size, 131 | baseline=baseline, 132 | explained_class=explained_class, 133 | steps=steps 134 | ) 135 | if baseline_type == 'noise': 136 | assert(num_noise) 137 | all_igs = [] 138 | for _ in range(num_noise): 139 | baseline = torch.rand_like(img_input) # uniform between 0 and 1 140 | baseline = transform(baseline) 141 | ig = integrated_gradients( 142 | inputs=img_input, 143 | model=model, 144 | batch_size=batch_size, 145 | baseline=baseline, 146 | explained_class=explained_class, 147 | steps=steps 148 | ) 149 | all_igs.append(ig) 150 | attributions = np.mean(np.array(all_igs), axis=0) 151 | else: 152 | print('Invalid analyzer:', analyzer) 153 | exit() 154 | 155 | # Create heatmap by summing across channels 156 | heatmap = np.sum(attributions, axis=0) 157 | 158 | # Perform perturbation 159 | perturb_scores = compute_perturbations( 160 | img_input=img_input, 161 | model=model, 162 | batch_size=batch_size, 163 | explained_class=explained_class, 164 | heatmap=heatmap, 165 | kernel_size=kernel_size, 166 | draw_mode=draw_mode, 167 | num_regions=num_regions, 168 | num_perturbs=num_perturbs 169 | ) 170 | 171 | # Prepare the perturb scores 172 | perturb_scores = [input_score] + perturb_scores # add original score 173 | perturb_scores = [x / math.fabs(input_score) for x in perturb_scores] # normalize into between 0 and 1 174 | aupc = simps(perturb_scores, dx=1) 175 | return heatmap, perturb_scores, aupc 176 | 177 | 178 | def compute_multiscaled_atv(heatmap, downscale, min_size, lp_norm): 179 | """ 180 | Args: 181 | heatmap (array): heatmap of the attribution (summed across channels) 182 | downscale (float): proportion to reduce dimension at each step 183 | min_size (tuple): minimum dimension to stop pyramid gaussian 184 | lp_norm (int): lp normalization to compute total variation 185 | """ 186 | multiscale_atvs = [] 187 | for (i, resized) in enumerate(pyramid_gaussian(heatmap, downscale=downscale)): 188 | # if the image is too small, break from the loop 189 | if resized.shape[0] < min_size[0] or resized.shape[1] < min_size[1]: 190 | break 191 | multiscale_atvs.append(average_total_variation(resized, norm=lp_norm)) 192 | multiscale_atvs = np.array(multiscale_atvs) 193 | return multiscale_atvs 194 | 195 | 196 | def compute_perturbations(img_input, model, batch_size, explained_class, heatmap, 197 | kernel_size, draw_mode, num_regions, num_perturbs): 198 | """ 199 | Args: 200 | img_input (array): original image 201 | model (PyTorch model): pretrained model 202 | batch_size (int): batch size to run per epoch 203 | explained_class (int): index of the class to be explained 204 | heatmap (array): attribution heatmap for the image 205 | kernel_size (int): size of the window of each perturbation 206 | draw_mode (int): perturb draw mode: 0 - uniform; 1 - gaussian according to image stats 207 | num_regions (int): number of regions to perturbate 208 | num_perturbs (int): number of random perturbations to evaluate 209 | """ 210 | # Find average pooling with kernel 211 | img_h, img_w = heatmap.shape 212 | avg_values = -np.inf * np.ones(img_h * img_w) 213 | 214 | for h in range(img_h - kernel_size): 215 | for w in range(img_w - kernel_size): 216 | avg_values[h+w*img_h] = np.mean(np.abs(heatmap[h:h+kernel_size, w:w+kernel_size])) 217 | most_relevant_idxs = np.argsort(-avg_values) # most relevant first 218 | 219 | # Load original image 220 | img = INVERSE_TRANSFORM(img_input) # TODO: maybe take transform as input? 221 | if draw_mode == 1: # from gaussian according to img stats 222 | # compute mean and std dev. from each channel 223 | channel_stats = np.zeros((2,3)) 224 | for c in range(3): 225 | channel_stats[0, c]= np.mean(img[c,:,:]) 226 | channel_stats[1, c]= np.std(img[c,:,:]) 227 | if np.isnan(channel_stats[1, c]): 228 | channel_stats[1, c] = 1e-3 229 | 230 | # Initialize perturb scores 231 | perturb_scores = [] 232 | 233 | # Select the most relevant point to perform the perturbation 234 | bad_idxs = set() 235 | for region_idx in range(num_regions): 236 | found = False 237 | for i, kernel_idx in enumerate(most_relevant_idxs): 238 | if kernel_idx not in bad_idxs: 239 | # get coordinates of point 240 | width = int(math.floor(kernel_idx / img_h)) 241 | height = int(kernel_idx - width * img_h) 242 | 243 | # ignore if the point is beyond the boundaries of kernel 244 | if (img_h - height) <= kernel_size or (img_w- width) <= kernel_size: 245 | continue 246 | 247 | # mark overlapping neighboring points as not to use in bad_idxs 248 | for h in range(-kernel_size + 1, kernel_size): 249 | for w in range(-kernel_size + 1, kernel_size): 250 | bad_idxs.add((height + h) + (width + w) * img_h) 251 | 252 | found=True 253 | break 254 | if not found: # no more useful points 255 | break 256 | 257 | # Prepare perturbation copies 258 | perturbs = [] 259 | perturbs_imgs = torch.stack([torch.zeros_like(img_input) for _ in range(num_perturbs)]) 260 | 261 | # Compute the perturbation 262 | for i in range(num_perturbs): 263 | if draw_mode == 0: # uniform 264 | # draw randomly from uniform distribution, space is 0,255 265 | perturb = np.random.uniform(low=0, high=255, size=(3, kernel_size, kernel_size)) 266 | elif draw_mode == 1: # from gaussian according to img stats 267 | perturb = np.zeros((3, kernel_size, kernel_size)) 268 | for c in range(3): 269 | perturb[c] = np.random.normal( 270 | loc=channel_stats[0, c], # mean 271 | scale=channel_stats[1, c], # std dev 272 | size=(kernel_size, kernel_size) 273 | ) 274 | # Ensure perturb does not exceed bounds 275 | perturb = np.maximum(perturb, np.zeros_like(perturb)) # element-wise max(p, 0) 276 | perturb = np.minimum(perturb, 255 * np.ones_like(perturb)) # element-wise min(p, 255) 277 | else: 278 | print('Invalid perturb draw mode') 279 | exit() 280 | 281 | # Apply the perturbation to the current point 282 | perturbs_imgs[i] = img 283 | perturb = perturb / 255. # normalize to 0 to 1 284 | perturbs_imgs[i, :, height:height+kernel_size, width:width+kernel_size] = torch.Tensor(perturb) 285 | perturbs_imgs[i] = NORMALIZE_TRANSFORM(perturbs_imgs[i]) # TODO: take transform as input? 286 | perturbs.append(perturb) 287 | 288 | # Perform the classification for the perturbations 289 | perturb_dataset = torch.utils.data.dataset.TensorDataset(perturbs_imgs) 290 | data_loader = DataLoader(perturb_dataset, batch_size=batch_size, shuffle=False) 291 | actual_perturb_idx, mean_score = classify_perturbations(data_loader, model, explained_class) 292 | 293 | # Apply the actual perturbation on image 294 | actual_perturb = perturbs[actual_perturb_idx] 295 | img[:, height:height+kernel_size, width:width+kernel_size] = torch.Tensor(actual_perturb) 296 | 297 | # Save the mean scores of the perturbations 298 | perturb_scores.append(mean_score) 299 | return perturb_scores 300 | 301 | 302 | def classify_perturbations(data_loader, model, explained_class): 303 | """ 304 | Args: 305 | data_loader (DataLoader): object that loads perturbated images 306 | model (model): pre-trained PyTorch model 307 | explained_class (int): index of the class to be explained 308 | """ 309 | # Find the scores for all perturbation candidates 310 | all_scores = [] 311 | for sample_batch in data_loader: 312 | inputs = sample_batch[0] 313 | inputs = inputs.to(DEVICE) 314 | with torch.no_grad(): 315 | out = model(inputs) 316 | scores = out[:, explained_class].cpu().numpy() 317 | all_scores.append(scores) 318 | all_scores = np.array(all_scores) 319 | all_scores = np.concatenate(all_scores) 320 | 321 | # Find the index of the perturbation closest to the median 322 | mean_score = np.mean(all_scores) 323 | median = np.median(all_scores) 324 | best_idx = 0 325 | best_val = np.inf 326 | for i in range(len(all_scores)): # for every perturbation candidate 327 | value = math.fabs(all_scores[i] - median) # absolute distance to median 328 | if value < best_val: 329 | best_val = value 330 | best_idx = i 331 | return best_idx, mean_score 332 | -------------------------------------------------------------------------------- /experiment_adaptive_noising.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import numpy as np 4 | from tqdm import tqdm 5 | import argparse 6 | import torch 7 | from torch.utils.data import DataLoader 8 | 9 | from attribution.eval import compute_aupc, compute_autvc 10 | from attribution.dataset import ImageNetValDataset 11 | from attribution.constants import * 12 | 13 | 14 | def parse_args(): 15 | # Parse arguments 16 | parser = argparse.ArgumentParser(description='Perform adaptive noising per image for smooth integrated gradients') 17 | parser.add_argument('-m', '--model_name', type=str, help='name of the model used to classify') 18 | parser.add_argument('-b', '--batch_size', type=int, help='batch size to use during each epoch', default=50) 19 | parser.add_argument('-r', '--num_roots', type=int, help='number of noised images used', default=150) 20 | parser.add_argument('-f', '--obj_function', type=str, help='objective function to optimize', default='aupc') 21 | parser.add_argument('-ds', '--downscale', type=float, help='factor to downscale heatmap', default=1.5) 22 | parser.add_argument('-wms', '--width_min_size', type=int, help='minimum width dimension for downscale', default=30) 23 | parser.add_argument('-hms', '--height_min_size', type=int, help='minimum height dimension for downscale', default=30) 24 | parser.add_argument('-lp', '--lp_norm', type=int, help='norm to use to calculate TV', default=1) 25 | parser.add_argument('-k', '--kernel_size', type=int, help='size of the window of each perturbation', default=15) 26 | parser.add_argument('-p', '--num_perturbs', type=int, help='number of random perturbations to evaluate', default=50) 27 | parser.add_argument('-l', '--num_regions', type=int, help='number of regions to perturbate', default=30) 28 | parser.add_argument('-d', '--draw_mode', type=int, help='perturb draw mode: 0 - uniform; 1 - gaussian according to image stats', default=0) 29 | parser.add_argument('-lr', '--learning_rate', type=float, help='learning rate for variable update', default=0.1) 30 | parser.add_argument('-y', '--learning_decay', type=float, help='decay rate of learning rate', default=0.9) 31 | parser.add_argument('-c', '--max_stop_count', type=int, help='maximum stop count to terminate search', default=3) 32 | parser.add_argument('-x', '--max_iteration', type=int, help='maximum iterations to search', default=20) 33 | parser.add_argument('-i', '--num_image', type=int, help='number of image data to use from the first', default=1000) 34 | parser.add_argument('-o', '--overwrite', action='store_true', help='overwrite the output') 35 | args = parser.parse_args() 36 | if args.model_name not in MODELS: 37 | print('Invalid model name:', args.model_name) 38 | exit() 39 | if args.obj_function not in ['aupc', 'autvc']: 40 | print('Invalid objective function:', args.obj_function) 41 | exit() 42 | return args 43 | 44 | 45 | def adapt_noise(img_input, model, num_roots, learning_rate, learning_decay, 46 | max_stop_count, max_iteration, obj_function, params): 47 | # Initialize values 48 | if obj_function == 'aupc': 49 | compute_auc = compute_aupc 50 | elif obj_function == 'autvc': 51 | compute_auc = compute_autvc 52 | current_noise = np.mean(np.abs(img_input.numpy())) # initialize noise = absolute mean 53 | params['noise_scale'] = current_noise 54 | current_heatmap, current_score, current_auc = compute_auc(**params) 55 | # print('initial noise:', current_noise) 56 | # print('initial auc:', current_auc) 57 | best_noise = current_noise 58 | best_auc = current_auc 59 | best_score = current_score 60 | best_heatmap = current_heatmap 61 | stop_count = 0 62 | lr = learning_rate 63 | iteration = 1 64 | 65 | while iteration <= max_iteration: 66 | # Find direction 67 | params['noise_scale'] = math.fabs(current_noise + lr) 68 | current_heatmap, current_score, search_auc = compute_auc(**params) 69 | if search_auc > current_auc: 70 | current_noise = math.fabs(current_noise - lr) 71 | params['noise_scale'] = current_noise 72 | current_heatmap, current_score, search_auc = compute_auc(**params) 73 | else: 74 | current_noise = math.fabs(current_noise + lr) 75 | print('update noise -', 'noise:', current_noise) 76 | 77 | # Early stopping 78 | if search_auc > current_auc: # worse 79 | if stop_count < max_stop_count: 80 | lr = lr * learning_decay # reduce lr 81 | stop_count += 1 82 | else: 83 | print('finished -', 'best auc:', best_auc, 'best noise:', best_noise) 84 | break # exit the loop 85 | else: # improved 86 | stop_count = 0 87 | if search_auc < best_auc: 88 | best_auc = search_auc 89 | best_noise = current_noise 90 | best_score = current_score 91 | best_heatmap = current_heatmap 92 | print('update best -', 'best auc:', best_auc, 'best noise:', best_noise) 93 | current_auc = search_auc 94 | iteration += 1 95 | return best_noise, best_heatmap, best_auc, best_score 96 | 97 | 98 | def run_adapt_noise_experiment(dataset, model, model_name, batch_size, transform, 99 | num_roots, obj_function, learning_rate, learning_decay, 100 | max_stop_count, max_iteration, downscale, min_size, lp_norm, 101 | kernel_size, num_regions, draw_mode, num_perturbs, 102 | percent=False, overwrite=False): 103 | # Read all top10idxs and all scores for that model (only for aupc objective function) 104 | input_dir = os.path.join('output/', model_name) 105 | if not os.path.exists(input_dir): 106 | print('Model classification output not found for:', model_name) 107 | exit() 108 | input_path = os.path.join(input_dir, 'all_scores.npy') 109 | all_model_scores = np.load(input_path) 110 | input_path = os.path.join(input_dir, 'all_top10_idxs.npy') 111 | all_top10_idxs = np.load(input_path) 112 | 113 | # Prepare heatmap output directory 114 | output_dir = os.path.join('adaptive/', model_name, obj_function) 115 | if not os.path.isdir(output_dir): 116 | os.makedirs(output_dir) 117 | heatmap_dir = os.path.join('heatmaps/', 'smooth-taylor', model_name, 'adaptive', obj_function, str(num_roots) + 'N') 118 | if not os.path.isdir(heatmap_dir): 119 | os.makedirs(heatmap_dir) 120 | 121 | # Prepare output file formats 122 | all_scores = [] 123 | if obj_function == 'aupc': 124 | params_format = ('{num_roots}N_{learning_rate}lr_{learning_decay}ld_' 125 | '{max_stop_count}s_{kernel_size}k_{num_perturbs}p_' 126 | '{num_regions}r_{draw_mode}d') 127 | single_params_format = params_format + '_{img_filename}' 128 | score_file_format = 'perturbations_scores_' + single_params_format + '.npy' 129 | all_scores_file_format = 'all_perturb_scores_' + params_format + '.npy' 130 | exp_params = { 131 | 'num_roots': num_roots, 132 | 'learning_rate': learning_rate, 133 | 'learning_decay': learning_decay, 134 | 'max_stop_count': max_stop_count, 135 | 'kernel_size': kernel_size, 136 | 'num_perturbs': num_perturbs, 137 | 'num_regions': num_regions, 138 | 'draw_mode': draw_mode 139 | } 140 | elif obj_function == 'autvc': 141 | params_format = ('{num_roots}N_{learning_rate}lr_{learning_decay}ld_' 142 | '{max_stop_count}s_{downscale}ds_{min_size}ms_{lp_norm}l') 143 | single_params_format = params_format + '_{img_filename}' 144 | score_file_format = 'atv_scores_' + single_params_format + '.npy' 145 | all_scores_file_format = 'all_atv_scores_' + params_format +'.npy' 146 | exp_params = { 147 | 'num_roots': num_roots, 148 | 'learning_rate': learning_rate, 149 | 'learning_decay': learning_decay, 150 | 'max_stop_count': max_stop_count, 151 | 'downscale': downscale, 152 | 'min_size': min_size, 153 | 'lp_norm': lp_norm 154 | } 155 | noise_file_format = 'noise_scores_' + single_params_format + '.npy' 156 | all_noise_file_format = 'all_noise_' + params_format + '.npy' 157 | 158 | # Prepare overall output file paths 159 | all_noise_scores_filename = all_noise_file_format.format(**exp_params) 160 | all_noise_scores_filepath = os.path.join(output_dir, all_noise_scores_filename) 161 | all_scores_filename = all_scores_file_format.format(**exp_params) 162 | all_scores_filepath = os.path.join(output_dir, all_scores_filename) 163 | 164 | # Initialize noise scores 165 | dataset_size = len(dataset) 166 | all_noise_scores = np.zeros((dataset_size, 2)) # noise, AUC score 167 | 168 | # Go through each image 169 | for img_idx, img_filepath in enumerate(tqdm(dataset.img_filepaths, desc='Image')): 170 | # Check if adaptive noise output already exists 171 | img_filename = os.path.basename(img_filepath) 172 | exp_params['img_filename'] = img_filename 173 | 174 | # Prepare noise score outpath 175 | noise_score_filename = noise_file_format.format(**exp_params) 176 | noise_score_outpath = os.path.join(output_dir, noise_score_filename) 177 | 178 | # Prepare score output 179 | scores_filename = score_file_format.format(**exp_params) 180 | scores_outpath = os.path.join(output_dir, scores_filename) 181 | 182 | # Prepare best heatmap output 183 | best_heatmap_outpath = os.path.join(heatmap_dir, img_filename + '_hm.npy') 184 | 185 | if not overwrite and os.path.exists(noise_score_outpath) and \ 186 | os.path.exists(scores_outpath) and \ 187 | os.path.exists(best_heatmap_outpath): # ignore if already generated 188 | print(img_filename, 'already has noise scores generated') 189 | all_noise_scores[img_idx] = np.load(noise_score_outpath) 190 | all_scores.append(np.load(scores_outpath)) 191 | continue 192 | 193 | # Initialize parameters 194 | img_input = dataset[img_idx]['image'] 195 | predicted_class = all_top10_idxs[img_idx, 0] 196 | if obj_function == 'aupc': 197 | # Retrieve the image data, predicted class and score 198 | input_score = all_model_scores[img_idx, predicted_class] 199 | params = { 200 | 'img_input': img_input, 201 | 'model': model, 202 | 'batch_size': batch_size, 203 | 'transform': transform, 204 | 'analyzer': 'smooth-taylor', 205 | 'explained_class': predicted_class, 206 | 'input_score': input_score, 207 | 'num_roots': num_roots, 208 | 'kernel_size': kernel_size, 209 | 'draw_mode': draw_mode, 210 | 'num_regions': num_regions, 211 | 'num_perturbs': num_perturbs, 212 | } 213 | elif obj_function == 'autvc': 214 | params = { 215 | 'img_input': img_input, 216 | 'model': model, 217 | 'batch_size': batch_size, 218 | 'transform': transform, 219 | 'analyzer': 'smooth-taylor', 220 | 'explained_class': predicted_class, 221 | 'num_roots': num_roots, 222 | 'downscale': downscale, 223 | 'min_size': min_size, 224 | 'lp_norm': lp_norm, 225 | } 226 | 227 | # Find best noise scale 228 | best_noise, best_heatmap, best_auc, best_scores = adapt_noise( 229 | img_input=img_input, 230 | model=model, 231 | num_roots=num_roots, 232 | learning_rate=learning_rate, 233 | learning_decay=learning_decay, 234 | max_stop_count=max_stop_count, 235 | max_iteration=max_iteration, 236 | obj_function=obj_function, 237 | params=params 238 | ) 239 | 240 | # Save the best heatmap 241 | np.save(best_heatmap_outpath, best_heatmap) 242 | 243 | # Save the best noise and auc scores 244 | all_noise_scores[img_idx, 0] = best_noise 245 | all_noise_scores[img_idx, 1] = best_auc 246 | all_scores.append(best_scores) 247 | scores_outpath = os.path.join(output_dir, scores_filename) 248 | np.save(scores_outpath, best_scores) 249 | np.save(noise_score_outpath, all_noise_scores[img_idx]) 250 | 251 | # Save all scores 252 | np.save(all_scores_filepath, np.array(all_scores)) 253 | np.save(all_noise_scores_filepath, all_noise_scores) 254 | 255 | 256 | if __name__ == "__main__": 257 | args = parse_args() 258 | 259 | from datetime import datetime, timedelta 260 | start_time = datetime.now() 261 | 262 | # Load the dataset 263 | dataset = ImageNetValDataset( 264 | root_dir='data/images', 265 | label_dir='data/annotations', 266 | synset_filepath='rsc/synset_words.txt', 267 | max_num=args.num_image 268 | ) 269 | 270 | # Load the pre-trained model 271 | model = MODELS[args.model_name](pretrained=True) 272 | model = model.to(DEVICE) 273 | model.eval() 274 | 275 | # Perform experiment 276 | run_adapt_noise_experiment( 277 | dataset=dataset, 278 | model=model, 279 | model_name=args.model_name, 280 | batch_size=args.batch_size, 281 | transform=NORMALIZE_TRANSFORM, 282 | obj_function=args.obj_function, 283 | num_roots=args.num_roots, 284 | learning_rate=args.learning_rate, 285 | learning_decay=args.learning_decay, 286 | max_stop_count=args.max_stop_count, 287 | max_iteration=args.max_iteration, 288 | overwrite=args.overwrite, 289 | # autvc params: 290 | downscale=args.downscale, 291 | min_size=(args.height_min_size, args.width_min_size), 292 | lp_norm=args.lp_norm, 293 | # aupc params: 294 | kernel_size=args.kernel_size, 295 | num_regions=args.num_regions, 296 | draw_mode=args.draw_mode, 297 | num_perturbs=args.num_perturbs 298 | ) 299 | 300 | end_time = datetime.now() 301 | elapsed_seconds = int((end_time - start_time).total_seconds()) 302 | print('Start time:', start_time.strftime('%d %b %Y %H:%M:%S')) 303 | print('End time:', end_time.strftime('%d %b %Y %H:%M:%S')) 304 | print('Elapsed time:', timedelta(seconds=elapsed_seconds)) -------------------------------------------------------------------------------- /rsc/imagenet_classes.txt: -------------------------------------------------------------------------------- 1 | tench, Tinca tinca 2 | goldfish, Carassius auratus 3 | great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias 4 | tiger shark, Galeocerdo cuvieri 5 | hammerhead, hammerhead shark 6 | electric ray, crampfish, numbfish, torpedo 7 | stingray 8 | cock 9 | hen 10 | ostrich, Struthio camelus 11 | brambling, Fringilla montifringilla 12 | goldfinch, Carduelis carduelis 13 | house finch, linnet, Carpodacus mexicanus 14 | junco, snowbird 15 | indigo bunting, indigo finch, indigo bird, Passerina cyanea 16 | robin, American robin, Turdus migratorius 17 | bulbul 18 | jay 19 | magpie 20 | chickadee 21 | water ouzel, dipper 22 | kite 23 | bald eagle, American eagle, Haliaeetus leucocephalus 24 | vulture 25 | great grey owl, great gray owl, Strix nebulosa 26 | European fire salamander, Salamandra salamandra 27 | common newt, Triturus vulgaris 28 | eft 29 | spotted salamander, Ambystoma maculatum 30 | axolotl, mud puppy, Ambystoma mexicanum 31 | bullfrog, Rana catesbeiana 32 | tree frog, tree-frog 33 | tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui 34 | loggerhead, loggerhead turtle, Caretta caretta 35 | leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea 36 | mud turtle 37 | terrapin 38 | box turtle, box tortoise 39 | banded gecko 40 | common iguana, iguana, Iguana iguana 41 | American chameleon, anole, Anolis carolinensis 42 | whiptail, whiptail lizard 43 | agama 44 | frilled lizard, Chlamydosaurus kingi 45 | alligator lizard 46 | Gila monster, Heloderma suspectum 47 | green lizard, Lacerta viridis 48 | African chameleon, Chamaeleo chamaeleon 49 | Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis 50 | African crocodile, Nile crocodile, Crocodylus niloticus 51 | American alligator, Alligator mississipiensis 52 | triceratops 53 | thunder snake, worm snake, Carphophis amoenus 54 | ringneck snake, ring-necked snake, ring snake 55 | hognose snake, puff adder, sand viper 56 | green snake, grass snake 57 | king snake, kingsnake 58 | garter snake, grass snake 59 | water snake 60 | vine snake 61 | night snake, Hypsiglena torquata 62 | boa constrictor, Constrictor constrictor 63 | rock python, rock snake, Python sebae 64 | Indian cobra, Naja naja 65 | green mamba 66 | sea snake 67 | horned viper, cerastes, sand viper, horned asp, Cerastes cornutus 68 | diamondback, diamondback rattlesnake, Crotalus adamanteus 69 | sidewinder, horned rattlesnake, Crotalus cerastes 70 | trilobite 71 | harvestman, daddy longlegs, Phalangium opilio 72 | scorpion 73 | black and gold garden spider, Argiope aurantia 74 | barn spider, Araneus cavaticus 75 | garden spider, Aranea diademata 76 | black widow, Latrodectus mactans 77 | tarantula 78 | wolf spider, hunting spider 79 | tick 80 | centipede 81 | black grouse 82 | ptarmigan 83 | ruffed grouse, partridge, Bonasa umbellus 84 | prairie chicken, prairie grouse, prairie fowl 85 | peacock 86 | quail 87 | partridge 88 | African grey, African gray, Psittacus erithacus 89 | macaw 90 | sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita 91 | lorikeet 92 | coucal 93 | bee eater 94 | hornbill 95 | hummingbird 96 | jacamar 97 | toucan 98 | drake 99 | red-breasted merganser, Mergus serrator 100 | goose 101 | black swan, Cygnus atratus 102 | tusker 103 | echidna, spiny anteater, anteater 104 | platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus 105 | wallaby, brush kangaroo 106 | koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus 107 | wombat 108 | jellyfish 109 | sea anemone, anemone 110 | brain coral 111 | flatworm, platyhelminth 112 | nematode, nematode worm, roundworm 113 | conch 114 | snail 115 | slug 116 | sea slug, nudibranch 117 | chiton, coat-of-mail shell, sea cradle, polyplacophore 118 | chambered nautilus, pearly nautilus, nautilus 119 | Dungeness crab, Cancer magister 120 | rock crab, Cancer irroratus 121 | fiddler crab 122 | king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica 123 | American lobster, Northern lobster, Maine lobster, Homarus americanus 124 | spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish 125 | crayfish, crawfish, crawdad, crawdaddy 126 | hermit crab 127 | isopod 128 | white stork, Ciconia ciconia 129 | black stork, Ciconia nigra 130 | spoonbill 131 | flamingo 132 | little blue heron, Egretta caerulea 133 | American egret, great white heron, Egretta albus 134 | bittern 135 | crane 136 | limpkin, Aramus pictus 137 | European gallinule, Porphyrio porphyrio 138 | American coot, marsh hen, mud hen, water hen, Fulica americana 139 | bustard 140 | ruddy turnstone, Arenaria interpres 141 | red-backed sandpiper, dunlin, Erolia alpina 142 | redshank, Tringa totanus 143 | dowitcher 144 | oystercatcher, oyster catcher 145 | pelican 146 | king penguin, Aptenodytes patagonica 147 | albatross, mollymawk 148 | grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus 149 | killer whale, killer, orca, grampus, sea wolf, Orcinus orca 150 | dugong, Dugong dugon 151 | sea lion 152 | Chihuahua 153 | Japanese spaniel 154 | Maltese dog, Maltese terrier, Maltese 155 | Pekinese, Pekingese, Peke 156 | Shih-Tzu 157 | Blenheim spaniel 158 | papillon 159 | toy terrier 160 | Rhodesian ridgeback 161 | Afghan hound, Afghan 162 | basset, basset hound 163 | beagle 164 | bloodhound, sleuthhound 165 | bluetick 166 | black-and-tan coonhound 167 | Walker hound, Walker foxhound 168 | English foxhound 169 | redbone 170 | borzoi, Russian wolfhound 171 | Irish wolfhound 172 | Italian greyhound 173 | whippet 174 | Ibizan hound, Ibizan Podenco 175 | Norwegian elkhound, elkhound 176 | otterhound, otter hound 177 | Saluki, gazelle hound 178 | Scottish deerhound, deerhound 179 | Weimaraner 180 | Staffordshire bullterrier, Staffordshire bull terrier 181 | American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier 182 | Bedlington terrier 183 | Border terrier 184 | Kerry blue terrier 185 | Irish terrier 186 | Norfolk terrier 187 | Norwich terrier 188 | Yorkshire terrier 189 | wire-haired fox terrier 190 | Lakeland terrier 191 | Sealyham terrier, Sealyham 192 | Airedale, Airedale terrier 193 | cairn, cairn terrier 194 | Australian terrier 195 | Dandie Dinmont, Dandie Dinmont terrier 196 | Boston bull, Boston terrier 197 | miniature schnauzer 198 | giant schnauzer 199 | standard schnauzer 200 | Scotch terrier, Scottish terrier, Scottie 201 | Tibetan terrier, chrysanthemum dog 202 | silky terrier, Sydney silky 203 | soft-coated wheaten terrier 204 | West Highland white terrier 205 | Lhasa, Lhasa apso 206 | flat-coated retriever 207 | curly-coated retriever 208 | golden retriever 209 | Labrador retriever 210 | Chesapeake Bay retriever 211 | German short-haired pointer 212 | vizsla, Hungarian pointer 213 | English setter 214 | Irish setter, red setter 215 | Gordon setter 216 | Brittany spaniel 217 | clumber, clumber spaniel 218 | English springer, English springer spaniel 219 | Welsh springer spaniel 220 | cocker spaniel, English cocker spaniel, cocker 221 | Sussex spaniel 222 | Irish water spaniel 223 | kuvasz 224 | schipperke 225 | groenendael 226 | malinois 227 | briard 228 | kelpie 229 | komondor 230 | Old English sheepdog, bobtail 231 | Shetland sheepdog, Shetland sheep dog, Shetland 232 | collie 233 | Border collie 234 | Bouvier des Flandres, Bouviers des Flandres 235 | Rottweiler 236 | German shepherd, German shepherd dog, German police dog, alsatian 237 | Doberman, Doberman pinscher 238 | miniature pinscher 239 | Greater Swiss Mountain dog 240 | Bernese mountain dog 241 | Appenzeller 242 | EntleBucher 243 | boxer 244 | bull mastiff 245 | Tibetan mastiff 246 | French bulldog 247 | Great Dane 248 | Saint Bernard, St Bernard 249 | Eskimo dog, husky 250 | malamute, malemute, Alaskan malamute 251 | Siberian husky 252 | dalmatian, coach dog, carriage dog 253 | affenpinscher, monkey pinscher, monkey dog 254 | basenji 255 | pug, pug-dog 256 | Leonberg 257 | Newfoundland, Newfoundland dog 258 | Great Pyrenees 259 | Samoyed, Samoyede 260 | Pomeranian 261 | chow, chow chow 262 | keeshond 263 | Brabancon griffon 264 | Pembroke, Pembroke Welsh corgi 265 | Cardigan, Cardigan Welsh corgi 266 | toy poodle 267 | miniature poodle 268 | standard poodle 269 | Mexican hairless 270 | timber wolf, grey wolf, gray wolf, Canis lupus 271 | white wolf, Arctic wolf, Canis lupus tundrarum 272 | red wolf, maned wolf, Canis rufus, Canis niger 273 | coyote, prairie wolf, brush wolf, Canis latrans 274 | dingo, warrigal, warragal, Canis dingo 275 | dhole, Cuon alpinus 276 | African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus 277 | hyena, hyaena 278 | red fox, Vulpes vulpes 279 | kit fox, Vulpes macrotis 280 | Arctic fox, white fox, Alopex lagopus 281 | grey fox, gray fox, Urocyon cinereoargenteus 282 | tabby, tabby cat 283 | tiger cat 284 | Persian cat 285 | Siamese cat, Siamese 286 | Egyptian cat 287 | cougar, puma, catamount, mountain lion, painter, panther, Felis concolor 288 | lynx, catamount 289 | leopard, Panthera pardus 290 | snow leopard, ounce, Panthera uncia 291 | jaguar, panther, Panthera onca, Felis onca 292 | lion, king of beasts, Panthera leo 293 | tiger, Panthera tigris 294 | cheetah, chetah, Acinonyx jubatus 295 | brown bear, bruin, Ursus arctos 296 | American black bear, black bear, Ursus americanus, Euarctos americanus 297 | ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus 298 | sloth bear, Melursus ursinus, Ursus ursinus 299 | mongoose 300 | meerkat, mierkat 301 | tiger beetle 302 | ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle 303 | ground beetle, carabid beetle 304 | long-horned beetle, longicorn, longicorn beetle 305 | leaf beetle, chrysomelid 306 | dung beetle 307 | rhinoceros beetle 308 | weevil 309 | fly 310 | bee 311 | ant, emmet, pismire 312 | grasshopper, hopper 313 | cricket 314 | walking stick, walkingstick, stick insect 315 | cockroach, roach 316 | mantis, mantid 317 | cicada, cicala 318 | leafhopper 319 | lacewing, lacewing fly 320 | dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk 321 | damselfly 322 | admiral 323 | ringlet, ringlet butterfly 324 | monarch, monarch butterfly, milkweed butterfly, Danaus plexippus 325 | cabbage butterfly 326 | sulphur butterfly, sulfur butterfly 327 | lycaenid, lycaenid butterfly 328 | starfish, sea star 329 | sea urchin 330 | sea cucumber, holothurian 331 | wood rabbit, cottontail, cottontail rabbit 332 | hare 333 | Angora, Angora rabbit 334 | hamster 335 | porcupine, hedgehog 336 | fox squirrel, eastern fox squirrel, Sciurus niger 337 | marmot 338 | beaver 339 | guinea pig, Cavia cobaya 340 | sorrel 341 | zebra 342 | hog, pig, grunter, squealer, Sus scrofa 343 | wild boar, boar, Sus scrofa 344 | warthog 345 | hippopotamus, hippo, river horse, Hippopotamus amphibius 346 | ox 347 | water buffalo, water ox, Asiatic buffalo, Bubalus bubalis 348 | bison 349 | ram, tup 350 | bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis 351 | ibex, Capra ibex 352 | hartebeest 353 | impala, Aepyceros melampus 354 | gazelle 355 | Arabian camel, dromedary, Camelus dromedarius 356 | llama 357 | weasel 358 | mink 359 | polecat, fitch, foulmart, foumart, Mustela putorius 360 | black-footed ferret, ferret, Mustela nigripes 361 | otter 362 | skunk, polecat, wood pussy 363 | badger 364 | armadillo 365 | three-toed sloth, ai, Bradypus tridactylus 366 | orangutan, orang, orangutang, Pongo pygmaeus 367 | gorilla, Gorilla gorilla 368 | chimpanzee, chimp, Pan troglodytes 369 | gibbon, Hylobates lar 370 | siamang, Hylobates syndactylus, Symphalangus syndactylus 371 | guenon, guenon monkey 372 | patas, hussar monkey, Erythrocebus patas 373 | baboon 374 | macaque 375 | langur 376 | colobus, colobus monkey 377 | proboscis monkey, Nasalis larvatus 378 | marmoset 379 | capuchin, ringtail, Cebus capucinus 380 | howler monkey, howler 381 | titi, titi monkey 382 | spider monkey, Ateles geoffroyi 383 | squirrel monkey, Saimiri sciureus 384 | Madagascar cat, ring-tailed lemur, Lemur catta 385 | indri, indris, Indri indri, Indri brevicaudatus 386 | Indian elephant, Elephas maximus 387 | African elephant, Loxodonta africana 388 | lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens 389 | giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca 390 | barracouta, snoek 391 | eel 392 | coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch 393 | rock beauty, Holocanthus tricolor 394 | anemone fish 395 | sturgeon 396 | gar, garfish, garpike, billfish, Lepisosteus osseus 397 | lionfish 398 | puffer, pufferfish, blowfish, globefish 399 | abacus 400 | abaya 401 | academic gown, academic robe, judge's robe 402 | accordion, piano accordion, squeeze box 403 | acoustic guitar 404 | aircraft carrier, carrier, flattop, attack aircraft carrier 405 | airliner 406 | airship, dirigible 407 | altar 408 | ambulance 409 | amphibian, amphibious vehicle 410 | analog clock 411 | apiary, bee house 412 | apron 413 | ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin 414 | assault rifle, assault gun 415 | backpack, back pack, knapsack, packsack, rucksack, haversack 416 | bakery, bakeshop, bakehouse 417 | balance beam, beam 418 | balloon 419 | ballpoint, ballpoint pen, ballpen, Biro 420 | Band Aid 421 | banjo 422 | bannister, banister, balustrade, balusters, handrail 423 | barbell 424 | barber chair 425 | barbershop 426 | barn 427 | barometer 428 | barrel, cask 429 | barrow, garden cart, lawn cart, wheelbarrow 430 | baseball 431 | basketball 432 | bassinet 433 | bassoon 434 | bathing cap, swimming cap 435 | bath towel 436 | bathtub, bathing tub, bath, tub 437 | beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon 438 | beacon, lighthouse, beacon light, pharos 439 | beaker 440 | bearskin, busby, shako 441 | beer bottle 442 | beer glass 443 | bell cote, bell cot 444 | bib 445 | bicycle-built-for-two, tandem bicycle, tandem 446 | bikini, two-piece 447 | binder, ring-binder 448 | binoculars, field glasses, opera glasses 449 | birdhouse 450 | boathouse 451 | bobsled, bobsleigh, bob 452 | bolo tie, bolo, bola tie, bola 453 | bonnet, poke bonnet 454 | bookcase 455 | bookshop, bookstore, bookstall 456 | bottlecap 457 | bow 458 | bow tie, bow-tie, bowtie 459 | brass, memorial tablet, plaque 460 | brassiere, bra, bandeau 461 | breakwater, groin, groyne, mole, bulwark, seawall, jetty 462 | breastplate, aegis, egis 463 | broom 464 | bucket, pail 465 | buckle 466 | bulletproof vest 467 | bullet train, bullet 468 | butcher shop, meat market 469 | cab, hack, taxi, taxicab 470 | caldron, cauldron 471 | candle, taper, wax light 472 | cannon 473 | canoe 474 | can opener, tin opener 475 | cardigan 476 | car mirror 477 | carousel, carrousel, merry-go-round, roundabout, whirligig 478 | carpenter's kit, tool kit 479 | carton 480 | car wheel 481 | cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM 482 | cassette 483 | cassette player 484 | castle 485 | catamaran 486 | CD player 487 | cello, violoncello 488 | cellular telephone, cellular phone, cellphone, cell, mobile phone 489 | chain 490 | chainlink fence 491 | chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour 492 | chain saw, chainsaw 493 | chest 494 | chiffonier, commode 495 | chime, bell, gong 496 | china cabinet, china closet 497 | Christmas stocking 498 | church, church building 499 | cinema, movie theater, movie theatre, movie house, picture palace 500 | cleaver, meat cleaver, chopper 501 | cliff dwelling 502 | cloak 503 | clog, geta, patten, sabot 504 | cocktail shaker 505 | coffee mug 506 | coffeepot 507 | coil, spiral, volute, whorl, helix 508 | combination lock 509 | computer keyboard, keypad 510 | confectionery, confectionary, candy store 511 | container ship, containership, container vessel 512 | convertible 513 | corkscrew, bottle screw 514 | cornet, horn, trumpet, trump 515 | cowboy boot 516 | cowboy hat, ten-gallon hat 517 | cradle 518 | crane 519 | crash helmet 520 | crate 521 | crib, cot 522 | Crock Pot 523 | croquet ball 524 | crutch 525 | cuirass 526 | dam, dike, dyke 527 | desk 528 | desktop computer 529 | dial telephone, dial phone 530 | diaper, nappy, napkin 531 | digital clock 532 | digital watch 533 | dining table, board 534 | dishrag, dishcloth 535 | dishwasher, dish washer, dishwashing machine 536 | disk brake, disc brake 537 | dock, dockage, docking facility 538 | dogsled, dog sled, dog sleigh 539 | dome 540 | doormat, welcome mat 541 | drilling platform, offshore rig 542 | drum, membranophone, tympan 543 | drumstick 544 | dumbbell 545 | Dutch oven 546 | electric fan, blower 547 | electric guitar 548 | electric locomotive 549 | entertainment center 550 | envelope 551 | espresso maker 552 | face powder 553 | feather boa, boa 554 | file, file cabinet, filing cabinet 555 | fireboat 556 | fire engine, fire truck 557 | fire screen, fireguard 558 | flagpole, flagstaff 559 | flute, transverse flute 560 | folding chair 561 | football helmet 562 | forklift 563 | fountain 564 | fountain pen 565 | four-poster 566 | freight car 567 | French horn, horn 568 | frying pan, frypan, skillet 569 | fur coat 570 | garbage truck, dustcart 571 | gasmask, respirator, gas helmet 572 | gas pump, gasoline pump, petrol pump, island dispenser 573 | goblet 574 | go-kart 575 | golf ball 576 | golfcart, golf cart 577 | gondola 578 | gong, tam-tam 579 | gown 580 | grand piano, grand 581 | greenhouse, nursery, glasshouse 582 | grille, radiator grille 583 | grocery store, grocery, food market, market 584 | guillotine 585 | hair slide 586 | hair spray 587 | half track 588 | hammer 589 | hamper 590 | hand blower, blow dryer, blow drier, hair dryer, hair drier 591 | hand-held computer, hand-held microcomputer 592 | handkerchief, hankie, hanky, hankey 593 | hard disc, hard disk, fixed disk 594 | harmonica, mouth organ, harp, mouth harp 595 | harp 596 | harvester, reaper 597 | hatchet 598 | holster 599 | home theater, home theatre 600 | honeycomb 601 | hook, claw 602 | hoopskirt, crinoline 603 | horizontal bar, high bar 604 | horse cart, horse-cart 605 | hourglass 606 | iPod 607 | iron, smoothing iron 608 | jack-o'-lantern 609 | jean, blue jean, denim 610 | jeep, landrover 611 | jersey, T-shirt, tee shirt 612 | jigsaw puzzle 613 | jinrikisha, ricksha, rickshaw 614 | joystick 615 | kimono 616 | knee pad 617 | knot 618 | lab coat, laboratory coat 619 | ladle 620 | lampshade, lamp shade 621 | laptop, laptop computer 622 | lawn mower, mower 623 | lens cap, lens cover 624 | letter opener, paper knife, paperknife 625 | library 626 | lifeboat 627 | lighter, light, igniter, ignitor 628 | limousine, limo 629 | liner, ocean liner 630 | lipstick, lip rouge 631 | Loafer 632 | lotion 633 | loudspeaker, speaker, speaker unit, loudspeaker system, speaker system 634 | loupe, jeweler's loupe 635 | lumbermill, sawmill 636 | magnetic compass 637 | mailbag, postbag 638 | mailbox, letter box 639 | maillot 640 | maillot, tank suit 641 | manhole cover 642 | maraca 643 | marimba, xylophone 644 | mask 645 | matchstick 646 | maypole 647 | maze, labyrinth 648 | measuring cup 649 | medicine chest, medicine cabinet 650 | megalith, megalithic structure 651 | microphone, mike 652 | microwave, microwave oven 653 | military uniform 654 | milk can 655 | minibus 656 | miniskirt, mini 657 | minivan 658 | missile 659 | mitten 660 | mixing bowl 661 | mobile home, manufactured home 662 | Model T 663 | modem 664 | monastery 665 | monitor 666 | moped 667 | mortar 668 | mortarboard 669 | mosque 670 | mosquito net 671 | motor scooter, scooter 672 | mountain bike, all-terrain bike, off-roader 673 | mountain tent 674 | mouse, computer mouse 675 | mousetrap 676 | moving van 677 | muzzle 678 | nail 679 | neck brace 680 | necklace 681 | nipple 682 | notebook, notebook computer 683 | obelisk 684 | oboe, hautboy, hautbois 685 | ocarina, sweet potato 686 | odometer, hodometer, mileometer, milometer 687 | oil filter 688 | organ, pipe organ 689 | oscilloscope, scope, cathode-ray oscilloscope, CRO 690 | overskirt 691 | oxcart 692 | oxygen mask 693 | packet 694 | paddle, boat paddle 695 | paddlewheel, paddle wheel 696 | padlock 697 | paintbrush 698 | pajama, pyjama, pj's, jammies 699 | palace 700 | panpipe, pandean pipe, syrinx 701 | paper towel 702 | parachute, chute 703 | parallel bars, bars 704 | park bench 705 | parking meter 706 | passenger car, coach, carriage 707 | patio, terrace 708 | pay-phone, pay-station 709 | pedestal, plinth, footstall 710 | pencil box, pencil case 711 | pencil sharpener 712 | perfume, essence 713 | Petri dish 714 | photocopier 715 | pick, plectrum, plectron 716 | pickelhaube 717 | picket fence, paling 718 | pickup, pickup truck 719 | pier 720 | piggy bank, penny bank 721 | pill bottle 722 | pillow 723 | ping-pong ball 724 | pinwheel 725 | pirate, pirate ship 726 | pitcher, ewer 727 | plane, carpenter's plane, woodworking plane 728 | planetarium 729 | plastic bag 730 | plate rack 731 | plow, plough 732 | plunger, plumber's helper 733 | Polaroid camera, Polaroid Land camera 734 | pole 735 | police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria 736 | poncho 737 | pool table, billiard table, snooker table 738 | pop bottle, soda bottle 739 | pot, flowerpot 740 | potter's wheel 741 | power drill 742 | prayer rug, prayer mat 743 | printer 744 | prison, prison house 745 | projectile, missile 746 | projector 747 | puck, hockey puck 748 | punching bag, punch bag, punching ball, punchball 749 | purse 750 | quill, quill pen 751 | quilt, comforter, comfort, puff 752 | racer, race car, racing car 753 | racket, racquet 754 | radiator 755 | radio, wireless 756 | radio telescope, radio reflector 757 | rain barrel 758 | recreational vehicle, RV, R.V. 759 | reel 760 | reflex camera 761 | refrigerator, icebox 762 | remote control, remote 763 | restaurant, eating house, eating place, eatery 764 | revolver, six-gun, six-shooter 765 | rifle 766 | rocking chair, rocker 767 | rotisserie 768 | rubber eraser, rubber, pencil eraser 769 | rugby ball 770 | rule, ruler 771 | running shoe 772 | safe 773 | safety pin 774 | saltshaker, salt shaker 775 | sandal 776 | sarong 777 | sax, saxophone 778 | scabbard 779 | scale, weighing machine 780 | school bus 781 | schooner 782 | scoreboard 783 | screen, CRT screen 784 | screw 785 | screwdriver 786 | seat belt, seatbelt 787 | sewing machine 788 | shield, buckler 789 | shoe shop, shoe-shop, shoe store 790 | shoji 791 | shopping basket 792 | shopping cart 793 | shovel 794 | shower cap 795 | shower curtain 796 | ski 797 | ski mask 798 | sleeping bag 799 | slide rule, slipstick 800 | sliding door 801 | slot, one-armed bandit 802 | snorkel 803 | snowmobile 804 | snowplow, snowplough 805 | soap dispenser 806 | soccer ball 807 | sock 808 | solar dish, solar collector, solar furnace 809 | sombrero 810 | soup bowl 811 | space bar 812 | space heater 813 | space shuttle 814 | spatula 815 | speedboat 816 | spider web, spider's web 817 | spindle 818 | sports car, sport car 819 | spotlight, spot 820 | stage 821 | steam locomotive 822 | steel arch bridge 823 | steel drum 824 | stethoscope 825 | stole 826 | stone wall 827 | stopwatch, stop watch 828 | stove 829 | strainer 830 | streetcar, tram, tramcar, trolley, trolley car 831 | stretcher 832 | studio couch, day bed 833 | stupa, tope 834 | submarine, pigboat, sub, U-boat 835 | suit, suit of clothes 836 | sundial 837 | sunglass 838 | sunglasses, dark glasses, shades 839 | sunscreen, sunblock, sun blocker 840 | suspension bridge 841 | swab, swob, mop 842 | sweatshirt 843 | swimming trunks, bathing trunks 844 | swing 845 | switch, electric switch, electrical switch 846 | syringe 847 | table lamp 848 | tank, army tank, armored combat vehicle, armoured combat vehicle 849 | tape player 850 | teapot 851 | teddy, teddy bear 852 | television, television system 853 | tennis ball 854 | thatch, thatched roof 855 | theater curtain, theatre curtain 856 | thimble 857 | thresher, thrasher, threshing machine 858 | throne 859 | tile roof 860 | toaster 861 | tobacco shop, tobacconist shop, tobacconist 862 | toilet seat 863 | torch 864 | totem pole 865 | tow truck, tow car, wrecker 866 | toyshop 867 | tractor 868 | trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi 869 | tray 870 | trench coat 871 | tricycle, trike, velocipede 872 | trimaran 873 | tripod 874 | triumphal arch 875 | trolleybus, trolley coach, trackless trolley 876 | trombone 877 | tub, vat 878 | turnstile 879 | typewriter keyboard 880 | umbrella 881 | unicycle, monocycle 882 | upright, upright piano 883 | vacuum, vacuum cleaner 884 | vase 885 | vault 886 | velvet 887 | vending machine 888 | vestment 889 | viaduct 890 | violin, fiddle 891 | volleyball 892 | waffle iron 893 | wall clock 894 | wallet, billfold, notecase, pocketbook 895 | wardrobe, closet, press 896 | warplane, military plane 897 | washbasin, handbasin, washbowl, lavabo, wash-hand basin 898 | washer, automatic washer, washing machine 899 | water bottle 900 | water jug 901 | water tower 902 | whiskey jug 903 | whistle 904 | wig 905 | window screen 906 | window shade 907 | Windsor tie 908 | wine bottle 909 | wing 910 | wok 911 | wooden spoon 912 | wool, woolen, woollen 913 | worm fence, snake fence, snake-rail fence, Virginia fence 914 | wreck 915 | yawl 916 | yurt 917 | web site, website, internet site, site 918 | comic book 919 | crossword puzzle, crossword 920 | street sign 921 | traffic light, traffic signal, stoplight 922 | book jacket, dust cover, dust jacket, dust wrapper 923 | menu 924 | plate 925 | guacamole 926 | consomme 927 | hot pot, hotpot 928 | trifle 929 | ice cream, icecream 930 | ice lolly, lolly, lollipop, popsicle 931 | French loaf 932 | bagel, beigel 933 | pretzel 934 | cheeseburger 935 | hotdog, hot dog, red hot 936 | mashed potato 937 | head cabbage 938 | broccoli 939 | cauliflower 940 | zucchini, courgette 941 | spaghetti squash 942 | acorn squash 943 | butternut squash 944 | cucumber, cuke 945 | artichoke, globe artichoke 946 | bell pepper 947 | cardoon 948 | mushroom 949 | Granny Smith 950 | strawberry 951 | orange 952 | lemon 953 | fig 954 | pineapple, ananas 955 | banana 956 | jackfruit, jak, jack 957 | custard apple 958 | pomegranate 959 | hay 960 | carbonara 961 | chocolate sauce, chocolate syrup 962 | dough 963 | meat loaf, meatloaf 964 | pizza, pizza pie 965 | potpie 966 | burrito 967 | red wine 968 | espresso 969 | cup 970 | eggnog 971 | alp 972 | bubble 973 | cliff, drop, drop-off 974 | coral reef 975 | geyser 976 | lakeside, lakeshore 977 | promontory, headland, head, foreland 978 | sandbar, sand bar 979 | seashore, coast, seacoast, sea-coast 980 | valley, vale 981 | volcano 982 | ballplayer, baseball player 983 | groom, bridegroom 984 | scuba diver 985 | rapeseed 986 | daisy 987 | yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum 988 | corn 989 | acorn 990 | hip, rose hip, rosehip 991 | buckeye, horse chestnut, conker 992 | coral fungus 993 | agaric 994 | gyromitra 995 | stinkhorn, carrion fungus 996 | earthstar 997 | hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa 998 | bolete 999 | ear, spike, capitulum 1000 | toilet tissue, toilet paper, bathroom tissue -------------------------------------------------------------------------------- /rsc/synset_words.txt: -------------------------------------------------------------------------------- 1 | n01440764 tench, Tinca tinca 2 | n01443537 goldfish, Carassius auratus 3 | n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias 4 | n01491361 tiger shark, Galeocerdo cuvieri 5 | n01494475 hammerhead, hammerhead shark 6 | n01496331 electric ray, crampfish, numbfish, torpedo 7 | n01498041 stingray 8 | n01514668 cock 9 | n01514859 hen 10 | n01518878 ostrich, Struthio camelus 11 | n01530575 brambling, Fringilla montifringilla 12 | n01531178 goldfinch, Carduelis carduelis 13 | n01532829 house finch, linnet, Carpodacus mexicanus 14 | n01534433 junco, snowbird 15 | n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea 16 | n01558993 robin, American robin, Turdus migratorius 17 | n01560419 bulbul 18 | n01580077 jay 19 | n01582220 magpie 20 | n01592084 chickadee 21 | n01601694 water ouzel, dipper 22 | n01608432 kite 23 | n01614925 bald eagle, American eagle, Haliaeetus leucocephalus 24 | n01616318 vulture 25 | n01622779 great grey owl, great gray owl, Strix nebulosa 26 | n01629819 European fire salamander, Salamandra salamandra 27 | n01630670 common newt, Triturus vulgaris 28 | n01631663 eft 29 | n01632458 spotted salamander, Ambystoma maculatum 30 | n01632777 axolotl, mud puppy, Ambystoma mexicanum 31 | n01641577 bullfrog, Rana catesbeiana 32 | n01644373 tree frog, tree-frog 33 | n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui 34 | n01664065 loggerhead, loggerhead turtle, Caretta caretta 35 | n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea 36 | n01667114 mud turtle 37 | n01667778 terrapin 38 | n01669191 box turtle, box tortoise 39 | n01675722 banded gecko 40 | n01677366 common iguana, iguana, Iguana iguana 41 | n01682714 American chameleon, anole, Anolis carolinensis 42 | n01685808 whiptail, whiptail lizard 43 | n01687978 agama 44 | n01688243 frilled lizard, Chlamydosaurus kingi 45 | n01689811 alligator lizard 46 | n01692333 Gila monster, Heloderma suspectum 47 | n01693334 green lizard, Lacerta viridis 48 | n01694178 African chameleon, Chamaeleo chamaeleon 49 | n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis 50 | n01697457 African crocodile, Nile crocodile, Crocodylus niloticus 51 | n01698640 American alligator, Alligator mississipiensis 52 | n01704323 triceratops 53 | n01728572 thunder snake, worm snake, Carphophis amoenus 54 | n01728920 ringneck snake, ring-necked snake, ring snake 55 | n01729322 hognose snake, puff adder, sand viper 56 | n01729977 green snake, grass snake 57 | n01734418 king snake, kingsnake 58 | n01735189 garter snake, grass snake 59 | n01737021 water snake 60 | n01739381 vine snake 61 | n01740131 night snake, Hypsiglena torquata 62 | n01742172 boa constrictor, Constrictor constrictor 63 | n01744401 rock python, rock snake, Python sebae 64 | n01748264 Indian cobra, Naja naja 65 | n01749939 green mamba 66 | n01751748 sea snake 67 | n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus 68 | n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus 69 | n01756291 sidewinder, horned rattlesnake, Crotalus cerastes 70 | n01768244 trilobite 71 | n01770081 harvestman, daddy longlegs, Phalangium opilio 72 | n01770393 scorpion 73 | n01773157 black and gold garden spider, Argiope aurantia 74 | n01773549 barn spider, Araneus cavaticus 75 | n01773797 garden spider, Aranea diademata 76 | n01774384 black widow, Latrodectus mactans 77 | n01774750 tarantula 78 | n01775062 wolf spider, hunting spider 79 | n01776313 tick 80 | n01784675 centipede 81 | n01795545 black grouse 82 | n01796340 ptarmigan 83 | n01797886 ruffed grouse, partridge, Bonasa umbellus 84 | n01798484 prairie chicken, prairie grouse, prairie fowl 85 | n01806143 peacock 86 | n01806567 quail 87 | n01807496 partridge 88 | n01817953 African grey, African gray, Psittacus erithacus 89 | n01818515 macaw 90 | n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita 91 | n01820546 lorikeet 92 | n01824575 coucal 93 | n01828970 bee eater 94 | n01829413 hornbill 95 | n01833805 hummingbird 96 | n01843065 jacamar 97 | n01843383 toucan 98 | n01847000 drake 99 | n01855032 red-breasted merganser, Mergus serrator 100 | n01855672 goose 101 | n01860187 black swan, Cygnus atratus 102 | n01871265 tusker 103 | n01872401 echidna, spiny anteater, anteater 104 | n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus 105 | n01877812 wallaby, brush kangaroo 106 | n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus 107 | n01883070 wombat 108 | n01910747 jellyfish 109 | n01914609 sea anemone, anemone 110 | n01917289 brain coral 111 | n01924916 flatworm, platyhelminth 112 | n01930112 nematode, nematode worm, roundworm 113 | n01943899 conch 114 | n01944390 snail 115 | n01945685 slug 116 | n01950731 sea slug, nudibranch 117 | n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore 118 | n01968897 chambered nautilus, pearly nautilus, nautilus 119 | n01978287 Dungeness crab, Cancer magister 120 | n01978455 rock crab, Cancer irroratus 121 | n01980166 fiddler crab 122 | n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica 123 | n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus 124 | n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish 125 | n01985128 crayfish, crawfish, crawdad, crawdaddy 126 | n01986214 hermit crab 127 | n01990800 isopod 128 | n02002556 white stork, Ciconia ciconia 129 | n02002724 black stork, Ciconia nigra 130 | n02006656 spoonbill 131 | n02007558 flamingo 132 | n02009229 little blue heron, Egretta caerulea 133 | n02009912 American egret, great white heron, Egretta albus 134 | n02011460 bittern 135 | n02012849 crane 136 | n02013706 limpkin, Aramus pictus 137 | n02017213 European gallinule, Porphyrio porphyrio 138 | n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana 139 | n02018795 bustard 140 | n02025239 ruddy turnstone, Arenaria interpres 141 | n02027492 red-backed sandpiper, dunlin, Erolia alpina 142 | n02028035 redshank, Tringa totanus 143 | n02033041 dowitcher 144 | n02037110 oystercatcher, oyster catcher 145 | n02051845 pelican 146 | n02056570 king penguin, Aptenodytes patagonica 147 | n02058221 albatross, mollymawk 148 | n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus 149 | n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca 150 | n02074367 dugong, Dugong dugon 151 | n02077923 sea lion 152 | n02085620 Chihuahua 153 | n02085782 Japanese spaniel 154 | n02085936 Maltese dog, Maltese terrier, Maltese 155 | n02086079 Pekinese, Pekingese, Peke 156 | n02086240 Shih-Tzu 157 | n02086646 Blenheim spaniel 158 | n02086910 papillon 159 | n02087046 toy terrier 160 | n02087394 Rhodesian ridgeback 161 | n02088094 Afghan hound, Afghan 162 | n02088238 basset, basset hound 163 | n02088364 beagle 164 | n02088466 bloodhound, sleuthhound 165 | n02088632 bluetick 166 | n02089078 black-and-tan coonhound 167 | n02089867 Walker hound, Walker foxhound 168 | n02089973 English foxhound 169 | n02090379 redbone 170 | n02090622 borzoi, Russian wolfhound 171 | n02090721 Irish wolfhound 172 | n02091032 Italian greyhound 173 | n02091134 whippet 174 | n02091244 Ibizan hound, Ibizan Podenco 175 | n02091467 Norwegian elkhound, elkhound 176 | n02091635 otterhound, otter hound 177 | n02091831 Saluki, gazelle hound 178 | n02092002 Scottish deerhound, deerhound 179 | n02092339 Weimaraner 180 | n02093256 Staffordshire bullterrier, Staffordshire bull terrier 181 | n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier 182 | n02093647 Bedlington terrier 183 | n02093754 Border terrier 184 | n02093859 Kerry blue terrier 185 | n02093991 Irish terrier 186 | n02094114 Norfolk terrier 187 | n02094258 Norwich terrier 188 | n02094433 Yorkshire terrier 189 | n02095314 wire-haired fox terrier 190 | n02095570 Lakeland terrier 191 | n02095889 Sealyham terrier, Sealyham 192 | n02096051 Airedale, Airedale terrier 193 | n02096177 cairn, cairn terrier 194 | n02096294 Australian terrier 195 | n02096437 Dandie Dinmont, Dandie Dinmont terrier 196 | n02096585 Boston bull, Boston terrier 197 | n02097047 miniature schnauzer 198 | n02097130 giant schnauzer 199 | n02097209 standard schnauzer 200 | n02097298 Scotch terrier, Scottish terrier, Scottie 201 | n02097474 Tibetan terrier, chrysanthemum dog 202 | n02097658 silky terrier, Sydney silky 203 | n02098105 soft-coated wheaten terrier 204 | n02098286 West Highland white terrier 205 | n02098413 Lhasa, Lhasa apso 206 | n02099267 flat-coated retriever 207 | n02099429 curly-coated retriever 208 | n02099601 golden retriever 209 | n02099712 Labrador retriever 210 | n02099849 Chesapeake Bay retriever 211 | n02100236 German short-haired pointer 212 | n02100583 vizsla, Hungarian pointer 213 | n02100735 English setter 214 | n02100877 Irish setter, red setter 215 | n02101006 Gordon setter 216 | n02101388 Brittany spaniel 217 | n02101556 clumber, clumber spaniel 218 | n02102040 English springer, English springer spaniel 219 | n02102177 Welsh springer spaniel 220 | n02102318 cocker spaniel, English cocker spaniel, cocker 221 | n02102480 Sussex spaniel 222 | n02102973 Irish water spaniel 223 | n02104029 kuvasz 224 | n02104365 schipperke 225 | n02105056 groenendael 226 | n02105162 malinois 227 | n02105251 briard 228 | n02105412 kelpie 229 | n02105505 komondor 230 | n02105641 Old English sheepdog, bobtail 231 | n02105855 Shetland sheepdog, Shetland sheep dog, Shetland 232 | n02106030 collie 233 | n02106166 Border collie 234 | n02106382 Bouvier des Flandres, Bouviers des Flandres 235 | n02106550 Rottweiler 236 | n02106662 German shepherd, German shepherd dog, German police dog, alsatian 237 | n02107142 Doberman, Doberman pinscher 238 | n02107312 miniature pinscher 239 | n02107574 Greater Swiss Mountain dog 240 | n02107683 Bernese mountain dog 241 | n02107908 Appenzeller 242 | n02108000 EntleBucher 243 | n02108089 boxer 244 | n02108422 bull mastiff 245 | n02108551 Tibetan mastiff 246 | n02108915 French bulldog 247 | n02109047 Great Dane 248 | n02109525 Saint Bernard, St Bernard 249 | n02109961 Eskimo dog, husky 250 | n02110063 malamute, malemute, Alaskan malamute 251 | n02110185 Siberian husky 252 | n02110341 dalmatian, coach dog, carriage dog 253 | n02110627 affenpinscher, monkey pinscher, monkey dog 254 | n02110806 basenji 255 | n02110958 pug, pug-dog 256 | n02111129 Leonberg 257 | n02111277 Newfoundland, Newfoundland dog 258 | n02111500 Great Pyrenees 259 | n02111889 Samoyed, Samoyede 260 | n02112018 Pomeranian 261 | n02112137 chow, chow chow 262 | n02112350 keeshond 263 | n02112706 Brabancon griffon 264 | n02113023 Pembroke, Pembroke Welsh corgi 265 | n02113186 Cardigan, Cardigan Welsh corgi 266 | n02113624 toy poodle 267 | n02113712 miniature poodle 268 | n02113799 standard poodle 269 | n02113978 Mexican hairless 270 | n02114367 timber wolf, grey wolf, gray wolf, Canis lupus 271 | n02114548 white wolf, Arctic wolf, Canis lupus tundrarum 272 | n02114712 red wolf, maned wolf, Canis rufus, Canis niger 273 | n02114855 coyote, prairie wolf, brush wolf, Canis latrans 274 | n02115641 dingo, warrigal, warragal, Canis dingo 275 | n02115913 dhole, Cuon alpinus 276 | n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus 277 | n02117135 hyena, hyaena 278 | n02119022 red fox, Vulpes vulpes 279 | n02119789 kit fox, Vulpes macrotis 280 | n02120079 Arctic fox, white fox, Alopex lagopus 281 | n02120505 grey fox, gray fox, Urocyon cinereoargenteus 282 | n02123045 tabby, tabby cat 283 | n02123159 tiger cat 284 | n02123394 Persian cat 285 | n02123597 Siamese cat, Siamese 286 | n02124075 Egyptian cat 287 | n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor 288 | n02127052 lynx, catamount 289 | n02128385 leopard, Panthera pardus 290 | n02128757 snow leopard, ounce, Panthera uncia 291 | n02128925 jaguar, panther, Panthera onca, Felis onca 292 | n02129165 lion, king of beasts, Panthera leo 293 | n02129604 tiger, Panthera tigris 294 | n02130308 cheetah, chetah, Acinonyx jubatus 295 | n02132136 brown bear, bruin, Ursus arctos 296 | n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus 297 | n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus 298 | n02134418 sloth bear, Melursus ursinus, Ursus ursinus 299 | n02137549 mongoose 300 | n02138441 meerkat, mierkat 301 | n02165105 tiger beetle 302 | n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle 303 | n02167151 ground beetle, carabid beetle 304 | n02168699 long-horned beetle, longicorn, longicorn beetle 305 | n02169497 leaf beetle, chrysomelid 306 | n02172182 dung beetle 307 | n02174001 rhinoceros beetle 308 | n02177972 weevil 309 | n02190166 fly 310 | n02206856 bee 311 | n02219486 ant, emmet, pismire 312 | n02226429 grasshopper, hopper 313 | n02229544 cricket 314 | n02231487 walking stick, walkingstick, stick insect 315 | n02233338 cockroach, roach 316 | n02236044 mantis, mantid 317 | n02256656 cicada, cicala 318 | n02259212 leafhopper 319 | n02264363 lacewing, lacewing fly 320 | n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk 321 | n02268853 damselfly 322 | n02276258 admiral 323 | n02277742 ringlet, ringlet butterfly 324 | n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus 325 | n02280649 cabbage butterfly 326 | n02281406 sulphur butterfly, sulfur butterfly 327 | n02281787 lycaenid, lycaenid butterfly 328 | n02317335 starfish, sea star 329 | n02319095 sea urchin 330 | n02321529 sea cucumber, holothurian 331 | n02325366 wood rabbit, cottontail, cottontail rabbit 332 | n02326432 hare 333 | n02328150 Angora, Angora rabbit 334 | n02342885 hamster 335 | n02346627 porcupine, hedgehog 336 | n02356798 fox squirrel, eastern fox squirrel, Sciurus niger 337 | n02361337 marmot 338 | n02363005 beaver 339 | n02364673 guinea pig, Cavia cobaya 340 | n02389026 sorrel 341 | n02391049 zebra 342 | n02395406 hog, pig, grunter, squealer, Sus scrofa 343 | n02396427 wild boar, boar, Sus scrofa 344 | n02397096 warthog 345 | n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius 346 | n02403003 ox 347 | n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis 348 | n02410509 bison 349 | n02412080 ram, tup 350 | n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis 351 | n02417914 ibex, Capra ibex 352 | n02422106 hartebeest 353 | n02422699 impala, Aepyceros melampus 354 | n02423022 gazelle 355 | n02437312 Arabian camel, dromedary, Camelus dromedarius 356 | n02437616 llama 357 | n02441942 weasel 358 | n02442845 mink 359 | n02443114 polecat, fitch, foulmart, foumart, Mustela putorius 360 | n02443484 black-footed ferret, ferret, Mustela nigripes 361 | n02444819 otter 362 | n02445715 skunk, polecat, wood pussy 363 | n02447366 badger 364 | n02454379 armadillo 365 | n02457408 three-toed sloth, ai, Bradypus tridactylus 366 | n02480495 orangutan, orang, orangutang, Pongo pygmaeus 367 | n02480855 gorilla, Gorilla gorilla 368 | n02481823 chimpanzee, chimp, Pan troglodytes 369 | n02483362 gibbon, Hylobates lar 370 | n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus 371 | n02484975 guenon, guenon monkey 372 | n02486261 patas, hussar monkey, Erythrocebus patas 373 | n02486410 baboon 374 | n02487347 macaque 375 | n02488291 langur 376 | n02488702 colobus, colobus monkey 377 | n02489166 proboscis monkey, Nasalis larvatus 378 | n02490219 marmoset 379 | n02492035 capuchin, ringtail, Cebus capucinus 380 | n02492660 howler monkey, howler 381 | n02493509 titi, titi monkey 382 | n02493793 spider monkey, Ateles geoffroyi 383 | n02494079 squirrel monkey, Saimiri sciureus 384 | n02497673 Madagascar cat, ring-tailed lemur, Lemur catta 385 | n02500267 indri, indris, Indri indri, Indri brevicaudatus 386 | n02504013 Indian elephant, Elephas maximus 387 | n02504458 African elephant, Loxodonta africana 388 | n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens 389 | n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca 390 | n02514041 barracouta, snoek 391 | n02526121 eel 392 | n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch 393 | n02606052 rock beauty, Holocanthus tricolor 394 | n02607072 anemone fish 395 | n02640242 sturgeon 396 | n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus 397 | n02643566 lionfish 398 | n02655020 puffer, pufferfish, blowfish, globefish 399 | n02666196 abacus 400 | n02667093 abaya 401 | n02669723 academic gown, academic robe, judge's robe 402 | n02672831 accordion, piano accordion, squeeze box 403 | n02676566 acoustic guitar 404 | n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier 405 | n02690373 airliner 406 | n02692877 airship, dirigible 407 | n02699494 altar 408 | n02701002 ambulance 409 | n02704792 amphibian, amphibious vehicle 410 | n02708093 analog clock 411 | n02727426 apiary, bee house 412 | n02730930 apron 413 | n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin 414 | n02749479 assault rifle, assault gun 415 | n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack 416 | n02776631 bakery, bakeshop, bakehouse 417 | n02777292 balance beam, beam 418 | n02782093 balloon 419 | n02783161 ballpoint, ballpoint pen, ballpen, Biro 420 | n02786058 Band Aid 421 | n02787622 banjo 422 | n02788148 bannister, banister, balustrade, balusters, handrail 423 | n02790996 barbell 424 | n02791124 barber chair 425 | n02791270 barbershop 426 | n02793495 barn 427 | n02794156 barometer 428 | n02795169 barrel, cask 429 | n02797295 barrow, garden cart, lawn cart, wheelbarrow 430 | n02799071 baseball 431 | n02802426 basketball 432 | n02804414 bassinet 433 | n02804610 bassoon 434 | n02807133 bathing cap, swimming cap 435 | n02808304 bath towel 436 | n02808440 bathtub, bathing tub, bath, tub 437 | n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon 438 | n02814860 beacon, lighthouse, beacon light, pharos 439 | n02815834 beaker 440 | n02817516 bearskin, busby, shako 441 | n02823428 beer bottle 442 | n02823750 beer glass 443 | n02825657 bell cote, bell cot 444 | n02834397 bib 445 | n02835271 bicycle-built-for-two, tandem bicycle, tandem 446 | n02837789 bikini, two-piece 447 | n02840245 binder, ring-binder 448 | n02841315 binoculars, field glasses, opera glasses 449 | n02843684 birdhouse 450 | n02859443 boathouse 451 | n02860847 bobsled, bobsleigh, bob 452 | n02865351 bolo tie, bolo, bola tie, bola 453 | n02869837 bonnet, poke bonnet 454 | n02870880 bookcase 455 | n02871525 bookshop, bookstore, bookstall 456 | n02877765 bottlecap 457 | n02879718 bow 458 | n02883205 bow tie, bow-tie, bowtie 459 | n02892201 brass, memorial tablet, plaque 460 | n02892767 brassiere, bra, bandeau 461 | n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty 462 | n02895154 breastplate, aegis, egis 463 | n02906734 broom 464 | n02909870 bucket, pail 465 | n02910353 buckle 466 | n02916936 bulletproof vest 467 | n02917067 bullet train, bullet 468 | n02927161 butcher shop, meat market 469 | n02930766 cab, hack, taxi, taxicab 470 | n02939185 caldron, cauldron 471 | n02948072 candle, taper, wax light 472 | n02950826 cannon 473 | n02951358 canoe 474 | n02951585 can opener, tin opener 475 | n02963159 cardigan 476 | n02965783 car mirror 477 | n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig 478 | n02966687 carpenter's kit, tool kit 479 | n02971356 carton 480 | n02974003 car wheel 481 | n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM 482 | n02978881 cassette 483 | n02979186 cassette player 484 | n02980441 castle 485 | n02981792 catamaran 486 | n02988304 CD player 487 | n02992211 cello, violoncello 488 | n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone 489 | n02999410 chain 490 | n03000134 chainlink fence 491 | n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour 492 | n03000684 chain saw, chainsaw 493 | n03014705 chest 494 | n03016953 chiffonier, commode 495 | n03017168 chime, bell, gong 496 | n03018349 china cabinet, china closet 497 | n03026506 Christmas stocking 498 | n03028079 church, church building 499 | n03032252 cinema, movie theater, movie theatre, movie house, picture palace 500 | n03041632 cleaver, meat cleaver, chopper 501 | n03042490 cliff dwelling 502 | n03045698 cloak 503 | n03047690 clog, geta, patten, sabot 504 | n03062245 cocktail shaker 505 | n03063599 coffee mug 506 | n03063689 coffeepot 507 | n03065424 coil, spiral, volute, whorl, helix 508 | n03075370 combination lock 509 | n03085013 computer keyboard, keypad 510 | n03089624 confectionery, confectionary, candy store 511 | n03095699 container ship, containership, container vessel 512 | n03100240 convertible 513 | n03109150 corkscrew, bottle screw 514 | n03110669 cornet, horn, trumpet, trump 515 | n03124043 cowboy boot 516 | n03124170 cowboy hat, ten-gallon hat 517 | n03125729 cradle 518 | n03126707 crane 519 | n03127747 crash helmet 520 | n03127925 crate 521 | n03131574 crib, cot 522 | n03133878 Crock Pot 523 | n03134739 croquet ball 524 | n03141823 crutch 525 | n03146219 cuirass 526 | n03160309 dam, dike, dyke 527 | n03179701 desk 528 | n03180011 desktop computer 529 | n03187595 dial telephone, dial phone 530 | n03188531 diaper, nappy, napkin 531 | n03196217 digital clock 532 | n03197337 digital watch 533 | n03201208 dining table, board 534 | n03207743 dishrag, dishcloth 535 | n03207941 dishwasher, dish washer, dishwashing machine 536 | n03208938 disk brake, disc brake 537 | n03216828 dock, dockage, docking facility 538 | n03218198 dogsled, dog sled, dog sleigh 539 | n03220513 dome 540 | n03223299 doormat, welcome mat 541 | n03240683 drilling platform, offshore rig 542 | n03249569 drum, membranophone, tympan 543 | n03250847 drumstick 544 | n03255030 dumbbell 545 | n03259280 Dutch oven 546 | n03271574 electric fan, blower 547 | n03272010 electric guitar 548 | n03272562 electric locomotive 549 | n03290653 entertainment center 550 | n03291819 envelope 551 | n03297495 espresso maker 552 | n03314780 face powder 553 | n03325584 feather boa, boa 554 | n03337140 file, file cabinet, filing cabinet 555 | n03344393 fireboat 556 | n03345487 fire engine, fire truck 557 | n03347037 fire screen, fireguard 558 | n03355925 flagpole, flagstaff 559 | n03372029 flute, transverse flute 560 | n03376595 folding chair 561 | n03379051 football helmet 562 | n03384352 forklift 563 | n03388043 fountain 564 | n03388183 fountain pen 565 | n03388549 four-poster 566 | n03393912 freight car 567 | n03394916 French horn, horn 568 | n03400231 frying pan, frypan, skillet 569 | n03404251 fur coat 570 | n03417042 garbage truck, dustcart 571 | n03424325 gasmask, respirator, gas helmet 572 | n03425413 gas pump, gasoline pump, petrol pump, island dispenser 573 | n03443371 goblet 574 | n03444034 go-kart 575 | n03445777 golf ball 576 | n03445924 golfcart, golf cart 577 | n03447447 gondola 578 | n03447721 gong, tam-tam 579 | n03450230 gown 580 | n03452741 grand piano, grand 581 | n03457902 greenhouse, nursery, glasshouse 582 | n03459775 grille, radiator grille 583 | n03461385 grocery store, grocery, food market, market 584 | n03467068 guillotine 585 | n03476684 hair slide 586 | n03476991 hair spray 587 | n03478589 half track 588 | n03481172 hammer 589 | n03482405 hamper 590 | n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier 591 | n03485407 hand-held computer, hand-held microcomputer 592 | n03485794 handkerchief, hankie, hanky, hankey 593 | n03492542 hard disc, hard disk, fixed disk 594 | n03494278 harmonica, mouth organ, harp, mouth harp 595 | n03495258 harp 596 | n03496892 harvester, reaper 597 | n03498962 hatchet 598 | n03527444 holster 599 | n03529860 home theater, home theatre 600 | n03530642 honeycomb 601 | n03532672 hook, claw 602 | n03534580 hoopskirt, crinoline 603 | n03535780 horizontal bar, high bar 604 | n03538406 horse cart, horse-cart 605 | n03544143 hourglass 606 | n03584254 iPod 607 | n03584829 iron, smoothing iron 608 | n03590841 jack-o'-lantern 609 | n03594734 jean, blue jean, denim 610 | n03594945 jeep, landrover 611 | n03595614 jersey, T-shirt, tee shirt 612 | n03598930 jigsaw puzzle 613 | n03599486 jinrikisha, ricksha, rickshaw 614 | n03602883 joystick 615 | n03617480 kimono 616 | n03623198 knee pad 617 | n03627232 knot 618 | n03630383 lab coat, laboratory coat 619 | n03633091 ladle 620 | n03637318 lampshade, lamp shade 621 | n03642806 laptop, laptop computer 622 | n03649909 lawn mower, mower 623 | n03657121 lens cap, lens cover 624 | n03658185 letter opener, paper knife, paperknife 625 | n03661043 library 626 | n03662601 lifeboat 627 | n03666591 lighter, light, igniter, ignitor 628 | n03670208 limousine, limo 629 | n03673027 liner, ocean liner 630 | n03676483 lipstick, lip rouge 631 | n03680355 Loafer 632 | n03690938 lotion 633 | n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system 634 | n03692522 loupe, jeweler's loupe 635 | n03697007 lumbermill, sawmill 636 | n03706229 magnetic compass 637 | n03709823 mailbag, postbag 638 | n03710193 mailbox, letter box 639 | n03710637 maillot 640 | n03710721 maillot, tank suit 641 | n03717622 manhole cover 642 | n03720891 maraca 643 | n03721384 marimba, xylophone 644 | n03724870 mask 645 | n03729826 matchstick 646 | n03733131 maypole 647 | n03733281 maze, labyrinth 648 | n03733805 measuring cup 649 | n03742115 medicine chest, medicine cabinet 650 | n03743016 megalith, megalithic structure 651 | n03759954 microphone, mike 652 | n03761084 microwave, microwave oven 653 | n03763968 military uniform 654 | n03764736 milk can 655 | n03769881 minibus 656 | n03770439 miniskirt, mini 657 | n03770679 minivan 658 | n03773504 missile 659 | n03775071 mitten 660 | n03775546 mixing bowl 661 | n03776460 mobile home, manufactured home 662 | n03777568 Model T 663 | n03777754 modem 664 | n03781244 monastery 665 | n03782006 monitor 666 | n03785016 moped 667 | n03786901 mortar 668 | n03787032 mortarboard 669 | n03788195 mosque 670 | n03788365 mosquito net 671 | n03791053 motor scooter, scooter 672 | n03792782 mountain bike, all-terrain bike, off-roader 673 | n03792972 mountain tent 674 | n03793489 mouse, computer mouse 675 | n03794056 mousetrap 676 | n03796401 moving van 677 | n03803284 muzzle 678 | n03804744 nail 679 | n03814639 neck brace 680 | n03814906 necklace 681 | n03825788 nipple 682 | n03832673 notebook, notebook computer 683 | n03837869 obelisk 684 | n03838899 oboe, hautboy, hautbois 685 | n03840681 ocarina, sweet potato 686 | n03841143 odometer, hodometer, mileometer, milometer 687 | n03843555 oil filter 688 | n03854065 organ, pipe organ 689 | n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO 690 | n03866082 overskirt 691 | n03868242 oxcart 692 | n03868863 oxygen mask 693 | n03871628 packet 694 | n03873416 paddle, boat paddle 695 | n03874293 paddlewheel, paddle wheel 696 | n03874599 padlock 697 | n03876231 paintbrush 698 | n03877472 pajama, pyjama, pj's, jammies 699 | n03877845 palace 700 | n03884397 panpipe, pandean pipe, syrinx 701 | n03887697 paper towel 702 | n03888257 parachute, chute 703 | n03888605 parallel bars, bars 704 | n03891251 park bench 705 | n03891332 parking meter 706 | n03895866 passenger car, coach, carriage 707 | n03899768 patio, terrace 708 | n03902125 pay-phone, pay-station 709 | n03903868 pedestal, plinth, footstall 710 | n03908618 pencil box, pencil case 711 | n03908714 pencil sharpener 712 | n03916031 perfume, essence 713 | n03920288 Petri dish 714 | n03924679 photocopier 715 | n03929660 pick, plectrum, plectron 716 | n03929855 pickelhaube 717 | n03930313 picket fence, paling 718 | n03930630 pickup, pickup truck 719 | n03933933 pier 720 | n03935335 piggy bank, penny bank 721 | n03937543 pill bottle 722 | n03938244 pillow 723 | n03942813 ping-pong ball 724 | n03944341 pinwheel 725 | n03947888 pirate, pirate ship 726 | n03950228 pitcher, ewer 727 | n03954731 plane, carpenter's plane, woodworking plane 728 | n03956157 planetarium 729 | n03958227 plastic bag 730 | n03961711 plate rack 731 | n03967562 plow, plough 732 | n03970156 plunger, plumber's helper 733 | n03976467 Polaroid camera, Polaroid Land camera 734 | n03976657 pole 735 | n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria 736 | n03980874 poncho 737 | n03982430 pool table, billiard table, snooker table 738 | n03983396 pop bottle, soda bottle 739 | n03991062 pot, flowerpot 740 | n03992509 potter's wheel 741 | n03995372 power drill 742 | n03998194 prayer rug, prayer mat 743 | n04004767 printer 744 | n04005630 prison, prison house 745 | n04008634 projectile, missile 746 | n04009552 projector 747 | n04019541 puck, hockey puck 748 | n04023962 punching bag, punch bag, punching ball, punchball 749 | n04026417 purse 750 | n04033901 quill, quill pen 751 | n04033995 quilt, comforter, comfort, puff 752 | n04037443 racer, race car, racing car 753 | n04039381 racket, racquet 754 | n04040759 radiator 755 | n04041544 radio, wireless 756 | n04044716 radio telescope, radio reflector 757 | n04049303 rain barrel 758 | n04065272 recreational vehicle, RV, R.V. 759 | n04067472 reel 760 | n04069434 reflex camera 761 | n04070727 refrigerator, icebox 762 | n04074963 remote control, remote 763 | n04081281 restaurant, eating house, eating place, eatery 764 | n04086273 revolver, six-gun, six-shooter 765 | n04090263 rifle 766 | n04099969 rocking chair, rocker 767 | n04111531 rotisserie 768 | n04116512 rubber eraser, rubber, pencil eraser 769 | n04118538 rugby ball 770 | n04118776 rule, ruler 771 | n04120489 running shoe 772 | n04125021 safe 773 | n04127249 safety pin 774 | n04131690 saltshaker, salt shaker 775 | n04133789 sandal 776 | n04136333 sarong 777 | n04141076 sax, saxophone 778 | n04141327 scabbard 779 | n04141975 scale, weighing machine 780 | n04146614 school bus 781 | n04147183 schooner 782 | n04149813 scoreboard 783 | n04152593 screen, CRT screen 784 | n04153751 screw 785 | n04154565 screwdriver 786 | n04162706 seat belt, seatbelt 787 | n04179913 sewing machine 788 | n04192698 shield, buckler 789 | n04200800 shoe shop, shoe-shop, shoe store 790 | n04201297 shoji 791 | n04204238 shopping basket 792 | n04204347 shopping cart 793 | n04208210 shovel 794 | n04209133 shower cap 795 | n04209239 shower curtain 796 | n04228054 ski 797 | n04229816 ski mask 798 | n04235860 sleeping bag 799 | n04238763 slide rule, slipstick 800 | n04239074 sliding door 801 | n04243546 slot, one-armed bandit 802 | n04251144 snorkel 803 | n04252077 snowmobile 804 | n04252225 snowplow, snowplough 805 | n04254120 soap dispenser 806 | n04254680 soccer ball 807 | n04254777 sock 808 | n04258138 solar dish, solar collector, solar furnace 809 | n04259630 sombrero 810 | n04263257 soup bowl 811 | n04264628 space bar 812 | n04265275 space heater 813 | n04266014 space shuttle 814 | n04270147 spatula 815 | n04273569 speedboat 816 | n04275548 spider web, spider's web 817 | n04277352 spindle 818 | n04285008 sports car, sport car 819 | n04286575 spotlight, spot 820 | n04296562 stage 821 | n04310018 steam locomotive 822 | n04311004 steel arch bridge 823 | n04311174 steel drum 824 | n04317175 stethoscope 825 | n04325704 stole 826 | n04326547 stone wall 827 | n04328186 stopwatch, stop watch 828 | n04330267 stove 829 | n04332243 strainer 830 | n04335435 streetcar, tram, tramcar, trolley, trolley car 831 | n04336792 stretcher 832 | n04344873 studio couch, day bed 833 | n04346328 stupa, tope 834 | n04347754 submarine, pigboat, sub, U-boat 835 | n04350905 suit, suit of clothes 836 | n04355338 sundial 837 | n04355933 sunglass 838 | n04356056 sunglasses, dark glasses, shades 839 | n04357314 sunscreen, sunblock, sun blocker 840 | n04366367 suspension bridge 841 | n04367480 swab, swob, mop 842 | n04370456 sweatshirt 843 | n04371430 swimming trunks, bathing trunks 844 | n04371774 swing 845 | n04372370 switch, electric switch, electrical switch 846 | n04376876 syringe 847 | n04380533 table lamp 848 | n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle 849 | n04392985 tape player 850 | n04398044 teapot 851 | n04399382 teddy, teddy bear 852 | n04404412 television, television system 853 | n04409515 tennis ball 854 | n04417672 thatch, thatched roof 855 | n04418357 theater curtain, theatre curtain 856 | n04423845 thimble 857 | n04428191 thresher, thrasher, threshing machine 858 | n04429376 throne 859 | n04435653 tile roof 860 | n04442312 toaster 861 | n04443257 tobacco shop, tobacconist shop, tobacconist 862 | n04447861 toilet seat 863 | n04456115 torch 864 | n04458633 totem pole 865 | n04461696 tow truck, tow car, wrecker 866 | n04462240 toyshop 867 | n04465501 tractor 868 | n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi 869 | n04476259 tray 870 | n04479046 trench coat 871 | n04482393 tricycle, trike, velocipede 872 | n04483307 trimaran 873 | n04485082 tripod 874 | n04486054 triumphal arch 875 | n04487081 trolleybus, trolley coach, trackless trolley 876 | n04487394 trombone 877 | n04493381 tub, vat 878 | n04501370 turnstile 879 | n04505470 typewriter keyboard 880 | n04507155 umbrella 881 | n04509417 unicycle, monocycle 882 | n04515003 upright, upright piano 883 | n04517823 vacuum, vacuum cleaner 884 | n04522168 vase 885 | n04523525 vault 886 | n04525038 velvet 887 | n04525305 vending machine 888 | n04532106 vestment 889 | n04532670 viaduct 890 | n04536866 violin, fiddle 891 | n04540053 volleyball 892 | n04542943 waffle iron 893 | n04548280 wall clock 894 | n04548362 wallet, billfold, notecase, pocketbook 895 | n04550184 wardrobe, closet, press 896 | n04552348 warplane, military plane 897 | n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin 898 | n04554684 washer, automatic washer, washing machine 899 | n04557648 water bottle 900 | n04560804 water jug 901 | n04562935 water tower 902 | n04579145 whiskey jug 903 | n04579432 whistle 904 | n04584207 wig 905 | n04589890 window screen 906 | n04590129 window shade 907 | n04591157 Windsor tie 908 | n04591713 wine bottle 909 | n04592741 wing 910 | n04596742 wok 911 | n04597913 wooden spoon 912 | n04599235 wool, woolen, woollen 913 | n04604644 worm fence, snake fence, snake-rail fence, Virginia fence 914 | n04606251 wreck 915 | n04612504 yawl 916 | n04613696 yurt 917 | n06359193 web site, website, internet site, site 918 | n06596364 comic book 919 | n06785654 crossword puzzle, crossword 920 | n06794110 street sign 921 | n06874185 traffic light, traffic signal, stoplight 922 | n07248320 book jacket, dust cover, dust jacket, dust wrapper 923 | n07565083 menu 924 | n07579787 plate 925 | n07583066 guacamole 926 | n07584110 consomme 927 | n07590611 hot pot, hotpot 928 | n07613480 trifle 929 | n07614500 ice cream, icecream 930 | n07615774 ice lolly, lolly, lollipop, popsicle 931 | n07684084 French loaf 932 | n07693725 bagel, beigel 933 | n07695742 pretzel 934 | n07697313 cheeseburger 935 | n07697537 hotdog, hot dog, red hot 936 | n07711569 mashed potato 937 | n07714571 head cabbage 938 | n07714990 broccoli 939 | n07715103 cauliflower 940 | n07716358 zucchini, courgette 941 | n07716906 spaghetti squash 942 | n07717410 acorn squash 943 | n07717556 butternut squash 944 | n07718472 cucumber, cuke 945 | n07718747 artichoke, globe artichoke 946 | n07720875 bell pepper 947 | n07730033 cardoon 948 | n07734744 mushroom 949 | n07742313 Granny Smith 950 | n07745940 strawberry 951 | n07747607 orange 952 | n07749582 lemon 953 | n07753113 fig 954 | n07753275 pineapple, ananas 955 | n07753592 banana 956 | n07754684 jackfruit, jak, jack 957 | n07760859 custard apple 958 | n07768694 pomegranate 959 | n07802026 hay 960 | n07831146 carbonara 961 | n07836838 chocolate sauce, chocolate syrup 962 | n07860988 dough 963 | n07871810 meat loaf, meatloaf 964 | n07873807 pizza, pizza pie 965 | n07875152 potpie 966 | n07880968 burrito 967 | n07892512 red wine 968 | n07920052 espresso 969 | n07930864 cup 970 | n07932039 eggnog 971 | n09193705 alp 972 | n09229709 bubble 973 | n09246464 cliff, drop, drop-off 974 | n09256479 coral reef 975 | n09288635 geyser 976 | n09332890 lakeside, lakeshore 977 | n09399592 promontory, headland, head, foreland 978 | n09421951 sandbar, sand bar 979 | n09428293 seashore, coast, seacoast, sea-coast 980 | n09468604 valley, vale 981 | n09472597 volcano 982 | n09835506 ballplayer, baseball player 983 | n10148035 groom, bridegroom 984 | n10565667 scuba diver 985 | n11879895 rapeseed 986 | n11939491 daisy 987 | n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum 988 | n12144580 corn 989 | n12267677 acorn 990 | n12620546 hip, rose hip, rosehip 991 | n12768682 buckeye, horse chestnut, conker 992 | n12985857 coral fungus 993 | n12998815 agaric 994 | n13037406 gyromitra 995 | n13040303 stinkhorn, carrion fungus 996 | n13044778 earthstar 997 | n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa 998 | n13054560 bolete 999 | n13133613 ear, spike, capitulum 1000 | n15075141 toilet tissue, toilet paper, bathroom tissue 1001 | --------------------------------------------------------------------------------