├── AMIL_Data └── readme_data_format.txt ├── AMIL_codes ├── amil_model.py ├── patch_data.py ├── run_for_all_zoom.sh └── train_n_test.py ├── ResNet ├── resnet_pre.py └── run_for_all_zoom.sh ├── VGG ├── run_for_all_zoom.sh └── vgg_pre.py ├── grad_cam ├── inputs │ ├── SOB_B_A-14-22549G-40-001.png │ └── vgg_224.png ├── results │ ├── vgg_224,layer=35_Guided_Grad_Cam.jpg │ ├── vgg_224,layer=35_Guided_Grad_Cam_gray.jpg │ ├── vgg_224,layer=35_guided_grads .jpg │ └── vgg_224.png └── src │ ├── cnn_layer_visualization.py │ ├── deep_dream.py │ ├── generate_class_specific_samples.py │ ├── gradcam.py │ ├── gradcam.pyc │ ├── guided_backprop.py │ ├── guided_backprop.pyc │ ├── guided_gradcam.py │ ├── inverted_representation.py │ ├── layer_activation_with_guided_backprop.py │ ├── misc_functions.py │ ├── misc_functions.pyc │ ├── net.py │ ├── net.pyc │ ├── smooth_grad.py │ └── vanilla_backprop.py ├── my_network ├── net.py └── run_for_all_zoom.sh ├── readme.md └── requirement.txt /AMIL_Data/readme_data_format.txt: -------------------------------------------------------------------------------- 1 | ################################################################################################## 2 | AMIL_Data 3 | ################################################################################################## 4 | we are using Breakhis dataset from the Kaggle datasets(link to the dataset) 5 | => https://www.kaggle.com/kritika397/breast-cancer-dataset-from-breakhis/downloads/fold1.zip/1 6 | -------------------------------------------------------------------------------------------------- 7 | Here, dataset is in this structure: 8 | fold1 9 | -test 10 | -100X 11 | -B_100X 12 | -(images) 13 | -M_100X 14 | -(images) 15 | -200X 16 | -B_200X 17 | -(images) 18 | -M_200X 19 | -(images) 20 | -400X 21 | -B_400X 22 | -(images) 23 | -M_400X 24 | -(images) 25 | -40X 26 | -B_40X 27 | -(images) 28 | -M_40X 29 | -(images) 30 | -train 31 | -100X 32 | -B_100X 33 | -(images) 34 | -M_100X 35 | -(images) 36 | -200X 37 | -B_200X 38 | -(images) 39 | -M_200X 40 | -(images) 41 | -400X 42 | -B_400X 43 | -(images) 44 | -M_400X 45 | -(images) 46 | -40X 47 | -B_40X 48 | -(images) 49 | -M_40X 50 | -(images) 51 | 52 | Now, we have to convert it in the following format: 53 | data_breakhis 54 | -100X 55 | -train 56 | -0 57 | -images 58 | -1 59 | -images 60 | -test 61 | -0 62 | -images 63 | -1 64 | -images 65 | -200X 66 | -train 67 | -0 68 | -images 69 | -1 70 | -images 71 | -test 72 | -0 73 | -images 74 | -1 75 | -images 76 | -400X 77 | -train 78 | -0 79 | -images 80 | -1 81 | -images 82 | -test 83 | -0 84 | -images 85 | -1 86 | -images 87 | -40X 88 | -train 89 | -0 90 | -images 91 | -1 92 | -images 93 | -test 94 | -0 95 | -images 96 | -1 97 | -images -------------------------------------------------------------------------------- /AMIL_codes/amil_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class Attention(nn.Module): 7 | def __init__(self): 8 | super(Attention, self).__init__() 9 | self.L = 500 10 | self.D = 128 11 | self.K = 1 12 | 13 | self.feature_extractor_part1 = nn.Sequential( 14 | nn.Conv2d(3, 20, kernel_size=5), 15 | nn.ReLU(), 16 | nn.MaxPool2d(2, stride=2), 17 | nn.Conv2d(20, 50, kernel_size=5), 18 | nn.ReLU(), 19 | nn.MaxPool2d(2, stride=2) 20 | ) 21 | 22 | self.feature_extractor_part2 = nn.Sequential( 23 | nn.Linear(50 * 4 * 4, self.L), 24 | nn.ReLU(), 25 | ) 26 | 27 | self.attention = nn.Sequential( 28 | nn.Linear(self.L, self.D), 29 | nn.Tanh(), 30 | nn.Linear(self.D, self.K) 31 | ) 32 | 33 | self.classifier = nn.Sequential( 34 | nn.Linear(self.L*self.K, 1), 35 | nn.Sigmoid() 36 | ) 37 | 38 | def forward(self, x): 39 | # print('========================') 40 | # print(x.shape) 41 | x = x.squeeze(0) 42 | # print(x.shape) 43 | # print('========================') 44 | H = self.feature_extractor_part1(x) 45 | H = H.view(-1, 50 * 4 * 4) 46 | H = self.feature_extractor_part2(H) # NxL 47 | 48 | A = self.attention(H) # NxK 49 | A = torch.transpose(A, 1, 0) # KxN 50 | A = F.softmax(A, dim=1) # softmax over N 51 | 52 | M = torch.mm(A, H) # KxL 53 | 54 | Y_prob = self.classifier(M) 55 | Y_hat = torch.ge(Y_prob, 0.5).float() 56 | 57 | return Y_prob, Y_hat, A 58 | 59 | # AUXILIARY METHODS 60 | def calculate_classification_error(self, X, Y): 61 | Y = Y.float() 62 | _, Y_hat, _ = self.forward(X) 63 | # error = 1. - Y_hat.eq(Y).cpu().float().mean().data[0] 64 | error = 1. - Y_hat.eq(Y).cpu().float().mean().item() 65 | 66 | return error, Y_hat 67 | 68 | def calculate_objective(self, X, Y): 69 | Y = Y.float() 70 | Y_prob, _, A = self.forward(X) 71 | Y_prob = torch.clamp(Y_prob, min=1e-5, max=1. - 1e-5) 72 | neg_log_likelihood = -1. * (Y * torch.log(Y_prob) + (1. - Y) * torch.log(1. - Y_prob)) # negative log bernoulli 73 | 74 | return neg_log_likelihood, A 75 | -------------------------------------------------------------------------------- /AMIL_codes/patch_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | from PIL import Image 4 | import numpy as np 5 | import torch 6 | from torchvision import datasets, transforms 7 | import random 8 | # dir structure would be: data/class_name(0 and 1)/dir_containing_img/img 9 | class PatchMethod(torch.utils.data.Dataset): 10 | def __init__(self, root = 'Desktop/screenshots/', mode = 'train', transform = None): 11 | self.root = root 12 | self.mode = mode 13 | self.raw_samples = glob.glob(root + '/*/*') 14 | # print(self.raw_samples) 15 | self.samples = [] 16 | for raw_sample in self.raw_samples: 17 | self.samples.append((raw_sample, int(raw_sample.split('/')[-2]))) 18 | # print(raw_sample,int(raw_sample.split('/')[-2])) 19 | # print(self.samples) 20 | 21 | def __len__(self): 22 | return len(self.samples) 23 | 24 | def __getitem__(self, index): 25 | if self.mode == 'train': 26 | random.shuffle(self.samples) 27 | 28 | image_dir, label = self.samples[index] 29 | images = glob.glob(image_dir) 30 | # images = glob.glob(image_dir + '/*') 31 | 32 | t = transforms.Compose([transforms.CenterCrop((448,700))]) #centercropping to 1200 to generate 9 400x400 patches 33 | 34 | transformations = transforms.Compose([ 35 | transforms.ToTensor() 36 | ]) 37 | 38 | array = [] 39 | 40 | for i, image_path in enumerate(images): 41 | # print(image_path) 42 | image = Image.open(image_path) 43 | image = np.array(t(image)) 44 | # print(image.shape) 45 | # image = np.array(image) 46 | r, c, _ = image.shape 47 | # print("image.shape",image.shape) 48 | for i in range(0,28*16,28): 49 | for j in range(0,28*25,28): 50 | array.append(transformations(image[i:i+28, j:j+28, :])) 51 | # array.append(transformations(image[i:i+400, j:j+400, :]).float()) 52 | # array.append(image[i:i+400, j:j+400, :]) 53 | # if (i+400 < r) and (j+400 < c): 54 | # array.append(transformations(image[i:i+400, j:j+400, :]).float()) 55 | # array.append(image[i:i+400, j:j+400, :]) 56 | 57 | 58 | array = tuple(array) 59 | # print("################### array ###################") 60 | # print(array) 61 | array = torch.stack(array, 0) 62 | 63 | return (array, label) -------------------------------------------------------------------------------- /AMIL_codes/run_for_all_zoom.sh: -------------------------------------------------------------------------------- 1 | mkdir terminal_logs 2 | declare -a ZOOM=(40 100 200 400) 3 | # declare -a epoch=(1 2 3 4 5 6 7 8 9 10) 4 | 5 | for i in "${ZOOM[@]}" 6 | do 7 | # for j in "${epoch[@]}" 8 | # do 9 | echo "$i" 10 | # echo "$j" 11 | python train_n_test.py --epochs=20 --zoom=$i |& tee terminal_logs/epoch_20_op_zoom_$i.txt 12 | # python vgg_pre.py --epochs=$j --zoom=$i |& tee terminal_logs/op_zoom_$i_epoch_$j.txt 13 | # done 14 | done 15 | -------------------------------------------------------------------------------- /AMIL_codes/train_n_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import numpy as np 4 | from scipy.misc import imsave 5 | import argparse 6 | import torch 7 | import torch.utils.data as data_utils 8 | import torch.optim as optim 9 | from torch.autograd import Variable 10 | 11 | # from dataloader import MnistBags 12 | from amil_model import Attention 13 | 14 | # from __future__ import print_function, division 15 | import os 16 | import glob 17 | from PIL import Image 18 | import numpy as np 19 | from torchvision import datasets, transforms 20 | from torchvision import models 21 | import random 22 | import torch 23 | import torch.nn as nn 24 | import torch.nn.functional as F 25 | import torch.optim as optim 26 | 27 | from amil_model import Attention 28 | from patch_data import PatchMethod 29 | from tensorboardX import SummaryWriter 30 | import argparse 31 | 32 | parser = argparse.ArgumentParser(description='Breakthis data_mynet') 33 | parser.add_argument("--zoom", help='zoom_level',default=400) 34 | parser.add_argument('--epochs',type=int, default=1, metavar='N', 35 | help='number of epochs to train (default: 10)') 36 | parser.add_argument('--seed', type=int, default=1, metavar='S', 37 | help='random seed (default: 1)') 38 | parser.add_argument('--no-cuda', action='store_true', default=False, 39 | help='disables CUDA training') 40 | parser.add_argument('--lr', type=float, default=0.0001, metavar='LR', 41 | help='learning rate (default: 0.01)') 42 | parser.add_argument('--reg', type=float, default=10e-5, metavar='R', 43 | help='weight decay') 44 | 45 | args = parser.parse_args() 46 | args.cuda = not args.no_cuda and torch.cuda.is_available() 47 | 48 | zoom_level_x =str(args.zoom) + 'X' 49 | 50 | 51 | print('zoom_level_{} epoch_{} learning_rate_{}'.format(zoom_level_x, args.epochs, args.lr)) 52 | writer = SummaryWriter(zoom_level_x+'/runs/'+"epoch:"+str(args.epochs)) 53 | 54 | # Training settings 55 | # parser = argparse.ArgumentParser(description='PyTorch MNIST bags Example') 56 | # parser.add_argument('--epochs', type=int, default=1, metavar='N', 57 | # help='number of epochs to train (default: 10)') 58 | # parser.add_argument('--lr', type=float, default=0.0001, metavar='LR', 59 | # help='learning rate (default: 0.01)') 60 | # parser.add_argument('--reg', type=float, default=10e-5, metavar='R', 61 | # help='weight decay') 62 | # parser.add_argument('--target_number', type=int, default=9, metavar='T', 63 | # help='bags have a positive labels if they contain at least one 9') 64 | # parser.add_argument('--mean_bag_length', type=int, default=10, metavar='ML', 65 | # help='average bag length') 66 | # parser.add_argument('--var_bag_length', type=int, default=2, metavar='VL', 67 | # help='variance of bag length') 68 | # parser.add_argument('--num_bags_train', type=int, default=200, metavar='NTrain', 69 | # help='number of bags in training set') 70 | # parser.add_argument('--num_bags_test', type=int, default=50, metavar='NTest', 71 | # help='number of bags in test set') 72 | # parser.add_argument('--seed', type=int, default=1, metavar='S', 73 | # help='random seed (default: 1)') 74 | # parser.add_argument('--no-cuda', action='store_true', default=False, 75 | # help='disables CUDA training') 76 | 77 | # args = parser.parse_args() 78 | # args.cuda = not args.no_cuda and torch.cuda.is_available() 79 | 80 | torch.manual_seed(args.seed) 81 | if args.cuda: 82 | torch.cuda.manual_seed(args.seed) 83 | print('\nGPU is ON!') 84 | 85 | print('Load Train and Test Set') 86 | loader_kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} 87 | 88 | # train_loader = data_utils.DataLoader(MnistBags(target_number=args.target_number, 89 | # mean_bag_length=args.mean_bag_length, 90 | # var_bag_length=args.var_bag_length, 91 | # num_bag=args.num_bags_train, 92 | # seed=args.seed, 93 | # train=True), 94 | # batch_size=1, 95 | # shuffle=True, 96 | # **loader_kwargs) 97 | 98 | # test_loader = data_utils.DataLoader(MnistBags(target_number=args.target_number, 99 | # mean_bag_length=args.mean_bag_length, 100 | # var_bag_length=args.var_bag_length, 101 | # num_bag=args.num_bags_test, 102 | # seed=args.seed, 103 | # train=False), 104 | # batch_size=1, 105 | # shuffle=False, 106 | # **loader_kwargs) 107 | 108 | # dir structure would be: data/class_name(0 and 1)/dir_containing_img/img 109 | # sftp://test1@10.107.42.42/home/Drive2/amil/data 110 | # /home/dipesh/test1/new_data_tree/40X 111 | # /home/dipesh/126/AMIL_project/AMIL_codes/amil_model.py 112 | # /home/dipesh/126/AMIL_project/ 113 | data_path_train = "../AMIL_Data/{}/train".format(zoom_level_x) 114 | data_path_test = "../AMIL_Data/{}/test".format(zoom_level_x) 115 | 116 | data = PatchMethod(root = data_path_train) 117 | val_data =PatchMethod(root = data_path_test, mode = 'test') 118 | # data = PatchMethod(root = '/Users/abhijeetpatil/Desktop/screenshots2/') 119 | # val_data =PatchMethod(root = '/Users/abhijeetpatil/Desktop/screenshots2/', mode = 'test') 120 | 121 | train_loader = torch.utils.data.DataLoader(data, shuffle = True, num_workers = 6, batch_size = 1) 122 | test_loader = torch.utils.data.DataLoader(val_data, shuffle = False, num_workers = 6, batch_size = 1) 123 | 124 | 125 | print('Init Model') 126 | model = Attention() 127 | if args.cuda: 128 | model.cuda() 129 | 130 | optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), weight_decay=args.reg) 131 | # optimizer = optim.Adam(model.parameters(), lr=0.001) 132 | 133 | 134 | def train(epoch): 135 | model.train() 136 | train_loss = 0. 137 | train_error = 0. 138 | correct_label_pred = 0 139 | for batch_idx, (data, label) in enumerate(train_loader): 140 | # print("label",label[0][0]) 141 | # print("label",label[0]) 142 | bag_label = label[0] 143 | if args.cuda: 144 | data, bag_label = data.cuda(), bag_label.cuda() 145 | data, bag_label = Variable(data), Variable(bag_label) 146 | data = data.squeeze(0) 147 | 148 | optimizer.zero_grad() 149 | # calculate loss and metrics 150 | loss, _ = model.calculate_objective(data, bag_label) 151 | train_loss += loss.data[0] 152 | error, predicted_label_train = model.calculate_classification_error(data, bag_label) 153 | # print("bag_label,predicted_label_train",bag_label,predicted_label_train) 154 | # print(int(bag_label) == int(predicted_label_train)) 155 | correct_label_pred += (int(bag_label) == int(predicted_label_train)) 156 | # exit() 157 | train_error += error 158 | # backward pass 159 | loss.backward() 160 | # step 161 | optimizer.step() 162 | # print(correct_label_pred) 163 | # print(len(train_loader)) 164 | # print(batch_idx) 165 | 166 | # calculate loss and error for epoch 167 | train_loss /= len(train_loader) 168 | train_error /= len(train_loader) 169 | train_acc = (1 - train_error)*100 170 | 171 | writer.add_scalar('data/train_acc', train_acc, epoch) 172 | writer.add_scalar('data/train_error', train_error, epoch) 173 | writer.add_scalar('data/train_loss', train_loss, epoch) 174 | 175 | result_train = 'Epoch: {}, Loss: {:.4f}, Train error: {:.4f}, Train accuracy: {:.2f}'.format(epoch, train_loss.cpu().numpy()[0], train_error, train_acc) 176 | 177 | print(result_train) 178 | return result_train 179 | 180 | def test(epoch): 181 | model.eval() 182 | test_loss = 0. 183 | test_error = 0. 184 | for batch_idx, (data, label) in enumerate(test_loader): 185 | # print(label) 186 | # print((data[0].shape)) 187 | 188 | bag_label = label[0] 189 | instance_labels = label[0] 190 | if args.cuda: 191 | data, bag_label = data.cuda(), bag_label.cuda() 192 | data, bag_label = Variable(data), Variable(bag_label) 193 | loss, attention_weights = model.calculate_objective(data, bag_label) 194 | test_loss += loss.data[0] 195 | error, predicted_label = model.calculate_classification_error(data, bag_label) 196 | test_error += error 197 | 198 | visualization_attention(data[0],attention_weights[0],batch_idx,epoch) 199 | if batch_idx < 1: # plot bag labels and instance labels for first 5 bags 200 | bag_level = (bag_label.cpu().data.numpy(), int(predicted_label.cpu().data.numpy())) 201 | # print(bag_level) 202 | # print(instance_labels) 203 | # visualization_attention(data[0],attention_weights[0],batch_idx,epoch) 204 | # print("attention_weights.shape",attention_weights.shape) 205 | # instance_level = list(zip(instance_labels.numpy().tolist(), 206 | # np.round(attention_weights.cpu().data.numpy(), decimals=3).tolist())) 207 | 208 | # print('\nTrue Bag Label, Predicted Bag Label: {}\n' 209 | # 'True Instance Labels, Attention Weights: {}'.format(bag_level, instance_level)) 210 | 211 | test_error /= len(test_loader) 212 | test_loss /= len(test_loader) 213 | test_acc = (1 - test_error)*100 214 | 215 | writer.add_scalar('data/test_acc', test_acc, epoch) 216 | writer.add_scalar('data/test_error', test_error, epoch) 217 | writer.add_scalar('data/test_loss', test_loss, epoch) 218 | result_test = 'Epoch: {}, Loss: {:.4f}, test error: {:.4f}, test accuracy: {:.2f}'.format(epoch, test_loss.cpu().numpy()[0], test_error, test_acc) 219 | print(result_test) 220 | return result_test 221 | # print('Test Set, Loss: {:.4f}, Test error: {:.4f}'.format(test_loss.cpu().numpy()[0], test_error)) 222 | 223 | def visualization_attention(data,attention_weights,batch_idx,epoch): 224 | img_save_dir = './{}/AMIL_visualization/epoch_{}'.format(zoom_level_x,epoch) 225 | img_save_name = img_save_dir + '/test_epoch_{}_no_{}.png'.format(epoch,batch_idx) 226 | if not os.path.exists(img_save_dir): 227 | os.makedirs(img_save_dir) 228 | 229 | data = data.cpu().data.numpy() 230 | attention_weights = attention_weights.cpu().data.numpy() 231 | # print("data.shape",data.shape) 232 | # print("attention_weights",attention_weights.shape) 233 | attention_weights = attention_weights/np.max(attention_weights) 234 | complete_image=np.zeros((3,480,700)) 235 | for height_no in range(16): 236 | for width_no in range(25): 237 | complete_image[:,height_no*28:height_no*28+28, width_no*28:width_no*28+28] = data[height_no*25+width_no,:,:,:] * attention_weights[height_no*25+width_no] 238 | complete_image = complete_image.transpose((1,2,0)) 239 | imsave(img_save_name,complete_image) 240 | # weighted_images_list = data * attention_weights 241 | 242 | 243 | 244 | if __name__ == "__main__": 245 | # img_save_dir = './AMIL_visualization/zoom_{}/epoch_{}'.format(zoom_level_x,epoch) 246 | main_dir = "./" + zoom_level_x +'/' 247 | folders = ["pt_files","txt_file"] 248 | for i in folders: 249 | if not os.path.exists(main_dir + i ): 250 | os.makedirs(main_dir + i ) 251 | 252 | save_string="AMIL_Breakthis_epochs: "+str(args.epochs)+"zoom:"+zoom_level_x 253 | save_name_txt = main_dir+"txt_file/"+save_string+".txt" 254 | 255 | model_file = open(save_name_txt,"w") 256 | for epoch in range(1, args.epochs + 1): 257 | print('----------Start Training----------') 258 | train_result = train(epoch) 259 | print('----------Start Testing----------') 260 | test_result = test(epoch) 261 | model_file.write(test_result + '\n') 262 | model_file.write(train_result + '\n') 263 | model_file.close() 264 | torch.save(model.state_dict(),main_dir+"pt_files/"+save_string+"AMIL_Breakthis_state_dict.pt") 265 | torch.save(model ,main_dir+"pt_files/"+save_string+"AMIL_Breakthis_model.pt") 266 | -------------------------------------------------------------------------------- /ResNet/resnet_pre.py: -------------------------------------------------------------------------------- 1 | #model 2 | from __future__ import print_function 3 | import os 4 | import argparse 5 | import torch 6 | import numpy as np 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torchvision 11 | from torchvision import datasets, transforms 12 | import time 13 | from tensorboardX import SummaryWriter 14 | from datetime import datetime 15 | import pickle 16 | import matplotlib.pyplot as plt 17 | # from sklearn.metrics import f1_score, classification_report, confusion_matrix 18 | 19 | parser = argparse.ArgumentParser(description='Breakthis data_mynet') 20 | parser.add_argument("--zoom", help='zoom_level',default=40) 21 | parser.add_argument('--epochs',type=int, default=1, metavar='N', 22 | help='number of epochs to train (default: 10)') 23 | args = parser.parse_args() 24 | zoom_level_x =str(args.zoom) + 'X' 25 | 26 | writer = SummaryWriter(zoom_level_x+'/runs/'+"epoch:"+str(args.epochs)) 27 | 28 | transform = transforms.Compose([transforms.RandomCrop(224),transforms.ToTensor(),]) 29 | 30 | data_train = torchvision.datasets.ImageFolder("../AMIL_Data/test/{}".format(zoom_level_x), transform=transform) 31 | print(data_train) 32 | data_test = torchvision.datasets.ImageFolder("../AMIL_Data/train/{}".format(zoom_level_x), transform=transform) 33 | print(data_test) 34 | 35 | resnet_based = torchvision.models.resnet18(pretrained=True) 36 | print(resnet_based) 37 | for param in resnet_based.parameters(): 38 | # print(param) 39 | param.requires_grad = False 40 | 41 | # Modify the last layer 42 | features = list(resnet_based.fc.children())[:-1] # Remove last layer 43 | # print(type(resnet_based.classifier.children())) 44 | features.extend([torch.nn.Linear(512, 2)]) 45 | resnet_based.fc = torch.nn.Sequential(*features) 46 | 47 | 48 | 49 | def train(args, model, device, train_loader, optimizer, epoch): 50 | model.train() 51 | correct = 0 52 | # print("-----------",train_loader) 53 | for batch_idx, (data, target) in enumerate(train_loader): 54 | data, target = data.to(device), target.to(device) 55 | # print(len(data),len(target)) 56 | optimizer.zero_grad() 57 | output = model(data) 58 | output = F.log_softmax(output, dim=1) 59 | # print(output.shape) 60 | # print(len(output), len(target)) 61 | loss = F.nll_loss(output, target) 62 | loss.backward() 63 | optimizer.step() 64 | if batch_idx % args.log_interval == 0: 65 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 66 | epoch, batch_idx * len(data), len(train_loader.dataset), 67 | 100. * batch_idx / len(train_loader), loss.item())) 68 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 69 | correct += pred.eq(target.view_as(pred)).sum().item() 70 | print('\nTrain_accuracy: {:.0f}%\n'.format(100. * correct / len(train_loader.dataset))) 71 | writer.add_scalar('train_Accuracy_epoch',100. * correct / len(train_loader.dataset),epoch) 72 | writer.add_scalar('train_loss_epoch',loss/len(train_loader.dataset),epoch) 73 | return (100. * correct / len(train_loader.dataset)) 74 | def test(args, model, device, test_loader,epoch): 75 | print("test started") 76 | model.eval() 77 | test_loss = 0 78 | correct = 0 79 | with torch.no_grad(): 80 | for data, target in test_loader: 81 | data, target = data.to(device), target.to(device) 82 | output = model(data) 83 | test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss 84 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 85 | correct += pred.eq(target.view_as(pred)).sum().item() 86 | test_loss /= len(test_loader.dataset) 87 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 88 | test_loss, correct, len(test_loader.dataset), 89 | 100. * correct / len(test_loader.dataset))) 90 | writer.add_scalar('test_loss_epoch',test_loss,epoch) 91 | writer.add_scalar('test_Accuracy_epoch',100. * correct / len(test_loader.dataset),epoch) 92 | return (100. * correct / len(test_loader.dataset)) 93 | 94 | def main(): 95 | start = time.time() 96 | print ("into the main") 97 | parser = argparse.ArgumentParser(description='Breakthis') 98 | 99 | parser.add_argument('--batch-size', type=int, default=16, metavar='N', 100 | help='input batch size for training (default: 64)') 101 | 102 | parser.add_argument('--test-batch-size', type=int, default=16, metavar='N', 103 | help='input batch size for testing (default: 1000)') 104 | 105 | parser.add_argument('--epochs', type=int, default=4, metavar='N', 106 | help='number of epochs to train (default: 10)') 107 | 108 | 109 | parser.add_argument('--lr', type=float, default=0.0003 , metavar='LR', 110 | help='learning rate (default: 0.01)') 111 | 112 | parser.add_argument('--momentum', type=float, default=0.4, metavar='M', 113 | help='SGD momentum (default: 0.9)') 114 | 115 | parser.add_argument('--no-cuda', action='store_true', default=False, 116 | help='disables CUDA training') 117 | 118 | parser.add_argument('--seed', type=int, default=1, metavar='S', 119 | help='random seed (default: 1)') 120 | 121 | parser.add_argument('--log_interval', type=int, default=20, metavar='N', 122 | help='how many batches to wait before logging training status') 123 | 124 | parser.add_argument('--save-model', action='store_true', default=True, 125 | help='For Saving the current Model') 126 | parser.add_argument("--zoom", help='zoom_level',default=40) 127 | 128 | args = parser.parse_args() 129 | 130 | use_cuda = not args.no_cuda and torch.cuda.is_available() 131 | device = torch.device("cuda" if use_cuda else "cpu") 132 | # device = "cpu" 133 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 134 | train_loader = torch.utils.data.DataLoader(data_train, batch_size=32 ,shuffle = True, **kwargs) 135 | test_loader = torch.utils.data.DataLoader(data_test, batch_size=32 ,shuffle = False, **kwargs) 136 | print("device: ",device) 137 | 138 | model = resnet_based.to(device) 139 | print ("model transferred to device") 140 | # optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) 141 | # optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) 142 | optimizer = optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.99, eps=1e-08, weight_decay=0, momentum=args.momentum, centered=False) 143 | print ("optimizer choosed") 144 | print("#######__parameters__######") 145 | print("learning rate: ", args.lr, "\nmomentum: ", args.momentum, "\nepochs: ", args.epochs) 146 | print("############################") 147 | print("model:\n",model) 148 | print("############################") 149 | print("optimizer:\n",optimizer) 150 | print("############################") 151 | 152 | # for epoch in range(2): 153 | for epoch in range(1, int(args.epochs) + 1): 154 | # print("-----",train_loader) 155 | train_acc=train(args, model, device, train_loader, optimizer, epoch) 156 | test_acc=test(args, model, device, test_loader, epoch) 157 | # writer.add_scalar('loss_fn2',loss.item(),epoch) 158 | # sftp://test1@10.107.42.42/home/Drive2/amil/my_network/runs 159 | # sftp://test1@10.107.42.42/home/Drive2/amil/res_net_pretrained 160 | main_dir = "./" + zoom_level_x +'/' 161 | folders = ["pt_files","pickel_files","txt_file"] 162 | for i in folders: 163 | if not os.path.exists(main_dir + i ): 164 | os.makedirs(main_dir + i ) 165 | 166 | 167 | save_string="Breakthis: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+"zoom:"+zoom_level_x 168 | if (args.save_model): 169 | torch.save(model.state_dict(),main_dir+"pt_files/"+save_string+"Breakthis_state_dict.pt") 170 | torch.save(model ,main_dir+"pt_files/"+save_string+"Breakthis_model.pt") 171 | save_name_pkl = main_dir+"pickel_files/"+save_string+".pkl" 172 | save_name_txt = main_dir+"txt_file/"+save_string+".txt" 173 | model_file = open(save_name_txt,"w") 174 | model_string = str(model) 175 | optimizer_string = str(optimizer) 176 | model_file.write(model_string) 177 | model_file.write(optimizer_string) 178 | model_file.write(save_name_txt) 179 | model_file.close() 180 | 181 | f=open(save_name_pkl,"wb") 182 | pickle.dump(model, f) 183 | end = time.time() 184 | print('time taken is ', (end-start)) 185 | 186 | 187 | if __name__ == '__main__': 188 | main() 189 | -------------------------------------------------------------------------------- /ResNet/run_for_all_zoom.sh: -------------------------------------------------------------------------------- 1 | #For resnet 2 | 3 | mkdir terminal_logs 4 | declare -a ZOOM=(40 100 200 400) 5 | declare -a epoch=(10) 6 | # declare -a epoch=(1 2 3 4 5 6 7 8 9 10) 7 | 8 | for i in "${ZOOM[@]}" 9 | do 10 | for j in "${epoch[@]}" 11 | do 12 | echo "$i" 13 | echo "$j" 14 | python resnet_pre.py --epochs=$j --zoom=$i |& tee terminal_logs/op_zoom_$i_epoch_$j.txt 15 | # python vgg_pre.py --epochs=$j --zoom=$i |& tee terminal_logs/op_zoom_$i_epoch_$j.txt 16 | done 17 | done 18 | -------------------------------------------------------------------------------- /VGG/run_for_all_zoom.sh: -------------------------------------------------------------------------------- 1 | #vgg_pre 2 | declare -a ZOOM=(40 100 200 400) 3 | mkdir terminal_logs 4 | # declare -a epoch=(1 2 3 4 5 6 7 8 9 10) 5 | declare -a epoch=(10) 6 | 7 | for i in "${ZOOM[@]}" 8 | 9 | do 10 | for j in "${epoch[@]}" 11 | do 12 | echo "$i" 13 | echo "$j" 14 | python vgg_pre.py --epochs=$j --zoom=$i |& tee terminal_logs/op_zoom_$i_epoch_$j.txt 15 | done 16 | done 17 | -------------------------------------------------------------------------------- /VGG/vgg_pre.py: -------------------------------------------------------------------------------- 1 | #model 2 | from __future__ import print_function 3 | import os 4 | import argparse 5 | import torch 6 | import numpy as np 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | import torchvision 11 | from torchvision import datasets, transforms 12 | import time 13 | from tensorboardX import SummaryWriter 14 | from datetime import datetime 15 | import pickle 16 | import matplotlib.pyplot as plt 17 | # from sklearn.metrics import f1_score, classification_report, confusion_matrix 18 | 19 | parser = argparse.ArgumentParser(description='Breakthis data_mynet') 20 | parser.add_argument('--epochs',type=int, default=1, metavar='N', 21 | help='number of epochs to train (default: 10)') 22 | parser.add_argument("--zoom", help='zoom_level',default=40) 23 | 24 | args = parser.parse_args() 25 | zoom_level_x =str(args.zoom) + 'X' 26 | writer = SummaryWriter(zoom_level_x+'/runs/'+"epoch:"+str(args.epochs)) 27 | 28 | transform = transforms.Compose([transforms.RandomCrop(224),transforms.ToTensor(),]) 29 | 30 | data_train = torchvision.datasets.ImageFolder("../AMIL_Data/test/{}".format(zoom_level_x), transform=transform) 31 | print(data_train) 32 | data_test = torchvision.datasets.ImageFolder("../AMIL_Data/train/{}".format(zoom_level_x), transform=transform) 33 | print(data_test) 34 | 35 | vgg_based = torchvision.models.vgg19(pretrained=True) 36 | print(vgg_based) 37 | for param in vgg_based.parameters(): 38 | # print(param) 39 | param.requires_grad = False 40 | 41 | # Modify the last layer 42 | features = list(vgg_based.classifier.children())[:-1] # Remove last layer 43 | # print(type(vgg_based.classifier.children())) 44 | features.extend([torch.nn.Linear(4096, 2)]) 45 | vgg_based.classifier = torch.nn.Sequential(*features) 46 | 47 | 48 | 49 | def train(args, model, device, train_loader, optimizer, epoch): 50 | model.train() 51 | correct = 0 52 | # print("-----------",train_loader) 53 | for batch_idx, (data, target) in enumerate(train_loader): 54 | data, target = data.to(device), target.to(device) 55 | # print(len(data),len(target)) 56 | optimizer.zero_grad() 57 | output = model(data) 58 | output = F.log_softmax(output, dim=1) 59 | # print(output.shape) 60 | # print(len(output), len(target)) 61 | loss = F.nll_loss(output, target) 62 | loss.backward() 63 | optimizer.step() 64 | if batch_idx % args.log_interval == 0: 65 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 66 | epoch, batch_idx * len(data), len(train_loader.dataset), 67 | 100. * batch_idx / len(train_loader), loss.item())) 68 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 69 | correct += pred.eq(target.view_as(pred)).sum().item() 70 | print('\nTrain_accuracy: {:.0f}%\n'.format(100. * correct / len(train_loader.dataset))) 71 | writer.add_scalar('train_Accuracy_epoch',100. * correct / len(train_loader.dataset),epoch) 72 | writer.add_scalar('train_loss_epoch',loss/len(train_loader.dataset),epoch) 73 | return (100. * correct / len(train_loader.dataset)) 74 | def test(args, model, device, test_loader,epoch): 75 | print("test started") 76 | model.eval() 77 | test_loss = 0 78 | correct = 0 79 | with torch.no_grad(): 80 | for data, target in test_loader: 81 | data, target = data.to(device), target.to(device) 82 | output = model(data) 83 | test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss 84 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 85 | correct += pred.eq(target.view_as(pred)).sum().item() 86 | test_loss /= len(test_loader.dataset) 87 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 88 | test_loss, correct, len(test_loader.dataset), 89 | 100. * correct / len(test_loader.dataset))) 90 | writer.add_scalar('test_loss_epoch',test_loss,epoch) 91 | writer.add_scalar('test_Accuracy_epoch',100. * correct / len(test_loader.dataset),epoch) 92 | return (100. * correct / len(test_loader.dataset)) 93 | 94 | def main(): 95 | start = time.time() 96 | print ("into the main") 97 | parser = argparse.ArgumentParser(description='Breakthis') 98 | 99 | parser.add_argument('--batch-size', type=int, default=16, metavar='N', 100 | help='input batch size for training (default: 64)') 101 | 102 | parser.add_argument('--test-batch-size', type=int, default=16, metavar='N', 103 | help='input batch size for testing (default: 1000)') 104 | 105 | parser.add_argument('--epochs', default=1, metavar='N', 106 | help='number of epochs to train (default: 10)') 107 | 108 | 109 | parser.add_argument('--lr', type=float, default=0.0003 , metavar='LR', 110 | help='learning rate (default: 0.01)') 111 | 112 | parser.add_argument('--momentum', type=float, default=0.4, metavar='M', 113 | help='SGD momentum (default: 0.9)') 114 | 115 | parser.add_argument('--no-cuda', action='store_true', default=False, 116 | help='disables CUDA training') 117 | 118 | parser.add_argument('--seed', type=int, default=1, metavar='S', 119 | help='random seed (default: 1)') 120 | 121 | parser.add_argument('--log_interval', type=int, default=20, metavar='N', 122 | help='how many batches to wait before logging training status') 123 | 124 | parser.add_argument('--save-model', action='store_true', default=True, 125 | help='For Saving the current Model') 126 | parser.add_argument("--zoom", help='zoom_level',default=40) 127 | 128 | args = parser.parse_args() 129 | 130 | use_cuda = not args.no_cuda and torch.cuda.is_available() 131 | device = torch.device("cuda" if use_cuda else "cpu") 132 | # device = "cpu" 133 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 134 | train_loader = torch.utils.data.DataLoader(data_train, batch_size=32 ,shuffle = True, **kwargs) 135 | test_loader = torch.utils.data.DataLoader(data_test, batch_size=32 ,shuffle = False, **kwargs) 136 | print("device: ",device) 137 | 138 | model = vgg_based.to(device) 139 | print ("model transferred to device") 140 | # optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) 141 | # optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) 142 | optimizer = optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.99, eps=1e-08, weight_decay=0, momentum=args.momentum, centered=False) 143 | print ("optimizer choosed") 144 | print("#######__parameters__######") 145 | print("learning rate: ", args.lr, "\nmomentum: ", args.momentum, "\nepochs: ", args.epochs) 146 | print("############################") 147 | print("model:\n",model) 148 | print("############################") 149 | print("optimizer:\n",optimizer) 150 | print("############################") 151 | 152 | # for epoch in range(2): 153 | for epoch in range(1, int(args.epochs) + 1): 154 | # print("-----",train_loader) 155 | train_acc=train(args, model, device, train_loader, optimizer, epoch) 156 | test_acc=test(args, model, device, test_loader, epoch) 157 | # writer.add_scalar('loss_fn2',loss.item(),epoch) 158 | # sftp://test1@10.107.42.42/home/Drive2/amil/my_network/runs 159 | 160 | main_dir = "./" + zoom_level_x +'/' 161 | folders = ["pt_files","pickel_files","txt_file"] 162 | for i in folders: 163 | if not os.path.exists(main_dir + i ): 164 | os.makedirs(main_dir + i ) 165 | 166 | 167 | save_string="Breakthis: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+"zoom:"+zoom_level_x 168 | if (args.save_model): 169 | torch.save(model.state_dict(),main_dir+"pt_files/"+save_string+"Breakthis_state_dict.pt") 170 | torch.save(model ,main_dir+"pt_files/"+save_string+"Breakthis_model.pt") 171 | save_name_pkl = main_dir+"pickel_files/"+save_string+".pkl" 172 | save_name_txt = main_dir+"txt_file/"+save_string+".txt" 173 | model_file = open(save_name_txt,"w") 174 | model_string = str(model) 175 | optimizer_string = str(optimizer) 176 | model_file.write(model_string) 177 | model_file.write(optimizer_string) 178 | model_file.write(save_name_txt) 179 | model_file.close() 180 | 181 | f=open(save_name_pkl,"wb") 182 | pickle.dump(model, f) 183 | end = time.time() 184 | print('time taken is ', (end-start)) 185 | 186 | 187 | if __name__ == '__main__': 188 | main() 189 | -------------------------------------------------------------------------------- /grad_cam/inputs/SOB_B_A-14-22549G-40-001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/inputs/SOB_B_A-14-22549G-40-001.png -------------------------------------------------------------------------------- /grad_cam/inputs/vgg_224.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/inputs/vgg_224.png -------------------------------------------------------------------------------- /grad_cam/results/vgg_224,layer=35_Guided_Grad_Cam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/results/vgg_224,layer=35_Guided_Grad_Cam.jpg -------------------------------------------------------------------------------- /grad_cam/results/vgg_224,layer=35_Guided_Grad_Cam_gray.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/results/vgg_224,layer=35_Guided_Grad_Cam_gray.jpg -------------------------------------------------------------------------------- /grad_cam/results/vgg_224,layer=35_guided_grads .jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/results/vgg_224,layer=35_guided_grads .jpg -------------------------------------------------------------------------------- /grad_cam/results/vgg_224.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/results/vgg_224.png -------------------------------------------------------------------------------- /grad_cam/src/cnn_layer_visualization.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Sat Nov 18 23:12:08 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import os 7 | import numpy as np 8 | 9 | import torch 10 | from torch.optim import Adam 11 | from torchvision import models 12 | 13 | from misc_functions import preprocess_image, recreate_image, save_image 14 | 15 | 16 | class CNNLayerVisualization(): 17 | """ 18 | Produces an image that minimizes the loss of a convolution 19 | operation for a specific layer and filter 20 | """ 21 | def __init__(self, model, selected_layer, selected_filter): 22 | self.model = model 23 | self.model.eval() 24 | self.selected_layer = selected_layer 25 | self.selected_filter = selected_filter 26 | self.conv_output = 0 27 | # Create the folder to export images if not exists 28 | if not os.path.exists('../generated'): 29 | os.makedirs('../generated') 30 | 31 | def hook_layer(self): 32 | def hook_function(module, grad_in, grad_out): 33 | # Gets the conv output of the selected filter (from selected layer) 34 | self.conv_output = grad_out[0, self.selected_filter] 35 | # Hook the selected layer 36 | self.model[self.selected_layer].register_forward_hook(hook_function) 37 | 38 | def visualise_layer_with_hooks(self): 39 | # Hook the selected layer 40 | self.hook_layer() 41 | # Generate a random image 42 | random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3))) 43 | # Process image and return variable 44 | processed_image = preprocess_image(random_image, False) 45 | # Define optimizer for the image 46 | optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6) 47 | for i in range(1, 31): 48 | optimizer.zero_grad() 49 | # Assign create image to a variable to move forward in the model 50 | x = processed_image 51 | for index, layer in enumerate(self.model): 52 | # Forward pass layer by layer 53 | # x is not used after this point because it is only needed to trigger 54 | # the forward hook function 55 | x = layer(x) 56 | # Only need to forward until the selected layer is reached 57 | if index == self.selected_layer: 58 | # (forward hook function triggered) 59 | break 60 | # Loss function is the mean of the output of the selected layer/filter 61 | # We try to minimize the mean of the output of that specific filter 62 | loss = -torch.mean(self.conv_output) 63 | print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy())) 64 | # Backward 65 | loss.backward() 66 | # Update image 67 | optimizer.step() 68 | # Recreate image 69 | self.created_image = recreate_image(processed_image) 70 | # Save image 71 | if i % 5 == 0: 72 | im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \ 73 | '_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg' 74 | save_image(self.created_image, im_path) 75 | 76 | def visualise_layer_without_hooks(self): 77 | # Process image and return variable 78 | # Generate a random image 79 | random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3))) 80 | # Process image and return variable 81 | processed_image = preprocess_image(random_image, False) 82 | # Define optimizer for the image 83 | optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6) 84 | for i in range(1, 31): 85 | optimizer.zero_grad() 86 | # Assign create image to a variable to move forward in the model 87 | x = processed_image 88 | for index, layer in enumerate(self.model): 89 | # Forward pass layer by layer 90 | x = layer(x) 91 | if index == self.selected_layer: 92 | # Only need to forward until the selected layer is reached 93 | # Now, x is the output of the selected layer 94 | break 95 | # Here, we get the specific filter from the output of the convolution operation 96 | # x is a tensor of shape 1x512x28x28.(For layer 17) 97 | # So there are 512 unique filter outputs 98 | # Following line selects a filter from 512 filters so self.conv_output will become 99 | # a tensor of shape 28x28 100 | self.conv_output = x[0, self.selected_filter] 101 | # Loss function is the mean of the output of the selected layer/filter 102 | # We try to minimize the mean of the output of that specific filter 103 | loss = -torch.mean(self.conv_output) 104 | print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy())) 105 | # Backward 106 | loss.backward() 107 | # Update image 108 | optimizer.step() 109 | # Recreate image 110 | self.created_image = recreate_image(processed_image) 111 | # Save image 112 | if i % 5 == 0: 113 | im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \ 114 | '_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg' 115 | save_image(self.created_image, im_path) 116 | 117 | 118 | if __name__ == '__main__': 119 | cnn_layer = 17 120 | filter_pos = 5 121 | # Fully connected layer is not needed 122 | pretrained_model = models.vgg16(pretrained=True).features 123 | layer_vis = CNNLayerVisualization(pretrained_model, cnn_layer, filter_pos) 124 | 125 | # Layer visualization with pytorch hooks 126 | layer_vis.visualise_layer_with_hooks() 127 | 128 | # Layer visualization without pytorch hooks 129 | # layer_vis.visualise_layer_without_hooks() 130 | -------------------------------------------------------------------------------- /grad_cam/src/deep_dream.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Mon Nov 21 21:57:29 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import os 7 | from PIL import Image 8 | 9 | import torch 10 | from torch.optim import SGD 11 | from torchvision import models 12 | 13 | from misc_functions import preprocess_image, recreate_image, save_image 14 | 15 | 16 | class DeepDream(): 17 | """ 18 | Produces an image that minimizes the loss of a convolution 19 | operation for a specific layer and filter 20 | """ 21 | def __init__(self, model, selected_layer, selected_filter, im_path): 22 | self.model = model 23 | self.model.eval() 24 | self.selected_layer = selected_layer 25 | self.selected_filter = selected_filter 26 | self.conv_output = 0 27 | # Generate a random image 28 | self.created_image = Image.open(im_path).convert('RGB') 29 | # Hook the layers to get result of the convolution 30 | self.hook_layer() 31 | # Create the folder to export images if not exists 32 | if not os.path.exists('../generated'): 33 | os.makedirs('../generated') 34 | 35 | def hook_layer(self): 36 | def hook_function(module, grad_in, grad_out): 37 | # Gets the conv output of the selected filter (from selected layer) 38 | self.conv_output = grad_out[0, self.selected_filter] 39 | 40 | # Hook the selected layer 41 | self.model[self.selected_layer].register_forward_hook(hook_function) 42 | 43 | def dream(self): 44 | # Process image and return variable 45 | self.processed_image = preprocess_image(self.created_image, True) 46 | # Define optimizer for the image 47 | # Earlier layers need higher learning rates to visualize whereas layer layers need less 48 | optimizer = SGD([self.processed_image], lr=12, weight_decay=1e-4) 49 | for i in range(1, 251): 50 | optimizer.zero_grad() 51 | # Assign create image to a variable to move forward in the model 52 | x = self.processed_image 53 | for index, layer in enumerate(self.model): 54 | # Forward 55 | x = layer(x) 56 | # Only need to forward until we the selected layer is reached 57 | if index == self.selected_layer: 58 | break 59 | # Loss function is the mean of the output of the selected layer/filter 60 | # We try to minimize the mean of the output of that specific filter 61 | loss = -torch.mean(self.conv_output) 62 | print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy())) 63 | # Backward 64 | loss.backward() 65 | # Update image 66 | optimizer.step() 67 | # Recreate image 68 | self.created_image = recreate_image(self.processed_image) 69 | # Save image every 20 iteration 70 | if i % 10 == 0: 71 | print(self.created_image.shape) 72 | im_path = '../generated/ddream_l' + str(self.selected_layer) + \ 73 | '_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg' 74 | save_image(self.created_image, im_path) 75 | 76 | 77 | if __name__ == '__main__': 78 | # THIS OPERATION IS MEMORY HUNGRY! # 79 | # Because of the selected image is very large 80 | # If it gives out of memory error or locks the computer 81 | # Try it with a smaller image 82 | cnn_layer = 34 83 | filter_pos = 94 84 | 85 | im_path = '../input_images/dd_tree.jpg' 86 | # Fully connected layer is not needed 87 | pretrained_model = models.vgg19(pretrained=True).features 88 | dd = DeepDream(pretrained_model, cnn_layer, filter_pos, im_path) 89 | # This operation can also be done without Pytorch hooks 90 | # See layer visualisation for the implementation without hooks 91 | dd.dream() 92 | -------------------------------------------------------------------------------- /grad_cam/src/generate_class_specific_samples.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 26 14:19:44 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import os 7 | import numpy as np 8 | 9 | from torch.optim import SGD 10 | from torchvision import models 11 | 12 | from misc_functions import preprocess_image, recreate_image, save_image 13 | 14 | 15 | class ClassSpecificImageGeneration(): 16 | """ 17 | Produces an image that maximizes a certain class with gradient ascent 18 | """ 19 | def __init__(self, model, target_class): 20 | self.mean = [-0.485, -0.456, -0.406] 21 | self.std = [1/0.229, 1/0.224, 1/0.225] 22 | self.model = model 23 | self.model.eval() 24 | self.target_class = target_class 25 | # Generate a random image 26 | self.created_image = np.uint8(np.random.uniform(0, 255, (224, 224, 3))) 27 | # Create the folder to export images if not exists 28 | if not os.path.exists('../generated'): 29 | os.makedirs('../generated') 30 | 31 | def generate(self): 32 | initial_learning_rate = 6 33 | for i in range(1, 150): 34 | # Process image and return variable 35 | self.processed_image = preprocess_image(self.created_image, False) 36 | # Define optimizer for the image 37 | optimizer = SGD([self.processed_image], lr=initial_learning_rate) 38 | # Forward 39 | output = self.model(self.processed_image) 40 | # Target specific class 41 | class_loss = -output[0, self.target_class] 42 | print('Iteration:', str(i), 'Loss', "{0:.2f}".format(class_loss.data.numpy())) 43 | # Zero grads 44 | self.model.zero_grad() 45 | # Backward 46 | class_loss.backward() 47 | # Update image 48 | optimizer.step() 49 | # Recreate image 50 | self.created_image = recreate_image(self.processed_image) 51 | if i % 10 == 0: 52 | # Save image 53 | im_path = '../generated/c_specific_iteration_'+str(i)+'.jpg' 54 | save_image(self.created_image, im_path) 55 | return self.processed_image 56 | 57 | 58 | if __name__ == '__main__': 59 | target_class = 130 # Flamingo 60 | pretrained_model = models.alexnet(pretrained=True) 61 | csig = ClassSpecificImageGeneration(pretrained_model, target_class) 62 | csig.generate() 63 | -------------------------------------------------------------------------------- /grad_cam/src/gradcam.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 26 11:06:51 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | from PIL import Image 7 | import numpy as np 8 | import torch 9 | 10 | from misc_functions import get_example_params, save_class_activation_images 11 | 12 | 13 | class CamExtractor(): 14 | """ 15 | Extracts cam features from the model 16 | """ 17 | def __init__(self, model, target_layer): 18 | self.model = model 19 | self.target_layer = target_layer 20 | self.gradients = None 21 | 22 | def save_gradient(self, grad): 23 | self.gradients = grad 24 | 25 | def forward_pass_on_convolutions(self, x): 26 | """ 27 | Does a forward pass on convolutions, hooks the function at given layer 28 | """ 29 | conv_output = None 30 | print(self.model) 31 | for module_pos, module in self.model.features._modules.items(): 32 | print("@@@@ x @@@",x.shape) 33 | print("module_pos",module) 34 | # print() 35 | x = module(x) # Forward 36 | print("X_module",x.shape) 37 | if int(module_pos) == self.target_layer: 38 | x.register_hook(self.save_gradient) 39 | conv_output = x # Save the convolution output on that layer 40 | return conv_output, x 41 | 42 | def forward_pass(self, x): 43 | """ 44 | Does a full forward pass on the model 45 | """ 46 | # Forward pass on the convolutions 47 | conv_output, x = self.forward_pass_on_convolutions(x) 48 | x = x.view(x.size(0), -1) # Flatten 49 | # Forward pass on the classifier 50 | print("X_in_forward_pass_before_classifier",x.shape) 51 | x = self.model.classifier(x) 52 | return conv_output, x 53 | 54 | 55 | class GradCam(): 56 | """ 57 | Produces class activation map 58 | """ 59 | def __init__(self, model, target_layer): 60 | self.model = model 61 | self.model.eval() 62 | # Define extractor 63 | self.extractor = CamExtractor(self.model, target_layer) 64 | 65 | def generate_cam(self, input_image, target_class=None): 66 | # Full forward pass 67 | # conv_output is the output of convolutions at specified layer 68 | # model_output is the final output of the model (1, 1000) 69 | conv_output, model_output = self.extractor.forward_pass(input_image) 70 | print("conv_output",conv_output.shape) 71 | print("model_output",model_output.shape) 72 | if target_class is None: 73 | target_class = np.argmax(model_output.data.numpy()) 74 | # Target for backprop 75 | one_hot_output = torch.FloatTensor(1, model_output.size()[-1]).zero_() 76 | one_hot_output[0][target_class] = 1 77 | # Zero grads 78 | self.model.features.zero_grad() 79 | self.model.classifier.zero_grad() 80 | # Backward pass with specified target 81 | model_output.backward(gradient=one_hot_output, retain_graph=True) 82 | # Get hooked gradients 83 | guided_gradients = self.extractor.gradients.data.numpy()[0] 84 | # Get convolution outputs 85 | target = conv_output.data.numpy()[0] 86 | # Get weights from gradients 87 | weights = np.mean(guided_gradients, axis=(1, 2)) # Take averages for each gradient 88 | # Create empty numpy array for cam 89 | cam = np.ones(target.shape[1:], dtype=np.float32) 90 | # Multiply each weight with its conv output and then, sum 91 | for i, w in enumerate(weights): 92 | cam += w * target[i, :, :] 93 | cam = np.maximum(cam, 0) 94 | cam = (cam - np.min(cam)) / (np.max(cam) - np.min(cam)) # Normalize between 0-1 95 | cam = np.uint8(cam * 255) 96 | print("cam_shape",cam.shape) # Scale between 0-255 to visualize 97 | cam = np.uint8(Image.fromarray(cam).resize((input_image.shape[2], 98 | input_image.shape[3]), Image.ANTIALIAS)) 99 | # ^ I am extremely unhappy with this line. Originally resizing was done in cv2 which 100 | # supports resizing numpy matrices, however, when I moved the repository to PIL, this 101 | # option is out of the window. So, in order to use resizing with ANTIALIAS feature of PIL, 102 | # I briefly convert matrix to PIL image and then back. 103 | # If there is a more beautiful way, send a PR. 104 | return cam 105 | 106 | 107 | if __name__ == '__main__': 108 | # Get params 109 | class_no=1 110 | image_no=84 111 | check_target_class=1 112 | 113 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 114 | get_example_params(class_no,image_no,check_target_class) 115 | # Grad cam 116 | grad_cam = GradCam(pretrained_model, target_layer='35') 117 | # Generate cam mask 118 | cam = grad_cam.generate_cam(prep_img, target_class) 119 | # Save mask 120 | save_class_activation_images(original_image, cam, file_name_to_export) 121 | print('Grad cam completed') 122 | -------------------------------------------------------------------------------- /grad_cam/src/gradcam.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/src/gradcam.pyc -------------------------------------------------------------------------------- /grad_cam/src/guided_backprop.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 26 11:23:47 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import torch 7 | from torch.nn import ReLU 8 | 9 | from misc_functions import (get_example_params, 10 | convert_to_grayscale, 11 | save_gradient_images, 12 | get_positive_negative_saliency) 13 | 14 | 15 | class GuidedBackprop(): 16 | """ 17 | Produces gradients generated with guided back propagation from the given image 18 | """ 19 | def __init__(self, model): 20 | self.model = model 21 | self.gradients = None 22 | self.forward_relu_outputs = [] 23 | # Put model in evaluation mode 24 | self.model.eval() 25 | self.update_relus() 26 | self.hook_layers() 27 | 28 | def hook_layers(self): 29 | def hook_function(module, grad_in, grad_out): 30 | self.gradients = grad_in[0] 31 | # Register hook to the first layer 32 | first_layer = list(self.model.features._modules.items())[0][1] 33 | first_layer.register_backward_hook(hook_function) 34 | 35 | def update_relus(self): 36 | """ 37 | Updates relu activation functions so that 38 | 1- stores output in forward pass 39 | 2- imputes zero for gradient values that are less than zero 40 | """ 41 | def relu_backward_hook_function(module, grad_in, grad_out): 42 | """ 43 | If there is a negative gradient, change it to zero 44 | """ 45 | # Get last forward output 46 | corresponding_forward_output = self.forward_relu_outputs[-1] 47 | corresponding_forward_output[corresponding_forward_output > 0] = 1 48 | modified_grad_out = corresponding_forward_output * torch.clamp(grad_in[0], min=0.0) 49 | del self.forward_relu_outputs[-1] # Remove last forward output 50 | return (modified_grad_out,) 51 | 52 | def relu_forward_hook_function(module, ten_in, ten_out): 53 | """ 54 | Store results of forward pass 55 | """ 56 | self.forward_relu_outputs.append(ten_out) 57 | 58 | # Loop through layers, hook up ReLUs 59 | for pos, module in self.model.features._modules.items(): 60 | if isinstance(module, ReLU): 61 | module.register_backward_hook(relu_backward_hook_function) 62 | module.register_forward_hook(relu_forward_hook_function) 63 | 64 | def generate_gradients(self, input_image, target_class): 65 | # Forward pass 66 | model_output = self.model(input_image) 67 | # Zero gradients 68 | self.model.zero_grad() 69 | # Target for backprop 70 | one_hot_output = torch.FloatTensor(1, model_output.size()[-1]).zero_() 71 | one_hot_output[0][target_class] = 1 72 | # Backward pass 73 | model_output.backward(gradient=one_hot_output) 74 | # Convert Pytorch variable to numpy array 75 | # [0] to get rid of the first channel (1,3,224,224) 76 | gradients_as_arr = self.gradients.data.numpy()[0] 77 | return gradients_as_arr 78 | 79 | 80 | if __name__ == '__main__': 81 | class_no=1 82 | image_no=84 83 | check_target_class=1 84 | 85 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 86 | get_example_params(class_no,image_no,check_target_class) 87 | 88 | # Guided backprop 89 | GBP = GuidedBackprop(pretrained_model) 90 | # Get gradients 91 | guided_grads = GBP.generate_gradients(prep_img, target_class) 92 | # Save colored gradients 93 | save_gradient_images(guided_grads, file_name_to_export + '_Guided_BP_color') 94 | # Convert to grayscale 95 | grayscale_guided_grads = convert_to_grayscale(guided_grads) 96 | # Save grayscale gradients 97 | save_gradient_images(grayscale_guided_grads, file_name_to_export + '_Guided_BP_gray') 98 | # Positive and negative saliency maps 99 | pos_sal, neg_sal = get_positive_negative_saliency(guided_grads) 100 | save_gradient_images(pos_sal, file_name_to_export + '_pos_sal') 101 | save_gradient_images(neg_sal, file_name_to_export + '_neg_sal') 102 | print('Guided backprop completed') 103 | -------------------------------------------------------------------------------- /grad_cam/src/guided_backprop.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/src/guided_backprop.pyc -------------------------------------------------------------------------------- /grad_cam/src/guided_gradcam.py: -------------------------------------------------------------------------------- 1 | """ 2 | This programs helps to test the grad_cam on different examples 3 | 4 | author: Dipesh Tamboli - https://github.com/Dipeshtamboli 5 | author: Parth Patil - https://github.com/Parth1811 6 | """ 7 | # sftp://test1@10.107.42.42/home/Drive2/amil/my_network/net.py 8 | import numpy as np 9 | from misc_functions import (get_example_params, convert_to_grayscale, save_gradient_images) 10 | from gradcam import GradCam 11 | from guided_backprop import GuidedBackprop 12 | from net import Net 13 | 14 | def guided_grad_cam(grad_cam_mask, guided_backprop_mask): 15 | """ 16 | Guided grad cam is just pointwise multiplication of cam mask and 17 | guided backprop mask 18 | 19 | Args: 20 | grad_cam_mask (np_arr): Class activation map mask 21 | guided_backprop_mask (np_arr):Guided backprop mask 22 | """ 23 | cam_gb = np.multiply(grad_cam_mask, guided_backprop_mask) 24 | return cam_gb 25 | 26 | 27 | if __name__ == '__main__': 28 | # Get params 29 | class_no=6 #denseresidential 30 | image_no=84 31 | check_target_class=1 32 | 33 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 34 | get_example_params(class_no,image_no,check_target_class) 35 | 36 | # Grad cam 37 | target_layer=35 38 | gcv2 = GradCam(pretrained_model, target_layer=target_layer) 39 | # Generate cam mask 40 | cam = gcv2.generate_cam(prep_img, target_class) 41 | print('Grad cam completed') 42 | 43 | # Guided backprop 44 | GBP = GuidedBackprop(pretrained_model) 45 | # Get gradients 46 | guided_grads = GBP.generate_gradients(prep_img, target_class) 47 | print('Guided backpropagation completed') 48 | 49 | # Guided Grad cam 50 | cam_gb = guided_grad_cam(cam, guided_grads) 51 | save_gradient_images(cam_gb, file_name_to_export +",layer="+str(target_layer) +'_Guided_Grad_Cam') 52 | grayscale_cam_gb = convert_to_grayscale(cam_gb) 53 | save_gradient_images(grayscale_cam_gb, file_name_to_export +",layer="+str(target_layer)+ '_Guided_Grad_Cam_gray') 54 | save_gradient_images(guided_grads, file_name_to_export+",layer="+str(target_layer)+'_guided_grads ') 55 | print('Guided grad cam completed') 56 | -------------------------------------------------------------------------------- /grad_cam/src/inverted_representation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Wed Jan 17 08:05:11 2018 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import torch 7 | from torch.autograd import Variable 8 | from torch.optim import SGD 9 | import os 10 | 11 | from misc_functions import get_example_params, recreate_image, save_image 12 | 13 | 14 | class InvertedRepresentation(): 15 | def __init__(self, model): 16 | self.model = model 17 | self.model.eval() 18 | if not os.path.exists('../generated'): 19 | os.makedirs('../generated') 20 | 21 | def alpha_norm(self, input_matrix, alpha): 22 | """ 23 | Converts matrix to vector then calculates the alpha norm 24 | """ 25 | alpha_norm = ((input_matrix.view(-1))**alpha).sum() 26 | return alpha_norm 27 | 28 | def total_variation_norm(self, input_matrix, beta): 29 | """ 30 | Total variation norm is the second norm in the paper 31 | represented as R_V(x) 32 | """ 33 | to_check = input_matrix[:, :-1, :-1] # Trimmed: right - bottom 34 | one_bottom = input_matrix[:, 1:, :-1] # Trimmed: top - right 35 | one_right = input_matrix[:, :-1, 1:] # Trimmed: top - right 36 | total_variation = (((to_check - one_bottom)**2 + 37 | (to_check - one_right)**2)**(beta/2)).sum() 38 | return total_variation 39 | 40 | def euclidian_loss(self, org_matrix, target_matrix): 41 | """ 42 | Euclidian loss is the main loss function in the paper 43 | ||fi(x) - fi(x_0)||_2^2& / ||fi(x_0)||_2^2 44 | """ 45 | distance_matrix = target_matrix - org_matrix 46 | euclidian_distance = self.alpha_norm(distance_matrix, 2) 47 | normalized_euclidian_distance = euclidian_distance / self.alpha_norm(org_matrix, 2) 48 | return normalized_euclidian_distance 49 | 50 | def get_output_from_specific_layer(self, x, layer_id): 51 | """ 52 | Saves the output after a forward pass until nth layer 53 | This operation could be done with a forward hook too 54 | but this one is simpler (I think) 55 | """ 56 | layer_output = None 57 | for index, layer in enumerate(self.model.features): 58 | x = layer(x) 59 | if str(index) == str(layer_id): 60 | layer_output = x[0] 61 | break 62 | return layer_output 63 | 64 | def generate_inverted_image_specific_layer(self, input_image, img_size, target_layer=3): 65 | # Generate a random image which we will optimize 66 | opt_img = Variable(1e-1 * torch.randn(1, 3, img_size, img_size), requires_grad=True) 67 | # Define optimizer for previously created image 68 | optimizer = SGD([opt_img], lr=1e4, momentum=0.9) 69 | # Get the output from the model after a forward pass until target_layer 70 | # with the input image (real image, NOT the randomly generated one) 71 | input_image_layer_output = \ 72 | self.get_output_from_specific_layer(input_image, target_layer) 73 | 74 | # Alpha regularization parametrs 75 | # Parameter alpha, which is actually sixth norm 76 | alpha_reg_alpha = 6 77 | # The multiplier, lambda alpha 78 | alpha_reg_lambda = 1e-7 79 | 80 | # Total variation regularization parameters 81 | # Parameter beta, which is actually second norm 82 | tv_reg_beta = 2 83 | # The multiplier, lambda beta 84 | tv_reg_lambda = 1e-8 85 | 86 | for i in range(201): 87 | optimizer.zero_grad() 88 | # Get the output from the model after a forward pass until target_layer 89 | # with the generated image (randomly generated one, NOT the real image) 90 | output = self.get_output_from_specific_layer(opt_img, target_layer) 91 | # Calculate euclidian loss 92 | euc_loss = 1e-1 * self.euclidian_loss(input_image_layer_output.detach(), output) 93 | # Calculate alpha regularization 94 | reg_alpha = alpha_reg_lambda * self.alpha_norm(opt_img, alpha_reg_alpha) 95 | # Calculate total variation regularization 96 | reg_total_variation = tv_reg_lambda * self.total_variation_norm(opt_img, 97 | tv_reg_beta) 98 | # Sum all to optimize 99 | loss = euc_loss + reg_alpha + reg_total_variation 100 | # Step 101 | loss.backward() 102 | optimizer.step() 103 | # Generate image every 5 iterations 104 | if i % 5 == 0: 105 | print('Iteration:', str(i), 'Loss:', loss.data.numpy()) 106 | recreated_im = recreate_image(opt_img) 107 | im_path = '../generated/Inv_Image_Layer_' + str(target_layer) + \ 108 | '_Iteration_' + str(i) + '.jpg' 109 | save_image(recreated_im, im_path) 110 | 111 | # Reduce learning rate every 40 iterations 112 | if i % 40 == 0: 113 | for param_group in optimizer.param_groups: 114 | param_group['lr'] *= 1/10 115 | 116 | 117 | if __name__ == '__main__': 118 | # Get params 119 | target_example = 0 # Snake 120 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 121 | get_example_params(target_example) 122 | 123 | inverted_representation = InvertedRepresentation(pretrained_model) 124 | image_size = 224 # width & height 125 | target_layer = 4 126 | inverted_representation.generate_inverted_image_specific_layer(prep_img, 127 | image_size, 128 | target_layer) 129 | -------------------------------------------------------------------------------- /grad_cam/src/layer_activation_with_guided_backprop.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 26 11:23:47 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import torch 7 | from torch.nn import ReLU 8 | 9 | from misc_functions import (get_example_params, 10 | convert_to_grayscale, 11 | save_gradient_images, 12 | get_positive_negative_saliency) 13 | 14 | 15 | class GuidedBackprop(): 16 | """ 17 | Produces gradients generated with guided back propagation from the given image 18 | """ 19 | def __init__(self, model): 20 | self.model = model 21 | self.gradients = None 22 | self.forward_relu_outputs = [] 23 | # Put model in evaluation mode 24 | self.model.eval() 25 | self.update_relus() 26 | self.hook_layers() 27 | 28 | def hook_layers(self): 29 | def hook_function(module, grad_in, grad_out): 30 | self.gradients = grad_in[0] 31 | # Register hook to the first layer 32 | first_layer = list(self.model.features._modules.items())[0][1] 33 | first_layer.register_backward_hook(hook_function) 34 | 35 | def update_relus(self): 36 | """ 37 | Updates relu activation functions so that 38 | 1- stores output in forward pass 39 | 2- imputes zero for gradient values that are less than zero 40 | """ 41 | def relu_backward_hook_function(module, grad_in, grad_out): 42 | """ 43 | If there is a negative gradient, change it to zero 44 | """ 45 | # Get last forward output 46 | corresponding_forward_output = self.forward_relu_outputs[-1] 47 | corresponding_forward_output[corresponding_forward_output > 0] = 1 48 | modified_grad_out = corresponding_forward_output * torch.clamp(grad_in[0], min=0.0) 49 | del self.forward_relu_outputs[-1] # Remove last forward output 50 | return (modified_grad_out,) 51 | 52 | def relu_forward_hook_function(module, ten_in, ten_out): 53 | """ 54 | Store results of forward pass 55 | """ 56 | self.forward_relu_outputs.append(ten_out) 57 | 58 | # Loop through layers, hook up ReLUs 59 | for pos, module in self.model.features._modules.items(): 60 | if isinstance(module, ReLU): 61 | module.register_backward_hook(relu_backward_hook_function) 62 | module.register_forward_hook(relu_forward_hook_function) 63 | 64 | def generate_gradients(self, input_image, target_class, cnn_layer, filter_pos): 65 | self.model.zero_grad() 66 | # Forward pass 67 | x = input_image 68 | for index, layer in enumerate(self.model.features): 69 | # Forward pass layer by layer 70 | # x is not used after this point because it is only needed to trigger 71 | # the forward hook function 72 | x = layer(x) 73 | # Only need to forward until the selected layer is reached 74 | if index == cnn_layer: 75 | # (forward hook function triggered) 76 | break 77 | conv_output = torch.sum(torch.abs(x[0, filter_pos])) 78 | # Backward pass 79 | conv_output.backward() 80 | # Convert Pytorch variable to numpy array 81 | # [0] to get rid of the first channel (1,3,224,224) 82 | gradients_as_arr = self.gradients.data.numpy()[0] 83 | return gradients_as_arr 84 | 85 | 86 | if __name__ == '__main__': 87 | cnn_layer = 10 88 | filter_pos = 5 89 | target_example = 2 # Spider 90 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 91 | get_example_params(target_example) 92 | 93 | # File export name 94 | file_name_to_export = file_name_to_export + '_layer' + str(cnn_layer) + '_filter' + str(filter_pos) 95 | # Guided backprop 96 | GBP = GuidedBackprop(pretrained_model) 97 | # Get gradients 98 | guided_grads = GBP.generate_gradients(prep_img, target_class, cnn_layer, filter_pos) 99 | # Save colored gradients 100 | save_gradient_images(guided_grads, file_name_to_export + '_Guided_BP_color') 101 | # Convert to grayscale 102 | grayscale_guided_grads = convert_to_grayscale(guided_grads) 103 | # Save grayscale gradients 104 | save_gradient_images(grayscale_guided_grads, file_name_to_export + '_Guided_BP_gray') 105 | # Positive and negative saliency maps 106 | pos_sal, neg_sal = get_positive_negative_saliency(guided_grads) 107 | save_gradient_images(pos_sal, file_name_to_export + '_pos_sal') 108 | save_gradient_images(neg_sal, file_name_to_export + '_neg_sal') 109 | print('Layer Guided backprop completed') 110 | -------------------------------------------------------------------------------- /grad_cam/src/misc_functions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 21 11:09:09 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import os 7 | import copy 8 | import numpy as np 9 | from PIL import Image 10 | import matplotlib.cm as mpl_color_map 11 | import cv2 12 | import torch 13 | from torch.autograd import Variable 14 | from torchvision import models 15 | from os.path import dirname, abspath, join 16 | from net import Net 17 | 18 | PROJECT_FILE_DIR = dirname(dirname(abspath(__file__))) 19 | 20 | def convert_to_grayscale(im_as_arr): 21 | """ 22 | Converts 3d image to grayscale 23 | 24 | Args: 25 | im_as_arr (numpy arr): RGB image with shape (D,W,H) 26 | 27 | returns: 28 | grayscale_im (numpy_arr): Grayscale image with shape (1,W,D) 29 | """ 30 | grayscale_im = np.sum(np.abs(im_as_arr), axis=0) 31 | im_max = np.percentile(grayscale_im, 99) 32 | im_min = np.min(grayscale_im) 33 | grayscale_im = (np.clip((grayscale_im - im_min) / (im_max - im_min), 0, 1)) 34 | grayscale_im = np.expand_dims(grayscale_im, axis=0) 35 | return grayscale_im 36 | 37 | 38 | def save_gradient_images(gradient, file_name): 39 | """ 40 | Exports the original gradient image 41 | 42 | Args: 43 | gradient (np arr): Numpy array of the gradient with shape (3, 224, 224) 44 | file_name (str): File name to be exported 45 | """ 46 | if not os.path.exists(join(PROJECT_FILE_DIR, 'results')): 47 | os.makedirs(join(PROJECT_FILE_DIR, 'results')) 48 | # Normalize 49 | gradient = gradient - gradient.min() 50 | gradient /= gradient.max() 51 | # Save image 52 | path_to_file = os.path.join(PROJECT_FILE_DIR, 'results', file_name + '.jpg') 53 | save_image(gradient, path_to_file) 54 | 55 | 56 | def save_class_activation_images(org_img, activation_map, file_name): 57 | """ 58 | Saves cam activation map and activation map on the original image 59 | 60 | Args: 61 | org_img (PIL img): Original image 62 | activation_map (numpy arr): Activation map (grayscale) 0-255 63 | file_name (str): File name of the exported image 64 | """ 65 | if not os.path.exists('../results'): 66 | os.makedirs('../results') 67 | # Grayscale activation map 68 | heatmap, heatmap_on_image = apply_colormap_on_image(org_img, activation_map, 'hsv') 69 | # Save colored heatmap 70 | path_to_file = os.path.join('../results', file_name+'_Cam_Heatmap.png') 71 | print(np.max(heatmap)) 72 | save_image(heatmap, path_to_file) 73 | # Save heatmap on iamge 74 | print() 75 | print(np.max(heatmap_on_image)) 76 | path_to_file = os.path.join('../results', file_name+'_Cam_On_Image.png') 77 | save_image(heatmap_on_image, path_to_file) 78 | # SAve grayscale heatmap 79 | print() 80 | print(np.max(activation_map)) 81 | path_to_file = os.path.join('../results', file_name+'_Cam_Grayscale.png') 82 | save_image(activation_map, path_to_file) 83 | 84 | 85 | def apply_colormap_on_image(org_im, activation, colormap_name): 86 | """ 87 | Apply heatmap on image 88 | Args: 89 | org_img (PIL img): Original image 90 | activation_map (numpy arr): Activation map (grayscale) 0-255 91 | colormap_name (str): Name of the colormap 92 | """ 93 | # Get colormap 94 | color_map = mpl_color_map.get_cmap(colormap_name) 95 | no_trans_heatmap = color_map(activation) 96 | # Change alpha channel in colormap to make sure original image is displayed 97 | heatmap = copy.copy(no_trans_heatmap) 98 | heatmap[:, :, 3] = 0.4 99 | heatmap = Image.fromarray((heatmap*255).astype(np.uint8)) 100 | no_trans_heatmap = Image.fromarray((no_trans_heatmap*255).astype(np.uint8)) 101 | 102 | # Apply heatmap on iamge 103 | heatmap_on_image = Image.new("RGBA", org_im.size) 104 | heatmap_on_image = Image.alpha_composite(heatmap_on_image, org_im.convert('RGBA')) 105 | heatmap_on_image = Image.alpha_composite(heatmap_on_image, heatmap) 106 | return no_trans_heatmap, heatmap_on_image 107 | 108 | 109 | def save_image(im, path): 110 | """ 111 | Saves a numpy matrix of shape D(1 or 3) x W x H as an image 112 | Args: 113 | im_as_arr (Numpy array): Matrix of shape DxWxH 114 | path (str): Path to the image 115 | 116 | TODO: Streamline image saving, it is ugly. 117 | """ 118 | if isinstance(im, np.ndarray): 119 | if len(im.shape) == 2: 120 | im = np.expand_dims(im, axis=0) 121 | print('3_channel_image') 122 | print(im.shape) 123 | if im.shape[0] == 1: 124 | # Converting an image with depth = 1 to depth = 3, repeating the same values 125 | # For some reason PIL complains when I want to save channel image as jpg without 126 | # additional format in the .save() 127 | print('1_channel_image') 128 | im = np.repeat(im, 3, axis=0) 129 | print(im.shape) 130 | # Convert to values to range 1-255 and W,H, D 131 | # A bandaid fix to an issue with gradcam 132 | if im.shape[0] == 3 and np.max(im) == 1: 133 | im = im.transpose(1, 2, 0) * 255 134 | elif im.shape[0] == 3 and np.max(im) > 1: 135 | im = im.transpose(1, 2, 0) 136 | im = Image.fromarray(im.astype(np.uint8)) 137 | im.save(path) 138 | 139 | 140 | def preprocess_image(pil_im, resize_im=True): 141 | """ 142 | Processes image for CNNs 143 | 144 | Args: 145 | PIL_img (PIL_img): Image to process 146 | resize_im (bool): Resize to 224 or not 147 | returns: 148 | im_as_var (torch variable): Variable that contains processed float tensor 149 | """ 150 | # mean and std list for channels (Imagenet) 151 | mean = [0.485, 0.456, 0.406] 152 | std = [0.229, 0.224, 0.225] 153 | # Resize image 154 | if resize_im: 155 | pil_im.thumbnail((224, 224)) 156 | im_as_arr = np.float32(pil_im) 157 | im_as_arr = im_as_arr.transpose(2, 0, 1) # Convert array to D,W,H 158 | # Normalize the channels 159 | for channel, _ in enumerate(im_as_arr): 160 | im_as_arr[channel] /= 255 161 | im_as_arr[channel] -= mean[channel] 162 | im_as_arr[channel] /= std[channel] 163 | # Convert to float tensor 164 | im_as_ten = torch.from_numpy(im_as_arr).float() 165 | # Add one more channel to the beginning. Tensor shape = 1,3,224,224 166 | im_as_ten.unsqueeze_(0) 167 | # Convert to Pytorch variable 168 | im_as_var = Variable(im_as_ten, requires_grad=True) 169 | return im_as_var 170 | 171 | 172 | def recreate_image(im_as_var): 173 | """ 174 | Recreates images from a torch variable, sort of reverse preprocessing 175 | Args: 176 | im_as_var (torch variable): Image to recreate 177 | returns: 178 | recreated_im (numpy arr): Recreated image in array 179 | """ 180 | reverse_mean = [-0.485, -0.456, -0.406] 181 | reverse_std = [1/0.229, 1/0.224, 1/0.225] 182 | recreated_im = copy.copy(im_as_var.data.numpy()[0]) 183 | for c in range(3): 184 | recreated_im[c] /= reverse_std[c] 185 | recreated_im[c] -= reverse_mean[c] 186 | recreated_im[recreated_im > 1] = 1 187 | recreated_im[recreated_im < 0] = 0 188 | recreated_im = np.round(recreated_im * 255) 189 | 190 | recreated_im = np.uint8(recreated_im).transpose(1, 2, 0) 191 | return recreated_im 192 | 193 | 194 | def get_positive_negative_saliency(gradient): 195 | """ 196 | Generates positive and negative saliency maps based on the gradient 197 | Args: 198 | gradient (numpy arr): Gradient of the operation to visualize 199 | 200 | returns: 201 | pos_saliency ( ) 202 | """ 203 | pos_saliency = (np.maximum(0, gradient) / gradient.max()) 204 | neg_saliency = (np.maximum(0, -gradient) / -gradient.min()) 205 | return pos_saliency, neg_saliency 206 | 207 | 208 | def get_example_params(class_no,image_no,check_target_class): 209 | """ 210 | Gets used variables for almost all visualizations, like the image, model etc. 211 | 212 | Args: 213 | example_index (int): Image id to use from examples 214 | 215 | returns: 216 | original_image (numpy arr): Original image read from the file 217 | prep_img (numpy_arr): Processed image 218 | target_class (int): Target class for the image 219 | file_name_to_export (string): File name to export the visualizations 220 | pretrained_model(Pytorch model): Model to use for the operations 221 | """ 222 | 223 | # class_list=[ 224 | # "agricultural" , # class_num =0 225 | # "airplane" , # class_num =1 226 | # "baseballdiamond" , # class_num =2 227 | # "beach" , # class_num =3 228 | # "buildings" , # class_num =4 229 | # "chaparral" , # class_num =5 230 | # "denseresidential" , # class_num =6 231 | # "forest" , # class_num =7 232 | # "freeway" , # class_num =8 233 | # "golfcourse" , # class_num =9 234 | # "harbor" , # class_num =10 235 | # "intersection" , # class_num =11 236 | # "mediumresidential" , # class_num =12 237 | # "mobilehomepark" , # class_num =13 238 | # "overpass" , # class_num =14 239 | # "parkinglot" , # class_num =15 240 | # "river" , # class_num =16 241 | # "runway" , # class_num =17 242 | # "sparseresidential" , # class_num =18 243 | # "storagetanks" , # class_num =19 244 | # "tenniscourt"] # class_num =20 245 | 246 | # Pick one of the examples 247 | # path_to_test_folder = join(PROJECT_FILE_DIR, "data") 248 | # print('###################################',path_to_test_folder) 249 | # img_path = join(path_to_test_folder, class_list[class_no], class_list[class_no] + str(image_no)+".tif") 250 | # print img_path 251 | 252 | # overide image path for testing 253 | custom_path = join(PROJECT_FILE_DIR, "inputs/vgg_224.png") 254 | img_path = custom_path 255 | 256 | target_class = check_target_class 257 | file_name_to_export = img_path[img_path.rfind('/')+1:img_path.rfind('.')] 258 | 259 | # Read image 260 | original_image = Image.open(img_path).convert('RGB') 261 | 262 | # Process image 263 | prep_img = preprocess_image(original_image) 264 | 265 | # Define model 266 | # sftp://test1@10.107.42.42/home/Drive2/amil/grad_cam/grad_cam/state_dict.pt 267 | # path_weights = join(PROJECT_FILE_DIR, "model.pt") 268 | # path_model = join(PROJECT_FILE_DIR, "state_dict.pt") 269 | # from 270 | path_model = join(PROJECT_FILE_DIR, "vgg_models/model.pt") 271 | path_weights = join(PROJECT_FILE_DIR, "vgg_models/statedict.pt") 272 | 273 | checkpoint = torch.load(path_weights,map_location="cpu") 274 | pretrained_model = torch.load(path_model,map_location="cpu") 275 | # pretrained_model = Net() 276 | pretrained_model.load_state_dict(checkpoint) 277 | 278 | return (original_image, 279 | prep_img, 280 | target_class, 281 | file_name_to_export, 282 | pretrained_model) 283 | -------------------------------------------------------------------------------- /grad_cam/src/misc_functions.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/src/misc_functions.pyc -------------------------------------------------------------------------------- /grad_cam/src/net.py: -------------------------------------------------------------------------------- 1 | #model 2 | from __future__ import print_function 3 | import argparse 4 | import torch 5 | import numpy as np 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import torch.optim as optim 9 | import torchvision 10 | from torchvision import datasets, transforms 11 | import time 12 | from tensorboardX import SummaryWriter 13 | from datetime import datetime 14 | import pickle 15 | import matplotlib.pyplot as plt 16 | # from sklearn.metrics import f1_score, classification_report, confusion_matrix 17 | 18 | writer = SummaryWriter() 19 | 20 | transform = transforms.Compose([transforms.CenterCrop(128),transforms.ToTensor(),]) 21 | 22 | data_train = torchvision.datasets.ImageFolder("/home/Drive2/amil/data/test/40X", transform=transform) 23 | print(data_train) 24 | data_test = torchvision.datasets.ImageFolder("/home/Drive2/amil/data/train/40X", transform=transform) 25 | print(data_test) 26 | 27 | class Net(nn.Module): 28 | def __init__(self): 29 | super(Net, self).__init__() 30 | self.conv1 = nn.Conv2d(3, 64, 3, 1, 1) 31 | self.conv2 = nn.Conv2d(64, 64, 3, 1, 1) 32 | self.conv3 = nn.Conv2d(64, 128, 3, 1, 1) 33 | self.conv4 = nn.Conv2d(128, 128, 3, 1, 1) 34 | self.conv5 = nn.Conv2d(128, 128, 3, 1, 1) 35 | self.dropout1=nn.Dropout(p=0.5, inplace=False) 36 | self.dropout2=nn.Dropout(p=0.5, inplace=False) 37 | self.fc1 = nn.Linear(4*4*128,1024) 38 | self.fc2 = nn.Linear(1024, 2) 39 | # self.fc3 = nn.Linear(200, 100) 40 | 41 | def forward(self, x): 42 | # torch.nn.LeakyReLU(negative_slope=0.01, inplace=False) 43 | # print("##@@@@@@@@@@@@@@@@@") 44 | m=nn.LeakyReLU(0.01) 45 | # print(x.shape) 46 | x = m(self.conv1(x)) 47 | # print(x.shape) 48 | x = F.max_pool2d(x, 2, 2) 49 | x = m(self.conv2(x)) 50 | x = F.max_pool2d(x, 2, 2) 51 | 52 | x = m(self.conv3(x)) 53 | x = F.max_pool2d(x, 2, 2) 54 | x = m(self.conv4(x)) 55 | x = F.max_pool2d(x, 2, 2) 56 | x = m(self.conv5(x)) 57 | x = F.max_pool2d(x, 2, 2) 58 | 59 | # x = F.max_pool2d(x, 2, 2) 60 | x = self.dropout1(x) 61 | x = x.view(-1, 4*4*128) 62 | x = m(self.fc1(x)) 63 | # x = m(self.fc2(x)) 64 | x = self.dropout2(x) 65 | x = self.fc2(x) 66 | # print("x.shape",x.shape) 67 | return F.log_softmax(x, dim=1) 68 | 69 | 70 | def train(args, model, device, train_loader, optimizer, epoch): 71 | model.train() 72 | correct = 0 73 | # print("-----------",train_loader) 74 | for batch_idx, (data, target) in enumerate(train_loader): 75 | data, target = data.to(device), target.to(device) 76 | # print(len(data),len(target)) 77 | optimizer.zero_grad() 78 | output = model(data) 79 | # print(len(output), len(target)) 80 | loss = F.nll_loss(output, target) 81 | loss.backward() 82 | optimizer.step() 83 | if batch_idx % args.log_interval == 0: 84 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 85 | epoch, batch_idx * len(data), len(train_loader.dataset), 86 | 100. * batch_idx / len(train_loader), loss.item())) 87 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 88 | correct += pred.eq(target.view_as(pred)).sum().item() 89 | print('\nTrain_accuracy: {:.0f}%\n'.format(100. * correct / len(train_loader.dataset))) 90 | writer.add_scalar('train_Accuracy_epoch',100. * correct / len(train_loader.dataset),epoch) 91 | writer.add_scalar('train_loss_epoch',loss/len(train_loader.dataset),epoch) 92 | return (100. * correct / len(train_loader.dataset)) 93 | def test(args, model, device, test_loader,epoch): 94 | print("test started") 95 | model.eval() 96 | test_loss = 0 97 | correct = 0 98 | with torch.no_grad(): 99 | for data, target in test_loader: 100 | data, target = data.to(device), target.to(device) 101 | output = model(data) 102 | test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss 103 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 104 | correct += pred.eq(target.view_as(pred)).sum().item() 105 | test_loss /= len(test_loader.dataset) 106 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 107 | test_loss, correct, len(test_loader.dataset), 108 | 100. * correct / len(test_loader.dataset))) 109 | writer.add_scalar('test_loss_epoch',test_loss,epoch) 110 | writer.add_scalar('test_Accuracy_epoch',100. * correct / len(test_loader.dataset),epoch) 111 | return (100. * correct / len(test_loader.dataset)) 112 | 113 | def main(): 114 | start = time.time() 115 | print ("into the main") 116 | parser = argparse.ArgumentParser(description='UC_Merced data_mynet') 117 | 118 | parser.add_argument('--batch-size', type=int, default=16, metavar='N', 119 | help='input batch size for training (default: 64)') 120 | 121 | parser.add_argument('--test-batch-size', type=int, default=16, metavar='N', 122 | help='input batch size for testing (default: 1000)') 123 | 124 | parser.add_argument('--epochs', type=int, default=10, metavar='N', 125 | help='number of epochs to train (default: 10)') 126 | 127 | 128 | parser.add_argument('--lr', type=float, default=0.0003 , metavar='LR', 129 | help='learning rate (default: 0.01)') 130 | 131 | parser.add_argument('--momentum', type=float, default=0.4, metavar='M', 132 | help='SGD momentum (default: 0.9)') 133 | 134 | parser.add_argument('--no-cuda', action='store_true', default=False, 135 | help='disables CUDA training') 136 | 137 | parser.add_argument('--seed', type=int, default=1, metavar='S', 138 | help='random seed (default: 1)') 139 | 140 | parser.add_argument('--log_interval', type=int, default=20, metavar='N', 141 | help='how many batches to wait before logging training status') 142 | 143 | parser.add_argument('--save-model', action='store_true', default=True, 144 | help='For Saving the current Model') 145 | 146 | args = parser.parse_args() 147 | 148 | use_cuda = not args.no_cuda and torch.cuda.is_available() 149 | device = torch.device("cuda" if use_cuda else "cpu") 150 | # device = "cpu" 151 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 152 | train_loader = torch.utils.data.DataLoader(data_train, batch_size=32 ,shuffle = True, **kwargs) 153 | test_loader = torch.utils.data.DataLoader(data_test, batch_size=32 ,shuffle = False, **kwargs) 154 | print("device: ",device) 155 | 156 | model = Net().to(device) 157 | print ("model transferred to device") 158 | # optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) 159 | # optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) 160 | optimizer = optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.99, eps=1e-08, weight_decay=0, momentum=args.momentum, centered=False) 161 | print ("optimizer choosed") 162 | print("#######__parameters__######") 163 | print("learning rate: ", args.lr, "\nmomentum: ", args.momentum, "\nepochs: ", args.epochs) 164 | print("############################") 165 | print("model:\n",model) 166 | print("############################") 167 | print("optimizer:\n",optimizer) 168 | print("############################") 169 | 170 | # for epoch in range(2): 171 | for epoch in range(1, args.epochs + 1): 172 | # print("-----",train_loader) 173 | train_acc=train(args, model, device, train_loader, optimizer, epoch) 174 | test_acc=test(args, model, device, test_loader, epoch) 175 | # writer.add_scalar('loss_fn2',loss.item(),epoch) 176 | # sftp://test1@10.107.42.42/home/Drive2/amil/my_network/runs 177 | if (args.save_model): 178 | torch.save(model.state_dict(),"/home/Drive2/amil/my_network/pt_files/file: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+"Breakthis_state_dict.pt") 179 | torch.save(model ,"/home/Drive2/amil/my_network/pt_files/file: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+"Breakthis_model.pt") 180 | save_name_pkl = "/home/Drive2/amil/my_network/pickel_files/file: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+" end.pkl" 181 | save_name_txt = "/home/Drive2/amil/my_network/txt_file/file: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+" end.txt" 182 | model_file = open(save_name_txt,"w") 183 | model_string = str(model) 184 | optimizer_string = str(optimizer) 185 | model_file.write(model_string) 186 | model_file.write(optimizer_string) 187 | model_file.write(save_name_txt) 188 | model_file.close() 189 | 190 | f=open(save_name_pkl,"wb") 191 | pickle.dump(model, f) 192 | end = time.time() 193 | print('time taken is ', (end-start)) 194 | 195 | 196 | if __name__ == '__main__': 197 | main() 198 | -------------------------------------------------------------------------------- /grad_cam/src/net.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dipeshtamboli/Image-Classification-and-Localization-using-Multiple-Instance-Learning/d05714517a55739f95bb0ed63aa6ec38ceb02c3c/grad_cam/src/net.pyc -------------------------------------------------------------------------------- /grad_cam/src/smooth_grad.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Wed Mar 28 10:12:13 2018 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import numpy as np 7 | 8 | from torch.autograd import Variable 9 | import torch 10 | 11 | from misc_functions import (get_example_params, 12 | convert_to_grayscale, 13 | save_gradient_images) 14 | from vanilla_backprop import VanillaBackprop 15 | # from guided_backprop import GuidedBackprop # To use with guided backprop 16 | 17 | 18 | def generate_smooth_grad(Backprop, prep_img, target_class, param_n, param_sigma_multiplier): 19 | """ 20 | Generates smooth gradients of given Backprop type. You can use this with both vanilla 21 | and guided backprop 22 | Args: 23 | Backprop (class): Backprop type 24 | prep_img (torch Variable): preprocessed image 25 | target_class (int): target class of imagenet 26 | param_n (int): Amount of images used to smooth gradient 27 | param_sigma_multiplier (int): Sigma multiplier when calculating std of noise 28 | """ 29 | # Generate an empty image/matrix 30 | smooth_grad = np.zeros(prep_img.size()[1:]) 31 | 32 | mean = 0 33 | sigma = param_sigma_multiplier / (torch.max(prep_img) - torch.min(prep_img)).item() 34 | for x in range(param_n): 35 | # Generate noise 36 | noise = Variable(prep_img.data.new(prep_img.size()).normal_(mean, sigma**2)) 37 | # Add noise to the image 38 | noisy_img = prep_img + noise 39 | # Calculate gradients 40 | vanilla_grads = Backprop.generate_gradients(noisy_img, target_class) 41 | # Add gradients to smooth_grad 42 | smooth_grad = smooth_grad + vanilla_grads 43 | # Average it out 44 | smooth_grad = smooth_grad / param_n 45 | return smooth_grad 46 | 47 | 48 | if __name__ == '__main__': 49 | # Get params 50 | target_example = 0 # Snake 51 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 52 | get_example_params(target_example) 53 | 54 | VBP = VanillaBackprop(pretrained_model) 55 | # GBP = GuidedBackprop(pretrained_model) # if you want to use GBP dont forget to 56 | # change the parametre in generate_smooth_grad 57 | 58 | param_n = 50 59 | param_sigma_multiplier = 4 60 | smooth_grad = generate_smooth_grad(VBP, # ^This parameter 61 | prep_img, 62 | target_class, 63 | param_n, 64 | param_sigma_multiplier) 65 | 66 | # Save colored gradients 67 | save_gradient_images(smooth_grad, file_name_to_export + '_SmoothGrad_color') 68 | # Convert to grayscale 69 | grayscale_smooth_grad = convert_to_grayscale(smooth_grad) 70 | # Save grayscale gradients 71 | save_gradient_images(grayscale_smooth_grad, file_name_to_export + '_SmoothGrad_gray') 72 | print('Smooth grad completed') 73 | -------------------------------------------------------------------------------- /grad_cam/src/vanilla_backprop.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Thu Oct 26 11:19:58 2017 3 | 4 | @author: Utku Ozbulak - github.com/utkuozbulak 5 | """ 6 | import torch 7 | 8 | from misc_functions import get_example_params, convert_to_grayscale, save_gradient_images 9 | 10 | 11 | class VanillaBackprop(): 12 | """ 13 | Produces gradients generated with vanilla back propagation from the image 14 | """ 15 | def __init__(self, model): 16 | self.model = model 17 | self.gradients = None 18 | # Put model in evaluation mode 19 | self.model.eval() 20 | # Hook the first layer to get the gradient 21 | self.hook_layers() 22 | 23 | def hook_layers(self): 24 | def hook_function(module, grad_in, grad_out): 25 | self.gradients = grad_in[0] 26 | 27 | # Register hook to the first layer 28 | first_layer = list(self.model.features._modules.items())[0][1] 29 | first_layer.register_backward_hook(hook_function) 30 | 31 | def generate_gradients(self, input_image, target_class): 32 | # Forward 33 | model_output = self.model(input_image) 34 | # Zero grads 35 | self.model.zero_grad() 36 | # Target for backprop 37 | one_hot_output = torch.FloatTensor(1, model_output.size()[-1]).zero_() 38 | one_hot_output[0][target_class] = 1 39 | # Backward pass 40 | model_output.backward(gradient=one_hot_output) 41 | # Convert Pytorch variable to numpy array 42 | # [0] to get rid of the first channel (1,3,224,224) 43 | gradients_as_arr = self.gradients.data.numpy()[0] 44 | return gradients_as_arr 45 | 46 | 47 | if __name__ == '__main__': 48 | # Get params 49 | target_example = 1 # Snake 50 | (original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\ 51 | get_example_params(target_example) 52 | # Vanilla backprop 53 | VBP = VanillaBackprop(pretrained_model) 54 | # Generate gradients 55 | vanilla_grads = VBP.generate_gradients(prep_img, target_class) 56 | # Save colored gradients 57 | save_gradient_images(vanilla_grads, file_name_to_export + '_Vanilla_BP_color') 58 | # Convert to grayscale 59 | grayscale_vanilla_grads = convert_to_grayscale(vanilla_grads) 60 | # Save grayscale gradients 61 | save_gradient_images(grayscale_vanilla_grads, file_name_to_export + '_Vanilla_BP_gray') 62 | print('Vanilla backprop completed') 63 | -------------------------------------------------------------------------------- /my_network/net.py: -------------------------------------------------------------------------------- 1 | #model 2 | from __future__ import print_function 3 | import argparse 4 | import torch 5 | import numpy as np 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import torch.optim as optim 9 | import torchvision 10 | from torchvision import datasets, transforms 11 | import time 12 | from tensorboardX import SummaryWriter 13 | from datetime import datetime 14 | import pickle 15 | import matplotlib.pyplot as plt 16 | import os 17 | # from sklearn.metrics import f1_score, classification_report, confusion_matrix 18 | parser = argparse.ArgumentParser(description='Breakthis data_mynet') 19 | parser.add_argument("--zoom", help='zoom_level',default=40) 20 | parser.add_argument('--epochs',type=int, default=1, metavar='N', 21 | help='number of epochs to train (default: 10)') 22 | args = parser.parse_args() 23 | 24 | zoom_level_x =str(args.zoom) + 'X' 25 | writer = SummaryWriter(zoom_level_x+'/runs/'+"epoch:"+str(args.epochs)) 26 | 27 | transform = transforms.Compose([transforms.RandomCrop(128),transforms.ToTensor(),]) 28 | 29 | data_train = torchvision.datasets.ImageFolder("../AMIL_Data/test/{}".format(zoom_level_x), transform=transform) 30 | # print(data_train) 31 | data_test = torchvision.datasets.ImageFolder("../AMIL_Data/train/{}".format(zoom_level_x), transform=transform) 32 | # print(data_test) 33 | 34 | # os.environ("tee output.txt") 35 | class Net(nn.Module): 36 | def __init__(self): 37 | super(Net, self).__init__() 38 | self.conv1 = nn.Conv2d(3, 64, 3, 1, 1) 39 | self.conv2 = nn.Conv2d(64, 64, 3, 1, 1) 40 | self.conv3 = nn.Conv2d(64, 128, 3, 1, 1) 41 | self.conv4 = nn.Conv2d(128, 128, 3, 1, 1) 42 | self.conv5 = nn.Conv2d(128, 128, 3, 1, 1) 43 | self.dropout1=nn.Dropout(p=0.5, inplace=False) 44 | self.dropout2=nn.Dropout(p=0.5, inplace=False) 45 | self.fc1 = nn.Linear(4*4*128,1024) 46 | self.fc2 = nn.Linear(1024, 2) 47 | 48 | def forward(self, x): 49 | m=nn.LeakyReLU(0.01) 50 | x = m(self.conv1(x)) 51 | x = F.max_pool2d(x, 2, 2) 52 | x = m(self.conv2(x)) 53 | x = F.max_pool2d(x, 2, 2) 54 | 55 | x = m(self.conv3(x)) 56 | x = F.max_pool2d(x, 2, 2) 57 | x = m(self.conv4(x)) 58 | x = F.max_pool2d(x, 2, 2) 59 | x = m(self.conv5(x)) 60 | x = F.max_pool2d(x, 2, 2) 61 | 62 | # x = F.max_pool2d(x, 2, 2) 63 | x = self.dropout1(x) 64 | x = x.view(-1, 4*4*128) 65 | x = m(self.fc1(x)) 66 | # x = m(self.fc2(x)) 67 | x = self.dropout2(x) 68 | x = self.fc2(x) 69 | # print("x.shape",x.shape) 70 | return F.log_softmax(x, dim=1) 71 | 72 | 73 | def train(args, model, device, train_loader, optimizer, epoch): 74 | model.train() 75 | correct = 0 76 | # print("-----------",train_loader) 77 | for batch_idx, (data, target) in enumerate(train_loader): 78 | data, target = data.to(device), target.to(device) 79 | # print(len(data),len(target)) 80 | optimizer.zero_grad() 81 | output = model(data) 82 | # print(len(output), len(target)) 83 | loss = F.nll_loss(output, target) 84 | loss.backward() 85 | optimizer.step() 86 | if batch_idx % args.log_interval == 0: 87 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 88 | epoch, batch_idx * len(data), len(train_loader.dataset), 89 | 100. * batch_idx / len(train_loader), loss.item())) 90 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 91 | correct += pred.eq(target.view_as(pred)).sum().item() 92 | print('\nTrain_accuracy: {:.0f}%\n'.format(100. * correct / len(train_loader.dataset))) 93 | writer.add_scalar('train_Accuracy_epoch',100. * correct / len(train_loader.dataset),epoch) 94 | writer.add_scalar('train_loss_epoch',loss/len(train_loader.dataset),epoch) 95 | return (100. * correct / len(train_loader.dataset)) 96 | def test(args, model, device, test_loader,epoch): 97 | print("test started") 98 | model.eval() 99 | test_loss = 0 100 | correct = 0 101 | with torch.no_grad(): 102 | for data, target in test_loader: 103 | data, target = data.to(device), target.to(device) 104 | output = model(data) 105 | test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss 106 | pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability 107 | correct += pred.eq(target.view_as(pred)).sum().item() 108 | test_loss /= len(test_loader.dataset) 109 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 110 | test_loss, correct, len(test_loader.dataset), 111 | 100. * correct / len(test_loader.dataset))) 112 | writer.add_scalar('test_loss_epoch',test_loss,epoch) 113 | writer.add_scalar('test_Accuracy_epoch',100. * correct / len(test_loader.dataset),epoch) 114 | return (100. * correct / len(test_loader.dataset)) 115 | 116 | def main(): 117 | start = time.time() 118 | print ("into the main") 119 | parser = argparse.ArgumentParser(description='Breakthis data_mynet') 120 | 121 | 122 | parser.add_argument('--batch-size', type=int, default=16, metavar='N', 123 | help='input batch size for training (default: 64)') 124 | 125 | parser.add_argument('--test-batch-size', type=int, default=16, metavar='N', 126 | help='input batch size for testing (default: 1000)') 127 | 128 | parser.add_argument('--epochs', type=int, default=10, metavar='N', 129 | help='number of epochs to train (default: 10)') 130 | 131 | 132 | parser.add_argument('--lr', type=float, default=0.0003 , metavar='LR', 133 | help='learning rate (default: 0.01)') 134 | 135 | parser.add_argument('--momentum', type=float, default=0.4, metavar='M', 136 | help='SGD momentum (default: 0.9)') 137 | 138 | parser.add_argument('--no-cuda', action='store_true', default=False, 139 | help='disables CUDA training') 140 | 141 | parser.add_argument('--seed', type=int, default=1, metavar='S', 142 | help='random seed (default: 1)') 143 | 144 | parser.add_argument('--log_interval', type=int, default=20, metavar='N', 145 | help='how many batches to wait before logging training status') 146 | 147 | parser.add_argument('--save-model', action='store_true', default=True, 148 | help='For Saving the current Model') 149 | parser.add_argument("--zoom", help='zoom_level',default=40) 150 | 151 | args = parser.parse_args() 152 | 153 | use_cuda = not args.no_cuda and torch.cuda.is_available() 154 | device = torch.device("cuda" if use_cuda else "cpu") 155 | # device = "cpu" 156 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 157 | train_loader = torch.utils.data.DataLoader(data_train, batch_size=32 ,shuffle = True, **kwargs) 158 | test_loader = torch.utils.data.DataLoader(data_test, batch_size=32 ,shuffle = False, **kwargs) 159 | print("device: ",device) 160 | 161 | model = Net().to(device) 162 | optimizer = optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.99, eps=1e-08, weight_decay=0, momentum=args.momentum, centered=False) 163 | print ("optimizer choosed") 164 | print("#######__parameters__######") 165 | print("learning rate: ", args.lr, "\nmomentum: ", args.momentum, "\nepochs: ", args.epochs) 166 | print("############################") 167 | print("model:\n",model) 168 | print("############################") 169 | print("optimizer:\n",optimizer) 170 | print("############################") 171 | 172 | # for epoch in range(2): 173 | for epoch in range(1, int(args.epochs) + 1): 174 | # print("-----",train_loader) 175 | train_acc=train(args, model, device, train_loader, optimizer, epoch) 176 | test_acc=test(args, model, device, test_loader, epoch) 177 | # writer.add_scalar('loss_fn2',loss.item(),epoch) 178 | # sftp://test1@10.107.42.42/home/Drive2/amil/my_network/runs 179 | main_dir = "./" + zoom_level_x +'/' 180 | folders = ["pt_files","pickel_files","txt_file"] 181 | for i in folders: 182 | if not os.path.exists(main_dir + i ): 183 | os.makedirs(main_dir + i ) 184 | # model_dir = 185 | # state_dict_dir = 186 | save_string="Breakthis: train_acc:"+str(train_acc)+" test-acc:"+str(test_acc)+" epochs: "+str(args.epochs)+"zoom:"+zoom_level_x 187 | if (args.save_model): 188 | torch.save(model.state_dict(),main_dir+"pt_files/"+save_string+"Breakthis_state_dict.pt") 189 | torch.save(model ,main_dir+"pt_files/"+save_string+"Breakthis_model.pt") 190 | save_name_pkl = main_dir+"pickel_files/"+save_string+".pkl" 191 | save_name_txt = main_dir+"txt_file/"+save_string+".txt" 192 | model_file = open(save_name_txt,"w") 193 | model_string = str(model) 194 | optimizer_string = str(optimizer) 195 | model_file.write(model_string) 196 | model_file.write(optimizer_string) 197 | model_file.write(save_name_txt) 198 | model_file.close() 199 | 200 | f=open(save_name_pkl,"wb") 201 | pickle.dump(model, f) 202 | end = time.time() 203 | print('time taken is ', (end-start)) 204 | 205 | 206 | if __name__ == '__main__': 207 | main() 208 | -------------------------------------------------------------------------------- /my_network/run_for_all_zoom.sh: -------------------------------------------------------------------------------- 1 | #net.py 2 | mkdir terminal_logs 3 | declare -a ZOOM=(40 100 200 400) 4 | # declare -a epoch=(1 2 3 4 5 6 7 8 9 10) 5 | declare -a epoch=(10) 6 | 7 | for i in "${ZOOM[@]}" 8 | do 9 | for j in "${epoch[@]}" 10 | do 11 | echo "$i" 12 | echo "$j" 13 | python net.py --epochs=$j --zoom=$i |& tee terminal_logs/op_zoom_$i_epoch_$j.txt 14 | done 15 | done 16 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | Deep learning in histopathology has developed an interest over the decade due to its improvements in classification and localization tasks. Breast cancer is a prominent cause of death in women. 3 | 4 | 5 | Computer-Aided Pathology is essential to analyze microscopic histopathology images for diagnosis with an 6 | increasing number of breast cancer patients. The convolutional neural network, a deep learning algorithm, provides significant results in classification among cancer and non-cancer tissue images but lacks in providing interpretation. Here in this blog, I am writing the streamlined version of the paper **"Breast cancer histopathology image classification and localization using multiple instance learning"** published in **WIECON-2019** in which we have aimed to provide a better interpretation of classification results by providing localization on microscopic histopathology images. We frame the image classification problem as weakly supervised multiple instance learning problems and use attention on instances to localize the tumour and normal regions in an image. **Attention-based multiple instance learning (A-MIL)** is applied on **BreakHis** and **BACH** datasets. The classification and visualization results are compared with other recent techniques. A method used in this paper produces better localization results without compromising classification accuracy. 7 | 8 | # About Grad-CAM Image Visualizations 9 | 10 | In the era of deep learning, understanding of the model's decision is important and the GradCAM is one of the first and good methods to visualize the outcome. Here is the paper and the following are the results taken from paper directly. 11 | 12 | 15 | 16 | ![Highlighting the part in the input image responsible for classification in that category. Image is taken from the paper directly.](https://dipeshtamboli.github.io/images/amil/grad_cam.png) 17 | 18 | Here, in the image, you can see the highlighted portion corresponding to the parts of the image which is responsible for the classification. Like in the first image of Torch, GradCAM is highlighting the portion in the image where the torch is present. Similarly, in the Car Mirror image, it's highlighting the portion where the car mirror is present. Thus this explains the reason behind the decision taken by the model. 19 | 20 | ![This is the image where the model's output is the cat and GradCAM is highlighting the portion responsible for that decision. Image is taken from the paper directly.](https://dipeshtamboli.github.io/images/amil/dog.png) 21 | 22 | This is the image where the model's output is the cat and GradCAM is highlighting the portion responsible for that decision. Image is taken from the paper directly.Now, these are the results where we see that the GradCAM and Guided GradCAM gives us the portion which is important for decision making. But it doesn't work well on the medical images(especially Histopathology images). 23 | 24 | 25 | 26 | ![GradCAM and Guided-GradCAM is not highlighting the useful portion of the image.](https://dipeshtamboli.github.io/images/amil/grad_amil.png) 27 | 28 | So we have proposed another visualization technique for it. It's an attention-based visualization method where we are doing multiple instance learning. 29 | 30 | # Attention-based multiple instance learning (A-MIL) 31 | 32 | In this method, we have cropped an image in small square patches and made a bag of it. This bag of images will act like a batch. We are fee 33 | AMIL ArchitectureFirst making a bag of the input image by taking the small patches from it. 34 | Passing it to the feature extractor which is basically a convolutional neural network block. 35 | Then we are passing the Instance level features to the classifier for getting Instance level attention. 36 | Here we are getting the attention weights which we are further using for attention aggregation to get the bag level features. 37 | 38 | ![The figure shows localization in sample malignant image using Attention - Multiple Instance Learning. A-MIL accurately highlights affected gland and ignores background region](https://dipeshtamboli.github.io/images/amil/amil_arc.png) 39 | 40 | 41 | Then we are applying Dense layer for the classification of the Benign, Malignant or Invasiveconsidering the 42 | 43 | So in the end, we have cropped image patches and their attention weights. We multiplied each attention weight with the corresponding patch and stitch the whole image to get the visualization of the complete input image. With this method, we are neither compromising the accuracy nor made the model complicated. This is just adding transparency to the whole process. 44 | 45 | ![The figure shows localization in sample malignant image using Attention - Multiple Instance Learning. A-MIL accurately highlights affected gland and ignores background region](https://dipeshtamboli.github.io/images/amil/amil.png) 46 | 47 | 48 | This is the comparison between the visualization of GradCAM and AMIL method. Here we cropped two portions from the image which is important for the classification and applied GradCAM on it. In another scene, AMIL visualization is there which is properly highlighting the useful portion. 49 | Comparison of the visualization output of GradCAM and A-MILAnother result of AMIL visualization of BACH image. 50 | 51 | 52 | ![Another result from the BACH dataset.](https://dipeshtamboli.github.io/images/amil/result.png) 53 | 54 | *********************** 55 | 56 | [Same article is also available on Medium](https://medium.com/@dipeshtamboli/a-sota-method-for-visualization-of-histopathology-images-1cc6cc3b76f3) 57 | And also on [my blog](https://dipeshtamboli.github.io/blog/2019/Visualization-of-Histopathology-images/) 58 | 59 | | | 63 | 64 | *********************** 65 | 66 | 67 | # How to run the code 68 | 69 | Here, in the folder AMIL_project, we have following folders: 70 | 71 | ## my_network 72 | -In this, we have trained a model using this dataset on our own architecture. Accuracies are comparable to the VGG_pretrained and ResNet_pretrained model. 73 | -In this folder, we have 74 | -net.py 75 | -use "python net.py" to run the code(without quotes) 76 | -this is the code which will train the model, and test it on the validation set 77 | -this code will save following things in corresponding zoom level folder 78 | -model(in pickel and pytorch format) 79 | -terminal logs 80 | -tensorboard run logs 81 | -text file summarizing the run 82 | -run_for_all_zoom.sh 83 | - use "bash run_for_all_zoom.sh" to run this script(without quotes) 84 | - this script will run vgg_pre.py for all the zoom level and for all the epochs 85 | - u can keep the number of epoch = 10 86 | ## ResNet 87 | -In this, we have used a pre-trained ResNet model and trained last fully connected layer using this dataset. 88 | -In this folder, we have 89 | -resnet_pre.py 90 | -use "python resnet_pre.py" to run the code(without quotes) 91 | -this is the code which will train the model, and test it on the validation set 92 | -this code will save following things in corresponding zoom level folder 93 | -model(in pickel and pytorch format) 94 | -terminal logs 95 | -tensorboard run logs 96 | -text file summarizing the run 97 | -run_for_all_zoom.sh 98 | - use "bash run_for_all_zoom.sh" to run this script(without quotes) 99 | - this script will run vgg_pre.py for all the zoom level and for all the epochs 100 | - u can keep the number of epoch = 10 101 | ## VGG 102 | -In this, we have used a pre-trained VGG model and trained last fully connected layer using this dataset. 103 | -In this folder, we have 104 | -vgg_pre.py 105 | -use "python vgg_pre.py" to run the code(without quotes) 106 | -this is the code which will train the model, and test it on the validation set 107 | -this code will save following things in corresponding zoom level folder 108 | -model(in pickel and pytorch format) 109 | -terminal logs 110 | -tensorboard run logs 111 | -text file summarizing the run 112 | -run_for_all_zoom.sh 113 | - use "bash run_for_all_zoom.sh" to run this script(without quotes) 114 | - this script will run vgg_pre.py for all the zoom level and for all the epochs 115 | - u can keep the number of epoch = 10 116 | 117 | ## AMIL_codes 118 | -In this folder, we have 119 | -amil_model.py 120 | -it contains attention model(architecture) 121 | -patch_data.py 122 | -data loader (takes images as input and crop it to 28*28 and creates a bag) 123 | -train_n_test.py 124 | -use "python train_n_test.py" to run the code(without quotes) 125 | -code which trains the AMIL and then test it on the validation set and saves visualization in the AMIL_visualization folder 126 | -this code will save following things in corresponding zoom level folder 127 | -model(pytorch format) 128 | -terminal logs 129 | -tensorboard run logs 130 | -text file summarizing the run 131 | -visualization of test images 132 | -run_for_all_zoom.sh 133 | - use "bash run_for_all_zoom.sh" to run this script(without quotes) 134 | - this script will run vgg_pre.py for all the zoom level and for all the epochs 135 | - u can keep the number of epoch = 20 136 | 137 | ## grad_cam 138 | -In this folder, we have "inputs" folder where you have to put test image 139 | -In folder "src", in misc_functions.py, on line number 253, you have to put the name of test image 140 | -run the code "python guided_gradcam.py" without quotes. 141 | -this will produce resultant visualization images in "results" folder 142 | 143 | ## Kaggle_Data: 144 | we are using Breakhis dataset from the Kaggle datasets(link to the dataset) 145 | -download the dataset from the following link: 146 | [Kaggle Dataset](https://www.kaggle.com/kritika397/breast-cancer-dataset-from-breakhis/downloads/fold1.zip/1) 147 | -rename it to Kaggle_Data 148 | -We will use this data for Resnet architecture, vgg architecture and mynet architecture 149 | 150 | ## AMIL_Data: 151 | -Here, for attention based multiple instance learning, we will re-arrange the dataset in the given format(readme_data_format.txt) 152 | 153 | ### Here, dataset is in this structure: 154 | fold1 155 | -test 156 | -100X 157 | -B_100X 158 | -(images) 159 | -M_100X 160 | -(images) 161 | -200X 162 | -B_200X 163 | -(images) 164 | -M_200X 165 | -(images) 166 | -400X 167 | -B_400X 168 | -(images) 169 | -M_400X 170 | -(images) 171 | -40X 172 | -B_40X 173 | -(images) 174 | -M_40X 175 | -(images) 176 | -train 177 | -100X 178 | -B_100X 179 | -(images) 180 | -M_100X 181 | -(images) 182 | -200X 183 | -B_200X 184 | -(images) 185 | -M_200X 186 | -(images) 187 | -400X 188 | -B_400X 189 | -(images) 190 | -M_400X 191 | -(images) 192 | -40X 193 | -B_40X 194 | -(images) 195 | -M_40X 196 | -(images) 197 | 198 | ### Now, we have to convert it in the following format: 199 | data_breakhis 200 | -100X 201 | -train 202 | -0 203 | -images 204 | -1 205 | -images 206 | -test 207 | -0 208 | -images 209 | -1 210 | -images 211 | -200X 212 | -train 213 | -0 214 | -images 215 | -1 216 | -images 217 | -test 218 | -0 219 | -images 220 | -1 221 | -images 222 | -400X 223 | -train 224 | -0 225 | -images 226 | -1 227 | -images 228 | -test 229 | -0 230 | -images 231 | -1 232 | -images 233 | -40X 234 | -train 235 | -0 236 | -images 237 | -1 238 | -images 239 | -test 240 | -0 241 | -images 242 | -1 243 | -images 244 | -rearrange the folders and rename it to AMIL_Data 245 | 246 | 247 | ************************************************ 248 | To cite this: 249 | Plain Text: 250 | A. Patil, D. Tamboli, S. Meena, D. Anand and A. Sethi, "Breast Cancer Histopathology Image Classification and Localization using Multiple Instance Learning," 2019 IEEE International WIE Conference on Electrical and Computer Engineering (WIECON-ECE), Bangalore, India, 2019, pp. 1-4. 251 | 252 | BibTex: 253 | @INPROCEEDINGS{9019916, author={A. {Patil} and D. {Tamboli} and S. {Meena} and D. {Anand} and A. {Sethi}}, booktitle={2019 IEEE International WIE Conference on Electrical and Computer Engineering (WIECON-ECE)}, title={Breast Cancer Histopathology Image Classification and Localization using Multiple Instance Learning}, year={2019}, volume={}, number={}, pages={1-4},} 254 | -------------------------------------------------------------------------------- /requirement.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.0 2 | astor==0.7.1 3 | backports.functools-lru-cache==1.5 4 | backports.weakref==1.0.post1 5 | cycler==0.10.0 6 | enum34==1.1.6 7 | funcsigs==1.0.2 8 | futures==3.2.0 9 | gast==0.2.2 10 | grpcio==1.19.0 11 | h5py==2.9.0 12 | Keras-Applications==1.0.7 13 | Keras-Preprocessing==1.0.9 14 | kiwisolver==1.0.1 15 | Markdown==3.0.1 16 | matplotlib==2.2.3 17 | mock==2.0.0 18 | numpy==1.15.0 19 | opencv-python==4.0.0.21 20 | pbr==5.1.3 21 | Pillow==5.3.0 22 | pkg-resources==0.0.0 23 | protobuf==3.7.0 24 | pydensecrf==1.0rc3 25 | pyparsing==2.2.2 26 | python-dateutil==2.7.4 27 | pytz==2018.6 28 | scipy==1.2.2 29 | six==1.11.0 30 | subprocess32==3.5.3 31 | tensorboard==1.13.1 32 | tensorboardX==1.6 33 | tensorflow==1.13.1 34 | tensorflow-estimator==1.13.0 35 | termcolor==1.1.0 36 | torch==0.4.1 37 | torchvision==0.2.1 38 | Werkzeug==0.14.1 39 | --------------------------------------------------------------------------------