├── README.md ├── Unet.py ├── basic_blocks.py └── main.py /README.md: -------------------------------------------------------------------------------- 1 | # Unsupervised Change Detection in Satellite Images Using Convolutional Neural Networks 2 | Convolutional NN for change detection 3 | 4 | This project deals with the task of detecting relevant changes between two satellite images taken of the same scene at different times. A convolutional neural network (CNN) and semantic segmentation is implemented to detect the changes between the images, as well as classify the changes into the correct semantic class. A difference image is created using the feature maps generated by the CNN, which means that the CNN does not need to learn the non-linear mapping between two images and is thus unsupervised in the task of change detection. 5 | 6 | Below is a diagram of the Unet architecture used in this project: 7 | 8 | ![Unet](https://user-images.githubusercontent.com/48834483/54834087-1a4dfe80-4cc8-11e9-96bb-017fd63be742.png) 9 | 10 | The research paper is available on ArXiV: https://arxiv.org/abs/1812.05815 11 | The paper has been published in the proceedings of the IEEE International Joint Conference on Neural Networks, 2019 12 | -------------------------------------------------------------------------------- /Unet.py: -------------------------------------------------------------------------------- 1 | from basic_blocks import * 2 | 3 | class UnetGenerator(nn.Module): 4 | 5 | def __init__(self, in_dim, out_dim, num_filter): 6 | super(UnetGenerator,self).__init__() 7 | self.in_dim = in_dim 8 | self.out_dim = out_dim 9 | self.num_filter = num_filter 10 | act_fn = nn.LeakyReLU(0.2, inplace=True) 11 | 12 | print("\n------Initiating U-Net------\n") 13 | 14 | self.down_1 = conv_block_2(self.in_dim,self.num_filter,act_fn) 15 | self.pool_1 = maxpool() 16 | self.down_2 = conv_block_2(self.num_filter*1,self.num_filter*2,act_fn) 17 | self.pool_2 = maxpool() 18 | self.down_3 = conv_block_2(self.num_filter*2,self.num_filter*4,act_fn) 19 | self.pool_3 = maxpool() 20 | self.down_4 = conv_block_2(self.num_filter*4,self.num_filter*8,act_fn) 21 | self.pool_4 = maxpool() 22 | 23 | self.bridge = conv_block_2(self.num_filter*8,self.num_filter*16,act_fn) 24 | 25 | self.trans_1 = conv_trans_block(self.num_filter*16,self.num_filter*8,act_fn) 26 | self.up_1 = conv_block_2(self.num_filter*16,self.num_filter*8,act_fn) 27 | self.trans_2 = conv_trans_block(self.num_filter*8,self.num_filter*4,act_fn) 28 | self.up_2 = conv_block_2(self.num_filter*8,self.num_filter*4,act_fn) 29 | self.trans_3 = conv_trans_block(self.num_filter*4,self.num_filter*2,act_fn) 30 | self.up_3 = conv_block_2(self.num_filter*4,self.num_filter*2,act_fn) 31 | self.trans_4 = conv_trans_block(self.num_filter*2,self.num_filter*1,act_fn) 32 | self.up_4 = conv_block_2(self.num_filter*2,self.num_filter*1,act_fn) 33 | 34 | self.out = nn.Sequential( 35 | nn.Conv2d(self.num_filter,self.out_dim,3,1,1) 36 | ) 37 | 38 | def forward(self, input, mode="train", comparison = False): 39 | 40 | down_1 = self.down_1(input) 41 | pool_1 = self.pool_1(down_1) 42 | down_2 = self.down_2(pool_1) 43 | pool_2 = self.pool_2(down_2) 44 | down_3 = self.down_3(pool_2) 45 | pool_3 = self.pool_3(down_3) 46 | down_4 = self.down_4(pool_3) 47 | pool_4 = self.pool_4(down_4) 48 | 49 | bridge = self.bridge(pool_4) 50 | 51 | if mode == "test" and comparison == False: 52 | torch.save(down_1, 'tensor_store\\temp_down_1.pt') 53 | torch.save(down_2, 'tensor_store\\temp_down_2.pt') 54 | torch.save(down_3, 'tensor_store\\temp_down_3.pt') 55 | torch.save(down_4, 'tensor_store\\temp_down_4.pt') 56 | torch.save(bridge, 'tensor_store\\temp_bridge.pt') 57 | 58 | if mode == "test" and comparison == True: 59 | prev_down_1 = torch.load('tensor_store\\temp_down_1.pt') 60 | prev_down_2 = torch.load('tensor_store\\temp_down_2.pt') 61 | prev_down_3 = torch.load('tensor_store\\temp_down_3.pt') 62 | prev_down_4 = torch.load('tensor_store\\temp_down_4.pt') 63 | prev_bridge = torch.load('tensor_store\\temp_bridge.pt') 64 | 65 | down_1 = difference(down_1, prev_down_1,threshhold=0.6) 66 | down_2 = difference(down_2, prev_down_2,threshhold=0.6) 67 | down_3 = difference(down_3, prev_down_3,threshhold=0.8) 68 | down_4 = difference(down_4, prev_down_4,threshhold=1.0) 69 | bridge = difference(bridge, prev_bridge,threshhold=1.2) 70 | 71 | #print(bridge.type()) 72 | 73 | trans_1 = self.trans_1(bridge) 74 | concat_1 = torch.cat([trans_1,down_4],dim=1) 75 | up_1 = self.up_1(concat_1) 76 | trans_2 = self.trans_2(up_1) 77 | concat_2 = torch.cat([trans_2,down_3],dim=1) 78 | up_2 = self.up_2(concat_2) 79 | trans_3 = self.trans_3(up_2) 80 | concat_3 = torch.cat([trans_3,down_2],dim=1) 81 | up_3 = self.up_3(concat_3) 82 | trans_4 = self.trans_4(up_3) 83 | concat_4 = torch.cat([trans_4,down_1],dim=1) 84 | up_4 = self.up_4(concat_4) 85 | 86 | out = self.out(up_4) 87 | 88 | return out 89 | 90 | 91 | -------------------------------------------------------------------------------- /basic_blocks.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.utils as utils 4 | import torch.nn.init as init 5 | import torch.utils.data as data 6 | import torchvision.utils as v_utils 7 | import torchvision.datasets as dset 8 | import torchvision.transforms as transforms 9 | from torch.autograd import Variable 10 | import numpy as np 11 | from random import shuffle 12 | import scipy.misc 13 | from PIL import Image 14 | 15 | import cv2 16 | import os 17 | import glob 18 | 19 | 20 | def conv_block(in_dim,out_dim,act_fn): 21 | model = nn.Sequential( 22 | nn.Conv2d(in_dim,out_dim, kernel_size=3, stride=1, padding=1), 23 | nn.BatchNorm2d(out_dim), 24 | act_fn, 25 | ) 26 | return model 27 | 28 | 29 | def conv_trans_block(in_dim,out_dim,act_fn): 30 | model = nn.Sequential( 31 | nn.ConvTranspose2d(in_dim,out_dim, kernel_size=3, stride=2, padding=1,output_padding=1), 32 | nn.BatchNorm2d(out_dim), 33 | act_fn, 34 | ) 35 | return model 36 | 37 | 38 | def maxpool(): 39 | pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0) 40 | return pool 41 | 42 | 43 | def conv_block_2(in_dim,out_dim,act_fn): 44 | model = nn.Sequential( 45 | conv_block(in_dim,out_dim,act_fn), 46 | nn.Conv2d(out_dim,out_dim, kernel_size=3, stride=1, padding=1), 47 | nn.BatchNorm2d(out_dim), 48 | ) 49 | return model 50 | 51 | 52 | def conv_block_3(in_dim,out_dim,act_fn): 53 | model = nn.Sequential( 54 | conv_block(in_dim,out_dim,act_fn), 55 | conv_block(out_dim,out_dim,act_fn), 56 | nn.Conv2d(out_dim,out_dim, kernel_size=3, stride=1, padding=1), 57 | nn.BatchNorm2d(out_dim), 58 | ) 59 | return model 60 | 61 | def difference(tensor1, tensor2, threshhold=1): 62 | 63 | bat = tensor1.shape[0] 64 | c = tensor1.shape[1] 65 | l = tensor1.shape[2] 66 | w = tensor1.shape[3] 67 | new_tensor = np.zeros((bat, c, l, w), dtype=np.float) 68 | temp_tensor1 = Variable(tensor1, requires_grad=False) 69 | temp_tensor2 = Variable(tensor2, requires_grad=False) 70 | temp_tensor1 = temp_tensor1.cpu().numpy() 71 | temp_tensor2 = temp_tensor2.cpu().numpy() 72 | for b in range(bat): 73 | for x in range(l): 74 | for y in range(w): 75 | for z in range(c): 76 | if abs(temp_tensor1.item((b, z, x, y)) - temp_tensor2.item((b, z, x, y))) <= threshhold: 77 | new_tensor[b,z,x,y] = 0 78 | else: 79 | new_tensor[b,z,x,y] = temp_tensor1.item((b, z, x, y)) 80 | 81 | new_tensor = torch.from_numpy(new_tensor) 82 | new_tensor = new_tensor.float().cuda(0) 83 | new_tensor = Variable(new_tensor, requires_grad=True) 84 | return new_tensor 85 | 86 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # Semantic Segmentation 2 | # Code by GunhoChoi 3 | 4 | from testUnet2 import * 5 | 6 | 7 | 8 | import matplotlib.pyplot as plt 9 | 10 | def noisy(noise_typ,image): 11 | if noise_typ == "gauss": 12 | row,col,ch= image.shape 13 | mean = 0 14 | var = 15 15 | sigma = var**0.5 16 | print(sigma) 17 | gauss = np.random.normal(mean,sigma,(row,col,ch)) 18 | gauss = gauss.reshape(row,col,ch) 19 | noisy = image + gauss + gauss + gauss 20 | return noisy 21 | elif noise_typ == "s&p": 22 | row,col,ch = image.shape 23 | s_vs_p = 0.5 24 | amount = 0.004 25 | out = np.copy(image) 26 | # Salt mode 27 | num_salt = np.ceil(amount * image.size * s_vs_p) 28 | coords = [np.random.randint(0, i - 1, int(num_salt)) 29 | for i in image.shape] 30 | out[coords] = 1 31 | 32 | # Pepper mode 33 | num_pepper = np.ceil(amount* image.size * (1. - s_vs_p)) 34 | coords = [np.random.randint(0, i - 1, int(num_pepper)) 35 | for i in image.shape] 36 | out[coords] = 0 37 | return out 38 | elif noise_typ == "poisson": 39 | vals = len(np.unique(image)) 40 | vals = 2 ** np.ceil(np.log2(vals)) 41 | noisy = np.random.poisson(image * vals) / float(vals) 42 | return noisy 43 | elif noise_typ =="speckle": 44 | row,col,ch = image.shape 45 | gauss = np.random.randn(row,col,ch) 46 | gauss = gauss.reshape(row,col,ch) 47 | noisy = image + image * gauss 48 | return noisy 49 | 50 | 51 | def getTrainingData(): 52 | 53 | transform = transforms.Compose([transforms.ToTensor(),]) 54 | 55 | img_dir = "cutpic\input" 56 | gt_dir = "cutpic\gtruth" 57 | data_path1 = os.path.join(img_dir,'*.png') 58 | data_path2 = os.path.join(gt_dir,'*.png') 59 | files1 = sorted(glob.glob(data_path1), key=os.path.getmtime) 60 | files2 = sorted(glob.glob(data_path2), key=os.path.getmtime) 61 | data_input = [] 62 | data_label = [] 63 | 64 | for f1 in files1: 65 | img = cv2.imread(f1) 66 | img = transform(img) 67 | img = img.unsqueeze(0) 68 | data_input.append(img) 69 | 70 | for f2 in files2: 71 | img = cv2.imread(f2) 72 | img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 73 | img = torch.from_numpy(img.astype(int)) 74 | img = img.long() 75 | img = img.unsqueeze(0) 76 | data_label.append(img) 77 | 78 | tuple_list = [] 79 | for x in range(len(data_input)): 80 | pair = (data_input[x], data_label[x]) 81 | tuple_list.append(pair) 82 | 83 | return tuple_list 84 | 85 | def getTestData(img_size=320): 86 | transform = transforms.Compose([transforms.ToTensor(),]) 87 | 88 | img_dir = "cutpic\\test" 89 | 90 | data_path1 = os.path.join(img_dir,'*.png') 91 | files1 = sorted(glob.glob(data_path1), key=os.path.getmtime) 92 | data_input = [] 93 | 94 | for f1 in files1: 95 | img = cv2.imread(f1) 96 | 97 | img = img[0:img_size,0:img_size,:] 98 | print(img.shape) 99 | cv2.imwrite("result\original.png", img) 100 | #scipy.misc.imsave('test\pic\gradient.png', arr_gtruth) 101 | img = noisy("gauss", img) 102 | cv2.imwrite("result\\noisy.png", img) 103 | img = transform(img) 104 | img = img.float() 105 | img = img.unsqueeze(0) 106 | data_input.append(img) 107 | 108 | return data_input 109 | 110 | def getBatchData(tuple_list, batch_size = 1): 111 | batch_tuple_list = [] 112 | shuffle(tuple_list) 113 | x = 0 114 | while x < len(tuple_list): 115 | arg1 = tuple_list[x][0] 116 | arg2 = tuple_list[x][1] 117 | x += 1 118 | for i in range(batch_size-1): 119 | if x >= len(tuple_list): 120 | break 121 | 122 | arg3 = tuple_list[x][0] 123 | arg1 = torch.cat((arg1, arg3),0) 124 | 125 | arg4 = tuple_list[x][1] 126 | arg2 = torch.cat((arg2, arg4),0) 127 | 128 | x +=1 129 | 130 | pair = (arg1, arg2) 131 | batch_tuple_list.append(pair) 132 | 133 | return batch_tuple_list 134 | 135 | def displayGenerated(gen, i, k, mode="standard"): 136 | bat = gen.shape[0] 137 | l = gen.shape[2] 138 | w = gen.shape[3] 139 | gen = gen.cpu().numpy() 140 | 141 | for z in range(bat): 142 | dis = np.zeros((l, w, 3), dtype=np.uint8) 143 | for x in range(l): 144 | for y in range(w): 145 | if gen.item((z, 0, x, y)) == 0: 146 | dis[x,y,0] = 255 147 | if mode == "difference": 148 | dis[x,y,1] = 255 149 | dis[x,y,2] = 255 150 | if gen.item((z, 0, x, y)) == 1: 151 | dis[x,y,1] = 255 152 | if gen.item((z, 0, x, y)) == 2: 153 | dis[x,y,2] = 255 154 | scipy.misc.imsave('result\\argmax\\argmax_{}_{}_{}.png'.format(i,k,z), dis) 155 | 156 | def displayTruth(truth, i, k): 157 | bat = truth.shape[0] 158 | l = truth.shape[1] 159 | w = truth.shape[2] 160 | 161 | truth = truth.cpu().numpy() 162 | for z in range(bat): 163 | dis = np.zeros((l, w), dtype=np.uint8) 164 | for x in range(l): 165 | for y in range(w): 166 | if truth.item((z,x,y)) == 0: 167 | dis[x,y] = 0 168 | if truth.item((z,x,y)) == 1: 169 | dis[x,y] = 100 170 | if truth.item((z,x,y)) == 2: 171 | dis[x,y] = 200 172 | scipy.misc.imsave('result\\truth_{}_{}_{}.png'.format(i,k,z), dis) 173 | 174 | 175 | if __name__ == '__main__': 176 | 177 | # hyperparameters 178 | mode = "test" 179 | batch_size = 2 180 | img_size = 320 181 | lr = 0.0002 182 | epoch = 1 183 | 184 | # initiate Generator 185 | 186 | generator = nn.DataParallel(UnetGenerator(3,3,64),device_ids=[i for i in range(1)]).cuda() 187 | 188 | # load pretrained model 189 | 190 | try: 191 | generator = torch.load('./model/{}.pkl'.format("unet")) 192 | print("\n--------model restored--------\n") 193 | except: 194 | print("\n--------model not restored--------\n") 195 | pass 196 | 197 | # loss function & optimizer 198 | 199 | recon_loss_func = nn.CrossEntropyLoss() 200 | gen_optimizer = torch.optim.Adam(generator.parameters(),lr=lr) 201 | 202 | # Training 203 | 204 | if mode == "train": 205 | 206 | train_data = getTrainingData() 207 | 208 | file = open('./{}_CE_loss.txt'.format("unet"), 'w') 209 | for i in range(epoch): 210 | 211 | total_loss = 0 212 | batchData = getBatchData(train_data, batch_size=batch_size) 213 | for k in range(len(batchData)): 214 | image = batchData[k][0] 215 | gtruth = batchData[k][1] 216 | 217 | gen_optimizer.zero_grad() 218 | 219 | x = Variable(image).cuda(0) 220 | y_ = Variable(gtruth).cuda(0) 221 | y = generator.forward(x) 222 | 223 | loss = recon_loss_func(y,y_) 224 | total_loss += loss 225 | loss.backward() 226 | gen_optimizer.step() 227 | 228 | 229 | 230 | if k % 100 == 0: 231 | print(i) 232 | print(loss) 233 | y_argmax = torch.argmax(y, dim=1) 234 | y_argmax = y_argmax.unsqueeze(1) 235 | displayGenerated(y_argmax, i, k) 236 | 237 | 238 | v_utils.save_image(x.cpu().data,"./result/original_image_{}_{}.png".format(i,k)) 239 | displayTruth(y_, i, k) 240 | v_utils.save_image(y.cpu().data,"./result/gen_image_{}_{}.png".format(i,k)) 241 | #torch.save(generator,'./model/{}.pkl'.format("unet")) 242 | 243 | ave_loss = total_loss / len(batchData) 244 | file.write(str(ave_loss)+"\n") 245 | print("average loss for batch") 246 | print(ave_loss) 247 | 248 | 249 | #Testing 250 | 251 | if mode == "test": 252 | 253 | test_data = getTestData() 254 | 255 | comparison = False 256 | i = 0 257 | while i < 3: 258 | 259 | image = test_data[i] 260 | 261 | #print(torch.sum(image).item()) 262 | x = Variable(image).cuda(0) 263 | y = generator.forward(x, mode=mode, comparison=comparison) 264 | 265 | if comparison == False: 266 | comparison = True 267 | v_utils.save_image(y.cpu().data,"./result/gen_image_{}.png".format(i)) 268 | else: 269 | v_utils.save_image(y.cpu().data,"./result/gen_image_{}.png".format(i)) 270 | y_argmax = torch.argmax(y, dim=1) 271 | y_argmax = y_argmax.unsqueeze(1) 272 | displayGenerated(y_argmax,i,0, mode="difference") 273 | comparison = False 274 | i +=1 275 | 276 | --------------------------------------------------------------------------------