├── README.md ├── final.py ├── flow.png ├── gabor.py ├── gabor_filter.py ├── gabor_scratch.py ├── main.py ├── output.png ├── segment.py ├── shadow.jpg ├── shadow_detect.py └── shadow_remove.py /README.md: -------------------------------------------------------------------------------- 1 | Digital Image processing - NITT course project 2 | 3 | ## Shadow detection and removal 4 | 5 | Reimplementation of the paper [Efficient Shadow Removal Using Subregion Matching Illumination Transfer](https://onlinelibrary.wiley.com/doi/full/10.1111/cgf.12250?casa_token=XaNTua352PwAAAAA%3AbThsqn8IUYmAwvGgR0-iVLmKTn8SI0YaYy1APLNI1hbzCpHLyakUAGy1ICcWy4YgvJCD2vvlaMeEH_Zt "Efficient Shadow Removal Using Subregion Matching Illumination Transfer") with a slight twist 6 | 7 | Implemented an unsupervised segmentation algorithm employing autoencoders for detection of shadow regions. 8 | Gabor filter is designed to identify the texture features in the images. Illuminance transfer techniques are deployed to remove the shadow regions with the help of the acquired textures. 9 | Implemented the algorithm on SBU Shadow Datasets obtaining good results. More work is needed on the boundary processing of the shadows to improve the accuracy. 10 | 11 | ![gesture detection](flow.png) 12 | 13 | The flow of the shadow detection and removal algorithm 14 | 15 | ## Reference: 16 | 1. [Efficient Shadow Removal Using Subregion Matching Illumination Transfer](https://onlinelibrary.wiley.com/doi/full/10.1111/cgf.12250?casa_token=XaNTua352PwAAAAA%3AbThsqn8IUYmAwvGgR0-iVLmKTn8SI0YaYy1APLNI1hbzCpHLyakUAGy1ICcWy4YgvJCD2vvlaMeEH_Zt "Efficient Shadow Removal Using Subregion Matching Illumination Transfer") 17 | 2. [Unsupervised Segmentation](https://github.com/kanezaki/pytorch-unsupervised-segmentation "Unsupervised Segmentation") 18 | -------------------------------------------------------------------------------- /final.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | 3 | # SHADOW REMOVAL WITH SUBREGION MATCHING AND ILLUMINATION TRANSFER 4 | 5 | # DONE BY: 6 | # BHARATH KUMAR 7 | # CHOCKALINGAM 8 | # DC VIVEK 9 | # HARINATH GOBI 10 | 11 | ############################################################################## 12 | 13 | import cv2 14 | import numpy as np 15 | import math 16 | 17 | import torch 18 | import torch.nn as nn 19 | import torch.nn.functional as F 20 | import torch.optim as optim 21 | from torchvision import datasets, transforms 22 | from torch.autograd import Variable 23 | import sys 24 | from skimage import segmentation 25 | import torch.nn.init 26 | import scipy 27 | 28 | use_cuda = torch.cuda.is_available() 29 | 30 | ############################# SEGMENTATION VARIABLES ######################### 31 | 32 | nChannel = 100 33 | maxIter = 100 34 | minLabels = 3 35 | lr = 0.1 36 | nConv = 2 37 | num_superpixels = 10000 38 | compactness = 100 39 | visualize = 1 40 | input_file = 'shadow.jpg' 41 | 42 | ############################################################################## 43 | 44 | img = cv2.imread(input_file) 45 | shadow_removed_img = cv2.imread(input_file) 46 | gray = cv2.imread(input_file, 0) 47 | blur = cv2.bilateralFilter(img,9,75,75) 48 | 49 | ############################# HSI CONVERSION ########################### 50 | 51 | blur = np.divide(blur, 255.0) 52 | 53 | hsi = np.zeros((blur.shape[0],blur.shape[1],blur.shape[2]),dtype=np.float) 54 | ratio_map = np.zeros((blur.shape[0],blur.shape[1]),dtype=np.uint8) 55 | 56 | for i in range(blur.shape[0]): 57 | for j in range(blur.shape[1]): 58 | hsi[i][j][2] = (blur[i][j][0]+blur[i][j][1]+blur[i][j][2])/3 59 | hsi[i][j][0] = math.acos(((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][0]))/(2*math.sqrt((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][1])+(blur[i][j][2]-blur[i][j][0])*(blur[i][j][1]-blur[i][j][0])))) 60 | hsi[i][j][1] = 1 - 3*min(blur[i][j][0],blur[i][j][1],blur[i][j][2])/hsi[i][j][2] 61 | ratio_map[i][j] = hsi[i][j][0]/(hsi[i][j][2]+0.01) 62 | 63 | ############################################################################### 64 | 65 | ######################### SHADOW DETECTION ############################### 66 | 67 | hist = np.histogram(ratio_map.ravel(),256,[0,256]) 68 | ret,th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) 69 | ret,inv_th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) 70 | bin_thresh = cv2.medianBlur(th,15) 71 | bin_inv_thresh = cv2.medianBlur(inv_th,15) 72 | 73 | ############################################################################### 74 | 75 | shadow_region = cv2.bitwise_and(img,img,mask = th) 76 | background_region = cv2.bitwise_and(img,img,mask = inv_th) 77 | 78 | ############################## SEGMENTATION ################################# 79 | 80 | # CNN model 81 | class MyNet(nn.Module): 82 | def __init__(self,input_dim): 83 | super(MyNet, self).__init__() 84 | self.conv1 = nn.Conv2d(input_dim, nChannel, kernel_size=3, stride=1, padding=1 ) 85 | self.bn1 = nn.BatchNorm2d(nChannel) 86 | self.conv2 = [] 87 | self.bn2 = [] 88 | for i in range(nConv-1): 89 | self.conv2.append( nn.Conv2d(nChannel, nChannel, kernel_size=3, stride=1, padding=1 ) ) 90 | self.bn2.append( nn.BatchNorm2d(nChannel) ) 91 | self.conv3 = nn.Conv2d(nChannel, nChannel, kernel_size=1, stride=1, padding=0 ) 92 | self.bn3 = nn.BatchNorm2d(nChannel) 93 | 94 | def forward(self, x): 95 | x = self.conv1(x) 96 | x = F.relu( x ) 97 | x = self.bn1(x) 98 | for i in range(nConv-1): 99 | x = self.conv2[i](x) 100 | x = F.relu( x ) 101 | x = self.bn2[i](x) 102 | x = self.conv3(x) 103 | x = self.bn3(x) 104 | return x 105 | 106 | def segment(im): 107 | data = torch.from_numpy( np.array([im.transpose( (2, 0, 1) ).astype('float32')/255.]) ) 108 | if use_cuda: 109 | data = data.cuda() 110 | data = Variable(data) 111 | 112 | # slic 113 | labels = segmentation.slic(im, compactness=compactness, n_segments=num_superpixels) 114 | labels = labels.reshape(im.shape[0]*im.shape[1]) 115 | u_labels = np.unique(labels) 116 | l_inds = [] 117 | for i in range(len(u_labels)): 118 | l_inds.append( np.where( labels == u_labels[ i ] )[ 0 ] ) 119 | 120 | # train 121 | model = MyNet( data.size(1) ) 122 | if use_cuda: 123 | model.cuda() 124 | for i in range(nConv-1): 125 | model.conv2[i].cuda() 126 | model.bn2[i].cuda() 127 | model.train() 128 | loss_fn = torch.nn.CrossEntropyLoss() 129 | optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) 130 | label_colours = np.random.randint(255,size=(100,3)) 131 | for batch_idx in range(maxIter): 132 | # forwarding 133 | optimizer.zero_grad() 134 | output = model( data )[ 0 ] 135 | output = output.permute( 1, 2, 0 ).contiguous().view( -1, nChannel ) 136 | ignore, target = torch.max( output, 1 ) 137 | im_target = target.data.cpu().numpy() 138 | nLabels = len(np.unique(im_target)) 139 | if visualize: 140 | im_target_rgb = np.array([label_colours[ c % 100 ] for c in im_target]) 141 | im_target_rgb = im_target_rgb.reshape( im.shape ).astype( np.uint8 ) 142 | cv2.imshow( "output", im_target_rgb ) 143 | cv2.waitKey(1) 144 | 145 | # superpixel refinement 146 | for i in range(len(l_inds)): 147 | labels_per_sp = im_target[ l_inds[ i ] ] 148 | u_labels_per_sp = np.unique( labels_per_sp ) 149 | hist = np.zeros( len(u_labels_per_sp) ) 150 | for j in range(len(hist)): 151 | hist[ j ] = len( np.where( labels_per_sp == u_labels_per_sp[ j ] )[ 0 ] ) 152 | im_target[ l_inds[ i ] ] = u_labels_per_sp[ np.argmax( hist ) ] 153 | target = torch.from_numpy( im_target ) 154 | if use_cuda: 155 | target = target.cuda() 156 | target = Variable( target ) 157 | loss = loss_fn(output, target) 158 | loss.backward() 159 | optimizer.step() 160 | 161 | print (batch_idx, '/', maxIter, ':', nLabels, loss.data[0]) 162 | if nLabels <= minLabels: 163 | print ("nLabels", nLabels, "reached minLabels", minLabels, ".") 164 | break 165 | return im_target_rgb 166 | 167 | shadow_region = segment(shadow_region) 168 | shadow_region = cv2.bitwise_and(shadow_region,shadow_region,mask = th) 169 | background_region = segment(background_region) 170 | background_region = cv2.bitwise_and(background_region,background_region,mask = inv_th) 171 | 172 | ############################################################################### 173 | 174 | ############################ SUBREGION MATCHING ############################# 175 | 176 | shadow_region_colors = list(np.unique(shadow_region.reshape(-1, shadow_region.shape[2]), axis=0)) 177 | background_region_colors = list(np.unique(background_region.reshape(-1, background_region.shape[2]), axis=0)) 178 | 179 | shadow_subregions = [] 180 | background_subregions = [] 181 | 182 | for i in range(len(shadow_region_colors)): 183 | shadow_subregions.append([]) 184 | for x in range(shadow_region.shape[0]): 185 | for y in range(shadow_region.shape[1]): 186 | if (shadow_region[x][y] == shadow_region_colors[i]).all(): 187 | shadow_subregions[i].append([x,y]) 188 | 189 | for i in range(len(background_region_colors)): 190 | background_subregions.append([]) 191 | for x in range(background_region.shape[0]): 192 | for y in range(background_region.shape[1]): 193 | if (background_region[x][y] == background_region_colors[i]).all(): 194 | background_subregions[i].append([x,y]) 195 | 196 | shadow_region_colors.pop(0) 197 | background_region_colors.pop(0) 198 | shadow_region_colors = np.array(shadow_region_colors) 199 | background_region_colors = np.array(background_region_colors) 200 | 201 | shadow_subregions.pop(0) 202 | background_subregions.pop(0) 203 | 204 | #print(background_region_colors) 205 | #print(shadow_region_colors) 206 | #print(len(shadow_subregions)) 207 | #print(len(shadow_subregions[0]),len(shadow_subregions[1]),len(shadow_subregions[2])) 208 | 209 | feature_vector = [] 210 | kernel_vector = [] 211 | scales = 4 212 | orientation = 6 213 | 214 | for s in range(scales): 215 | for o in range(orientation): 216 | g_kernel = cv2.getGaborKernel((21, 21), (4.0+s*2), (np.pi*o)/6, 10.0, 0.5, 0, ktype=cv2.CV_32F) 217 | filtered_img = cv2.filter2D(gray, cv2.CV_8UC3, g_kernel) 218 | kernel_vector.append(g_kernel) 219 | feature_vector.append(filtered_img) 220 | 221 | shadow_subregion_feature_vector = [] 222 | background_subregion_feature_vector = [] 223 | 224 | shadow_region_coord_mean = [] 225 | background_region_coord_mean = [] 226 | 227 | feature_vector_mean = [np.mean(vector) for vector in feature_vector] 228 | feature_vector_std = [np.std(vector) for vector in feature_vector] 229 | 230 | iterator = 0 231 | 232 | for vector in feature_vector: 233 | shadow_region_coord_mean.append([]) 234 | shadow_subregion_feature_vector.append([]) 235 | background_region_coord_mean.append([]) 236 | background_subregion_feature_vector.append([]) 237 | 238 | for region in shadow_subregions: 239 | mean = 0 240 | std = 0 241 | x_mean = 0 242 | y_mean = 0 243 | for x,y in region: 244 | mean = mean + vector[x][y] 245 | x_mean = x_mean + x 246 | y_mean = y_mean + y 247 | mean = mean/(vector.shape[0]*vector.shape[1]) 248 | for x,y in region: 249 | std = std + abs((vector[x][y]-mean)*(vector[x][y]-mean)) 250 | shadow_subregion_feature_vector[iterator].append([mean,math.sqrt(std/(vector.shape[0]*vector.shape[1]))]) 251 | shadow_region_coord_mean[iterator].append([x_mean/(len(region)),y_mean/(len(region))]) 252 | 253 | for region in background_subregions: 254 | mean = 0 255 | std = 0 256 | x_mean = 0 257 | y_mean = 0 258 | for x,y in region: 259 | mean = mean + vector[x][y] 260 | x_mean = x_mean + x 261 | y_mean = y_mean + y 262 | mean = mean/(vector.shape[0]*vector.shape[1]) 263 | for x,y in region: 264 | std = std + abs((vector[x][y]-mean)*(vector[x][y]-mean)) 265 | background_subregion_feature_vector[iterator].append([mean,math.sqrt(std/(vector.shape[0]*vector.shape[1]))]) 266 | background_region_coord_mean[iterator].append([x_mean/(len(region)),y_mean/(len(region))]) 267 | iterator = iterator + 1 268 | 269 | #print(len(background_subregions),len(shadow_subregions),len(shadow_subregion_feature_vector),len(shadow_region_coord_mean)) 270 | #print(len(background_region_coord_mean),len(background_subregion_feature_vector),len(background_region_coord_mean[0]),len(background_subregion_feature_vector[0]),len(background_subregion_feature_vector[0][0]),len(background_region_coord_mean[0][0])) 271 | #print(shadow_region_coord_mean[0],shadow_region_coord_mean[1]) 272 | #print(shadow_subregion_feature_vector[0],shadow_subregion_feature_vector[1],shadow_subregion_feature_vector[2]) 273 | 274 | dist_texture = [] 275 | dist_space = [] 276 | index = 0 277 | dist = 0 278 | text = 0 279 | iterator = 0 280 | _iterator = 0 281 | 282 | #for s in range(scales): 283 | # for o in range(orientation): 284 | # dist_texture.append([]) 285 | # index = (orientation*s)+o 286 | # for shadow_reg in range(len(shadow_subregion_feature_vector[index])): 287 | # dist_texture[index].append([]) 288 | # for background_reg in range(len(background_subregion_feature_vector[index])): 289 | # text = text + abs((shadow_subregion_feature_vector[index][shadow_reg][0]-background_subregion_feature_vector[index][shadow_reg][0])/feature_vector_mean[index][0]) + abs((shadow_subregion_feature_vector[index][shadow_reg][1]-background_subregion_feature_vector[index][shadow_reg][1])/feature_vector_mean[index][1]) 290 | # dist_texture[index][iterator].append(text) 291 | # iterator = iterator + 1 292 | 293 | #print(feature_vector_mean[0],shadow_subregion_feature_vector[0],background_subregion_feature_vector[0]) 294 | 295 | for shadow_reg in range(len(shadow_subregion_feature_vector[index])): 296 | dist_texture.append({}) 297 | _iterator = 0 298 | for background_reg in range(len(background_subregion_feature_vector[index])): 299 | text = 0 300 | for s in range(scales): 301 | for o in range(orientation): 302 | index = (orientation*s)+o 303 | text = text + abs((shadow_subregion_feature_vector[index][shadow_reg][0]-background_subregion_feature_vector[index][shadow_reg][0])/feature_vector_mean[index]) + abs((shadow_subregion_feature_vector[index][shadow_reg][1]-background_subregion_feature_vector[index][shadow_reg][1])/feature_vector_mean[index]) 304 | dist_texture[iterator][_iterator] = text 305 | _iterator = _iterator + 1 306 | iterator = iterator + 1 307 | 308 | iterator = 0 309 | _iterator = 0 310 | 311 | for shadow_reg in range(len(shadow_subregion_feature_vector[index])): 312 | dist_space.append({}) 313 | _iterator = 0 314 | for background_reg in range(len(background_subregion_feature_vector[index])): 315 | dist = math.sqrt((shadow_region_coord_mean[0][shadow_reg][0]-background_region_coord_mean[0][background_reg][0])*(shadow_region_coord_mean[0][shadow_reg][0]-background_region_coord_mean[0][background_reg][0])+(shadow_region_coord_mean[0][shadow_reg][1]-background_region_coord_mean[0][background_reg][1])*(shadow_region_coord_mean[0][shadow_reg][1]-background_region_coord_mean[0][background_reg][1])) 316 | dist_space[iterator][_iterator] = dist 317 | _iterator = _iterator + 1 318 | iterator = iterator + 1 319 | 320 | #print(dist_texture,dist_space) 321 | 322 | for i in range(len(dist_texture)): 323 | dist_texture[i] = sorted(dist_texture[i].items(), key=lambda kv: kv[1]) 324 | dist_space[i] = sorted(dist_space[i].items(), key=lambda kv: kv[1]) 325 | 326 | match = [] 327 | 328 | for i in range(len(dist_texture)): 329 | min = len(dist_texture[0])+len(dist_space[0]) 330 | minimum = 0 331 | for j in range(len(dist_texture[0])): 332 | for k in range(len(dist_space[0])): 333 | if dist_texture[i][j][0] == dist_space[i][k][0]: 334 | if j+k < min: 335 | min = j+k 336 | minimum = dist_texture[i][j][0] 337 | match.append(minimum) 338 | 339 | #print(match) 340 | ############################################################################### 341 | 342 | ############################ SHADOW REMOVAL ############################# 343 | 344 | luminance = np.zeros((img.shape[0],img.shape[1])) 345 | #shadow_removed_image = np.zeros((img.shape[0],img.shape[1],img.shape[2])) 346 | #sigma_shadow = [1]*len(shadow_subregion_feature_vector[0]) 347 | #sigma_background = [1]*len(background_subregion_feature_vector[0]) 348 | 349 | for x in range(img.shape[0]): 350 | for y in range(img.shape[1]): 351 | luminance[x][y] = img[x][y][2]*0.2126 + img[x][y][1]*0.7152 + img[x][y][0]*0.0722 352 | 353 | iterator_ = 0 354 | 355 | shad_reg_mean = [] 356 | back_reg_mean = [] 357 | 358 | for region in shadow_subregions: 359 | r = 0 360 | g = 0 361 | b = 0 362 | for x,y in region: 363 | b = b + img[x][y][0] 364 | g = g + img[x][y][1] 365 | r = r + img[x][y][2] 366 | b = b/len(region) 367 | g = g/len(region) 368 | r = r/len(region) 369 | shad_reg_mean.append([b,g,r]) 370 | 371 | for region in background_subregions: 372 | r = 0 373 | g = 0 374 | b = 0 375 | for x,y in region: 376 | b = b + img[x][y][0] 377 | g = g + img[x][y][1] 378 | r = r + img[x][y][2] 379 | b = b/len(region) 380 | g = g/len(region) 381 | r = r/len(region) 382 | back_reg_mean.append([b,g,r]) 383 | 384 | luminance_shad_std = [] 385 | luminance_back_std = [] 386 | 387 | for region in shadow_subregions: 388 | std = 0 389 | mean = 0 390 | for x,y in region: 391 | mean = mean + luminance[x][y] 392 | mean = mean/len(region) 393 | for x,y in region: 394 | std = std + abs((luminance[x][y]-mean)*(luminance[x][y]-mean)) 395 | luminance_shad_std.append(math.sqrt(std/len(region))) 396 | 397 | 398 | for region in background_subregions: 399 | std = 0 400 | mean = 0 401 | for x,y in region: 402 | mean = mean + luminance[x][y] 403 | mean = mean/len(region) 404 | for x,y in region: 405 | std = std + abs((luminance[x][y]-mean)*(luminance[x][y]-mean)) 406 | luminance_back_std.append(math.sqrt(std/len(region))) 407 | 408 | #match = [0,1,2] 409 | _iterator_ = 0 410 | for region in shadow_subregions: 411 | for x,y in region: 412 | shadow_removed_img[x][y][0] = back_reg_mean[match[_iterator_]][0] + (luminance_back_std[match[_iterator_]]/luminance_shad_std[_iterator_])*(img[x][y][0]-shad_reg_mean[_iterator_][0]) 413 | shadow_removed_img[x][y][1] = back_reg_mean[match[_iterator_]][1] + (luminance_back_std[match[_iterator_]]/luminance_shad_std[_iterator_])*(img[x][y][1]-shad_reg_mean[_iterator_][1]) 414 | shadow_removed_img[x][y][2] = back_reg_mean[match[_iterator_]][2] + (luminance_back_std[match[_iterator_]]/luminance_shad_std[_iterator_])*(img[x][y][2]-shad_reg_mean[_iterator_][2]) 415 | _iterator_ = _iterator_ + 1 416 | 417 | #print(luminance_back_std,luminance_shad_std,back_reg_mean,shad_reg_mean) 418 | print(match) 419 | 420 | ############################################################################### 421 | 422 | cv2.imshow("original_image",img) 423 | cv2.imshow("detected_shadow",bin_thresh) 424 | cv2.imshow("shadow_region",shadow_region) 425 | cv2.imshow("background_region",background_region) 426 | cv2.imshow("shadow_removed_image",shadow_removed_img) 427 | 428 | cv2.waitKey(0) 429 | cv2.destroyAllWindows(0) 430 | 431 | -------------------------------------------------------------------------------- /flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cdbharath/shadow_detection_and_removal/40cefe8c448642d7f9f17a02f6a61787fadf2fc5/flow.png -------------------------------------------------------------------------------- /gabor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | # cv2.getGaborKernel(ksize, sigma, theta, lambda, gamma, psi, ktype) 5 | # ksize - size of gabor filter (n, n) 6 | # sigma - standard deviation of the gaussian function 7 | # theta - orientation of the normal to the parallel stripes 8 | # lambda - wavelength of the sunusoidal factor 9 | # gamma - spatial aspect ratio 10 | # psi - phase offset 11 | # ktype - type and range of values that each pixel in the gabor kernel can hold 12 | 13 | g_kernel = cv2.getGaborKernel((21, 21), 8.0, np.pi/4, 10.0, 0.5, 0, ktype=cv2.CV_32F) 14 | 15 | img = cv2.imread('108004.jpg') 16 | img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 17 | filtered_img = cv2.filter2D(img, cv2.CV_8UC3, g_kernel) 18 | 19 | cv2.imshow('image', img) 20 | cv2.imshow('filtered image', filtered_img) 21 | cv2.waitKey(0) 22 | cv2.destroyAllWindows() 23 | -------------------------------------------------------------------------------- /gabor_filter.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | # cv2.getGaborKernel(ksize, sigma, theta, lambda, gamma, psi, ktype) 5 | # ksize - size of gabor filter (n, n) 6 | # sigma - standard deviation of the gaussian function 7 | # theta - orientation of the normal to the parallel stripes 8 | # lambda - wavelength of the sunusoidal factor 9 | # gamma - spatial aspect ratio 10 | # psi - phase offset 11 | # ktype - type and range of values that each pixel in the gabor kernel can hold 12 | 13 | img = cv2.imread('shadow.jpg') 14 | #img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 15 | 16 | feature_vector = [] 17 | kernel_vector = [] 18 | scales = 4 19 | orientation = 6 20 | 21 | for s in range(scales): 22 | for o in range(orientation): 23 | g_kernel = cv2.getGaborKernel((21, 21), (4.0+s*2), (np.pi*o)/6, 10.0, 0.5, 0, ktype=cv2.CV_32F) 24 | filtered_img = cv2.filter2D(img, cv2.CV_8UC3, g_kernel) 25 | kernel_vector.append(g_kernel) 26 | feature_vector.append(filtered_img) 27 | 28 | print(filtered_img.shape) 29 | cv2.imshow('image', img) 30 | cv2.imshow('filtered image', filtered_img) 31 | cv2.waitKey(0) 32 | cv2.destroyAllWindows() 33 | -------------------------------------------------------------------------------- /gabor_scratch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cdbharath/shadow_detection_and_removal/40cefe8c448642d7f9f17a02f6a61787fadf2fc5/gabor_scratch.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | 3 | # SHADOW REMOVAL WITH SUBREGION MATCHING AND ILLUMINATION TRANSFER 4 | 5 | # DONE BY: 6 | # BHARATH KUMAR 7 | # CHOCKALINGAM 8 | # DC VIVEK 9 | # HARINATH GOBI 10 | 11 | ############################################################################## 12 | 13 | import cv2 14 | import numpy as np 15 | import math 16 | 17 | import torch 18 | import torch.nn as nn 19 | import torch.nn.functional as F 20 | import torch.optim as optim 21 | from torchvision import datasets, transforms 22 | from torch.autograd import Variable 23 | import sys 24 | from skimage import segmentation 25 | import torch.nn.init 26 | import scipy 27 | 28 | use_cuda = torch.cuda.is_available() 29 | 30 | ############################# SEGMENTATION VARIABLES ######################### 31 | 32 | nChannel = 100 33 | maxIter = 100 34 | minLabels = 3 35 | lr = 0.1 36 | nConv = 2 37 | num_superpixels = 10000 38 | compactness = 100 39 | visualize = 1 40 | input_file = 'shadow.jpg' 41 | 42 | ############################################################################## 43 | 44 | img = cv2.imread(input_file) 45 | shadow_removed_img = cv2.imread(input_file) 46 | gray = cv2.imread(input_file, 0) 47 | blur = cv2.bilateralFilter(img,9,75,75) 48 | 49 | ############################# HSI CONVERSION ########################### 50 | 51 | blur = np.divide(blur, 255.0) 52 | 53 | hsi = np.zeros((blur.shape[0],blur.shape[1],blur.shape[2]),dtype=np.float) 54 | ratio_map = np.zeros((blur.shape[0],blur.shape[1]),dtype=np.uint8) 55 | 56 | for i in range(blur.shape[0]): 57 | for j in range(blur.shape[1]): 58 | hsi[i][j][2] = (blur[i][j][0]+blur[i][j][1]+blur[i][j][2])/3 59 | hsi[i][j][0] = math.acos(((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][0]))/(2*math.sqrt((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][1])+(blur[i][j][2]-blur[i][j][0])*(blur[i][j][1]-blur[i][j][0])))) 60 | hsi[i][j][1] = 1 - 3*min(blur[i][j][0],blur[i][j][1],blur[i][j][2])/hsi[i][j][2] 61 | ratio_map[i][j] = hsi[i][j][0]/(hsi[i][j][2]+0.01) 62 | 63 | ############################################################################### 64 | 65 | ######################### SHADOW DETECTION ############################### 66 | 67 | hist = np.histogram(ratio_map.ravel(),256,[0,256]) 68 | ret,th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) 69 | ret,inv_th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) 70 | bin_thresh = cv2.medianBlur(th,15) 71 | bin_inv_thresh = cv2.medianBlur(inv_th,15) 72 | 73 | ############################################################################### 74 | 75 | shadow_region = cv2.bitwise_and(img,img,mask = th) 76 | background_region = cv2.bitwise_and(img,img,mask = inv_th) 77 | 78 | ############################## SEGMENTATION ################################# 79 | 80 | # CNN model 81 | class MyNet(nn.Module): 82 | def __init__(self,input_dim): 83 | super(MyNet, self).__init__() 84 | self.conv1 = nn.Conv2d(input_dim, nChannel, kernel_size=3, stride=1, padding=1 ) 85 | self.bn1 = nn.BatchNorm2d(nChannel) 86 | self.conv2 = [] 87 | self.bn2 = [] 88 | for i in range(nConv-1): 89 | self.conv2.append( nn.Conv2d(nChannel, nChannel, kernel_size=3, stride=1, padding=1 ) ) 90 | self.bn2.append( nn.BatchNorm2d(nChannel) ) 91 | self.conv3 = nn.Conv2d(nChannel, nChannel, kernel_size=1, stride=1, padding=0 ) 92 | self.bn3 = nn.BatchNorm2d(nChannel) 93 | 94 | def forward(self, x): 95 | x = self.conv1(x) 96 | x = F.relu( x ) 97 | x = self.bn1(x) 98 | for i in range(nConv-1): 99 | x = self.conv2[i](x) 100 | x = F.relu( x ) 101 | x = self.bn2[i](x) 102 | x = self.conv3(x) 103 | x = self.bn3(x) 104 | return x 105 | 106 | def segment(im): 107 | data = torch.from_numpy( np.array([im.transpose( (2, 0, 1) ).astype('float32')/255.]) ) 108 | if use_cuda: 109 | data = data.cuda() 110 | data = Variable(data) 111 | 112 | # slic 113 | labels = segmentation.slic(im, compactness=compactness, n_segments=num_superpixels) 114 | labels = labels.reshape(im.shape[0]*im.shape[1]) 115 | u_labels = np.unique(labels) 116 | l_inds = [] 117 | for i in range(len(u_labels)): 118 | l_inds.append( np.where( labels == u_labels[ i ] )[ 0 ] ) 119 | 120 | # train 121 | model = MyNet( data.size(1) ) 122 | if use_cuda: 123 | model.cuda() 124 | for i in range(nConv-1): 125 | model.conv2[i].cuda() 126 | model.bn2[i].cuda() 127 | model.train() 128 | loss_fn = torch.nn.CrossEntropyLoss() 129 | optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) 130 | label_colours = np.random.randint(255,size=(100,3)) 131 | for batch_idx in range(maxIter): 132 | # forwarding 133 | optimizer.zero_grad() 134 | output = model( data )[ 0 ] 135 | output = output.permute( 1, 2, 0 ).contiguous().view( -1, nChannel ) 136 | ignore, target = torch.max( output, 1 ) 137 | im_target = target.data.cpu().numpy() 138 | nLabels = len(np.unique(im_target)) 139 | if visualize: 140 | im_target_rgb = np.array([label_colours[ c % 100 ] for c in im_target]) 141 | im_target_rgb = im_target_rgb.reshape( im.shape ).astype( np.uint8 ) 142 | cv2.imshow( "output", im_target_rgb ) 143 | cv2.waitKey(1) 144 | 145 | # superpixel refinement 146 | for i in range(len(l_inds)): 147 | labels_per_sp = im_target[ l_inds[ i ] ] 148 | u_labels_per_sp = np.unique( labels_per_sp ) 149 | hist = np.zeros( len(u_labels_per_sp) ) 150 | for j in range(len(hist)): 151 | hist[ j ] = len( np.where( labels_per_sp == u_labels_per_sp[ j ] )[ 0 ] ) 152 | im_target[ l_inds[ i ] ] = u_labels_per_sp[ np.argmax( hist ) ] 153 | target = torch.from_numpy( im_target ) 154 | if use_cuda: 155 | target = target.cuda() 156 | target = Variable( target ) 157 | loss = loss_fn(output, target) 158 | loss.backward() 159 | optimizer.step() 160 | 161 | print (batch_idx, '/', maxIter, ':', nLabels, loss.data[0]) 162 | if nLabels <= minLabels: 163 | print ("nLabels", nLabels, "reached minLabels", minLabels, ".") 164 | break 165 | return im_target_rgb 166 | 167 | shadow_region = segment(shadow_region) 168 | shadow_region = cv2.bitwise_and(shadow_region,shadow_region,mask = th) 169 | background_region = segment(background_region) 170 | background_region = cv2.bitwise_and(background_region,background_region,mask = inv_th) 171 | 172 | ############################################################################### 173 | 174 | ############################ SUBREGION MATCHING ############################# 175 | 176 | shadow_region_colors = list(np.unique(shadow_region.reshape(-1, shadow_region.shape[2]), axis=0)) 177 | background_region_colors = list(np.unique(background_region.reshape(-1, background_region.shape[2]), axis=0)) 178 | 179 | shadow_subregions = [] 180 | background_subregions = [] 181 | 182 | for i in range(len(shadow_region_colors)): 183 | shadow_subregions.append([]) 184 | for x in range(shadow_region.shape[0]): 185 | for y in range(shadow_region.shape[1]): 186 | if (shadow_region[x][y] == shadow_region_colors[i]).all(): 187 | shadow_subregions[i].append([x,y]) 188 | 189 | for i in range(len(background_region_colors)): 190 | background_subregions.append([]) 191 | for x in range(background_region.shape[0]): 192 | for y in range(background_region.shape[1]): 193 | if (background_region[x][y] == background_region_colors[i]).all(): 194 | background_subregions[i].append([x,y]) 195 | 196 | shadow_region_colors.pop(0) 197 | background_region_colors.pop(0) 198 | shadow_region_colors = np.array(shadow_region_colors) 199 | background_region_colors = np.array(background_region_colors) 200 | 201 | shadow_subregions.pop(0) 202 | background_subregions.pop(0) 203 | 204 | #print(background_region_colors) 205 | #print(shadow_region_colors) 206 | #print(len(shadow_subregions)) 207 | #print(len(shadow_subregions[0]),len(shadow_subregions[1]),len(shadow_subregions[2])) 208 | 209 | feature_vector = [] 210 | kernel_vector = [] 211 | scales = 4 212 | orientation = 6 213 | 214 | for s in range(scales): 215 | for o in range(orientation): 216 | g_kernel = cv2.getGaborKernel((21, 21), (4.0+s*2), (np.pi*o)/6, 10.0, 0.5, 0, ktype=cv2.CV_32F) 217 | filtered_img = cv2.filter2D(gray, cv2.CV_8UC3, g_kernel) 218 | kernel_vector.append(g_kernel) 219 | feature_vector.append(filtered_img) 220 | 221 | shadow_subregion_feature_vector = [] 222 | background_subregion_feature_vector = [] 223 | 224 | shadow_region_coord_mean = [] 225 | background_region_coord_mean = [] 226 | 227 | feature_vector_mean = [np.mean(vector) for vector in feature_vector] 228 | feature_vector_std = [np.std(vector) for vector in feature_vector] 229 | 230 | iterator = 0 231 | 232 | for vector in feature_vector: 233 | shadow_region_coord_mean.append([]) 234 | shadow_subregion_feature_vector.append([]) 235 | background_region_coord_mean.append([]) 236 | background_subregion_feature_vector.append([]) 237 | 238 | for region in shadow_subregions: 239 | mean = 0 240 | std = 0 241 | x_mean = 0 242 | y_mean = 0 243 | for x,y in region: 244 | mean = mean + vector[x][y] 245 | x_mean = x_mean + x 246 | y_mean = y_mean + y 247 | mean = mean/(vector.shape[0]*vector.shape[1]) 248 | for x,y in region: 249 | std = std + abs((vector[x][y]-mean)*(vector[x][y]-mean)) 250 | shadow_subregion_feature_vector[iterator].append([mean,math.sqrt(std/(vector.shape[0]*vector.shape[1]))]) 251 | shadow_region_coord_mean[iterator].append([x_mean/(len(region)),y_mean/(len(region))]) 252 | 253 | for region in background_subregions: 254 | mean = 0 255 | std = 0 256 | x_mean = 0 257 | y_mean = 0 258 | for x,y in region: 259 | mean = mean + vector[x][y] 260 | x_mean = x_mean + x 261 | y_mean = y_mean + y 262 | mean = mean/(vector.shape[0]*vector.shape[1]) 263 | for x,y in region: 264 | std = std + abs((vector[x][y]-mean)*(vector[x][y]-mean)) 265 | background_subregion_feature_vector[iterator].append([mean,math.sqrt(std/(vector.shape[0]*vector.shape[1]))]) 266 | background_region_coord_mean[iterator].append([x_mean/(len(region)),y_mean/(len(region))]) 267 | iterator = iterator + 1 268 | 269 | #print(len(background_subregions),len(shadow_subregions),len(shadow_subregion_feature_vector),len(shadow_region_coord_mean)) 270 | #print(len(background_region_coord_mean),len(background_subregion_feature_vector),len(background_region_coord_mean[0]),len(background_subregion_feature_vector[0]),len(background_subregion_feature_vector[0][0]),len(background_region_coord_mean[0][0])) 271 | #print(shadow_region_coord_mean[0],shadow_region_coord_mean[1]) 272 | #print(shadow_subregion_feature_vector[0],shadow_subregion_feature_vector[1],shadow_subregion_feature_vector[2]) 273 | 274 | dist_texture = [] 275 | dist_space = [] 276 | index = 0 277 | dist = 0 278 | text = 0 279 | iterator = 0 280 | _iterator = 0 281 | 282 | #for s in range(scales): 283 | # for o in range(orientation): 284 | # dist_texture.append([]) 285 | # index = (orientation*s)+o 286 | # for shadow_reg in range(len(shadow_subregion_feature_vector[index])): 287 | # dist_texture[index].append([]) 288 | # for background_reg in range(len(background_subregion_feature_vector[index])): 289 | # text = text + abs((shadow_subregion_feature_vector[index][shadow_reg][0]-background_subregion_feature_vector[index][shadow_reg][0])/feature_vector_mean[index][0]) + abs((shadow_subregion_feature_vector[index][shadow_reg][1]-background_subregion_feature_vector[index][shadow_reg][1])/feature_vector_mean[index][1]) 290 | # dist_texture[index][iterator].append(text) 291 | # iterator = iterator + 1 292 | 293 | #print(feature_vector_mean[0],shadow_subregion_feature_vector[0],background_subregion_feature_vector[0]) 294 | 295 | for shadow_reg in range(len(shadow_subregion_feature_vector[index])): 296 | dist_texture.append({}) 297 | _iterator = 0 298 | for background_reg in range(len(background_subregion_feature_vector[index])): 299 | text = 0 300 | for s in range(scales): 301 | for o in range(orientation): 302 | index = (orientation*s)+o 303 | text = text + abs((shadow_subregion_feature_vector[index][shadow_reg][0]-background_subregion_feature_vector[index][shadow_reg][0])/feature_vector_mean[index]) + abs((shadow_subregion_feature_vector[index][shadow_reg][1]-background_subregion_feature_vector[index][shadow_reg][1])/feature_vector_mean[index]) 304 | dist_texture[iterator][_iterator] = text 305 | _iterator = _iterator + 1 306 | iterator = iterator + 1 307 | 308 | iterator = 0 309 | _iterator = 0 310 | 311 | for shadow_reg in range(len(shadow_subregion_feature_vector[index])): 312 | dist_space.append({}) 313 | _iterator = 0 314 | for background_reg in range(len(background_subregion_feature_vector[index])): 315 | dist = math.sqrt((shadow_region_coord_mean[0][shadow_reg][0]-background_region_coord_mean[0][background_reg][0])*(shadow_region_coord_mean[0][shadow_reg][0]-background_region_coord_mean[0][background_reg][0])+(shadow_region_coord_mean[0][shadow_reg][1]-background_region_coord_mean[0][background_reg][1])*(shadow_region_coord_mean[0][shadow_reg][1]-background_region_coord_mean[0][background_reg][1])) 316 | dist_space[iterator][_iterator] = dist 317 | _iterator = _iterator + 1 318 | iterator = iterator + 1 319 | 320 | #print(dist_texture,dist_space) 321 | 322 | for i in range(len(dist_texture)): 323 | dist_texture[i] = sorted(dist_texture[i].items(), key=lambda kv: kv[1]) 324 | dist_space[i] = sorted(dist_space[i].items(), key=lambda kv: kv[1]) 325 | 326 | match = [] 327 | 328 | for i in range(len(dist_texture)): 329 | min = len(dist_texture[0])+len(dist_space[0]) 330 | minimum = 0 331 | for j in range(len(dist_texture[0])): 332 | for k in range(len(dist_space[0])): 333 | if dist_texture[i][j][0] == dist_space[i][k][0]: 334 | if j+k < min: 335 | min = j+k 336 | minimum = dist_texture[i][j][0] 337 | match.append(minimum) 338 | 339 | #print(match) 340 | ############################################################################### 341 | 342 | ############################ SHADOW REMOVAL ############################# 343 | 344 | luminance = np.zeros((img.shape[0],img.shape[1])) 345 | #shadow_removed_image = np.zeros((img.shape[0],img.shape[1],img.shape[2])) 346 | #sigma_shadow = [1]*len(shadow_subregion_feature_vector[0]) 347 | #sigma_background = [1]*len(background_subregion_feature_vector[0]) 348 | 349 | for x in range(img.shape[0]): 350 | for y in range(img.shape[1]): 351 | luminance[x][y] = img[x][y][2]*0.2126 + img[x][y][1]*0.7152 + img[x][y][0]*0.0722 352 | 353 | iterator_ = 0 354 | 355 | shad_reg_mean = [] 356 | back_reg_mean = [] 357 | 358 | for region in shadow_subregions: 359 | r = 0 360 | g = 0 361 | b = 0 362 | for x,y in region: 363 | b = b + img[x][y][0] 364 | g = g + img[x][y][1] 365 | r = r + img[x][y][2] 366 | b = b/len(region) 367 | g = g/len(region) 368 | r = r/len(region) 369 | shad_reg_mean.append([b,g,r]) 370 | 371 | for region in background_subregions: 372 | r = 0 373 | g = 0 374 | b = 0 375 | for x,y in region: 376 | b = b + img[x][y][0] 377 | g = g + img[x][y][1] 378 | r = r + img[x][y][2] 379 | b = b/len(region) 380 | g = g/len(region) 381 | r = r/len(region) 382 | back_reg_mean.append([b,g,r]) 383 | 384 | luminance_shad_std = [] 385 | luminance_back_std = [] 386 | 387 | for region in shadow_subregions: 388 | std = 0 389 | mean = 0 390 | for x,y in region: 391 | mean = mean + luminance[x][y] 392 | mean = mean/len(region) 393 | for x,y in region: 394 | std = std + abs((luminance[x][y]-mean)*(luminance[x][y]-mean)) 395 | luminance_shad_std.append(math.sqrt(std/len(region))) 396 | 397 | 398 | for region in background_subregions: 399 | std = 0 400 | mean = 0 401 | for x,y in region: 402 | mean = mean + luminance[x][y] 403 | mean = mean/len(region) 404 | for x,y in region: 405 | std = std + abs((luminance[x][y]-mean)*(luminance[x][y]-mean)) 406 | luminance_back_std.append(math.sqrt(std/len(region))) 407 | 408 | #match = [0,1,2] 409 | _iterator_ = 0 410 | for region in shadow_subregions: 411 | for x,y in region: 412 | shadow_removed_img[x][y][0] = back_reg_mean[match[_iterator_]][0] + (luminance_back_std[match[_iterator_]]/luminance_shad_std[_iterator_])*(img[x][y][0]-shad_reg_mean[_iterator_][0]) 413 | shadow_removed_img[x][y][1] = back_reg_mean[match[_iterator_]][1] + (luminance_back_std[match[_iterator_]]/luminance_shad_std[_iterator_])*(img[x][y][1]-shad_reg_mean[_iterator_][1]) 414 | shadow_removed_img[x][y][2] = back_reg_mean[match[_iterator_]][2] + (luminance_back_std[match[_iterator_]]/luminance_shad_std[_iterator_])*(img[x][y][2]-shad_reg_mean[_iterator_][2]) 415 | _iterator_ = _iterator_ + 1 416 | 417 | #print(luminance_back_std,luminance_shad_std,back_reg_mean,shad_reg_mean) 418 | print(match) 419 | 420 | ############################################################################### 421 | 422 | cv2.imshow("original_image",img) 423 | cv2.imshow("detected_shadow",bin_thresh) 424 | cv2.imshow("shadow_region",shadow_region) 425 | cv2.imshow("background_region",background_region) 426 | cv2.imshow("shadow_removed_image",shadow_removed_img) 427 | 428 | cv2.waitKey(0) 429 | cv2.destroyAllWindows(0) 430 | 431 | -------------------------------------------------------------------------------- /output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cdbharath/shadow_detection_and_removal/40cefe8c448642d7f9f17a02f6a61787fadf2fc5/output.png -------------------------------------------------------------------------------- /segment.py: -------------------------------------------------------------------------------- 1 | #from __future__ import print_function 2 | import argparse 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | from torchvision import datasets, transforms 8 | from torch.autograd import Variable 9 | import cv2 10 | import sys 11 | import numpy as np 12 | from skimage import segmentation 13 | import torch.nn.init 14 | 15 | use_cuda = torch.cuda.is_available() 16 | 17 | parser = argparse.ArgumentParser(description='PyTorch Unsupervised Segmentation') 18 | parser.add_argument('--nChannel', metavar='N', default=100, type=int, 19 | help='number of channels') 20 | parser.add_argument('--maxIter', metavar='T', default=1000, type=int, 21 | help='number of maximum iterations') 22 | parser.add_argument('--minLabels', metavar='minL', default=3, type=int, 23 | help='minimum number of labels') 24 | parser.add_argument('--lr', metavar='LR', default=0.1, type=float, 25 | help='learning rate') 26 | parser.add_argument('--nConv', metavar='M', default=2, type=int, 27 | help='number of convolutional layers') 28 | parser.add_argument('--num_superpixels', metavar='K', default=10000, type=int, 29 | help='number of superpixels') 30 | parser.add_argument('--compactness', metavar='C', default=100, type=float, 31 | help='compactness of superpixels') 32 | parser.add_argument('--visualize', metavar='1 or 0', default=1, type=int, 33 | help='visualization flag') 34 | parser.add_argument('--input', metavar='FILENAME', 35 | help='input image file name', required=True) 36 | args = parser.parse_args() 37 | 38 | # CNN model 39 | class MyNet(nn.Module): 40 | def __init__(self,input_dim): 41 | super(MyNet, self).__init__() 42 | self.conv1 = nn.Conv2d(input_dim, args.nChannel, kernel_size=3, stride=1, padding=1 ) 43 | self.bn1 = nn.BatchNorm2d(args.nChannel) 44 | self.conv2 = [] 45 | self.bn2 = [] 46 | for i in range(args.nConv-1): 47 | self.conv2.append( nn.Conv2d(args.nChannel, args.nChannel, kernel_size=3, stride=1, padding=1 ) ) 48 | self.bn2.append( nn.BatchNorm2d(args.nChannel) ) 49 | self.conv3 = nn.Conv2d(args.nChannel, args.nChannel, kernel_size=1, stride=1, padding=0 ) 50 | self.bn3 = nn.BatchNorm2d(args.nChannel) 51 | 52 | def forward(self, x): 53 | x = self.conv1(x) 54 | x = F.relu( x ) 55 | x = self.bn1(x) 56 | for i in range(args.nConv-1): 57 | x = self.conv2[i](x) 58 | x = F.relu( x ) 59 | x = self.bn2[i](x) 60 | x = self.conv3(x) 61 | x = self.bn3(x) 62 | return x 63 | 64 | # load image 65 | im = cv2.imread(args.input) 66 | data = torch.from_numpy( np.array([im.transpose( (2, 0, 1) ).astype('float32')/255.]) ) 67 | if use_cuda: 68 | data = data.cuda() 69 | data = Variable(data) 70 | 71 | # slic 72 | labels = segmentation.slic(im, compactness=args.compactness, n_segments=args.num_superpixels) 73 | labels = labels.reshape(im.shape[0]*im.shape[1]) 74 | u_labels = np.unique(labels) 75 | l_inds = [] 76 | for i in range(len(u_labels)): 77 | l_inds.append( np.where( labels == u_labels[ i ] )[ 0 ] ) 78 | 79 | # train 80 | model = MyNet( data.size(1) ) 81 | if use_cuda: 82 | model.cuda() 83 | for i in range(args.nConv-1): 84 | model.conv2[i].cuda() 85 | model.bn2[i].cuda() 86 | model.train() 87 | loss_fn = torch.nn.CrossEntropyLoss() 88 | optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9) 89 | label_colours = np.random.randint(255,size=(100,3)) 90 | for batch_idx in range(args.maxIter): 91 | # forwarding 92 | optimizer.zero_grad() 93 | output = model( data )[ 0 ] 94 | output = output.permute( 1, 2, 0 ).contiguous().view( -1, args.nChannel ) 95 | ignore, target = torch.max( output, 1 ) 96 | im_target = target.data.cpu().numpy() 97 | nLabels = len(np.unique(im_target)) 98 | if args.visualize: 99 | im_target_rgb = np.array([label_colours[ c % 100 ] for c in im_target]) 100 | im_target_rgb = im_target_rgb.reshape( im.shape ).astype( np.uint8 ) 101 | cv2.imshow( "output", im_target_rgb ) 102 | cv2.waitKey(10) 103 | 104 | # superpixel refinement 105 | for i in range(len(l_inds)): 106 | labels_per_sp = im_target[ l_inds[ i ] ] 107 | u_labels_per_sp = np.unique( labels_per_sp ) 108 | hist = np.zeros( len(u_labels_per_sp) ) 109 | for j in range(len(hist)): 110 | hist[ j ] = len( np.where( labels_per_sp == u_labels_per_sp[ j ] )[ 0 ] ) 111 | im_target[ l_inds[ i ] ] = u_labels_per_sp[ np.argmax( hist ) ] 112 | target = torch.from_numpy( im_target ) 113 | if use_cuda: 114 | target = target.cuda() 115 | target = Variable( target ) 116 | loss = loss_fn(output, target) 117 | loss.backward() 118 | optimizer.step() 119 | 120 | print (batch_idx, '/', args.maxIter, ':', nLabels, loss.data[0]) 121 | if nLabels <= args.minLabels: 122 | print ("nLabels", nLabels, "reached minLabels", args.minLabels, ".") 123 | break 124 | 125 | # save output image 126 | if not args.visualize: 127 | output = model( data )[ 0 ] 128 | output = output.permute( 1, 2, 0 ).contiguous().view( -1, args.nChannel ) 129 | ignore, target = torch.max( output, 1 ) 130 | im_target = target.data.cpu().numpy() 131 | im_target_rgb = np.array([label_colours[ c % 100 ] for c in im_target]) 132 | im_target_rgb = im_target_rgb.reshape( im.shape ).astype( np.uint8 ) 133 | cv2.imwrite( "output.png", im_target_rgb ) 134 | -------------------------------------------------------------------------------- /shadow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cdbharath/shadow_detection_and_removal/40cefe8c448642d7f9f17a02f6a61787fadf2fc5/shadow.jpg -------------------------------------------------------------------------------- /shadow_detect.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import math 4 | 5 | img = cv2.imread('shadow.jpg') 6 | gray = cv2.imread('shadow.jpg', 0) 7 | blur = cv2.bilateralFilter(img,9,75,75) 8 | 9 | ############################# HSI CONVERSION ########################### 10 | 11 | blur = np.divide(blur, 255.0) 12 | 13 | hsi = np.zeros((blur.shape[0],blur.shape[1],blur.shape[2]),dtype=np.float) 14 | ratio_map = np.zeros((blur.shape[0],blur.shape[1]),dtype=np.uint8) 15 | 16 | for i in range(blur.shape[0]): 17 | for j in range(blur.shape[1]): 18 | hsi[i][j][2] = (blur[i][j][0]+blur[i][j][1]+blur[i][j][2])/3 19 | hsi[i][j][0] = math.acos(((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][0]))/(2*math.sqrt((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][1])+(blur[i][j][2]-blur[i][j][0])*(blur[i][j][1]-blur[i][j][0])))) 20 | hsi[i][j][1] = 1 - 3*min(blur[i][j][0],blur[i][j][1],blur[i][j][2])/hsi[i][j][2] 21 | ratio_map[i][j] = hsi[i][j][0]/(hsi[i][j][2]+0.01) 22 | 23 | ############################################################################### 24 | 25 | ######################### OTSU'S METHOD ################################# 26 | 27 | hist = np.histogram(ratio_map.ravel(),256,[0,256]) 28 | ret,th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) 29 | median = cv2.medianBlur(th,15) 30 | ############################################################################### 31 | 32 | cv2.imshow("original_image",img) 33 | cv2.imshow("detected_shadow",median) 34 | 35 | cv2.waitKey(0) 36 | cv2.destroyAllWindows(0) 37 | 38 | -------------------------------------------------------------------------------- /shadow_remove.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import math 4 | 5 | seg = cv2.imread("output.png") 6 | img = cv2.imread('shadow.jpg') 7 | gray = cv2.imread('shadow.jpg', 0) 8 | blur = cv2.bilateralFilter(img,9,75,75) 9 | 10 | ############################# HSI CONVERSION ########################### 11 | 12 | blur = np.divide(blur, 255.0) 13 | 14 | hsi = np.zeros((blur.shape[0],blur.shape[1],blur.shape[2]),dtype=np.float) 15 | ratio_map = np.zeros((blur.shape[0],blur.shape[1]),dtype=np.uint8) 16 | 17 | for i in range(blur.shape[0]): 18 | for j in range(blur.shape[1]): 19 | hsi[i][j][2] = (blur[i][j][0]+blur[i][j][1]+blur[i][j][2])/3 20 | hsi[i][j][0] = math.acos(((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][0]))/(2*math.sqrt((blur[i][j][2]-blur[i][j][1])*(blur[i][j][2]-blur[i][j][1])+(blur[i][j][2]-blur[i][j][0])*(blur[i][j][1]-blur[i][j][0])))) 21 | hsi[i][j][1] = 1 - 3*min(blur[i][j][0],blur[i][j][1],blur[i][j][2])/hsi[i][j][2] 22 | ratio_map[i][j] = hsi[i][j][0]/(hsi[i][j][2]+0.01) 23 | 24 | ############################################################################### 25 | 26 | ######################### OTSU'S METHOD ################################# 27 | 28 | hist = np.histogram(ratio_map.ravel(),256,[0,256]) 29 | ret,th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) 30 | ret,inv_th = cv2.threshold(ratio_map,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) 31 | bin_thresh = cv2.medianBlur(th,15) 32 | bin_inv_thresh = cv2.medianBlur(inv_th,15) 33 | ############################################################################### 34 | 35 | ############################################################################### 36 | shadow_region = cv2.bitwise_and(seg,seg,mask = bin_thresh) 37 | background_region = cv2.bitwise_and(seg,seg,mask = bin_inv_thresh) 38 | 39 | print(np.unique(seg.reshape(-1, seg.shape[2]), axis=0)) 40 | 41 | cv2.imshow("original_image",img) 42 | cv2.imshow("detected_shadow",bin_thresh) 43 | cv2.imshow("segmented_image",seg) 44 | cv2.imshow("shadow_region",shadow_region) 45 | cv2.imshow("background_region",background_region) 46 | 47 | cv2.waitKey(0) 48 | cv2.destroyAllWindows(0) 49 | 50 | --------------------------------------------------------------------------------