├── .gitignore ├── demo.py ├── pascal_voc_util.pkl ├── readme.md └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pth 2 | *.pyc 3 | *.ipynb_checkpoints/* -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from utils import * 3 | import pickle 4 | from docopt import docopt 5 | import time 6 | 7 | docstr = """Find semantic segmentation metrics for given predictions and ground truth images(For PASCAL VOC 2012). 8 | 9 | Usage: 10 | demo.py convert_prediction 11 | demo.py convert_gt 12 | demo.py find_metrics [options] 13 | demo.py (-h | --help) 14 | demo.py --version 15 | 16 | Options: 17 | -h --help Show this screen. 18 | --version Show version. 19 | --batch_size= batch_size for processing in gpu[default: 20] 20 | --gpu= if GPU is to be used, and which GPU to be used. Set blank for no GPU.[default: 0] 21 | --classes= number of classes[default: 21] 22 | --ignore_label= label to be ignored[default: 255] 23 | 24 | """ 25 | if __name__ == '__main__': 26 | args = docopt(docstr, version='v0.1') 27 | if(args['--gpu']): 28 | torch.cuda.set_device(int(args['--gpu'])) 29 | start_time = time.time() 30 | if(args['find_metrics']): 31 | ########### 32 | pascal_voc_util = pickle.load( open( "pascal_voc_util.pkl", "rb" ) ) 33 | class_dict = pascal_voc_util[1] 34 | class_names = [class_dict[i] for i in class_dict.keys()][:-1] 35 | ########### 36 | file_ids = get_file_ids(args['']) 37 | st = time.time() 38 | print('Making Histogram') 39 | hist = hist_maker(args[''],args[''],file_ids, 40 | int(args['--batch_size']),int(args['--ignore_label']),int(args['--classes']),bool(args['--gpu'])) 41 | print('Histogram Made!') 42 | print(time.time()-st) 43 | mean_acc = mean_pixel_accuracy(hist.numpy(),class_names) 44 | overall_acc = pixel_accuracy(hist.numpy()) 45 | mean_ious = mean_iou(hist.numpy(),class_names) 46 | fmiou = freq_weighted_miou(hist.numpy(), class_names) 47 | print('Total time taken: ',time.time()-start_time) 48 | 49 | if(args['convert_prediction']): 50 | ######## 51 | pascal_voc_util = pickle.load( open( "pascal_voc_util.pkl", "rb" ) ) 52 | color_map = pascal_voc_util[0] 53 | ######## 54 | file_ids = get_file_ids(args['']) 55 | mat_to_png(args[''],file_ids,color_map) 56 | print('Total time taken: ',time.time()-start_time) 57 | 58 | if(args['convert_gt']): 59 | ######## 60 | pascal_voc_util = pickle.load( open( "pascal_voc_util.pkl", "rb" ) ) 61 | color_map = pascal_voc_util[0] 62 | ######## 63 | file_ids = get_file_ids(args['']) 64 | img_dim_reductor(args[''],file_ids,color_map) 65 | print('Total time taken: ',time.time()-start_time) -------------------------------------------------------------------------------- /pascal_voc_util.pkl: -------------------------------------------------------------------------------- 1 | ((dp0 2 | (F128.0 3 | F64.0 4 | F0.0 5 | tp1 6 | I17 7 | s(F0.0 8 | F192.0 9 | F0.0 10 | tp2 11 | I18 12 | s(F192.0 13 | F128.0 14 | F0.0 15 | tp3 16 | I11 17 | s(F64.0 18 | F0.0 19 | F0.0 20 | tp4 21 | I8 22 | s(F0.0 23 | F0.0 24 | F128.0 25 | tp5 26 | I4 27 | s(F0.0 28 | F128.0 29 | F0.0 30 | tp6 31 | I2 32 | s(F192.0 33 | F128.0 34 | F128.0 35 | tp7 36 | I15 37 | s(F64.0 38 | F0.0 39 | F128.0 40 | tp8 41 | I12 42 | s(F192.0 43 | F0.0 44 | F0.0 45 | tp9 46 | I9 47 | s(F0.0 48 | F64.0 49 | F0.0 50 | tp10 51 | I16 52 | s(F128.0 53 | F0.0 54 | F128.0 55 | tp11 56 | I5 57 | s(F0.0 58 | F64.0 59 | F128.0 60 | tp12 61 | I20 62 | s(F128.0 63 | F0.0 64 | F0.0 65 | tp13 66 | I1 67 | s(F128.0 68 | F128.0 69 | F128.0 70 | tp14 71 | I7 72 | s(F192.0 73 | F0.0 74 | F128.0 75 | tp15 76 | I13 77 | s(F64.0 78 | F128.0 79 | F128.0 80 | tp16 81 | I14 82 | s(F128.0 83 | F192.0 84 | F0.0 85 | tp17 86 | I19 87 | s(F224.0 88 | F224.0 89 | F192.0 90 | tp18 91 | I255 92 | s(F128.0 93 | F128.0 94 | F0.0 95 | tp19 96 | I3 97 | s(F64.0 98 | F128.0 99 | F0.0 100 | tp20 101 | I10 102 | s(F0.0 103 | F0.0 104 | F0.0 105 | tp21 106 | I0 107 | s(F0.0 108 | F128.0 109 | F128.0 110 | tp22 111 | I6 112 | s(dp23 113 | I0 114 | S'background' 115 | p24 116 | sI1 117 | S'aeroplane' 118 | p25 119 | sI2 120 | S'bicycle' 121 | p26 122 | sI3 123 | S'bird' 124 | p27 125 | sI4 126 | S'boat' 127 | p28 128 | sI5 129 | S'bottle' 130 | p29 131 | sI6 132 | S'bus' 133 | p30 134 | sI7 135 | S'car' 136 | p31 137 | sI8 138 | S'cat' 139 | p32 140 | sI9 141 | S'chair' 142 | p33 143 | sI10 144 | S'cow' 145 | p34 146 | sI11 147 | S'diningtable' 148 | p35 149 | sI12 150 | S'dog' 151 | p36 152 | sI13 153 | S'horse' 154 | p37 155 | sI14 156 | S'motorbike' 157 | p38 158 | sI15 159 | S'person' 160 | p39 161 | sI16 162 | S'potted-plant' 163 | p40 164 | sI17 165 | S'sheep' 166 | p41 167 | sI18 168 | S'sofa' 169 | p42 170 | sI19 171 | S'train' 172 | p43 173 | sI20 174 | S'tv/monitor' 175 | p44 176 | sI255 177 | S'ignorelabel' 178 | p45 179 | stp46 180 | . -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Semantic Segmentation Metrics on Pytorch 2 | 3 | 4 | ## Metrics used: 5 | 6 | * Pixel Accuracy 7 | * mean Accuracy(of per-class pixel accuracy) 8 | * mean IOU(of per-class Mean IOU) 9 | * Frequency weighted IOU 10 | 11 | For more information, kindly refer [Fully Convolutional Networks for Semantic Segmentation 12 | ](https://people.eecs.berkeley.edu/~jonlong/long_shelhamer_fcn.pdf) 13 | 14 | ## Functions 15 | 16 | ### Convert .mat files to .png files 17 | Use 18 | 19 | ` python demo.py convert_prediction predict_loc id_file` 20 | 21 | ### Convert Pascal VOC Validation filesfrom 3d-color to 2d-class-id .png format 22 | Use 23 | 24 | `python demo.py convert_gt gt_loc id_file` 25 | 26 | ### Calculate the metrics 27 | Use 28 | 29 | `python demo.py find_metrics predict_path gt_path id_file [--options]` 30 | 31 | ## Files 32 | 33 | * `utils.py`: contains functions. 34 | * `demo.py`: contains a brief demo of how to use the functions.[use demo.py -h] 35 | 36 | ## Acknowledgement 37 | 38 | 39 | A few parts have been adopted from the code present in [martinkersner/py_img_seg_eval](https://github.com/martinkersner/py_img_seg_eval/tree/c0bf9787ebbe3e5e2c7833efe78b5b2d392afaf1). Although the formulations are slightly wrong, it was very helpful. 40 | 41 | Also, a big thanks to [Video Analytics Lab](http://val.serc.iisc.ernet.in/valweb/). 42 | 43 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import skimage.io as sio 2 | import os 3 | import numpy as np 4 | import cPickle as pickle 5 | import torch 6 | import numpy as np 7 | import time 8 | import os 9 | import math 10 | import scipy.io 11 | 12 | def get_file_ids(id_path): 13 | file_ids = open(id_path) 14 | ids = [] 15 | for line in file_ids.readlines(): 16 | ids.append(line[:-1]) 17 | return ids 18 | 19 | def mat_to_png(location,file_ids,conv_dict): 20 | ### 21 | post_fix = '.png' 22 | ### 23 | if not os.path.isdir(location+'_converted/'): 24 | os.makedirs(location+'_converted/') 25 | for id in file_ids: 26 | print(os.path.join(location,id+'.mat')) 27 | img = scipy.io.loadmat(os.path.join(location,id+'.mat'))['data'] 28 | img = img.swapaxes(3,0).swapaxes(1,2)[0] 29 | img = np.argmax(img,0) 30 | sio.imsave(location+'_converted/'+id+post_fix,img) 31 | print('Saved:',id+post_fix) 32 | 33 | def img_dim_reductor(location,file_ids,conv_dict): 34 | ### 35 | post_fix = '.png' 36 | ### 37 | if not os.path.isdir(location+'_converted/'): 38 | os.makedirs(location+'_converted/') 39 | for id in file_ids: 40 | img = sio.imread(os.path.join(location,id+post_fix)) 41 | img_2d = img_to_2d(img,conv_dict) 42 | sio.imsave(os.path.join(location+'_converted',id+post_fix),img_2d) 43 | print('Saved:',id+post_fix) 44 | 45 | def img_to_2d(img,conv_dict): 46 | img_2d = np.zeros((img.shape[0],img.shape[1]), dtype=np.uint8) 47 | for key,value in conv_dict.iteritems(): 48 | m = np.all(img==np.array(key).reshape(1, 1, 3), axis=2) 49 | img_2d[m] = value 50 | return img_2d 51 | 52 | def tensor_maker(cur_batch, predict_loc,gt_loc,gpu=True): 53 | #returns the tensors of GT and IMG(use 513 and crops) 54 | ######### 55 | post_fix = '.png' 56 | ######### 57 | batch_size = len(cur_batch) 58 | height = 513 59 | width = 513 60 | prediction_tensor = np.full((batch_size,height,width),255) 61 | gt_tensor = np.full((batch_size,height,width),255) 62 | counter = 0 63 | for id in cur_batch: 64 | img = sio.imread(os.path.join(predict_loc,id+post_fix)) 65 | prediction_tensor[counter,:img.shape[0],:img.shape[1]] = np.copy(img) 66 | img = sio.imread(os.path.join(gt_loc,id+post_fix)) 67 | gt_tensor[counter,:img.shape[0],:img.shape[1]] = np.copy(img) 68 | counter +=1 69 | prediction_tensor = torch.from_numpy(prediction_tensor).long() 70 | gt_tensor = torch.from_numpy(gt_tensor).long() 71 | if(gpu): 72 | prediction_tensor = prediction_tensor.cuda() 73 | gt_tensor = gt_tensor.cuda() 74 | return (prediction_tensor,gt_tensor) 75 | 76 | def hist_per_batch(tensor_1, tensor_2, ignore_label=255, classes=21): 77 | hist_tensor = torch.zeros(classes,classes) 78 | for class_2_int in range(classes): 79 | tensor_2_class = torch.eq(tensor_2,class_2_int).long() 80 | for class_1_int in range(classes): 81 | tensor_1_class = torch.eq(tensor_1,class_1_int).long() 82 | tensor_1_class = torch.mul(tensor_2_class,tensor_1_class) 83 | count = torch.sum(tensor_1_class) 84 | hist_tensor[class_2_int,class_1_int] +=count 85 | return hist_tensor 86 | 87 | def hist_maker(predict_loc,gt_loc,file_id_list,batch_size= 20,ignore_label = 255,classes=21,gpu=True): 88 | hist_tensor = torch.zeros(classes,classes) 89 | max_iter = int(math.ceil(len(file_id_list)/batch_size)+1) 90 | for i in range(max_iter): 91 | cur_batch = file_id_list[batch_size*i:min(len(file_id_list),batch_size*(i+1))] 92 | predict_tensor, gt_tensor = tensor_maker(cur_batch, predict_loc,gt_loc,gpu = gpu) 93 | hist_batch = hist_per_batch(predict_tensor, gt_tensor, ignore_label=255, classes=21) 94 | hist_tensor = torch.add(hist_tensor,hist_batch) 95 | return hist_tensor 96 | 97 | def mean_iou(hist_matrix, class_names): 98 | classes = len(class_names) 99 | class_scores = np.zeros((classes)) 100 | for i in range(classes): 101 | class_scores[i] = hist_matrix[i,i]/(max(1,np.sum(hist_matrix[i,:])+np.sum(hist_matrix[:,i])-hist_matrix[i,i])) 102 | print('class',class_names[i],'miou',class_scores[i]) 103 | print('Mean IOU:',np.mean(class_scores)) 104 | return class_scores 105 | 106 | def mean_pixel_accuracy(hist_matrix, class_names): 107 | classes = len(class_names) 108 | class_scores = np.zeros((classes)) 109 | for i in range(classes): 110 | class_scores[i] = hist_matrix[i,i]/(max(1,np.sum(hist_matrix[i,:]))) 111 | print('class',class_names[i],'mean_pixel_accuracy',class_scores[i]) 112 | return class_scores 113 | 114 | def pixel_accuracy(hist_matrix): 115 | num = np.trace(hist_matrix) 116 | p_a = num/max(1,np.sum(hist_matrix).astype('float')) 117 | print('Pixel accuracy:',p_a) 118 | return p_a 119 | 120 | def freq_weighted_miou(hist_matrix, class_names): 121 | classes = len(class_names) 122 | class_scores = np.zeros((classes)) 123 | for i in range(classes): 124 | class_scores[i] = np.sum(hist_matrix[i,:])*hist_matrix[i,i]/(max(1,np.sum(hist_matrix[i,:]))) 125 | fmiou = np.sum(class_scores)/np.sum(hist_matrix).astype('float') 126 | print('Frequency Weighted mean accuracy:',fmiou) 127 | return fmiou 128 | --------------------------------------------------------------------------------