├── README.md ├── box_overlaps.c ├── box_overlaps.pyx ├── evaluation.py └── setup.py /README.md: -------------------------------------------------------------------------------- 1 | # WiderFace-Eval-Python 2 | wider face验证集的测评代码 3 | 4 | 一、生成预测结果 5 | 生成以下预测结果： 6 | ```python 7 | 0--Parade 8 | 0_Parade_marchingband_1_20.txt 9 | 0_Parade_marchingband_1_74.txt 10 | ... 11 | 1--Handshaking 12 | ``` 13 | 其中，0--Parade是不同场景的文件夹，wider face总共有61种场景，0_Parade_marchingband_1_20.txt是对应某个图片的预测结果，其具有以下格式： 14 | ```python 15 | image_name 16 | the number fo faces # 检测出多少张人脸 17 | x, y, w, h, confidence # x和y是检测框左上角的坐标 18 | ``` 19 | 举个例子： 20 | ```python 21 | 0_Parade_marchingband_1_309.jpg 22 | 536 23 | 499.62817 73.10439 34.215393 38.730423 0.93176836 24 | 47.55735 86.14974 21.215218 25.779213 0.7041396 25 | ``` 26 | 27 | 二、下载ground truth数据 28 | 可以从官网下载，得到这四个文件：`wider_easy_val.mat, wider_face_val.mat, wider_hard_val.mat, wider_medium_val.mat` 29 | 官网下载可能有点慢，可以从这里下载：https://pan.baidu.com/s/1AErRlTlYaok6p7OGV7VShQ 30 | 31 | 三、编译工具 32 | 执行命令： 33 | 34 | python3 setup.py build_ext --inplace 35 | 36 | 四、测AP 37 | ```python 38 | python3 evaluation.py -p -g # 测easy,medium,hard的结果 39 | ``` 40 | ```python 41 | python3 evaluation.py -p -g --all # 将easy,medium,hard一起测 42 | ``` 43 | 44 | -------------------------------------------------------------------------------- /box_overlaps.pyx: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------- 2 | # Fast R-CNN 3 | # Copyright (c) 2015 Microsoft 4 | # Licensed under The MIT License [see LICENSE for details] 5 | # Written by Sergey Karayev 6 | # -------------------------------------------------------- 7 | 8 | cimport cython 9 | import numpy as np 10 | cimport numpy as np 11 | 12 | DTYPE = np.float 13 | ctypedef np.float_t DTYPE_t 14 | 15 | def bbox_overlaps( 16 | np.ndarray[DTYPE_t, ndim=2] boxes, 17 | np.ndarray[DTYPE_t, ndim=2] query_boxes): 18 | """ 19 | Parameters 20 | ---------- 21 | boxes: (N, 4) ndarray of float 22 | query_boxes: (K, 4) ndarray of float 23 | Returns 24 | ------- 25 | overlaps: (N, K) ndarray of overlap between boxes and query_boxes 26 | """ 27 | cdef unsigned int N = boxes.shape[0] 28 | cdef unsigned int K = query_boxes.shape[0] 29 | cdef np.ndarray[DTYPE_t, ndim=2] overlaps = np.zeros((N, K), dtype=DTYPE) 30 | cdef DTYPE_t iw, ih, box_area 31 | cdef DTYPE_t ua 32 | cdef unsigned int k, n 33 | for k in range(K): 34 | box_area = ( 35 | (query_boxes[k, 2] - query_boxes[k, 0] + 1) * 36 | (query_boxes[k, 3] - query_boxes[k, 1] + 1) 37 | ) 38 | for n in range(N): 39 | iw = ( 40 | min(boxes[n, 2], query_boxes[k, 2]) - 41 | max(boxes[n, 0], query_boxes[k, 0]) + 1 42 | ) 43 | if iw > 0: 44 | ih = ( 45 | min(boxes[n, 3], query_boxes[k, 3]) - 46 | max(boxes[n, 1], query_boxes[k, 1]) + 1 47 | ) 48 | if ih > 0: 49 | ua = float( 50 | (boxes[n, 2] - boxes[n, 0] + 1) * 51 | (boxes[n, 3] - boxes[n, 1] + 1) + 52 | box_area - iw * ih 53 | ) 54 | overlaps[n, k] = iw * ih / ua 55 | return overlaps -------------------------------------------------------------------------------- /evaluation.py: -------------------------------------------------------------------------------- 1 | #-*-coding:utf-8-*- 2 | from __future__ import division 3 | """ 4 | WiderFace evaluation code 5 | author: wondervictor 6 | mail: tianhengcheng@gmail.com 7 | copyright@wondervictor 8 | """ 9 | import os 10 | import tqdm 11 | import pickle 12 | import argparse 13 | import numpy as np 14 | from scipy.io import loadmat 15 | from bbox import bbox_overlaps 16 | from IPython import embed 17 | 18 | 19 | def get_gt_boxes(gt_dir): 20 | """ gt dir: (wider_face_val.mat, wider_easy_val.mat, wider_medium_val.mat, wider_hard_val.mat)""" 21 | 22 | gt_mat = loadmat(os.path.join(gt_dir, 'wider_face_val.mat')) 23 | hard_mat = loadmat(os.path.join(gt_dir, 'wider_hard_val.mat')) 24 | medium_mat = loadmat(os.path.join(gt_dir, 'wider_medium_val.mat')) 25 | easy_mat = loadmat(os.path.join(gt_dir, 'wider_easy_val.mat')) 26 | 27 | facebox_list = gt_mat['face_bbx_list'] 28 | event_list = gt_mat['event_list'] 29 | file_list = gt_mat['file_list'] 30 | 31 | hard_gt_list = hard_mat['gt_list'] 32 | medium_gt_list = medium_mat['gt_list'] 33 | easy_gt_list = easy_mat['gt_list'] 34 | 35 | return facebox_list, event_list, file_list, hard_gt_list, medium_gt_list, easy_gt_list 36 | 37 | 38 | def get_gt_boxes_from_txt(gt_path, cache_dir): 39 | 40 | cache_file = os.path.join(cache_dir, 'gt_cache.pkl') 41 | if os.path.exists(cache_file): 42 | f = open(cache_file, 'rb') 43 | boxes = pickle.load(f) 44 | f.close() 45 | return boxes 46 | 47 | f = open(gt_path, 'r') 48 | state = 0 49 | lines = f.readlines() 50 | lines = list(map(lambda x: x.rstrip('\r\n'), lines)) 51 | boxes = {} 52 | f.close() 53 | current_boxes = [] 54 | current_name = None 55 | for line in lines: 56 | if state == 0 and '--' in line: 57 | state = 1 58 | current_name = line 59 | continue 60 | if state == 1: 61 | state = 2 62 | continue 63 | 64 | if state == 2 and '--' in line: 65 | state = 1 66 | boxes[current_name] = np.array(current_boxes).astype('float32') 67 | current_name = line 68 | current_boxes = [] 69 | continue 70 | 71 | if state == 2: 72 | box = [float(x) for x in line.split(' ')[:4]] 73 | current_boxes.append(box) 74 | continue 75 | 76 | f = open(cache_file, 'wb') 77 | pickle.dump(boxes, f) 78 | f.close() 79 | return boxes 80 | 81 | 82 | def read_pred_file(filepath): 83 | 84 | with open(filepath, 'r') as f: 85 | lines = f.readlines() 86 | img_file = lines[0].rstrip('\n\r') 87 | lines = lines[2:] 88 | 89 | boxes = np.array(list(map(lambda x: [float(a) for a in x.rstrip('\r\n').split(' ')], lines))).astype('float') 90 | return img_file.split('/')[-1], boxes 91 | 92 | 93 | def get_preds(pred_dir): 94 | events = os.listdir(pred_dir) 95 | boxes = dict() 96 | pbar = tqdm.tqdm(events) 97 | 98 | for event in pbar: 99 | pbar.set_description('Reading Predictions ') 100 | event_dir = os.path.join(pred_dir, event) 101 | event_images = os.listdir(event_dir) 102 | current_event = dict() 103 | for imgtxt in event_images: 104 | imgname, _boxes = read_pred_file(os.path.join(event_dir, imgtxt)) 105 | current_event[imgname.rstrip('.jpg')] = _boxes 106 | boxes[event] = current_event 107 | return boxes 108 | 109 | 110 | def norm_score(pred): 111 | """ norm score 112 | pred {key: [[x1,y1,x2,y2,s]]} 113 | """ 114 | 115 | max_score = 0 116 | min_score = 1 117 | 118 | for _, k in pred.items(): 119 | for _, v in k.items(): 120 | if len(v) == 0: 121 | continue 122 | _min = np.min(v[:, -1]) 123 | _max = np.max(v[:, -1]) 124 | max_score = max(_max, max_score) 125 | min_score = min(_min, min_score) 126 | 127 | diff = max_score - min_score 128 | for _, k in pred.items(): 129 | for _, v in k.items(): 130 | if len(v) == 0: 131 | continue 132 | v[:, -1] = (v[:, -1] - min_score)/diff 133 | 134 | 135 | def image_eval(pred, gt, ignore, iou_thresh): 136 | """ single image evaluation 137 | pred: Nx5 138 | gt: Nx4 139 | ignore: 140 | """ 141 | _pred = pred.copy() 142 | _gt = gt.copy() 143 | pred_recall = np.zeros(_pred.shape[0]) 144 | recall_list = np.zeros(_gt.shape[0]) 145 | proposal_list = np.ones(_pred.shape[0]) 146 | 147 | _pred[:, 2] = _pred[:, 2] + _pred[:, 0] 148 | _pred[:, 3] = _pred[:, 3] + _pred[:, 1] 149 | _gt[:, 2] = _gt[:, 2] + _gt[:, 0] 150 | _gt[:, 3] = _gt[:, 3] + _gt[:, 1] 151 | 152 | overlaps = bbox_overlaps(_pred[:, :4], _gt) 153 | 154 | for h in range(_pred.shape[0]): 155 | 156 | gt_overlap = overlaps[h] 157 | max_overlap, max_idx = gt_overlap.max(), gt_overlap.argmax() 158 | if max_overlap >= iou_thresh: 159 | if ignore[max_idx] == 0: 160 | recall_list[max_idx] = -1 161 | proposal_list[h] = -1 162 | elif recall_list[max_idx] == 0: 163 | recall_list[max_idx] = 1 164 | 165 | r_keep_index = np.where(recall_list == 1)[0] 166 | pred_recall[h] = len(r_keep_index) 167 | return pred_recall, proposal_list 168 | 169 | 170 | def img_pr_info(thresh_num, pred_info, proposal_list, pred_recall): 171 | pr_info = np.zeros((thresh_num, 2)).astype('float') 172 | for t in range(thresh_num): 173 | 174 | thresh = 1 - (t+1)/thresh_num 175 | r_index = np.where(pred_info[:, 4] >= thresh)[0] 176 | if len(r_index) == 0: 177 | pr_info[t, 0] = 0 178 | pr_info[t, 1] = 0 179 | else: 180 | r_index = r_index[-1] 181 | p_index = np.where(proposal_list[:r_index+1] == 1)[0] 182 | pr_info[t, 0] = len(p_index) 183 | pr_info[t, 1] = pred_recall[r_index] 184 | return pr_info 185 | 186 | 187 | def dataset_pr_info(thresh_num, pr_curve, count_face): 188 | _pr_curve = np.zeros((thresh_num, 2)) 189 | for i in range(thresh_num): 190 | _pr_curve[i, 0] = pr_curve[i, 1] / pr_curve[i, 0] 191 | _pr_curve[i, 1] = pr_curve[i, 1] / count_face 192 | return _pr_curve 193 | 194 | 195 | def voc_ap(rec, prec): 196 | 197 | # correct AP calculation 198 | # first append sentinel values at the end 199 | mrec = np.concatenate(([0.], rec, [1.])) 200 | mpre = np.concatenate(([0.], prec, [0.])) 201 | 202 | # compute the precision envelope 203 | for i in range(mpre.size - 1, 0, -1): 204 | mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) 205 | 206 | # to calculate area under PR curve, look for points 207 | # where X axis (recall) changes value 208 | i = np.where(mrec[1:] != mrec[:-1])[0] 209 | 210 | # and sum (\Delta recall) * prec 211 | ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) 212 | return ap 213 | 214 | 215 | def evaluation(pred, gt_path, all, iou_thresh=0.5): 216 | pred = get_preds(pred) 217 | norm_score(pred) 218 | facebox_list, event_list, file_list, hard_gt_list, medium_gt_list, easy_gt_list = get_gt_boxes(gt_path) 219 | event_num = len(event_list) 220 | thresh_num = 1000 221 | settings = ['easy', 'medium', 'hard'] 222 | setting_gts = [easy_gt_list, medium_gt_list, hard_gt_list] 223 | 224 | if not all: 225 | aps = [] 226 | for setting_id in range(3): 227 | # different setting 228 | gt_list = setting_gts[setting_id] 229 | count_face = 0 230 | pr_curve = np.zeros((thresh_num, 2)).astype('float') 231 | # [hard, medium, easy] 232 | pbar = tqdm.tqdm(range(event_num)) # 61 233 | error_count = 0 234 | for i in pbar: 235 | pbar.set_description('Processing {}'.format(settings[setting_id])) 236 | event_name = str(event_list[i][0][0]) 237 | img_list = file_list[i][0] 238 | pred_list = pred[event_name] 239 | sub_gt_list = gt_list[i][0] 240 | print("shape of sub_gt_list is: ",sub_gt_list.shape) 241 | gt_bbx_list = facebox_list[i][0] 242 | 243 | for j in range(len(img_list)): 244 | try: 245 | pred_info = pred_list[str(img_list[j][0][0])] 246 | except: 247 | error_count+=1 248 | continue 249 | 250 | gt_boxes = gt_bbx_list[j][0].astype('float') 251 | keep_index = sub_gt_list[j][0] 252 | count_face += len(keep_index) 253 | if len(gt_boxes) == 0 or len(pred_info) == 0: 254 | continue 255 | ignore = np.zeros(gt_boxes.shape[0]) 256 | if len(keep_index) != 0: 257 | ignore[keep_index-1] = 1 258 | pred_recall, proposal_list = image_eval(pred_info, gt_boxes, ignore, iou_thresh) 259 | 260 | _img_pr_info = img_pr_info(thresh_num, pred_info, proposal_list, pred_recall) 261 | 262 | pr_curve += _img_pr_info 263 | print("error_count is: ",error_count) 264 | pr_curve = dataset_pr_info(thresh_num, pr_curve, count_face) 265 | 266 | propose = pr_curve[:, 0] 267 | recall = pr_curve[:, 1] 268 | 269 | ap = voc_ap(recall, propose) 270 | aps.append(ap) 271 | 272 | print("==================== Results ====================") 273 | print("Easy Val AP: {}".format(aps[0])) 274 | print("Medium Val AP: {}".format(aps[1])) 275 | print("Hard Val AP: {}".format(aps[2])) 276 | print("=================================================") 277 | else: 278 | aps = [] 279 | # different setting 280 | count_face = 0 281 | pr_curve = np.zeros((thresh_num, 2)).astype('float') # control calcultate how many samples 282 | # [hard, medium, easy] 283 | pbar = tqdm.tqdm(range(event_num)) 284 | error_count = 0 285 | for i in pbar: 286 | pbar.set_description('Processing {}'.format("all")) 287 | # print("event_list is: ",event_list) 288 | event_name = str(event_list[i][0][0]) # '0--Parade', '1--Handshaking' 289 | img_list = file_list[i][0] 290 | pred_list = pred[event_name] # 每个文件夹的所有检测结果 291 | sub_gt_list = [ setting_gts[0][i][0], setting_gts[1][i][0], setting_gts[2][i][0] ] 292 | 293 | gt_bbx_list = facebox_list[i][0] 294 | for j in range(len(img_list)): 295 | try: 296 | pred_info = pred_list[str(img_list[j][0][0])] # # str(img_list[j][0][0] 是每个folder下面的图片名字 297 | except: 298 | error_count+=1 299 | continue 300 | 301 | gt_boxes = gt_bbx_list[j][0].astype('float') 302 | temp_i = [] 303 | for ii in range(3): 304 | if len(sub_gt_list[ii][j][0])!=0: 305 | temp_i.append(ii) 306 | if len(temp_i)!=0: 307 | keep_index = np.concatenate(tuple([sub_gt_list[xx][j][0] for xx in temp_i])) 308 | else: 309 | keep_index = [] 310 | count_face += len(keep_index) 311 | 312 | if len(gt_boxes) == 0 or len(pred_info) == 0: 313 | continue 314 | ignore = np.zeros(gt_boxes.shape[0]) # # no ignore 315 | if len(keep_index) != 0: 316 | ignore[keep_index-1] = 1 317 | pred_recall, proposal_list = image_eval(pred_info, gt_boxes, ignore, iou_thresh) 318 | 319 | _img_pr_info = img_pr_info(thresh_num, pred_info, proposal_list, pred_recall) 320 | 321 | pr_curve += _img_pr_info 322 | 323 | pr_curve = dataset_pr_info(thresh_num, pr_curve, count_face) 324 | 325 | propose = pr_curve[:, 0] 326 | recall = pr_curve[:, 1] 327 | 328 | ap = voc_ap(recall, propose) 329 | aps.append(ap) 330 | 331 | print("==================== Results ====================") 332 | print("All Val AP: {}".format(aps[0])) 333 | print("=================================================") 334 | 335 | 336 | 337 | if __name__ == '__main__': 338 | 339 | parser = argparse.ArgumentParser() 340 | parser.add_argument('-p', '--pred') 341 | parser.add_argument('-g', '--gt', default='./ground_truth') 342 | parser.add_argument('--all', help='if test all together', action='store_true') 343 | 344 | args = parser.parse_args() 345 | evaluation(args.pred, args.gt, args.all) 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | WiderFace evaluation code 3 | author: wondervictor 4 | mail: tianhengcheng@gmail.com 5 | copyright@wondervictor 6 | """ 7 | 8 | from distutils.core import setup, Extension 9 | from Cython.Build import cythonize 10 | import numpy 11 | 12 | package = Extension('bbox', ['box_overlaps.pyx'], include_dirs=[numpy.get_include()]) 13 | setup(ext_modules=cythonize([package])) 14 | --------------------------------------------------------------------------------