└── benchmark.py


/benchmark.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | from __future__ import division
  3 | from __future__ import print_function
  4 | import cv2
  5 | import os
  6 | import numpy as np
  7 | import time
  8 | import pickle
  9 | 
 10 | test_path = "../coco/val2014"
 11 | save_path = "../coco/val2014/predictions"
 12 | 
 13 | test_classes = ['person', 'bicycle', 'car', 'motorbike', 'aeroplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'sofa', 'pottedplant', 'bed', 'diningtable', 'toilet', 'tvmonitor', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']
 14 | 
 15 | #topk
 16 | topk = 40
 17 | 
 18 | 
 19 | images_folder = os.path.join(test_path, "images")
 20 | annotations_folder = os.path.join(test_path, "annotations")
 21 | 
 22 | images = os.listdir(images_folder)
 23 | 
 24 | def detect(image_name):
 25 | 	prediction_path = os.path.join(save_path, image_name)
 26 | 	with open(prediction_path, "rb") as f:
 27 | 		predictions = pickle.load(f)
 28 | 	return predictions[1]
 29 | 
 30 | def sign(x):
 31 |     if x>=0: return 1
 32 |     else: return -1
 33 | 
 34 | def get_IoU(boxA, boxB):
 35 |     classA, confidenceA, positionA = boxA
 36 |     classB, confidenceB, positionB = boxB
 37 |     if classA != classB: return 0
 38 | 
 39 |     xA, yA, wA, hA = positionA
 40 |     xB, yB, wB, hB = positionB
 41 |     
 42 |     tx = sign(xB - xA)
 43 |     ty = sign(yB - yA)
 44 | 
 45 |     Ix = (wA/2 + wB/2 + tx*(xA - xB))
 46 |     Iy = (hA/2 + hB/2 + ty*(yA - yB))
 47 | 
 48 |     if Ix < 0 or Iy < 0:    I = 0
 49 |     else:                   I = Ix * Iy
 50 | 
 51 |     U = wA*hA + wB*hB - I
 52 |     return I/U
 53 | 
 54 | TPs = np.zeros(topk)
 55 | ABs = np.zeros(topk)
 56 | PBs = np.zeros(topk)
 57 | 
 58 | n = 0
 59 | for image_name in images:
 60 |     n += 1
 61 |     print("{} Image {}".format(n, image_name))
 62 |     image_path = os.path.join(images_folder, image_name)
 63 |     annotation_path = os.path.join(annotations_folder, image_name[:-4] + ".txt")
 64 |     if not os.path.exists(annotation_path): continue
 65 | 
 66 |     img = cv2.imread(image_path)
 67 | 
 68 |     with open(annotation_path, "r") as f:
 69 |         annotation_raw = f.read()
 70 |     annotation = []
 71 |     for box_raw in annotation_raw.split("\n"):
 72 |         pack = box_raw.split(" ")
 73 |         try: 
 74 |             class_name, x, y, w, h = pack[0], pack[1], pack[2], pack[3], pack[4]
 75 |         except IndexError:
 76 |             continue
 77 |         class_name = test_classes[int(class_name)]
 78 |         x = img.shape[1]*float(x)
 79 |         y = img.shape[0]*float(y)
 80 |         w = img.shape[1]*float(w)
 81 |         h = img.shape[0]*float(h)
 82 |         box = (class_name, 1, (x, y, w, h))
 83 |         annotation.append(box)
 84 | 
 85 |     prediction = detect(image_name)
 86 |     if len(prediction) == 0: continue
 87 | 
 88 | 
 89 |     #print("predicted {} over {} images".format(len(prediction), len(annotation)))
 90 | 
 91 | 
 92 |     TP = np.zeros(topk)
 93 | 
 94 | 
 95 |     for k in range(topk):
 96 |       	if k < len(prediction):
 97 |             IoU_list = []
 98 |             for box in annotation:
 99 |                 pred_box = prediction[k]
100 |                 IoU = get_IoU(box, pred_box)
101 |                 IoU_list.append(IoU)
102 |             if max(IoU_list) > 0.5:
103 |                 TP[k] = 1 + TP[k-1]
104 |             else:
105 |                 TP[k] = 0 + TP[k-1]
106 |         else:
107 |             TP[k] = TP[len(prediction)-1]
108 | 
109 |     PB = np.zeros(topk)
110 |     if topk <= len(prediction):
111 |         for k in range(topk):
112 |             PB[k] = k+1
113 |     else:
114 |         for k in range(len(prediction)):
115 |             PB[k] = k+1
116 |         for k in range(len(prediction), topk):
117 |             PB[k] = len(prediction)
118 | 
119 |     AB = np.zeros(topk)
120 |     AB += len(annotation)
121 | 
122 |                 #k top k predictions
123 |     TPs += TP   #true positive
124 |     PBs += PB   #predicted boxes
125 |     ABs += AB   #annotated boxes
126 |     #print("TP {}\nPB {}\nAB {}".format(TP, PB, AB))
127 |     #import pdb
128 |     #pdb.set_trace()
129 |     #print("TPs {}\nPBs {}\n ABs{}".format(TPs, PBs, ABs))
130 | 
131 |     precision   = TPs/PBs
132 |     recall      = TPs/ABs
133 | 
134 |     #print("precision    {}".format(precision))
135 |     #print( "recall      {}".format(recall))
136 | 
137 | import matplotlib.pyplot as plt
138 | 
139 | recall 		= np.hstack((0,recall,1))
140 | precision 	= np.hstack((1, precision, 0))
141 | 
142 | mAP = 0
143 | for i in range(len(recall) -1):
144 |     dw = recall[i+1] - recall[i]
145 |     h = (precision[i+1] + precision[i])/2
146 |     mAP += h*dw
147 | 
148 | print(mAP)
149 | 
150 | plt.plot(recall, precision, "ro")
151 | plt.show()
152 | 


--------------------------------------------------------------------------------