├── figures
    ├── iou.jpg
    ├── concepts.jpg
    ├── different_iou.jpg
    └── precision_recall_curve.jpg
├── example.py
├── utils.py
├── example_data_generator.py
├── README.md
└── ap.py


/figures/iou.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Ao-Lee/AveragePrecision/HEAD/figures/iou.jpg


--------------------------------------------------------------------------------
/figures/concepts.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Ao-Lee/AveragePrecision/HEAD/figures/concepts.jpg


--------------------------------------------------------------------------------
/figures/different_iou.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Ao-Lee/AveragePrecision/HEAD/figures/different_iou.jpg


--------------------------------------------------------------------------------
/figures/precision_recall_curve.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Ao-Lee/AveragePrecision/HEAD/figures/precision_recall_curve.jpg


--------------------------------------------------------------------------------
/example.py:
--------------------------------------------------------------------------------
1 | from example_data_generator import GenFakeData
2 | from ap import AveragePrecisionOnImages
3 | 
4 | gts, predictions, num_gt = GenFakeData(size=20)
5 | ap, _, _ = AveragePrecisionOnImages(gts, predictions, num_gt, min_overlap=0.5, validate_input=True)
6 | print(ap)
7 | 
8 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | 
 3 | '''
 4 | Compute IoU between detect box and ground truth boxes
 5 | 
 6 | Parameters:
 7 |     box:            detection box, numpy array, shape (4,): x1, y1, x2, y2
 8 |     
 9 |     boxes:          ground truth boxes, shape (n, 4): x1, y1, x2, y2
10 | 
11 | Returns:
12 |     iou:            numpy array, shape (n, )
13 | '''
14 | def IoU(box, boxes):
15 |     
16 |     box_area = (box[2] - box[0] + 1) * (box[3] - box[1] + 1)
17 |     area = (boxes[:, 2] - boxes[:, 0] + 1) * (boxes[:, 3] - boxes[:, 1] + 1)
18 |     xx1 = np.maximum(box[0], boxes[:, 0])
19 |     yy1 = np.maximum(box[1], boxes[:, 1])
20 |     xx2 = np.minimum(box[2], boxes[:, 2])
21 |     yy2 = np.minimum(box[3], boxes[:, 3])
22 |     
23 |     # compute the width and height of the bounding box
24 |     w = np.maximum(0, xx2 - xx1 + 1)
25 |     h = np.maximum(0, yy2 - yy1 + 1)
26 | 
27 |     inter = w * h
28 |     iou = inter / (box_area + area - inter)
29 |     return iou


--------------------------------------------------------------------------------
/example_data_generator.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | from utils import IoU
 3 | 
 4 | def _GenRandomBoxes(size=10):
 5 |     def _GenRandomBox():
 6 |         rand = np.random.randint(low=0, high=101, size=4)
 7 |         box = np.array([rand[0], rand[1], rand[0]+rand[2], rand[1]+rand[3]])
 8 |         box[box<0]=0
 9 |         return box
10 | 
11 |     arrays = []
12 |     for _ in range(size):
13 |         arrays.append(_GenRandomBox())
14 |     return np.stack(arrays)
15 |     
16 | def _GenPredictionBox(box, fp=False):
17 |     w = box[2] - box[0]
18 |     h = box[3] - box[1]
19 |     dw = int(w*0.7) if fp else int(w*0.3)
20 |     dh = int(h*0.7) if fp else int(h*0.3)
21 |     rand_x = np.random.randint(low=-dw, high=dw+1, size=2)
22 |     rand_y = np.random.randint(low=-dh, high=dh+1, size=2)
23 |     new_box = [box[0]+rand_x[0], box[1]+rand_y[0], box[2]+rand_x[1], box[3]+rand_y[1]]
24 | 
25 |     new_box = np.array(new_box)
26 |     new_box[new_box<0]=0
27 |     if new_box[2]<new_box[0]: new_box[2]=new_box[0]+1
28 |     if new_box[3]<new_box[1]: new_box[3]=new_box[1]+1
29 | 
30 |     return new_box
31 | 
32 | def _GenFileNames(size=100):
33 |     result = []
34 |     while len(result) < size:
35 |         n = np.random.randint(low=0, high=10000)
36 |         s = str(n).zfill(4)
37 |         s+='.jpg'
38 |         if s not in result:
39 |             result.append(s)
40 |     return result
41 |     
42 | def _GenImgData(file_id):
43 |     num_gt = np.random.randint(low=1, high=6)
44 |     gt_boxes = _GenRandomBoxes(size=num_gt)
45 |     gt_key = file_id
46 |     gt_value = gt_boxes
47 | 
48 |     prediction = []
49 |     for idx in range(num_gt):
50 |         if np.random.random() < 0.2:
51 |             continue
52 |         fp = True if np.random.random()<0.2 else False
53 |         pred_box = _GenPredictionBox(gt_boxes[idx], fp)
54 |         iou = np.max(IoU(pred_box, gt_boxes))
55 |         dc = (np.random.random()-0.5)/3
56 |         confidence = iou + dc
57 |         result = {'bbox':pred_box, 'confidence':confidence, 'file_id':file_id}
58 |         prediction.append(result)
59 |         
60 |     return gt_key, gt_value, prediction, num_gt
61 |     
62 | def GenFakeData(size=10):
63 |     files = _GenFileNames(size=size)
64 |     gts = {}
65 |     predictions = []
66 |     num_gt = 0
67 |     for file in files:
68 |         gt_key, gt_value, prediction, num_gt_this_img = _GenImgData(file)
69 |         gts[gt_key] = gt_value
70 |         predictions += prediction
71 |         num_gt += num_gt_this_img
72 |     return gts, predictions, num_gt


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## Some important concepts
 2 | ### Intersection over Union (IoU) 
 3 | Intersection over Union is simply a ratio. In the numerator we compute the area of overlap between the predicted bounding box and the ground-truth bounding box. The denominator is the area of union, or more simply, the area encompassed by both the predicted bounding box and the ground-truth bounding box.
 4 | ![Alt text](https://github.com/Ao-Lee/AveragePrecision/raw/master/figures/iou.jpg)
 5 | In the above figure I have included examples of good and bad IoU scores. As you can see, predicted bounding boxes that heavily overlap with the ground-truth bounding boxes have higher scores than those with less overlap. This makes Intersection over Union an excellent metric for evaluating custom object detectors.
 6 | ![Alt text](https://github.com/Ao-Lee/AveragePrecision/raw/master/figures/different_iou.jpg)
 7 | 
 8 | ### TP (true positive):
 9 | Correctly predicted positive examples. In the object detection context, it means a correct detection. The IOU between the detected box and any ground truth is greater than the threshold (in the PASCAL VOC Challenge, this threshold is set to 0.5. In this library, APIs are provided to change this threshold)
10 | 
11 | ### FP (false positive):
12 | Wrongly predicted negative examples. In the object detection context, it means a wrong detection. The IOU between the detected box and all ground truth bounding boxes is less than the threshold.
13 | 
14 | ### FN (false negative): 
15 | Wrongly predicted positive examples. In the object detection context, it means a ground truth object is not detected
16 | 
17 | ### TN (true negative):
18 | Correctly predicted negative examples. In the object detection context, it means non-object bounding boxes which are not detected by the model. Since there are infinite possible non-object bounding boxes in an image, TN is never used in object detection.
19 | 
20 | 
21 | 
22 | ### Precision 
23 | Precision = TP / (TP + FP) = TP / number of detected bounding boxes
24 | 
25 | ### Recall
26 | Recall = TP / (TP + FN) = TP / number of ground truth bounding boxes
27 | 
28 | ## How is Average Precision (AP) computed?
29 | Computing AP for a particular object detection pipeline is essentially a three step process:
30 | 1. Compute all the precisions which the confidence of the model varies.
31 | 2. Compute all the recalls which the confidence of the model varies.
32 | 3. Draw a precision-recall curve for all precisions and recalls, AP is the area under this Precision-Recall curve. In the implementation, an Interpolated precision-recall curve is used, as shown in the figure.
33 | ![Alt text](https://github.com/Ao-Lee/AveragePrecision/raw/master/figures/precision_recall_curve.jpg)
34 | 
35 | ## APIs
36 | in order to compute average protection, you need to provoide three piece of information, the ground truth Information, the predicton information and the total number of ground truth objects.
37 | ### Ground truth Information
38 | a dictionary of files, representing the ground truth information. Each dictionary contains an image id and ground truth bounding boxes of that image, for example:
39 | ```python
40 | {'1.jpg':array([[95, 95, 188, 176]]), '2.jpg': array([[21, 94, 104, 157], [ 9, 88, 56, 178]]}
41 | ```                
42 | ### Prediction Information
43 | a list, each element in the list representing a prediction, for example:
44 | ```python
45 | [
46 | 	{'bbox': array([ 83, 73, 156, 101]), 'confidence': 0.53, 'file_id': '0969.jpg'},
47 | 	{'bbox': array([ 30, 71, 122, 98]), 'confidence': 0.83, 'file_id': '1440.jpg'}
48 | ]
49 | ```
50 | ### Total number of ground truth objects
51 | The total number of ground truth objects must be provided to correctly compute recall, as discussed above.
52 | 
53 | ## How to run the example?
54 | Run example.py, this is a complete example. Note that it is able to adjust the IOU threshold.
55 | 
56 | ## Common questions:
57 | * __What if many detections have high IOUs to the same ground truth object? Are those detections all considered as true positive?__
58 | No. In this case, only one detection is counted as TP, other detections are false positives
59 | 
60 | * __I have a multiclass detection scenario, and how to compute MAP among all classes?__
61 | provoide the ground truth, the predicton and the total number of ground truth objects for each paticular class, compute AP for each class, and average the results.
62 | 
63 | 


--------------------------------------------------------------------------------
/ap.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | from utils import IoU
  3 | 
  4 | '''
  5 | Compute average precision given each result of prediction and the confidence
  6 | 
  7 | Parameters:
  8 |     prediction:     a list of dictionary. 
  9 |                     for example [{'TP':True, 'confidence':0.65}, {'TP':False, 'confidence':0.4}]
 10 |                     set TP to True if the prediction is true positive
 11 |                     set TP to False if the prediction is false positive
 12 |                     
 13 |     num_gt_example: int, number of ground truth examples
 14 |     
 15 |     validate:       bool, turn it on to check the inputs
 16 | 
 17 | Returns:
 18 |     ap:             int, average precision. note that we assume the inputs belongs to the same class
 19 |                     it is possible to utilize this functionality to compute mean average precision via multiple classes
 20 |                     
 21 |     recall:         recalls computed over different confidence threshold, numpy array of shape (N,)
 22 |     
 23 |     precision:      precision computed over different confidence threshold, numpy array of shape (N,) 
 24 |     
 25 | '''
 26 | def VocAveragePrecision(prediction, num_gt_example, validate=False):
 27 |     def CheckInput(prediction, num_gt_example):
 28 |         num_true_positive = 0
 29 |         for d in prediction:
 30 |             keys = list(d.keys())
 31 |             assert 'TP' in keys and 'confidence' in keys and type(d['TP']) is bool
 32 |             assert len(keys) == 2
 33 |             if d['TP']: num_true_positive +=1
 34 |         assert num_true_positive <= num_gt_example
 35 |         return
 36 |     
 37 |     if validate: CheckInput(prediction, num_gt_example)
 38 |     
 39 |     prediction.sort(key=lambda x:x['confidence'], reverse=True)
 40 |     prediction = [info['TP'] for info in prediction]
 41 |     prediction = np.array(prediction)
 42 |     
 43 |     tp = (prediction==True).astype(np.int)
 44 |     fp = (prediction==False).astype(np.int)
 45 |     
 46 |     cumulative_tp = np.cumsum(tp)
 47 |     cumulative_fp = np.cumsum(fp)
 48 |     
 49 |     # precision_scores = cumulative_tp / (np.arange(len(prediction)) + 1)
 50 |     precision = cumulative_tp / (cumulative_tp + cumulative_fp)
 51 |     recall = cumulative_tp / num_gt_example
 52 |     
 53 |     '''
 54 |     metric      rank(confidence)    extream_value
 55 |     precision   inf                 1
 56 |     precision   -inf                0
 57 |     recall      inf                 0
 58 |     recall      -inf                1
 59 |     '''
 60 |     precision = np.array([1.0] + list(precision) + [0.0])
 61 |     recall = np.array([0.0] + list(recall) + [1.0])
 62 |     
 63 |     for i in range(len(precision)-2, -1, -1):
 64 |         precision[i] = max(precision[i], precision[i+1])
 65 | 
 66 |     i_list = []
 67 |     for i in range(1, len(recall)):
 68 |         if recall[i] != recall[i-1]:
 69 |             i_list.append(i)
 70 |     
 71 |     ap = 0.0
 72 |     for i in i_list:
 73 |         ap += ((recall[i]-recall[i-1])*precision[i])
 74 |     return ap, recall, precision
 75 |    
 76 | '''
 77 | Compute average precision given predictions of images
 78 | 
 79 | Parameters:     
 80 |     gt_info:        a dictionary of files, representing the ground truth infomation 
 81 |                     of each image, for example
 82 |                         {'1.jpg':bboxes_01, '2.jpg':bboxes_02}
 83 |                     each key represents each image file, and the bboxes_01 is 
 84 |                     the numpy array of shape (K, 4), representing K ground truth
 85 |                     objects.
 86 |                     
 87 |     predictions_info:
 88 |                     a list, each element in the list representing a prediction, for example: [prediction01, prediction02]
 89 |                     each prediction is a dictionary, for example
 90 |                         {'bbox': array([165., 214., 230., 255.]), 'confidence':0.85, 'file_id':'2.jpg'}
 91 |                     bbox is numpy array of shape (4, )
 92 |                     file_id should be consistent with the files in gt_info
 93 |                     
 94 |     num_gt_example: int, number of ground truth examples
 95 |     
 96 |     validate_input: bool, turn it on to check the inputs
 97 |     
 98 |     min_overlap:    should be in (0, 1) default value is 0.5 (defined in the PASCAL VOC2012 challenge)
 99 |                     
100 | Returns:
101 |     ap:             int, average precision. note that we assume the inputs belongs to the same class
102 |                     it is possible to utilize this functionality to compute mean average precision via multiple classes
103 |                     
104 |     recall:         recalls computed over different confidence threshold, numpy array of shape (N,)
105 |     
106 |     precision:      precision computed over different confidence threshold, numpy array of shape (N,) 
107 |     
108 | '''
109 | def AveragePrecisionOnImages(gt_info, predictions_info, num_gt_example, min_overlap=0.5, validate_input=False):
110 |     result = []
111 |     used_info = {}
112 |     # set 'used' tag for each ground truch bbox
113 |     for file_id in gt_info:
114 |         boxes = gt_info[file_id]
115 |         size = boxes.shape[0]
116 |         used = np.zeros(shape=[size]).astype(np.bool)
117 |         used_info[file_id] = used
118 | 
119 |     for idx, prediction in enumerate(predictions_info):
120 |         file_id = prediction['file_id']
121 |         box_pred = prediction['bbox']
122 | 
123 |         boxes_gt = gt_info[file_id]
124 |         iou = IoU(box_pred, boxes_gt)
125 |         idx = iou.argmax()
126 |         used = used_info[file_id][idx]
127 |         confidence = prediction['confidence']
128 | 
129 |         if iou[idx] >= min_overlap and not used:
130 |             result.append({'TP':True, 'confidence':confidence})
131 |             used_info[file_id][idx] = True
132 |         else:
133 |             result.append({'TP':False, 'confidence':confidence})
134 | 
135 |     ap, recall, precision = VocAveragePrecision(result, num_gt_example, validate=validate_input)
136 |     return ap, recall, precision
137 |     
138 |     
139 |     
140 |     
141 |     


--------------------------------------------------------------------------------