├── detection_eval ├── example │ ├── PR_curve.png │ ├── example_result_format.txt │ ├── anno_example2.txt │ ├── anno_example1.txt │ └── eval_result.txt ├── README.md └── detection_eval.py ├── README.md ├── ensemble └── README.md └── annotaion_files_process ├── README.md ├── imglab_xml2txt.py ├── txt2imglab_xml.py ├── VOC_XML2txt.py ├── pascal2coco.py └── txt2VOC_XML.py /detection_eval/example/PR_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seefun/object_detection_utils/HEAD/detection_eval/example/PR_curve.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # object_detection_utils 2 | data augmentation; annotation format transform; evaluation methods; TTA(test time augmentation); model fusion 3 | -------------------------------------------------------------------------------- /ensemble/README.md: -------------------------------------------------------------------------------- 1 | TODO: 2 | 3 | 参考 https://github.com/ahrnbom/ensemble-objdet 4 | 5 | - [ ] TTA 6 | 7 | - [ ] Ensemble 8 | 9 | - [ ] SoftNMS 10 | -------------------------------------------------------------------------------- /detection_eval/example/example_result_format.txt: -------------------------------------------------------------------------------- 1 | Images/01.jpg 2 | 2 3 | 99 102 149 152 0.95 4 | 23 23 55 55 0.55 5 | Image/02.jpg 6 | 5 7 | 99 102 149 152 0.95 8 | 23 23 55 55 0.77 9 | 19 12 49 52 0.75 10 | 66 66 77 77 0.55 11 | 324 422 534 523 0.15 12 | Images/03.jpg 13 | 1 14 | 23 23 55 55 0.22 15 | 16 | -------------------------------------------------------------------------------- /detection_eval/example/anno_example2.txt: -------------------------------------------------------------------------------- 1 | Image/1.jpg 1571 477 1653 559 214 375 262 423 442 310 496 364 498 308 536 346 530 313 592 375 2 | Image/2.jpg 1305 736 1443 874 3 | Image/3.jpg 607 340 687 420 782 438 864 520 4 | Image/4.jpg 734 256 796 318 618 282 654 318 541 308 583 350 1722 654 1796 728 179 394 223 438 5 | -------------------------------------------------------------------------------- /annotaion_files_process/README.md: -------------------------------------------------------------------------------- 1 | # Convert Annotation Format 2 | 3 | this code can only be used in the single type object detection task. 4 | 5 | STEP1 (not necessary) : use "imglab_xml2txt" to convert the XML file generated by imglab(an annotation toolbox from [dlib](http://dlib.net/)) to txt file. 6 | 7 | eg. in 'annotation.txt' 8 | 9 | ``` 10 | image1.jpg x1 y1 x2 y2 11 | image1.jpg x1 y1 x2 y2 12 | image2.jpg x1 y1 x2 y2 13 | image3.jpg x1 y1 x2 y2 14 | image3.jpg x1 y1 x2 y2 15 | image3.jpg x1 y1 x2 y2 16 | ...... 17 | ``` 18 | 19 | STEP2 : use txt file generated by step1, convert it to pascal voc format (.XML). 20 | -------------------------------------------------------------------------------- /detection_eval/example/anno_example1.txt: -------------------------------------------------------------------------------- 1 | Image/1.jpg 1571 477 1653 559 2 | Image/1.jpg 214 375 262 423 3 | Image/1.jpg 442 310 496 364 4 | Image/1.jpg 498 308 536 346 5 | Image/1.jpg 530 313 592 375 6 | Image/1.jpg 916 600 1014 698 7 | Image/1.jpg 1199 783 1329 913 8 | Image/1.jpg 1402 669 1516 783 9 | Image/1.jpg 1644 611 1724 691 10 | Image/1.jpg 1730 562 1810 642 11 | Image/2.jpg 1305 736 1443 874 12 | Image/3.jpg 607 340 687 420 13 | Image/3.jpg 782 438 864 520 14 | Image/4.jpg 734 256 796 318 15 | Image/4.jpg 618 282 654 318 16 | Image/4.jpg 541 308 583 350 17 | Image/4.jpg 1722 654 1796 728 18 | Image/4.jpg 179 394 223 438 19 | Image/5.jpg 1459 638 1583 762 20 | Image/5.jpg 473 321 529 377 21 | Image/5.jpg 772 263 820 311 22 | -------------------------------------------------------------------------------- /annotaion_files_process/imglab_xml2txt.py: -------------------------------------------------------------------------------- 1 | # convert imglab (a dlib tool) annotation files to txt file 2 | # "image_name.jpg x1 y1 x2 y2" (each line in output txt file) 3 | # for single type object annotation 4 | 5 | import xml.dom.minidom 6 | 7 | INPUT_FILE_NAME = 'imglab_anno.xml' 8 | OUTPUT_FILE_NAME = 'output.txt' 9 | MIN_BBOX_SIZE = 20 10 | 11 | f = open(OUTPUT_FILE_NAME, 'w') 12 | 13 | dom = xml.dom.minidom.parse(INPUT_FILE_NAME) 14 | root = dom.documentElement 15 | 16 | images = root.getElementsByTagName('images')[0] 17 | for image in images.getElementsByTagName('image'): 18 | filename = image.getAttribute('file') 19 | filename = filename.split('/')[-1] 20 | for box in image.getElementsByTagName('box'): 21 | top = eval(box.getAttribute('top')) 22 | left = eval(box.getAttribute('left')) 23 | width = eval(box.getAttribute('width')) 24 | height = eval(box.getAttribute('height')) 25 | if width < MIN_BBOX_SIZE: 26 | continue 27 | x1 = left 28 | y1 = top 29 | x2 = x1 + width 30 | y2 = y1 + height 31 | f.write(filename + ' ' + str(x1) + ' ' + str(y1) + ' ' + str(x2) + ' ' + str(y2) + '\n') 32 | print(filename) 33 | 34 | f.close() 35 | -------------------------------------------------------------------------------- /annotaion_files_process/txt2imglab_xml.py: -------------------------------------------------------------------------------- 1 | import os 2 | import xml.dom.minidom 3 | from xml.dom.minidom import Document 4 | 5 | doc = Document() 6 | 7 | # creat root 8 | images = doc.createElement('images') 9 | # creat dom tree 10 | doc.appendChild(images) 11 | 12 | 13 | def writeInfoToXml(filename, info): 14 | image = doc.createElement('image') 15 | image.setAttribute('file',filename) 16 | #image.attrib = {'file':filename} 17 | images.appendChild(image) 18 | 19 | for xyxy in info: 20 | box = doc.createElement('box') 21 | x = int(xyxy[0]) 22 | y = int(xyxy[1]) 23 | w = int(xyxy[2]) - x 24 | h = int(xyxy[3]) - y 25 | box.setAttribute('top',str(y)) 26 | box.setAttribute('left',str(x)) 27 | box.setAttribute('width',str(w)) 28 | box.setAttribute('height',str(h)) 29 | #box.attrib = {'top':str(x),'left':str(y),'width':str(w),'height':str(h)} 30 | image.appendChild(box) 31 | 32 | 33 | 34 | with open('anno_all.txt') as label: 35 | line = label.readline() 36 | while line: 37 | suffix = '.' + line.split()[0].split('/')[-1].split('.')[-1] 38 | filename = line.split()[0].split('/')[-1].split(suffix)[0] + '.jpg' 39 | info = [] 40 | info.append(line.split()[1:5]) 41 | line = label.readline() 42 | while line and ((line.split()[0].split('/')[-1].split(suffix)[0] + '.jpg') == filename): 43 | info.append(line.split()[1:5]) 44 | line = label.readline() 45 | writeInfoToXml(filename, info) 46 | 47 | with open('output.xml', 'wb') as f: 48 | f.write(doc.toprettyxml(indent='\t', encoding='utf-8')) 49 | -------------------------------------------------------------------------------- /annotaion_files_process/VOC_XML2txt.py: -------------------------------------------------------------------------------- 1 | # convert pascal voc format annotation files to txt file 2 | # "image_name.jpg x1 y1 x2 y2" (each line in output txt file) 3 | # for single type object annotation 4 | 5 | import os 6 | import xml.dom.minidom 7 | 8 | VOC_XML_FOLDER = 'Annotations' 9 | OUTPUT_FILE = 'annotation.txt' 10 | MIN_BBOX_SIZE = 20 11 | 12 | f = open(OUTPUT_FILE, 'w') 13 | anno_dir = VOC_XML_FOLDER 14 | images = os.listdir(anno_dir) 15 | 16 | for name in images: 17 | if name.split('.')[-1]!='xml': 18 | continue 19 | name = name.split('.xml')[0] 20 | dom = xml.dom.minidom.parse(anno_dir + '/' + name + '.xml') 21 | root = dom.documentElement 22 | heads = root.getElementsByTagName('object') 23 | filename = root.getElementsByTagName('filename')[0].childNodes[0].data 24 | size = root.getElementsByTagName('size')[0] 25 | w = eval(size.getElementsByTagName('width')[0].childNodes[0].data) 26 | h = eval(size.getElementsByTagName('height')[0].childNodes[0].data) 27 | for head in heads: 28 | bbox = head.getElementsByTagName('bndbox')[0] 29 | xmin = bbox.getElementsByTagName('xmin')[0] 30 | ymin = bbox.getElementsByTagName('ymin')[0] 31 | xmax = bbox.getElementsByTagName('xmax')[0] 32 | ymax = bbox.getElementsByTagName('ymax')[0] 33 | xmin = max(eval(xmin.childNodes[0].data),1) 34 | ymin = max(eval(ymin.childNodes[0].data),1) 35 | xmax = min(eval(xmax.childNodes[0].data),w-1) 36 | ymax = min(eval(ymax.childNodes[0].data),h-1) 37 | if (xmax-xmin >= MIN_BBOX_SIZE): 38 | f.write(filename + ' ' + str(xmin) + ' ' + str(ymin) + ' ' + str(xmax) + ' ' + str(ymax) + '\n') 39 | 40 | f.close() 41 | -------------------------------------------------------------------------------- /detection_eval/README.md: -------------------------------------------------------------------------------- 1 | # Detection Result Evaluation 2 | this is a simple code to evaluate the performance of one type object detection algorithm. 3 | 4 | ## Usage 5 | Input two files: ground truth ('anno.txt') and prediction result ('result.txt'). 6 | 7 | run: 8 | 9 | ``` python3 detection_eval.py ``` 10 | 11 | We can get the P-R curve and the AP value: 12 | 13 | ![P-R curve](example/PR_curve.png) 14 | 15 | ``` 16 | [[[ IoU threshold is 0.50 ]]] 17 | []Threshold:0.000 Precision:0.091 Recall:0.945 18 | []Threshold:0.100 Precision:0.091 Recall:0.945 19 | []Threshold:0.200 Precision:0.283 Recall:0.910 20 | []Threshold:0.300 Precision:0.547 Recall:0.869 21 | []Threshold:0.400 Precision:0.753 Recall:0.823 22 | []Threshold:0.500 Precision:0.874 Recall:0.749 23 | []Threshold:0.600 Precision:0.932 Recall:0.576 24 | []Threshold:0.700 Precision:0.968 Recall:0.330 25 | []Threshold:0.800 Precision:0.986 Recall:0.126 26 | []Threshold:0.900 Precision:1.000 Recall:0.006 27 | [AP]:0.83046 28 | ``` 29 | 30 | ## Input Format 31 | The format of input files 32 | (annotation files & detection results file) 33 | 34 | ### annotation file 35 | annotations file should follow the format like this: 36 | 37 | eg. in 'annotation.txt' 38 | 39 | ``` 40 | path/to/image_name.jpg x1 y1 x2 y2 x1 y1 x2 y2 ...... 41 | path/to/image_name.jpg x1 y1 x2 y2 x1 y1 x2 y2 ...... 42 | ...... 43 | ``` 44 | 45 | or 46 | 47 | ``` 48 | image1.jpg x1 y1 x2 y2 49 | image1.jpg x1 y1 x2 y2 50 | image2.jpg x1 y1 x2 y2 51 | image3.jpg x1 y1 x2 y2 52 | image3.jpg x1 y1 x2 y2 53 | image3.jpg x1 y1 x2 y2 54 | ...... 55 | ``` 56 | 57 | or see 'anno_example1.txt' or 'anno_example2.txt' in example folder 58 | ### detection results file 59 | detection results should follow the format like this: 60 | 61 | eg. in 'result.txt' 62 | 63 | ``` 64 | path/to/image.jpg 65 | number_of_detection 66 | x1 y1 x2 y2 p1 67 | x1 y1 x2 y2 p2 68 | .... 69 | .... 70 | Images/01.jpg 71 | 2 72 | 99 102 149 152 0.95 73 | 23 23 55 55 0.55 74 | ``` 75 | 76 | or see 'example_result_format.txt' in example folder 77 | 78 | *the 'path/to/image.jpg' is the index, and it should be the same in 'anno.txt' and 'result.txt' for the same image* 79 | 80 | ## TODO 81 | - [ ] increase test data by filping the test images and annotations 82 | - [ ] add TTA (test time augmentation) 83 | -------------------------------------------------------------------------------- /annotaion_files_process/pascal2coco.py: -------------------------------------------------------------------------------- 1 | # https://github.com/JialianW/pascal2coco/blob/master/pascal2coco.py 2 | 3 | import json 4 | import xml.etree.ElementTree as ET 5 | import os 6 | 7 | def load_load_image_labels(LABEL_PATH, class_name=[]): 8 | # temp=[] 9 | images=[] 10 | type="instances" 11 | annotations=[] 12 | #assign your categories which contain the classname and calss id 13 | #the order must be same as the class_nmae 14 | categories = [ 15 | { 16 | "id" : 1, 17 | "name" : "xxx", 18 | "supercategory" : "none" 19 | }, 20 | { 21 | "id": 2, 22 | "name": "xxx", 23 | "supercategory": "none" 24 | }, 25 | ] 26 | # load ground-truth from xml annotations 27 | id_number=0 28 | for image_id, label_file_name in enumerate(os.listdir(LABEL_PATH)): 29 | print(str(image_id)+' '+label_file_name) 30 | label_file=LABEL_PATH + label_file_name 31 | image_file = label_file_name.split('.')[0] + '.jpg' 32 | tree = ET.parse(label_file) 33 | root = tree.getroot() 34 | 35 | size=root.find('size') 36 | width = float(size.find('width').text) 37 | height = float(size.find('height').text) 38 | 39 | images.append({ 40 | "file_name": image_file, 41 | "height": height, 42 | "width": width, 43 | "id": image_id 44 | })# id of the image. referenced in the annotation "image_id" 45 | 46 | for anno_id, obj in enumerate(root.iter('object')): 47 | name = obj.find('name').text 48 | bbox=obj.find('bndbox') 49 | cls_id = class_name.index(name) 50 | xmin = float(bbox.find('xmin').text) 51 | ymin = float(bbox.find('ymin').text) 52 | xmax = float(bbox.find('xmax').text) 53 | ymax = float(bbox.find('ymax').text) 54 | xlen = xmax-xmin 55 | ylen = ymax-ymin 56 | annotations.append({ 57 | "segmentation" : [[xmin, ymin, xmin, ymax, xmax, ymax, xmax, ymin],], 58 | "area" : xlen*ylen, 59 | "iscrowd": 0, 60 | "image_id": image_id, 61 | "bbox" : [xmin, ymin, xlen, ylen], 62 | "category_id": cls_id, 63 | "id": id_number, 64 | "ignore":0 65 | }) 66 | # print([image_file,image_id, cls_id, xmin, ymin, xlen, ylen]) 67 | id_number += 1 68 | 69 | return {"images":images,"annotations":annotations,"categories":categories} 70 | 71 | if __name__=='__main__': 72 | LABEL_PATH='your pascal voc annotation path' 73 | classes=['background','add your class name'] 74 | 75 | label_dict = load_load_image_labels(LABEL_PATH,classes) 76 | jsonfile='/home/xxx/train.json'#location where you would like to save the coco format annotations 77 | with open('/home/wjl/DataSet/707dataset/label/train.json','w') as json_file: 78 | json_file.write(json.dumps(label_dict, ensure_ascii=False)) 79 | json_file.close() 80 | -------------------------------------------------------------------------------- /annotaion_files_process/txt2VOC_XML.py: -------------------------------------------------------------------------------- 1 | # convert txt file to pascal voc format 2 | # "image_name.jpg x1 y1 x2 y2" (each line in input txt file) 3 | # for single type object annotation 4 | 5 | import os 6 | from xml.dom.minidom import Document 7 | import cv2 8 | 9 | IMG_FOLDER = 'Images' # images folder 10 | XML_FOLDER = 'Annotations/' # output folder 11 | INPUT_TXT = 'anno.txt' # input txt file 12 | #foldername = 'data' 13 | object_name = 'face' # object name (eg. person, face, car ...) 14 | 15 | def writeInfoToXml(filename, info): 16 | img_name = 'JPEGImages/' + filename 17 | img_read = cv2.imread(img_name) 18 | try: 19 | h = img_read.shape[0] 20 | w = img_read.shape[1] 21 | except: # img not exist 22 | return 23 | 24 | # creat dom document 25 | doc = Document() 26 | 27 | # creat root node 28 | annotation = doc.createElement('annotation') 29 | # creat dom tree 30 | doc.appendChild(annotation) 31 | 32 | folder = doc.createElement('folder') 33 | folder_text = doc.createTextNode(IMG_FOLDER) 34 | folder.appendChild(folder_text) 35 | annotation.appendChild(folder) 36 | 37 | file_name = doc.createElement('filename') 38 | file_name_text = doc.createTextNode(filename) 39 | file_name.appendChild(file_name_text) 40 | annotation.appendChild(file_name) 41 | 42 | size = doc.createElement('size') 43 | width = doc.createElement('width') 44 | width_text = doc.createTextNode(str(w)) 45 | width.appendChild(width_text) 46 | size.appendChild(width) 47 | height = doc.createElement('height') 48 | height_text = doc.createTextNode(str(h)) 49 | height.appendChild(height_text) 50 | size.appendChild(height) 51 | depth = doc.createElement('depth') 52 | depth_text = doc.createTextNode('3') 53 | depth.appendChild(depth_text) 54 | size.appendChild(depth) 55 | annotation.appendChild(size) 56 | 57 | for xyxy in info: 58 | object = doc.createElement('object') 59 | bndbox = doc.createElement('bndbox') 60 | xmin = doc.createElement('xmin') 61 | xmin_text = doc.createTextNode(xyxy[0]) 62 | xmin.appendChild(xmin_text) 63 | bndbox.appendChild(xmin) 64 | ymin = doc.createElement('ymin') 65 | ymin_text = doc.createTextNode(xyxy[1]) 66 | ymin.appendChild(ymin_text) 67 | bndbox.appendChild(ymin) 68 | xmax = doc.createElement('xmax') 69 | xmax_text = doc.createTextNode(xyxy[2]) 70 | xmax.appendChild(xmax_text) 71 | bndbox.appendChild(xmax) 72 | ymax = doc.createElement('ymax') 73 | ymax_text = doc.createTextNode(xyxy[3]) 74 | ymax.appendChild(ymax_text) 75 | bndbox.appendChild(ymax) 76 | name = doc.createElement('name') 77 | name_text = doc.createTextNode(object_name) 78 | name.appendChild(name_text) 79 | difficult = doc.createElement('difficult') 80 | difficult_text = doc.createTextNode('0') 81 | difficult.appendChild(difficult_text) 82 | object.appendChild(name) 83 | object.appendChild(difficult) 84 | object.appendChild(bndbox) 85 | 86 | annotation.appendChild(object) 87 | 88 | # write xml files 89 | xml_name = os.path.join(XML_FOLDER, filename.split('.')[0] + '.xml') 90 | print (xml_name) 91 | with open(xml_name, 'wb') as f: 92 | f.write(doc.toprettyxml(indent='\t', encoding='utf-8')) 93 | 94 | 95 | with open(INPUT_TXT) as label: 96 | line = label.readline() 97 | while line: 98 | suffix = '.' + line.split()[0].split('/')[-1].split('.')[-1] 99 | filename = line.split()[0].split('/')[-1].split(suffix)[0] + '.jpg' 100 | info = [] 101 | info.append(line.split()[1:5]) 102 | line = label.readline() 103 | while line and ((line.split()[0].split('/')[-1].split(suffix)[0] + '.jpg') == filename): 104 | info.append(line.split()[1:5]) 105 | line = label.readline() 106 | writeInfoToXml(filename, info) 107 | 108 | -------------------------------------------------------------------------------- /detection_eval/example/eval_result.txt: -------------------------------------------------------------------------------- 1 | [[[ IoU threshold is 0.50 ]]] 2 | []Threshold:0.000 Precision:0.091 Recall:0.945 3 | []Threshold:0.010 Precision:0.091 Recall:0.945 4 | []Threshold:0.020 Precision:0.091 Recall:0.945 5 | []Threshold:0.030 Precision:0.091 Recall:0.945 6 | []Threshold:0.040 Precision:0.091 Recall:0.945 7 | []Threshold:0.050 Precision:0.091 Recall:0.945 8 | []Threshold:0.060 Precision:0.091 Recall:0.945 9 | []Threshold:0.070 Precision:0.091 Recall:0.945 10 | []Threshold:0.080 Precision:0.091 Recall:0.945 11 | []Threshold:0.090 Precision:0.091 Recall:0.945 12 | []Threshold:0.100 Precision:0.091 Recall:0.945 13 | []Threshold:0.110 Precision:0.106 Recall:0.941 14 | []Threshold:0.120 Precision:0.122 Recall:0.939 15 | []Threshold:0.130 Precision:0.139 Recall:0.936 16 | []Threshold:0.140 Precision:0.157 Recall:0.932 17 | []Threshold:0.150 Precision:0.175 Recall:0.928 18 | []Threshold:0.160 Precision:0.195 Recall:0.924 19 | []Threshold:0.170 Precision:0.215 Recall:0.921 20 | []Threshold:0.180 Precision:0.237 Recall:0.917 21 | []Threshold:0.190 Precision:0.259 Recall:0.913 22 | []Threshold:0.200 Precision:0.283 Recall:0.910 23 | []Threshold:0.210 Precision:0.309 Recall:0.907 24 | []Threshold:0.220 Precision:0.336 Recall:0.903 25 | []Threshold:0.230 Precision:0.363 Recall:0.899 26 | []Threshold:0.240 Precision:0.389 Recall:0.895 27 | []Threshold:0.250 Precision:0.416 Recall:0.890 28 | []Threshold:0.260 Precision:0.442 Recall:0.886 29 | []Threshold:0.270 Precision:0.469 Recall:0.881 30 | []Threshold:0.280 Precision:0.494 Recall:0.878 31 | []Threshold:0.290 Precision:0.521 Recall:0.873 32 | []Threshold:0.300 Precision:0.547 Recall:0.869 33 | []Threshold:0.310 Precision:0.573 Recall:0.866 34 | []Threshold:0.320 Precision:0.596 Recall:0.862 35 | []Threshold:0.330 Precision:0.618 Recall:0.856 36 | []Threshold:0.340 Precision:0.639 Recall:0.853 37 | []Threshold:0.350 Precision:0.659 Recall:0.849 38 | []Threshold:0.360 Precision:0.680 Recall:0.843 39 | []Threshold:0.370 Precision:0.700 Recall:0.839 40 | []Threshold:0.380 Precision:0.718 Recall:0.834 41 | []Threshold:0.390 Precision:0.736 Recall:0.829 42 | []Threshold:0.400 Precision:0.753 Recall:0.823 43 | []Threshold:0.410 Precision:0.768 Recall:0.817 44 | []Threshold:0.420 Precision:0.782 Recall:0.809 45 | []Threshold:0.430 Precision:0.795 Recall:0.802 46 | []Threshold:0.440 Precision:0.809 Recall:0.797 47 | []Threshold:0.450 Precision:0.823 Recall:0.791 48 | []Threshold:0.460 Precision:0.835 Recall:0.784 49 | []Threshold:0.470 Precision:0.846 Recall:0.776 50 | []Threshold:0.480 Precision:0.855 Recall:0.767 51 | []Threshold:0.490 Precision:0.864 Recall:0.758 52 | []Threshold:0.500 Precision:0.874 Recall:0.749 53 | []Threshold:0.510 Precision:0.881 Recall:0.735 54 | []Threshold:0.520 Precision:0.889 Recall:0.722 55 | []Threshold:0.530 Precision:0.897 Recall:0.710 56 | []Threshold:0.540 Precision:0.903 Recall:0.695 57 | []Threshold:0.550 Precision:0.908 Recall:0.678 58 | []Threshold:0.560 Precision:0.912 Recall:0.661 59 | []Threshold:0.570 Precision:0.917 Recall:0.642 60 | []Threshold:0.580 Precision:0.922 Recall:0.623 61 | []Threshold:0.590 Precision:0.928 Recall:0.599 62 | []Threshold:0.600 Precision:0.932 Recall:0.576 63 | []Threshold:0.610 Precision:0.937 Recall:0.551 64 | []Threshold:0.620 Precision:0.942 Recall:0.525 65 | []Threshold:0.630 Precision:0.945 Recall:0.501 66 | []Threshold:0.640 Precision:0.948 Recall:0.476 67 | []Threshold:0.650 Precision:0.952 Recall:0.451 68 | []Threshold:0.660 Precision:0.957 Recall:0.426 69 | []Threshold:0.670 Precision:0.961 Recall:0.403 70 | []Threshold:0.680 Precision:0.965 Recall:0.376 71 | []Threshold:0.690 Precision:0.968 Recall:0.353 72 | []Threshold:0.700 Precision:0.968 Recall:0.330 73 | []Threshold:0.710 Precision:0.972 Recall:0.312 74 | []Threshold:0.720 Precision:0.973 Recall:0.287 75 | []Threshold:0.730 Precision:0.974 Recall:0.266 76 | []Threshold:0.740 Precision:0.973 Recall:0.242 77 | []Threshold:0.750 Precision:0.974 Recall:0.224 78 | []Threshold:0.760 Precision:0.974 Recall:0.205 79 | []Threshold:0.770 Precision:0.973 Recall:0.184 80 | []Threshold:0.780 Precision:0.973 Recall:0.164 81 | []Threshold:0.790 Precision:0.976 Recall:0.147 82 | []Threshold:0.800 Precision:0.986 Recall:0.126 83 | []Threshold:0.810 Precision:0.988 Recall:0.110 84 | []Threshold:0.820 Precision:0.988 Recall:0.095 85 | []Threshold:0.830 Precision:0.989 Recall:0.078 86 | []Threshold:0.840 Precision:0.991 Recall:0.065 87 | []Threshold:0.850 Precision:0.989 Recall:0.054 88 | []Threshold:0.860 Precision:0.989 Recall:0.039 89 | []Threshold:0.870 Precision:0.990 Recall:0.028 90 | []Threshold:0.880 Precision:0.992 Recall:0.018 91 | []Threshold:0.890 Precision:0.987 Recall:0.012 92 | []Threshold:0.900 Precision:1.000 Recall:0.006 93 | []Threshold:0.910 Precision:1.000 Recall:0.002 94 | []Threshold:0.920 Precision:1.000 Recall:0.001 95 | []Threshold:0.930 Precision:1.000 Recall:0.000 96 | []Threshold:0.940 Precision:1.000 Recall:0.000 97 | []Threshold:0.950 Precision:1.000 Recall:0.000 98 | []Threshold:0.960 Precision:1.000 Recall:0.000 99 | []Threshold:0.970 Precision:1.000 Recall:0.000 100 | []Threshold:0.980 Precision:1.000 Recall:0.000 101 | []Threshold:0.990 Precision:1.000 Recall:0.000 102 | [AP]:0.83237 103 | -------------------------------------------------------------------------------- /detection_eval/detection_eval.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from collections import defaultdict 4 | import seaborn as sns 5 | from tqdm import tqdm 6 | 7 | def read_bbox_txt(file): 8 | ''' 9 | read bbox annotation .txt file : eg. "filename x1 y1 x2 y2\n" 10 | :param file: path to ground truth annotation 11 | :return: a dictionary {img_name:[[x1, y1, x2, y2],[x1_, y1_, x2_, y2_], ......]} 12 | ''' 13 | anno_dict = defaultdict() 14 | with open(file, 'r') as label: 15 | line = label.readline() 16 | while line and line.strip()!='': 17 | filename = line.split()[0] 18 | anno = [] 19 | line_split = line.split() 20 | for i in range((len(line_split)-1)//4): 21 | anno.append(line_split[i*4+1:i*4+5]) 22 | line = label.readline() 23 | while line and line.strip()!='' and (line.split()[0] == filename): 24 | line_split = line.split() 25 | for i in range((len(line_split) - 1) // 4): 26 | anno.append(line_split[i * 4 + 1:i * 4 + 5]) 27 | line = label.readline() 28 | anno_dict[filename] = anno 29 | 30 | return anno_dict 31 | 32 | # def read_bbox_txt_split_folder_and_suffix(file, type='png'): 33 | # ''' 34 | # read bbox annotation .txt file : eg. "filename x1 y1 x2 y2\n" 35 | # :param file: path to ground truth annotation 36 | # :param type: imgs type (eg: jpg , png) 37 | # :return: a dictionary {img_name:[[x1, y1, x2, y2],[x1_, y1_, x2_, y2_], ......]} 38 | # ''' 39 | # anno_dict = defaultdict() 40 | # with open(file, 'r') as label: 41 | # line = label.readline() 42 | # while line: 43 | # filename = line.split()[0].split('/')[-1].split('.')[0] + '.' + type 44 | # anno = [] 45 | # anno.append(line.split()[1:5]) 46 | # line = label.readline() 47 | # while line and ((line.split()[0].split('/')[-1].split('.')[0] + '.' + type) == filename): 48 | # anno.append(line.split()[1:5]) 49 | # line = label.readline() 50 | # anno_dict[filename] = anno 51 | # 52 | # return anno_dict 53 | 54 | def IoU(bbox_predict: list, bbox_label: list, type = 'xyxy'): 55 | ''' 56 | calculate IoU of two box 57 | :param bbox_predict: predict result box [num1,num2,num3,num4] 58 | :param bbox_label: ground truth box [num1,num2,num3,num4] 59 | :param type: annotation type :'xyxy' or 'xywh' 60 | :return: IoU ratio 61 | ''' 62 | x1 = int(bbox_predict[0]) 63 | y1 = int(bbox_predict[1]) 64 | if type=='xyxy': 65 | width1 = int(bbox_predict[2]) - x1 66 | height1 = int(bbox_predict[3]) - y1 67 | elif type=='xywh': 68 | width1 = int(bbox_predict[2]) 69 | height1 = int(bbox_predict[3]) 70 | else: 71 | print('Error! IoU calculate:annotation type error') 72 | width1 = int(bbox_predict[2]) - x1 73 | height1 = int(bbox_predict[3]) - y1 74 | 75 | x2 = int(bbox_label[0]) 76 | y2 = int(bbox_label[1]) 77 | width2 = int(bbox_label[2]) - x2 78 | height2 = int(bbox_label[3]) - y2 79 | 80 | endx = max(x1 + width1, x2 + width2) 81 | startx = min(x1, x2) 82 | width = width1 + width2 - (endx - startx) 83 | 84 | endy = max(y1 + height1, y2 + height2) 85 | starty = min(y1, y2) 86 | height = height1 + height2 - (endy - starty) 87 | 88 | if width <= 0 or height <= 0: 89 | ratio = 0 90 | else: 91 | Area = width * height 92 | Area1 = width1 * height1 93 | Area2 = width2 * height2 94 | ratio = Area * 1. / (Area1 + Area2 - Area) 95 | # return IOU 96 | return ratio 97 | 98 | 99 | def precision_recall(predict_list, truth_list, IoU_thre): 100 | ''' 101 | calculate number of true posible, false posible and false negative 102 | :param predict_list: a list of prediction bbox eg. [[x1, y1, x2, y2], [x1_, y1_, x2_, y2_], ......] 103 | :param truth_list: a list of ground truth anno eg. [[x1, y1, x2, y2], [x1_, y1_, x2_, y2_], ......] 104 | :param IoU_thre: IoU threshold 105 | :return: (true posible, false posible, false negative) 106 | ''' 107 | tp = 0 108 | fp = 0 109 | fn = 0 110 | is_recall_truth_list = [False] * len(truth_list) 111 | for positive in predict_list: 112 | is_find = False 113 | for k, truth in enumerate(truth_list): 114 | if IoU(positive,truth) > IoU_thre: 115 | is_find = True 116 | is_recall_truth_list[k] = True 117 | break 118 | if is_find: 119 | tp += 1 120 | else: 121 | fp += 1 122 | 123 | for is_recall in is_recall_truth_list: 124 | if not is_recall: 125 | fn += 1 126 | 127 | return (tp, fp, fn) 128 | 129 | 130 | def detection(anno_path, detection_result_txt, thre_list, IoU_thre = 0.5): 131 | ''' 132 | detection all images and return the all number of true posible, false posible and false negative in each level of thre 133 | :param anno_path: path of ground truth anno txt file 134 | :param detection_result_txt: path of detection results file 135 | :param thre_list: several detection threshold level 136 | :param IoU_thre: IoU threshold 137 | :return: a list [[tp,fp,fn], ...] in different detection threshold level 138 | ''' 139 | anno_dict = read_bbox_txt(anno_path) 140 | 141 | pbar = tqdm(total=len(anno_dict)) 142 | count = 0 143 | 144 | result_list = [] 145 | for i in range(len(thre_list)): 146 | result_list.append([0,0,0]) # [tp,fp,fn] 147 | 148 | det_txt = open(detection_result_txt, 'r') 149 | 150 | line = det_txt.readline() 151 | 152 | while line and line.strip(): 153 | img_name = line.strip() 154 | num_detections = int(det_txt.readline().strip()) 155 | det_result = [] 156 | for i in range(num_detections): 157 | line = det_txt.readline().split() 158 | det_result.append(line) 159 | 160 | for k, thre in enumerate(thre_list): 161 | predict_list = [] 162 | for det in det_result: 163 | score = float(det[4]) 164 | if score > thre: ### threshould 165 | x = float(det[0]) 166 | y = float(det[1]) 167 | right = float(det[2]) 168 | bottom = float(det[3]) 169 | predict_list.append([x,y,right,bottom]) 170 | 171 | detection_result = precision_recall(predict_list,anno_dict[img_name],IoU_thre) 172 | tp = detection_result[0] 173 | fp = detection_result[1] 174 | fn = detection_result[2] 175 | result_list[k][0] += tp 176 | result_list[k][1] += fp 177 | result_list[k][2] += fn 178 | 179 | line = det_txt.readline() 180 | count += 1 181 | if count%10 == 0: 182 | pbar.update(10) 183 | #print('number:%d , finish:%.3f'%(count ,count/len(imgs))) 184 | return result_list 185 | 186 | 187 | def main(): 188 | # ------ change this param --------- 189 | ANNO_PATH = 'anno.txt' 190 | DETECTION_RESULT = 'result.txt' 191 | thre_list = list(np.arange(0.0001,1.0001,0.01)) 192 | IoU_thre = 0.5 193 | # ---------------------------------- 194 | result_list = detection(ANNO_PATH, DETECTION_RESULT, thre_list, IoU_thre) 195 | precision_list = [] 196 | recall_list = [] 197 | result_txt = open('eval_result.txt', 'w') 198 | print('[[[ IoU threshold is %.2f ]]]'%IoU_thre) 199 | result_txt.write('[[[ IoU threshold is %.2f ]]] \n'%IoU_thre) 200 | for i in range(len(thre_list)): 201 | thre = thre_list[i] 202 | tp = result_list[i][0] 203 | fp = result_list[i][1] 204 | fn = result_list[i][2] 205 | precision = (tp + 0.00001) / (tp + fp + 0.00001) 206 | recall = (tp) / (tp + fn + 0.00001) 207 | precision_list.append(precision) 208 | recall_list.append(recall) 209 | print("[]Threshold:%.3f Precision:%.3f Recall:%.3f"%(thre,precision,recall)) 210 | result_txt.write("[]Threshold:%.3f Precision:%.3f Recall:%.3f \n"%(thre,precision,recall)) 211 | 212 | AP = precision_list[0] * (1-recall_list[0]) * 0.5 213 | precision_list.append(1.0) 214 | recall_list.append(0.0) 215 | for i in range(len(precision_list)-1): 216 | h = (precision_list[i+1] + precision_list[i]) / 2 217 | w = recall_list[i] - recall_list[i+1] 218 | AP = AP + h * w 219 | print('[AP]:%.5f'%AP) 220 | result_txt.write('[AP]:%.5f \n'%AP) 221 | result_txt.close() 222 | 223 | ## draw the P-R curve 224 | sns.set() 225 | plt.title('P-R curve') 226 | plt.xlabel('Recall') 227 | plt.ylabel('Precision') 228 | plt.plot(recall_list[:-1:5], precision_list[:-1:5]) 229 | #plt.plot(recall_list[:-1], precision_list[:-1]) 230 | plt.savefig("PR_curve.png") 231 | plt.show() 232 | 233 | 234 | if __name__ == '__main__': 235 | main() 236 | 237 | # TODO: add flip image and annotation eval; 238 | # TODO: add TTA(test time augmentation) 239 | --------------------------------------------------------------------------------