├── COCO ├── COCO_xml_parser.py ├── README.md ├── calculate_objects.py ├── data_migrate.sh ├── parse.sh └── voc_data_migrate.py ├── DETRAC ├── DETRAC_xmlParser.py ├── compare.py ├── generate_tfrecord.py ├── generate_tfrecord.sh ├── run.sh ├── statistics_analysi.py ├── train_test_split.py ├── voc_data_migrate.py └── xml_to_csv.py ├── KITTI └── txt_parser.py ├── README.md └── VOC2007_2012 ├── ImageSets_Convert.py ├── generate_tfrecord.py ├── test_xml_to_csv.py ├── train_test_split.py └── xml_to_csv.py /COCO/COCO_xml_parser.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import xml.etree.ElementTree as ET 4 | from xml.dom.minidom import Document 5 | import time 6 | import sys 7 | from tqdm import tqdm 8 | 9 | def ConvertVOCXml(file_path="",file_name=""): 10 | 11 | # 创建dom文档 12 | doc = Document() 13 | # 创建根节点 14 | annotation = doc.createElement('annotation') 15 | # 根节点插入dom树 16 | doc.appendChild(annotation) 17 | 18 | tree = ET.parse(file_name) 19 | root = tree.getroot() 20 | 21 | #读xml操作 22 | object_lists=[] 23 | for child in root: 24 | if(child.tag=="folder"): 25 | #print(child.tag, child.text) 26 | folder = doc.createElement("folder") 27 | # folder.appendChild(doc.createTextNode(child.text)) 28 | folder.appendChild(doc.createTextNode("VOC2007")) 29 | annotation.appendChild(folder) 30 | elif (child.tag == "filename"): 31 | #print(child.tag, child.text) 32 | filename = doc.createElement("filename") 33 | filename.appendChild(doc.createTextNode(child.text)) 34 | annotation.appendChild(filename) 35 | elif (child.tag == "size"): #解析size 36 | sizeimage = doc.createElement("size") 37 | imagewidth = doc.createElement("width") 38 | imageheight = doc.createElement("height") 39 | imagedepth = doc.createElement("depth") 40 | for size_child in child: 41 | if(size_child.tag=="width"): 42 | # print(size_child.tag,size_child.text) 43 | imagewidth.appendChild(doc.createTextNode(size_child.text)) 44 | elif (size_child.tag == "height"): 45 | # print(size_child.tag, size_child.text) 46 | # height_value = int(size_child.text) 47 | #print("图片的高度", height_value) 48 | imageheight.appendChild(doc.createTextNode(size_child.text)) 49 | elif (size_child.tag == "depth"): 50 | #print(size_child.tag, size_child.text) 51 | imagedepth.appendChild(doc.createTextNode(size_child.text)) 52 | 53 | sizeimage.appendChild(imagewidth) 54 | sizeimage.appendChild(imageheight) 55 | sizeimage.appendChild(imagedepth) 56 | annotation.appendChild(sizeimage) 57 | 58 | elif (child.tag == "object"): #解析object 59 | singleObject={} 60 | for object_child in child: 61 | if (object_child.tag == "name"): 62 | if(object_child.text=="person"): 63 | singleObject["name"] = "person" 64 | elif object_child.text=="car": 65 | singleObject["name"] = "car" 66 | elif object_child.text=="bus": 67 | singleObject["name"] = "car" 68 | elif object_child.text=="motorcycle": 69 | singleObject["name"] = "car" 70 | elif object_child.text=="train": 71 | singleObject["name"] = "car" 72 | elif object_child.text=="truck": 73 | singleObject["name"] = "car" 74 | else: 75 | # print(file_name) 76 | # print(object_child.text) 77 | #singleObject["name"] = object_child.text 78 | singleObject={} 79 | break 80 | #singleObject["name"] = object_child.text 81 | elif(object_child.tag == "pose"): 82 | singleObject["pose"] = object_child.text 83 | # print(object_child.text) 84 | elif(object_child.tag == "truncated"): 85 | singleObject["truncated"] = object_child.text 86 | # print(object_child.text) 87 | elif(object_child.tag == "difficult"): 88 | singleObject["difficult"] = object_child.text 89 | # print(object_child.text) 90 | elif (object_child.tag == "bndbox"): 91 | for bndbox_child in object_child: 92 | if (bndbox_child.tag == "xmin"): 93 | singleObject["xmin"] = bndbox_child.text 94 | # print(bndbox_child.tag, bndbox_child.text) 95 | elif (bndbox_child.tag == "ymin"): 96 | # print(bndbox_child.tag, bndbox_child.text) 97 | singleObject["ymin"] = bndbox_child.text 98 | elif (bndbox_child.tag == "xmax"): 99 | singleObject["xmax"] = bndbox_child.text 100 | elif (bndbox_child.tag == "ymax"): 101 | singleObject["ymax"] = bndbox_child.text 102 | object_length=len(singleObject) 103 | if(object_length>0): 104 | object_lists.append(singleObject) 105 | # print(object_lists) 106 | if(len(object_lists)==0): #如果解析出来没有所需要的car person目标，则直接舍弃，不保留 107 | return False 108 | 109 | ''' 110 | 写入xml操作 111 | ''' 112 | for singleObject in object_lists: 113 | object = doc.createElement('object') 114 | 115 | name = doc.createElement('name') 116 | 117 | truncated=doc.createElement('truncated') 118 | difficult=doc.createElement('difficult') 119 | pose=doc.createElement('pose') 120 | 121 | 122 | name.appendChild(doc.createTextNode(singleObject['name'])) 123 | 124 | if(truncated in singleObject): 125 | truncated.appendChild(doc.createTextNode(singleObject['truncated'])) 126 | else: 127 | truncated.appendChild(doc.createTextNode("0")) 128 | 129 | if(difficult in singleObject): 130 | difficult.appendChild(doc.createTextNode(singleObject['difficult'])) 131 | else: 132 | difficult.appendChild(doc.createTextNode("0")) 133 | 134 | if(pose in singleObject): 135 | pose.appendChild(doc.createTextNode(singleObject['pose'])) 136 | else: 137 | pose.appendChild(doc.createTextNode("Unspecified")) 138 | 139 | 140 | object.appendChild(name) 141 | object.appendChild(pose) 142 | object.appendChild(truncated) 143 | object.appendChild(difficult) 144 | 145 | bndbox = doc.createElement("bndbox") 146 | xmin = doc.createElement("xmin") 147 | ymin = doc.createElement("ymin") 148 | xmax = doc.createElement("xmax") 149 | ymax = doc.createElement("ymax") 150 | 151 | xmin.appendChild(doc.createTextNode(singleObject['xmin'])) 152 | ymin.appendChild(doc.createTextNode(singleObject['ymin'])) 153 | xmax.appendChild(doc.createTextNode(singleObject['xmax'])) 154 | ymax.appendChild(doc.createTextNode(singleObject['ymax'])) 155 | 156 | bndbox.appendChild(xmin) 157 | bndbox.appendChild(ymin) 158 | bndbox.appendChild(xmax) 159 | bndbox.appendChild(ymax) 160 | object.appendChild(bndbox) 161 | annotation.appendChild(object) 162 | #print(bndbox_child.tag, bndbox_child.text) 163 | file_name=file_name.split("/")[-1] 164 | output_file=os.path.join(file_path,file_name) 165 | #file_path=os.path.join(file_path,file_name) 166 | 167 | #print(file_path) 168 | f = open(output_file, 'w') 169 | f.write(doc.toprettyxml(indent=' ' * 4)) 170 | f.close() 171 | return True 172 | 173 | def main(): 174 | 175 | #train 176 | # basePath="COCO/instance_train_annotation" 177 | # saveBasePath="COCO/coco_train_xml" 178 | #val 179 | # basePath="COCO/instance_val_annotation" 180 | # saveBasePath="COCO/coco_val_xml" 181 | basePath=sys.argv[1] 182 | saveBasePath=sys.argv[2] 183 | 184 | xml_base_paths=os.listdir(basePath) 185 | total_num=0 186 | flag=False 187 | print("正在转换") 188 | if os.path.exists(saveBasePath)==False: #判断文件夹是否存在 189 | os.makedirs(saveBasePath) 190 | 191 | # ConvertVOCXml(file_path=saveBasePath,file_name="COCO_train2014_000000000036.xml") 192 | # Start time 193 | start = time.time() 194 | 195 | log=open("xml_statistical.txt","w") #分析日志，进行排错 196 | for xml_base_path in tqdm(xml_base_paths): 197 | xml_path=os.path.join(basePath,xml_base_path) 198 | # print(xml_path) 199 | list_xml=os.listdir(xml_path) 200 | for xml_file in list_xml: 201 | xml_file_path=os.path.join(xml_path,xml_file) 202 | # print(xml_file_path) 203 | _file_name=xml_file_path.split("/")[-1] 204 | _output_file=os.path.join(saveBasePath,_file_name) 205 | if os.path.exists(_output_file)==True: 206 | # print(_output_file) 207 | continue #判断文件夹是否存在 208 | flag=ConvertVOCXml(file_path=saveBasePath,file_name=xml_file_path) 209 | if flag: 210 | total_num+=1 211 | if(total_num%10000==0): 212 | print(total_num) 213 | else: 214 | log.write(xml_file_path+" "+str(total_num)+"\n") 215 | 216 | 217 | # End time 218 | end = time.time() 219 | seconds=end-start 220 | print( "Time taken : {0} seconds".format(seconds)) 221 | print(total_num) 222 | log.write(str(total_num)+"\n") 223 | 224 | if __name__ == '__main__': 225 | main() -------------------------------------------------------------------------------- /COCO/README.md: -------------------------------------------------------------------------------- 1 | # Dataset_to_VOC_converter 2 | this is the extend files, if you want to run the whole problem, pleasure refer to the repository:https://github.com/CasiaFan/Dataset_to_VOC_converter 3 | 4 | ``` 5 | git clone https://github.com/CasiaFan/Dataset_to_VOC_converter.git 6 | ``` 7 | 8 | - then you can run these python scripts in this directory 9 | 10 | **extract cars and person,and save to Annotations directory** 11 | ``` 12 | sh parse.sh 13 | ``` 14 | 15 | **copy image to JPEGImages directory** 16 | ``` 17 | sh data_migrate.sh 18 | ``` 19 | 20 | # environment 21 | - python 3.5 22 | 23 | 24 | # reference 25 | [markdown语法连接、代码块、段落](https://www.jianshu.com/p/9ab34d075bba) -------------------------------------------------------------------------------- /COCO/calculate_objects.py: -------------------------------------------------------------------------------- 1 | import os 2 | import xml.etree.ElementTree as ET 3 | from xml.dom.minidom import Document 4 | import time 5 | import sys 6 | 7 | 8 | def total_objects(file_name=""): 9 | tree = ET.parse(file_name) 10 | root = tree.getroot() 11 | count=0 12 | person_count=0 13 | car_count=0 14 | #读xml操作 15 | for child in root: 16 | if (child.tag == "object"): #解析object 17 | count+=1 18 | for object_child in child: 19 | if (object_child.tag == "name"): 20 | if(object_child.text=="person"): 21 | person_count+=1 22 | elif object_child.text=="car": 23 | car_count+=1 24 | # print(count) 25 | return count,person_count,car_count 26 | 27 | def main(): 28 | basePath=sys.argv[1] 29 | xml_base_paths=os.listdir(basePath) 30 | # print(xml_base_paths) 31 | objects_count=0 32 | persons_count=0 33 | cars_count=0 34 | total_count=0 35 | for xml_file in xml_base_paths: 36 | xml_file_path=os.path.join(basePath,xml_file) 37 | object_value,person_value,car_value=total_objects(xml_file_path) 38 | 39 | objects_count+=object_value 40 | persons_count+=person_value 41 | cars_count+=car_value 42 | total_count=total_count+1 43 | # print(objects_count) 44 | if(total_count%1000==0): 45 | print(total_count) 46 | print("total xml files: "+str(total_count)) 47 | print("total objects: "+str(objects_count)) 48 | print("total persons: "+str(persons_count)) 49 | print("total cars: "+str(cars_count)) 50 | print("average objects per file: "+str(objects_count/float(total_count))) 51 | print("average persons per file: "+str(persons_count/float(total_count))) 52 | print("average cars per file: "+str(cars_count/float(total_count))) 53 | 54 | 55 | if __name__ == '__main__': 56 | main() -------------------------------------------------------------------------------- /COCO/data_migrate.sh: -------------------------------------------------------------------------------- 1 | #val+train 2 | python3 voc_data_migrate.py ./COCO/Annotations /media/data/COCO/val2014 /media/data/COCO/train2014 ./COCO/JPEGImages 3 | -------------------------------------------------------------------------------- /COCO/parse.sh: -------------------------------------------------------------------------------- 1 | #train 2 | python3 COCO_xml_parser.py /media/data/COCO/instance_train_annotation_2 ./COCO/Annotations 3 | #val 4 | python3 COCO_xml_parser.py /media/data/COCO/instance_val_annotation ./COCO/Annotations -------------------------------------------------------------------------------- /COCO/voc_data_migrate.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import random 4 | import shutil 5 | import sys 6 | from tqdm import tqdm 7 | 8 | #xml路径的地址 train 9 | # XmlPath=r'./COCO/coco_xml' 10 | #原图片的地址 train 11 | # pictureBathPath=r"./COCO/train2014" 12 | #保存图片的地址 13 | #train 14 | # saveBasePath=r"./COCO/coco_train_images" 15 | #val 16 | # XmlPath=r'./COCO/coco_val_xml' 17 | # #原图片的地址 train 18 | # pictureBathPath=r"./COCO/val2014" 19 | # saveBasePath=r"./COCO/coco_val_images" 20 | XmlPath=sys.argv[1] 21 | val_pictureBasePath=sys.argv[2] 22 | train_pictureBasePath=sys.argv[3] 23 | saveBasePath=sys.argv[4] 24 | 25 | total_xml = os.listdir(XmlPath) 26 | num=len(total_xml) 27 | list=range(num) 28 | print("xml file total number",num) 29 | 30 | count=0 31 | for xml in tqdm(total_xml): 32 | filename=xml.split(".")[0] 33 | filename=filename+".jpg" 34 | # print(filename) 35 | val_filePath=os.path.join(val_pictureBasePath,filename) 36 | train_filePath=os.path.join(train_pictureBasePath,filename) 37 | # print(filePath) 38 | if os.path.exists(val_filePath): 39 | # print(filePath+" not exists") 40 | filePath=val_filePath 41 | elif os.path.exists(train_filePath): 42 | filePath=train_filePath 43 | else: 44 | print(filename+" not exists") 45 | continue 46 | 47 | newfile=os.path.join(saveBasePath,filename) 48 | shutil.copyfile(filePath, newfile) 49 | count+=1 50 | if(count%1000==0): 51 | print(count) -------------------------------------------------------------------------------- /DETRAC/DETRAC_xmlParser.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import xml.etree.ElementTree as ET 4 | from xml.dom.minidom import Document 5 | import os 6 | import cv2 7 | import time 8 | 9 | def ConvertVOCXml(file_path="",file_name=""): 10 | tree = ET.parse(file_name) 11 | root = tree.getroot() 12 | # print(root.tag) 13 | 14 | num=0 #计数 15 | #读xml操作 16 | 17 | frame_lists=[] 18 | output_file_name="" 19 | for child in root: 20 | 21 | if(child.tag=="frame"): 22 | # 创建dom文档 23 | doc = Document() 24 | # 创建根节点 25 | annotation = doc.createElement('annotation') 26 | # 根节点插入dom树 27 | doc.appendChild(annotation) 28 | 29 | #print(child.tag, child.attrib["num"]) 30 | pic_id= child.attrib["num"].zfill(5) 31 | #print(pic_id) 32 | output_file_name=root.attrib["name"]+"__img"+pic_id+".xml" 33 | # print(output_file_name) 34 | 35 | folder = doc.createElement("folder") 36 | folder.appendChild(doc.createTextNode("VOC2007")) 37 | annotation.appendChild(folder) 38 | 39 | filename = doc.createElement("filename") 40 | pic_name=root.attrib["name"]+"__img"+pic_id+".jpg" 41 | filename.appendChild(doc.createTextNode(pic_name)) 42 | annotation.appendChild(filename) 43 | 44 | sizeimage = doc.createElement("size") 45 | imagewidth = doc.createElement("width") 46 | imageheight = doc.createElement("height") 47 | imagedepth = doc.createElement("depth") 48 | 49 | imagewidth.appendChild(doc.createTextNode("960")) 50 | imageheight.appendChild(doc.createTextNode("540")) 51 | imagedepth.appendChild(doc.createTextNode("3")) 52 | 53 | sizeimage.appendChild(imagedepth) 54 | sizeimage.appendChild(imagewidth) 55 | sizeimage.appendChild(imageheight) 56 | annotation.appendChild(sizeimage) 57 | 58 | target_list=child.getchildren()[0] #获取target_list 59 | #print(target_list.tag) 60 | object=None 61 | for target in target_list: 62 | if(target.tag=="target"): 63 | #print(target.tag) 64 | object = doc.createElement('object') 65 | bndbox = doc.createElement("bndbox") 66 | 67 | for target_child in target: 68 | if(target_child.tag=="box"): 69 | xmin = doc.createElement("xmin") 70 | ymin = doc.createElement("ymin") 71 | xmax = doc.createElement("xmax") 72 | ymax = doc.createElement("ymax") 73 | xmin_value=int(float(target_child.attrib["left"])) 74 | ymin_value=int(float(target_child.attrib["top"])) 75 | box_width_value=int(float(target_child.attrib["width"])) 76 | box_height_value=int(float(target_child.attrib["height"])) 77 | xmin.appendChild(doc.createTextNode(str(xmin_value))) 78 | ymin.appendChild(doc.createTextNode(str(ymin_value))) 79 | if(xmin_value+box_width_value>960): 80 | xmax.appendChild(doc.createTextNode(str(960))) 81 | else: 82 | xmax.appendChild(doc.createTextNode(str(xmin_value+box_width_value))) 83 | if(ymin_value+box_height_value>540): 84 | ymax.appendChild(doc.createTextNode(str(540))) 85 | else: 86 | ymax.appendChild(doc.createTextNode(str(ymin_value+box_height_value))) 87 | 88 | if(target_child.tag=="attribute"): 89 | name = doc.createElement('name') 90 | pose=doc.createElement('pose') 91 | truncated=doc.createElement('truncated') 92 | difficult=doc.createElement('difficult') 93 | 94 | name.appendChild(doc.createTextNode("car")) 95 | pose.appendChild(doc.createTextNode("Left")) #随意指定 96 | truncated.appendChild(doc.createTextNode("0")) #随意指定 97 | difficult.appendChild(doc.createTextNode("0")) #随意指定 98 | 99 | 100 | object.appendChild(name) 101 | object.appendChild(pose) 102 | object.appendChild(truncated) 103 | object.appendChild(difficult) 104 | 105 | bndbox.appendChild(xmin) 106 | bndbox.appendChild(ymin) 107 | bndbox.appendChild(xmax) 108 | bndbox.appendChild(ymax) 109 | object.appendChild(bndbox) 110 | annotation.appendChild(object) 111 | 112 | 113 | file_path_out=os.path.join(file_path,output_file_name) 114 | f = open(file_path_out, 'w') 115 | f.write(doc.toprettyxml(indent=' ' * 4)) 116 | f.close() 117 | num=num+1 118 | return num 119 | 120 | 121 | 122 | 123 | ''' 124 | 画方框 125 | ''' 126 | def bboxes_draw_on_img(img, bbox, color=[255, 0, 0], thickness=2): 127 | 128 | # Draw bounding box... 129 | print(bbox) 130 | p1 = (int(float(bbox["xmin"])), int(float(bbox["ymin"]))) 131 | p2 = (int(float(bbox["xmax"])), int(float(bbox["ymax"]))) 132 | cv2.rectangle(img, p1, p2, color, thickness) 133 | 134 | 135 | def visualization_image(image_name,xml_file_name): 136 | tree = ET.parse(xml_file_name) 137 | root = tree.getroot() 138 | 139 | object_lists=[] 140 | for child in root: 141 | if(child.tag=="folder"): 142 | print(child.tag, child.text) 143 | elif (child.tag == "filename"): 144 | print(child.tag, child.text) 145 | elif (child.tag == "size"): #解析size 146 | for size_child in child: 147 | if(size_child.tag=="width"): 148 | print(size_child.tag,size_child.text) 149 | elif (size_child.tag == "height"): 150 | print(size_child.tag, size_child.text) 151 | elif (size_child.tag == "depth"): 152 | print(size_child.tag, size_child.text) 153 | elif (child.tag == "object"): #解析object 154 | singleObject={} 155 | for object_child in child: 156 | if (object_child.tag == "name"): 157 | # print(object_child.tag,object_child.text) 158 | singleObject["name"] = object_child.text 159 | elif (object_child.tag == "bndbox"): 160 | for bndbox_child in object_child: 161 | if (bndbox_child.tag == "xmin"): 162 | singleObject["xmin"] = bndbox_child.text 163 | # print(bndbox_child.tag, bndbox_child.text) 164 | elif (bndbox_child.tag == "ymin"): 165 | # print(bndbox_child.tag, bndbox_child.text) 166 | singleObject["ymin"] = bndbox_child.text 167 | elif (bndbox_child.tag == "xmax"): 168 | singleObject["xmax"] = bndbox_child.text 169 | elif (bndbox_child.tag == "ymax"): 170 | singleObject["ymax"] = bndbox_child.text 171 | object_length=len(singleObject) 172 | if(object_length>0): 173 | object_lists.append(singleObject) 174 | img = cv2.imread(image_name) 175 | for object_coordinate in object_lists: 176 | bboxes_draw_on_img(img,object_coordinate) 177 | cv2.imshow("capture", img) 178 | cv2.waitKey (0) 179 | cv2.destroyAllWindows() 180 | 181 | 182 | if ( __name__ == "__main__"): 183 | #print("main") 184 | basePath="DETRAC-Train-Annotations-XML" 185 | totalxml=os.listdir(basePath) 186 | total_num=0 187 | flag=False 188 | print("正在转换") 189 | saveBasePath="xml_test" 190 | if os.path.exists(saveBasePath)==False: #判断文件夹是否存在 191 | os.makedirs(saveBasePath) 192 | 193 | #ConvertVOCXml(file_path="samplexml",file_name="000009.xml") 194 | # Start time 195 | start = time.time() 196 | log=open("xml_statistical.txt","w") #分析日志，进行排错 197 | for xml in totalxml: 198 | file_name=os.path.join(basePath,xml) 199 | print(file_name) 200 | num=ConvertVOCXml(file_path=saveBasePath,file_name=file_name) 201 | print(num) 202 | total_num=total_num+num 203 | log.write(file_name+" "+str(num)+"\n") 204 | # End time 205 | end = time.time() 206 | seconds=end-start 207 | print( "Time taken : {0} seconds".format(seconds)) 208 | print(total_num) 209 | log.write(str(total_num)+"\n") 210 | visualization_image("Insight-MVT_Annotation_Train/MVI_40212/img00396.jpg","xml_test/MVI_40212__img00396.xml") 211 | -------------------------------------------------------------------------------- /DETRAC/compare.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | xml_file=open("xml_statistical.txt","r") 4 | pic_lines=open("picture_statistical.txt","r") 5 | 6 | xml_lines=xml_file.readlines() 7 | xml_dict={} 8 | xml_total_num=len(xml_lines) 9 | for j in range(0,xml_total_num-1): 10 | # print(xml_lines[j]) 11 | folder=xml_lines[j].split(".")[0].split("/")[1] 12 | # print(folder) 13 | xml_dict[folder]=xml_lines[j].split(" ")[1] 14 | 15 | # print(xml_dict) 16 | 17 | pic_lines=pic_lines.readlines() 18 | num=len(pic_lines) 19 | num_of_gap=0 20 | for i in range(0,num-1): 21 | folder=pic_lines[i].split(" ")[0].split("/")[1] 22 | xml_num=int(xml_dict[folder]) 23 | picture_num=int(pic_lines[i].split(" ")[1]) 24 | if(xml_num!=picture_num): 25 | print(pic_lines[i]) 26 | print(xml_num) 27 | num_of_gap=num_of_gap+(picture_num-xml_num) 28 | print(picture_num) 29 | print(num_of_gap) 30 | # def migrate_xml(): 31 | -------------------------------------------------------------------------------- /DETRAC/generate_tfrecord.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | # From tensorflow/models/ 4 | # Create train data: 5 | python generate_tfrecord.py --csv_input=data/train_labels.csv --output_path=train.record 6 | 7 | # Create test data: 8 | python generate_tfrecord.py --csv_input=data/test_labels.csv --output_path=test.record 9 | """ 10 | from __future__ import division 11 | from __future__ import print_function 12 | from __future__ import absolute_import 13 | 14 | import os 15 | import io 16 | import pandas as pd 17 | import tensorflow as tf 18 | 19 | from PIL import Image 20 | from object_detection.utils import dataset_util 21 | from collections import namedtuple, OrderedDict 22 | 23 | flags = tf.app.flags 24 | flags.DEFINE_string('csv_input', '', 'Path to the CSV input') 25 | flags.DEFINE_string('output_path', '', 'Path to output TFRecord') 26 | FLAGS = flags.FLAGS 27 | 28 | 29 | # TO-DO replace this with label map 30 | def class_text_to_int(row_label): 31 | if row_label == 'car': 32 | return 1 33 | elif row_label == 'person': 34 | return 2 35 | else: 36 | None 37 | 38 | 39 | def split(df, group): 40 | data = namedtuple('data', ['filename', 'object']) 41 | gb = df.groupby(group) 42 | return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)] 43 | 44 | 45 | def create_tf_example(group, path): 46 | with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid: 47 | encoded_jpg = fid.read() 48 | encoded_jpg_io = io.BytesIO(encoded_jpg) 49 | image = Image.open(encoded_jpg_io) 50 | width, height = image.size 51 | 52 | filename = group.filename.encode('utf8') 53 | image_format = b'jpg' 54 | xmins = [] 55 | xmaxs = [] 56 | ymins = [] 57 | ymaxs = [] 58 | classes_text = [] 59 | classes = [] 60 | 61 | for index, row in group.object.iterrows(): 62 | xmins.append(row['xmin'] / width) 63 | xmaxs.append(row['xmax'] / width) 64 | ymins.append(row['ymin'] / height) 65 | ymaxs.append(row['ymax'] / height) 66 | classes_text.append(row['class'].encode('utf8')) 67 | classes.append(class_text_to_int(row['class'])) 68 | 69 | tf_example = tf.train.Example(features=tf.train.Features(feature={ 70 | 'image/height': dataset_util.int64_feature(height), 71 | 'image/width': dataset_util.int64_feature(width), 72 | 'image/filename': dataset_util.bytes_feature(filename), 73 | 'image/source_id': dataset_util.bytes_feature(filename), 74 | 'image/encoded': dataset_util.bytes_feature(encoded_jpg), 75 | 'image/format': dataset_util.bytes_feature(image_format), 76 | 'image/object/bbox/xmin': dataset_util.float_list_feature(xmins), 77 | 'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs), 78 | 'image/object/bbox/ymin': dataset_util.float_list_feature(ymins), 79 | 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs), 80 | 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), 81 | 'image/object/class/label': dataset_util.int64_list_feature(classes), 82 | })) 83 | return tf_example 84 | 85 | 86 | def main(_): 87 | writer = tf.python_io.TFRecordWriter(FLAGS.output_path) 88 | path = os.path.join(os.getcwd(), 'picture_test') 89 | examples = pd.read_csv(FLAGS.csv_input) 90 | grouped = split(examples, 'filename') 91 | num=0 92 | for group in grouped: 93 | num+=1 94 | tf_example = create_tf_example(group, path) 95 | writer.write(tf_example.SerializeToString()) 96 | if(num%100==0): #每完成100个转换，打印一次 97 | print(num) 98 | 99 | writer.close() 100 | output_path = os.path.join(os.getcwd(), FLAGS.output_path) 101 | print('Successfully created the TFRecords: {}'.format(output_path)) 102 | 103 | 104 | if __name__ == '__main__': 105 | tf.app.run() 106 | -------------------------------------------------------------------------------- /DETRAC/generate_tfrecord.sh: -------------------------------------------------------------------------------- 1 | python3 generate_tfrecord.py --csv_input=data/DETRAC_train_labels.csv --output_path=data/DETRAC_train.tfrecord 2 | python3 generate_tfrecord.py --csv_input=data/DETRAC_test_labels.csv --output_path=data/DETRAC_test.tfrecord 3 | python3 generate_tfrecord.py --csv_input=data/DETRAC_validation_labels.csv --output_path=data/DETRAC_validation.tfrecord 4 | -------------------------------------------------------------------------------- /DETRAC/run.sh: -------------------------------------------------------------------------------- 1 | python3 DETRAC_xmlParser.py -------------------------------------------------------------------------------- /DETRAC/statistics_analysi.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | #原图片的地址 5 | pictureBasePath=r"Insight-MVT_Annotation_Train" 6 | picturefolder = os.listdir(pictureBasePath) 7 | 8 | num=0 9 | log=open("picture_statistical.txt","w") 10 | for pic_folder_item in picturefolder: 11 | pic_file_path=os.path.join(pictureBasePath,pic_folder_item) 12 | print(pic_file_path) 13 | file_lists=os.listdir(pic_file_path) 14 | num=num+len(file_lists) 15 | print(len(file_lists)) 16 | log.write(pic_file_path+" "+str(len(file_lists))+"\n") 17 | print(num) 18 | log.write(str(num)+"\n") 19 | -------------------------------------------------------------------------------- /DETRAC/train_test_split.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import random 4 | import time 5 | import shutil 6 | 7 | xmlfilepath=r'xml_test' 8 | saveBasePath=r"./annotations" 9 | 10 | trainval_percent=0.9 11 | train_percent=0.85 12 | total_xml = os.listdir(xmlfilepath) 13 | num=len(total_xml) 14 | list=range(num) 15 | tv=int(num*trainval_percent) 16 | tr=int(tv*train_percent) 17 | trainval= random.sample(list,tv) 18 | train=random.sample(trainval,tr) 19 | 20 | print("train and val size",tv) 21 | print("train size",tr) 22 | # print(total_xml[1]) 23 | start = time.time() 24 | 25 | # print(trainval) 26 | # print(train) 27 | 28 | test_num=0 29 | val_num=0 30 | train_num=0 31 | # for directory in ['train','test',"val"]: 32 | # xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 33 | # if(not os.path.exists(xml_path)): 34 | # os.mkdir(xml_path) 35 | # # shutil.copyfile(filePath, newfile) 36 | # print(xml_path) 37 | for i in list: 38 | name=total_xml[i] 39 | # print(i) 40 | if i in trainval: #train and val set 41 | # ftrainval.write(name) 42 | if i in train: 43 | # ftrain.write(name) 44 | # print("train") 45 | # print(name) 46 | # print("train: "+name+" "+str(train_num)) 47 | directory="train" 48 | train_num+=1 49 | if(train_num%1000==0): 50 | print("train: "+name+" "+str(train_num)) 51 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 52 | if(not os.path.exists(xml_path)): 53 | os.mkdir(xml_path) 54 | filePath=os.path.join(xmlfilepath,name) 55 | newfile=os.path.join(saveBasePath,os.path.join(directory,name)) 56 | shutil.copyfile(filePath, newfile) 57 | 58 | else: 59 | # fval.write(name) 60 | # print("val") 61 | # print("val: "+name+" "+str(val_num)) 62 | directory="validation" 63 | if(val_num%1000==0): 64 | print("val: "+name+" "+str(val_num)) 65 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 66 | if(not os.path.exists(xml_path)): 67 | os.mkdir(xml_path) 68 | val_num+=1 69 | filePath=os.path.join(xmlfilepath,name) 70 | newfile=os.path.join(saveBasePath,os.path.join(directory,name)) 71 | shutil.copyfile(filePath, newfile) 72 | # print(name) 73 | else: #test set 74 | # ftest.write(name) 75 | # print("test") 76 | # print("test: "+name+" "+str(test_num)) 77 | directory="test" 78 | if(test_num%1000==0): 79 | print("test: "+name+" "+str(test_num)) 80 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 81 | if(not os.path.exists(xml_path)): 82 | os.mkdir(xml_path) 83 | test_num+=1 84 | filePath=os.path.join(xmlfilepath,name) 85 | newfile=os.path.join(saveBasePath,os.path.join(directory,name)) 86 | shutil.copyfile(filePath, newfile) 87 | # print(name) 88 | 89 | # End time 90 | end = time.time() 91 | seconds=end-start 92 | print("train total : "+str(train_num)) 93 | print("validation total : "+str(val_num)) 94 | print("test total : "+str(test_num)) 95 | total_num=train_num+val_num+test_num 96 | print("total number : "+str(total_num)) 97 | print( "Time taken : {0} seconds".format(seconds)) 98 | -------------------------------------------------------------------------------- /DETRAC/voc_data_migrate.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import random 4 | import shutil 5 | 6 | #xml路径的地址 7 | XmlPath=r'xml_test' 8 | #原图片的地址 9 | pictureBasePath=r"Insight-MVT_Annotation_Train" 10 | #保存图片的地址 11 | saveBasePath=r"picture_test" 12 | 13 | total_xml = os.listdir(XmlPath) 14 | num=len(total_xml) 15 | list=range(num) 16 | if os.path.exists(saveBasePath)==False: #判断文件夹是否存在 17 | os.makedirs(saveBasePath) 18 | 19 | 20 | for xml in total_xml: 21 | xml_temp=xml.split("__") 22 | folder=xml_temp[0] 23 | filename=xml_temp[1].split(".")[0]+".jpg" 24 | # print(folder) 25 | # print(filename) 26 | temp_pictureBasePath=os.path.join(pictureBasePath,folder) 27 | filePath=os.path.join(temp_pictureBasePath,filename) 28 | # print(filePath) 29 | newfile=xml.split(".")[0]+".jpg" 30 | newfile_path=os.path.join(saveBasePath,newfile) 31 | print(newfile_path) 32 | shutil.copyfile(filePath, newfile_path) 33 | print("xml file total number",num) 34 | -------------------------------------------------------------------------------- /DETRAC/xml_to_csv.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import pandas as pd 4 | import xml.etree.ElementTree as ET 5 | 6 | 7 | def xml_to_csv(path): 8 | xml_list = [] 9 | for xml_file in glob.glob(path + '/*.xml'): 10 | tree = ET.parse(xml_file) 11 | root = tree.getroot() 12 | # print(root) 13 | print(root.find('filename').text) 14 | for member in root.findall('object'): 15 | value = (root.find('filename').text, 16 | int(root.find('size')[1].text), #width 17 | int(root.find('size')[2].text), #height 18 | member[0].text, 19 | int(member[4][0].text), 20 | int(float(member[4][1].text)), 21 | int(member[4][2].text), 22 | int(member[4][3].text) 23 | ) 24 | xml_list.append(value) 25 | column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax'] 26 | xml_df = pd.DataFrame(xml_list, columns=column_name) 27 | return xml_df 28 | 29 | 30 | def main(): 31 | for directory in ['train','test','validation']: 32 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 33 | # image_path = os.path.join(os.getcwd(), 'merged_xml') 34 | xml_df = xml_to_csv(xml_path) 35 | # xml_df.to_csv('whsyxt.csv', index=None) 36 | xml_df.to_csv('data/DETRAC_{}_labels.csv'.format(directory), index=None) 37 | print('Successfully converted xml to csv.') 38 | 39 | 40 | main() 41 | -------------------------------------------------------------------------------- /KITTI/txt_parser.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import os 4 | ''' 5 | KITTI_LABELS = { 6 | 'none': (0, 'Background'), 7 | 'Car': (1, 'Vehicle'), 8 | 'Van': (2, 'Vehicle'), 9 | 'Truck': (3, 'Vehicle'), 10 | 'Cyclist': (4, 'Vehicle'), 11 | 'Pedestrian': (5, 'Person'), 12 | 'Person_sitting': (6, 'Person'), 13 | 'Tram': (7, 'Vehicle'), 14 | 'Misc': (8, 'Misc'), 15 | 'DontCare': (9, 'DontCare'), 16 | } 17 | ''' 18 | 19 | def filter_KITTI_Data(txt_file_path,output_file_base_path): 20 | file = open(txt_file_path) 21 | #results/****.txt 22 | # print(txt_file_path) 23 | txt_name=txt_file_path.split("/")[1] 24 | output_file_path=os.path.join(output_file_base_path,txt_name) 25 | print(output_file_path) 26 | output_object = open(output_file_path, 'w') 27 | 28 | file_lines=file.readlines() 29 | for line in file_lines: 30 | # print(line) 31 | line_array=line.split(" ") 32 | # print(line_array) 33 | if(line_array[0]=="Truck"): 34 | line=str(line).replace("Truck", "Car") 35 | elif(line_array[0]=="Van"): 36 | line=str(line).replace("Van", "Car") 37 | # print(line) 38 | elif(line_array[0]=="Car"): 39 | line_array[0]="Car" 40 | elif(line_array[0]=="Cyclist"): 41 | # line_array[0]="Cyclist" 42 | line=str(line).replace("Cyclist", "Car") 43 | elif(line_array[0]=="Person_sitting"): 44 | # line_array[0]="Person" 45 | line=str(line).replace("Person_sitting", "Person") 46 | elif(line_array[0]=="Pedestrian"): 47 | line_array[0]="Person" 48 | line=str(line).replace("Pedestrian", "Person") 49 | elif(line_array[0]=="Tram"): 50 | # line_array[0]="Car" 51 | line=str(line).replace("Tram", "Car") 52 | elif(line_array[0]=="none"): 53 | line_array[0]="none" 54 | else: 55 | continue 56 | output_object.write(line) 57 | output_object.close( ) 58 | if __name__ == '__main__': 59 | base_path=r"label_2" 60 | file_lists=os.listdir(base_path) 61 | output_path="results" 62 | num=0 63 | for single_file in file_lists: 64 | single_file_path=os.path.join(base_path,single_file) 65 | # print(single_file_path,output_path) 66 | filter_KITTI_Data(single_file_path,output_path) 67 | num=num+1 68 | print("total num of files %s" %(num)) 69 | print("already finished") 70 | # print(file_lists) 71 | # filter_KITTI_Data() 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # datasets-preprocessing-for-object-detection 2 | this repository includes some python parser scripts for converting other public datasets into voc data format 3 | # note 4 | - I don't mean to provide the usage for these scripts, it's very simple to modified these files 5 | - most files in this repository are very similar, you can run for your own purpose 6 | # environment 7 | - python 3.5 -------------------------------------------------------------------------------- /VOC2007_2012/ImageSets_Convert.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import random 4 | import time 5 | 6 | xmlfilepath=r'./VOC2007/Annotations' 7 | saveBasePath=r"./" 8 | 9 | trainval_percent=0.8 10 | train_percent=0.85 11 | total_xml = os.listdir(xmlfilepath) 12 | num=len(total_xml) 13 | list=range(num) 14 | tv=int(num*trainval_percent) 15 | tr=int(tv*train_percent) 16 | trainval= random.sample(list,tv) 17 | train=random.sample(trainval,tr) 18 | 19 | print("train and val size",tv) 20 | print("traub suze",tr) 21 | ftrainval = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/trainval.txt'), 'w') 22 | ftest = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/test.txt'), 'w') 23 | ftrain = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/train.txt'), 'w') 24 | fval = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/val.txt'), 'w') 25 | # Start time 26 | start = time.time() 27 | for i in list: 28 | name=total_xml[i][:-4]+'\n' 29 | if i in trainval: 30 | ftrainval.write(name) 31 | if i in train: 32 | ftrain.write(name) 33 | else: 34 | fval.write(name) 35 | else: 36 | ftest.write(name) 37 | # End time 38 | end = time.time() 39 | seconds=end-start 40 | print( "Time taken : {0} seconds".format(seconds)) 41 | 42 | ftrainval.close() 43 | ftrain.close() 44 | fval.close() 45 | ftest .close() 46 | -------------------------------------------------------------------------------- /VOC2007_2012/generate_tfrecord.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | # From tensorflow/models/ 4 | # Create train data: 5 | python generate_tfrecord.py --csv_input=data/train_labels.csv --output_path=train.record 6 | 7 | # Create test data: 8 | python generate_tfrecord.py --csv_input=data/test_labels.csv --output_path=test.record 9 | """ 10 | from __future__ import division 11 | from __future__ import print_function 12 | from __future__ import absolute_import 13 | 14 | import os 15 | import io 16 | import pandas as pd 17 | import tensorflow as tf 18 | 19 | from PIL import Image 20 | from object_detection.utils import dataset_util 21 | from collections import namedtuple, OrderedDict 22 | 23 | flags = tf.app.flags 24 | flags.DEFINE_string('csv_input', '', 'Path to the CSV input') 25 | flags.DEFINE_string('output_path', '', 'Path to output TFRecord') 26 | FLAGS = flags.FLAGS 27 | 28 | 29 | # TO-DO replace this with label map 30 | def class_text_to_int(row_label): 31 | if row_label == 'car': 32 | return 1 33 | elif row_label == 'person': 34 | return 2 35 | else: 36 | None 37 | 38 | 39 | def split(df, group): 40 | data = namedtuple('data', ['filename', 'object']) 41 | gb = df.groupby(group) 42 | return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)] 43 | 44 | 45 | def create_tf_example(group, path): 46 | with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid: 47 | encoded_jpg = fid.read() 48 | encoded_jpg_io = io.BytesIO(encoded_jpg) 49 | image = Image.open(encoded_jpg_io) 50 | width, height = image.size 51 | 52 | filename = group.filename.encode('utf8') 53 | image_format = b'jpg' 54 | xmins = [] 55 | xmaxs = [] 56 | ymins = [] 57 | ymaxs = [] 58 | classes_text = [] 59 | classes = [] 60 | 61 | for index, row in group.object.iterrows(): 62 | xmins.append(row['xmin'] / width) 63 | xmaxs.append(row['xmax'] / width) 64 | ymins.append(row['ymin'] / height) 65 | ymaxs.append(row['ymax'] / height) 66 | classes_text.append(row['class'].encode('utf8')) 67 | classes.append(class_text_to_int(row['class'])) 68 | 69 | tf_example = tf.train.Example(features=tf.train.Features(feature={ 70 | 'image/height': dataset_util.int64_feature(height), 71 | 'image/width': dataset_util.int64_feature(width), 72 | 'image/filename': dataset_util.bytes_feature(filename), 73 | 'image/source_id': dataset_util.bytes_feature(filename), 74 | 'image/encoded': dataset_util.bytes_feature(encoded_jpg), 75 | 'image/format': dataset_util.bytes_feature(image_format), 76 | 'image/object/bbox/xmin': dataset_util.float_list_feature(xmins), 77 | 'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs), 78 | 'image/object/bbox/ymin': dataset_util.float_list_feature(ymins), 79 | 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs), 80 | 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), 81 | 'image/object/class/label': dataset_util.int64_list_feature(classes), 82 | })) 83 | return tf_example 84 | 85 | 86 | def main(_): 87 | writer = tf.python_io.TFRecordWriter(FLAGS.output_path) 88 | path = os.path.join(os.getcwd(), 'images') 89 | examples = pd.read_csv(FLAGS.csv_input) 90 | grouped = split(examples, 'filename') 91 | num=0 92 | for group in grouped: 93 | num+=1 94 | tf_example = create_tf_example(group, path) 95 | writer.write(tf_example.SerializeToString()) 96 | if(num%100==0): #每完成100个转换，打印一次 97 | print(num) 98 | 99 | writer.close() 100 | output_path = os.path.join(os.getcwd(), FLAGS.output_path) 101 | print('Successfully created the TFRecords: {}'.format(output_path)) 102 | 103 | 104 | if __name__ == '__main__': 105 | tf.app.run() 106 | -------------------------------------------------------------------------------- /VOC2007_2012/test_xml_to_csv.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import os 3 | import tempfile 4 | import unittest 5 | import xml_to_csv 6 | from xml.etree import ElementTree as ET 7 | 8 | 9 | class XMLToCSVTest(unittest.TestCase): 10 | def test_one_raccoon_one_xml(self): 11 | xml_file_one = """ 12 | 13 | images 14 | raccoon-1.png 15 | raccoon-1.png 16 | 17 | Unknown 18 | 19 | 20 | 3 21 | 256 22 | 256 23 | 24 | 0 25 | 26 | raccoon 27 | Unspecified 28 | 0 29 | 0 30 | 31 | 96 32 | 96 33 | 128 34 | 128 35 | 36 | 37 | 38 | """ 39 | 40 | xml = ET.fromstring(xml_file_one) 41 | with tempfile.TemporaryDirectory() as tmpdirname: 42 | tree = ET.ElementTree(xml) 43 | tree.write(tmpdirname + '/test_raccoon_one.xml') 44 | raccoon_df = xml_to_csv.xml_to_csv(tmpdirname) 45 | self.assertEqual(raccoon_df.columns.values.tolist(), 46 | ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']) 47 | self.assertEqual(raccoon_df.values.tolist()[0], ['raccoon-1.png', 256, 256, 'raccoon', 96, 96, 128, 128]) 48 | 49 | def test_multiple_raccoon_one_xml(self): 50 | xml_file_one = """ 51 | 52 | images 53 | raccoon-1.png 54 | raccoon-1.png 55 | 56 | Unknown 57 | 58 | 59 | 256 60 | 256 61 | 3 62 | 63 | 0 64 | 65 | raccoon 66 | Unspecified 67 | 0 68 | 0 69 | 70 | 96 71 | 96 72 | 128 73 | 128 74 | 75 | 76 | 77 | raccoon 78 | Unspecified 79 | 0 80 | 0 81 | 82 | 32 83 | 32 84 | 64 85 | 64 86 | 87 | 88 | 89 | """ 90 | 91 | xml = ET.fromstring(xml_file_one) 92 | with tempfile.TemporaryDirectory() as tmpdirname: 93 | tree = ET.ElementTree(xml) 94 | tree.write(tmpdirname + '/test_raccoon_one.xml') 95 | raccoon_df = xml_to_csv.xml_to_csv(tmpdirname) 96 | self.assertEqual(raccoon_df.columns.values.tolist(), 97 | ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']) 98 | self.assertEqual(raccoon_df.values.tolist()[0], ['raccoon-1.png', 256, 256, 'raccoon', 96, 96, 128, 128]) 99 | self.assertEqual(raccoon_df.values.tolist()[1], ['raccoon-1.png', 256, 256, 'raccoon', 32, 32, 64, 64]) 100 | 101 | def test_one_raccoon_multiple_xml(self): 102 | xml_file_one = """ 103 | 104 | images 105 | raccoon-1.png 106 | raccoon-1.png 107 | 108 | Unknown 109 | 110 | 111 | 256 112 | 256 113 | 3 114 | 115 | 0 116 | 117 | raccoon 118 | Unspecified 119 | 0 120 | 0 121 | 122 | 96 123 | 96 124 | 128 125 | 128 126 | 127 | 128 | 129 | """ 130 | xml_file_two = """ 131 | 132 | images 133 | raccoon-2.png 134 | raccoon-2.png 135 | 136 | Unknown 137 | 138 | 139 | 256 140 | 256 141 | 3 142 | 143 | 0 144 | 145 | raccoon 146 | Unspecified 147 | 0 148 | 0 149 | 150 | 128 151 | 128 152 | 194 153 | 194 154 | 155 | 156 | 157 | """ 158 | xml_list = [xml_file_one, xml_file_two] 159 | tmpdirname = tempfile.mkdtemp() 160 | for index, x in enumerate(xml_list): 161 | xml = ET.fromstring(x) 162 | tree = ET.ElementTree(xml) 163 | tree.write(tmpdirname + '/test_raccoon_{}.xml'.format(index)) 164 | 165 | raccoon_df = xml_to_csv.xml_to_csv(tmpdirname) 166 | self.assertEqual(raccoon_df.columns.values.tolist(), 167 | ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']) 168 | self.assertEqual(raccoon_df.values.tolist()[0], ['raccoon-1.png', 256, 256, 'raccoon', 96, 96, 128, 128]) 169 | self.assertEqual(raccoon_df.values.tolist()[1], ['raccoon-2.png', 256, 256, 'raccoon', 128, 128, 194, 194]) 170 | shutil.rmtree(tmpdirname) 171 | -------------------------------------------------------------------------------- /VOC2007_2012/train_test_split.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import random 4 | import time 5 | import shutil 6 | 7 | xmlfilepath=r'merged_xml' 8 | saveBasePath=r"./annotations" 9 | 10 | trainval_percent=0.9 11 | train_percent=0.85 12 | total_xml = os.listdir(xmlfilepath) 13 | num=len(total_xml) 14 | list=range(num) 15 | tv=int(num*trainval_percent) 16 | tr=int(tv*train_percent) 17 | trainval= random.sample(list,tv) 18 | train=random.sample(trainval,tr) 19 | 20 | print("train and val size",tv) 21 | print("train size",tr) 22 | # print(total_xml[1]) 23 | start = time.time() 24 | 25 | # print(trainval) 26 | # print(train) 27 | 28 | test_num=0 29 | val_num=0 30 | train_num=0 31 | # for directory in ['train','test',"val"]: 32 | # xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 33 | # if(not os.path.exists(xml_path)): 34 | # os.mkdir(xml_path) 35 | # # shutil.copyfile(filePath, newfile) 36 | # print(xml_path) 37 | for i in list: 38 | name=total_xml[i] 39 | # print(i) 40 | if i in trainval: #train and val set 41 | # ftrainval.write(name) 42 | if i in train: 43 | # ftrain.write(name) 44 | # print("train") 45 | # print(name) 46 | # print("train: "+name+" "+str(train_num)) 47 | directory="train" 48 | train_num+=1 49 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 50 | if(not os.path.exists(xml_path)): 51 | os.mkdir(xml_path) 52 | filePath=os.path.join(xmlfilepath,name) 53 | newfile=os.path.join(saveBasePath,os.path.join(directory,name)) 54 | shutil.copyfile(filePath, newfile) 55 | 56 | else: 57 | # fval.write(name) 58 | # print("val") 59 | # print("val: "+name+" "+str(val_num)) 60 | directory="validation" 61 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 62 | if(not os.path.exists(xml_path)): 63 | os.mkdir(xml_path) 64 | val_num+=1 65 | filePath=os.path.join(xmlfilepath,name) 66 | newfile=os.path.join(saveBasePath,os.path.join(directory,name)) 67 | shutil.copyfile(filePath, newfile) 68 | # print(name) 69 | else: #test set 70 | # ftest.write(name) 71 | # print("test") 72 | # print("test: "+name+" "+str(test_num)) 73 | directory="test" 74 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 75 | if(not os.path.exists(xml_path)): 76 | os.mkdir(xml_path) 77 | test_num+=1 78 | filePath=os.path.join(xmlfilepath,name) 79 | newfile=os.path.join(saveBasePath,os.path.join(directory,name)) 80 | shutil.copyfile(filePath, newfile) 81 | # print(name) 82 | 83 | # End time 84 | end = time.time() 85 | seconds=end-start 86 | print("train total : "+str(train_num)) 87 | print("validation total : "+str(val_num)) 88 | print("test total : "+str(test_num)) 89 | total_num=train_num+val_num+test_num 90 | print("total number : "+str(total_num)) 91 | print( "Time taken : {0} seconds".format(seconds)) 92 | ''' 93 | ftrainval = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/trainval.txt'), 'w') 94 | ftest = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/test.txt'), 'w') 95 | ftrain = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/train.txt'), 'w') 96 | fval = open(os.path.join(saveBasePath,'VOC2007/ImageSets/Main/val.txt'), 'w') 97 | # Start time 98 | 99 | for i in list: 100 | name=total_xml[i][:-4]+'\n' 101 | if i in trainval: 102 | ftrainval.write(name) 103 | if i in train: 104 | ftrain.write(name) 105 | else: 106 | fval.write(name) 107 | else: 108 | ftest.write(name) 109 | 110 | 111 | ftrainval.close() 112 | ftrain.close() 113 | fval.close() 114 | ftest .close() 115 | ''' 116 | -------------------------------------------------------------------------------- /VOC2007_2012/xml_to_csv.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import pandas as pd 4 | import xml.etree.ElementTree as ET 5 | 6 | 7 | def xml_to_csv(path): 8 | xml_list = [] 9 | for xml_file in glob.glob(path + '/*.xml'): 10 | tree = ET.parse(xml_file) 11 | root = tree.getroot() 12 | # print(root) 13 | print(root.find('filename').text) 14 | for member in root.findall('object'): 15 | value = (root.find('filename').text, 16 | int(root.find('size')[1].text), #width 17 | int(root.find('size')[2].text), #height 18 | member[0].text, 19 | int(member[4][0].text), 20 | int(float(member[4][1].text)), 21 | int(member[4][2].text), 22 | int(member[4][3].text) 23 | ) 24 | xml_list.append(value) 25 | column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax'] 26 | xml_df = pd.DataFrame(xml_list, columns=column_name) 27 | return xml_df 28 | 29 | 30 | def main(): 31 | for directory in ['train','test','validation']: 32 | xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory)) 33 | # image_path = os.path.join(os.getcwd(), 'merged_xml') 34 | xml_df = xml_to_csv(xml_path) 35 | # xml_df.to_csv('whsyxt.csv', index=None) 36 | xml_df.to_csv('data/whsyxt_{}_labels.csv'.format(directory), index=None) 37 | print('Successfully converted xml to csv.') 38 | 39 | 40 | main() 41 | --------------------------------------------------------------------------------