├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── bmw.txt ├── convert-dataset.py └── datasets ├── __init__.py ├── kitti.py ├── lisa.py ├── voc.py └── yolo.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Eddie Weill 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # convert-datasets 2 | Convert object detection datasets to different formats 3 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eweill/convert-datasets/6e2192807a7ef0146ca29b2480d13226165d6b05/__init__.py -------------------------------------------------------------------------------- /bmw.txt: -------------------------------------------------------------------------------- 1 | Label 2 | -------------------------------------------------------------------------------- /convert-dataset.py: -------------------------------------------------------------------------------- 1 | # Import necessary libraries 2 | import os, sys, shutil, glob, argparse 3 | import numpy as np 4 | from PIL import Image 5 | 6 | # Import datasets dependent files 7 | from datasets import kitti 8 | from datasets import lisa 9 | from datasets import voc 10 | from datasets import yolo 11 | 12 | def parse_args(): 13 | """ 14 | Definition: Parse command line arguments. 15 | 16 | Parameters: None 17 | Returns: args - list of arguments 18 | """ 19 | parser = argparse.ArgumentParser(description= 20 | 'Convert object detection datasets.') 21 | parser._action_groups.pop() 22 | required = parser.add_argument_group('required arguments') 23 | optional = parser.add_argument_group('optional arguments') 24 | required.add_argument('--from', 25 | dest='from_key', 26 | required=True, 27 | help='Format to convert dataset from.', 28 | choices=['kitti','lisa','voc','yolo'], 29 | type=str, nargs=1) 30 | required.add_argument('--from-path', 31 | dest='from_path', 32 | required=True, 33 | help='Path to dataset you wish to convert.', 34 | type=str, nargs=1) 35 | required.add_argument('--to', 36 | dest='to_key', 37 | required=True, 38 | help='Format to convert dataset to', 39 | choices=['kitti','lisa','voc','yolo'], 40 | type=str, nargs=1) 41 | required.add_argument('--to-path', 42 | dest='to_path', 43 | required=True, 44 | help='Path to output dataset to convert to.', 45 | type=str, nargs=1) 46 | optional.add_argument('-l', '--label', 47 | dest='label', 48 | required=False, 49 | help='Label file necessary for yolo conversion.', 50 | type=str, nargs=1) 51 | optional.add_argument('-v','--verbose', 52 | dest='verbose', 53 | required=False, 54 | help='Print out during execution of the script.') 55 | 56 | args = parser.parse_args() 57 | return args 58 | 59 | if __name__ == '__main__': 60 | # Parse command line arguments 61 | args = parse_args() 62 | 63 | # If conversion types are same, no conversion necessary (ex. both 'kitti') 64 | if args.from_key == args.to_key: 65 | print ("No conversion necessary.") 66 | exit(0) 67 | 68 | # If yolo is part of the conversion (either 'to' or 'from' type) 69 | if 'yolo' in args.to_key or 'yolo' in args.from_key: 70 | # Must contain a label file 71 | if not args.label: 72 | print ("Error: A label file is necessary for yolo conversion.") 73 | exit(0) 74 | 75 | # Parameters including the label file 76 | params = ("'" + args.from_path[0] + "', '" + args.to_path[0] + "', '" + 77 | args.label[0] + "'") 78 | 79 | # Otherwise set up parameters without a label file 80 | else: 81 | # Parameters without the label file 82 | params = ("'" + args.from_path[0] + "', '" + args.to_path[0] + "'") 83 | 84 | # Evaluate the conversion based on command line parameters 85 | eval (args.from_key[0] + '.' + args.to_key[0] + '(' + params + ')') 86 | 87 | print ("Conversion complete!!") 88 | -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eweill/convert-datasets/6e2192807a7ef0146ca29b2480d13226165d6b05/datasets/__init__.py -------------------------------------------------------------------------------- /datasets/kitti.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ########## KITTI format ########## 3 | """ 4 | Values Name Description 5 | ---------------------------------------------------------------------------- 6 | 1 type Describes the type of object: 'Car', 'Van', 'Truck', 7 | 'Pedestrian', 'Person_sitting', 'Cyclist', 'Tram', 8 | 'Misc' or 'DontCare' 9 | 1 truncated Float from 0 (non-truncated) to 1 (truncated), where 10 | truncated refers to the object leaving image boundaries 11 | 1 occluded Integer (0,1,2,3) indicating occlusion state: 12 | 0 = fully visible, 1 = partly occluded 13 | 2 = largely occluded, 3 = unknown 14 | 1 alpha Observation angle of object, ranging [-pi..pi] 15 | 4 bbox 2D bounding box of object in the image (0-based index): 16 | contains left, top, right, bottom pixel coordinates 17 | 3 dimensions 3D object dimensions: height, width, length (in meters) 18 | 3 location 3D object location x,y,z in camera coordinates (in meters) 19 | 1 rotation_y Rotation ry around Y-axis in camera coordinates [-pi..pi] 20 | 1 score Only for results: Float, indicating confidence in 21 | detection, needed for p/r curves, higher is better. 22 | """ 23 | ############################################################################### 24 | 25 | # Import necessary libraries 26 | import os, sys, shutil, glob, argparse 27 | import numpy as np 28 | from PIL import Image 29 | from lxml import etree 30 | 31 | python_version = sys.version_info.major 32 | 33 | ########################################################### 34 | ########## KITTI to YOLO Conversion ########## 35 | ########################################################### 36 | def determine_label_yolo(label, labels): 37 | """ 38 | Definition: Converts label to index in label set 39 | 40 | Parameters: label - label from file 41 | labels - list of labels 42 | Returns: index of label in labels (in str format) 43 | """ 44 | return str(labels.index(label)) 45 | 46 | def parse_labels_yolo(label_file, labels, img_width, img_height): 47 | """ 48 | Definition: Parses label files to extract label and bounding box 49 | coordinates. Converts (x1, y1, x1, y2) KITTI format to 50 | (x, y, width, height) normalized YOLO format. 51 | 52 | Parameters: label_file - file with KITTI label(s) inside 53 | labels - list of labels in dataset 54 | img_width - width of input image 55 | img_height - height of input image 56 | Return: all_labels - contains a list of labels for objects in image 57 | all_coords - contains a list of coordinate for objects in image 58 | """ 59 | lfile = open(label_file) 60 | coords = [] 61 | all_coords = [] 62 | all_labels = [] 63 | for line in lfile: 64 | l = line.split(" ") 65 | all_labels.append(determine_label_yolo(l[0], labels)) 66 | coords = list(map(int, list(map(float, l[4:8])))) 67 | x = float((float(coords[2]) + float(coords[0])) / 2.0) / float(img_width) 68 | y = float((float(coords[3]) + float(coords[1])) / 2.0) / float(img_height) 69 | width = float(float(coords[2]) - float(coords[0])) / float(img_width) 70 | height = float(float(coords[3]) - float(coords[1])) / float(img_height) 71 | tmp = [x, y, width, height] 72 | all_coords.append(tmp) 73 | lfile.close() 74 | return all_labels, all_coords 75 | 76 | def copy_images_yolo(kitti, yolo): 77 | """ 78 | Definition: Copy all images from the training and validation image sets 79 | in kitti format to training and validation image sets in yolo format. 80 | This means converting from .png to .jpg 81 | 82 | Parameters: kitti - path to kitti directory (contains 'train' and 'val') 83 | yolo - path to yolo output directory 84 | Returns: None 85 | """ 86 | for filename in glob.glob(os.path.join(kitti + "train/images/", "*.*")): 87 | shutil.copy(filename, yolo + "train/images/") 88 | for filename in glob.glob(os.path.join(kitti + "val/images/", "*.*")): 89 | shutil.copy(filename, yolo + "val/images/") 90 | 91 | for filename in glob.glob(os.path.join(yolo + "train/images", "*.*")): 92 | im = Image.open(filename) 93 | im.save(filename.split(".png")[0] + ".jpg", "jpeg") 94 | os.remove(filename) 95 | for filename in glob.glob(os.path.join(yolo + "val/images", "*.*")): 96 | im = Image.open(filename) 97 | im.save(filename.split(".png")[0] + ".jpg", "jpeg") 98 | os.remove(filename) 99 | 100 | def write_txt_files_yolo(yolo, f_train, f_val): 101 | """ 102 | Definition: Fill in a text file containing a list of all images in the 103 | training and validation sets. 104 | 105 | Parameters: yolo - path to yolo dataset directory (contains 'train' and 'val') 106 | f_train - file open for adding training examples 107 | f_val - file open for adding validation examples 108 | Returns: None 109 | """ 110 | for filename in glob.glob(os.path.join(yolo + "train/images/", "*.*")): 111 | f_train.write('%s\n' % (filename)) 112 | for filename in glob.glob(os.path.join(yolo + "val/images/", "*.*")): 113 | f_val.write('%s\n' % (filename)) 114 | 115 | def make_yolo_directories(yolo): 116 | """ 117 | Definition: Make directories for yolo images and labels. 118 | Removes previously created yolo image and label directories. 119 | 120 | Parameters: yolo - path to yolo directory to be created 121 | Returns: None 122 | """ 123 | if os.path.exists(yolo): 124 | if python_version == 3: 125 | prompt = input('Directory already exists. Overwrite? (yes, no): ') 126 | else: 127 | prompt = raw_input('Directory already exists. Overwrite? (yes, no): ') 128 | if prompt == 'no': 129 | exit(0) 130 | shutil.rmtree(yolo) 131 | os.makedirs(yolo) 132 | os.makedirs(yolo + "train") 133 | os.makedirs(yolo + "train/images") 134 | os.makedirs(yolo + "train/labels") 135 | os.makedirs(yolo + "val") 136 | os.makedirs(yolo + "val/images") 137 | os.makedirs(yolo + "val/labels") 138 | 139 | def yolo(kitti_dir, yolo_dir, label=None): 140 | print ("Converting kitti to yolo") 141 | 142 | # Split label file 143 | label_file = open(label) 144 | labels_split = label_file.read().split('\n') 145 | 146 | # Make all directories for yolo dataset 147 | make_yolo_directories(yolo_dir) 148 | 149 | # Iterate through kitti training data 150 | for f in os.listdir(kitti_dir + "train/labels/"): 151 | fname = (kitti_dir + "train/images/" + f).split(".txt")[0] + ".png" 152 | if os.path.isfile(fname): 153 | img = Image.open(fname) 154 | w, h = img.size 155 | img.close() 156 | labels, coords = parse_labels_yolo(os.path.join(kitti_dir + 157 | "train/labels/" + f), labels_split, w, h) 158 | yolof = open(yolo_dir + "train/labels/" + f, "a+") 159 | for l, c in zip(labels, coords): 160 | yolof.write(l + " " + str(c[0]) + " " + str(c[1]) + 161 | " " + str(c[2]) + " " + str(c[3]) + "\n") 162 | yolof.close() 163 | 164 | # Iterate through kitti validation data 165 | for f in os.listdir(kitti_dir + "val/labels/"): 166 | fname = (kitti_dir + "val/images/" + f).split(".txt")[0] + ".png" 167 | if os.path.isfile(fname): 168 | img = Image.open(fname) 169 | w, h = img.size 170 | img.close() 171 | labels, coords = parse_labels_yolo(os.path.join(kitti_dir + 172 | "val/labels/" + f), labels_split, w, h) 173 | yolof = open(yolo_dir + "val/labels/" + f, "a+") 174 | for l, c in zip(labels, coords): 175 | yolof.write(l + " " + str(c[0]) + " " + str(c[1]) + 176 | " " + str(c[2]) + " " + str(c[3]) + "\n") 177 | yolof.close() 178 | 179 | # Copy images from kitti to yolo 180 | copy_images_yolo(kitti_dir, yolo_dir) 181 | 182 | # Create train.txt and val.txt and populate them 183 | f_train = open(yolo_dir + "train.txt", "a") 184 | f_val = open(yolo_dir + "val.txt", "a") 185 | write_txt_files_yolo(yolo_dir, f_train, f_val) 186 | f_train.close() 187 | f_val.close() 188 | 189 | 190 | ########################################################### 191 | ########## KITTI to VOC Conversion ########## 192 | ########################################################### 193 | def write_voc_file(fname, labels, coords, img_width, img_height): 194 | """ 195 | Definition: Writes label into VOC (XML) format. 196 | 197 | Parameters: fname - full file path to label file 198 | labels - list of objects in file 199 | coords - list of position of objects in file 200 | img_width - width of image 201 | img_height - height of image 202 | Returns: annotation - XML tree for image file 203 | """ 204 | annotation = etree.Element('annotation') 205 | filename = etree.Element('filename') 206 | f = fname.split("/") 207 | filename.text = f[-1] 208 | annotation.append(filename) 209 | folder = etree.Element('folder') 210 | folder.text = "/".join(f[:-1]) 211 | annotation.append(folder) 212 | for i in range(len(coords)): 213 | object = etree.Element('object') 214 | annotation.append(object) 215 | name = etree.Element('name') 216 | name.text = labels[i] 217 | object.append(name) 218 | bndbox = etree.Element('bndbox') 219 | object.append(bndbox) 220 | xmax = etree.Element('xmax') 221 | xmax.text = str(coords[i][2]) 222 | bndbox.append(xmax) 223 | xmin = etree.Element('xmin') 224 | xmin.text = str(coords[i][0]) 225 | bndbox.append(xmin) 226 | ymax = etree.Element('ymax') 227 | ymax.text = str(coords[i][3]) 228 | bndbox.append(ymax) 229 | ymin = etree.Element('ymin') 230 | ymin.text = str(coords[i][1]) 231 | bndbox.append(ymin) 232 | difficult = etree.Element('difficult') 233 | difficult.text = '0' 234 | object.append(difficult) 235 | occluded = etree.Element('occluded') 236 | occluded.text = '0' 237 | object.append(occluded) 238 | pose = etree.Element('pose') 239 | pose.text = 'Unspecified' 240 | object.append(pose) 241 | truncated = etree.Element('truncated') 242 | truncated.text = '1' 243 | object.append(truncated) 244 | img_size = etree.Element('size') 245 | annotation.append(img_size) 246 | depth = etree.Element('depth') 247 | depth.text = '3' 248 | img_size.append(depth) 249 | height = etree.Element('height') 250 | height.text = str(img_height) 251 | img_size.append(height) 252 | width = etree.Element('width') 253 | width.text = str(img_width) 254 | img_size.append(width) 255 | 256 | return annotation 257 | 258 | def parse_labels_voc(label_file): 259 | """ 260 | Definition: Parses label file to extract label and bounding box 261 | coordintates. 262 | 263 | Parameters: label_file - list of labels in images 264 | Returns: all_labels - contains a list of labels for objects in the image 265 | all_coords - contains a list of coordinates for objects in image 266 | """ 267 | lfile = open(label_file) 268 | coords = [] 269 | all_coords = [] 270 | all_labels = [] 271 | for line in lfile: 272 | l = line.split(" ") 273 | all_labels.append(l[0]) 274 | coords = list(map(int, list(map(float, l[4:8])))) 275 | xmin = coords[0] 276 | ymin = coords[1] 277 | xmax = coords[2] 278 | ymax = coords[3] 279 | tmp = [xmin, ymin, xmax, ymax] 280 | all_coords.append(list(map(int, tmp))) 281 | lfile.close() 282 | return all_labels, all_coords 283 | 284 | def copy_images_voc(kitti, voc): 285 | """ 286 | Definition: Copy all images from the training and validation sets 287 | in kitti format to training and validation image sets in voc 288 | format. 289 | 290 | Parameters: kitti - path to kitti directory (contains 'train' and 'val') 291 | voc - path to voc output directory 292 | Returns: None 293 | """ 294 | for filename in glob.glob(os.path.join(kitti + "train/images/", "*.*")): 295 | shutil.copy(filename, voc + "train/images/") 296 | for filename in glob.glob(os.path.join(kitti + "val/images/", "*.*")): 297 | shutil.copy(filename, voc + "val/images/") 298 | 299 | def make_voc_directories(voc): 300 | """ 301 | Definition: Make directories for voc images and labels. 302 | Removes previously created voc image and label directories. 303 | 304 | Parameters: yolo - path to voc directory to be created 305 | Returns: None 306 | """ 307 | if os.path.exists(voc): 308 | if python_version == 3: 309 | prompt = input('Directory already exists. Overwrite? (yes, no): ') 310 | else: 311 | prompt = raw_input('Directory already exists. Overwrite? (yes, no): ') 312 | if prompt == 'no': 313 | exit(0) 314 | shutil.rmtree(voc) 315 | os.makedirs(voc) 316 | os.makedirs(voc + "train") 317 | os.makedirs(voc + "train/images") 318 | os.makedirs(voc + "train/labels") 319 | os.makedirs(voc + "val") 320 | os.makedirs(voc + "val/images") 321 | os.makedirs(voc + "val/labels") 322 | 323 | def voc(kitti_dir, voc_dir, label=None): 324 | print ("Convert kitti to voc") 325 | 326 | # Make all directories for voc dataset 327 | make_voc_directories(voc_dir) 328 | 329 | # Iterate through kitti training data 330 | for f in os.listdir(kitti_dir + "train/labels/"): 331 | fname = (kitti_dir + "train/images/" + f).split(".txt")[0] + ".png" 332 | if os.path.isfile(fname): 333 | img = Image.open(fname) 334 | w, h = img.size 335 | img.close() 336 | labels, coords = parse_labels_voc(os.path.join(kitti_dir + 337 | "train/labels/" + f)) 338 | annotation = write_voc_file(fname, labels, coords, w, h) 339 | et = etree.ElementTree(annotation) 340 | et.write(voc_dir + "train/labels/" + f.split(".txt")[0] + ".xml", pretty_print=True) 341 | 342 | # Iterate through kitti validation data 343 | for f in os.listdir(kitti_dir + "val/labels/"): 344 | fname = (kitti_dir + "val/images/" + f).split(".txt")[0] + ".png" 345 | if os.path.isfile(fname): 346 | img = Image.open(fname) 347 | w, h = img.size 348 | img.close() 349 | labels, coords = parse_labels_voc(os.path.join(kitti_dir + 350 | "val/labels/" + f)) 351 | annotation = write_voc_file(fname, labels, coords, w, h) 352 | et = etree.ElementTree(annotation) 353 | et.write(voc_dir + "val/labels/" + f.split(".txt")[0] + ".xml", pretty_print=True) 354 | 355 | # Copy images from kitti to voc 356 | copy_images_voc(kitti_dir, voc_dir) 357 | 358 | ########################################################### 359 | ########## KITTI to LISA Conversion ######### 360 | ########################################################### 361 | def lisa(kitti_dir, output, label=None): 362 | print ("Convert kitti to lisa") 363 | pass -------------------------------------------------------------------------------- /datasets/lisa.py: -------------------------------------------------------------------------------- 1 | # Import necessary libraries 2 | import os, sys, shutil, glob, argparse 3 | import numpy as np 4 | from PIL import Image 5 | import csv, ntpath 6 | 7 | def make_directories(dataset): 8 | """ 9 | Definition: Make directories for kitti images and labels. 10 | Removes previously create kitti image and label directories. 11 | 12 | Parameters: dataset - path to {kitti, yolo, etc.} directory to be created 13 | Returns: None 14 | """ 15 | if os.path.exists(dataset): 16 | prompt = input("Directory already exists. Overwrite? (yes, no): ") 17 | if prompt == 'no': 18 | exit(0) 19 | shutil.rmtree(dataset) 20 | os.makedirs(dataset) 21 | os.makedirs(dataset + "train") 22 | os.makedirs(dataset + "train/images") 23 | os.makedirs(dataset + "train/labels") 24 | os.makedirs(dataset + "val") 25 | os.makedirs(dataset + "val/images") 26 | os.makedirs(dataset + "val/labels") 27 | 28 | def split_train_val(data_split): 29 | """ 30 | Definition: Split the csv files for training and testing 31 | 32 | Parameters: data_split - split for data (i.e. '80' splits 80/20) 33 | Return: None 34 | """ 35 | os.system("python " + viva_signs + "scripts/filterAnnotationFile.py " + 36 | data_split + " " + lisa_annotations) 37 | os.system("python " + viva_signs + "scripts/filterAnnotationFile.py " + 38 | data_split + " " + lisa_ext_annotations) 39 | 40 | ########################################################### 41 | ########## LISA to KITTI Conversion ########## 42 | ########################################################### 43 | def create_labels_kitti(fp_train, fp_val, out_train, out_val): 44 | f_train = open(fp_train, "r") 45 | f_val = open(fp_val, "r") 46 | 47 | csvf_train = csv.reader(f_train, delimiter=';') 48 | csvf_val = csv.reader(f_val, delimiter=';') 49 | header_train = csvf_train.next() 50 | header_val = csvf_val.next() 51 | 52 | fnameIdx_train = header_train.index("Filename") 53 | fnameIdx_val = header_val.index("Filename") 54 | tagIdx_train = header_train.index("Annotation tag") 55 | tagIdx_val = header_val.index("Annotation tag") 56 | upleftXIdx_train = header_train.index("Upper left corner X") 57 | upleftXIdx_val = header_val.index("Upper left corner X") 58 | upleftYIdx_train = header_train.index("Upper left corner Y") 59 | upleftYIdx_val = header_val.index("Upper left corner Y") 60 | lowrightXIdx_train = header_train.index("Lower right corner X") 61 | lowrightXIdx_val = header_val.index("Lower right corner X") 62 | lowrightYIdx_train = header_train.index("Lower right corner Y") 63 | lowrightYIdx_val = header_val.index("Lower right corner Y") 64 | 65 | for row in csvf_train: 66 | fname = ntpath.basename(row[fnameIdx_train]) 67 | with open(out_train + fname + ".txt", 'a') as file: 68 | file.write('%s 0 0 0 %s %s %s %s 0 0 0 0 0 0 0 0\n' 69 | % (row[tagIdx_train], row[upleftXIdx_train], row[upleftYIdx_train], 70 | row[lowrightXIdx_train], row[lowrightYIdx_train])) 71 | 72 | for row in csvf_val: 73 | fname = ntpath.basename(row[fnameIdx_val]) 74 | with open(out_val + fname + ".txt", 'a') as file: 75 | file.write('%s 0 0 0 %s %s %s %s 0 0 0 0 0 0 0 0\n' 76 | % (row[tagIdx_val], row[upleftXIdx_val], row[upleftYIdx_val], 77 | row[lowrightXIdx_val], row[lowrightYIdx_val])) 78 | 79 | def kitti(lisa_dir, kitti_dir, label=None): 80 | print ("Converting lisa to kitti") 81 | 82 | # Make all directories for kitti dataset 83 | make_directories(kitti_dir) 84 | 85 | # Split into training and validation 86 | #split_train_val(70) 87 | 88 | # Write labels 89 | 90 | # Convert png.txt to .txt 91 | 92 | # Copy all images 93 | pass 94 | 95 | ########################################################### 96 | ########## LISA to YOLO Conversion ########## 97 | ########################################################### 98 | def yolo(lisa_dir, yolo_dir, label=None): 99 | print ("Converting lisa to yolo") 100 | 101 | # Make all directories for yolo dataset 102 | make_directories(yolo_dir) 103 | pass 104 | 105 | ########################################################### 106 | ########## LISA to VOC Conversion ########## 107 | ########################################################### 108 | def voc(lisa_dir, voc_dir, label=None): 109 | print ("Converting lisa to voc") 110 | 111 | # Make all directories for voc dataset 112 | make_directories(voc_dir) 113 | pass -------------------------------------------------------------------------------- /datasets/voc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eweill/convert-datasets/6e2192807a7ef0146ca29b2480d13226165d6b05/datasets/voc.py -------------------------------------------------------------------------------- /datasets/yolo.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ########## YOLO format ########## 3 | """ 4 | Values Name Description 5 | ---------------------------------------------------------------------------- 6 | 1 type Describes the type of object by the index value from the 7 | label file for each class (i.e. 0=Car, 1=Pedestrian) 8 | 4 bbox 2D bounding box of object in the image (0-based index): 9 | contains x, y, width, height where x and y are the center 10 | of the image and all coordinates are normalized based on 11 | the image size 12 | """ 13 | ############################################################################### 14 | 15 | # Import necessary libraries 16 | import os, sys, shutil, glob, argparse 17 | import numpy as np 18 | from PIL import Image 19 | from lxml import etree 20 | 21 | python_version = sys.version_info.major 22 | 23 | ########################################################### 24 | ########## YOLO to KITTI Conversion ########## 25 | ########################################################### 26 | def determine_label_kitti(label, labels): 27 | """ 28 | Definition: Converts label index to label from label set 29 | 30 | Parameters: label - label from file 31 | labels - list of labels 32 | Returns: label corresponding to index in yolo label 33 | """ 34 | return str(labels[int(label)]) 35 | 36 | def parse_labels_kitti(label_file, labels, img_width, img_height): 37 | """ 38 | Definition: Parsers label file to extract label and bounding box 39 | coordinates. Converts (x, y, width, height) YOLO format to 40 | (x1, y1, x2, y2) KITTI format. 41 | 42 | Parameters: label_file - file with YOLO label(s) inside 43 | labels - list of labels in dataset 44 | img_width - width of input image 45 | img_height - height of input image 46 | Returns: all_labels - contains a list of labels for objects in image 47 | all_coords - contains a list of coordinates for objects in image 48 | """ 49 | lfile = open(label_file) 50 | coords = [] 51 | all_coords = [] 52 | all_labels = [] 53 | for line in lfile: 54 | l = line.split(" ") 55 | all_labels.append(determine_label_kitti(l[0], labels)) 56 | coords = list(map(float, list(map(float, l[1:5])))) 57 | x1 = float(img_width) * (2.0 * float(coords[0]) - float(coords[2])) / 2.0 58 | y1 = float(img_height) * (2.0 * float(coords[1]) - float(coords[3])) / 2.0 59 | x2 = float(img_width) * (2.0 * float(coords[0]) + float(coords[2])) / 2.0 60 | y2 = float(img_height) * (2.0 * float(coords[1]) + float(coords[3])) / 2.0 61 | tmp = [x1, y1, x2, y2] 62 | all_coords.append(list(map(int, tmp))) 63 | lfile.close() 64 | return all_labels, all_coords 65 | 66 | def copy_images_kitti(yolo, kitti): 67 | """ 68 | Definition: Copy all images from the training and validation sets 69 | in yolo format to training and validation image sets in kitti 70 | format. This means converting from .jpg to .png 71 | 72 | Parameters: yolo - path to yolo directory (contains 'train' and 'val') 73 | kitti - path to kitti output directory 74 | Returns: None 75 | """ 76 | for filename in glob.glob(os.path.join(yolo + "train/images/", "*.*")): 77 | shutil.copy(filename, kitti + "train/images/") 78 | for filename in glob.glob(os.path.join(yolo + "val/images/", "*.*")): 79 | shutil.copy(filename, kitti + "val/images/") 80 | 81 | for filename in glob.glob(os.path.join(kitti + "train/images/", "*.*")): 82 | im = Image.open(filename) 83 | im.save(filename.split(".jpg")[0] + ".png", "png") 84 | os.remove(filename) 85 | for filename in glob.glob(os.path.join(kitti + "val/images/", "*.*")): 86 | im = Image.open(filename) 87 | im.save(filename.split(".jpg")[0] + ".png", "png") 88 | os.remove(filename) 89 | 90 | def make_kitti_directories(kitti): 91 | """ 92 | Definition: Make directories for kitti images and labels. 93 | Removes previously created kitti image and label directories. 94 | 95 | Parameters: kitti - path to kitti directory to be created 96 | Returns: None 97 | """ 98 | if os.path.exists(kitti): 99 | if python_version == 3: 100 | prompt = input('Directory already exists. Overwrite? (yes, no): ') 101 | else: 102 | prompt = raw_input('Directory already exists. Overwrite? (yes, no): ') 103 | if prompt == 'no': 104 | exit(0) 105 | shutil.rmtree(kitti) 106 | os.makedirs(kitti) 107 | os.makedirs(kitti + "train") 108 | os.makedirs(kitti + "train/images") 109 | os.makedirs(kitti + "train/labels") 110 | os.makedirs(kitti + "val") 111 | os.makedirs(kitti + "val/images") 112 | os.makedirs(kitti + "val/labels") 113 | 114 | def kitti(yolo_dir, kitti_dir, label=None): 115 | print ("Converting yolo to kitti") 116 | 117 | # Split label file 118 | label_file = open(label) 119 | labels_split = label_file.read().split('\n') 120 | 121 | # Make all directories for kitti dataset 122 | make_kitti_directories(kitti_dir) 123 | 124 | # Iterate through yolo training data 125 | for f in os.listdir(yolo_dir + "train/labels/"): 126 | fname = (yolo_dir + "train/images/" + f).split(".txt")[0] + ".jpg" 127 | if os.path.isfile(fname): 128 | img = Image.open(fname) 129 | w, h = img.size 130 | img.close() 131 | labels, coords = parse_labels_kitti(os.path.join(yolo_dir + 132 | "train/labels/" + f), labels_split, w, h) 133 | kittif = open(kitti_dir + "train/labels/" + f, "a+") 134 | for l, c in zip(labels, coords): 135 | kittif.write(l + " 0 0 0 " + str(c[0]) + " " + str(c[1]) + 136 | " " + str(c[2]) + " " + str(c[3]) + " 0 0 0 0 0 0 0 0\n") 137 | kittif.close() 138 | 139 | # Iterate through yolo validation data 140 | for f in os.listdir(yolo_dir + "val/labels/"): 141 | fname = (yolo_dir + "val/images/" + f).split(".txt")[0] + ".jpg" 142 | if os.path.isfile(fname): 143 | img = Image.open(fname) 144 | w, h = img.size 145 | img.close() 146 | labels, coords = parse_labels_kitti(os.path.join(yolo_dir + 147 | "val/labels/" + f), labels_split, w, h) 148 | kittif = open(kitti_dir + "val/labels/" + f, "a+") 149 | for l, c in zip(labels, coords): 150 | kittif.write(l + " 0 0 0 " + str(c[0]) + " " + str(c[1]) + 151 | " " + str(c[2]) + " " + str(c[3]) + " 0 0 0 0 0 0 0 0\n") 152 | kittif.close() 153 | 154 | 155 | # Copy images from yolo to kitti 156 | copy_images_kitti(yolo_dir, kitti_dir) 157 | 158 | ########################################################### 159 | ########## YOLO to LISA Conversion ########## 160 | ########################################################### 161 | def lisa(yolo_dir, lisa_dir, label=None): 162 | pass 163 | 164 | ########################################################### 165 | ########## YOLO to VOC Conversion ########## 166 | ########################################################### 167 | def write_voc_file(fname, labels, coords, img_width, img_height): 168 | """ 169 | Definition: Writes label into VOC (XML) format. 170 | 171 | Parameters: fname - full file path to label file 172 | labels - list of objects in file 173 | coords - list of position of objects in file 174 | img_width - width of image 175 | img_height - height of image 176 | Returns: annotation - XML tree for image file 177 | """ 178 | annotation = etree.Element('annotation') 179 | filename = etree.Element('filename') 180 | f = fname.split("/") 181 | filename.text = f[-1] 182 | annotation.append(filename) 183 | folder = etree.Element('folder') 184 | folder.text = "/".join(f[:-1]) 185 | annotation.append(folder) 186 | for i in range(len(coords)): 187 | object = etree.Element('object') 188 | annotation.append(object) 189 | name = etree.Element('name') 190 | name.text = labels[i] 191 | object.append(name) 192 | bndbox = etree.Element('bndbox') 193 | object.append(bndbox) 194 | xmax = etree.Element('xmax') 195 | xmax.text = str(coords[i][2]) 196 | bndbox.append(xmax) 197 | xmin = etree.Element('xmin') 198 | xmin.text = str(coords[i][0]) 199 | bndbox.append(xmin) 200 | ymax = etree.Element('ymax') 201 | ymax.text = str(coords[i][3]) 202 | bndbox.append(ymax) 203 | ymin = etree.Element('ymin') 204 | ymin.text = str(coords[i][1]) 205 | bndbox.append(ymin) 206 | difficult = etree.Element('difficult') 207 | difficult.text = '0' 208 | object.append(difficult) 209 | occluded = etree.Element('occluded') 210 | occluded.text = '0' 211 | object.append(occluded) 212 | pose = etree.Element('pose') 213 | pose.text = 'Unspecified' 214 | object.append(pose) 215 | truncated = etree.Element('truncated') 216 | truncated.text = '1' 217 | object.append(truncated) 218 | img_size = etree.Element('size') 219 | annotation.append(img_size) 220 | depth = etree.Element('depth') 221 | depth.text = '3' 222 | img_size.append(depth) 223 | height = etree.Element('height') 224 | height.text = str(img_height) 225 | img_size.append(height) 226 | width = etree.Element('width') 227 | width.text = str(img_width) 228 | img_size.append(width) 229 | 230 | return annotation 231 | 232 | def determine_label_voc(label, labels): 233 | """ 234 | Definition: Converts label index to label from label set 235 | 236 | Parameters: label - label from file 237 | labels - list of labels 238 | Returns: label corresponding to index in yolo label 239 | """ 240 | return str(labels[int(label)]) 241 | 242 | 243 | def parse_labels_voc(label_file, labels, img_width, img_height): 244 | """ 245 | Definition: Parses label file to extract label and bounding box 246 | coordintates. 247 | 248 | Parameters: label_file - list of labels in images 249 | Returns: all_labels - contains a list of labels for objects in the image 250 | all_coords - contains a list of coordinates for objects in image 251 | """ 252 | lfile = open(label_file) 253 | coords = [] 254 | all_coords = [] 255 | all_labels = [] 256 | for line in lfile: 257 | l = line.split(" ") 258 | all_labels.append(determine_label_voc(l[0], labels)) 259 | coords = list(map(float, list(map(float, l[1:5])))) 260 | xmin = float(img_width) * (2.0 * float(coords[0]) - float(coords[2])) / 2.0 261 | ymin = float(img_height) * (2.0 * float(coords[1]) - float(coords[3])) / 2.0 262 | xmax = float(img_width) * (2.0 * float(coords[0]) + float(coords[2])) / 2.0 263 | ymax = float(img_height) * (2.0 * float(coords[1]) + float(coords[3])) / 2.0 264 | tmp = [xmin, ymin, xmax, ymax] 265 | all_coords.append(list(map(int, tmp))) 266 | lfile.close() 267 | return all_labels, all_coords 268 | 269 | def copy_images_voc(yolo, voc): 270 | """ 271 | Definition: Copy all images from the training and validation sets 272 | in kitti format to training and validation image sets in voc 273 | format. 274 | 275 | Parameters: yolo - path to yolo directory (contains 'train' and 'val') 276 | voc - path to voc output directory 277 | Returns: None 278 | """ 279 | for filename in glob.glob(os.path.join(yolo + "train/images/", "*.*")): 280 | shutil.copy(filename, voc + "train/images/") 281 | for filename in glob.glob(os.path.join(yolo + "val/images/", "*.*")): 282 | shutil.copy(filename, voc + "val/images/") 283 | 284 | for filename in glob.glob(os.path.join(voc + "train/images/", "*.*")): 285 | im = Image.open(filename) 286 | im.save(filename.split(".jpg")[0] + ".png", "png") 287 | os.remove(filename) 288 | for filename in glob.glob(os.path.join(voc + "val/images/", "*.*")): 289 | im = Image.open(filename) 290 | im.save(filename.split(".jpg")[0] + ".png", "png") 291 | os.remove(filename) 292 | 293 | def make_voc_directories(voc): 294 | """ 295 | Definition: Make directories for voc images and labels. 296 | Removes previously created voc image and label directories. 297 | 298 | Parameters: yolo - path to voc directory to be created 299 | Returns: None 300 | """ 301 | if os.path.exists(voc): 302 | if python_version == 3: 303 | prompt = input('Directory already exists. Overwrite? (yes, no): ') 304 | else: 305 | prompt = raw_input('Directory already exists. Overwrite? (yes, no): ') 306 | if prompt == 'no': 307 | exit(0) 308 | shutil.rmtree(voc) 309 | os.makedirs(voc) 310 | os.makedirs(voc + "train") 311 | os.makedirs(voc + "train/images") 312 | os.makedirs(voc + "train/labels") 313 | os.makedirs(voc + "val") 314 | os.makedirs(voc + "val/images") 315 | os.makedirs(voc + "val/labels") 316 | 317 | def voc(yolo_dir, voc_dir, label=None): 318 | print ("Convert yolo to voc") 319 | 320 | # Split label file 321 | label_file = open(label) 322 | labels_split = label_file.read().split('\n') 323 | 324 | # Make all directories for voc dataset 325 | make_voc_directories(voc_dir) 326 | 327 | # Iterate through kitti training data 328 | for f in os.listdir(yolo_dir + "train/labels/"): 329 | fname = (yolo_dir + "train/images/" + f).split(".txt")[0] + ".jpg" 330 | if os.path.isfile(fname): 331 | img = Image.open(fname) 332 | w, h = img.size 333 | img.close() 334 | labels, coords = parse_labels_voc(os.path.join(yolo_dir + 335 | "train/labels/" + f), labels_split, w, h) 336 | annotation = write_voc_file(fname, labels, coords, w, h) 337 | et = etree.ElementTree(annotation) 338 | et.write(voc_dir + "train/labels/" + f.split(".txt")[0] + ".xml", pretty_print=True) 339 | 340 | # Iterate through kitti validation data 341 | for f in os.listdir(yolo_dir + "val/labels/"): 342 | fname = (yolo_dir + "val/images/" + f).split(".txt")[0] + ".jpg" 343 | if os.path.isfile(fname): 344 | img = Image.open(fname) 345 | w, h = img.size 346 | img.close() 347 | labels, coords = parse_labels_voc(os.path.join(yolo_dir + 348 | "val/labels/" + f), labels_split, w, h) 349 | annotation = write_voc_file(fname, labels, coords, w, h) 350 | et = etree.ElementTree(annotation) 351 | et.write(voc_dir + "val/labels/" + f.split(".txt")[0] + ".xml", pretty_print=True) 352 | 353 | # Copy images from kitti to voc 354 | copy_images_voc(yolo_dir, voc_dir) 355 | 356 | --------------------------------------------------------------------------------