630 | 631 | 632 | 633 | 634 | 635 | 636 | 637 | 638 | 639 | 640 | 641 | 642 |

643 |

644 | 645 |

651 | 652 |
653 | 654 |
655 |
656 | 657 | Watch 658 | 659 |
662 |
663 | Notifications 664 |
665 |
666 | 677 | 678 | 689 | 690 | 701 | 702 | 713 |
714 | 715 |
716 | 719 | 2,810 720 | 721 |
722 |
725 |
726 |
727 | 728 | 732 | 85,492 733 | 734 |
735 |
736 | 737 | 741 | 85,492 742 | 743 |

744 | 745 |
748 |
749 |
752 | 754 | 41,961 755 | 756 |

758 | 759 |

760 | 761 | github/gitignore 764 | 765 | 766 |

767 | 768 |

769 | 770 | 810 | 811 |

812 |

815 | 816 | 817 | 821 | 822 | 823 | 824 | 829 | 830 | 831 | 836 | 837 | 838 | 842 | 845 | 846 | 849 | 850 |

851 |

852 | 853 | 854 |

855 |

856 |

857 | 858 | 859 | 860 | 861 | 862 | 863 | 864 | 865 | 866 | 867 | 868 |

869 | 870 | 871 | 886 | 887 |

888 | 892 | Find file 893 | 894 | 895 | Copy path 896 | 897 |

898 | 899 | 902 | 903 |

904 | 908 | Find file 909 | 910 | 911 | Copy path 912 | 913 |

914 |

915 | 916 | 917 | 918 | 919 |

920 |

921 | 922 |

923 | captainpete 924 | 925 | Add db.sqlite3-journal to Python.gitignore (#3063 ) 940 | 941 | 942 | 943 | c1faae4 944 | May 25, 2019 945 | 946 |

947 | 948 |

949 |

950 |

951 | 73 contributors 952 |

953 | 957 |

958 | 961 |

962 | Users who have contributed to this file 963 |

964 |

965 | 966 | 967 |

968 | 969 | 970 |

1021 | 1022 | 1023 | 1024 | 1025 |

1026 |

1027 | 1028 | 1029 | 1030 | 1031 | 1032 |

1033 | 1034 |

1035 | 1036 |

1037 | 125 lines (101 sloc) 1038 | 1039 | 1.67 KB 1040 |

1041 | 1042 |

1043 | 1044 |

1045 | Raw 1046 | Blame 1047 | History 1048 |

1049 | 1050 | 1051 |

1052 | 1053 | 1059 |

1065 |

1066 |

1067 | 1068 | 1069 | 1070 | 1071 | 1072 | 1073 |

1074 | 1075 | 1076 | 1077 | 1078 | 1079 | 1080 | 1081 | 1082 | 1083 | 1084 | 1085 | 1086 | 1087 | 1088 | 1089 | 1090 | 1091 | 1092 | 1093 | 1094 | 1096 | 1097 | 1098 | 1099 | 1100 | 1101 | 1102 | 1103 | 1104 | 1105 | 1106 | 1107 | 1109 | 1110 | 1111 | 1112 | 1113 | 1114 | 1115 | 1116 | 1117 | 1118 | 1119 | 1120 | 1121 | 1122 | 1123 | 1124 | 1125 | 1126 | 1127 | 1128 | 1129 | 1130 | 1131 | 1132 | 1133 | 1134 | 1135 | 1136 | 1137 | 1138 | 1139 | 1140 | 1141 | 1142 | 1143 | 1144 | 1145 | 1146 | 1147 | 1148 | 1149 | 1150 | 1151 | 1152 | 1153 | 1154 | 1155 | 1156 | 1157 | 1158 | 1159 | 1160 | 1161 | 1162 | 1163 | 1164 | 1165 | 1166 | 1167 | 1168 | 1169 | 1170 | 1171 | 1172 | 1173 | 1174 | 1175 | 1176 | 1177 | 1178 | 1179 | 1180 | 1181 | 1182 | 1183 | 1184 | 1185 | 1186 | 1187 | 1188 | 1189 | 1190 | 1191 | 1192 | 1194 | 1195 | 1196 | 1197 | 1198 | 1199 | 1200 | 1201 | 1202 | 1203 | 1204 | 1205 | 1206 | 1207 | 1208 | 1209 | 1210 | 1211 | 1212 | 1213 | 1214 | 1215 | 1216 | 1217 | 1219 | 1220 | 1221 | 1222 | 1223 | 1224 | 1225 | 1226 | 1227 | 1228 | 1229 | 1230 | 1231 | 1232 | 1233 | 1234 | 1236 | 1237 | 1238 | 1239 | 1240 | 1241 | 1242 | 1243 | 1244 | 1245 | 1246 | 1247 | 1248 | 1249 | 1250 | 1251 | 1252 | 1253 | 1254 | 1255 | 1256 | 1257 | 1258 | 1259 | 1260 | 1261 | 1262 | 1263 | 1264 | 1265 | 1266 | 1267 | 1268 | 1269 | 1270 | 1271 | 1272 | 1273 | 1274 | 1275 | 1276 | 1277 | 1278 | 1279 | 1280 | 1281 | 1282 | 1283 | 1284 | 1285 | 1286 | 1287 | 1289 | 1290 | 1291 | 1292 | 1293 | 1294 | 1295 | 1296 | 1297 | 1298 | 1299 | 1300 | 1301 | 1302 | 1303 | 1304 | 1306 | 1307 | 1308 | 1309 | 1310 | 1311 | 1312 | 1313 | 1314 | 1315 | 1316 | 1317 | 1318 | 1319 | 1320 | 1321 | 1322 | 1323 | 1324 | 1325 | 1326 | 1327 | 1328 | 1329 | 1331 | 1332 | 1333 | 1334 | 1335 | 1336 | 1337 | 1338 | 1339 | 1340 | 1341 | 1342 | 1343 | 1344 | 1345 | 1346 | 1348 | 1349 | 1350 | 1351 | 1352 | 1353 | 1354 | 1355 | 1356 | 1357 | 1358 | 1359 | 1361 | 1362 | 1363 | 1364 | 1365 | 1366 | 1367 | 1368 | 1369 | 1370 | 1371 | 1372 | 1374 | 1375 | 1376 | 1377 | 1378 | 1379 | 1380 | 1381 | 1382 | 1383 | 1384 | 1385 | 1387 | 1388 | 1389 | 1390 | 1391 | 1392 | 1393 | 1394 | 1395 | 1396 | 1397 | 1398 | 1400 | 1401 | 1402 | 1403 | 1404 | 1405 | 1406 | 1407 | 1408 | 1409 | 1410 | 1411 | 1412 | 1413 | 1414 | 1415 | 1417 | 1418 | 1419 | 1420 | 1421 | 1422 | 1423 | 1424 | 1425 | 1426 | 1427 | 1428 | 1430 | 1431 | 1432 | 1433 | 1434 | 1435 | 1436 | 1437 | 1438 | 1439 | 1440 | 1441 | 1442 | 1443 | 1444 | 1445 | 1446 | 1447 | 1448 | 1449 | 1450 | 1451 | 1452 | 1453 | 1454 | 1455 | 1456 | 1457 | 1459 | 1460 | 1461 | 1462 | 1463 | 1464 | 1465 | 1466 | 1467 | 1468 | 1469 | 1470 | 1472 | 1473 | 1474 | 1475 | 1476 | 1477 | 1478 | 1479 | 1480 | 1481 | 1482 | 1483 | 1485 | 1486 | 1487 | 1488 | 1489 | 1490 | 1491 | 1492 | 1493 | 1494 | 1495 | 1496 | 1497 | 1498 | 1499 | 1500 | 1501 | 1502 | 1503 | 1504 | 1505 | 1506 | 1507 | 1508 | 1509 | 1510 | 1511 | 1512 | 1513 | 1514 | 1515 | 1516 | 1517 | 1518 | 1519 | 1520 | 1522 | 1523 | 1524 | 1525 | 1526 | 1527 | 1528 | 1529 | 1530 | 1531 | 1532 | 1533 | 1534 | 1535 | 1536 | 1537 | 1539 | 1540 | 1541 | 1542 | 1543 | 1544 | 1545 | 1546 | 1547 | 1548 | 1549 | 1550 | 1552 | 1553 | 1554 | 1555 | 1556 | 1557 | 1558 | 1559 | 1560 | 1561 | 1562 | 1563 | 1565 | 1566 | 1567 | 1568 | 1569 | 1570 | 1571 | 1572 | 1573 | 1574 | 1575 | 1576 | 1577 | 1578 | 1579 | 1580 | 1581 | 1582 | 1583 | 1584 | 1586 | 1587 | 1588 | 1589 | 1590 | 1591 | 1592 | 1593 | 1594 | 1595 |

	# Byte-compiled / optimized / DLL files
	__pycache__/
	*.py[cod]
	*$py.class
	1095 \|
	# C extensions
	*.so
	1108 \|
	# Distribution / packaging
	.Python
	build/
	develop-eggs/
	dist/
	downloads/
	eggs/
	.eggs/
	lib/
	lib64/
	parts/
	sdist/
	var/
	wheels/
	pip-wheel-metadata/
	share/python-wheels/
	*.egg-info/
	.installed.cfg
	*.egg
	MANIFEST
	1193 \|
	# PyInstaller
	# Usually these files are written by a python script from a template
	# before PyInstaller builds the exe, so as to inject date/other infos into it.
	*.manifest
	*.spec
	1218 \|
	# Installer logs
	pip-log.txt
	pip-delete-this-directory.txt
	1235 \|
	# Unit test / coverage reports
	htmlcov/
	.tox/
	.nox/
	.coverage
	.coverage.*
	.cache
	nosetests.xml
	coverage.xml
	*.cover
	.hypothesis/
	.pytest_cache/
	1288 \|
	# Translations
	*.mo
	*.pot
	1305 \|
	# Django stuff:
	*.log
	local_settings.py
	db.sqlite3
	db.sqlite3-journal
	1330 \|
	# Flask stuff:
	instance/
	.webassets-cache
	1347 \|
	# Scrapy stuff:
	.scrapy
	1360 \|
	# Sphinx documentation
	docs/_build/
	1373 \|
	# PyBuilder
	target/
	1386 \|
	# Jupyter Notebook
	.ipynb_checkpoints
	1399 \|
	# IPython
	profile_default/
	ipython_config.py
	1416 \|
	# pyenv
	.python-version
	1429 \|
	# pipenv
	# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
	# However, in case of collaboration, if having platform-specific dependencies or dependencies
	# having no cross-platform support, pipenv may install dependencies that don't work, or not
	# install all needed dependencies.
	#Pipfile.lock
	1458 \|
	# celery beat schedule file
	celerybeat-schedule
	1471 \|
	# SageMath parsed files
	*.sage.py
	1484 \|
	# Environments
	.env
	.venv
	env/
	venv/
	ENV/
	env.bak/
	venv.bak/
	1521 \|
	# Spyder project settings
	.spyderproject
	.spyproject
	1538 \|
	# Rope project settings
	.ropeproject
	1551 \|
	# mkdocs documentation
	/site
	1564 \|
	# mypy
	.mypy_cache/
	.dmypy.json
	dmypy.json
	1585 \|
	# Pyre type checker
	.pyre/

1596 | 1597 | 1609 | 1610 |

1611 | 1612 |

1613 | 1614 | 1615 | 1616 |

1617 |

1618 | 1619 | 1623 |

1624 | 1625 | 1626 | 1627 |

1628 | 1629 |

1630 | 1631 |

~ OIDv4 ToolKit ~

Train

Validation

Test

#Classes

Images

1,743,042

41,620

125,436

Boxes

14,610,229

204,621

625,282

600

Train

Validation

Test

#Classes

Images

9,011,219

41,620

125,436

Machine-Generated Labels

78,977,695

512,093

1,545,835

7,870

Human-Verified Labels

27,894,289

551,390

1,667,399

19,794

241 |

242 |

") 153 | image_dir = input("> ") 154 | flag = 1 155 | 156 | if image_dir == 'exit': 157 | exit(1) 158 | 159 | class_image_dir = os.path.join(dataset_dir, image_dir) 160 | 161 | print("Which class? ") 162 | show_classes(os.listdir(class_image_dir)) 163 | 164 | class_name = input("> ") 165 | if class_name == 'exit': 166 | exit(1) 167 | 168 | download_dir = os.path.join(dataset_dir, image_dir, class_name) 169 | label_dir = os.path.join(dataset_dir, image_dir, class_name, 'Label') 170 | 171 | if not os.path.isdir(download_dir): 172 | print("[ERROR] Images folder not found") 173 | exit(1) 174 | if not os.path.isdir(label_dir): 175 | print("[ERROR] Labels folder not found") 176 | exit(1) 177 | 178 | index = 0 179 | 180 | print(dedent(""" 181 | -------------------------------------------------------- 182 | INFO: 183 | - Press 'd' to select next image 184 | - Press 'a' to select previous image 185 | - Press 'e' to select a new class 186 | - Press 'w' to select a new folder 187 | - Press 'q' to exit 188 | You can resize the window if it's not optimal 189 | -------------------------------------------------------- 190 | """)) 191 | 192 | show(class_name, download_dir, label_dir,len(os.listdir(download_dir))-1, index) 193 | 194 | while True: 195 | 196 | progression_bar(len(os.listdir(download_dir))-1, index+1) 197 | 198 | k = cv2.waitKey(0) & 0xFF 199 | 200 | if k == ord('d'): 201 | cv2.destroyAllWindows() 202 | if index < (len(os.listdir(download_dir)) - 2): 203 | index += 1 204 | show(class_name, download_dir, label_dir,len(os.listdir(download_dir))-1, index) 205 | elif k == ord('a'): 206 | cv2.destroyAllWindows() 207 | if index > 0: 208 | index -= 1 209 | show(class_name, download_dir, label_dir,len(os.listdir(download_dir))-1, index) 210 | elif k == ord('e'): 211 | cv2.destroyAllWindows() 212 | break 213 | elif k == ord('w'): 214 | flag = 0 215 | cv2.destroyAllWindows() 216 | break 217 | elif k == ord('q'): 218 | cv2.destroyAllWindows() 219 | exit(1) 220 | break 221 | -------------------------------------------------------------------------------- /modules/csv_downloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import urllib.request 5 | import pandas as pd 6 | 7 | from modules.utils import bcolors as bc 8 | 9 | OID_URL = 'https://storage.googleapis.com/openimages/2018_04/' 10 | 11 | def TTV(csv_dir, name_file, args_y): 12 | ''' 13 | Manage error_csv and read the correct .csv file. 14 | 15 | :param csv_dir: folder of the .csv files 16 | :param name_file: name of the correct .csv file 17 | :return: None 18 | ''' 19 | CSV = os.path.join(csv_dir, name_file) 20 | error_csv(name_file, csv_dir, args_y) 21 | df_val = pd.read_csv(CSV) 22 | return df_val 23 | 24 | def error_csv(file, csv_dir, args_y): 25 | ''' 26 | Check the presence of the required .csv files. 27 | 28 | :param file: .csv file missing 29 | :param csv_dir: folder of the .csv files 30 | :return: None 31 | ''' 32 | if not os.path.isfile(os.path.join(csv_dir, file)): 33 | print(bc.FAIL + "Missing the {} file.".format(os.path.basename(file)) + bc.ENDC) 34 | if args_y: 35 | ans = 'y' 36 | print(bc.OKBLUE + "Automatic download." + bc.ENDC) 37 | else: 38 | ans = input(bc.OKBLUE + "Do you want to download the missing file? [Y/n] " + bc.ENDC) 39 | 40 | if ans.lower() == 'y': 41 | folder = str(os.path.basename(file)).split('-')[0] 42 | if folder != 'class': 43 | FILE_URL = str(OID_URL + folder + '/' + file) 44 | else: 45 | FILE_URL = str(OID_URL + file) 46 | 47 | FILE_PATH = os.path.join(csv_dir, file) 48 | save(FILE_URL, FILE_PATH) 49 | print('\n' + bc.OKBLUE + "File {} downloaded into {}.".format(file, FILE_PATH) + bc.ENDC) 50 | 51 | else: 52 | exit(1) 53 | 54 | def save(url, filename): 55 | ''' 56 | Download the .csv file. 57 | 58 | :param url: Google url for download .csv files 59 | :param filename: .csv file name 60 | :return: None 61 | ''' 62 | urllib.request.urlretrieve(url, filename, reporthook) 63 | 64 | def reporthook(count, block_size, total_size): 65 | ''' 66 | Print the progression bar for the .csv file download. 67 | 68 | :param count: 69 | :param block_size: 70 | :param total_size: 71 | :return: 72 | ''' 73 | global start_time 74 | if count == 0: 75 | start_time = time.time() 76 | return 77 | duration = time.time() - start_time 78 | progress_size = int(count * block_size) 79 | speed = int(progress_size / ((1024 * duration) + 1e-5)) 80 | percent = int(count * block_size * 100 / (total_size + 1e-5)) 81 | sys.stdout.write("\r...%d%%, %d MB, %d KB/s, %d seconds passed" % 82 | (percent, progress_size / (1024 * 1024), speed, duration)) 83 | sys.stdout.flush() 84 | -------------------------------------------------------------------------------- /modules/downloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | from tqdm import tqdm 4 | from modules.utils import images_options 5 | from modules.utils import bcolors as bc 6 | from multiprocessing.dummy import Pool as ThreadPool 7 | 8 | def download(args, df_val, folder, dataset_dir, class_name, class_code, class_list=None, threads = 20): 9 | ''' 10 | Manage the download of the images and the label maker. 11 | :param args: argument parser. 12 | :param df_val: DataFrame Values 13 | :param folder: train, validation or test 14 | :param dataset_dir: self explanatory 15 | :param class_name: self explanatory 16 | :param class_code: self explanatory 17 | :param class_list: list of the class if multiclasses is activated 18 | :param threads: number of threads 19 | :return: None 20 | ''' 21 | if os.name == 'posix': 22 | rows, columns = os.popen('stty size', 'r').read().split() 23 | elif os.name == 'nt': 24 | try: 25 | columns, rows = os.get_terminal_size(0) 26 | except OSError: 27 | columns, rows = os.get_terminal_size(1) 28 | else: 29 | columns = 50 30 | l = int((int(columns) - len(class_name))/2) 31 | 32 | print ('\n' + bc.HEADER + '-'*l + class_name + '-'*l + bc.ENDC) 33 | print(bc.INFO + 'Downloading {} images.'.format(args.type_csv) + bc.ENDC) 34 | df_val_images = images_options(df_val, args) 35 | 36 | images_list = df_val_images['ImageID'][df_val_images.LabelName == class_code].values 37 | images_list = set(images_list) 38 | print(bc.INFO + '[INFO] Found {} online images for {}.'.format(len(images_list), folder) + bc.ENDC) 39 | 40 | if args.limit is not None: 41 | import itertools 42 | print(bc.INFO + 'Limiting to {} images.'.format(args.limit) + bc.ENDC) 43 | images_list = set(itertools.islice(images_list, args.limit)) 44 | 45 | if class_list is not None: 46 | class_name_list = '_'.join(class_list) 47 | else: 48 | class_name_list = class_name 49 | 50 | download_img(folder, dataset_dir, class_name_list, images_list, threads) 51 | if not args.sub: 52 | get_label(folder, dataset_dir, class_name, class_code, df_val, class_name_list, args) 53 | 54 | 55 | def download_img(folder, dataset_dir, class_name, images_list, threads): 56 | ''' 57 | Download the images. 58 | :param folder: train, validation or test 59 | :param dataset_dir: self explanatory 60 | :param class_name: self explanatory 61 | :param images_list: list of the images to download 62 | :param threads: number of threads 63 | :return: None 64 | ''' 65 | image_dir = folder 66 | download_dir = os.path.join(dataset_dir, image_dir, class_name) 67 | downloaded_images_list = [f.split('.')[0] for f in os.listdir(download_dir)] 68 | images_list = list(set(images_list) - set(downloaded_images_list)) 69 | 70 | pool = ThreadPool(threads) 71 | 72 | if len(images_list) > 0: 73 | print(bc.INFO + 'Download of {} images in {}.'.format(len(images_list), folder) + bc.ENDC) 74 | commands = [] 75 | for image in images_list: 76 | path = image_dir + '/' + str(image) + '.jpg ' + '"' + download_dir + '"' 77 | command = 'aws s3 --no-sign-request --only-show-errors cp s3://open-images-dataset/' + path 78 | commands.append(command) 79 | 80 | list(tqdm(pool.imap(os.system, commands), total = len(commands) )) 81 | 82 | print(bc.INFO + 'Done!' + bc.ENDC) 83 | pool.close() 84 | pool.join() 85 | else: 86 | print(bc.INFO + 'All images already downloaded.' +bc.ENDC) 87 | 88 | 89 | def get_label(folder, dataset_dir, class_name, class_code, df_val, class_list, args): 90 | ''' 91 | Make the label.txt files 92 | :param folder: trai, validation or test 93 | :param dataset_dir: self explanatory 94 | :param class_name: self explanatory 95 | :param class_code: self explanatory 96 | :param df_val: DataFrame values 97 | :param class_list: list of the class if multiclasses is activated 98 | :return: None 99 | ''' 100 | if not args.noLabels: 101 | print(bc.INFO + 'Creating labels for {} of {}.'.format(class_name, folder) + bc.ENDC) 102 | 103 | image_dir = folder 104 | if class_list is not None: 105 | download_dir = os.path.join(dataset_dir, image_dir, class_list) 106 | label_dir = os.path.join(dataset_dir, folder, class_list, 'Label') 107 | else: 108 | download_dir = os.path.join(dataset_dir, image_dir, class_name) 109 | label_dir = os.path.join(dataset_dir, folder, class_name, 'Label') 110 | 111 | downloaded_images_list = [f.split('.')[0] for f in os.listdir(download_dir) if f.endswith('.jpg')] 112 | images_label_list = list(set(downloaded_images_list)) 113 | 114 | groups = df_val[(df_val.LabelName == class_code)].groupby(df_val.ImageID) 115 | for image in images_label_list: 116 | try: 117 | current_image_path = os.path.join(download_dir, image + '.jpg') 118 | dataset_image = cv2.imread(current_image_path) 119 | boxes = groups.get_group(image.split('.')[0])[['XMin', 'XMax', 'YMin', 'YMax']].values.tolist() 120 | file_name = str(image.split('.')[0]) + '.txt' 121 | file_path = os.path.join(label_dir, file_name) 122 | if os.path.isfile(file_path): 123 | f = open(file_path, 'a') 124 | else: 125 | f = open(file_path, 'w') 126 | 127 | for box in boxes: 128 | box[0] *= int(dataset_image.shape[1]) 129 | box[1] *= int(dataset_image.shape[1]) 130 | box[2] *= int(dataset_image.shape[0]) 131 | box[3] *= int(dataset_image.shape[0]) 132 | 133 | # each row in a file is name of the class_name, XMin, YMix, XMax, YMax (left top right bottom) 134 | print(class_name, box[0], box[2], box[1], box[3], file=f) 135 | 136 | except Exception as e: 137 | pass 138 | 139 | print(bc.INFO + 'Labels creation completed.' + bc.ENDC) 140 | -------------------------------------------------------------------------------- /modules/image_level.py: -------------------------------------------------------------------------------- 1 | import os 2 | from modules.utils import * 3 | from modules.downloader import * 4 | from modules.show import * 5 | from modules.csv_downloader import * 6 | 7 | from modules.utils import bcolors as bc 8 | 9 | def image_level(args, DEFAULT_OID_DIR): 10 | 11 | if not args.Dataset: 12 | dataset_dir = os.path.join(DEFAULT_OID_DIR, 'Dataset_nl') 13 | csv_dir = os.path.join(DEFAULT_OID_DIR, 'csv_folder_nl') 14 | else: 15 | dataset_dir = os.path.join(DEFAULT_OID_DIR, args.Dataset) 16 | csv_dir = os.path.join(DEFAULT_OID_DIR, 'csv_folder_nl') 17 | 18 | name_file_class = 'class-descriptions.csv' 19 | CLASSES_CSV = os.path.join(csv_dir, name_file_class) 20 | 21 | if args.sub is None: 22 | print(bc.FAIL + 'Missing subset argument.' + bc.ENDC) 23 | exit(1) 24 | 25 | if args.sub == 'h': 26 | 27 | file_list = ['train-annotations-human-imagelabels.csv', \ 28 | 'validation-annotations-human-imagelabels.csv', \ 29 | 'test-annotations-human-imagelabels.csv'] 30 | 31 | if args.sub == 'm': 32 | 33 | file_list = ['train-annotations-machine-imagelabels.csv', \ 34 | 'validation-annotations-machine-imagelabels.csv', \ 35 | 'test-annotations-machine-imagelabels.csv'] 36 | 37 | if args.sub == 'h' or args.sub == 'm': 38 | 39 | logo(args.command) 40 | 41 | if args.type_csv is None: 42 | print(bc.FAIL + 'Missing type_csv argument.' + bc.ENDC) 43 | exit(1) 44 | if args.classes is None: 45 | print(bc.FAIL + 'Missing classes argument.' + bc.ENDC) 46 | exit(1) 47 | if args.multiclasses is None: 48 | args.multiclasses = 0 49 | 50 | folder = ['train', 'validation', 'test'] 51 | 52 | if args.classes[0].endswith('.txt'): 53 | with open(args.classes[0]) as f: 54 | args.classes = f.readlines() 55 | args.classes = [x.strip() for x in args.classes] 56 | else: 57 | args.classes = [arg.replace('_', ' ') for arg in args.classes] 58 | 59 | if args.multiclasses == '0': 60 | 61 | mkdirs(dataset_dir, csv_dir, args.classes, args.type_csv) 62 | 63 | for classes in args.classes: 64 | 65 | class_name = classes 66 | 67 | error_csv(name_file_class, csv_dir, args.yes) 68 | df_classes = pd.read_csv(CLASSES_CSV, header=None) 69 | 70 | class_code = df_classes.loc[df_classes[1] == class_name].values[0][0] 71 | 72 | if args.type_csv == 'train': 73 | name_file = file_list[0] 74 | df_val = TTV(csv_dir, name_file, args.yes) 75 | if not args.n_threads: 76 | download(args, df_val, folder[0], dataset_dir, class_name, class_code) 77 | else: 78 | download(args, df_val, folder[0], dataset_dir, class_name, class_code, threads = int(args.n_threads)) 79 | 80 | elif args.type_csv == 'validation': 81 | name_file = file_list[1] 82 | df_val = TTV(csv_dir, name_file, args.yes) 83 | if not args.n_threads: 84 | download(args, df_val, folder[1], dataset_dir, class_name, class_code) 85 | else: 86 | download(args, df_val, folder[1], dataset_dir, class_name, class_code, threads = int(args.n_threads)) 87 | 88 | elif args.type_csv == 'test': 89 | name_file = file_list[2] 90 | df_val = TTV(csv_dir, name_file, args.yes) 91 | if not args.n_threads: 92 | download(args, df_val, folder[2], dataset_dir, class_name, class_code) 93 | else: 94 | download(args, df_val, folder[2], dataset_dir, class_name, class_code, threads = int(args.n_threads)) 95 | 96 | elif args.type_csv == 'all': 97 | for i in range(3): 98 | name_file = file_list[i] 99 | df_val = TTV(csv_dir, name_file, args.yes) 100 | if not args.n_threads: 101 | download(args, df_val, folder[i], dataset_dir, class_name, class_code) 102 | else: 103 | download(args, df_val, folder[i], dataset_dir, class_name, class_code, threads = int(args.n_threads)) 104 | else: 105 | print(bc.FAIL + 'csv file not specified' + bc.ENDC) 106 | exit(1) 107 | 108 | elif args.multiclasses == '1': 109 | 110 | class_list = args.classes 111 | print(bc.INFO + "Downloading {} together.".format(class_list) + bc.ENDC) 112 | multiclass_name = ['_'.join(class_list)] 113 | mkdirs(dataset_dir, csv_dir, multiclass_name, args.type_csv) 114 | 115 | error_csv(name_file_class, csv_dir, args.yes) 116 | df_classes = pd.read_csv(CLASSES_CSV, header=None) 117 | 118 | class_dict = {} 119 | for class_name in class_list: 120 | class_dict[class_name] = df_classes.loc[df_classes[1] == class_name].values[0][0] 121 | 122 | for class_name in class_list: 123 | 124 | if args.type_csv == 'train': 125 | name_file = file_list[0] 126 | df_val = TTV(csv_dir, name_file, args.yes) 127 | if not args.n_threads: 128 | download(args, df_val, folder[0], dataset_dir, class_name, class_dict[class_name], class_list) 129 | else: 130 | download(args, df_val, folder[0], dataset_dir, class_name, class_dict[class_name], class_list, int(args.n_threads)) 131 | 132 | elif args.type_csv == 'validation': 133 | name_file = file_list[1] 134 | df_val = TTV(csv_dir, name_file, args.yes) 135 | if not args.n_threads: 136 | download(args, df_val, folder[1], dataset_dir, class_name, class_dict[class_name], class_list) 137 | else: 138 | download(args, df_val, folder[1], dataset_dir, class_name, class_dict[class_name], class_list, int(args.n_threads)) 139 | 140 | elif args.type_csv == 'test': 141 | name_file = file_list[2] 142 | df_val = TTV(csv_dir, name_file, args.yes) 143 | if not args.n_threads: 144 | download(args, df_val, folder[2], dataset_dir, class_name, class_dict[class_name], class_list) 145 | else: 146 | download(args, df_val, folder[2], dataset_dir, class_name, class_dict[class_name], class_list, int(args.n_threads)) 147 | 148 | elif args.type_csv == 'all': 149 | for i in range(3): 150 | name_file = file_list[i] 151 | df_val = TTV(csv_dir, name_file, args.yes) 152 | if not args.n_threads: 153 | download(args, df_val, folder[i], dataset_dir, class_name, class_dict[class_name], class_list) 154 | else: 155 | download(args, df_val, folder[i], dataset_dir, class_name, class_dict[class_name], class_list, int(args.n_threads)) 156 | -------------------------------------------------------------------------------- /modules/parser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def parser_arguments(): 4 | ''' 5 | Manage the input from the terminal. 6 | :return: parser 7 | ''' 8 | parser = argparse.ArgumentParser(description='Open Image Dataset Downloader') 9 | 10 | parser.add_argument("command", 11 | metavar=" 'downloader', 'visualizer' or 'ill_downloader'.", 12 | help="'downloader', 'visualizer' or 'ill_downloader'.") 13 | parser.add_argument('--Dataset', required=False, 14 | metavar="/path/to/OID/csv/", 15 | help='Directory of the OID dataset folder') 16 | parser.add_argument('-y', '--yes', required=False, action='store_true', 17 | #metavar="Yes to download missing files", 18 | help='ans Yes to possible download of missing files') 19 | parser.add_argument('--classes', required=False, nargs='+', 20 | metavar="list of classes", 21 | help="Sequence of 'strings' of the wanted classes") 22 | parser.add_argument('--type_csv', required=False, choices=['train', 'test', 'validation', 'all'], 23 | metavar="'train' or 'validation' or 'test' or 'all'", 24 | help='From what csv search the images') 25 | 26 | parser.add_argument('--sub', required=False, choices=['h', 'm'], 27 | metavar="Subset of human verified images or machine generated (h or m)", 28 | help='Download from the human verified dataset or from the machine generated one.') 29 | 30 | parser.add_argument('--image_IsOccluded', required=False, choices=['0', '1'], 31 | metavar="1 or 0", 32 | help='Optional characteristic of the images. Indicates that the object is occluded by another object in the image.') 33 | parser.add_argument('--image_IsTruncated', required=False, choices=['0', '1'], 34 | metavar="1 or 0", 35 | help='Optional characteristic of the images. Indicates that the object extends beyond the boundary of the image.') 36 | parser.add_argument('--image_IsGroupOf', required=False, choices=['0', '1'], 37 | metavar="1 or 0", 38 | help='Optional characteristic of the images. Indicates that the box spans a group of objects (min 5).') 39 | parser.add_argument('--image_IsDepiction', required=False, choices=['0', '1'], 40 | metavar="1 or 0", 41 | help='Optional characteristic of the images. Indicates that the object is a depiction.') 42 | parser.add_argument('--image_IsInside', required=False, choices=['0', '1'], 43 | metavar="1 or 0", 44 | help='Optional characteristic of the images. Indicates a picture taken from the inside of the object.') 45 | 46 | parser.add_argument('--multiclasses', required=False, default='0', choices=['0', '1'], 47 | metavar="0 (default) or 1", 48 | help='Download different classes separately (0) or together (1)') 49 | 50 | parser.add_argument('--n_threads', required=False, metavar="[default 20]", 51 | help='Num of the threads to use') 52 | 53 | parser.add_argument('--noLabels', required=False, action='store_true', 54 | help='No labels creations') 55 | 56 | parser.add_argument('--limit', required=False, type=int, default=None, 57 | metavar="integer number", 58 | help='Optional limit on number of images to download') 59 | 60 | return parser.parse_args() 61 | -------------------------------------------------------------------------------- /modules/show.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import re 4 | import numpy as np 5 | 6 | class_list = [] 7 | color_dic = dict() 8 | flag = 0 9 | 10 | def color_gen(): 11 | ''' 12 | Generate a new color. As first color generates (0, 255, 0) 13 | ''' 14 | global flag 15 | 16 | if flag == 0: 17 | color = (0, 255, 0) 18 | flag += 1 19 | else: 20 | np.random.seed() 21 | color = tuple(255 * np.random.rand(3)) 22 | return color 23 | 24 | def show(class_name, download_dir, label_dir,total_images, index): 25 | ''' 26 | Show the images with the labeled boxes. 27 | 28 | :param class_name: self explanatory 29 | :param download_dir: folder that contains the images 30 | :param label_dir: folder that contains the labels 31 | :param index: self explanatory 32 | :return: None 33 | ''' 34 | 35 | global class_list, color_dic 36 | 37 | if not os.listdir(download_dir)[index].endswith('.jpg'): 38 | index += 2 39 | img_file = os.listdir(download_dir)[index] 40 | current_image_path = str(os.path.join(download_dir, img_file)) 41 | img = cv2.imread(current_image_path) 42 | file_name = str(img_file.split('.')[0]) + '.txt' 43 | file_path = os.path.join(label_dir, file_name) 44 | f = open(file_path, 'r') 45 | 46 | window_name = "Visualizer: {}/{}".format(index+1, total_images) 47 | 48 | cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) 49 | width = 500 50 | height = int((img.shape[0] * width) / img.shape[1]) 51 | cv2.resizeWindow(window_name, width, height) 52 | 53 | for line in f: 54 | # each row in a file is class_name, XMin, YMix, XMax, YMax 55 | match_class_name = re.compile('^[a-zA-Z]+(\s+[a-zA-Z]+)*').match(line) 56 | class_name = line[:match_class_name.span()[1]] 57 | ax = line[match_class_name.span()[1]:].lstrip().rstrip().split(' ') 58 | # opencv top left bottom right 59 | 60 | if class_name not in class_list: 61 | class_list.append(class_name) 62 | color = color_gen() 63 | color_dic[class_name] = color 64 | 65 | font = cv2.FONT_HERSHEY_SIMPLEX 66 | r ,g, b = color_dic[class_name] 67 | cv2.putText(img,class_name,(int(float(ax[0]))+5,int(float(ax[1]))-7), font, 0.8,(b, g, r), 2,cv2.LINE_AA) 68 | cv2.rectangle(img, (int(float(ax[-2])), int(float(ax[-1]))), 69 | (int(float(ax[-4])), 70 | int(float(ax[-3]))), (b, g, r), 3) 71 | 72 | cv2.imshow(window_name, img) 73 | -------------------------------------------------------------------------------- /modules/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from textwrap import dedent 3 | 4 | def images_options(df_val, args): 5 | ''' 6 | Manage the options for the images downloader. 7 | 8 | :param df_val: DataFrame Value. 9 | :param args: argument parser. 10 | :return: modified df_val 11 | ''' 12 | if args.image_IsOccluded is not None: 13 | rejectedID = df_val.ImageID[df_val.IsOccluded != int(args.image_IsOccluded)].values 14 | df_val = df_val[~df_val.ImageID.isin(rejectedID)] 15 | 16 | if args.image_IsTruncated is not None: 17 | rejectedID = df_val.ImageID[df_val.IsTruncated != int(args.image_IsTruncated)].values 18 | df_val = df_val[~df_val.ImageID.isin(rejectedID)] 19 | 20 | if args.image_IsGroupOf is not None: 21 | rejectedID = df_val.ImageID[df_val.IsGroupOf != int(args.image_IsGroupOf)].values 22 | df_val = df_val[~df_val.ImageID.isin(rejectedID)] 23 | 24 | if args.image_IsDepiction is not None: 25 | rejectedID = df_val.ImageID[df_val.IsDepiction != int(args.image_IsDepiction)].values 26 | df_val = df_val[~df_val.ImageID.isin(rejectedID)] 27 | 28 | if args.image_IsInside is not None: 29 | rejectedID = df_val.ImageID[df_val.IsInside != int(args.image_IsInside)].values 30 | df_val = df_val[~df_val.ImageID.isin(rejectedID)] 31 | 32 | return df_val 33 | 34 | def mkdirs(Dataset_folder, csv_folder, classes, type_csv): 35 | ''' 36 | Make the folder structure for the system. 37 | 38 | :param Dataset_folder: Self explanatory 39 | :param csv_folder: folder path of csv files 40 | :param classes: list of classes to download 41 | :param type_csv: train, validation, test or all 42 | :return: None 43 | ''' 44 | 45 | directory_list = ['train', 'validation', 'test'] 46 | 47 | if not type_csv == 'all': 48 | for class_name in classes: 49 | if not Dataset_folder.endswith('_nl'): 50 | folder = os.path.join(Dataset_folder, type_csv, class_name, 'Label') 51 | else: 52 | folder = os.path.join(Dataset_folder, type_csv, class_name) 53 | if not os.path.exists(folder): 54 | os.makedirs(folder) 55 | filelist = [f for f in os.listdir(folder) if f.endswith(".txt")] 56 | for f in filelist: 57 | os.remove(os.path.join(folder, f)) 58 | 59 | else: 60 | for directory in directory_list: 61 | for class_name in classes: 62 | if not Dataset_folder.endswith('_nl'): 63 | folder = os.path.join(Dataset_folder, directory, class_name, 'Label') 64 | else: 65 | folder = os.path.join(Dataset_folder, directory, class_name, 'Label') 66 | if not os.path.exists(folder): 67 | os.makedirs(folder) 68 | filelist = [f for f in os.listdir(folder) if f.endswith(".txt")] 69 | for f in filelist: 70 | os.remove(os.path.join(folder, f)) 71 | 72 | if not os.path.exists(csv_folder): 73 | os.makedirs(csv_folder) 74 | 75 | def progression_bar(total_images, index): 76 | ''' 77 | Print the progression bar for the download of the images. 78 | 79 | :param total_images: self explanatory 80 | :param index: self explanatory 81 | :return: None 82 | ''' 83 | # for windows os 84 | if os.name == 'nt': 85 | from ctypes import windll, create_string_buffer 86 | 87 | h = windll.kernel32.GetStdHandle(-12) 88 | csbi = create_string_buffer(22) 89 | res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) 90 | 91 | if res: 92 | import struct 93 | (bufx, bufy, curx, cury, wattr, 94 | left, top, right, bottom, maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) 95 | columns = right - left + 1 96 | rows = bottom - top + 1 97 | else: 98 | columns, rows = 80, 25 # can't determine actual size - return default values 99 | # for linux/gnu os 100 | else: 101 | rows, columns = os.popen('stty size', 'r').read().split() 102 | toolbar_width = int(columns) - 10 103 | image_index = index 104 | index = int(index / total_images * toolbar_width) 105 | 106 | print(' ' * (toolbar_width), end='\r') 107 | bar = "[{}{}] {}/{}".format('-' * index, ' ' * (toolbar_width - index), image_index, total_images) 108 | print(bar.rjust(int(columns)), end='\r') 109 | 110 | def show_classes(classes): 111 | '''imag 112 | Show the downloaded classes in the selected folder during visualization mode 113 | ''' 114 | for n in classes: 115 | print("- {}".format(n)) 116 | print("\n") 117 | 118 | def logo(command): 119 | ''' 120 | Print the logo for the downloader and the visualizer when selected 121 | ''' 122 | bc = bcolors 123 | 124 | print(bc.OKGREEN + """ 125 | ___ _____ ______ _ _ 126 | .' `.|_ _||_ _ `. | | | | 127 | / .-. \ | | | | `. \ _ __ | |__| |_ 128 | | | | | | | | | | |[ \ [ ]|____ _| 129 | \ `-' /_| |_ _| |_.' / \ \/ / _| |_ 130 | `.___.'|_____||______.' \__/ |_____| 131 | """ + bc.ENDC) 132 | 133 | if command == 'downloader': 134 | print(bc.OKGREEN + ''' 135 | _____ _ _ 136 | (____ \ | | | | 137 | _ \ \ ___ _ _ _ ____ | | ___ ____ _ | | ____ ____ 138 | | | | / _ \| | | | _ \| |/ _ \ / _ |/ || |/ _ )/ ___) 139 | | |__/ / |_| | | | | | | | | |_| ( ( | ( (_| ( (/ /| | 140 | |_____/ \___/ \____|_| |_|_|\___/ \_||_|\____|\____)_| 141 | 142 | ''' + bc.ENDC) 143 | 144 | if command == 'visualizer': 145 | print(bc.OKGREEN + """ 146 | _ _ _ _ _ 147 | | | | (_) | (_) 148 | | | | |_ ___ _ _ ____| |_ _____ ____ ____ 149 | \ \/ /| |/___) | | |/ _ | | (___ ) _ )/ ___) 150 | \ / | |___ | |_| ( ( | | | |/ __( (/ /| | 151 | \/ |_(___/ \____|\_||_|_|_(_____)____)_| 152 | 153 | """ + bc.ENDC) 154 | 155 | if command == 'downloader_ill': 156 | print(bc.OKGREEN + ''' 157 | 158 | _____ _ _ _____ _ _ 159 | | __ \ | | | | |_ _| | | | 160 | | | | | _____ ___ __ | | ___ __ _ __| | ___ _ __ | | | | | | 161 | | | | |/ _ \ \ /\ / / '_ \| |/ _ \ / _` |/ _` |/ _ \ '__| | | | | | | 162 | | |__| | (_) \ V V /| | | | | (_) | (_| | (_| | __/ | _| |_| |____| |____ 163 | |_____/ \___/ \_/\_/ |_| |_|_|\___/ \__,_|\__,_|\___|_| |_____|______|______| 164 | 165 | 166 | ''' + bc.ENDC) 167 | 168 | class bcolors: 169 | HEADER = '\033[95m' 170 | 171 | INFO = ' [INFO] | ' 172 | OKBLUE = '\033[94m[DOWNLOAD] | ' 173 | WARNING = '\033[93m [WARN] | ' 174 | FAIL = '\033[91m [ERROR] | ' 175 | 176 | OKGREEN = '\033[92m' 177 | ENDC = '\033[0m' -------------------------------------------------------------------------------- /oid_to_pascal_voc_xml.py: -------------------------------------------------------------------------------- 1 | import os 2 | from tqdm import tqdm 3 | from sys import exit 4 | import argparse 5 | import cv2 6 | from textwrap import dedent 7 | from lxml import etree 8 | 9 | XML_DIR = '' 10 | 11 | #os.chdir('Dataset') 12 | os.chdir(os.path.join("OID", "Dataset")) 13 | DIRS = os.listdir(os.getcwd()) 14 | 15 | for DIR in DIRS: 16 | if os.path.isdir(DIR): 17 | os.chdir(DIR) 18 | 19 | print("Currently in Subdirectory:", DIR) 20 | CLASS_DIRS = os.listdir(os.getcwd()) 21 | for CLASS_DIR in CLASS_DIRS: 22 | if " " in CLASS_DIR: 23 | os.rename(CLASS_DIR, CLASS_DIR.replace(" ", "_")) 24 | 25 | CLASS_DIRS = os.listdir(os.getcwd()) 26 | for CLASS_DIR in CLASS_DIRS: 27 | #if " " in CLASS_DIR: 28 | # os.rename(CLASS_DIR, CLASS_DIR.replace(" ", "_")) 29 | if os.path.isdir(CLASS_DIR): 30 | os.chdir(CLASS_DIR) 31 | 32 | print("\n" + "Creating PASCAL VOC XML Files for Class:", CLASS_DIR) 33 | # Create Directory for annotations if it does not exist yet 34 | #if not os.path.exists(XML_DIR): 35 | # os.makedirs(XML_DIR) 36 | 37 | #Read Labels from OIDv4 ToolKit 38 | os.chdir("Label") 39 | 40 | #Create PASCAL XML 41 | for filename in tqdm(os.listdir(os.getcwd())): 42 | if filename.endswith(".txt"): 43 | filename_str = str.split(filename, ".")[0] 44 | 45 | 46 | annotation = etree.Element("annotation") 47 | 48 | os.chdir("..") 49 | folder = etree.Element("folder") 50 | folder.text = os.path.basename(os.getcwd()) 51 | annotation.append(folder) 52 | 53 | filename_xml = etree.Element("filename") 54 | filename_xml.text = filename_str + ".jpg" 55 | annotation.append(filename_xml) 56 | 57 | path = etree.Element("path") 58 | path.text = os.path.join(os.path.dirname(os.path.abspath(filename)), filename_str + ".jpg") 59 | annotation.append(path) 60 | 61 | source = etree.Element("source") 62 | annotation.append(source) 63 | 64 | database = etree.Element("database") 65 | database.text = "Unknown" 66 | source.append(database) 67 | 68 | size = etree.Element("size") 69 | annotation.append(size) 70 | 71 | width = etree.Element("width") 72 | height = etree.Element("height") 73 | depth = etree.Element("depth") 74 | 75 | img = cv2.imread(filename_xml.text) 76 | 77 | try: 78 | width.text = str(img.shape[1]) 79 | except AttributeError: 80 | #os.chdir("..") 81 | os.chdir("Label") 82 | continue 83 | height.text = str(img.shape[0]) 84 | depth.text = str(img.shape[2]) 85 | 86 | size.append(width) 87 | size.append(height) 88 | size.append(depth) 89 | 90 | segmented = etree.Element("segmented") 91 | segmented.text = "0" 92 | annotation.append(segmented) 93 | 94 | os.chdir("Label") 95 | label_original = open(filename, 'r') 96 | 97 | # Labels from OIDv4 Toolkit: name_of_class X_min Y_min X_max Y_max 98 | for line in label_original: 99 | line = line.strip() 100 | l = line.split(' ') 101 | 102 | class_name_len = len(l) - 4 # 4 coordinates 103 | class_name = l[0] 104 | for i in range(1,class_name_len): 105 | class_name = f"{class_name}_{l[i]}" 106 | 107 | addi = class_name_len 108 | 109 | xmin_l = str(int(round(float(l[0+addi])))) 110 | ymin_l = str(int(round(float(l[1+addi])))) 111 | xmax_l = str(int(round(float(l[2+addi])))) 112 | ymax_l = str(int(round(float(l[3+addi])))) 113 | 114 | obj = etree.Element("object") 115 | annotation.append(obj) 116 | 117 | name = etree.Element("name") 118 | name.text = class_name 119 | obj.append(name) 120 | 121 | pose = etree.Element("pose") 122 | pose.text = "Unspecified" 123 | obj.append(pose) 124 | 125 | truncated = etree.Element("truncated") 126 | truncated.text = "0" 127 | obj.append(truncated) 128 | 129 | difficult = etree.Element("difficult") 130 | difficult.text = "0" 131 | obj.append(difficult) 132 | 133 | bndbox = etree.Element("bndbox") 134 | obj.append(bndbox) 135 | 136 | xmin = etree.Element("xmin") 137 | xmin.text = xmin_l 138 | bndbox.append(xmin) 139 | 140 | ymin = etree.Element("ymin") 141 | ymin.text = ymin_l 142 | bndbox.append(ymin) 143 | 144 | xmax = etree.Element("xmax") 145 | xmax.text = xmax_l 146 | bndbox.append(xmax) 147 | 148 | ymax = etree.Element("ymax") 149 | ymax.text = ymax_l 150 | bndbox.append(ymax) 151 | 152 | os.chdir("..") 153 | 154 | #os.chdir(XML_DIR) 155 | 156 | # write xml to file 157 | s = etree.tostring(annotation, pretty_print=True) 158 | with open(filename_str + ".xml", 'wb') as f: 159 | f.write(s) 160 | f.close() 161 | 162 | #os.chdir("..") 163 | os.chdir("Label") 164 | 165 | os.chdir("..") 166 | os.chdir("..") 167 | 168 | os.chdir("..") 169 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | awscli 4 | 5 | urllib3 6 | 7 | tqdm 8 | 9 | opencv-python -------------------------------------------------------------------------------- /voc_to_YOLOv3.py: -------------------------------------------------------------------------------- 1 | import xml.etree.ElementTree as ET 2 | from os import getcwd 3 | import os 4 | 5 | 6 | dataset_train = 'OID\\Dataset\\train\\' 7 | dataset_file = '4_CLASS_test.txt' 8 | classes_file = dataset_file[:-4]+'_classes.txt' 9 | 10 | 11 | CLS = os.listdir(dataset_train) 12 | classes =[dataset_train+CLASS for CLASS in CLS] 13 | wd = getcwd() 14 | 15 | 16 | def test(fullname): 17 | bb = "" 18 | in_file = open(fullname) 19 | tree=ET.parse(in_file) 20 | root = tree.getroot() 21 | for i, obj in enumerate(root.iter('object')): 22 | difficult = obj.find('difficult').text 23 | cls = obj.find('name').text 24 | if cls not in CLS or int(difficult)==1: 25 | continue 26 | cls_id = CLS.index(cls) 27 | xmlbox = obj.find('bndbox') 28 | b = (int(xmlbox.find('xmin').text), int(xmlbox.find('ymin').text), int(xmlbox.find('xmax').text), int(xmlbox.find('ymax').text)) 29 | bb += (" " + ",".join([str(a) for a in b]) + ',' + str(cls_id)) 30 | 31 | # we need this because I don't know overlapping or something like that 32 | if cls == 'Traffic_light': 33 | list_file = open(dataset_file, 'a') 34 | file_string = str(fullname)[:-4]+'.jpg'+bb+'\n' 35 | list_file.write(file_string) 36 | list_file.close() 37 | bb = "" 38 | 39 | if bb != "": 40 | list_file = open(dataset_file, 'a') 41 | file_string = str(fullname)[:-4]+'.jpg'+bb+'\n' 42 | list_file.write(file_string) 43 | list_file.close() 44 | 45 | 46 | 47 | for CLASS in classes: 48 | for filename in os.listdir(CLASS): 49 | if not filename.endswith('.xml'): 50 | continue 51 | fullname = os.getcwd()+'\\'+CLASS+'\\'+filename 52 | test(fullname) 53 | 54 | for CLASS in CLS: 55 | list_file = open(classes_file, 'a') 56 | file_string = str(CLASS)+"\n" 57 | list_file.write(file_string) 58 | list_file.close() 59 | --------------------------------------------------------------------------------

Edit status

Suggestions:

760 | 761 | github/gitignore 764 | 765 | 766 |

962 | Users who have contributed to this file 963 |

Forked this repository from official creators and added 2 python scripts:

~ OIDv4 ToolKit ~

760 | 761 | github/gitignore 764 | 765 | 766 |

900 | gitignore/Python.gitignore 901 |

962 | Users who have contributed to this file 963 |

Forked this repository from official creators and added 2 python scripts:

~ OIDv4 ToolKit ~