├── .gitignore ├── README.md ├── cityscapes_to_voc.ipynb └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cityscapes_to_voc 2 | Python code for converting Cityscapes dataset to PASCAL VOC format 3 | -------------------------------------------------------------------------------- /cityscapes_to_voc.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Cityscapes_to_voc\n", 8 | "##### This code converts cityscapes to Pascal VOC 2007 format to be used for object detection tasks\n", 9 | "##### Written by: Hamza Rawal (mscs18004@itu.edu.pk / hamzarawal@gmail.com)\n" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 21, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from pascal_voc_writer import Writer\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "import numpy as np\n", 21 | "import os\n", 22 | "import json\n", 23 | "import glob\n", 24 | "import time\n", 25 | "from shutil import move, copy" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "#### Global Parameters" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 9, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "#----------\n", 42 | "#arguments\n", 43 | "#----------\n", 44 | "cityscapes_dir = '../datasets/cityscapes/cityscapes/'\n", 45 | "save_path = './cityscapes_voc_annotations/'\n", 46 | "\n", 47 | "cityscapes_dir_gt = os.path.join(cityscapes_dir, 'gtFine')" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "#### Pascal VOC classes" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 10, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "#--------------------------------------------------------------\n", 64 | "# Valid classes dictionary.\n", 65 | "# motorcycle is renamed to motorbike, change the key if this is\n", 66 | "# not is required\n", 67 | "#--------------------------------------------------------------\n", 68 | "classes = {'bicycle':'bicycle', 'bus':'bus', 'car':'car', 'motorcycle':'motorbike', \n", 69 | " 'person':'person', 'rider': 'rider', 'train':'train', 'truck':'truck'}\n", 70 | "classes_keys = list(classes.keys())\n" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "#### Function for making Directories" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 11, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "#------------------------\n", 87 | "#function to make folder\n", 88 | "#------------------------\n", 89 | "def make_dir(path):\n", 90 | " if not os.path.isdir(path):\n", 91 | " os.makedirs(path)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "#### Function for converting Polygons to Bounding Boxes" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 12, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "#----------------------------------------------------------------------------------------------------------------\n", 108 | "#convert polygon to bounding box\n", 109 | "#code from: \n", 110 | "#https://stackoverflow.com/questions/46335488/how-to-efficiently-find-the-bounding-box-of-a-collection-of-points\n", 111 | "#----------------------------------------------------------------------------------------------------------------\n", 112 | "def polygon_to_bbox(polygon):\n", 113 | " x_coordinates, y_coordinates = zip(*polygon)\n", 114 | " return [min(x_coordinates), min(y_coordinates), max(x_coordinates), max(y_coordinates)]" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "#### Function to read json files and return Polygons with classes" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 17, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "#--------------------------------------------\n", 131 | "#read a json file and convert to voc format\n", 132 | "#--------------------------------------------\n", 133 | "def read_json(file):\n", 134 | " \n", 135 | " #if no relevant objects found in the image,\n", 136 | " #don't save the xml for the image\n", 137 | " relevant_file = False\n", 138 | " \n", 139 | " data = []\n", 140 | " with open(file, 'r') as f:\n", 141 | " file_data = json.load(f)\n", 142 | "\n", 143 | " for object in file_data['objects']:\n", 144 | " label, polygon = object['label'], object['polygon']\n", 145 | " \n", 146 | " #process only if label found in voc\n", 147 | " if label in classes_keys:\n", 148 | " polygon = np.array([x for x in polygon])\n", 149 | " bbox = polygon_to_bbox(polygon)\n", 150 | " data.append([classes[label]]+bbox)\n", 151 | "\n", 152 | " #if relevant objects found in image, set the flag to True\n", 153 | " if data:\n", 154 | " relevant_file = True\n", 155 | "\n", 156 | " return data, relevant_file" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "#### Create and save xml file for each image in Cityscapes" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 14, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "#---------------------------\n", 173 | "#function to save xml file\n", 174 | "#---------------------------\n", 175 | "def save_xml(img_path, img_shape, data, save_path):\n", 176 | " writer = Writer(img_path,img_shape[0], img_shape[1])\n", 177 | " for element in data:\n", 178 | " writer.addObject(element[0],element[1],element[2],element[3],element[4])\n", 179 | " writer.save(save_path) " 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "#### Main Code" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 18, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "Total Time taken: 502.8435652256012\n" 199 | ] 200 | } 201 | ], 202 | "source": [ 203 | "#------------------------------------------\n", 204 | "#reading json files from each subdirectory\n", 205 | "#------------------------------------------\n", 206 | "valid_files = []\n", 207 | "trainval_files = []\n", 208 | "test_files = []\n", 209 | "\n", 210 | "#make Annotations target directory if already doesn't exist\n", 211 | "ann_dir = os.path.join(save_path, 'VOC2007','Annotations')\n", 212 | "make_dir(ann_dir)\n", 213 | "\n", 214 | "start = time.time()\n", 215 | "for category in os.listdir(cityscapes_dir_gt):\n", 216 | " \n", 217 | " #no GT for test data\n", 218 | " if category == 'test': continue\n", 219 | " \n", 220 | " for city in os.listdir(os.path.join(cityscapes_dir_gt, category)):\n", 221 | "\n", 222 | " #read files\n", 223 | " files = glob.glob(os.path.join(cityscapes_dir, 'gtFine', category, city)+'/*.json')\n", 224 | " \n", 225 | " #process json files\n", 226 | " for file in files:\n", 227 | " data, relevant_file = read_json(file)\n", 228 | " \n", 229 | " if relevant_file:\n", 230 | " base_filename = os.path.basename(file)[:-21]\n", 231 | " xml_filepath = os.path.join(ann_dir,base_filename + '_leftImg8bit.xml')\n", 232 | " img_name = base_filename+'_leftImg8bit.png'\n", 233 | " img_path = os.path.join(cityscapes_dir, 'leftImg8bit', category, city, base_filename+'_leftImg8bit.png')\n", 234 | " img_shape = plt.imread(img_path).shape\n", 235 | " valid_files.append([img_path, img_name])\n", 236 | " \n", 237 | " #make list of trainval and test files for voc format \n", 238 | " #lists will be stored in txt files\n", 239 | " trainval_files.append(img_name[:-4]) if category == 'train' else test_files.append(img_name[:-4])\n", 240 | " \n", 241 | " #save xml file\n", 242 | " save_xml(img_path, img_shape, data, xml_filepath)\n", 243 | " \n", 244 | "end = time.time() - start\n", 245 | "print('Total Time taken: ', end)" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "#### Copy images from all sub-dirs of Cityscapes to a single directory for voc format" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 165, 258 | "metadata": {}, 259 | "outputs": [ 260 | { 261 | "name": "stdout", 262 | "output_type": "stream", 263 | "text": [ 264 | "Total Time taken: 5.453717470169067\n" 265 | ] 266 | } 267 | ], 268 | "source": [ 269 | "#----------------------------\n", 270 | "#copy files into target path\n", 271 | "#----------------------------\n", 272 | "images_savepath = os.path.join(save_path, 'VOC2007', 'JPEGImages')\n", 273 | "make_dir(images_savepath)\n", 274 | "\n", 275 | "start = time.time()\n", 276 | "for file in valid_files:\n", 277 | " copy(file[0], os.path.join(images_savepath, file[1]))\n", 278 | " \n", 279 | "print('Total Time taken: ', end)" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": {}, 285 | "source": [ 286 | "#### Create txt files for trainval and test images" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": 19, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [ 295 | "#---------------------------------------------\n", 296 | "#create text files of trainval and test files\n", 297 | "#---------------------------------------------\n", 298 | "textfiles_savepath = os.path.join(save_path, 'VOC2007', 'ImageSets', 'Main')\n", 299 | "make_dir(textfiles_savepath)\n", 300 | "\n", 301 | "traival_files_wr = [x+'\\n' for x in trainval_files]\n", 302 | "test_files_wr = [x+'\\n' for x in test_files]\n", 303 | "\n", 304 | "with open(os.path.join(textfiles_savepath, 'trainval.txt'), 'w') as f:\n", 305 | " f.writelines(traival_files_wr)\n", 306 | " \n", 307 | "with open(os.path.join(textfiles_savepath, 'test.txt'), 'w') as f:\n", 308 | " f.writelines(test_files_wr)" 309 | ] 310 | } 311 | ], 312 | "metadata": { 313 | "kernelspec": { 314 | "display_name": "Python 3", 315 | "language": "python", 316 | "name": "python3" 317 | }, 318 | "language_info": { 319 | "codemirror_mode": { 320 | "name": "ipython", 321 | "version": 3 322 | }, 323 | "file_extension": ".py", 324 | "mimetype": "text/x-python", 325 | "name": "python", 326 | "nbconvert_exporter": "python", 327 | "pygments_lexer": "ipython3", 328 | "version": "3.6.10" 329 | } 330 | }, 331 | "nbformat": 4, 332 | "nbformat_minor": 2 333 | } 334 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | numpy 3 | pascal_voc_writer 4 | --------------------------------------------------------------------------------