├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── append_voc_dataset_prefix.py ├── bdd100k2voc.py ├── check_voc_dataset.py ├── cityscape2voc.py ├── cityscapes2foggy_cityscapes.py ├── coco2voc.py ├── coco_names.txt ├── copy_voc_dataset.py ├── delete_voc_dataset_object.py ├── get_voc_classes.py ├── kitti2voc.py ├── labelme2voc.py ├── merge_voc_dataset_v1.py ├── merge_voc_dataset_v2.py ├── mot2voc.py ├── random_split_voc_dataset.py ├── rename_voc_dataset_object_label.py ├── video2voc_dataset.py ├── voc2coco.py └── voc2labelme.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 DaiPuWei 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Object_Detection_Dataset_Utils 2 | 这是各种目标检测数据集预处理工具脚本集合，主要功能实现各种数据集之间的格式格式转换和VOC数据集各种预处理功能。 3 | 4 | --- 5 | 6 | # 环境配置 7 | - cv2 8 | - xml 9 | - json 10 | - tqdm 11 | - numpy 12 | - pycocotools 13 | - multiprocessing 14 | - pascal_voc_writer 15 | 16 | --- 17 | # 功能介绍 18 | ## X2VOC 19 | 以下脚本是将各种数据集转换成VOC数据集格式的脚本，具体功能如下： 20 | - `cityscapes2voc.py`是利用多线程将Cityscapes数据集转换为VOC数据集格式的脚本； 21 | - `cityscapes2foggy_cityscapes.py`是根据雾深度图像数据集和Cityscapes数据集生成Foggy_Cityscapes数据集； 22 | - `bdd100k2voc.py`是利用多线程将BDD100k数据集转换VOC数据集格式的脚本； 23 | - `kitti2voc.py`是利用多线程将KITTI数据集转换为VOC数据集格式的脚本； 24 | - `coco2voc.py`是利用多线程将COCO数据集转换为VOC数据集格式的脚本； 25 | - `mot2voc.py`是利用多线程将MOT目标跟踪数据集转换为VOC数据集格式的脚本； 26 | 27 | ## VOC相关预处理 28 | 以下脚本是对VOC格式数据集进行各种预处理功能，具体功能如下： 29 | - `delete_voc_dataset_object.py`是利用多线程删除VOC格式数据集XML标签文件中指定目标分类以外的所有目标框的脚本； 30 | - `check_voc_dataset.py`是利用多线程检查VOC格式数据集XML文件与图像文件是否一一对应，若不意义对应则删除相关XML文件或图像文件的脚本； 31 | - `merge_voc_dataset.py`是利用多线程将多个VOC格式数据集合并成一个VOC格式数据集的脚本； 32 | - `random_split_voc_dataset.py`是将VOC格式数据集随机划分成训练集和测试集，并生成对应子数据集txt文件的脚本； 33 | - `raname_voc_dataset_object_label.py`是利用多线程将VOC格式数据集中指定分类的目标进行重命名的脚本 34 | - `video2voc_dataset.py`是利用多线程将视频按照指定间隔进行抽帧生成VOC数据集的函数； 35 | - `get_voc_dataset.py`是利用多线程获取VOC格式数据集的目标分类数组的脚本 36 | - `copy_voc_dataset.py`是利用多线程复制VOC格式数据集，并只保留指定目标分类的函数 37 | 38 | --- 39 | # How to Use 40 | ## Cityscapes --> VOC 41 | 下载Cityscapes数据集并完成解压，其文档结构如下所示： 42 | ``` 43 | cityscapes 44 | - leftImg8bit 45 | - train 46 | - city1 47 | - image1.png 48 | - image2.png 49 | - ... 50 | - city2 51 | - ... 52 | - val 53 | - test 54 | - gtFine 55 | - train 56 | - city1 57 | - label1.json 58 | - label2.json 59 | - ... 60 | - city2 61 | - ... 62 | - val 63 | - test 64 | ``` 65 | 根据开发需要修改`cityscape_dataset_dir`、`voc_dataset_dir`和`class_names`,其中`cityscape_dataset_dir`代表原始Cityscapes数据集目录，`voc_dataset_dir`代表转换为VOC数据集格式后Cityscapes数据集目录，`class_names`代表目标名称数组，然后运行如下代码即可生成VOC数据集格式的Cityscapes数据集。 66 | ```bash 67 | python cityscape2voc.py 68 | ``` 69 | 70 | ## Foggy Cityscapes --> VOC 71 | 首先下载Cityscapes的数据集和Foggy Cityscapes的图像数据集，并完成两个数据集的解压，Foggy Cityscapes图像数据集文件结构如下所示： 72 | ``` 73 | cityscapes_foggy 74 | - leftImg8bit_trainvaltest_foggy 75 | - leftImg8bit_foggy 76 | - train 77 | - city1 78 | - image1_foggy_beta_0.005.png 79 | - image1_foggy_beta_0.01.png 80 | - image1_foggy_beta_0.02.png 81 | - ... 82 | - city2 83 | - ... 84 | - val 85 | - test 86 | ``` 87 | 然后根据`cityscape2voc.py`脚本将Cityscapes数据集转换为VOC数据集格式，最后根据开发需要修改`cityscapes_dataset_dir`、`foggy_cityscapes_dataset_dir`、`foggy_image_dir`和`beta`，其中`beta`控制雾浓度，候选值有`0.005`、`0.01`和`0.02`，`cityscapes_dataset_dir`为VOC数据集格式的Cityscapes数据集目录、`foggy_cityscapes_dataset_dir`为voc数据集格式的Foggy Cityscapes数据集目录、`foggy_image_dir`为Foggy Cityscapes图像数据集目录，然后运行如下代码即可生成VOC数据集格式的Foggy Cityscapes数据集。 88 | ```bash 89 | pyhon cityscapes2foggy_cityscapes.py 90 | ``` 91 | 92 | # BDD100k --> VOC 93 | 下载BDD100k数据集并完成解压，其文件结构如下所示： 94 | ``` 95 | bdd100k 96 | - images 97 | - 10k 98 | - 100k 99 | - train 100 | - image1.png 101 | - image2.png 102 | - ... 103 | - val 104 | - test 105 | - labels 106 | - 10k 107 | - 100k 108 | - train 109 | - label1.json 110 | - label2.json 111 | - ... 112 | - val 113 | - test 114 | ``` 115 | 然后根据开发需要修改`bdd100k_dataset_dir`、`voc_dataset_dir`、`class_names`和`dataset_type`,其中`bdd100k_dataset_dir`代表原始BDD100k数据集目录，`voc_dataset_dir`代表VOC数据集格式的BDD100k数据集目录，`class_names`代表目标名称数组，该参数控制VOC数据集格式的BDD100k数据集包含的目标种类，`dataset_type`代表数据集类型，候选值有‘all’、‘daytime’和‘night’，‘all’代表转化全部数据，‘daytime’代表转化白天数据，‘night’代表转化夜晚数据。最后运行如下代码即可生成VOC数据集格式的BDD100k数据集。 116 | ```bash 117 | python bdd100k2voc.py 118 | ``` 119 | 120 | ## KITTI --> VOC 121 | 首先下载KITTI数据集并完成解压，其文件结构如下所示： 122 | ``` 123 | kitti 124 | - train 125 | - image_2 126 | - name1.png 127 | - name2.png 128 | - ... 129 | - label_2 130 | - name1.txt 131 | - name2.txt 132 | - ... 133 | - test 134 | ``` 135 | 然后根据开发需要自行修改`kitti_dataset_dir`、`voc_dataset_dir`、`train_ratio`和`class_names`，其中`kitti_dataset_dir`为原始KITTI数据集目录路径,`voc_dataset_dir`为VOC数据集格式的KITTI数据集目录路径,`train_ratio`为训练集比例，默认为0.8,用于随机划分训练集和验证集使用，`class_names`为目标名称数组，该参数控制VOC数据集格式的BDD100k数据集包含的目标种类，默认为['Person_sitting',"Pedestrian",'Cyclist',"Truck","Car","Tram","Van"]， 最后运行如下命令即可生成VOC数据集格式的KITTI数据集。 136 | ```bash 137 | python kitti2voc.py 138 | ``` 139 | ## COCO --> VOC 140 | 下载COCO数据集并完成解压，其文件结构如下： 141 | ``` 142 | coco 143 | - train 144 | - name1.jpg 145 | - name2.jpg 146 | - ... 147 | - val 148 | - annotations 149 | - instances_train_year.json 150 | - instances_val_year.json 151 | - ... 152 | ``` 153 | 然后根据开发需要自行修改`coco_dataset_dir`、`voc_dataset_dir`和`year`， 其中`coco_dataset_dir`代表原始COCO数据集目录,`voc_dataset_dir`代表VOC数据集格式的COCO数据集目录,`year`代表原始COCO数据集的年份, 最后运行如下命令即可生成VOC数据集格式的COCO数据集。 154 | ```bash 155 | python coco2voc.py 156 | ``` 157 | ## MOT --> VOC 158 | 下载MOT数据集并完成解压，其文件结构如下： 159 | ``` 160 | MOT 161 | - train 162 | - subset_name1 163 | - det 164 | - det.txt 165 | - gt 166 | - gt.txt 167 | - img1 168 | - name1.jpg 169 | - name2.jpg 170 | - ... 171 | - subset_name2 172 | - ... 173 | - test 174 | - subset_name1 175 | - det 176 | - det.txt 177 | - img1 178 | - name1.jpg 179 | - name2.jpg 180 | - ... 181 | - subset_name2 182 | - ... 183 | ``` 184 | 然后根据开发需要自行修改`mot_dataste_dir`、`voc_dataset_dir`和`type`， 其中`mot_dataset_dir`代表原始MOT数据集目录，`voc_dataset_dir`代表VOC数据集格式的COCO数据集目录，`type`代表数据集类型，最后运行如下命令即可生成VOC数据集格式的MOT数据集。 185 | ```bash 186 | python mot2voc.py 187 | ``` 188 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/7/31 下午1:51 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : __init__.py.py 6 | # @Software: PyCharm 7 | 8 | def run_main(): 9 | """ 10 | 这是主函数 11 | """ 12 | 13 | 14 | if __name__ == '__main__': 15 | run_main() 16 | -------------------------------------------------------------------------------- /append_voc_dataset_prefix.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2024/4/21 下午9:22 3 | # @Author : DaiPuWei 4 | # @Email : daipuwei@qq.com 5 | # @File : append_voc_dataset_prefix.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是给VOC数据集图像及其标签文件加上前缀的脚本 10 | """ 11 | 12 | import os 13 | import shutil 14 | import numpy as np 15 | import xml.etree.ElementTree as ET 16 | from tqdm import tqdm 17 | from pascal_voc_writer import Writer 18 | from multiprocessing import Pool,cpu_count 19 | 20 | def append_voc_dataset_prefix(voc_dataset_dir, voc_dataset_prefix): 21 | """ 22 | 这是给VOC数据集图像及其标签文件加上前缀的函数 23 | Args: 24 | voc_dataset_dir: VOC数据集地址 25 | voc_dataset_prefix: VOC数据集前缀 26 | Returns: 27 | """ 28 | # 初始化voc数据集相关路径 29 | voc_image_dir = os.path.join(voc_dataset_dir, 'JPEGImages') 30 | if not os.path.exists(voc_image_dir): 31 | voc_image_dir = os.path.join(voc_dataset_dir, 'images') 32 | voc_annotation_dir = os.path.join(voc_dataset_dir, "Annotations") 33 | voc_main_dir = os.path.join(voc_dataset_dir,'ImageSets',"Main") 34 | 35 | # 更新子集txt文件 36 | for choice in ["train","val","trainval","test"]: 37 | choice_txt_path = os.path.join(voc_main_dir, choice + '.txt') 38 | if not os.path.exists(choice_txt_path): 39 | continue 40 | lines = [] 41 | with open(choice_txt_path,'r',encoding='utf-8') as f: 42 | for line in f.readlines(): 43 | lines.append(line.strip()) 44 | with open(choice_txt_path,'w',encoding='utf-8') as f: 45 | for line in lines: 46 | f.write("{0}_{1}\n".format(voc_dataset_prefix,line)) 47 | 48 | # 初始化相关路径 49 | voc_image_paths = [] 50 | voc_annotation_paths = [] 51 | new_voc_image_paths = [] 52 | new_voc_annotation_paths = [] 53 | for image_name in os.listdir(voc_image_dir): 54 | fname,ext = os.path.splitext(image_name) 55 | voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 56 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,fname+".xml")) 57 | new_voc_image_paths.append(os.path.join(voc_image_dir,"{0}_{1}".format(voc_dataset_prefix,image_name))) 58 | new_voc_annotation_paths.append(os.path.join(voc_annotation_dir,"{0}_{1}.xml".format(voc_dataset_prefix,fname))) 59 | voc_image_paths = np.array(voc_image_paths) 60 | voc_annotation_paths = np.array(voc_annotation_paths) 61 | new_voc_image_paths = np.array(new_voc_image_paths) 62 | new_voc_annotation_paths = np.array(new_voc_annotation_paths) 63 | 64 | # 多线程处理复制图像和xml文件 65 | size = len(voc_image_paths) 66 | if size // cpu_count() != 0: 67 | num_threads = cpu_count() 68 | elif size // (cpu_count() // 2) != 0: 69 | num_threads = cpu_count() // 2 70 | elif size // (cpu_count() // 4) != 0: 71 | num_threads = cpu_count() // 4 72 | else: 73 | num_threads = 1 74 | batch_size = size // num_threads 75 | pool = Pool(processes=num_threads) 76 | for start in np.arange(0,size,batch_size): 77 | end = int(np.min([start+batch_size,size])) 78 | batch_voc_image_paths = voc_image_paths[start:end] 79 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 80 | batch_new_voc_image_paths = new_voc_image_paths[start:end] 81 | batch_new_voc_annotation_paths = new_voc_annotation_paths[start:end] 82 | pool.apply_async(batch_process_image_xml,error_callback=print_error, 83 | args=(batch_voc_image_paths,batch_voc_annotation_paths, 84 | batch_new_voc_image_paths,batch_new_voc_annotation_paths)) 85 | pool.close() 86 | pool.join() 87 | 88 | def print_error(value): 89 | """ 90 | 定义错误回调函数 91 | Args: 92 | value: 出错误值 93 | Returns: 94 | """ 95 | print("error: ", value) 96 | 97 | def parse_xml(xml_path,class_names=None): 98 | """ 99 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 100 | Args: 101 | xml_path: XML标签文件路径 102 | class_names: 目标名称数组，默认为None 103 | Returns: 104 | """ 105 | # 获取XML文件的根结点 106 | root = ET.parse(xml_path).getroot() 107 | h = int(root.find("size").find("height").text) 108 | w = int(root.find("size").find("width").text) 109 | # 遍历所有目标 110 | objects = [] 111 | for obj in root.findall('object'): 112 | obj_name = obj.find('name').text 113 | bndbox = obj.find('bndbox') 114 | xmin = bndbox.find('xmin').text 115 | ymin = bndbox.find('ymin').text 116 | xmax = bndbox.find('xmax').text 117 | ymax = bndbox.find('ymax').text 118 | if class_names is None: 119 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 120 | else: 121 | if obj_name in class_names: 122 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 123 | return objects,(h,w) 124 | 125 | def process_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path): 126 | """ 127 | 这是复制VOC图像及其xml文件的函数 128 | Args: 129 | voc_image_path: voc图像文件路径 130 | voc_xml_path: voc标签文件路径 131 | new_voc_image_path: voc图像文件新路径 132 | new_voc_xml_path: voc标签文件新路径 133 | Returns: 134 | """ 135 | # 复制图像 136 | shutil.copy(voc_image_path,new_voc_image_path) 137 | 138 | # xml解析复制制定类别目标 139 | objects,(h,w) = parse_xml(voc_xml_path) 140 | writer = Writer(new_voc_image_path,w,h) 141 | for cls_name,x1,y1,x2,y2 in objects: 142 | writer.addObject(cls_name,x1,y1,x2,y2) 143 | writer.save(new_voc_xml_path) 144 | 145 | def batch_process_image_xml(batch_voc_image_paths,batch_voc_annotation_paths, 146 | batch_new_voc_image_paths,batch_new_voc_annotation_paths): 147 | """ 148 | 这是批量处理VOC图像及其标签函数 149 | Args: 150 | batch_voc_image_paths: 批量voc图像路径数组 151 | batch_voc_annotation_paths: 批量voc标签路径数组 152 | batch_new_voc_image_paths: 批量voc图像新路径数组 153 | batch_new_voc_annotation_paths: 批量voc标签新路径数组 154 | Returns: 155 | """ 156 | size = len(batch_voc_image_paths) 157 | for i in tqdm(np.arange(size)): 158 | voc_image_path = batch_voc_image_paths[i] 159 | voc_xml_path = batch_voc_annotation_paths[i] 160 | new_voc_image_path = batch_new_voc_image_paths[i] 161 | new_voc_xml_path = batch_new_voc_annotation_paths[i] 162 | process_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path) 163 | 164 | 165 | def run_main(): 166 | """ 167 | 这是主函数 168 | """ 169 | 170 | 171 | if __name__ == '__main__': 172 | run_main() 173 | -------------------------------------------------------------------------------- /bdd100k2voc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/7/31 下午6:12 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : bdd100k2voc.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将BDD100k数据集转换为VOC数据集格式的脚本，根据开发需要自行修改 10 | bdd100k_dataset_dir、voc_dataset_dir、class_names和dataset_type即可。 11 | 12 | 其中: 13 | - bdd100k_dataset_dir代表原始BDD100k数据集目录； 14 | - voc_dataset_dir代表VOC数据集格式的BDD100k数据集目录； 15 | - class_names代表目标名称数组，该参数控制VOC数据集格式的BDD100k数据集包含的目标种类， 16 | 默认为['car', 'person', 'rider', 'truck', 'bus','train', 'motorcycle', 'bicycle','traffic sign','traffic light']； 17 | - dataset_type代表数据集类型，候选值有‘all’、‘daytime’和‘night’， 18 | ‘all’代表转化全部数据，‘daytime’代表转化白天数据，‘night’代表转化夜晚数据； 19 | """ 20 | 21 | import os 22 | import cv2 23 | import json 24 | import numpy as np 25 | from tqdm import tqdm 26 | from multiprocessing import Pool 27 | from multiprocessing import cpu_count 28 | from pascal_voc_writer import Writer 29 | 30 | def bdd100k2voc(bdd100k_dataset_dir,voc_dataset_dir,dataset_type='all', 31 | class_names=['car','person','rider','truck','bus','train','motorcycle','bicycle','traffic sign','traffic light']): 32 | """ 33 | 这是将BDD100k数据集转换为VOC数据集格式的文件 34 | :param bdd100k_dataset_dir: bdd100k数据集目录地址 35 | :param voc_dataset_dir: voc数据集目录地址 36 | :param dataset_type: 数据集类型，默认为‘all’，即使用全部bdd100k数据集，候选值有['all','daytime','night'], 37 | 若为‘daytime’则是只筛选白天场景,若为‘night’则只筛选夜晚场景 38 | :param class_names: 目标名称数组，默认为['car','person','rider','truck','bus','train', 39 | 'motorcycle','bicycle','traffic sign','traffic light'] 40 | :return: 41 | """ 42 | # 初始化bdd100k数据集下的相关路径 43 | bdd100k_image_dir = os.path.join(bdd100k_dataset_dir,'images','100k') 44 | bdd100k_label_dir = os.path.join(bdd100k_dataset_dir,'labels','100k') 45 | 46 | # 初始化voc数据集相关路径 47 | voc_image_dir = os.path.join(voc_dataset_dir,'JPEGImages') 48 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 49 | voc_imagesets_main_dir = os.path.join(voc_dataset_dir,"ImageSets","Main") 50 | if not os.path.exists(voc_image_dir): 51 | os.makedirs(voc_image_dir) 52 | if not os.path.exists(voc_annotation_dir): 53 | os.makedirs(voc_annotation_dir) 54 | if not os.path.exists(voc_imagesets_main_dir): 55 | os.makedirs(voc_imagesets_main_dir) 56 | 57 | # 初始化bdd100k图像和标签文件数组 58 | bdd100k_image_paths = [] 59 | bdd100k_json_paths = [] 60 | voc_image_paths = [] 61 | voc_annotation_paths = [] 62 | for choice in ['train','val']: 63 | # 初始化每个子数据集相关数组 64 | _bdd100k_image_paths = [] 65 | _bdd100k_json_paths = [] 66 | _bdd100k_scene_mask_array = [] # 是否白天还是夜晚场景掩膜数组 67 | _voc_image_paths = [] 68 | _voc_annotation_paths = [] 69 | # 初始化子数据集目录路径 70 | _bdd100k_image_dir = os.path.join(bdd100k_image_dir, choice) 71 | _bdd100k_json_dir = os.path.join(bdd100k_label_dir, choice) 72 | for image_name in tqdm(os.listdir(_bdd100k_image_dir)): # 遍历子集下的所有图片 73 | # 初始化_bdd100k数据集和voc数据集图像和标签文件路径 74 | name,ext = os.path.splitext(image_name) 75 | bdd100k_image_path = os.path.join(_bdd100k_image_dir,image_name) 76 | bdd100k_json_path = os.path.join(_bdd100k_json_dir,name+".json") 77 | is_contain_object, scene_type = is_conitain_object_and_scene_type(bdd100k_json_path,class_names) 78 | if is_contain_object: # 判断图片是否包含候选目标实例 79 | _bdd100k_image_paths.append(bdd100k_image_path) 80 | _bdd100k_json_paths.append(bdd100k_json_path) 81 | _bdd100k_scene_mask_array.append(scene_type) 82 | _voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 83 | _voc_annotation_paths.append(os.path.join(voc_annotation_dir,name+".xml")) 84 | _bdd100k_image_paths = np.array(_bdd100k_image_paths) 85 | _bdd100k_json_paths = np.array(_bdd100k_json_paths) 86 | _bdd100k_scene_mask_array = np.array(_bdd100k_scene_mask_array,dtype=np.int32) 87 | _voc_image_paths = np.array(_voc_image_paths) 88 | _voc_annotation_paths = np.array(_voc_annotation_paths) 89 | # 筛选不同场景的数据 90 | if dataset_type == 'daytime': 91 | mask_index = _bdd100k_scene_mask_array == 0 92 | elif dataset_type == 'night': 93 | mask_index = _bdd100k_scene_mask_array == 1 94 | else: 95 | mask_index = np.array([True]*len(_bdd100k_scene_mask_array)) 96 | _bdd100k_image_paths = _bdd100k_image_paths[mask_index] 97 | _bdd100k_json_paths = _bdd100k_json_paths[mask_index] 98 | _voc_image_paths = _voc_image_paths[mask_index] 99 | _voc_annotation_paths = _voc_annotation_paths[mask_index] 100 | 101 | # 将子数据及图像名称写入voc数据集的txt文件 102 | with open(os.path.join(voc_imagesets_main_dir, choice + '.txt'), "w") as f: 103 | for voc_image_path in _voc_image_paths: 104 | dir,image_name = os.path.split(voc_image_path) 105 | name,ext = os.path.splitext(image_name) 106 | f.write(name+"\n") 107 | bdd100k_image_paths.append(_bdd100k_image_paths) 108 | bdd100k_json_paths.append(_bdd100k_json_paths) 109 | voc_image_paths.append(_voc_image_paths) 110 | voc_annotation_paths.append(_voc_annotation_paths) 111 | bdd100k_image_paths = np.concatenate(bdd100k_image_paths,axis=0) 112 | bdd100k_json_paths = np.concatenate(bdd100k_json_paths,axis=0) 113 | voc_image_paths = np.concatenate(voc_image_paths,axis=0) 114 | voc_annotation_paths = np.concatenate(voc_annotation_paths, axis=0) 115 | # 将数据集图像名称写入voc数据集的trainval.txt文件 116 | with open(os.path.join(voc_imagesets_main_dir, 'trainval.txt'), "w") as f: 117 | for voc_image_path in voc_image_paths: 118 | dir, image_name = os.path.split(voc_image_path) 119 | name, ext = os.path.splitext(image_name) 120 | f.write(name + "\n") 121 | 122 | # 利用多线程将cityscape数据集转化为voc数据集 123 | size = len(bdd100k_image_paths) 124 | batch_size = size // (cpu_count() - 1) 125 | pool = Pool(processes=cpu_count() - 1) 126 | for i, start in enumerate(np.arange(0, size, batch_size)): 127 | # 获取小批量数据 128 | end = int(np.min([start + batch_size, size])) 129 | batch_bdd100k_image_paths = bdd100k_image_paths[start:end] 130 | batch_bdd100k_json_paths = bdd100k_json_paths[start:end] 131 | batch_voc_image_paths = voc_image_paths[start:end] 132 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 133 | print("线程{}处理{}张图像及其标签".format(i, len(batch_bdd100k_json_paths))) 134 | pool.apply_async(batch_image_label_process, error_callback=print_error, 135 | args=(batch_bdd100k_image_paths, batch_bdd100k_json_paths, 136 | batch_voc_image_paths, batch_voc_annotation_paths, class_names)) 137 | pool.close() 138 | pool.join() 139 | 140 | def print_error(value): 141 | """ 142 | 定义错误回调函数 143 | :param value: 144 | :return: 145 | """ 146 | print("error: ", value) 147 | 148 | def is_conitain_object_and_scene_type(bdd100k_json_path,class_names): 149 | """ 150 | 这是判断bdd100k数据集的json标签中是否包含候选目标实例并判断图片是否属于白天场景的函数 151 | :param bdd100k_json_path: cityscapes数据集的json标签文件路径 152 | :param class_names: 目标分类数组 153 | :return: 154 | """ 155 | json_dict = json.load(open(bdd100k_json_path, 'r')) # 加载json标签 156 | is_contain_object = False 157 | for obj in json_dict['frames'][0]["objects"]: # load_dict['objects'] -> 目标的几何框体 158 | obj_label = obj['category'] # 目标的类型 159 | if obj_label not in class_names: 160 | continue 161 | else: 162 | is_contain_object = True 163 | break 164 | if json_dict['attributes']["timeofday"] == "daytime": # 白天 165 | scene_type = 0 166 | elif json_dict['attributes']["timeofday"] == "night": # 夜晚 167 | scene_type = 1 168 | else: # 黄昏或者黎明 169 | scene_type = 2 170 | return is_contain_object,scene_type 171 | 172 | def single_image_label_process(bdd100k_image_path,bdd100k_json_path, 173 | voc_image_path,voc_annotation_path,class_names): 174 | """ 175 | 这是将单张bdd100k图像及其JSON标签转换为VOC数据集格式的函数 176 | :param bdd100k_image_path: bdd100k图像路径 177 | :param bdd100k_json_path: bdd100k标签路径 178 | :param voc_image_path: voc图像路径 179 | :param voc_annotation_path: voc标签路径 180 | :param class_names: 目标名称数组 181 | :return: 182 | """ 183 | # 初始化VOC标签写类 184 | image = cv2.imread(bdd100k_image_path) 185 | h, w, c = np.shape(image) 186 | dir,image_name = os.path.split(bdd100k_image_path) 187 | writer = Writer(image_name,w,h) 188 | 189 | # JSON标签转化为VOC标签 190 | flag = False 191 | json_dict = json.load(open(bdd100k_json_path,'r')) # 加载json标签 192 | for obj in json_dict["frames"][0]['objects']: # load_dict['objects'] -> 目标的几何框体 193 | obj_label = obj['category'] # 目标的类型 194 | if obj_label not in class_names: # 目标分类标签不在候选目标分类标签数组中，跳过，只处理候选目标名称数组里的目标实例 195 | continue 196 | else: 197 | # 获取目标的矩形框标签,并添加到写类中 198 | if obj_label in ['traffic sign','traffic light']: # 对于有空格的标签名称，删除空格 199 | strs = obj_label.split(" ") 200 | obj_label = strs[0]+"_"+strs[1] 201 | # 处理bbox坐标 202 | xmin = max(int(obj['box2d']['x1'])-1,0) 203 | ymin = max(int(obj['box2d']['y1'])-1,0) 204 | xmax = min(int(obj['box2d']['x2'])+1,w) 205 | ymax = min(int(obj['box2d']['y2'])+1,h) 206 | writer.addObject(obj_label,xmin,ymin,xmax,ymax) 207 | writer.save(voc_annotation_path) 208 | flag = True 209 | if flag: # 复制图像 210 | cv2.imwrite(voc_image_path, image) 211 | 212 | 213 | def batch_image_label_process(batch_bdd100k_image_paths,batch_bdd100k_json_paths, 214 | batch_voc_image_paths,batch_voc_annotation_paths,class_names): 215 | """ 216 | 批量处理cityscape数据，转化为voc数据 217 | :param batch_bdd100k_image_paths: 批量cityscapes图像路径数组 218 | :param batch_bdd100k_json_paths: 批量cityscapes标签路径数组 219 | :param batch_voc_image_paths: 批量voc图像路径数组 220 | :param batch_voc_annotation_paths: 批量voc标签路径数组 221 | :param class_names: 目标名称数组 222 | :return: 223 | """ 224 | size = len(batch_bdd100k_image_paths) 225 | for i in tqdm(np.arange(size)): 226 | bdd100k_image_path = batch_bdd100k_image_paths[i] 227 | bdd100k_json_path = batch_bdd100k_json_paths[i] 228 | voc_image_path = batch_voc_image_paths[i] 229 | voc_annotation_path = batch_voc_annotation_paths[i] 230 | #print(bdd100k_image_path,bdd100k_json_path,voc_image_path,voc_annotation_path) 231 | single_image_label_process(bdd100k_image_path,bdd100k_json_path, 232 | voc_image_path,voc_annotation_path,class_names) 233 | 234 | def run_main(): 235 | """ 236 | 这是主函数 237 | """ 238 | # BDD100k --> VOC 239 | print("BDD100k --> VOC Start") 240 | bdd100k_dataset_dir = os.path.abspath("../object_detection_dataset/BDD100K/bdd100k") 241 | voc_dataset_dir = os.path.abspath("../dataset/BDD100k") 242 | class_names = ['car', 'person', 'rider', 'truck', 'bus', 'train', 'motorcycle', 'bicycle', 'traffic sign', 243 | 'traffic light'] 244 | dataset_type = 'all' 245 | bdd100k2voc(bdd100k_dataset_dir, voc_dataset_dir, dataset_type,class_names) 246 | print("BDD100k --> VOC Finish") 247 | 248 | # BDD100k --> VOC 249 | print("BDD100k Daytime --> VOC Start") 250 | bdd100k_dataset_dir = os.path.abspath("../object_detection_dataset/BDD100K/bdd100k") 251 | voc_dataset_dir = os.path.abspath("../dataset/BDD100k-Daytime") 252 | class_names = ['car', 'person', 'rider', 'truck', 'bus', 'train', 'motorcycle', 'bicycle', 'traffic sign', 253 | 'traffic light'] 254 | dataset_type = 'daytime' 255 | bdd100k2voc(bdd100k_dataset_dir, voc_dataset_dir, dataset_type,class_names) 256 | print("BDD100k Daytime--> VOC Finish") 257 | 258 | # BDD100k Night --> VOC 259 | print("BDD100k Night --> VOC Start") 260 | bdd100k_dataset_dir = os.path.abspath("../object_detection_dataset/BDD100K/bdd100k") 261 | voc_dataset_dir = os.path.abspath("../dataset/BDD100k-Night") 262 | class_names = ['car', 'person', 'rider', 'truck', 'bus', 'train', 'motorcycle', 'bicycle', 'traffic sign', 263 | 'traffic light'] 264 | dataset_type = 'night' 265 | bdd100k2voc(bdd100k_dataset_dir, voc_dataset_dir, dataset_type,class_names) 266 | print("BDD100k Night --> VOC Finish") 267 | 268 | if __name__ == '__main__': 269 | run_main() 270 | -------------------------------------------------------------------------------- /check_voc_dataset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/5/10 12:56 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : check_voc_dataset.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是检查VOC数据集是否存在异常的脚本,检测XML文件与图像文件是否一一对应，XML文件中是否不存目标 10 | """ 11 | 12 | import os 13 | import cv2 14 | import numpy as np 15 | import xml.etree.ElementTree as ET 16 | from tqdm import tqdm 17 | from multiprocessing import Pool 18 | from multiprocessing import cpu_count 19 | 20 | def check_voc_dataset(voc_dataset_dir): 21 | """ 22 | 这是检查VOC数据集是否存在异常的函数 23 | Args: 24 | voc_dataset_dir: VOC数据集文件夹地址 25 | Returns: 26 | """ 27 | # 初始化相关路径 28 | voc_image_dir = os.path.join(voc_dataset_dir,"JPEGImages") 29 | if not os.path.exists(voc_image_dir): 30 | voc_image_dir = os.path.join(voc_dataset_dir, "images") 31 | voc_annotatiion_dir = os.path.join(voc_dataset_dir,"Annotations") 32 | 33 | # 初始化xml和图像路径 34 | voc_image_paths = [] 35 | voc_annotatiion_paths = [] 36 | for image_name in os.listdir(voc_image_dir): 37 | fname,ext = os.path.splitext(image_name) 38 | voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 39 | voc_annotatiion_paths.append(os.path.join(voc_annotatiion_dir,fname+".xml")) 40 | voc_image_paths = np.array(voc_image_paths) 41 | voc_annotatiion_paths = np.array(voc_annotatiion_paths) 42 | 43 | # 多线程检查VOC数据集 44 | size = len(voc_image_paths) 45 | batch_size = size // (cpu_count()-1) 46 | pool = Pool(processes=cpu_count()-1) 47 | for start in np.arange(0,size,batch_size): 48 | end = int(np.min([start+batch_size,size])) 49 | batch_voc_image_paths = voc_image_paths[start:end] 50 | batch_voc_annotation_paths = voc_annotatiion_paths[start:end] 51 | pool.apply_async(batch_check_images_annotations,callback=print_error, 52 | args=(batch_voc_image_paths,batch_voc_annotation_paths)) 53 | pool.close() 54 | pool.join() 55 | 56 | def batch_check_images_annotations(batch_voc_image_paths,batch_voc_annotation_paths): 57 | """ 58 | 这是批量检查图像文件和标签文件是否一一对应的函数 59 | :param batch_voc_image_paths: 批量VOC图像路径数组 60 | :param batch_voc_annotation_paths: 批量VOC标签文件路径数组 61 | :return: 62 | """ 63 | size = len(batch_voc_image_paths) 64 | for i in tqdm(np.arange(size)): 65 | check_image_annotation(batch_voc_image_paths[i],batch_voc_annotation_paths[i]) 66 | 67 | def check_image_annotation(voc_image_path,voc_annotation_path): 68 | """ 69 | 这是图像文件和标签文件是否对应的函数 70 | :param voc_image_path: VOC图像路径 71 | :param voc_annotation_path: VOC标签文件路径 72 | :return: 73 | """ 74 | if not os.path.exists(voc_annotation_path): # XML文件不存在，则删除图片 75 | os.remove(voc_image_path) 76 | else: 77 | if get_object_num(voc_annotation_path) == 0: # XML文件中目标数为0，则将图片和标签全部删除 78 | os.remove(voc_image_path) 79 | os.remove(voc_annotation_path) 80 | else: 81 | image = cv2.imread(voc_image_path) 82 | if image is None: 83 | os.remove(voc_image_path) 84 | os.remove(voc_annotation_path) 85 | 86 | def print_error(value): 87 | """ 88 | 定义错误回调函数 89 | :param value: 90 | :return: 91 | """ 92 | print("error: ", value) 93 | 94 | def get_object_num(xml_path): 95 | """ 96 | 这是解析XML文件目标个数的函数 97 | :param xml_path: XML文件路径 98 | :return: 99 | """ 100 | tree = ET.parse(xml_path) 101 | return len(tree.findall('object')) 102 | 103 | 104 | def run_main(): 105 | """ 106 | 这是主函数 107 | """ 108 | voc_dataset_dirs = [os.path.abspath("../dataset/BDD100k"), 109 | os.path.abspath("../dataset/Cityscapes"), 110 | os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.01"), 111 | os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.02"), 112 | os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.005"), 113 | os.path.abspath("../dataset/KITTI")] 114 | for voc_dataset_dir in voc_dataset_dirs: 115 | check_voc_dataset(voc_dataset_dir) 116 | 117 | 118 | if __name__ == '__main__': 119 | run_main() 120 | -------------------------------------------------------------------------------- /cityscape2voc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/7/31 下午1:52 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : cityscape2voc.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将Cityscapes数据集转换为VOC数据集格式的工具脚本。 10 | 根据开发需要修改cityscape_dataset_dir，voc_dataset_dir，class_names即可。 11 | 12 | 其中: 13 | - cityscape_dataset_dir是原生cityscapes数据集目录路径； 14 | - voc_dataset_dir是VOC数据集格式的cityscapes数据集目录路径； 15 | - class_names为指定目标名称数组，默认为['car', 'person', 'rider', 'truck', 'bus', 16 | 'train', 'motorcycle', 'bicycle','traffic sign','traffic light'],即只处理class_names出现的目标实例； 17 | """ 18 | 19 | import os 20 | import cv2 21 | import json 22 | import numpy as np 23 | from tqdm import tqdm 24 | from multiprocessing import Pool 25 | from multiprocessing import cpu_count 26 | from pascal_voc_writer import Writer 27 | 28 | def cityscapes2voc(cityscapes_dataset_dir,voc_dataset_dir, 29 | class_names=['car','person','rider','truck','bus','train','motorcycle','bicycle','traffic sign','traffic light']): 30 | """ 31 | 这是Cityscapes数据集转化为VOC数据集格式的函数 32 | :param cityscapes_dataset_dir: cityscapes数据集目录 33 | :param voc_dataset_dir: VOC数据集目录 34 | :param class_names: 目标名称数组，默认为['car','person','rider','truck','bus','train', 35 | 'motorcycle','bicycle','traffic sign','traffic light'] 36 | :return: 37 | """ 38 | # 初始化cityscapes数据集相关路径 39 | cityscapes_image_dir = os.path.join(cityscapes_dataset_dir,"leftImg8bit") 40 | cityscapes_label_dir = os.path.join(cityscapes_dataset_dir,'gtFine') 41 | 42 | # 初始化voc数据集相关路径 43 | voc_image_dir = os.path.join(voc_dataset_dir,'JPEGImages') 44 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 45 | voc_imagesets_main_dir = os.path.join(voc_dataset_dir,"ImageSets","Main") 46 | if not os.path.exists(voc_image_dir): 47 | os.makedirs(voc_image_dir) 48 | if not os.path.exists(voc_annotation_dir): 49 | os.makedirs(voc_annotation_dir) 50 | if not os.path.exists(voc_imagesets_main_dir): 51 | os.makedirs(voc_imagesets_main_dir) 52 | 53 | # 初始化图像和标签文件数组 54 | cityscapes_image_paths = [] 55 | cityscapes_json_paths = [] 56 | voc_image_paths = [] 57 | voc_annotation_paths = [] 58 | for choice in ['train','val']: 59 | # 初始化每个子数据集相关数组 60 | _cityscapes_image_paths = [] 61 | _cityscapes_json_paths = [] 62 | _cityscapes_foggy_mask_paths = [] 63 | _voc_image_paths = [] 64 | _voc_annotation_paths = [] 65 | # 初始化子数据集目录路径 66 | _cityscapes_image_dir = os.path.join(cityscapes_image_dir, choice) 67 | for city_name in tqdm(os.listdir(_cityscapes_image_dir)): # 遍历每个城市文件夹 68 | # 初始化每个城市目录路径 69 | city_dir = os.path.join(_cityscapes_image_dir,city_name) 70 | for image_name in os.listdir(city_dir): # 遍历每个城市下的所有图片 71 | # 初始化cityscape数据集和voc数据集图像和标签文件路径 72 | name,ext = os.path.splitext(image_name) 73 | cityscapes_image_path = os.path.join(city_dir,image_name) 74 | cityscapes_json_path = os.path.join(cityscapes_label_dir,choice,city_name, 75 | name.replace("leftImg8bit","gtFine_polygons")+".json") 76 | if is_conitain_object(cityscapes_json_path,class_names): # 判断图片是否包含候选目标实例 77 | _cityscapes_image_paths.append(cityscapes_image_path) 78 | _cityscapes_json_paths.append(cityscapes_json_path) 79 | _voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 80 | _voc_annotation_paths.append(os.path.join(voc_annotation_dir,name+".xml")) 81 | # 将子数据及图像名称写入voc数据集的txt文件 82 | with open(os.path.join(voc_imagesets_main_dir, choice + '.txt'), "w+") as f: 83 | for voc_image_path in _voc_image_paths: 84 | dir,image_name = os.path.split(voc_image_path) 85 | name,ext = os.path.splitext(image_name) 86 | f.write(name+"\n") 87 | cityscapes_image_paths.append(_cityscapes_image_paths) 88 | cityscapes_json_paths.append(_cityscapes_json_paths) 89 | voc_image_paths.append(_voc_image_paths) 90 | voc_annotation_paths.append(_voc_annotation_paths) 91 | cityscapes_image_paths = np.concatenate(cityscapes_image_paths,axis=0) 92 | cityscapes_json_paths = np.concatenate(cityscapes_json_paths,axis=0) 93 | voc_image_paths = np.concatenate(voc_image_paths,axis=0) 94 | voc_annotation_paths = np.concatenate(voc_annotation_paths, axis=0) 95 | # 将数据集图像名称写入voc数据集的trainval.txt文件 96 | with open(os.path.join(voc_imagesets_main_dir, 'trainval.txt'), "w+") as f: 97 | for voc_image_path in voc_image_paths: 98 | dir, image_name = os.path.split(voc_image_path) 99 | name, ext = os.path.splitext(image_name) 100 | f.write(name + "\n") 101 | 102 | # 利用多线程将cityscape数据集转化为voc数据集 103 | size = len(cityscapes_image_paths) 104 | batch_size = size // (cpu_count() - 1) 105 | pool = Pool(processes=cpu_count()-1) 106 | for i,start in enumerate(np.arange(0,size,batch_size)): 107 | # 获取小批量数据 108 | end = int(np.min([start+batch_size,size])) 109 | batch_cityscapes_image_paths = cityscapes_image_paths[start:end] 110 | batch_cityscapes_json_paths = cityscapes_json_paths[start:end] 111 | batch_voc_image_paths = voc_image_paths[start:end] 112 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 113 | print("线程{}处理{}张图像及其标签".format(i,len(batch_cityscapes_json_paths))) 114 | pool.apply_async(batch_image_label_process,error_callback=print, 115 | args=(batch_cityscapes_image_paths,batch_cityscapes_json_paths, 116 | batch_voc_image_paths,batch_voc_annotation_paths,class_names)) 117 | pool.close() 118 | pool.join() 119 | 120 | def print_error(value): 121 | """ 122 | 定义错误回调函数 123 | :param value: 124 | :return: 125 | """ 126 | print("error: ", value) 127 | 128 | def is_conitain_object(cityscapes_json_path,class_names): 129 | """ 130 | 这是判断cityscapes数据集的json标签中是否包含候选目标实例的函数 131 | :param cityscapes_json_path: cityscapes数据集的json标签文件路径 132 | :param class_names: 目标分类数组 133 | :return: 134 | """ 135 | json_dict = json.load(open(cityscapes_json_path, 'r')) # 加载json标签 136 | flag = False 137 | for obj in json_dict['objects']: # load_dict['objects'] -> 目标的几何框体 138 | obj_label = obj['label'] # 目标的类型 139 | if obj_label not in class_names: 140 | continue 141 | else: 142 | flag = True 143 | break 144 | return flag 145 | 146 | def find_box(points): 147 | """ 148 | 这是通过多边形分割标签数组获取目标检测矩形框的函数 149 | :param points: 多边形分割标签数组 150 | :return: 151 | """ 152 | x = [] 153 | y = [] 154 | for point in points: 155 | x.append(point[0]) 156 | y.append(point[1]) 157 | xmin = int(np.min(x)) 158 | ymin = int(np.min(y)) 159 | xmax = int(np.max(x)) 160 | ymax = int(np.max(y)) 161 | return xmin,ymin,xmax,ymax 162 | 163 | def single_image_label_process(cityscapes_image_path,cityscapes_json_path, 164 | voc_image_path,voc_annotation_path,class_names): 165 | """ 166 | 这是将单张cityscape图像及其JSON标签转换为VOC数据集格式的函数 167 | :param cityscapes_image_path: cityscape图像路径 168 | :param cityscapes_json_path: cityscape标签路径 169 | :param voc_image_path: voc图像路径 170 | :param voc_annotation_path: voc标签路径 171 | :param class_names: 目标名称数组 172 | :return: 173 | """ 174 | # 初始化VOC标签写类 175 | image = cv2.imread(cityscapes_image_path) 176 | h, w, c = np.shape(image) 177 | dir,image_name = os.path.split(cityscapes_image_path) 178 | writer = Writer(image_name,w,h) 179 | 180 | # JSON标签转化为VOC标签 181 | json_dict = json.load(open(cityscapes_json_path,'r')) # 加载json标签 182 | flag = False 183 | for obj in json_dict['objects']: # load_dict['objects'] -> 目标的几何框体 184 | obj_label = obj['label'] # 目标的类型 185 | if obj_label in ['out of roi', 'ego vehicle']: # 直接跳过这两种类型 注意测试集里只有这两种类型 跳过的话测试集合里将为空的标签 186 | continue 187 | elif obj_label not in class_names: # 目标分类标签不在候选目标分类标签数组中，跳过，只处理候选目标名称数组里的目标实例 188 | continue 189 | else: 190 | # 获取目标的矩形框标签,并添加到写类中 191 | if obj_label in ['traffic sign','traffic light']: # 对于有空格的标签名称，删除空格 192 | strs = obj_label.split(" ") 193 | obj_label = strs[0]+"_"+strs[1] 194 | xmin,ymin,xmax,ymax = find_box(obj['polygon']) 195 | writer.addObject(obj_label,xmin,ymin,xmax,ymax) 196 | writer.save(voc_annotation_path) 197 | flag = True 198 | if flag: # 写入目标标签则复制图像，否则不复制图像 199 | cv2.imwrite(voc_image_path, image) 200 | 201 | 202 | def batch_image_label_process(batch_cityscapes_image_paths,batch_cityscapes_json_paths, 203 | batch_voc_image_paths,batch_voc_annotation_paths,class_names): 204 | """ 205 | 批量处理cityscape数据转化为voc数据 206 | :param batch_cityscapes_image_paths: 批量cityscapes图像路径数组 207 | :param batch_cityscapes_json_paths: 批量cityscapes标签路径数组 208 | :param batch_voc_image_paths: 批量voc图像路径数组 209 | :param batch_voc_annotation_paths: 批量voc标签路径数组 210 | :param class_names: 目标名称数组 211 | :return: 212 | """ 213 | size = len(batch_cityscapes_image_paths) 214 | for i in tqdm(np.arange(size)): 215 | cityscapes_image_path = batch_cityscapes_image_paths[i] 216 | cityscapes_json_path = batch_cityscapes_json_paths[i] 217 | voc_image_path = batch_voc_image_paths[i] 218 | voc_annotation_path = batch_voc_annotation_paths[i] 219 | single_image_label_process(cityscapes_image_path,cityscapes_json_path, 220 | voc_image_path,voc_annotation_path,class_names) 221 | 222 | def run_main(): 223 | """ 224 | 这是主函数 225 | """ 226 | # Cityscapes --> VOC 227 | print("Cityscapes --> VOC Start") 228 | cityscape_dataset_dir = os.path.abspath("../object_detection_dataset/cityscapes") 229 | voc_dataset_dir = os.path.abspath("../dataset/Cityscapes") 230 | class_names = ['car', 'person', 'rider', 'truck', 'bus', 'train', 'motorcycle', 'bicycle','traffic sign','traffic light'] 231 | cityscapes2voc(cityscape_dataset_dir,voc_dataset_dir,class_names) 232 | print("Cityscapes --> VOC Finish") 233 | 234 | if __name__ == '__main__': 235 | run_main() -------------------------------------------------------------------------------- /cityscapes2foggy_cityscapes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/7/31 下午4:55 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : cityscapes2foggy_cityscapes.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是根据Cityscapes数据集和已有的Foggy Cityscapes图像集生成Foggy Cityscapes 10 | 数据集的工具脚本，默认Cityscapes数据集已经转化为VOC数据集格式。 根据开发需要修改beta、 11 | cityscapes_dataset_dir、foggy_cityscapes_dataset_dir和foggy_image_dir即可 12 | 完成Foggy Cityscapes数据集的生成。 13 | 14 | 其中: 15 | - cityscapes_dataset_dir代表已经转化为VOC数据集格式的cityscapes数据集目录地址； 16 | - foggy_cityscapes_dataset_dir代表VOC数据集格式的foggy cityscapes数据集目录地址； 17 | - foggy_image_dir为人工生成的带雾cityscapes图像数据目录地址； 18 | - beta代表雾浓度，候选值有0.005、0.01和0.02； 19 | """ 20 | 21 | import os 22 | import cv2 23 | import numpy as np 24 | import xml.etree.ElementTree as ET 25 | 26 | from tqdm import tqdm 27 | from multiprocessing import Pool 28 | from multiprocessing import cpu_count 29 | 30 | def cityscapes2foggy_cityscapes(cityscapes_dataset_dir,foggy_cityscapes_dataset_dir,foggy_image_dir,beta=0.01): 31 | """ 32 | 这是根据Cityscapes数据集和为Foggy Cityscapes图像集生成Foggy Cityscapes数据集的函数，假定Cityscapes数据集已经转化成VOC数据集格式 33 | :param cityscapes_dataset_dir: cityscapes数据集目录 34 | :param foggy_cityscapes_dataset_dir: foggy_cityscapes数据集目录 35 | :param foggy_image_dir: 雾深度图像数据集目录 36 | :param beta: beta参数，用于控制雾的浓度，默认为0.01，候选值有[0.005,0.01,0.02] 37 | :return: 38 | """ 39 | # 初始化Cityscapes和Foggy_Cityscape两个数据集相关路径 40 | foggy_cityscapes_image_dir = os.path.join(foggy_cityscapes_dataset_dir,"JPEGImages") 41 | foggy_cityscapes_label_dir = os.path.join(foggy_cityscapes_dataset_dir,"Annotations") 42 | foggy_cityscapes_main_dir = os.path.join(foggy_cityscapes_dataset_dir,"ImageSets","Main") 43 | if not os.path.exists(foggy_cityscapes_image_dir): 44 | os.makedirs(foggy_cityscapes_image_dir) 45 | if not os.path.exists(foggy_cityscapes_label_dir): 46 | os.makedirs(foggy_cityscapes_label_dir) 47 | if not os.path.exists(foggy_cityscapes_main_dir): 48 | os.makedirs(foggy_cityscapes_main_dir) 49 | 50 | # 初始化Foggy_CItyscapes数据集的XML文件路径 51 | cityscapes_label_dir = os.path.join(cityscapes_dataset_dir,"Annotations") 52 | cityscapes_label_paths = [] 53 | foggy_cityscapes_label_paths = [] 54 | for annotation_name in os.listdir(cityscapes_label_dir): 55 | name,ext = os.path.splitext(annotation_name) 56 | cityscapes_label_paths.append(os.path.join(cityscapes_label_dir,annotation_name)) 57 | foggy_cityscapes_label_paths.append(os.path.join(foggy_cityscapes_label_dir,name+"_foggy_beta_{}".format(beta)+ext)) 58 | 59 | # 初始化Foggy_Cityscapes数据集图像路径 60 | foggy_image_paths = [] 61 | foggy_cityscapes_image_paths = [] 62 | for choice in ["train","val"]: 63 | # 初始化子数据集目录路径 64 | _foggy_cityscapes_image_dir = os.path.join(foggy_image_dir, choice) 65 | with open(os.path.join(foggy_cityscapes_main_dir,choice+".txt"),'w') as f: 66 | for city_name in os.listdir(_foggy_cityscapes_image_dir): 67 | city_dir = os.path.join(_foggy_cityscapes_image_dir,city_name) 68 | for image_name in os.listdir(city_dir): 69 | if "beta_{}".format(beta) in image_name: 70 | name,ext = os.path.splitext(image_name) 71 | if is_contain_object(name,foggy_cityscapes_label_paths): # Foggy Cityscapes图像存在目标 72 | foggy_image_paths.append(os.path.join(city_dir,image_name)) 73 | foggy_cityscapes_image_paths.append(os.path.join(foggy_cityscapes_image_dir,image_name)) 74 | f.write(name+"\n") # 写入文件名称到指定txt文件 75 | # 将数据集图像名称写入voc数据集的trainval.txt文件 76 | with open(os.path.join(foggy_cityscapes_main_dir, 'trainval.txt'), "w+") as f: 77 | for cityscapes_image_path in foggy_cityscapes_image_paths: 78 | dir, image_name = os.path.split(cityscapes_image_path) 79 | name, ext = os.path.splitext(image_name) 80 | f.write(name + "\n") 81 | 82 | # 多线程将Foggy_Cityscapes数据集图像复制到Cityscapes数据集目录里，假定Cityscapes数据集已经转化成VOC数据集格式 83 | size = len(cityscapes_label_paths) 84 | batch_size = size // (cpu_count()-1) 85 | pool = Pool(processes=cpu_count()-1) 86 | for i,start in enumerate(np.arange(0,size,batch_size)): 87 | end = int(np.min([start+batch_size,size])) 88 | batch_cityscapes_label_paths = cityscapes_label_paths[start:end] 89 | batch_foggy_cityscapes_label_paths = foggy_cityscapes_label_paths[start:end] 90 | batch_foggy_image_paths = foggy_image_paths[start:end] 91 | batch_foggy_cityscapes_image_paths = foggy_cityscapes_image_paths[start:end] 92 | print("线程{}处理{}张图像".format(i,len(batch_foggy_cityscapes_image_paths))) 93 | pool.apply_async(batch_copy_image_label,error_callback=print_error, 94 | args=(batch_foggy_image_paths,batch_foggy_cityscapes_image_paths, 95 | batch_cityscapes_label_paths,batch_foggy_cityscapes_label_paths)) 96 | pool.close() 97 | pool.join() 98 | 99 | def batch_copy_image_label(batch_foggy_image_paths,batch_foggy_cityscapes_image_paths, 100 | batch_cityscapes_label_paths,batch_foggy_cityscapes_label_paths): 101 | """ 102 | 这是批量复制图像函数，将Foggy_Cityscapes数据集图像复制到Cityscapes数据集里 103 | :param batch_foggy_image_paths: 批量foggy图像路径数组 104 | :param batch_foggy_cityscapes_image_paths: 批量foggy_cityscapes数据集图像路径数组 105 | :param batch_cityscapes_label_paths: 批量cityscapes数据集标签路径数组 106 | :param batch_foggy_cityscapes_label_paths: 批量foggy_cityscape数据集标签路径数组 107 | :return: 108 | """ 109 | size = len(batch_foggy_image_paths) 110 | for i in tqdm(np.arange(size)): 111 | foggy_image_path = batch_foggy_image_paths[i] 112 | foggy_cityscapes_image_path = batch_foggy_cityscapes_image_paths[i] 113 | cityscapes_label_path = batch_cityscapes_label_paths[i] 114 | foggy_cityscapes_label_path = batch_foggy_cityscapes_label_paths[i] 115 | copy_image_label(foggy_image_path,foggy_cityscapes_image_path, 116 | cityscapes_label_path,foggy_cityscapes_label_path) 117 | 118 | def copy_image_label(foggy_image_path,foggy_cityscapes_image_path, 119 | cityscapes_label_path,foggy_cityscapes_label_path): 120 | """ 121 | 这是复制图像及其XML标签的函数 122 | :param foggy_image_path: foggy图像路径 123 | :param foggy_cityscapes_image_path: foggy_cityscapes数据集图像路径 124 | :param cityscapes_label_path: cityscapes数据集标签路径 125 | :param foggy_cityscapes_label_path: foggy_cityscape数据集标签路径 126 | :return: 127 | """ 128 | # 复制图像 129 | image = cv2.imread(foggy_image_path) 130 | cv2.imwrite(foggy_cityscapes_image_path,image) 131 | 132 | # 修改XML文件名称与路径，并复制XML文件到指定路径 133 | dir,image_name = os.path.split(foggy_cityscapes_image_path) 134 | in_file = open(cityscapes_label_path) 135 | tree = ET.parse(in_file) 136 | root = tree.getroot() 137 | root.find("filename").text = image_name 138 | root.find("path").text = foggy_cityscapes_image_path 139 | tree.write(foggy_cityscapes_label_path) 140 | 141 | def print_error(value): 142 | """ 143 | 定义错误回调函数 144 | :param value: 145 | :return: 146 | """ 147 | print("error: ", value) 148 | 149 | def is_contain_object(image_name,foggy_cityscapes_label_paths): 150 | """ 151 | 这是Foggy Cityscapes数据集中一张图像是否存在目标的函数 152 | :param image_name: 图像名称 153 | :param foggy_cityscapes_label_paths: Foggy Cityscapes数据集图像路径数组 154 | :return: 155 | """ 156 | flag = False 157 | for foggy_image_path in foggy_cityscapes_label_paths: 158 | if image_name in foggy_image_path: 159 | flag = True 160 | break 161 | return flag 162 | 163 | def run_main(): 164 | """ 165 | 这是主函数 166 | """ 167 | # Cityscapes --> Foggy_Cityscapes_beta_0.005 168 | print("Cityscapes --> Foggy_Cityscapes_beta=0.005 Start") 169 | beta = 0.005 170 | cityscapes_dataset_dir = os.path.abspath("../dataset/Cityscapes") 171 | foggy_cityscapes_dataset_dir = os.path.abspath("../dataset/Foggy_Cityscapes_beta_{}".format(beta)) 172 | foggy_image_dir = os.path.abspath("../object_detection_dataset/cityscapes_foggy/leftImg8bit_trainvaltest_foggy/leftImg8bit_foggy") 173 | cityscapes2foggy_cityscapes(cityscapes_dataset_dir,foggy_cityscapes_dataset_dir,foggy_image_dir,beta) 174 | print("Cityscapes --> Foggy_Cityscapes_beta_0.005 Finish") 175 | 176 | # Cityscapes --> Foggy_Cityscapes_beta=0.01 177 | print("Cityscapes --> Foggy_Cityscapes_beta_0.01 Start") 178 | beta = 0.01 179 | cityscapes_dataset_dir = os.path.abspath("../dataset/Cityscapes") 180 | foggy_cityscapes_dataset_dir = os.path.abspath("../dataset/Foggy_Cityscapes_beta_{}".format(beta)) 181 | foggy_image_dir = os.path.abspath("../object_detection_dataset/cityscapes_foggy/leftImg8bit_trainvaltest_foggy/leftImg8bit_foggy") 182 | cityscapes2foggy_cityscapes(cityscapes_dataset_dir,foggy_cityscapes_dataset_dir,foggy_image_dir,beta) 183 | print("Cityscapes --> Foggy_Cityscapes_beta_0.01 Finish") 184 | 185 | # Cityscapes --> Foggy_Cityscapes_beta=0.02 186 | print("Cityscapes --> Foggy_Cityscapes_beta_0.02 Start") 187 | beta = 0.02 188 | cityscapes_dataset_dir = os.path.abspath("../dataset/Cityscapes") 189 | foggy_cityscapes_dataset_dir = os.path.abspath("../dataset/Foggy_Cityscapes_beta_{}".format(beta)) 190 | foggy_image_dir = os.path.abspath("../object_detection_dataset/cityscapes_foggy/leftImg8bit_trainvaltest_foggy/leftImg8bit_foggy") 191 | cityscapes2foggy_cityscapes(cityscapes_dataset_dir,foggy_cityscapes_dataset_dir,foggy_image_dir,beta) 192 | print("Cityscapes --> Foggy_Cityscapes_beta_0.02 Finish") 193 | 194 | if __name__ == '__main__': 195 | run_main() -------------------------------------------------------------------------------- /coco2voc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/8/7 下午12:52 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : coco2voc.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将COCO数据集转换为VOC数据集格式的脚本。 10 | 11 | """ 12 | 13 | import os 14 | import cv2 15 | import json 16 | import numpy as np 17 | from tqdm import tqdm 18 | from multiprocessing import Pool 19 | from multiprocessing import cpu_count 20 | from pascal_voc_writer import Writer 21 | 22 | def coco2voc(coco_dataset_dir,voc_dataset_dir,year=2017,choices=["train","val"]): 23 | """ 24 | 这是将COCO数据集转换为VOC数据集格式的函数 25 | Args: 26 | coco_dataset_dir: coco数据集目录路径 27 | voc_dataset_dir: voc数据集目录路径 28 | year: 年份，默认为2017 29 | choice: 子集列表，默认为[‘train’，‘val’] 30 | Returns: 31 | """ 32 | # 初始化coco数据集相关路径 33 | coco_image_json_paths = [] 34 | for choice in choices: 35 | _coco_image_dir = os.path.join(coco_dataset_dir, '{0}{1}'.format(choice,year)) 36 | _coco_json_path = os.path.join(coco_dataset_dir, 'annotations', '{0}{1}.json'.format(choice,year)) 37 | if not os.path.exists(_coco_image_dir): 38 | _coco_image_dir = os.path.join(coco_dataset_dir, choice) 39 | _coco_json_path = os.path.join(coco_dataset_dir, 'annotations', '{0}.json'.format(choice)) 40 | coco_image_json_paths.append((_coco_image_dir,_coco_json_path)) 41 | 42 | # 初始化voc数据集相关路径 43 | voc_image_dir = os.path.join(voc_dataset_dir, 'JPEGImages') 44 | voc_annotation_dir = os.path.join(voc_dataset_dir, "Annotations") 45 | voc_imagesets_main_dir = os.path.join(voc_dataset_dir, "ImageSets", "Main") 46 | _,voc_dataset_name = os.path.split(voc_dataset_dir) 47 | if not os.path.exists(voc_image_dir): 48 | os.makedirs(voc_image_dir) 49 | if not os.path.exists(voc_annotation_dir): 50 | os.makedirs(voc_annotation_dir) 51 | if not os.path.exists(voc_imagesets_main_dir): 52 | os.makedirs(voc_imagesets_main_dir) 53 | 54 | # 遍历所有子集 55 | coco_id_category_dict = {} 56 | coco_image_id_dict = {} 57 | coco_image_gt_dict = {} 58 | coco_voc_image_dict = {} 59 | cnt = 0 60 | for choice,coco_image_json_path in zip(choices,coco_image_json_paths): 61 | print("解析{}子集".format(choice)) 62 | coco_image_dir,coco_json_path = coco_image_json_path 63 | voc_txt_path = os.path.join(voc_imagesets_main_dir,choice+".txt") 64 | # 读取COCO JSON标签文件 65 | with open(coco_json_path, "r", encoding='utf-8') as f: 66 | with open(voc_txt_path,'w', encoding='utf-8') as g: 67 | json_data = json.load(f) 68 | image_infos = json_data['images'] 69 | gts = json_data['annotations'] 70 | category_id_dict_list = json_data['categories'] 71 | # 初始化目标名称及其对应id关系 72 | for _dict in category_id_dict_list: 73 | coco_id_category_dict[_dict['id']] = _dict["name"] 74 | # 初始化coco图像id与coco图像路径和voc图像及其标签路径之间的关系 75 | for image_info in tqdm(image_infos): 76 | # 初始化coco图像领、voc图像路径和voc标签路径 77 | coco_image_path = os.path.join(coco_image_dir, image_info['file_name']) 78 | voc_image_path = os.path.join(voc_image_dir, "{0}_{1:07d}.jpg".format(voc_dataset_name,cnt)) 79 | voc_annotation_path = os.path.join(voc_annotation_dir, "{0}_{1:07d}.xml".format(voc_dataset_name, cnt)) 80 | # is_contain_object = False 81 | # for gt in gts: 82 | # if image_info['id'] == gt['image_id']: 83 | # if len(gt['bbox']) != 0: 84 | # is_contain_object = True 85 | # break 86 | # if is_contain_object: 87 | # # 将VOC图像名称写入子集txt文件中 88 | # g.write("{0}_{1:07d}\n".format(voc_dataset_name,cnt)) 89 | # coco_image_id_dict[image_info['id']] = coco_image_path 90 | # coco_voc_image_dict[coco_image_path] = (voc_image_path,voc_annotation_path) 91 | # 将VOC图像名称写入子集txt文件中 92 | g.write("{0}_{1:07d}\n".format(voc_dataset_name,cnt)) 93 | coco_image_id_dict[image_info['id']] = coco_image_path 94 | coco_voc_image_dict[coco_image_path] = (voc_image_path, voc_annotation_path) 95 | cnt += 1 96 | for gt in gts: 97 | image_id = gt['image_id'] 98 | x1, y1, w, h = gt['bbox'] 99 | cls_id = gt['category_id'] 100 | image_path = coco_image_id_dict[image_id] 101 | if image_path not in coco_image_gt_dict.keys(): 102 | coco_image_gt_dict[image_path] = [[coco_id_category_dict[cls_id], x1, y1, w, h]] 103 | else: 104 | coco_image_gt_dict[image_path].append([coco_id_category_dict[cls_id], x1, y1, w, h]) 105 | 106 | # 初始化COCO与VOC数据集相关文件路径 107 | coco_image_paths = [] 108 | coco_labels = [] 109 | voc_image_paths = [] 110 | voc_annotation_paths = [] 111 | for coco_image_path,coco_label_ in coco_image_gt_dict.items(): 112 | voc_image_path,voc_annotation_path = coco_voc_image_dict[coco_image_path] 113 | coco_image_paths.append(coco_image_path) 114 | coco_labels.append({"gt":coco_label_}) 115 | voc_image_paths.append(voc_image_path) 116 | voc_annotation_paths.append(voc_annotation_path) 117 | coco_image_paths = np.array(coco_image_paths) 118 | coco_labels = np.array(coco_labels) 119 | voc_image_paths = np.array(voc_image_paths) 120 | voc_annotation_paths = np.array(voc_annotation_paths) 121 | print(len(coco_image_paths),len(voc_image_paths)) 122 | print(len(coco_labels),len(voc_annotation_paths)) 123 | 124 | print("开始多线程处理COCO图像及其标签并转换VOC格式") 125 | size = len(coco_image_paths) 126 | if size // cpu_count() != 0: 127 | num_threads = cpu_count() 128 | elif size // (cpu_count() // 2) != 0: 129 | num_threads = cpu_count() // 2 130 | elif size // (cpu_count() // 4) != 0: 131 | num_threads = cpu_count() // 4 132 | else: 133 | num_threads = 1 134 | batch_size = size // num_threads 135 | pool = Pool(processes=num_threads) 136 | for start in np.arange(0, size, batch_size): 137 | end = int(np.min([start + batch_size, size])) 138 | batch_coco_image_paths = coco_image_paths[start:end] 139 | batch_coco_labels = coco_labels[start:end] 140 | batch_voc_image_paths = voc_image_paths[start:end] 141 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 142 | pool.apply_async(batch_images_labels_process, error_callback=print_error, 143 | args=(batch_coco_image_paths, batch_coco_labels, 144 | batch_voc_image_paths,batch_voc_annotation_paths)) 145 | pool.close() 146 | pool.join() 147 | print("结束多线程处理") 148 | 149 | def print_error(value): 150 | """ 151 | 定义错误回调函数 152 | Args: 153 | value: 出错误值 154 | Returns: 155 | """ 156 | print("error: ", value) 157 | 158 | def single_image_label_process(coco_image_path, coco_label,voc_image_path,voc_annotation_path): 159 | """ 160 | 这是对单张coco图像及其标签进行处理转换为VOC格式的函数 161 | Args: 162 | coco_image_path: coco图像路径 163 | coco_label: coco标签 164 | voc_image_path: voc图像路径 165 | voc_annotation_path: voc标签路径 166 | Returns: 167 | """ 168 | # 复制图像 169 | image = cv2.imread(coco_image_path) 170 | h,w,_ = np.shape(image) 171 | cv2.imwrite(voc_image_path, image) 172 | 173 | # 将目标框标签写入xml 174 | writer = Writer(voc_image_path,w,h) 175 | for bbox_label in coco_label['gt']: 176 | cls_name, x1, y1, w, h = bbox_label 177 | x2 = x1 + w 178 | y2 = y1 + h 179 | x1 = int(round(x1)) 180 | y1 = int(round(y1)) 181 | x2 = int(round(x2)) 182 | y2 = int(round(y2)) 183 | writer.addObject(cls_name,x1,y1,x2,y2) 184 | writer.save(voc_annotation_path) 185 | 186 | def batch_images_labels_process(batch_coco_image_paths, batch_coco_labels, 187 | batch_voc_image_paths,batch_voc_annotation_paths): 188 | """ 189 | 这是对批量coco图像及其标签进行处理转换为VOC格式的函数 190 | Args: 191 | batch_coco_image_paths: 批量coco图像路径数组 192 | batch_coco_labels: 批量coco标签数组 193 | batch_voc_image_paths: 批量voc图像路径数组 194 | batch_voc_annotation_paths: 批量voc标签路径数组 195 | Returns: 196 | """ 197 | size = len(batch_coco_image_paths) 198 | for i in tqdm(np.arange(size)): 199 | coco_image_path = batch_coco_image_paths[i] 200 | coco_label = batch_coco_labels[i] 201 | voc_image_path = batch_voc_image_paths[i] 202 | voc_annotation_path = batch_voc_annotation_paths[i] 203 | single_image_label_process(coco_image_path, coco_label, voc_image_path,voc_annotation_path) 204 | 205 | def run_main(): 206 | """ 207 | 这是主函数 208 | """ 209 | # COCO --> VOC 210 | coco_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/origin/COCO2017") 211 | voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/coco2017") 212 | year = 2017 213 | choices = ['train','val'] 214 | coco2voc(coco_dataset_dir,voc_dataset_dir,year,choices) 215 | 216 | # 217 | # # VOC2007_coco --> VOC2007 218 | # coco_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2007_coco") 219 | # voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2007") 220 | # choices = ['train', 'val','test'] 221 | # coco2voc(coco_dataset_dir,voc_dataset_dir,choices=choices) 222 | # 223 | # # VOC2012_coco --> VOC2012 224 | # coco_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2012_coco") 225 | # voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2012") 226 | # choices = ['train', 'val'] 227 | # coco2voc(coco_dataset_dir,voc_dataset_dir,choices=choices) 228 | 229 | if __name__ == '__main__': 230 | run_main() 231 | -------------------------------------------------------------------------------- /coco_names.txt: -------------------------------------------------------------------------------- 1 | person 2 | bicycle 3 | car 4 | motorcycle 5 | airplane 6 | bus 7 | train 8 | truck 9 | boat 10 | traffic light 11 | fire hydrant 12 | stop sign 13 | parking meter 14 | bench 15 | bird 16 | cat 17 | dog 18 | horse 19 | sheep 20 | cow 21 | elephant 22 | bear 23 | zebra 24 | giraffe 25 | backpack 26 | umbrella 27 | handbag 28 | tie 29 | suitcase 30 | frisbee 31 | skis 32 | snowboard 33 | sports ball 34 | kite 35 | baseball bat 36 | baseball glove 37 | skateboard 38 | surfboard 39 | tennis racket 40 | bottle 41 | wine glass 42 | cup 43 | fork 44 | knife 45 | spoon 46 | bowl 47 | banana 48 | apple 49 | sandwich 50 | orange 51 | broccoli 52 | carrot 53 | hot dog 54 | pizza 55 | donut 56 | cake 57 | chair 58 | couch 59 | potted plant 60 | bed 61 | dining table 62 | toilet 63 | tv 64 | laptop 65 | mouse 66 | remote 67 | keyboard 68 | cell phone 69 | microwave 70 | oven 71 | toaster 72 | sink 73 | refrigerator 74 | book 75 | clock 76 | vase 77 | scissors 78 | teddy bear 79 | hair drier 80 | toothbrush 81 | -------------------------------------------------------------------------------- /copy_voc_dataset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/8/5 下午1:52 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : copy_voc_dataset.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是根据指定目标类别复制VOC数据集的脚本,包含对图像及其xml重命名功能 10 | """ 11 | 12 | import os 13 | import shutil 14 | import numpy as np 15 | import xml.etree.ElementTree as ET 16 | 17 | from tqdm import tqdm 18 | from multiprocessing import Pool 19 | from multiprocessing import cpu_count 20 | from pascal_voc_writer import Writer 21 | 22 | def copy_voc_dataset(voc_dataset_dir,new_voc_dataset_dir,class_names=None): 23 | """ 24 | 这是根据指定目标类别复制VOC数据集的函数 25 | Args: 26 | voc_dataset_dir: voc数据集地址 27 | new_voc_dataset_dir: 新voc数据集地址 28 | class_names: 目标类别数组,默认为None 29 | Returns: 30 | """ 31 | # 初始化voc数据集相关路径 32 | voc_image_dir = os.path.join(voc_dataset_dir, 'JPEGImages') 33 | if not os.path.exists(voc_image_dir): 34 | voc_image_dir = os.path.join(voc_dataset_dir, 'images') 35 | voc_annotation_dir = os.path.join(voc_dataset_dir, "Annotations") 36 | new_voc_image_dir = os.path.join(new_voc_dataset_dir, 'JPEGImages') 37 | new_voc_annotation_dir = os.path.join(new_voc_dataset_dir, "Annotations") 38 | new_voc_imagesets_main_dir = os.path.join(new_voc_dataset_dir, "ImageSets", "Main") 39 | _,new_voc_dataset_name = os.path.split(new_voc_dataset_dir) 40 | if not os.path.exists(new_voc_image_dir): 41 | os.makedirs(new_voc_image_dir) 42 | if not os.path.exists(new_voc_annotation_dir): 43 | os.makedirs(new_voc_annotation_dir) 44 | if not os.path.exists(new_voc_imagesets_main_dir): 45 | os.makedirs(new_voc_imagesets_main_dir) 46 | 47 | # 筛选包含指定目标的VOC数据集的图像和xml文件路径 48 | voc_image_paths = [] 49 | voc_annotation_paths = [] 50 | new_voc_image_paths = [] 51 | new_voc_annotation_paths = [] 52 | cnt = 0 53 | for image_name in os.listdir(voc_image_dir): 54 | name,ext = os.path.splitext(image_name) 55 | image_path = os.path.join(voc_image_dir,image_name) 56 | xml_path = os.path.join(voc_annotation_dir,name+".xml") 57 | if is_contain_classes(xml_path,class_names): 58 | new_name = "{0}_{1:06d}".format(new_voc_dataset_name, cnt) 59 | new_image_path = os.path.join(new_voc_image_dir,new_name+".jpg") 60 | new_xml_path = os.path.join(new_voc_annotation_dir, new_name+".xml") 61 | voc_image_paths.append(image_path) 62 | voc_annotation_paths.append(xml_path) 63 | new_voc_image_paths.append(new_image_path) 64 | new_voc_annotation_paths.append(new_xml_path) 65 | cnt += 1 66 | voc_image_paths = np.array(voc_image_paths) 67 | voc_annotation_paths = np.array(voc_annotation_paths) 68 | new_voc_image_paths = np.array(new_voc_image_paths) 69 | new_voc_annotation_paths = np.array(new_voc_annotation_paths) 70 | 71 | # 多线程处理复制图像和xml文件 72 | size = len(voc_annotation_paths) 73 | if size // cpu_count() != 0: 74 | num_threads = cpu_count() 75 | elif size // (cpu_count() // 2) != 0: 76 | num_threads = cpu_count() // 2 77 | elif size // (cpu_count() // 4) != 0: 78 | num_threads = cpu_count() // 4 79 | else: 80 | num_threads = 1 81 | batch_size = size // num_threads 82 | pool = Pool(processes=num_threads) 83 | for start in np.arange(0,size,batch_size): 84 | end = int(np.min([start+batch_size,size])) 85 | batch_voc_image_paths = voc_image_paths[start:end] 86 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 87 | batch_new_voc_image_paths = new_voc_image_paths[start:end] 88 | batch_new_voc_annotation_paths = new_voc_annotation_paths[start:end] 89 | pool.apply_async(batch_process_image_xml,error_callback=print_error, 90 | args=(batch_voc_image_paths,batch_voc_annotation_paths, 91 | batch_new_voc_image_paths,batch_new_voc_annotation_paths,class_names)) 92 | pool.close() 93 | pool.join() 94 | 95 | def is_contain_classes(xml_path,class_names=None): 96 | """ 97 | 这是判断xml文件中是否包含指定名称标签的函数 98 | Args: 99 | xml_path: xml文件路径 100 | class_names: 目标分类数组，默认为None 101 | Returns: 102 | """ 103 | if class_names is None: 104 | flag = True 105 | else: 106 | flag = False 107 | if os.path.exists(xml_path): 108 | # 解析XML文件 109 | in_file = open(xml_path) 110 | tree = ET.parse(in_file) 111 | root = tree.getroot() 112 | 113 | # 遍历所有目标结点,判断是否存在指定目标 114 | for obj in root.findall('object'): 115 | cls_name = obj.find('name').text 116 | if class_names is not None: 117 | if cls_name in class_names: 118 | flag = True 119 | break 120 | else: 121 | continue 122 | 123 | return flag 124 | 125 | def print_error(value): 126 | """ 127 | 定义错误回调函数 128 | Args: 129 | value: 出错误值 130 | Returns: 131 | """ 132 | print("error: ", value) 133 | 134 | def parse_xml(xml_path,class_names=None): 135 | """ 136 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 137 | Args: 138 | xml_path: XML标签文件路径 139 | class_names: 目标名称数组，默认为None 140 | Returns: 141 | """ 142 | # 获取XML文件的根结点 143 | root = ET.parse(xml_path).getroot() 144 | h = int(root.find("size").find("height").text) 145 | w = int(root.find("size").find("width").text) 146 | # 遍历所有目标 147 | objects = [] 148 | for obj in root.findall('object'): 149 | obj_name = obj.find('name').text 150 | bndbox = obj.find('bndbox') 151 | xmin = bndbox.find('xmin').text 152 | ymin = bndbox.find('ymin').text 153 | xmax = bndbox.find('xmax').text 154 | ymax = bndbox.find('ymax').text 155 | if class_names is None: 156 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 157 | else: 158 | if obj_name in class_names: 159 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 160 | return objects,(h,w) 161 | 162 | def process_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path,class_names): 163 | """ 164 | 这是复制VOC图像及其xml文件的函数 165 | Args: 166 | voc_image_path: voc图像文件路径 167 | voc_xml_path: voc标签文件路径 168 | new_voc_image_path: voc图像文件新路径 169 | new_voc_xml_path: voc标签文件新路径 170 | class_names: 目标分类数组 171 | Returns: 172 | """ 173 | # 复制图像 174 | shutil.copy(voc_image_path,new_voc_image_path) 175 | 176 | # xml解析复制制定类别目标 177 | objects,(h,w) = parse_xml(voc_xml_path,class_names) 178 | writer = Writer(new_voc_image_path,w,h) 179 | for cls_name,x1,y1,x2,y2 in objects: 180 | writer.addObject(cls_name,x1,y1,x2,y2) 181 | writer.save(new_voc_xml_path) 182 | 183 | def batch_process_image_xml(batch_voc_image_paths,batch_voc_annotation_paths, 184 | batch_new_voc_image_paths,batch_new_voc_annotation_paths,class_names): 185 | """ 186 | 这是批量处理VOC图像及其标签函数 187 | Args: 188 | batch_voc_image_paths: 批量voc图像路径数组 189 | batch_voc_annotation_paths: 批量voc标签路径数组 190 | batch_new_voc_image_paths: 批量voc图像新路径数组 191 | batch_new_voc_annotation_paths: 批量voc标签新路径数组 192 | class_names: 目标类别数组 193 | Returns: 194 | """ 195 | size = len(batch_voc_image_paths) 196 | for i in tqdm(np.arange(size)): 197 | voc_image_path = batch_voc_image_paths[i] 198 | voc_xml_path = batch_voc_annotation_paths[i] 199 | new_voc_image_path = batch_new_voc_image_paths[i] 200 | new_voc_xml_path = batch_new_voc_annotation_paths[i] 201 | process_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path,class_names) 202 | 203 | def run_main(): 204 | """ 205 | 这是主函数 206 | """ 207 | # VOC2007 --> VOT_VOC2007 208 | voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2007") 209 | new_voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/vot_voc2007") 210 | class_names = ['person','horse','cat','cow','bird','train','sleep',"bus","car","boat","aeroplane"] 211 | copy_voc_dataset(voc_dataset_dir, new_voc_dataset_dir, class_names,) 212 | 213 | # VOC2012 --> VOT_VOC2012 214 | voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2012") 215 | new_voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/vot_voc2012") 216 | class_names = ['person','horse','cat','cow','bird','train','sleep',"bus","car","boat","aeroplane"] 217 | copy_voc_dataset(voc_dataset_dir, new_voc_dataset_dir, class_names,) 218 | 219 | # COCO2017 --> VOT_COCO2017 220 | voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/coco2017") 221 | new_voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/vot_coco2017") 222 | class_names = ['person',"airplane","bus","train","truck","boat","bird","cat","dog","horse","sheep","cow", 223 | "elephant","bear","zebra","giraffe","fork",] 224 | copy_voc_dataset(voc_dataset_dir, new_voc_dataset_dir, class_names) 225 | 226 | 227 | if __name__ == '__main__': 228 | run_main() 229 | -------------------------------------------------------------------------------- /delete_voc_dataset_object.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/5/23 12:50 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : delete_voc_dataset_object.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是删除VOC数据集中制定目标名称标签的脚本 10 | """ 11 | 12 | import os 13 | import numpy as np 14 | import xml.etree.ElementTree as ET 15 | 16 | from tqdm import tqdm 17 | from multiprocessing import Pool 18 | from multiprocessing import cpu_count 19 | from pascal_voc_writer import Writer 20 | 21 | def delete_voc_dataset_object(voc_dataset_dir,class_names,is_save_class_names=True): 22 | """ 23 | 这是删除VOC数据集中指定目标分类的函数 24 | Args: 25 | voc_dataset_dir: VOC数据集文件夹路径 26 | class_names: 目标名称列表 27 | is_class_names: 是否为需要保留下来目标名称列表标志位，默认为True 28 | Returns: 29 | """ 30 | # 初始化XML标签文件数组 31 | voc_image_dir = os.path.join(voc_dataset_dir, 'JPEGImages') 32 | if not os.path.exists(voc_image_dir): 33 | voc_image_dir = os.path.join(voc_dataset_dir, 'images') 34 | voc_annotation_dir = os.path.join(voc_dataset_dir, "Annotations") 35 | voc_image_paths = [] 36 | voc_annotation_paths = [] 37 | for image_name in os.listdir(voc_image_dir): 38 | fname,ext = os.path.split(image_name) 39 | voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 40 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,fname+".xml")) 41 | voc_image_paths = np.array(voc_image_paths) 42 | voc_annotation_paths = np.array(voc_annotation_paths) 43 | 44 | # 多线程删除目标名称 45 | size = len(voc_annotation_paths) 46 | if size // cpu_count() != 0: 47 | num_threads = cpu_count() 48 | elif size // (cpu_count() // 2) != 0: 49 | num_threads = cpu_count() // 2 50 | elif size // (cpu_count() // 4) != 0: 51 | num_threads = cpu_count() // 4 52 | else: 53 | num_threads = 1 54 | batch_size = size // num_threads 55 | pool = Pool(processes=num_threads) 56 | for start in np.arange(0,size,batch_size): 57 | end = int(np.min([start+batch_size,size])) 58 | batch_voc_image_paths = voc_image_paths[start:end] 59 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 60 | pool.apply_async(batch_delete_voc_object,callback=print_error, 61 | args=(batch_voc_image_paths,batch_voc_annotation_paths,class_names,is_save_class_names)) 62 | pool.close() 63 | pool.join() 64 | 65 | def parse_xml(xml_path,class_names,is_save_class_names=True): 66 | """ 67 | 这是解析VOC数据集XML标签文件，获取需要保留目标的检测框数组 68 | Args: 69 | xml_path: voc数据集XML文件路径 70 | class_names: 目标名称列表 71 | is_class_names: 是否为需要保留下来目标名称列表标志位，默认为True 72 | Returns: 73 | """ 74 | # 获取XML文件的根结点 75 | root = ET.parse(xml_path).getroot() 76 | h = int(root.find("size").find("height").text) 77 | w = int(root.find("size").find("width").text) 78 | # 遍历所有目标 79 | objects = [] 80 | for obj in root.findall('object'): 81 | obj_name = obj.find('name').text 82 | bndbox = obj.find('bndbox') 83 | xmin = bndbox.find('xmin').text 84 | ymin = bndbox.find('ymin').text 85 | xmax = bndbox.find('xmax').text 86 | ymax = bndbox.find('ymax').text 87 | if is_save_class_names: 88 | if obj_name in class_names: 89 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 90 | else: 91 | if obj_name not in class_names: 92 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 93 | return objects,(h,w) 94 | 95 | def delete_voc_object(voc_image_path,voc_annotation_path,class_names,is_save_class_names=True): 96 | """ 97 | 这是删除单张VOC图像中指定名称目标标签的函数 98 | Args: 99 | voc_image_path: VOC数据集图像文件路径 100 | voc_annotation_path: VOC数据集XML文件路径 101 | class_names: 目标名称列表 102 | is_class_names: 是否为需要保留下来目标名称列表标志位，默认为True 103 | Returns: 104 | """ 105 | # 解析需要保留目标 106 | save_objects,(h,w) = parse_xml(voc_annotation_path,class_names,is_save_class_names) 107 | # 重新写入xml文件 108 | writer = Writer(voc_image_path,w,h) 109 | for cls_name,x1,y1,x2,y2 in save_objects: 110 | writer.addObject(cls_name,x1,y1,x2,y2) 111 | writer.save(voc_annotation_path) 112 | 113 | def batch_delete_voc_object(batch_voc_image_paths,batch_voc_annotation_paths,class_names,is_save_class_names=True): 114 | """ 115 | 这是删除批量VOC数据集图像中指定名称目标标签的函数 116 | Args: 117 | batch_voc_image_paths: 批量VOC图像文件路径数组 118 | batch_voc_annotation_paths: 批量VOC数据集XML文件路径数组 119 | class_names: 目标名称列表 120 | is_class_names: 是否为需要保留下来目标名称列表标志位，默认为True 121 | Returns: 122 | """ 123 | # 遍历所有XML标签文件，重命名目标标签 124 | size = len(batch_voc_annotation_paths) 125 | for i in tqdm(np.arange(size)): 126 | delete_voc_object(batch_voc_image_paths[i],batch_voc_annotation_paths[i],class_names,is_save_class_names) 127 | 128 | def print_error(value): 129 | """ 130 | 定义错误回调函数 131 | Args: 132 | value: 出错误值 133 | Returns: 134 | """ 135 | print("error: ", value) 136 | 137 | def run_main(): 138 | """ 139 | 这是主函数 140 | """ 141 | # BDD100k 142 | voc_dataset_dir = os.path.abspath("../dataset/BDD100k") 143 | save_object_names = ['person','rider'] 144 | delete_voc_dataset_object(voc_dataset_dir,save_object_names) 145 | 146 | # Cityscapes 147 | voc_dataset_dir = os.path.abspath("../dataset/Cityscapes") 148 | save_object_names = ['person','rider'] 149 | delete_voc_dataset_object(voc_dataset_dir,save_object_names) 150 | 151 | # Foggy_Cityscapes_beta=0.005 152 | voc_dataset_dir = os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.01") 153 | save_object_names = ['person','rider'] 154 | delete_voc_dataset_object(voc_dataset_dir,save_object_names) 155 | 156 | # Foggy_Cityscapes_beta=0.01 157 | voc_dataset_dir = os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.02") 158 | save_object_names = ['person','rider'] 159 | delete_voc_dataset_object(voc_dataset_dir,save_object_names) 160 | 161 | # Foggy_Cityscapes_beta=0.02 162 | voc_dataset_dir = os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.005") 163 | save_object_names = ['person','rider'] 164 | delete_voc_dataset_object(voc_dataset_dir,save_object_names) 165 | 166 | # KITTI 167 | voc_dataset_dir = os.path.abspath("../dataset/KITTI") 168 | save_object_names = ['person','rider'] 169 | delete_voc_dataset_object(voc_dataset_dir,save_object_names) 170 | 171 | if __name__ == '__main__': 172 | run_main() 173 | -------------------------------------------------------------------------------- /get_voc_classes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/9/1 17:38 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : get_voc_classes.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是利用多线程获取VOC数据集目标分类名称数组的函数 10 | """ 11 | 12 | import os 13 | import numpy as np 14 | import xml.etree.ElementTree as ET 15 | 16 | from tqdm import tqdm 17 | from multiprocessing import Pool 18 | from multiprocessing import cpu_count 19 | 20 | def get_voc_classes(voc_dataset_dir,class_names_txt_path): 21 | """ 22 | 这是获取VOC数据集的目标分类数组的函数 23 | Args: 24 | voc_dataset_dir: VOC数据集目录 25 | class_names_txt_path: 目标类别txt文件路径 26 | Returns: 27 | """ 28 | # 初始化VOC数据集XML标签文件路径 29 | annotation_paths = [] 30 | annotation_dir = os.path.join(voc_dataset_dir, "Annotations") 31 | for annotation_name in os.listdir(annotation_dir): 32 | annotation_paths.append(os.path.join(annotation_dir, annotation_name)) 33 | annotation_paths = np.array(annotation_paths) 34 | 35 | # 利用异步多线程获取每个小批量数据集包含的目标分类名称 36 | size = len(annotation_paths) 37 | batch_size = size // (cpu_count()-1) 38 | pool = Pool(processes=cpu_count()-1) 39 | results = [] 40 | for start in np.arange(0,size,batch_size): 41 | end = int(np.min([start+batch_size,size])) 42 | batch_annotation_paths = annotation_paths[start:end] 43 | results.append(pool.apply_async(get_batch_annotation_classes,error_callback=print_error, 44 | args=(batch_annotation_paths,))) 45 | pool.close() 46 | pool.join() 47 | 48 | # 合并每个小批量数据集所包含的批量数据名称 49 | classes_set = set() 50 | for result in results: 51 | classes_set = classes_set.union(result.get()) 52 | classes = list(classes_set) 53 | 54 | # 将目标类别写入txt 55 | with open(class_names_txt_path,'w' ,encoding='utf-8') as f: 56 | for class_name in classes: 57 | f.write("{}\n".format(class_name)) 58 | 59 | return classes 60 | 61 | def parse_xml_classes(xml_path): 62 | """ 63 | 这是解析XML文件中包含所有目标分类的函数 64 | Args: 65 | xml_path: XML文件路径 66 | Returns: 67 | """ 68 | tree = ET.parse(xml_path) 69 | objects = [] 70 | for obj in tree.findall('object'): 71 | name = obj.find('name').text 72 | objects.append(name) 73 | return objects 74 | 75 | def get_batch_annotation_classes(batch_annotation_paths): 76 | """ 77 | 这是获取小批量XML标签文件中目标分类名称的函数 78 | Args: 79 | batch_annotation_paths: 小批量标注文件路径数组 80 | Returns: 81 | """ 82 | classes_set = set() 83 | for i in tqdm(np.arange(len(batch_annotation_paths))): 84 | # 获取每个XML文件中所包含的目标分类名称 85 | classes_names = parse_xml_classes(batch_annotation_paths[i]) 86 | # 更新目标分类集合 87 | for cls in classes_names: 88 | classes_set.add(cls) 89 | return classes_set 90 | 91 | def print_error(value): 92 | """ 93 | 定义错误回调函数 94 | Args: 95 | value: 96 | Returns: 97 | """ 98 | print("error: ", value) 99 | 100 | def run_main(): 101 | """ 102 | 这是主函数 103 | """ 104 | dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/origin/VOC2007") 105 | class_names_txt_path = "./voc_names.txt" 106 | classes_set = get_voc_classes(dataset_dir,class_names_txt_path) 107 | for class_name in classes_set: 108 | print(class_name) 109 | 110 | if __name__ == '__main__': 111 | run_main() 112 | -------------------------------------------------------------------------------- /kitti2voc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2021/8/7 下午1:01 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : kitti2voc.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是KITTI数据集转化为VOC数据集格式的脚本，根据开发需要自行修改 10 | kitti_dataset_dir、voc_dataset_dir、train_ratio、class_names即可。 11 | 12 | 其中： 13 | - kitti_dataset_dir为原始KITTI数据集目录路径； 14 | - voc_dataset_dir为VOC数据集格式的KITTI数据集目录路径； 15 | - train_ratio为训练集比例，默认为0.8； 16 | - class_names为目标名称数组，默认为['Person_sitting',"Pedestrian",'Cyclist',"Truck","Car","Tram","Van"]； 17 | """ 18 | 19 | import os 20 | import cv2 21 | import numpy as np 22 | from tqdm import tqdm 23 | from multiprocessing import Pool 24 | from multiprocessing import cpu_count 25 | from pascal_voc_writer import Writer 26 | 27 | def kitti2voc(kitti_dataset_dir,voc_dataset_dir,train_ratio=0.8, 28 | class_names=['Person_sitting',"Pedestrian",'Cyclist',"Truck","Car","Tram","Van"]): 29 | """ 30 | 这是将KITTI数据集转化为VOC数据集的函数 31 | :param kitti_dataset_dir: kitti数据集目录路径 32 | :param voc_dataset_dir: voc数据集目录路径 33 | :param train_ratio: 训练集比例，默认为0.8 34 | :param class_names: 目标名称数组，默认为[] 35 | :return: 36 | """ 37 | # 初始化kitti数据集相关路径 38 | kitti_image_dir = os.path.join(kitti_dataset_dir,'training','image_2') 39 | kitti_label_dir = os.path.join(kitti_dataset_dir,'training','label_2') 40 | #print(kitti_image_dir,kitti_label_dir) 41 | 42 | # 初始化voc数据集相关路径 43 | voc_image_dir = os.path.join(voc_dataset_dir, 'JPEGImages') 44 | voc_annotation_dir = os.path.join(voc_dataset_dir, "Annotations") 45 | voc_imagesets_main_dir = os.path.join(voc_dataset_dir, "ImageSets", "Main") 46 | if not os.path.exists(voc_image_dir): 47 | os.makedirs(voc_image_dir) 48 | if not os.path.exists(voc_annotation_dir): 49 | os.makedirs(voc_annotation_dir) 50 | if not os.path.exists(voc_imagesets_main_dir): 51 | os.makedirs(voc_imagesets_main_dir) 52 | 53 | # 初始化图像和标签文件路径 54 | kitti_image_paths = [] 55 | kitti_label_paths = [] 56 | voc_image_paths = [] 57 | voc_annotation_paths = [] 58 | for txt_name in os.listdir(kitti_label_dir): 59 | txt_label_path = os.path.join(kitti_label_dir,txt_name) 60 | #print(txt_label_path) 61 | #print(is_conitain_object(txt_label_path,class_names)) 62 | if is_conitain_object(txt_label_path,class_names): 63 | name,ext = os.path.splitext(txt_name) 64 | kitti_image_paths.append(os.path.join(kitti_image_dir,name+".png")) 65 | kitti_label_paths.append(os.path.join(kitti_label_dir,txt_name)) 66 | voc_image_paths.append(os.path.join(voc_image_dir,name+".jpg")) 67 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,name+".xml")) 68 | kitti_image_paths = np.array(kitti_image_paths) 69 | kitti_label_paths = np.array(kitti_label_paths) 70 | voc_image_paths = np.array(voc_image_paths) 71 | voc_annotation_paths = np.array(voc_annotation_paths) 72 | print(len(kitti_image_paths),len(kitti_label_paths),len(voc_image_paths),len(voc_annotation_paths)) 73 | 74 | # 随机打乱数据集 75 | size = len(kitti_label_paths) 76 | random_index = np.random.permutation(size) 77 | kitti_image_paths = kitti_image_paths[random_index] 78 | kitti_label_paths = kitti_label_paths[random_index] 79 | voc_image_paths = voc_image_paths[random_index] 80 | voc_annotation_paths = voc_annotation_paths[random_index] 81 | 82 | # 随机划分训练集和测试集 83 | train_size = int(size*train_ratio) 84 | train_kitti_image_paths = kitti_image_paths[0:train_size] 85 | val_kitti_image_paths = kitti_image_paths[train_size:] 86 | for choice,kitti_image_paths in [('train',train_kitti_image_paths), 87 | ('val',val_kitti_image_paths), 88 | ('trainval',kitti_image_paths)]: 89 | voc_txt_path = os.path.join(voc_imagesets_main_dir,choice+".txt") 90 | with open(voc_txt_path,'w') as f: 91 | for image_path in kitti_image_paths: 92 | dir,image_name = os.path.split(image_path) 93 | name,ext = os.path.splitext(image_name) 94 | f.write(name+"\n") 95 | # 利用多线程将KITTI数据集转换为VOC数据集格式 96 | batch_size = size // (cpu_count()-1) 97 | pool = Pool(processes=cpu_count()-1) 98 | for start in np.arange(0,size,batch_size): 99 | end = int(np.min([start+batch_size,size])) 100 | batch_kitti_image_paths = kitti_image_paths[start:end] 101 | batch_kitti_label_paths = kitti_label_paths[start:end] 102 | batch_voc_image_paths = voc_image_paths[start:end] 103 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 104 | pool.apply_async(batch_image_label_process,error_callback=print_error, 105 | args=(batch_kitti_image_paths,batch_kitti_label_paths, 106 | batch_voc_image_paths,batch_voc_annotation_paths,class_names)) 107 | pool.close() 108 | pool.join() 109 | 110 | def is_conitain_object(kitti_txt_path,class_names): 111 | """ 112 | 这是判断kitti数据集的txt标签中是否包含候选目标实例的函数 113 | :param kitti_txt_path: kitti数据集的json标签文件路径 114 | :param class_names: 目标分类数组 115 | :return: 116 | """ 117 | flag = False 118 | with open(kitti_txt_path,'r') as f: 119 | for line in f.readlines(): 120 | strs = line.strip().split(" ") 121 | if strs[0] not in class_names: 122 | continue 123 | else: 124 | flag = True 125 | break 126 | return flag 127 | 128 | def print_error(value): 129 | """ 130 | 定义错误回调函数 131 | :param value: 132 | :return: 133 | """ 134 | print("error: ", value) 135 | 136 | def single_image_label_process(kitti_image_path,kitti_txt_path,voc_image_path,voc_annotation_path,class_names): 137 | """ 138 | 这是将单张kitti图像及其txt标签转换为VOC数据集格式的函数 139 | :param kitti_image_path: kitti图像路径 140 | :param kitti_txt_path: kitti标签路径 141 | :param voc_image_path: voc图像路径 142 | :param voc_annotation_path: voc标签路径 143 | :param class_names: 目标名称数组 144 | :return: 145 | """ 146 | # 初始化VOC标签写类,复制图像 147 | image = cv2.imread(kitti_image_path) 148 | cv2.imwrite(voc_image_path,image) 149 | h, w, c = np.shape(image) 150 | dir,image_name = os.path.split(kitti_image_path) 151 | writer = Writer(image_name,w,h) 152 | 153 | # TXT标签转化为VOC标签 154 | with open(kitti_txt_path,'r') as f: # 加载txt标签 155 | for line in f.readlines(): 156 | strs = line.strip().split(" ") 157 | if strs[0] not in class_names: 158 | continue 159 | else: 160 | xmin = max(int(float(strs[4]))-1,0) 161 | ymin = max(int(float(strs[5]))-1,0) 162 | xmax = min(int(float(strs[6]))+1,w) 163 | ymax = min(int(float(strs[7]))+1,h) 164 | label = strs[0] 165 | if label in ['Person_sitting',"Pedestrian"]: 166 | label = 'person' 167 | elif label in ['Cyclist']: 168 | label = 'rider' 169 | elif label in ['Car','Van']: 170 | label = 'car' 171 | elif label in ['Truck']: 172 | label = 'truck' 173 | elif label in ['Tram']: 174 | label = 'bus' 175 | writer.addObject(label,xmin,ymin,xmax,ymax) 176 | writer.save(voc_annotation_path) 177 | 178 | def batch_image_label_process(batch_kitti_image_paths,batch_kitti_txt_paths, 179 | batch_voc_image_paths,batch_voc_annotation_paths,class_names): 180 | """ 181 | 批量处理kitti数据转化为voc数据 182 | :param batch_kitti_image_paths: 批量kitti图像路径数组 183 | :param batch_kitti_txt_paths: 批量kitti标签路径数组 184 | :param batch_voc_image_paths: 批量voc图像路径数组 185 | :param batch_voc_annotation_paths: 批量voc标签路径数组 186 | :param class_names: 目标名称数组 187 | :return: 188 | """ 189 | size = len(batch_kitti_image_paths) 190 | for i in tqdm(np.arange(size)): 191 | kitti_image_path = batch_kitti_image_paths[i] 192 | kitti_txt_path = batch_kitti_txt_paths[i] 193 | voc_image_path = batch_voc_image_paths[i] 194 | voc_annotation_path = batch_voc_annotation_paths[i] 195 | single_image_label_process(kitti_image_path,kitti_txt_path, 196 | voc_image_path,voc_annotation_path,class_names) 197 | 198 | def run_main(): 199 | """ 200 | 这是主函数 201 | """ 202 | # KITTI --> VOC 203 | print("KITTI --> VOC Start") 204 | kitti_dataset_dir = os.path.abspath("../KITTI") 205 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/KITTI") 206 | train_ratio = 0.8 207 | class_names=['Person_sitting',"Pedestrian",'Cyclist',"Truck","Car","Tram","Van"] 208 | kitti2voc(kitti_dataset_dir,voc_dataset_dir,train_ratio,class_names) 209 | print("KITTI --> VOC Start") 210 | 211 | if __name__ == '__main__': 212 | run_main() 213 | -------------------------------------------------------------------------------- /labelme2voc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2024/4/22 下午11:37 3 | # @Author : DaiPuWei 4 | # @Email : daipuwei@qq.com 5 | # @File : labelme2voc.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将Labelme数据集转换为VOC数据集的脚本 10 | """ 11 | 12 | import os 13 | import json 14 | import shutil 15 | import numpy as np 16 | from tqdm import tqdm 17 | from pascal_voc_writer import Writer 18 | from multiprocessing import Pool,cpu_count 19 | 20 | def labelme2voc(labelme_dataset_dir,voc_dataset_dir,choices=["train","val"]): 21 | """ 22 | 这是Labelme数据集转VOC数据集的函数 23 | Args: 24 | labelme_dataset_dir: labelme数据集地址 25 | voc_dataset_dir: voc数据集地址 26 | choices: 子集列表，默认为["train","val"] 27 | Returns: 28 | """ 29 | # 初始化Labelme数据集相关路径 30 | labelme_image_dir = os.path.join(labelme_dataset_dir,'images') 31 | 32 | # 初始化VOC数据集相关路径 33 | _, voc_dataset_name = os.path.split(voc_dataset_dir) 34 | voc_image_dir = os.path.join(voc_dataset_dir,"JPEGImages") 35 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 36 | voc_main_dir = os.path.join(voc_dataset_dir,"ImageSets","Main") 37 | if not os.path.exists(voc_image_dir): 38 | os.makedirs(voc_image_dir) 39 | if not os.path.exists(voc_annotation_dir): 40 | os.makedirs(voc_annotation_dir) 41 | if not os.path.exists(voc_main_dir): 42 | os.makedirs(voc_main_dir) 43 | 44 | # 初始化数据集文件路径数组 45 | labelme_image_names = [] 46 | labelme_image_paths = [] 47 | labelme_json_paths = [] 48 | voc_image_paths = [] 49 | voc_annotation_paths = [] 50 | cnt = 0 51 | for image_name in os.listdir(labelme_image_dir): 52 | # json文件略过 53 | if image_name.endswith(".json"): 54 | continue 55 | fname,ext = os.path.splitext(image_name) 56 | labelme_image_names.append(fname) 57 | labelme_image_paths.append(os.path.join(labelme_image_dir,image_name)) 58 | labelme_json_paths.append(os.path.join(labelme_image_dir, "{0}.json".format(fname))) 59 | voc_image_paths.append(os.path.join(voc_image_dir,"{0}_{1:07d}.jpg".format(voc_dataset_name, cnt))) 60 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,"{0}_{1:07d}.xml".format(voc_dataset_name, cnt))) 61 | cnt += 1 62 | voc_image_paths = np.array(voc_image_paths) 63 | voc_annotation_paths = np.array(voc_annotation_paths) 64 | labelme_image_paths = np.array(labelme_image_paths) 65 | labelme_json_paths = np.array(labelme_json_paths) 66 | 67 | # 生成voc子集列表 68 | print("开始生成VOC数据集子集列表") 69 | for choice in choices: 70 | voc_txt_path = os.path.join(voc_main_dir,"{}.txt".format(choice)) 71 | labelme_txt_path = os.path.join(labelme_dataset_dir,"{}.txt".format(choice)) 72 | with open(voc_txt_path,'w',encoding='utf-8') as g: 73 | with open(labelme_txt_path,'r',encoding='utf-8') as f: 74 | for line in tqdm(f.readlines()): 75 | labelme_fname = line.strip() 76 | if labelme_fname in labelme_image_names: 77 | index = labelme_image_names.index(labelme_fname) 78 | _,voc_image_name = os.path.split(voc_image_paths[index]) 79 | voc_fname,_ = os.path.splitext(voc_image_name) 80 | g.write(voc_fname+"\n") 81 | print("结束生成VOC数据集子集列表") 82 | 83 | print("开始多线程处理Labelme数据集图像及其标签并转换为VOC格式") 84 | size = len(labelme_image_paths) 85 | if size // cpu_count() != 0: 86 | num_threads = cpu_count() 87 | elif size // (cpu_count() // 2) != 0: 88 | num_threads = cpu_count() // 2 89 | elif size // (cpu_count() // 4) != 0: 90 | num_threads = cpu_count() // 4 91 | else: 92 | num_threads = 1 93 | batch_size = size // num_threads 94 | pool = Pool(processes=num_threads) 95 | for start in np.arange(0, size, batch_size): 96 | end = int(np.min([start + batch_size, size])) 97 | batch_voc_image_paths = voc_image_paths[start:end] 98 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 99 | batch_labelme_image_paths = labelme_image_paths[start:end] 100 | batch_labelme_json_paths = labelme_json_paths[start:end] 101 | pool.apply_async(process_batch_images_labels, error_callback=print_error, 102 | args=(batch_labelme_image_paths,batch_labelme_json_paths, 103 | batch_voc_image_paths,batch_voc_annotation_paths,)) 104 | pool.close() 105 | pool.join() 106 | print("结束多线程处理") 107 | 108 | def process_batch_images_labels(batch_labelme_image_paths,batch_labelme_json_paths,batch_voc_image_paths,batch_voc_annotation_paths): 109 | """ 110 | 这是将批量Labeme数据集图像及其JSOn标签转换为VOC数据集格式的函数 111 | Args: 112 | batch_labelme_image_paths: 批量Labelme数据集图像文件路径数组 113 | batch_labelme_json_paths: 批量Labelme数据集Json标签文件路径数组 114 | batch_voc_image_paths: 批量VOC数据集图像文件路径数组 115 | batch_voc_annotation_paths: 批量VOC数据集XML标签文件路径数组 116 | Returns: 117 | """ 118 | size = len(batch_labelme_image_paths) 119 | for i in tqdm(np.arange(size)): 120 | process_single_image_label(batch_labelme_image_paths[i],batch_labelme_json_paths[i], 121 | batch_voc_image_paths[i],batch_voc_annotation_paths[i]) 122 | def process_single_image_label(labelme_image_path,labelme_json_path,voc_image_path,voc_annotation_path): 123 | """ 124 | 这是将单张Lableme数据集图像及其Json标签转换为VOC数据集格式的函数 125 | Args: 126 | labelme_image_path: Labelme数据集图像文件路径 127 | labelme_json_path: Labelme数据集Json标签文件路径 128 | voc_image_path: VOC数据集图像文件路径 129 | voc_annotation_path: VOC数据集XML标签文件路径 130 | Returns: 131 | """ 132 | # 复制图像 133 | shutil.copy(labelme_image_path,voc_image_path) 134 | 135 | # 读取labelme的json标签 136 | with open(labelme_json_path, 'r', encoding='utf-8') as f: 137 | # 读取json文件中的检测标签 138 | json_data = json.load(f) 139 | image_h = json_data["imageHeight"] 140 | image_w = json_data["imageWidth"] 141 | writer = Writer(voc_image_path,image_w,image_h) 142 | for shape in json_data['shapes']: 143 | x1,y1,x2,y2 = np.reshape(shape['points'],(-1,4))[0] 144 | cls_name = shape['label'] 145 | writer.addObject(cls_name,x1,y1,x2,y2) 146 | writer.save(voc_annotation_path) 147 | 148 | def print_error(value): 149 | """ 150 | 定义错误回调函数 151 | Args: 152 | value: 出错误值 153 | Returns: 154 | """ 155 | print("error: ", value) 156 | 157 | 158 | 159 | def run_main(): 160 | """ 161 | 这是主函数 162 | """ 163 | # COCO2017 164 | labelme_dataset_dir = "/home/dpw/deeplearning/dataset/labelme/coco2017" 165 | voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc1/coco2017" 166 | choices = ["train", "val"] 167 | labelme2voc(labelme_dataset_dir,voc_dataset_dir,choices) 168 | 169 | # VOC2007 170 | labelme_dataset_dir = "/home/dpw/deeplearning/dataset/labelme/voc2007" 171 | voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc1/voc2007" 172 | choices = ["train", "val","test"] 173 | labelme2voc(labelme_dataset_dir,voc_dataset_dir,choices) 174 | 175 | # VOC2012 176 | labelme_dataset_dir = "/home/dpw/deeplearning/dataset/labelme/voc2012" 177 | voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc1/voc2012" 178 | choices = ["train", "val"] 179 | labelme2voc(labelme_dataset_dir,voc_dataset_dir,choices) 180 | 181 | 182 | if __name__ == '__main__': 183 | run_main() -------------------------------------------------------------------------------- /merge_voc_dataset_v1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/4/8 10:21 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : merge_voc_dataset_v1.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将多个VOC格式的数据集合并成1个VOC格式数据集的脚本,根据子集对应关系进行合并 10 | """ 11 | 12 | import os 13 | import cv2 14 | import shutil 15 | import numpy as np 16 | import xml.etree.ElementTree as ET 17 | 18 | from tqdm import tqdm 19 | from multiprocessing import Pool 20 | from multiprocessing import cpu_count 21 | from pascal_voc_writer import Writer 22 | 23 | def merge_voc_dataset(voc_dataset_dir_list,new_voc_dataset_dir): 24 | """ 25 | 这是合并VOC数据集的函数 26 | Args: 27 | voc_dataset_dir_list: VOC数据集地址列表 28 | new_voc_dataset_dir: 合并后的VOC数据集地址 29 | Returns: 30 | """ 31 | # 初始化新VOC数据集相关目录地址 32 | new_voc_image_dir = os.path.join(new_voc_dataset_dir,'JPEGImages') 33 | new_voc_annotation_dir = os.path.join(new_voc_dataset_dir,'Annotations') 34 | new_voc_main_dir = os.path.join(new_voc_dataset_dir,"ImageSets","Main") 35 | if not os.path.exists(new_voc_image_dir): 36 | os.makedirs(new_voc_image_dir) 37 | if not os.path.exists(new_voc_annotation_dir): 38 | os.makedirs(new_voc_annotation_dir) 39 | if not os.path.exists(new_voc_main_dir): 40 | os.makedirs(new_voc_main_dir) 41 | 42 | # 合并子集txt文件 43 | for choice in ['train','val','test','trainval']: 44 | new_choice_txt_path = os.path.join(new_voc_main_dir, '{}.txt'.format(choice)) 45 | with open(new_choice_txt_path,'w',encoding='utf-8') as f: 46 | for voc_dataset_dir in voc_dataset_dir_list: 47 | voc_main_dir = os.path.join(voc_dataset_dir, "ImageSets", "Main") 48 | choice_txt_path = os.path.join(voc_main_dir, '{}.txt'.format(choice)) 49 | if os.path.exists(choice_txt_path): 50 | with open(choice_txt_path,'r',encoding='utf-8') as g: 51 | for line in g.readlines(): 52 | f.write(line) 53 | 54 | # 遍历所有VOC数据集，获取所有图像及其XML标签文件地址 55 | voc_image_paths = [] 56 | voc_annotation_paths = [] 57 | new_voc_image_paths = [] 58 | new_voc_annotation_paths = [] 59 | for voc_dataset_dir in voc_dataset_dir_list: 60 | voc_image_dir = os.path.join(voc_dataset_dir, 'JPEGImages') 61 | if not os.path.exists(voc_image_dir): 62 | voc_image_dir = os.path.join(voc_dataset_dir,"images") 63 | voc_annotation_dir = os.path.join(voc_dataset_dir, 'Annotations') 64 | # 遍历文件夹，生成图像和XML文件夹路径 65 | for image_name in os.listdir(voc_image_dir): 66 | fname,ext = os.path.splitext(image_name) 67 | voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 68 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,fname+".xml")) 69 | new_voc_image_paths.append(os.path.join(new_voc_image_dir,image_name)) 70 | new_voc_annotation_paths.append(os.path.join(new_voc_annotation_dir,fname+".xml")) 71 | voc_image_paths = np.array(voc_image_paths) 72 | voc_annotation_paths = np.array(voc_annotation_paths) 73 | new_voc_image_paths = np.array(new_voc_image_paths) 74 | new_voc_annotation_paths = np.array(new_voc_annotation_paths) 75 | 76 | # 多线程合并VOC数据集 77 | size = len(voc_image_paths) 78 | if size // cpu_count() != 0: 79 | num_threads = cpu_count() 80 | elif size // (cpu_count() // 2) != 0: 81 | num_threads = cpu_count() // 2 82 | elif size // (cpu_count() // 4) != 0: 83 | num_threads = cpu_count() // 4 84 | else: 85 | num_threads = 1 86 | batch_size = size // num_threads 87 | pool = Pool(processes=num_threads) 88 | for start in np.arange(0,size,batch_size): 89 | end = int(np.min([start+batch_size,size])) 90 | batch_voc_image_paths = voc_image_paths[start:end] 91 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 92 | batch_new_voc_image_paths = new_voc_image_paths[start:end] 93 | batch_new_voc_annotation_paths = new_voc_annotation_paths[start:end] 94 | pool.apply_async(copy_batch_images_xmls,callback=print_error, 95 | args=(batch_voc_image_paths,batch_voc_annotation_paths, 96 | batch_new_voc_image_paths,batch_new_voc_annotation_paths)) 97 | pool.close() 98 | pool.join() 99 | 100 | def print_error(value): 101 | """ 102 | 定义错误回调函数 103 | Args: 104 | value: 出错误值 105 | Returns: 106 | """ 107 | print("error: ", value) 108 | 109 | def parse_xml(xml_path,class_names=None): 110 | """ 111 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 112 | Args: 113 | xml_path: XML标签文件路径 114 | class_names: 目标名称数组，默认为None 115 | Returns: 116 | """ 117 | # 获取XML文件的根结点 118 | root = ET.parse(xml_path).getroot() 119 | h = int(root.find("size").find("height").text) 120 | w = int(root.find("size").find("width").text) 121 | # 遍历所有目标 122 | objects = [] 123 | for obj in root.findall('object'): 124 | obj_name = obj.find('name').text 125 | bndbox = obj.find('bndbox') 126 | xmin = bndbox.find('xmin').text 127 | ymin = bndbox.find('ymin').text 128 | xmax = bndbox.find('xmax').text 129 | ymax = bndbox.find('ymax').text 130 | if class_names is None: 131 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 132 | else: 133 | if obj_name in class_names: 134 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 135 | return objects,(h,w) 136 | 137 | 138 | def copy_single_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path): 139 | """ 140 | 这是复制单张VOC图像及其对应xml标签文件的函数 141 | Args: 142 | voc_image_path: voc图像文件路径 143 | voc_xml_path: voc标签文件路径 144 | new_voc_image_path: 新voc图像文件路径 145 | new_voc_xml_path: 新voc标签文件路径 146 | Returns: 147 | """ 148 | # 复制图像 149 | shutil.copy(voc_image_path,new_voc_image_path) 150 | 151 | # 解析XML文件 152 | objects,(h,w) = parse_xml(voc_xml_path) 153 | 154 | # xml文件复制 155 | writer = Writer(new_voc_image_path,w,h) 156 | for cls_name,x1,y1,x2,y2 in objects: 157 | writer.addObject(cls_name,x1,y1,x2,y2) 158 | writer.save(new_voc_xml_path) 159 | 160 | def copy_batch_images_xmls(batch_voc_image_paths,batch_voc_annotation_paths, 161 | batch_new_voc_image_paths,batch_new_voc_annotation_paths): 162 | """ 163 | 这是复制批量VOC图像及其对应xml标签文件的函数 164 | Args: 165 | batch_voc_image_paths: 批量voc图像文件路径数组 166 | batch_voc_annotation_paths: 批量voc标签文件路径数组 167 | batch_new_voc_image_paths: 批量新voc图像文件路径数组 168 | batch_new_voc_annotation_paths: 批量新voc标签文件路径数组 169 | Returns: 170 | """ 171 | size = len(batch_voc_image_paths) 172 | for i in tqdm(np.arange(size)): 173 | voc_image_path = batch_voc_image_paths[i] 174 | voc_xml_path = batch_voc_annotation_paths[i] 175 | new_voc_image_path = batch_new_voc_image_paths[i] 176 | new_voc_xml_path = batch_new_voc_annotation_paths[i] 177 | copy_single_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path) 178 | 179 | def run_main(): 180 | """ 181 | 这是主函数 182 | """ 183 | voc_dataset_dir_list = [os.path.abspath("../dataset/Cityscapes"), 184 | os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.01"), 185 | os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.02"), 186 | os.path.abspath("../dataset/Foggy_Cityscapes_beta_0.005"), 187 | os.path.abspath("../dataset/BDD100k"), 188 | os.path.abspath("../dataset/KITTI")] 189 | new_voc_dataset_dir = os.path.abspath("../dataset/VOC_Ours") 190 | merge_voc_dataset(voc_dataset_dir_list,new_voc_dataset_dir) 191 | 192 | if __name__ == '__main__': 193 | run_main() 194 | -------------------------------------------------------------------------------- /merge_voc_dataset_v2.py: -------------------------------------------------------------------------------- 1 | # @Time : 2024/4/21 下午1:52 2 | # @Author : DaiPuWei 3 | # @Email : daipuwei@qq.com 4 | # @File : merge_voc_dataset_v2.py 5 | # @Software: PyCharm 6 | 7 | """ 8 | 这是合并多VOC数据集的脚本 9 | """ 10 | 11 | import os 12 | import shutil 13 | import numpy as np 14 | import xml.etree.ElementTree as ET 15 | from tqdm import tqdm 16 | from multiprocessing import Pool, cpu_count 17 | from pascal_voc_writer import Writer 18 | 19 | def merge_voc_datasets(origin_voc_dataset_dirs, merge_voc_dataset_dir, choice_dict): 20 | """ 21 | 这是将多个不同VOC检测数据集合并成一个VOC检测数据集的函数 22 | Args: 23 | origin_voc_dataset_dirs: 原始VOC检测数据集地址数组 24 | merge_voc_dataset_dir：归并后VOC检测数据集地址 25 | choice_dict: 原始VOC检测数据集子集和合并VOC检测子集对应字典 26 | Returns: 27 | """ 28 | # 初始化合并数据集相关路径 29 | merge_voc_image_dir = os.path.join(merge_voc_dataset_dir, 'JPEGImages') 30 | merge_voc_annotation_dir = os.path.join(merge_voc_dataset_dir, 'Annotations') 31 | merge_voc_main_dir = os.path.join(merge_voc_dataset_dir, "ImageSets", "Main") 32 | if not os.path.exists(merge_voc_image_dir): 33 | os.makedirs(merge_voc_image_dir) 34 | if not os.path.exists(merge_voc_annotation_dir): 35 | os.makedirs(merge_voc_annotation_dir) 36 | if not os.path.exists(merge_voc_main_dir): 37 | os.makedirs(merge_voc_main_dir) 38 | 39 | # 遍历整个子集对应字典 40 | print("开始遍历所有数据集，初始化所有VOC数据集图像及其标签路径") 41 | origin_voc_image_paths = [] 42 | origin_voc_annotation_paths = [] 43 | merge_voc_image_paths = [] 44 | merge_voc_annotation_paths = [] 45 | for merge_choice, origin_choice_list in choice_dict.items(): 46 | merge_voc_txt_path = os.path.join(merge_voc_main_dir,merge_choice+".txt") 47 | with open(merge_voc_txt_path, 'w', encoding='utf-8') as f: 48 | # 遍历各个数据集指定子集 49 | for origin_voc_dataset_dir, choice_list in zip(origin_voc_dataset_dirs, origin_choice_list): 50 | origin_voc_image_dir = os.path.join(origin_voc_dataset_dir,"JPEGImages") 51 | origin_voc_annotation_dir = os.path.join(origin_voc_dataset_dir,"Annotations") 52 | origin_voc_main_dir = os.path.join(origin_voc_dataset_dir, "ImageSets","Main") 53 | if len(choice_list) > 0: 54 | for origin_choice in choice_list: 55 | origin_voc_txt_path = os.path.join(origin_voc_main_dir, origin_choice + ".txt") 56 | with open(origin_voc_txt_path, 'r',encoding='utf-8') as g: 57 | for line in tqdm(g.readlines()): 58 | image_name = line.strip() 59 | origin_voc_image_path = os.path.join(origin_voc_image_dir,image_name+".jpg") 60 | origin_voc_annotation_path = os.path.join(origin_voc_annotation_dir, image_name + ".xml") 61 | merge_voc_image_path = os.path.join(merge_voc_image_dir,image_name+".jpg") 62 | merge_voc_annotation_path = os.path.join(merge_voc_annotation_dir,image_name+".xml") 63 | if os.path.exists(origin_voc_image_path): 64 | origin_voc_image_paths.append(origin_voc_image_path) 65 | origin_voc_annotation_paths.append(origin_voc_annotation_path) 66 | merge_voc_image_paths.append(merge_voc_image_path) 67 | merge_voc_annotation_paths.append(merge_voc_annotation_path) 68 | f.write("{0}\n".format(image_name)) 69 | origin_voc_image_paths = np.array(origin_voc_image_paths) 70 | origin_voc_annotation_paths = np.array(origin_voc_annotation_paths) 71 | merge_voc_image_paths = np.array(merge_voc_image_paths) 72 | merge_voc_annotation_paths = np.array(merge_voc_annotation_paths) 73 | print("结束遍历所有数据集，共计{0}张图像及其标签需要处理".format(len(origin_voc_image_paths))) 74 | 75 | # 多线程复制图像 76 | size = len(origin_voc_image_paths) 77 | # batch_size = size // cpu_count() 78 | if size // cpu_count() != 0: 79 | num_threads = cpu_count() 80 | elif size // (cpu_count() // 2) != 0: 81 | num_threads = cpu_count() // 2 82 | elif size // (cpu_count() // 4) != 0: 83 | num_threads = cpu_count() // 4 84 | else: 85 | num_threads = 1 86 | batch_size = size // num_threads 87 | pool = Pool(processes=num_threads) 88 | for start in np.arange(0, size, batch_size): 89 | end = int(np.min([start + batch_size, size])) 90 | batch_origin_voc_image_paths = origin_voc_image_paths[start:end] 91 | batch_origin_voc_annotation_paths = origin_voc_annotation_paths[start:end] 92 | batch_merge_voc_image_paths = merge_voc_image_paths[start:end] 93 | batch_merge_voc_annotation_paths = merge_voc_annotation_paths[start:end] 94 | pool.apply_async(copy_batch_images_xmls, error_callback=print_error, 95 | args=(batch_origin_voc_image_paths, batch_origin_voc_annotation_paths, 96 | batch_merge_voc_image_paths,batch_merge_voc_annotation_paths)) 97 | pool.close() 98 | pool.join() 99 | 100 | def print_error(value): 101 | """ 102 | 定义错误回调函数 103 | Args: 104 | value: 出错误值 105 | Returns: 106 | """ 107 | print("error: ", value) 108 | 109 | def parse_xml(xml_path,class_names=None): 110 | """ 111 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 112 | Args: 113 | xml_path: XML标签文件路径 114 | class_names: 目标名称数组，默认为None 115 | Returns: 116 | """ 117 | # 获取XML文件的根结点 118 | root = ET.parse(xml_path).getroot() 119 | h = int(root.find("size").find("height").text) 120 | w = int(root.find("size").find("width").text) 121 | # 遍历所有目标 122 | objects = [] 123 | for obj in root.findall('object'): 124 | obj_name = obj.find('name').text 125 | bndbox = obj.find('bndbox') 126 | xmin = bndbox.find('xmin').text 127 | ymin = bndbox.find('ymin').text 128 | xmax = bndbox.find('xmax').text 129 | ymax = bndbox.find('ymax').text 130 | if class_names is None: 131 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 132 | else: 133 | if obj_name in class_names: 134 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 135 | return objects,(h,w) 136 | 137 | 138 | def copy_single_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path): 139 | """ 140 | 这是复制单张VOC图像及其对应xml标签文件的函数 141 | Args: 142 | voc_image_path: voc图像文件路径 143 | voc_xml_path: voc标签文件路径 144 | new_voc_image_path: 新voc图像文件路径 145 | new_voc_xml_path: 新voc标签文件路径 146 | Returns: 147 | """ 148 | # 复制图像 149 | shutil.copy(voc_image_path,new_voc_image_path) 150 | 151 | # 解析XML文件 152 | objects,(h,w) = parse_xml(voc_xml_path) 153 | 154 | # xml文件复制 155 | writer = Writer(new_voc_image_path,w,h) 156 | for cls_name,x1,y1,x2,y2 in objects: 157 | writer.addObject(cls_name,x1,y1,x2,y2) 158 | writer.save(new_voc_xml_path) 159 | 160 | def copy_batch_images_xmls(batch_voc_image_paths,batch_voc_annotation_paths, 161 | batch_new_voc_image_paths,batch_new_voc_annotation_paths): 162 | """ 163 | 这是复制批量VOC图像及其对应xml标签文件的函数 164 | Args: 165 | batch_voc_image_paths: 批量voc图像文件路径数组 166 | batch_voc_annotation_paths: 批量voc标签文件路径数组 167 | batch_new_voc_image_paths: 批量新voc图像文件路径数组 168 | batch_new_voc_annotation_paths: 批量新voc标签文件路径数组 169 | Returns: 170 | """ 171 | size = len(batch_voc_image_paths) 172 | for i in tqdm(np.arange(size)): 173 | voc_image_path = batch_voc_image_paths[i] 174 | voc_xml_path = batch_voc_annotation_paths[i] 175 | new_voc_image_path = batch_new_voc_image_paths[i] 176 | new_voc_xml_path = batch_new_voc_annotation_paths[i] 177 | copy_single_image_xml(voc_image_path,voc_xml_path,new_voc_image_path,new_voc_xml_path) 178 | 179 | def run_main(): 180 | """ 181 | 这是主函数 182 | """ 183 | # voc07++12+coco2017 184 | voc_dataset_dirs = ["/home/dpw/deeplearning/dataset/voc/coco2017", 185 | "/home/dpw/deeplearning/dataset/voc/voc2007", 186 | "/home/dpw/deeplearning/dataset/voc/voc2012"] 187 | merge_voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc/coco2017+voc07++12" 188 | choice_dict = { 189 | "train": [['train'],['train'],['train']], 190 | "test": [['val'],['test','val'],['val']] 191 | } 192 | merge_voc_datasets(voc_dataset_dirs,merge_voc_dataset_dir,choice_dict) 193 | 194 | if __name__ == '__main__': 195 | run_main() 196 | -------------------------------------------------------------------------------- /mot2voc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/5/23 15:41 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : mot2voc.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将MOT数据集转换为VOC数据集格式的脚本，根据开发需要自行修改 10 | mot_dataste_dir、voc_dataset_dir和type即可。 11 | 12 | 其中: 13 | - mot_dataste_dir代表原始MOT数据集目录； 14 | - voc_dataset_dir代表VOC数据集格式的MOT数据集目录； 15 | - type代表数据集类型，候选值有‘all’、‘train’和‘test’， 16 | ‘all’代表转化全部数据，‘train’代表转化训练数据集，‘test’代表转化测试数据集； 17 | """ 18 | 19 | import os 20 | import cv2 21 | import numpy as np 22 | from tqdm import tqdm 23 | from pascal_voc_writer import Writer 24 | 25 | from multiprocessing import Pool 26 | from multiprocessing import cpu_count 27 | 28 | def mot2voc(mot_dataste_dir,voc_dataset_dir,type='train'): 29 | """ 30 | 这是利用多线程将MOT数据集转换为VOC数据集的函数 31 | Args: 32 | mot_dataste_dir: MOT数据集文件夹路径 33 | voc_dataset_dir: VOC数据集文件夹路径 34 | type: 类型，默认为'train',候选值有['train','test'，'all'] 35 | Returns: 36 | """ 37 | # 初始化VOC数据集相关路径 38 | voc_image_dir = os.path.join(voc_dataset_dir,'JPEGImages') 39 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 40 | voc_main_dir = os.path.join(voc_dataset_dir,'ImageSets',"Main") 41 | if not os.path.exists(voc_image_dir): 42 | os.makedirs(voc_image_dir) 43 | if not os.path.exists(voc_annotation_dir): 44 | os.makedirs(voc_annotation_dir) 45 | if not os.path.exists(voc_main_dir): 46 | os.makedirs(voc_main_dir) 47 | 48 | mot_annotations = {} 49 | voc_image_paths = [] 50 | voc_annotation_paths = [] 51 | cnt = 0 52 | # 初始化训练集或者训练测试集的初始化MOT图像及其标签，VOC图像及其标签 53 | if type == 'train' or type == 'all': 54 | # 初始化MOT图像及其标签 55 | sub_dataset_dir = os.path.join(mot_dataste_dir, 'train') 56 | #print(sub_dataset_dir) 57 | for imageset_name in os.listdir(sub_dataset_dir): 58 | sub_imageset_dir = os.path.join(sub_dataset_dir,imageset_name,'img1') 59 | sub_label_txt_path = os.path.join(sub_dataset_dir,imageset_name,"gt",'gt.txt') 60 | #print(sub_imageset_dir,sub_label_txt_path) 61 | with open(sub_label_txt_path,'r') as f: 62 | for line in f.readlines(): 63 | strs = line.strip().split(",") 64 | mot_image_path = os.path.join(sub_imageset_dir,"{0:06d}.jpg".format(int(strs[0]))) 65 | if mot_image_path not in mot_annotations.keys(): 66 | mot_annotations[mot_image_path] = [] 67 | voc_image_paths.append(os.path.join(voc_image_dir,"{0:06d}.jpg".format(cnt))) 68 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,"{0:06d}.xml".format(cnt))) 69 | cnt += 1 70 | x1 = int(round(float(strs[2]))) 71 | y1 = int(round(float(strs[3]))) 72 | w = int(round(float(strs[4]))) 73 | h = int(round(float(strs[3]))) 74 | x2 = x1+w 75 | y2 = y1+h 76 | mot_annotations[mot_image_path].append(['person',x1,y1,x2,y2]) 77 | # 初始化测试集或者训练测试集的初始化MOT图像及其标签，VOC图像及其标签 78 | if type == 'test' or type == 'all': 79 | # 初始化MOT图像及其标签 80 | sub_dataset_dir = os.path.join(mot_dataste_dir, 'test') 81 | for imageset_name in os.listdir(sub_dataset_dir): 82 | sub_imageset_dir = os.path.join(sub_dataset_dir,imageset_name,'img1') 83 | sub_label_txt_path = os.path.join(sub_dataset_dir, imageset_name, "det", 'det.txt') 84 | with open(sub_label_txt_path,'r') as f: 85 | for line in f.readlines(): 86 | strs = line.strip().split(",") 87 | mot_image_path = os.path.join(sub_imageset_dir,"{0:06d}.jpg".format(int(strs[0]))) 88 | if mot_image_path not in mot_annotations.keys(): 89 | mot_annotations[mot_image_path] = [] 90 | voc_image_paths.append(os.path.join(voc_image_dir,"{0:06d}.jpg".format(cnt))) 91 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,"{0:06d}.xml".format(cnt))) 92 | cnt += 1 93 | x1 = int(round(float(strs[2]))) 94 | y1 = int(round(float(strs[3]))) 95 | w = int(round(float(strs[4]))) 96 | h = int(round(float(strs[3]))) 97 | x2 = x1+w 98 | y2 = y1+h 99 | mot_annotations[mot_image_path].append(['person',x1,y1,x2,y2]) 100 | mot_image_paths = np.array(list(mot_annotations.keys())) 101 | voc_image_paths = np.array(voc_image_paths) 102 | voc_annotation_paths = np.array(voc_annotation_paths) 103 | 104 | # 多线程生成VOC数据集 105 | size = len(voc_image_paths) 106 | batch_size = size // (cpu_count()-1) 107 | #print(size,cpu_count()-1,batch_size) 108 | pool = Pool(processes=cpu_count()-1) 109 | for start in np.arange(0,size,batch_size): 110 | end = int(np.min([start+batch_size,size])) 111 | batch_voc_image_paths = voc_image_paths[start:end] 112 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 113 | batch_mot_image_paths = mot_image_paths[start:end] 114 | pool.apply_async(batch_generate_voc_image_annotations,callback=print_error, 115 | args=(batch_mot_image_paths,batch_voc_image_paths,batch_voc_annotation_paths,mot_annotations)) 116 | pool.close() 117 | pool.join() 118 | 119 | def batch_generate_voc_image_annotations(batch_mot_image_paths,batch_voc_image_paths,batch_voc_annotation_paths,mot_annotations): 120 | """ 121 | 这是批量生成VOC数据集的图像及其标签的函数 122 | Args: 123 | batch_mot_image_paths: 批量MOT图像路径列表 124 | batch_voc_image_paths: 批量VOC图像路径列表 125 | batch_voc_annotation_paths: 批量VOC标签路径列表 126 | mot_annotations: MOT图像标签字典 127 | Returns: 128 | """ 129 | size = len(batch_voc_image_paths) 130 | for i in tqdm(np.arange(size)): 131 | generate_voc_image_annotation(batch_mot_image_paths[i],batch_voc_image_paths[i], 132 | batch_voc_annotation_paths[i],mot_annotations) 133 | 134 | def generate_voc_image_annotation(mot_image_path,voc_image_path,voc_annotation_path,mot_annotations): 135 | """ 136 | 这是生成VOC数据集的图像及其标签的函数 137 | Args: 138 | mot_image_path: MOT图像路径 139 | voc_image_path: VOC图像路径 140 | voc_annotation_path: VOC标签路径 141 | mot_annotations: MOT图像标签字典 142 | Returns: 143 | """ 144 | # 复制图像 145 | image = cv2.imread(mot_image_path) 146 | h,w,c = np.shape(image) 147 | if c == 1: 148 | image = cv2.cvtColor(image,cv2.COLOR_GRAY2BGR) 149 | if c == 4: 150 | image = cv2.cvtColor(image,cv2.COLOR_BGRA2BGR) 151 | cv2.imwrite(voc_image_path,image) 152 | 153 | # 生成XML文件 154 | writer = Writer(voc_annotation_path,w,h) 155 | for name,xmin,ymin,xmax,ymax in mot_annotations[mot_image_path]: 156 | writer.addObject(name,xmin,ymin,xmax,ymax) 157 | writer.save(voc_annotation_path) 158 | 159 | def print_error(value): 160 | """ 161 | 定义错误回调函数 162 | :param value: 163 | :return: 164 | """ 165 | print("error: ", value) 166 | 167 | def run_main(): 168 | """ 169 | 这是主函数 170 | """ 171 | # MOT15 172 | mot_dataste_dir = os.path.abspath("../MOT15") 173 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/MOT15") 174 | type='train' 175 | print("MOT15 --> VOC Start") 176 | mot2voc(mot_dataste_dir, voc_dataset_dir, type) 177 | print("MOT15 --> VOC Finish") 178 | 179 | # MOT16 180 | mot_dataste_dir = os.path.abspath("../MOT16") 181 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/MOT16") 182 | type='train' 183 | print("MOT16 --> VOC Start") 184 | mot2voc(mot_dataste_dir, voc_dataset_dir, type) 185 | print("MOT16 --> VOC Finish") 186 | 187 | # MOT17 188 | mot_dataste_dir = os.path.abspath("../MOT17") 189 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/MOT17") 190 | type='train' 191 | print("MOT17 --> VOC Start") 192 | mot2voc(mot_dataste_dir, voc_dataset_dir, type) 193 | print("MOT17 --> VOC Finish") 194 | 195 | if __name__ == '__main__': 196 | run_main() 197 | -------------------------------------------------------------------------------- /random_split_voc_dataset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/4/8 11:07 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : random_split_voc_dataset.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是随机划分VOC数据集的脚本 10 | """ 11 | 12 | import os 13 | import numpy as np 14 | 15 | def random_split_voc_dataset(voc_dataset_dir,train_ratio=0.8): 16 | """ 17 | 这是随机划分VOC数据集的函数 18 | Args: 19 | voc_dataset_dir: voc数据集地址 20 | train_ratio: 训练集比例，默认为0.8 21 | Returns: 22 | """ 23 | # 初始化相关文件和文件夹路径 24 | voc_main_dir = os.path.join(voc_dataset_dir,'ImageSets','Main') 25 | voc_image_dir = os.path.join(voc_dataset_dir,'JPEGImages') 26 | train_txt_path = os.path.join(voc_main_dir,'train.txt') 27 | trainval_txt_path = os.path.join(voc_main_dir,'trainval.txt') 28 | val_txt_path = os.path.join(voc_main_dir,'val.txt') 29 | if not os.path.exists(voc_main_dir): 30 | os.makedirs(voc_main_dir) 31 | 32 | # 遍历图像文件夹，获取所有图像 33 | image_name_list = [] 34 | for image_name in os.listdir(voc_image_dir): 35 | image_name_list.append(image_name) 36 | image_name_list = np.array(image_name_list) 37 | image_name_list = np.random.permutation(image_name_list) 38 | 39 | # 划分训练集和测试集 40 | size = len(image_name_list) 41 | random_index = np.random.permutation(size) 42 | train_size = int(size*train_ratio) 43 | train_image_name_list = image_name_list[random_index[0:train_size]] 44 | val_image_name_list = image_name_list[random_index[train_size:]] 45 | 46 | # 生成trainval 47 | with open(trainval_txt_path,'w') as f: 48 | for image_name in image_name_list: 49 | fname,ext = os.path.splitext(image_name) 50 | f.write(fname+"\n") 51 | # 生成train 52 | with open(train_txt_path, 'w') as f: 53 | for image_name in train_image_name_list: 54 | fname, ext = os.path.splitext(image_name) 55 | f.write(fname + "\n") 56 | # 生成val 57 | with open(val_txt_path,'w') as f: 58 | for image_name in val_image_name_list: 59 | fname, ext = os.path.splitext(image_name) 60 | f.write(fname + "\n") 61 | 62 | def run_main(): 63 | """ 64 | 这是主函数 65 | """ 66 | train_ratio = 0.8 67 | voc_dataset_dir = os.path.abspath("../dataset/Cityscapes"), 68 | random_split_voc_dataset(voc_dataset_dir, train_ratio) 69 | 70 | if __name__ == '__main__': 71 | run_main() 72 | -------------------------------------------------------------------------------- /rename_voc_dataset_object_label.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/5/23 9:17 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : rename_voc_dataset_object_label.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是利用多线程改变VOC数据集中目标分类标签的脚本 10 | """ 11 | 12 | import os 13 | import numpy as np 14 | import xml.etree.ElementTree as ET 15 | 16 | from tqdm import tqdm 17 | from pascal_voc_writer import Writer 18 | 19 | from multiprocessing import Pool 20 | from multiprocessing import cpu_count 21 | 22 | def rename_voc_dataset_object_label(voc_dataset_dir,object_name_dict): 23 | """ 24 | 这是改变VOC数据集中目标分类的函数 25 | Args: 26 | voc_dataset_dir: VOC数据集文件夹路径 27 | object_name_dict: 目标名称字典，键为更改后目标名称，值为待修改过的目标名称数组 28 | Returns: 29 | """ 30 | # 初始化VOCs数据集相关路径 31 | voc_image_dir = os.path.join(voc_dataset_dir,"JPEGImages") 32 | if not os.path.exists(voc_image_dir): 33 | voc_image_dir = os.path.join(voc_dataset_dir,"images") 34 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 35 | 36 | # 初始化VOC图像及其标签路径 37 | voc_image_paths = [] 38 | voc_annotation_paths = [] 39 | for image_name in os.listdir(voc_image_dir): 40 | fname,ext = os.path.splitext(image_name) 41 | voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 42 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,fname+".xml")) 43 | voc_image_paths = np.array(voc_image_paths) 44 | voc_annotation_paths = np.array(voc_annotation_paths) 45 | 46 | # 多线程更改目标名称 47 | size = len(voc_annotation_paths) 48 | if size // cpu_count() != 0: 49 | num_threads = cpu_count() 50 | elif size // (cpu_count() // 2) != 0: 51 | num_threads = cpu_count() // 2 52 | elif size // (cpu_count() // 4) != 0: 53 | num_threads = cpu_count() // 4 54 | else: 55 | num_threads = 1 56 | batch_size = size // num_threads 57 | pool = Pool(processes=num_threads) 58 | for start in np.arange(0,size,batch_size): 59 | end = int(np.min([start+batch_size,size])) 60 | batch_image_paths = voc_image_paths[start:end] 61 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 62 | pool.apply_async(batch_rename_voc_object,callback=print_error, 63 | args=(batch_image_paths,batch_voc_annotation_paths,object_name_dict)) 64 | pool.close() 65 | pool.join() 66 | # batch_rename_voc_object(voc_annotation_paths,src_object_names,target_object_name) 67 | 68 | def parse_xml(xml_path,class_names=None): 69 | """ 70 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 71 | Args: 72 | xml_path: XML标签文件路径 73 | class_names: 目标名称数组，默认为None 74 | Returns: 75 | """ 76 | # 获取XML文件的根结点 77 | root = ET.parse(xml_path).getroot() 78 | h = int(root.find("size").find("height").text) 79 | w = int(root.find("size").find("width").text) 80 | # 遍历所有目标 81 | objects = [] 82 | for obj in root.findall('object'): 83 | obj_name = obj.find('name').text 84 | bndbox = obj.find('bndbox') 85 | xmin = bndbox.find('xmin').text 86 | ymin = bndbox.find('ymin').text 87 | xmax = bndbox.find('xmax').text 88 | ymax = bndbox.find('ymax').text 89 | if class_names is None: 90 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 91 | else: 92 | if obj_name in class_names: 93 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 94 | return objects,(h,w) 95 | 96 | 97 | def rename_voc_object(voc_image_path,voc_annotation_path,object_name_dict): 98 | """ 99 | 这是对一张VOC数据集XML标签文件重命名目标标签的函数 100 | Args: 101 | voc_image_path: VOC数据集图像路径 102 | voc_annotation_path: VOC数据集XML文件路径 103 | object_name_dict: 目标名称字典，键为更改后目标名称，值为待修改过的目标名称数组 104 | Returns: 105 | """ 106 | # 获取XML标签 107 | objects,(h,w) = parse_xml(voc_annotation_path) 108 | 109 | # 重新写入目标标签，并完成重命名 110 | writer = Writer(voc_image_path,w,h) 111 | for cls_name,x1,y1,x2,y2 in objects: 112 | rename_cls_name = cls_name 113 | # 遍历目标名称修改字典 114 | for _rename_cls_name,src_cls_names in object_name_dict.items(): 115 | # 目标名称在待修改目标名称列表中 116 | if cls_name in src_cls_names: 117 | rename_cls_name = _rename_cls_name 118 | break 119 | writer.addObject(rename_cls_name,x1,y2,x2,y2) 120 | writer.save(voc_annotation_path) 121 | 122 | def batch_rename_voc_object(batch_voc_annotation_paths,src_object_names,target_object_name): 123 | """ 124 | 这是批量重命名目标标签的函数 125 | Args: 126 | batch_voc_annotation_paths: 批量VOC数据集XML文件路径数组 127 | src_object_names: 原始目标名称数组 128 | target_object_name: 重命名后目标名称 129 | Returns: 130 | """ 131 | # 遍历所有XML标签文件，重命名目标标签 132 | size = len(batch_voc_annotation_paths) 133 | for i in tqdm(np.arange(size)): 134 | rename_voc_object(batch_voc_annotation_paths[i],src_object_names,target_object_name) 135 | 136 | def print_error(value): 137 | """ 138 | 定义错误回调函数 139 | Args: 140 | value: 出错误值 141 | Returns: 142 | """ 143 | print("error: ", value) 144 | 145 | def run_main(): 146 | """ 147 | 这是主函数 148 | """ 149 | # BDD100k 150 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/BDD100k") 151 | src_object_names = ['rider'] 152 | target_object_name = "non-car" 153 | rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 154 | 155 | # Cityscapes 156 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/Cityscapes") 157 | src_object_names = ['rider'] 158 | target_object_name = "non-car" 159 | rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 160 | 161 | # Foggy_Cityscapes_beta=0.005 162 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/Foggy_Cityscapes_beta_0.01") 163 | src_object_names = ['rider'] 164 | target_object_name = "non-car" 165 | rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 166 | 167 | # Foggy_Cityscapes_beta=0.01 168 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/Foggy_Cityscapes_beta_0.02") 169 | src_object_names = ['rider'] 170 | target_object_name = "non-car" 171 | rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 172 | 173 | # Foggy_Cityscapes_beta=0.02 174 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/Foggy_Cityscapes_beta_0.005") 175 | src_object_names = ['rider'] 176 | target_object_name = "non-car" 177 | rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 178 | 179 | # KITTI 180 | voc_dataset_dir = os.path.abspath("../dataset/Person-NonCar/KITTI") 181 | src_object_names = ['rider'] 182 | target_object_name = "non-car" 183 | rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 184 | 185 | # # VOC2007 186 | # voc_dataset_dir = os.path.abspath("/data/daipuwei/ObjectDetection/dataset/Person-NonCar/VOC2007") 187 | # src_object_names = ['Pedestrian'] 188 | # target_object_name = "person" 189 | # rename_voc_dataset_object_label(voc_dataset_dir,src_object_names,target_object_name) 190 | 191 | if __name__ == '__main__': 192 | run_main() 193 | -------------------------------------------------------------------------------- /video2voc_dataset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2022/5/9 11:21 3 | # @Author : DaiPuWei 4 | # @Email : 771830171@qq.com 5 | # @File : video2voc_dataset.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是利用多线程对视频文件进行切割并转换为VOC格式数据集的脚本 10 | """ 11 | 12 | import os 13 | import cv2 14 | import numpy as np 15 | from tqdm import tqdm 16 | from multiprocessing import Pool 17 | from multiprocessing import cpu_count 18 | 19 | def video2voc_dataset(video_paths,voc_dataset_save_dir,bin=1): 20 | """ 21 | 这是将视频集进行切割并转换成VOC格式数据集的函数 22 | Args: 23 | video_paths: 视频文件路径数组 24 | voc_dataset_save_dir: VOC数据集保存文件夹 25 | bin: 间隔时长，默认为1s 26 | Returns: 27 | """ 28 | # 遍历视频，初始化VOC数据集文件夹路径 29 | voc_dataset_dirs = [] 30 | for video_path in video_paths: 31 | _,video_name = os.path.split(video_path) 32 | fname,ext = os.path.splitext(video_name) 33 | voc_dataset_dirs.append(os.path.join(voc_dataset_save_dir,fname)) 34 | 35 | # 多线切割视频并生成VOC数据集 36 | #print(video_paths) 37 | #print(voc_dataset_dirs) 38 | size = len(video_paths) 39 | batch_size = size // (cpu_count()-1) 40 | if batch_size == 0: 41 | batch_size = 1 42 | #print(size,batch_size) 43 | pool = Pool(processes=cpu_count()-1) 44 | for start in np.arange(0,size,batch_size): 45 | end = int(np.min([start+batch_size,size])) 46 | batch_video_paths = video_paths[start:end] 47 | batch_voc_dataset_dirs = voc_dataset_dirs[start:end] 48 | pool.apply_async(batch_videos2voc_datasets,callback=print_error, 49 | args=(batch_video_paths,batch_voc_dataset_dirs,bin)) 50 | pool.close() 51 | pool.join() 52 | 53 | def batch_videos2voc_datasets(batch_video_paths,batch_voc_dataset_dirs,bin=1): 54 | """ 55 | 这是将批量视频进行切割并转换VOC格式数据集的函数 56 | Args: 57 | batch_video_paths: 批量视频文件名路径数组 58 | batch_voc_dataset_dirs: 批量VOC数据集文件夹路径数组 59 | bin: 间隔时长 ，默认为1s 60 | Returns: 61 | """ 62 | for i in tqdm(np.arange(len(batch_video_paths))): 63 | single_video2voc_dataset(batch_video_paths[i],batch_voc_dataset_dirs[i],bin) 64 | 65 | def single_video2voc_dataset(video_path,voc_dataset_dir,bin=1): 66 | """ 67 | 这是将单个视频进行切割并转换为VOC数据集的函数 68 | Args: 69 | video_path: 视频文件路径 70 | voc_dataset_dir: VOC数据集文件夹路径 71 | bin: 间隔时长，默认为1s 72 | Returns: 73 | """ 74 | # 初始化视频名称 75 | _, video_name = os.path.split(video_path) 76 | fname, ext = os.path.splitext(video_name) 77 | # 初始化voc数据集路径 78 | voc_image_dir = os.path.join(voc_dataset_dir,"JPEGImages") 79 | voc_annotatiion_dir = os.path.join(voc_dataset_dir,"Annotations") 80 | if not os.path.exists(voc_image_dir): 81 | os.makedirs(voc_image_dir) 82 | if not os.path.exists(voc_annotatiion_dir): 83 | os.makedirs(voc_annotatiion_dir) 84 | 85 | # 读取视频进行抽帧保存 86 | vid = cv2.VideoCapture(video_path) 87 | if not vid.isOpened(): 88 | raise IOError("Couldn't open webcam or video") 89 | video_fps = int(round(vid.get(cv2.CAP_PROP_FPS))) # 视频的fps 90 | #print(video_fps) 91 | frame_bin = video_fps*bin # 帧间隔 92 | cnt = 0 93 | index = 0 94 | while True: 95 | return_value, frame = vid.read() 96 | if return_value: 97 | if cnt % frame_bin == 0: 98 | # 图像预处理 99 | h = frame.shape[0] 100 | w = frame.shape[1] 101 | cv2.imwrite(os.path.join(voc_image_dir, fname + "_frame_{0:06d}.jpg".format(index)),frame) 102 | index += 1 103 | cnt += 1 104 | else: 105 | break 106 | vid.release() 107 | 108 | def print_error(value): 109 | """ 110 | 定义错误回调函数 111 | Args: 112 | value: 113 | Returns: 114 | """ 115 | print("error: ", value) 116 | 117 | def run_main(): 118 | """ 119 | 这是主函数 120 | """ 121 | video_paths = [os.path.abspath("./1.mp4"), 122 | os.path.abspath("./2.mp4"), 123 | os.path.abspath("./3.mp4")] 124 | voc_dataset_save_dir = os.path.abspath("./VOCdevkit") 125 | bin = 1 126 | video2voc_dataset(video_paths,voc_dataset_save_dir,bin) 127 | 128 | 129 | if __name__ == '__main__': 130 | run_main() 131 | -------------------------------------------------------------------------------- /voc2coco.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2024/4/14 下午4:42 3 | # @Author : DaiPuWei 4 | # @Email : daipuwei@qq.com 5 | # @File : voc2coco.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将VOC数据集转换为COCO数据集的脚本 10 | """ 11 | 12 | import os 13 | import cv2 14 | import json 15 | import numpy as np 16 | import xml.etree.ElementTree as ET 17 | from tqdm import tqdm 18 | from multiprocessing import Pool,cpu_count 19 | 20 | def voc2coco(voc_dataset_dir,coco_dataset_dir,class_names,save_image=False,choices=['train','val']): 21 | """ 22 | 这是将VOC转换为COCO数据集的函数 23 | Args: 24 | voc_dataset_dir: VOC数据集地址 25 | coco_dataset_dir: COCO数据集地址 26 | class_names: 类别名称列表 27 | save_image: 是否保存图片，默认为False 28 | choices: 子集列表或者字符串，默认为['train','val'] 29 | Returns: 30 | """ 31 | if type(choices) == 'str': 32 | choices = [choices] 33 | if "trainval" in choices: 34 | choices.remove("trainval") 35 | 36 | # 初始化VOC数据集相关路径 37 | voc_image_dir = os.path.join(voc_dataset_dir,"JPEGImages") 38 | if not os.path.exists(voc_image_dir): 39 | voc_image_dir = os.path.join(voc_dataset_dir, "images") 40 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 41 | voc_main_dir = os.path.join(voc_dataset_dir,"ImageSets","Main") 42 | voc_txt_paths = [] 43 | for choice in choices: 44 | _voc_txt_path = os.path.join(voc_main_dir,'{0}.txt'.format(choice)) 45 | voc_txt_paths.append(_voc_txt_path) 46 | 47 | # 初始化COCO数据集路径 48 | coco_annotations_dir = os.path.join(coco_dataset_dir, 'annotations') 49 | if not os.path.exists(coco_annotations_dir): 50 | os.makedirs(coco_annotations_dir) 51 | coco_image_dirs = [] 52 | coco_json_paths = [] 53 | for choice in choices: 54 | _coco_image_dir = os.path.join(coco_dataset_dir,choice) 55 | _coco_json_path = os.path.join(coco_annotations_dir,"{}.json".format(choice)) 56 | if not os.path.exists(_coco_image_dir): 57 | os.makedirs(_coco_image_dir) 58 | coco_image_dirs.append(_coco_image_dir) 59 | coco_json_paths.append(_coco_json_path) 60 | coco_image_dirs = np.array(coco_image_dirs) 61 | coco_json_paths = np.array(coco_json_paths) 62 | 63 | # 初始化图像及其标签路径 64 | image_cnt = 0 65 | annotation_cnt = 0 66 | voc_image_paths = [] 67 | coco_image_paths = [] 68 | for i,choice in enumerate(choices): 69 | print("VOC-->COCO {}子集标签转换开始".format(choice)) 70 | voc_train_image_paths, coco_train_image_paths, image_cnt, annotation_cnt \ 71 | = xml2json(voc_image_dir, voc_annotation_dir, voc_txt_paths[i], 72 | coco_image_dirs[i], coco_json_paths[i], class_names, image_cnt, annotation_cnt) 73 | voc_image_paths.append(voc_train_image_paths) 74 | coco_image_paths.append(coco_train_image_paths) 75 | print("VOC-->COCO {}子集标签转换结束".format(choice)) 76 | voc_image_paths = np.concatenate(voc_image_paths) 77 | coco_image_paths = np.concatenate(coco_image_paths) 78 | 79 | # 多线程复制图像 80 | if save_image: 81 | print("开始复制图片") 82 | size = len(coco_image_paths) 83 | if size // cpu_count() != 0: 84 | num_threads = cpu_count() 85 | elif size // (cpu_count() // 2) != 0: 86 | num_threads = cpu_count() // 2 87 | elif size // (cpu_count() // 4) != 0: 88 | num_threads = cpu_count() // 4 89 | else: 90 | num_threads = 1 91 | batch_size = size // num_threads 92 | pool = Pool(processes=num_threads) 93 | for start in np.arange(0,size,batch_size): 94 | end = int(np.min([start+batch_size,size])) 95 | batch_coco_image_paths = coco_image_paths[start:end] 96 | batch_voc_image_paths = voc_image_paths[start:end] 97 | pool.apply_async(batch_image_copy,error_callback=print_error, 98 | args=(batch_voc_image_paths,batch_coco_image_paths)) 99 | pool.close() 100 | pool.join() 101 | print("结束复制图片") 102 | 103 | def xml2json(voc_image_dir,voc_annotation_dir,voc_txt_path,coco_image_dir,coco_json_path,class_names,image_cnt,annotation_cnt): 104 | """ 105 | 这是将VOC数据集的XML标签转换为COCO数据集JSON标签的函数 106 | Args: 107 | voc_image_dir: VOC数据集图像文件夹路径 108 | voc_annotation_dir: VOC数据集标签文件夹路径 109 | voc_txt_path: VOC数据集子集txt文件路径 110 | coco_image_dir: COCO数据集图像文件夹路径 111 | coco_json_path: COCO数据集JSON文件路径 112 | class_names: 目标分类名称数组 113 | image_cnt: 图片计数器 114 | annotation_cnt：标签计数器 115 | Returns: 116 | """ 117 | voc_dataset_dir,_ = os.path.split(voc_image_dir) 118 | _,voc_dataset_name = os.path.split(voc_dataset_dir) 119 | coco_dataset_dir,_ = os.path.split(coco_image_dir) 120 | _,coco_dataset_name = os.path.split(coco_dataset_dir) 121 | # 初始化相关文件路径 122 | voc_image_paths = [] 123 | voc_xml_paths = [] 124 | coco_image_paths = [] 125 | coco_image_ids = [] 126 | with open(voc_txt_path, 'r') as f: 127 | for line in f.readlines(): 128 | voc_image_paths.append(os.path.join(voc_image_dir, "{0}.jpg".format(line.strip()))) 129 | voc_xml_paths.append(os.path.join(voc_annotation_dir, "{0}.xml".format(line.strip()))) 130 | coco_image_paths.append(os.path.join(coco_image_dir, "{0}_{1:06d}.jpg".format(coco_dataset_name,image_cnt))) 131 | coco_image_ids.append(image_cnt) 132 | image_cnt += 1 133 | voc_image_paths = np.array(voc_image_paths) 134 | voc_xml_paths = np.array(voc_xml_paths) 135 | coco_image_paths = np.array(coco_image_paths) 136 | coco_image_ids = np.array(coco_image_ids) 137 | 138 | image_infos = [] 139 | detection_results = [] 140 | for i in tqdm(np.arange(len(voc_image_paths))): 141 | voc_xml_path = voc_xml_paths[i] 142 | coco_image_id = coco_image_ids[i] 143 | coco_image_path = coco_image_paths[i] 144 | _,coco_image_name = os.path.split(coco_image_path) 145 | if is_contain_object(voc_xml_path): 146 | objects,(h,w) = parse_xml(voc_xml_path) 147 | image_infos.append({'file_name': coco_image_name, 'id': coco_image_id,'width':w,'height':h}) 148 | for obj in objects: 149 | cls_name, xmin, ymin, xmax, ymax = obj 150 | w = xmax - xmin 151 | h = ymax - ymin 152 | detection_results.append({'image_id': coco_image_id, 153 | 'iscrowd': 0, 154 | 'bbox': [xmin,ymin, w, h], 155 | 'area': int(w * h), 156 | "category_id": class_names.index(cls_name), 157 | 'id': annotation_cnt}) 158 | annotation_cnt += 1 159 | gt_result = {} 160 | gt_result['images'] = image_infos 161 | gt_result["annotations"] = detection_results 162 | gt_result["categories"] = [{"id": id, "name": cls_name} for id,cls_name in enumerate(class_names)] 163 | gt_result_json_data = json.dumps(gt_result, indent=4, separators=(',', ': '), cls=NpEncoder) 164 | with open(coco_json_path, 'w+', encoding="utf-8") as f: 165 | f.write(gt_result_json_data) 166 | return voc_image_paths,coco_image_paths,image_cnt,annotation_cnt 167 | 168 | def is_contain_object(xml_path): 169 | """ 170 | 这是判断VOC数据集XML标签文件中是否包含目标的函数 171 | Args: 172 | xml_path: VOC数据集XML标签文件路径 173 | Returns: 174 | """ 175 | # 获取XML文件的根结点 176 | root = ET.parse(xml_path).getroot() 177 | return len(root.findall('object')) > 0 178 | 179 | def parse_xml(xml_path,class_names=None): 180 | """ 181 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 182 | Args: 183 | xml_path: XML标签文件路径 184 | class_names: 目标名称数组，默认为None 185 | Returns: 186 | """ 187 | # 获取XML文件的根结点 188 | root = ET.parse(xml_path).getroot() 189 | h = int(root.find("size").find("height").text) 190 | w = int(root.find("size").find("width").text) 191 | # 遍历所有目标 192 | objects = [] 193 | for obj in root.findall('object'): 194 | obj_name = obj.find('name').text 195 | bndbox = obj.find('bndbox') 196 | xmin = bndbox.find('xmin').text 197 | ymin = bndbox.find('ymin').text 198 | xmax = bndbox.find('xmax').text 199 | ymax = bndbox.find('ymax').text 200 | if class_names is None: 201 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 202 | else: 203 | if obj_name in class_names: 204 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 205 | return objects,(h,w) 206 | 207 | def single_image_copy(voc_image_path,coco_image_path): 208 | """ 209 | 这是单张图像复制函数 210 | Args: 211 | voc_image_path: VOC图像路径 212 | coco_image_path: COCO图像路径 213 | Returns: 214 | """ 215 | # 初始化VOC标签写类 216 | image = cv2.imread(voc_image_path) 217 | cv2.imwrite(coco_image_path, image) 218 | 219 | def batch_image_copy(batch_voc_image_paths,batch_coco_image_paths): 220 | """ 221 | 批量复制图像的函数 222 | Args: 223 | batch_voc_image_paths: 批量voc图像路径数组 224 | batch_coco_image_paths: 批量coco图像路径数组 225 | Returns: 226 | """ 227 | size = len(batch_voc_image_paths) 228 | for i in tqdm(np.arange(size)): 229 | coco_image_path = batch_coco_image_paths[i] 230 | voc_image_path = batch_voc_image_paths[i] 231 | single_image_copy(voc_image_path,coco_image_path) 232 | 233 | def print_error(value): 234 | """ 235 | 定义错误回调函数 236 | Args: 237 | value: 出错误值 238 | Returns: 239 | """ 240 | print("error: ", value) 241 | def get_classes(classes_path): 242 | """ 243 | 这是获取目标分类名称的函数 244 | Args: 245 | classes_path: 目标分类名称txt文件路径 246 | Returns: 247 | """ 248 | classes_names = [] 249 | with open(classes_path, 'r') as f: 250 | for line in f.readlines(): 251 | classes_names.append(line.strip()) 252 | return classes_names 253 | 254 | class NpEncoder(json.JSONEncoder): 255 | 256 | def default(self, obj): 257 | if isinstance(obj, np.integer): 258 | return int(obj) 259 | elif isinstance(obj, np.floating): 260 | return float(obj) 261 | elif isinstance(obj, np.ndarray): 262 | return list(obj) 263 | elif isinstance(obj,bytes): 264 | return str(obj,encoding='utf-8') 265 | else: 266 | return super(NpEncoder, self).default(obj) 267 | 268 | def run_main(): 269 | """ 270 | 这是主函数 271 | """ 272 | # voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/coco2017_voc") 273 | # coco_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/coco2017") 274 | # class_name_path = os.path.abspath("./coco_names.txt") 275 | # save_image = True 276 | # choices = ['train','val'] 277 | # class_names = get_classes(class_name_path) 278 | # voc2coco(voc_dataset_dir,coco_dataset_dir,class_names,save_image,choices) 279 | 280 | voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/origin/VOC2007") 281 | coco_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2007_coco") 282 | class_name_path = os.path.abspath("./voc_names.txt") 283 | save_image = True 284 | choices = ['train','val','test'] 285 | class_names = get_classes(class_name_path) 286 | voc2coco(voc_dataset_dir,coco_dataset_dir,class_names,save_image,choices) 287 | 288 | voc_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/origin/VOC2012") 289 | coco_dataset_dir = os.path.abspath("/home/dpw/deeplearning/dataset/voc/voc2012_coco") 290 | class_name_path = os.path.abspath("./voc_names.txt") 291 | save_image = True 292 | choices = ['train','val'] 293 | class_names = get_classes(class_name_path) 294 | voc2coco(voc_dataset_dir,coco_dataset_dir,class_names,save_image,choices) 295 | 296 | if __name__ == '__main__': 297 | run_main() 298 | -------------------------------------------------------------------------------- /voc2labelme.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2024/4/22 下午10:27 3 | # @Author : DaiPuWei 4 | # @Email : daipuwei@qq.com 5 | # @File : voc2labelme.py 6 | # @Software: PyCharm 7 | 8 | """ 9 | 这是将VOC数据集转换为Labelme数据集的脚本 10 | """ 11 | 12 | import os 13 | import json 14 | import shutil 15 | import labelme 16 | import numpy as np 17 | import xml.etree.ElementTree as ET 18 | from tqdm import tqdm 19 | from multiprocessing import Pool,cpu_count 20 | 21 | def voc2labelme(voc_dataset_dir,labelme_dataset_dir,choices=["train","val"]): 22 | """ 23 | 这是VOC数据集转Labelme数据集的函数 24 | Args: 25 | voc_dataset_dir: voc数据集地址 26 | labelme_dataset_dir: labelme数据集地址 27 | choices: 子集列表，默认为["train","val"] 28 | Returns: 29 | """ 30 | # 初始化VOC数据集相关路径 31 | voc_image_dir = os.path.join(voc_dataset_dir,"JPEGImages") 32 | if not os.path.exists(voc_image_dir): 33 | voc_image_dir = os.path.join(voc_dataset_dir,"images") 34 | voc_annotation_dir = os.path.join(voc_dataset_dir,"Annotations") 35 | voc_main_dir = os.path.join(voc_dataset_dir,"ImageSets","Main") 36 | 37 | # 初始化Labelme数据集相关路径 38 | _,labelme_dataset_name = os.path.split(labelme_dataset_dir) 39 | labelme_image_dir = os.path.join(labelme_dataset_dir,'images') 40 | if not os.path.exists(labelme_image_dir): 41 | os.makedirs(labelme_image_dir) 42 | 43 | # 初始化数据集文件路径数组 44 | voc_image_names = [] 45 | voc_image_paths = [] 46 | voc_annotation_paths = [] 47 | labelme_image_paths = [] 48 | labelme_json_paths = [] 49 | cnt = 0 50 | for image_name in os.listdir(voc_image_dir): 51 | fname,ext = os.path.splitext(image_name) 52 | voc_image_names.append(fname) 53 | voc_image_paths.append(os.path.join(voc_image_dir,image_name)) 54 | voc_annotation_paths.append(os.path.join(voc_annotation_dir,"{0}.xml".format(fname))) 55 | labelme_image_paths.append(os.path.join(labelme_image_dir,"{0}_{1:07d}.jpg".format(labelme_dataset_name, cnt))) 56 | labelme_json_paths.append(os.path.join(labelme_image_dir, "{0}_{1:07d}.json".format(labelme_dataset_name, cnt))) 57 | cnt += 1 58 | voc_image_paths = np.array(voc_image_paths) 59 | voc_annotation_paths = np.array(voc_annotation_paths) 60 | labelme_image_paths = np.array(labelme_image_paths) 61 | labelme_json_paths = np.array(labelme_json_paths) 62 | 63 | # 生成labelme子集列表 64 | print("开始生成Labelme子集列表") 65 | for choice in choices: 66 | voc_txt_path = os.path.join(voc_main_dir,"{}.txt".format(choice)) 67 | labelme_txt_path = os.path.join(labelme_dataset_dir,"{}.txt".format(choice)) 68 | with open(labelme_txt_path,'w',encoding='utf-8') as g: 69 | with open(voc_txt_path,'r',encoding='utf-8') as f: 70 | for line in tqdm(f.readlines()): 71 | voc_fname = line.strip() 72 | if voc_fname in voc_image_names: 73 | index = voc_image_names.index(voc_fname) 74 | _,labelme_image_name = os.path.split(labelme_image_paths[index]) 75 | labelme_fname,_ = os.path.splitext(labelme_image_name) 76 | g.write(labelme_fname+"\n") 77 | print("结束生成Labelme子集列表") 78 | 79 | print("开始多线程处理VOC图像及其标签并转换为Labelme格式") 80 | size = len(voc_image_paths) 81 | if size // cpu_count() != 0: 82 | num_threads = cpu_count() 83 | elif size // (cpu_count() // 2) != 0: 84 | num_threads = cpu_count() // 2 85 | elif size // (cpu_count() // 4) != 0: 86 | num_threads = cpu_count() // 4 87 | else: 88 | num_threads = 1 89 | batch_size = size // num_threads 90 | pool = Pool(processes=num_threads) 91 | for start in np.arange(0, size, batch_size): 92 | end = int(np.min([start + batch_size, size])) 93 | batch_voc_image_paths = voc_image_paths[start:end] 94 | batch_voc_annotation_paths = voc_annotation_paths[start:end] 95 | batch_labelme_image_paths = labelme_image_paths[start:end] 96 | batch_labelme_json_paths = labelme_json_paths[start:end] 97 | pool.apply_async(process_batch_images_labels, error_callback=print_error, 98 | args=(batch_voc_image_paths,batch_voc_annotation_paths, 99 | batch_labelme_image_paths,batch_labelme_json_paths)) 100 | pool.close() 101 | pool.join() 102 | print("结束多线程处理") 103 | 104 | def process_batch_images_labels(batch_voc_image_paths,batch_voc_annotation_paths,batch_labelme_image_paths,batch_labelme_json_paths): 105 | """ 106 | 这是将批量VOC数据集图像及其XML标签转换为labelme数据集格式的函数 107 | Args: 108 | batch_voc_image_paths: 批量VOC数据集图像文件路径数组 109 | batch_voc_annotation_paths: 批量VOC数据集XML标签文件路径数组 110 | batch_labelme_image_paths: 批量Labelme数据集图像文件路径数组 111 | batch_labelme_json_paths: 批量Labelme数据集Json标签文件路径数组 112 | Returns: 113 | """ 114 | size = len(batch_voc_image_paths) 115 | for i in tqdm(np.arange(size)): 116 | process_single_image_label(batch_voc_image_paths[i],batch_voc_annotation_paths[i], 117 | batch_labelme_image_paths[i],batch_labelme_json_paths[i]) 118 | def process_single_image_label(voc_image_path,voc_annotation_path,labelme_image_path,labelme_json_path): 119 | """ 120 | 这是将单张VOC数据集图像及其XML标签转换为labelme数据集格式的函数 121 | Args: 122 | voc_image_path: VOC数据集图像文件路径 123 | voc_annotation_path: VOC数据集XML标签文件路径 124 | labelme_image_path: Labelme数据集图像文件路径 125 | labelme_json_path: Labelme数据集Json标签文件路径 126 | Returns: 127 | """ 128 | # 复制图像 129 | _,labelme_image_name = os.path.split(labelme_image_path) 130 | shutil.copy(voc_image_path,labelme_image_path) 131 | 132 | # 解析xml文件,写入json 133 | objects,(h,w) = parse_xml(voc_annotation_path) 134 | 135 | # 将目标框写入json文件 136 | json_data = {"version": labelme.__version__, 137 | "flags": {}, 138 | "shapes": [], 139 | "imagePath": labelme_image_name, 140 | "imageData": None, 141 | "imageHeight": h, 142 | "imageWidth": w} 143 | for cls_name,x1,y1,x2,y2 in objects: 144 | json_data["shapes"].append( 145 | {"label": cls_name, 146 | "points": [[x1, y1], [x2, y2]], 147 | "group_id": None, 148 | "shape_type": "rectangle", 149 | "flags": {}} 150 | ) 151 | with open(labelme_json_path, 'w', encoding='utf-8') as f: 152 | json_data = json.dumps(json_data, indent=4, 153 | separators=(',', ': '), ensure_ascii=False) 154 | f.write(json_data) 155 | 156 | def parse_xml(xml_path,class_names=None): 157 | """ 158 | 这是解析VOC数据集XML标签文件，获取每个目标分类与定位的函数 159 | Args: 160 | xml_path: XML标签文件路径 161 | class_names: 目标名称数组，默认为None 162 | Returns: 163 | """ 164 | # 获取XML文件的根结点 165 | root = ET.parse(xml_path).getroot() 166 | h = int(root.find("size").find("height").text) 167 | w = int(root.find("size").find("width").text) 168 | # 遍历所有目标 169 | objects = [] 170 | for obj in root.findall('object'): 171 | obj_name = obj.find('name').text 172 | bndbox = obj.find('bndbox') 173 | xmin = bndbox.find('xmin').text 174 | ymin = bndbox.find('ymin').text 175 | xmax = bndbox.find('xmax').text 176 | ymax = bndbox.find('ymax').text 177 | if class_names is None: 178 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 179 | else: 180 | if obj_name in class_names: 181 | objects.append([obj_name, int(xmin), int(ymin), int(xmax), int(ymax)]) 182 | return objects,(h,w) 183 | 184 | 185 | def print_error(value): 186 | """ 187 | 定义错误回调函数 188 | Args: 189 | value: 出错误值 190 | Returns: 191 | """ 192 | print("error: ", value) 193 | 194 | 195 | 196 | def run_main(): 197 | """ 198 | 这是主函数 199 | """ 200 | # COCO2017 201 | voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc/coco2017" 202 | labelme_dataset_dir = "/home/dpw/deeplearning/dataset/labelme/coco2017" 203 | choices = ["train", "val"] 204 | voc2labelme(voc_dataset_dir,labelme_dataset_dir,choices) 205 | 206 | # VOC2007 207 | voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc/voc2007" 208 | labelme_dataset_dir = "/home/dpw/deeplearning/dataset/labelme/voc2007" 209 | choices = ["train", "val","test"] 210 | voc2labelme(voc_dataset_dir,labelme_dataset_dir,choices) 211 | 212 | # VOC2012 213 | voc_dataset_dir = "/home/dpw/deeplearning/dataset/voc/voc2012" 214 | labelme_dataset_dir = "/home/dpw/deeplearning/dataset/labelme/voc2012" 215 | choices = ["train", "val"] 216 | voc2labelme(voc_dataset_dir,labelme_dataset_dir,choices) 217 | 218 | 219 | if __name__ == '__main__': 220 | run_main() 221 | --------------------------------------------------------------------------------