├── code ├── poisson-image-editing │ ├── mask.png │ ├── 002389.jpg │ ├── 002395.jpg │ ├── target_mask.png │ ├── target_result.png │ ├── figs │ │ ├── demo │ │ │ ├── draw_mask.png │ │ │ └── move_mask.png │ │ └── example1 │ │ │ ├── all.png │ │ │ ├── mask1.png │ │ │ ├── source1.jpg │ │ │ ├── target1.jpg │ │ │ ├── possion1.png │ │ │ └── direct_merge1.png │ ├── ref_Poisson image editing.pdf │ ├── .gitignore │ ├── README.md │ ├── paint_mask.py │ ├── main.py │ ├── move_mask.py │ └── poisson_image_editing.py ├── Weighted-Boxes-Fusion │ ├── ensemble_boxes │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── ensemble_boxes_nms.cpython-37.pyc │ │ │ ├── ensemble_boxes_nmw.cpython-37.pyc │ │ │ └── ensemble_boxes_wbf.cpython-37.pyc │ │ ├── __init__.py │ │ ├── ensemble_boxes_nmw.py │ │ ├── ensemble_boxes_wbf.py │ │ └── ensemble_boxes_nms.py │ ├── setup.py │ ├── README.md │ ├── example.py │ └── example_oid.py ├── Readme.md ├── get_testjson.py ├── draw_pred_bbox.ipynb ├── mmdet │ ├── losses │ │ └── cross_entropy_loss.py │ └── datasets │ │ └── pipelines │ │ └── transforms.py ├── Retinex.py └── draw_bbox.ipynb ├── Readme.md ├── information ├── config.py └── logs ├── Logs.md └── AHF_EXPERIMENT.md /code/poisson-image-editing/mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/mask.png -------------------------------------------------------------------------------- /code/poisson-image-editing/002389.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/002389.jpg -------------------------------------------------------------------------------- /code/poisson-image-editing/002395.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/002395.jpg -------------------------------------------------------------------------------- /code/poisson-image-editing/target_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/target_mask.png -------------------------------------------------------------------------------- /code/poisson-image-editing/target_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/target_result.png -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/demo/draw_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/demo/draw_mask.png -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/demo/move_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/demo/move_mask.png -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/example1/all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/example1/all.png -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/example1/mask1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/example1/mask1.png -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/example1/source1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/example1/source1.jpg -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/example1/target1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/example1/target1.jpg -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/example1/possion1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/example1/possion1.png -------------------------------------------------------------------------------- /code/poisson-image-editing/ref_Poisson image editing.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/ref_Poisson image editing.pdf -------------------------------------------------------------------------------- /code/poisson-image-editing/figs/example1/direct_merge1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/poisson-image-editing/figs/example1/direct_merge1.png -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/ensemble_boxes_nms.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/ensemble_boxes_nms.cpython-37.pyc -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/ensemble_boxes_nmw.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/ensemble_boxes_nmw.cpython-37.pyc -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/ensemble_boxes_wbf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zml-ai/Underwater_detection/HEAD/code/Weighted-Boxes-Fusion/ensemble_boxes/__pycache__/ensemble_boxes_wbf.cpython-37.pyc -------------------------------------------------------------------------------- /code/poisson-image-editing/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # compiled output 3 | *.pyc 4 | build/ 5 | dist/ 6 | dist-server/ 7 | 8 | # IDE and system 9 | .vs/ 10 | .idea 11 | .vscode 12 | *.suo 13 | *.ntvs* 14 | *.njsproj 15 | *.sln 16 | *.sw* 17 | *.DS_Store 18 | .env/ 19 | .env.local 20 | .env.*.local 21 | *.pptx 22 | 23 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | from .ensemble_boxes_wbf import weighted_boxes_fusion 5 | from .ensemble_boxes_nmw import non_maximum_weighted 6 | from .ensemble_boxes_nms import nms_method 7 | from .ensemble_boxes_nms import nms 8 | from .ensemble_boxes_nms import soft_nms 9 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/setup.py: -------------------------------------------------------------------------------- 1 | try: 2 | from setuptools import setup 3 | except: 4 | from distutils.core import setup 5 | 6 | setup( 7 | name='ensemble_boxes', 8 | version='1.0.0', 9 | author='Roman Solovyev (ZFTurbo)', 10 | packages=['ensemble_boxes', ], 11 | url='https://github.com/ZFTurbo/Weighted-Boxes-Fusion', 12 | license='MIT License', 13 | description='Python implementation of several methods for ensembling boxes from object detection models: Non-maximum Suppression (NMS), Soft-NMS, Non-maximum weighted (NMW), Weighted boxes fusion (WBF)', 14 | long_description='Python implementation of several methods for ensembling boxes from object detection models: Non-maximum Suppression (NMS), Soft-NMS, Non-maximum weighted (NMW), Weighted boxes fusion (WBF)' 15 | 'More details: https://github.com/ZFTurbo/Weighted-Boxes-Fusion', 16 | install_requires=[ 17 | "numpy", 18 | "pandas", 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /code/poisson-image-editing/README.md: -------------------------------------------------------------------------------- 1 | # Poisson Image Editing 2 | 3 | This is a Python implementation of the [poisson image editing](https://www.cs.virginia.edu/~connelly/class/2014/comp_photo/proj2/poisson.pdf) paper to seamlessly blend two images. 4 | 5 | ![](figs/example1/all.png) 6 | 7 | For example, I'd like to put the Statue of Liberty under the shinning ocean. The right side compares the results from direct copying and the Poisson image editing. 8 | 9 | In this [notebook](https://github.com/PPPW/poisson-image-editing/tree/master/poisson_image_editing.ipynb), I build the algorithm from scratch and explain each steps in detail. The code used in the notebook is extracted in the `*.py` files, you can also run it directly (see the next section for more instructions about how to run). 10 | 11 | ## How to run 12 | 13 | ``` 14 | python main.py -s -t [-m ] 15 | ``` 16 | 17 | If the mask image is not specified, a window will pop up for you to draw the mask on the source image: 18 | 19 | ![](figs/demo/draw_mask.png) 20 | 21 | The green region will be used as the mask. Press `s` to save the result, press `r` to reset. 22 | 23 | After the mask is defined, a window will pop up for you to adjust the mask position on the target image: 24 | 25 | ![](figs/demo/move_mask.png) 26 | 27 | The mask corresponds to the region of source image that will be copied, you can move the mask to put the copied part into desired location in the target image. Press `s` to save the result, press `r` to reset. 28 | 29 | Then the Poisson image editing process will start. The blended image will be named as `target_result.png`, in the same directory as the source image. 30 | 31 | 32 | ## Structure 33 | Here's a brief description of each file's functionality: 34 | 35 | * `main.py`: take command line argument and call `paint_mask.py`, `move_mask.py` and `poisson_image_editing.py`. 36 | 37 | * `paint_mask.py`: pop up a window for drawing the mask on the source image. 38 | 39 | * `move_mask.py`: pop up a window for adjusting the mask location on the target image. 40 | 41 | * `poisson_image.editing.py`: take source, target, mask image and mask offset, run the Poisson image editing algorithm for seamless image blending. 42 | 43 | * `poisson_image.editing.ipynb`: a notebook demonstrating the process. 44 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # 代码说明 2 | 3 | + 数据分析：data_analysis.ipynb 4 | 5 | + 获取test的json：get_testjson.py 6 | 7 | + Retinex与加速测试代码：Retinex.py 8 | 9 | 已集成在mmdet的transform里，此处仅为测试用 10 | 11 | + WBF: Weighted-Boxes-Fusion/ensemble.ipynb 12 | 13 | + 泊松融合：poisson-image-editing和poisson_blending.ipynb 14 | 15 | + 画标注框到训练集图上：draw_bbox.ipynb 16 | 17 | + 画预测框到测试集图上：draw_pred_bbox.ipynb 18 | 19 | + 实例平衡增强：instance_balanced_augmentation.ipynb 20 | 21 | + 动态模糊可视化样本图代码：Motion_Blurring.ipynb 22 | 23 | + 动态模糊，Mixup和Retinex加入mmdetection后的代码：mmdet/datasets/pipelines/transforms.py，使用时请自行在__init__.py下加入它们的名称。 24 | 25 | + 配置文件按照需求修改以下内容： 26 | 27 | ```python 28 | train_pipeline = [ 29 | dict(type='LoadImageFromFile'), 30 | dict(type='LoadAnnotations', with_bbox=True), 31 | dict(type='Mixup', prob=0.5, lambd=0.8, mixup=True, 32 | json_path='data/seacoco/train_waterweeds.json', 33 | img_path='data/seacoco/train/'), 34 | dict(type='MotionBlur', p=0.3), 35 | dict(type='Resize', img_scale=[(4096, 600), (4096, 1000)], 36 | multiscale_mode='range', keep_ratio=True), 37 | dict(type='Retinex', model='MSR', sigma=[30, 150, 300], 38 | restore_factor=2.0, color_gain=6.0, gain=128.0, offset=128.0), 39 | dict(type='RandomFlip', flip_ratio=0.5), 40 | dict(type='Pad', size_divisor=32), 41 | dict(type='Albu', 42 | transforms=albu_train_transforms, 43 | bbox_params=dict(type='BboxParams', 44 | format='pascal_voc', 45 | label_fields=['gt_labels'], 46 | min_visibility=0.0, 47 | filter_lost_elements=True), 48 | keymap={ 49 | 'img': 'image', 50 | 'gt_bboxes': 'bboxes' 51 | }, 52 | update_pad_shape=False, 53 | skip_img_without_anno=True), 54 | dict(type='Normalize', **img_norm_cfg), 55 | dict(type='DefaultFormatBundle'), 56 | dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), 57 | ] 58 | ``` 59 | 60 | + 标签平滑：mmdet/losses/cross_entropy_loss.py加入了标签平滑的方法。在配置文件需要修改（rpn_head下的CEloss的平滑指数为0.0，bbox_head下的CEloss的平滑指数>0.0即可）如下： 61 | 62 | ```python 63 | oss_cls=dict( type='CrossEntropyLoss', 64 | use_sigmoid=False, 65 | loss_weight=1.0， 66 | smoothing=0.001) 67 | ``` 68 | 69 | -------------------------------------------------------------------------------- /code/Readme.md: -------------------------------------------------------------------------------- 1 | # 代码说明 2 | 3 | + 数据分析：data_analysis.ipynb 4 | 5 | + 获取test的json：get_testjson.py 6 | 7 | + Retinex与加速测试代码：Retinex.py 8 | 9 | 已集成在mmdet的transform里，此处仅为测试用 10 | 11 | + WBF: Weighted-Boxes-Fusion/ensemble.ipynb 12 | 13 | + 泊松融合：poisson-image-editing和poisson_blending.ipynb 14 | 15 | + 画标注框到训练集图上：draw_bbox.ipynb 16 | 17 | + 画预测框到测试集图上：draw_pred_bbox.ipynb 18 | 19 | + 实例平衡增强：instance_balanced_augmentation.ipynb 20 | 21 | + 动态模糊可视化样本图代码：Motion_Blurring.ipynb 22 | 23 | + 动态模糊，Mixup和Retinex加入mmdetection后的代码：mmdet/datasets/pipelines/transforms.py，使用时请自行在__init__.py下加入它们的名称。 24 | 25 | + 配置文件按照需求修改以下内容： 26 | 27 | ```python 28 | train_pipeline = [ 29 | dict(type='LoadImageFromFile'), 30 | dict(type='LoadAnnotations', with_bbox=True), 31 | dict(type='Mixup', prob=0.5, lambd=0.8, mixup=True, 32 | json_path='data/seacoco/train_waterweeds.json', 33 | img_path='data/seacoco/train/'), 34 | dict(type='MotionBlur', p=0.3), 35 | dict(type='Resize', img_scale=[(4096, 600), (4096, 1000)], 36 | multiscale_mode='range', keep_ratio=True), 37 | dict(type='Retinex', model='MSR', sigma=[30, 150, 300], 38 | restore_factor=2.0, color_gain=6.0, gain=128.0, offset=128.0), 39 | dict(type='RandomFlip', flip_ratio=0.5), 40 | dict(type='Pad', size_divisor=32), 41 | dict(type='Albu', 42 | transforms=albu_train_transforms, 43 | bbox_params=dict(type='BboxParams', 44 | format='pascal_voc', 45 | label_fields=['gt_labels'], 46 | min_visibility=0.0, 47 | filter_lost_elements=True), 48 | keymap={ 49 | 'img': 'image', 50 | 'gt_bboxes': 'bboxes' 51 | }, 52 | update_pad_shape=False, 53 | skip_img_without_anno=True), 54 | dict(type='Normalize', **img_norm_cfg), 55 | dict(type='DefaultFormatBundle'), 56 | dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), 57 | ] 58 | ``` 59 | 60 | + 标签平滑：mmdet/losses/cross_entropy_loss.py加入了标签平滑的方法。在配置文件需要修改（rpn_head下的CEloss的平滑指数为0.0，bbox_head下的CEloss的平滑指数>0.0即可）如下： 61 | 62 | ```python 63 | oss_cls=dict( type='CrossEntropyLoss', 64 | use_sigmoid=False, 65 | loss_weight=1.0， 66 | smoothing=0.001) 67 | ``` 68 | 69 | -------------------------------------------------------------------------------- /code/poisson-image-editing/paint_mask.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import cv2 4 | from os import path 5 | 6 | 7 | class MaskPainter(): 8 | def __init__(self, image_path): 9 | self.image_path = image_path 10 | self.image = cv2.imread(image_path) 11 | self.image_copy = self.image.copy() 12 | 13 | self.mask = np.zeros(self.image.shape) 14 | self.mask_copy = self.mask.copy() 15 | self.size = 4 16 | self.to_draw = False 17 | 18 | self.window_name = "Draw mask. s:save; r:reset; q:quit" 19 | 20 | 21 | def _paint_mask_handler(self, event, x, y, flags, param): 22 | if event == cv2.EVENT_LBUTTONDOWN: 23 | self.to_draw = True 24 | 25 | elif event == cv2.EVENT_MOUSEMOVE: 26 | if self.to_draw: 27 | cv2.rectangle(self.image, (x-self.size, y-self.size), 28 | (x+self.size, y+self.size), 29 | (0, 255, 0), -1) 30 | cv2.rectangle(self.mask, (x-self.size, y-self.size), 31 | (x+self.size, y+self.size), 32 | (255, 255, 255), -1) 33 | cv2.imshow(self.window_name, self.image) 34 | 35 | elif event == cv2.EVENT_LBUTTONUP: 36 | self.to_draw = False 37 | 38 | 39 | def paint_mask(self): 40 | cv2.namedWindow(self.window_name) 41 | cv2.setMouseCallback(self.window_name, 42 | self._paint_mask_handler) 43 | 44 | while True: 45 | cv2.imshow(self.window_name, self.image) 46 | key = cv2.waitKey(1) & 0xFF 47 | 48 | if key == ord("r"): 49 | self.image = self.image_copy.copy() 50 | self.mask = self.mask_copy.copy() 51 | 52 | elif key == ord("s"): 53 | break 54 | 55 | elif key == ord("q"): 56 | cv2.destroyAllWindows() 57 | exit() 58 | 59 | roi = self.mask 60 | cv2.imshow("Press any key to save the mask", roi) 61 | cv2.waitKey(0) 62 | maskPath = path.join(path.dirname(self.image_path), 63 | 'mask.png') 64 | cv2.imwrite(maskPath, self.mask) 65 | 66 | # close all open windows 67 | cv2.destroyAllWindows() 68 | return maskPath 69 | 70 | 71 | if __name__ == '__main__': 72 | ap = argparse.ArgumentParser() 73 | ap.add_argument("-i", "--image", required=True, help="Path to the image") 74 | args = vars(ap.parse_args()) 75 | 76 | mp = MaskPainter(args["image"]) 77 | mp.paint_mask() 78 | -------------------------------------------------------------------------------- /information: -------------------------------------------------------------------------------- 1 | 赛事基本信息 2 | [toc] 3 | 4 | 赛题链接：https://www.kesci.com/home/competition/5e535a612537a0002ca864ac/content/2 5 | 6 | 参赛成员（排名不分先后）： 7 | 8 | 李智敏 华中科技大学 研一 9 | 10 | 罗文斌 电子科技大学 研一 11 | 12 | 艾宏峰 曼切斯特大学 应届研究生 13 | 14 | 赛事背景 15 | “水下目标检测算法赛”属于2020年全国水下机器人（湛江）大赛的线上赛，本次算法赛由国家自然科学基金委员会、鹏城实验室和湛江市人民政府主办，将面向国内及海外各院校和科研机构、科技企业招募优秀选手，深化和拓宽水下机器人和水下目标检测领域的相关研究，推进算法技术向实际产业应用进行赋能。 16 | 17 | 算法赛日程 18 | 报名阶段：2020-2-25 (12:00:00 中午） 至 2020-4-10 (12:00:00 中午） 19 | 20 | 初赛阶段：2020-2-28 (12:00:00 中午） 至 2020-4-11 (12:00:00 中午） 21 | 22 | 综合评审：2020-4-13 (12:00:00 中午） 至 2020-4-17 (12:00:00 中午） 23 | 24 | 线上答辩：4 月 21 日 - 4 月 22 日（暂定） 25 | 26 | 赛题 27 | 初赛任务：在真实海底图片数据中检测出不同海产品（海参、海胆、扇贝、海星）的位置 28 | 线上答辩：采取网上远程答辩，形式包括PPT、视频等进行展示。由评委提问、打分决定最终排名。 29 | 数据描述 30 | 1、初赛训练集： 31 | 32 | 初赛训练集：提供5543幅训练图像（含人工标注真值数据），数据集结构如下， 33 | 34 | 一级目录 二级目录 解释说明 格式 35 | train/ image/ 所有训练图片 jpg 36 | box/ 同名图像文件的对应标注结果 xml 37 | 其中train/image文件夹中包含所有训练数据，这些图片之间不存在帧间连续性。 38 | 39 | 图片路径示例如下：train/image/000001.jpg，其对应的目标检测标注真值位于路径 train/box/000001.xml文件中，该文件包含了对应图像中所有物体的类别以及目标框参数（位置和尺寸）。 40 | 41 | 本届比赛需检测的目标类别包括海参“holothurian”，海胆“echinus”，扇贝**“scallop”和海星“starfish”四类。训练数据真值中可能存在水草“waterweeds”**这一类别，请忽略这一类。比赛不限制参赛队伍使用其他来源的数据进行预训练/训练。 42 | 43 | 2、初赛测试集： 44 | 45 | A榜测试集：800幅测试图像，详见数据下载 46 | 47 | B榜测试集：1200幅测试图像（与A榜不重复），详见数据下载 48 | 49 | 评审规则 50 | 一、初赛评审 51 | 1、客观评审 52 | 1）精度评审 53 | 评测方法： 54 | 55 | 初赛阶段的评审采用COCO mAP[@0.5:0.05:0.95] 指标（mean Average Precision) 进行计算，即将10个不同IOU阈值下的mAP取平均值作为最终结果 56 | 57 | 对于任意一IOU阈值，其对应的mAP计算公式如下： 58 | 59 | img 60 | 61 | img 62 | 63 | img 64 | 65 | p(r) 为当召回率（recall）为r时，检测结果的准确率(precision) 66 | 67 | mAP指标的基本介绍 68 | 69 | AP (Average Precision) 的计算方式采用PASCAL VOC 2010年之后的版本，关于该版本的mAP细节和原理参考The PASCAL Visual Object Classes (VOC) Challenge 70 | 71 | 本次客观评审的测评算法的实现参考Facebook Research Detectron（注意该IOU阈值默认为0.5） 72 | 73 | 测评指标的关键参数： 74 | 75 | IOU 阈值：0.5 + 0.05*i (i = 0,1,2...9) 76 | 评审说明： 77 | 78 | 初赛排行榜采用 A/B 榜机制，即在初赛第一阶段（ A 榜：2020-2-28 (12:00:00 中午） 至 2020-4-10 (12:00:00 中午）），每个队伍拥有每天 1 次提交与测评排名的机会，大赛页面实时更新排行榜，从高到低排序。 79 | 80 | 在初赛第二阶段（ B榜: 2020-4-10（12:00:01 中午） 至 2020-4-11（12:00:00 中午）），每个队伍拥有共计 2 次提交与测评排名的机会，精度评审阶段的最终有效成绩与有效排名将以第二阶段（B榜）排行榜为准。 81 | 82 | 2）精度和速度综合评审 83 | ​ 评审说明： 84 | 85 | ​ B榜排行榜前【20】支团队需要在初赛结束后的4天内，在组委会提供的云计算平台上部署测试代码（包括训练好的模型），组委会负责确认该测试代码的精度和速度。初赛综合排行榜将根据代码的精度和速度计算综合得分，此榜的成绩与排名将作为团队的初赛最终成绩。 86 | 87 | *备注：综合测评”精度和速度“的具体规则将于3月中旬左右发布。 88 | 89 | ​ 云计算平台部署指南将于初赛结束后12小时内发布。 90 | 91 | 2、主观评审 92 | B榜排行榜前【20】支团队需要在初赛结束后 24 小时内，在指定的提交入口提交审核材料，审核材料有K-Lab Notebook（含有算法思路、亮点解读、建模算力与环境、使用的预训练模型相关论文及模型下载链接的说明文档）、参赛项目 Github 链接，并配合提交其他技术委员会要求追加的材料。 93 | 94 | 主观评审结果决定初赛综合排行榜的成绩是否有效，初赛综合排行榜前【15】支团队将晋级线上答辩环节。 95 | 96 | 二、线上答辩 97 | 评审说明：采取网上远程答辩，形式包括 PPT、视频等进行展示。由评委提问、打分决定最终排名。 98 | 99 | 三、算法赛总成绩 100 | 评测方法：总成绩 = 0.7 * 初赛成绩 + 0.3 * 答辩成绩 101 | -------------------------------------------------------------------------------- /code/poisson-image-editing/main.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | 3 | from paint_mask import MaskPainter 4 | from move_mask import MaskMover 5 | from poisson_image_editing import poisson_edit 6 | 7 | #import argparse 8 | import getopt 9 | import sys 10 | from os import path 11 | 12 | 13 | def usage(): 14 | print("Usage: python main.py [options] \n\n\ 15 | Options: \n\ 16 | \t-h\tPrint a brief help message and exits..\n\ 17 | \t-s\t(Required) Specify a source image.\n\ 18 | \t-t\t(Required) Specify a target image.\n\ 19 | \t-m\t(Optional) Specify a mask image with the object in white and other part in black, ignore this option if you plan to draw it later.") 20 | 21 | 22 | if __name__ == '__main__': 23 | # parse command line arguments 24 | args = {} 25 | 26 | try: 27 | opts, _ = getopt.getopt(sys.argv[1:], "hs:t:m:p:") 28 | except getopt.GetoptError as err: 29 | # print help information and exit: 30 | print(err) # will print something like "option -a not recognized" 31 | print("See help: main.py -h") 32 | exit(2) 33 | for o, a in opts: 34 | if o in ("-h"): 35 | usage() 36 | exit() 37 | elif o in ("-s"): 38 | args["source"] = a 39 | elif o in ("-t"): 40 | args["target"] = a 41 | elif o in ("-m"): 42 | args["mask"] = a 43 | else: 44 | assert False, "unhandled option" 45 | 46 | # 47 | if ("source" not in args) or ("target" not in args): 48 | usage() 49 | exit() 50 | 51 | # 52 | source = cv2.imread(args["source"]) 53 | target = cv2.imread(args["target"]) 54 | 55 | if source is None or target is None: 56 | print('Source or target image not exist.') 57 | exit() 58 | 59 | if source.shape[0] > target.shape[0] or source.shape[1] > target.shape[1]: 60 | print('Source image cannot be larger than target image.') 61 | exit() 62 | 63 | # draw the mask 64 | mask_path = "" 65 | if "mask" not in args: 66 | print('Please highlight the object to disapparate.\n') 67 | mp = MaskPainter(args["source"]) 68 | mask_path = mp.paint_mask() 69 | else: 70 | mask_path = args["mask"] 71 | 72 | # adjust mask position for target image 73 | print('Please move the object to desired location to apparate.\n') 74 | mm = MaskMover(args["target"], mask_path) 75 | offset_x, offset_y, target_mask_path = mm.move_mask() 76 | 77 | # blend 78 | print('Blending ...') 79 | target_mask = cv2.imread(target_mask_path, cv2.IMREAD_GRAYSCALE) 80 | offset = offset_x, offset_y 81 | 82 | poisson_blend_result = poisson_edit(source, target, target_mask, offset) 83 | 84 | cv2.imwrite(path.join(path.dirname(args["source"]), 'target_result.png'), 85 | poisson_blend_result) 86 | 87 | print('Done.\n') 88 | -------------------------------------------------------------------------------- /code/get_testjson.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import cv2 4 | import mmcv 5 | from tqdm import tqdm 6 | 7 | # Refernce = r'F:/study/0_Project/sea_detection/mmdet_luo/data/train_new.json' 8 | # with open(Refernce,'r') as load_f: 9 | # f = json.load(load_f) 10 | # 11 | # a=f["images"] 12 | # b=f["annotations"] 13 | # # license=f["license"] 14 | # # info=f["info"] 15 | # categories=f["categories"] 16 | # categories= sorted(categories, key=lambda d:d['id'], reverse = False) 17 | # del categories[0] 18 | # print(categories) 19 | # 根路径，里面包含images(图片文件夹)，annos.txt(bbox标注)，classes.txt(类别标签),以及annotations文件夹(如果没有则会自动创建，用于保存最后的json) 20 | root_path = r'F:\\study\\0_Project\\sea_detection\\2020dataset\\test-B-image' 21 | # 用于创建训练集或验证集 22 | phase = 'test' 23 | 24 | 25 | # 读取images文件夹的图片名称 26 | indexes = [f for f in os.listdir(os.path.join(root_path, 'image'))] 27 | 28 | 29 | img=[] 30 | img1=[] 31 | for k, index in enumerate(tqdm(indexes)): 32 | # 用opencv读取图片，得到图像的宽和高 33 | im = cv2.imread(os.path.join(root_path, 'image/') + index) 34 | 35 | height,width,dim = im.shape 36 | 37 | # 添加图像的信息到dataset中 38 | img.append({'file_name': index, 39 | 'id': k+1, 40 | 'width': width, 41 | 'height': height}) 42 | # if width < 500: 43 | # img.append({'file_name': index, 44 | # 'id': k, 45 | # 'width': width, 46 | # 'height': height}) 47 | # else: 48 | # img1.append({'file_name': index, 49 | # 'id': k, 50 | # 'width': width, 51 | # 'height': height}) 52 | # print(len(img)) 53 | print(len(img)) 54 | 55 | categories1=[{"supercategory": "holothurian", "id": 1, "name": "holothurian"}, 56 | {"supercategory": "echinus", "id": 2, "name": "echinus"}, 57 | {"supercategory": "scallop", "id": 3, "name": "scallop"}, 58 | {"supercategory": "starfish", "id": 4, "name": "starfish"}, 59 | {"supercategory": "waterweeds", "id": 5, "name": "waterweeds"}, 60 | ] 61 | # categories1= sorted(categories1, key=lambda d:d['id'], reverse = False) 62 | print(categories1) 63 | # categories2=[{"supercategory": "\u6807\u8d34\u6c14\u6ce1", "id": 3, "name": "\u6807\u8d34\u6c14\u6ce1"}, 64 | # {"supercategory": "\u6807\u8d34\u6b6a\u659c", "id": 1, "name": "\u6807\u8d34\u6b6a\u659c"}, 65 | # {"supercategory": "\u6807\u8d34\u8d77\u76b1", "id": 2, "name": "\u6807\u8d34\u8d77\u76b1"}] 66 | # categories2= sorted(categories2, key=lambda d:d['id'], reverse = False) 67 | # print(categories2) 68 | seatestdataset={ 69 | "images": img, 70 | "annotations": [], 71 | "categories": categories1 72 | } 73 | # ptestdataset={ 74 | # "info": info, 75 | # "licenses": license, 76 | # "images":img1, 77 | # "annotations": [], 78 | # "categories": categories2 79 | # } 80 | 81 | with open(r"F:/study/0_Project/sea_detection/mmdet_luo/data/testB.json", 'w') as f: 82 | json.dump(seatestdataset, f) 83 | # with open(r"D:\chongqing1_round1_testA_20191223\ptest2017.json", 'w') as f: 84 | # json.dump(ptestdataset, f) 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /code/poisson-image-editing/move_mask.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import cv2 4 | from os import path 5 | 6 | 7 | class MaskMover(): 8 | def __init__(self, image_path, mask_path): 9 | self.image_path, self.mask_path = image_path, mask_path 10 | self.image = cv2.imread(image_path) 11 | self.image_copy = self.image.copy() 12 | 13 | self.original_mask = cv2.imread(mask_path) 14 | self.original_mask_copy = np.zeros(self.image.shape) 15 | self.original_mask_copy[np.where(self.original_mask!=0)] = 255 16 | 17 | self.mask = self.original_mask_copy.copy() 18 | 19 | self.to_move = False 20 | self.x0 = 0 21 | self.y0 = 0 22 | self.is_first = True 23 | self.xi = 0 24 | self.yi = 0 25 | 26 | self.window_name = "Move the mask. s:save; r:reset; q:quit" 27 | 28 | 29 | def _blend(self, image, mask): 30 | ret = image.copy() 31 | alpha = 0.3 32 | ret[mask != 0] = ret[mask != 0]*alpha + 255*(1-alpha) 33 | return ret.astype(np.uint8) 34 | 35 | 36 | def _move_mask_handler(self, event, x, y, flags, param): 37 | if event == cv2.EVENT_LBUTTONDOWN: 38 | self.to_move = True 39 | if self.is_first: 40 | self.x0, self.y0 = x, y 41 | self.is_first = False 42 | 43 | self.xi, self.yi = x, y 44 | 45 | elif event == cv2.EVENT_MOUSEMOVE: 46 | if self.to_move: 47 | M = np.float32([[1,0,x-self.xi], 48 | [0,1,y-self.yi]]) 49 | self.mask = cv2.warpAffine(self.mask,M, 50 | (self.mask.shape[1], 51 | self.mask.shape[0])) 52 | cv2.imshow(self.window_name, 53 | self._blend(self.image, self.mask)) 54 | self.xi, self.yi = x, y 55 | 56 | elif event == cv2.EVENT_LBUTTONUP: 57 | self.to_move = False 58 | 59 | 60 | def move_mask(self): 61 | cv2.namedWindow(self.window_name) 62 | cv2.setMouseCallback(self.window_name, 63 | self._move_mask_handler) 64 | 65 | while True: 66 | cv2.imshow(self.window_name, 67 | self._blend(self.image, self.mask)) 68 | key = cv2.waitKey(1) & 0xFF 69 | 70 | if key == ord("r"): 71 | self.image = self.image_copy.copy() 72 | self.mask = self.original_mask_copy.copy() 73 | 74 | elif key == ord("s"): 75 | break 76 | 77 | elif key == ord("q"): 78 | cv2.destroyAllWindows() 79 | exit() 80 | 81 | roi = self.mask 82 | cv2.imshow("Press any key to save the mask", roi) 83 | cv2.waitKey(0) 84 | new_mask_path = path.join(path.dirname(self.image_path), 85 | 'target_mask.png') 86 | cv2.imwrite(new_mask_path, self.mask) 87 | 88 | # close all open windows 89 | cv2.destroyAllWindows() 90 | return self.xi-self.x0, self.yi-self.y0, new_mask_path 91 | 92 | 93 | if __name__ == '__main__': 94 | ap = argparse.ArgumentParser() 95 | ap.add_argument("-i", "--image", required=True, help="Path to the image") 96 | ap.add_argument("-m", "--mask", required=True, help="Path to the mask") 97 | args = vars(ap.parse_args()) 98 | 99 | mm = MaskMover(args["image"], args["mask"]) 100 | offset_x, offset_y, _ = mm.move_mask() 101 | print(offset_x, offset_y) 102 | -------------------------------------------------------------------------------- /code/poisson-image-editing/poisson_image_editing.py: -------------------------------------------------------------------------------- 1 | """Poisson image editing. 2 | 3 | """ 4 | 5 | import numpy as np 6 | import cv2 7 | import scipy.sparse 8 | from scipy.sparse.linalg import spsolve 9 | 10 | from os import path 11 | 12 | def laplacian_matrix(n, m): 13 | """Generate the Poisson matrix. 14 | 15 | Refer to: 16 | https://en.wikipedia.org/wiki/Discrete_Poisson_equation 17 | 18 | Note: it's the transpose of the wiki's matrix 19 | """ 20 | mat_D = scipy.sparse.lil_matrix((m, m)) 21 | mat_D.setdiag(-1, -1) 22 | mat_D.setdiag(4) 23 | mat_D.setdiag(-1, 1) 24 | 25 | mat_A = scipy.sparse.block_diag([mat_D] * n).tolil() 26 | 27 | mat_A.setdiag(-1, 1*m) 28 | mat_A.setdiag(-1, -1*m) 29 | 30 | return mat_A 31 | 32 | 33 | def poisson_edit(source, target, mask, offset): 34 | """The poisson blending function. 35 | 36 | Refer to: 37 | Perez et. al., "Poisson Image Editing", 2003. 38 | """ 39 | 40 | # Assume: 41 | # target is not smaller than source. 42 | # shape of mask is same as shape of target. 43 | y_max, x_max = target.shape[:-1] 44 | y_min, x_min = 0, 0 45 | 46 | x_range = x_max - x_min 47 | y_range = y_max - y_min 48 | 49 | M = np.float32([[1,0,offset[0]],[0,1,offset[1]]]) 50 | source = cv2.warpAffine(source,M,(x_range,y_range)) 51 | 52 | mask = mask[y_min:y_max, x_min:x_max] 53 | mask[mask != 0] = 1 54 | #mask = cv2.threshold(mask, 127, 1, cv2.THRESH_BINARY) 55 | 56 | mat_A = laplacian_matrix(y_range, x_range) 57 | 58 | # for \Delta g 59 | laplacian = mat_A.tocsc() 60 | 61 | # set the region outside the mask to identity 62 | for y in range(1, y_range - 1): 63 | for x in range(1, x_range - 1): 64 | if mask[y, x] == 0: 65 | k = x + y * x_range 66 | mat_A[k, k] = 1 67 | mat_A[k, k + 1] = 0 68 | mat_A[k, k - 1] = 0 69 | mat_A[k, k + x_range] = 0 70 | mat_A[k, k - x_range] = 0 71 | 72 | # corners 73 | # mask[0, 0] 74 | # mask[0, y_range-1] 75 | # mask[x_range-1, 0] 76 | # mask[x_range-1, y_range-1] 77 | 78 | mat_A = mat_A.tocsc() 79 | 80 | mask_flat = mask.flatten() 81 | for channel in range(source.shape[2]): 82 | source_flat = source[y_min:y_max, x_min:x_max, channel].flatten() 83 | target_flat = target[y_min:y_max, x_min:x_max, channel].flatten() 84 | 85 | #concat = source_flat*mask_flat + target_flat*(1-mask_flat) 86 | 87 | # inside the mask: 88 | # \Delta f = div v = \Delta g 89 | alpha = 1 90 | mat_b = laplacian.dot(source_flat)*alpha 91 | 92 | # outside the mask: 93 | # f = t 94 | mat_b[mask_flat==0] = target_flat[mask_flat==0] 95 | 96 | x = spsolve(mat_A, mat_b) 97 | #print(x.shape) 98 | x = x.reshape((y_range, x_range)) 99 | #print(x.shape) 100 | x[x > 255] = 255 101 | x[x < 0] = 0 102 | x = x.astype('uint8') 103 | #x = cv2.normalize(x, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) 104 | #print(x.shape) 105 | 106 | target[y_min:y_max, x_min:x_max, channel] = x 107 | 108 | return target 109 | 110 | def main(): 111 | scr_dir = 'figs/example1' 112 | out_dir = scr_dir 113 | source = cv2.imread(path.join(scr_dir, "source1.jpg")) 114 | target = cv2.imread(path.join(scr_dir, "target1.jpg")) 115 | mask = cv2.imread(path.join(scr_dir, "mask1.png"), 116 | cv2.IMREAD_GRAYSCALE) 117 | offset = (0,66) 118 | result = poisson_edit(source, target, mask, offset) 119 | 120 | cv2.imwrite(path.join(out_dir, "possion1.png"), result) 121 | 122 | 123 | if __name__ == '__main__': 124 | main() 125 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/README.md: -------------------------------------------------------------------------------- 1 | [![DOI](https://zenodo.org/badge/217881799.svg)](https://zenodo.org/badge/latestdoi/217881799) 2 | 3 | ## Weighted boxes fusion 4 | 5 | Repository contains Python implementation of several methods for ensembling boxes from object detection models: 6 | 7 | * Non-maximum Suppression (NMS) 8 | * Soft-NMS [[1]](https://arxiv.org/abs/1704.04503) 9 | * Non-maximum weighted (NMW) [[2]](http://openaccess.thecvf.com/content_ICCV_2017_workshops/papers/w14/Zhou_CAD_Scale_Invariant_ICCV_2017_paper.pdf) 10 | * **Weighted boxes fusion (WBF)** [[3]](https://arxiv.org/abs/1910.13302) - new method which gives better results comparing to others 11 | 12 | ## Requirements 13 | 14 | Python 3.*, Numpy 15 | 16 | # Installation 17 | 18 | `pip install ensemble-boxes` 19 | 20 | ## Usage examples 21 | 22 | Coordinates for boxes expected to be normalized e.g in range [0; 1]. Order: x1, y1, x2, y2. 23 | 24 | Example of boxes ensembling for 2 models below. 25 | * First model predicts 5 boxes, second model predicts 4 boxes. 26 | * Confidence scores for each box model 1: [0.9, 0.8, 0.2, 0.4, 0.7] 27 | * Confidence scores for each box model 2: [0.5, 0.8, 0.7, 0.3] 28 | * Labels (classes) for each box model 1: [0, 1, 0, 1, 1] 29 | * Labels (classes) for each box model 2: [1, 1, 1, 0] 30 | * We set weight for 1st model to be 2, and weight for second model to be 1. 31 | * We set intersection over union for boxes to be match: iou_thr = 0.5 32 | * We skip boxes with confidence lower than skip_box_thr = 0.0001 33 | 34 | ```python 35 | from ensemble_boxes import * 36 | 37 | boxes_list = [[ 38 | [0.00, 0.51, 0.81, 0.91], 39 | [0.10, 0.31, 0.71, 0.61], 40 | [0.01, 0.32, 0.83, 0.93], 41 | [0.02, 0.53, 0.11, 0.94], 42 | [0.03, 0.24, 0.12, 0.35], 43 | ],[ 44 | [0.04, 0.56, 0.84, 0.92], 45 | [0.12, 0.33, 0.72, 0.64], 46 | [0.38, 0.66, 0.79, 0.95], 47 | [0.08, 0.49, 0.21, 0.89], 48 | ]] 49 | scores_list = [[0.9, 0.8, 0.2, 0.4, 0.7], [0.5, 0.8, 0.7, 0.3]] 50 | labels_list = [[0, 1, 0, 1, 1], [1, 1, 1, 0]] 51 | weights = [2, 1] 52 | 53 | iou_thr = 0.5 54 | skip_box_thr = 0.0001 55 | sigma = 0.1 56 | 57 | boxes, scores, labels = nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr) 58 | boxes, scores, labels = soft_nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr, sigma=sigma, thresh=skip_box_thr) 59 | boxes, scores, labels = non_maximum_weighted(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr) 60 | boxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr) 61 | ``` 62 | 63 | #### Single model 64 | 65 | If you need to apply NMS or any other method to single model predictions you can call function like that: 66 | 67 | ```python 68 | from ensemble_boxes import * 69 | # Merge boxes for single model predictions 70 | boxes, scores, labels = weighted_boxes_fusion([boxes_list], [scores_list], [labels_list], weights=None, method=method, iou_thr=iou_thr, thresh=thresh) 71 | ``` 72 | 73 | More examples can be found in [example.py](./example.py) 74 | 75 | ## Accuracy and speed comparison 76 | 77 | Comparison was made for ensemble of 5 different object detection models predictions trained on [Open Images Dataset](https://storage.googleapis.com/openimages/web/index.html) (500 classes). 78 | 79 | Model scores at local validation: 80 | * Model 1: mAP(0.5) 0.5164 81 | * Model 2: mAP(0.5) 0.5019 82 | * Model 3: mAP(0.5) 0.5144 83 | * Model 4: mAP(0.5) 0.5152 84 | * Model 5: mAP(0.5) 0.4910 85 | 86 | | Method | mAP(0.5) Result | Best params | Elapsed time (sec) | 87 | | ------ | --------------- | ----------- | ------------ | 88 | | NMS | **0.5642** | IOU Thr: 0.5 | 47 | 89 | | Soft-NMS | **0.5616** | Sigma: 0.1, Confidence Thr: 0.001 | 88 | 90 | | NMW | **0.5667** | IOU Thr: 0.5 | 171 | 91 | | WBF | **0.5982** | IOU Thr: 0.6 | 249 | 92 | 93 | You can download model predictions as well as ground truth labels from here: [test_data.zip](https://github.com/ZFTurbo/Weighted-Boxes-Fusion/releases/download/v1.0/test_data.zip) 94 | 95 | Ensemble script for them is available here: [example_oid.py](./example_oid.py) 96 | 97 | ## Description of WBF method 98 | 99 | * https://arxiv.org/abs/1910.13302 100 | -------------------------------------------------------------------------------- /code/draw_pred_bbox.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 8, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import cv2\n", 10 | "import os\n", 11 | "from tqdm import tqdm\n", 12 | "import json\n", 13 | "import matplotlib.pyplot as plt" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 9, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "ann_path = r'/media/alvinai/Documents/underwater/ahf_work_dirs/testb/testBresult_epoch_12_no_waterseeds.json' # annotation json\n", 23 | "train_path = 'seacoco/testB.json'\n", 24 | "img_path = 'seacoco/testB/'\n", 25 | "save_path = 'seacoco/pred_testb_with_bbox/' # the path of saveing image with annotated bboxes" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 10, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "with open(ann_path,'r') as f:\n", 35 | " ann = json.load(f)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 11, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/plain": [ 46 | "{'image_id': 1,\n", 47 | " 'bbox': [726.2694091796875,\n", 48 | " 451.71466064453125,\n", 49 | " 187.04254150390625,\n", 50 | " 101.61077880859375],\n", 51 | " 'score': 0.4945487380027771,\n", 52 | " 'category_id': 1}" 53 | ] 54 | }, 55 | "execution_count": 11, 56 | "metadata": {}, 57 | "output_type": "execute_result" 58 | } 59 | ], 60 | "source": [ 61 | "ann[0]" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 12, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "with open(train_path, 'r') as f1:\n", 71 | " train = json.load(f1)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 13, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "ann = {'images':train['images'], 'annotations':ann, 'categories':train['categories']}" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 14, 86 | "metadata": { 87 | "scrolled": false 88 | }, 89 | "outputs": [ 90 | { 91 | "name": "stderr", 92 | "output_type": "stream", 93 | "text": [ 94 | "100%|██████████| 1200/1200 [03:03<00:00, 6.53it/s]\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "for ann_img in tqdm(ann['images']):\n", 100 | " img = cv2.imread(img_path + ann_img['file_name'])\n", 101 | " img_id = ann_img['id']\n", 102 | " for ann_ann in ann['annotations']:\n", 103 | " if ann_ann['image_id'] == img_id and ann_ann['score']>=0.1:\n", 104 | " x1 = int(ann_ann['bbox'][0])\n", 105 | " y1 = int(ann_ann['bbox'][1])\n", 106 | " x2 = int(ann_ann['bbox'][0] + ann_ann['bbox'][2])\n", 107 | " y2 = int(ann_ann['bbox'][1] + ann_ann['bbox'][3])\n", 108 | " img = cv2.rectangle(img, (x1,y1), (x2,y2), (255,0,0), 8)\n", 109 | " for cat in ann['categories']:\n", 110 | " if cat['id'] == ann_ann['category_id']:\n", 111 | " catname = cat['name']\n", 112 | " break\n", 113 | " txt = catname + str(round(ann_ann['score'],1))\n", 114 | " img = cv2.putText(img, txt, (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 4)\n", 115 | " cv2.imwrite(save_path + ann_img['file_name'], img)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [] 124 | } 125 | ], 126 | "metadata": { 127 | "kernelspec": { 128 | "display_name": "Python 3", 129 | "language": "python", 130 | "name": "python3" 131 | }, 132 | "language_info": { 133 | "codemirror_mode": { 134 | "name": "ipython", 135 | "version": 3 136 | }, 137 | "file_extension": ".py", 138 | "mimetype": "text/x-python", 139 | "name": "python", 140 | "nbconvert_exporter": "python", 141 | "pygments_lexer": "ipython3", 142 | "version": "3.7.3" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 2 147 | } 148 | -------------------------------------------------------------------------------- /code/mmdet/losses/cross_entropy_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from ..registry import LOSSES 6 | from .utils import weight_reduce_loss 7 | 8 | 9 | def label_smooth_cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None, smoothing=0.0): 10 | confidence = 1.0 - smoothing 11 | label_shape = torch.Size((label.size(0), pred.size(1))) # N,6 12 | pred = pred.log_softmax(-1) 13 | with torch.no_grad(): 14 | true_dist = torch.zeros_like(pred) 15 | true_dist.fill_(smoothing / (pred.size(1) - 1)) # s / (C-1) 16 | true_dist.scatter_(1, label.data.unsqueeze(1), confidence) 17 | return torch.mean(torch.sum(-true_dist * pred, dim=-1)) 18 | 19 | def cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None): 20 | 21 | # element-wise losses 22 | loss = F.cross_entropy(pred, label, reduction='none') 23 | 24 | # apply weights and do the reduction 25 | if weight is not None: 26 | weight = weight.float() 27 | loss = weight_reduce_loss( 28 | loss, weight=weight, reduction=reduction, avg_factor=avg_factor) 29 | 30 | return loss 31 | 32 | 33 | def _expand_binary_labels(labels, label_weights, label_channels): 34 | bin_labels = labels.new_full((labels.size(0), label_channels), 0) 35 | inds = torch.nonzero(labels >= 1).squeeze() 36 | if inds.numel() > 0: 37 | bin_labels[inds, labels[inds] - 1] = 1 38 | if label_weights is None: 39 | bin_label_weights = None 40 | else: 41 | bin_label_weights = label_weights.view(-1, 1).expand( 42 | label_weights.size(0), label_channels) 43 | return bin_labels, bin_label_weights 44 | 45 | 46 | def binary_cross_entropy(pred, 47 | label, 48 | weight=None, 49 | reduction='mean', 50 | avg_factor=None): 51 | if pred.dim() != label.dim(): 52 | label, weight = _expand_binary_labels(label, weight, pred.size(-1)) 53 | 54 | # weighted element-wise losses 55 | if weight is not None: 56 | weight = weight.float() 57 | loss = F.binary_cross_entropy_with_logits( 58 | pred, label.float(), weight, reduction='none') 59 | # do the reduction for the weighted loss 60 | loss = weight_reduce_loss(loss, reduction=reduction, avg_factor=avg_factor) 61 | 62 | return loss 63 | 64 | 65 | def mask_cross_entropy(pred, target, label, reduction='mean', avg_factor=None): 66 | # TODO: handle these two reserved arguments 67 | assert reduction == 'mean' and avg_factor is None 68 | num_rois = pred.size()[0] 69 | inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) 70 | pred_slice = pred[inds, label].squeeze(1) 71 | return F.binary_cross_entropy_with_logits( 72 | pred_slice, target, reduction='mean')[None] 73 | 74 | 75 | @LOSSES.register_module 76 | class CrossEntropyLoss(nn.Module): 77 | 78 | def __init__(self, 79 | use_sigmoid=False, 80 | use_mask=False, 81 | reduction='mean', 82 | loss_weight=1.0, 83 | smoothing=0.0): 84 | super(CrossEntropyLoss, self).__init__() 85 | assert (use_sigmoid is False) or (use_mask is False) 86 | self.use_sigmoid = use_sigmoid 87 | self.use_mask = use_mask 88 | self.reduction = reduction 89 | self.loss_weight = loss_weight 90 | self.smoothing = smoothing 91 | 92 | if self.use_sigmoid: 93 | self.cls_criterion = binary_cross_entropy 94 | elif self.use_mask: 95 | self.cls_criterion = mask_cross_entropy 96 | elif self.smoothing > 0.0: 97 | self.cls_criterion = label_smooth_cross_entropy 98 | else: 99 | self.cls_criterion = cross_entropy 100 | 101 | def forward(self, 102 | cls_score, 103 | label, 104 | weight=None, 105 | avg_factor=None, 106 | reduction_override=None, 107 | **kwargs): 108 | assert reduction_override in (None, 'none', 'mean', 'sum') 109 | reduction = ( 110 | reduction_override if reduction_override else self.reduction) 111 | if self.smoothing > 0.0: 112 | loss_cls = self.loss_weight * self.cls_criterion( 113 | cls_score, 114 | label, 115 | weight, 116 | reduction=reduction, 117 | avg_factor=avg_factor, 118 | smoothing = self.smoothing, 119 | **kwargs) 120 | else: 121 | loss_cls = self.loss_weight * self.cls_criterion( 122 | cls_score, 123 | label, 124 | weight, 125 | reduction=reduction, 126 | avg_factor=avg_factor, 127 | **kwargs) 128 | return loss_cls 129 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/ensemble_boxes_nmw.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | """ 5 | Method described in: 6 | CAD: Scale Invariant Framework for Real-Time Object Detection 7 | http://openaccess.thecvf.com/content_ICCV_2017_workshops/papers/w14/Zhou_CAD_Scale_Invariant_ICCV_2017_paper.pdf 8 | """ 9 | 10 | import numpy as np 11 | 12 | 13 | def bb_intersection_over_union(A, B): 14 | xA = max(A[0], B[0]) 15 | yA = max(A[1], B[1]) 16 | xB = min(A[2], B[2]) 17 | yB = min(A[3], B[3]) 18 | 19 | # compute the area of intersection rectangle 20 | interArea = max(0, xB - xA) * max(0, yB - yA) 21 | 22 | if interArea == 0: 23 | return 0.0 24 | 25 | # compute the area of both the prediction and ground-truth rectangles 26 | boxAArea = (A[2] - A[0]) * (A[3] - A[1]) 27 | boxBArea = (B[2] - B[0]) * (B[3] - B[1]) 28 | 29 | iou = interArea / float(boxAArea + boxBArea - interArea) 30 | return iou 31 | 32 | 33 | def prefilter_boxes(boxes, scores, labels, weights, thr): 34 | # Create dict with boxes stored by its label 35 | new_boxes = dict() 36 | for t in range(len(boxes)): 37 | for j in range(len(boxes[t])): 38 | label = int(labels[t][j]) 39 | score = scores[t][j] 40 | if score < thr: 41 | break 42 | box_part = boxes[t][j] 43 | b = [int(label), float(score) * weights[t], float(box_part[0]), float(box_part[1]), float(box_part[2]), float(box_part[3])] 44 | if label not in new_boxes: 45 | new_boxes[label] = [] 46 | new_boxes[label].append(b) 47 | 48 | # Sort each list in dict and transform it to numpy array 49 | for k in new_boxes: 50 | current_boxes = np.array(new_boxes[k]) 51 | new_boxes[k] = current_boxes[current_boxes[:, 1].argsort()[::-1]] 52 | 53 | return new_boxes 54 | 55 | 56 | def get_weighted_box(boxes): 57 | """ 58 | Create weighted box for set of boxes 59 | :param boxes: set of boxes to fuse 60 | :return: weighted box 61 | """ 62 | 63 | box = np.zeros(6, dtype=np.float32) 64 | best_box = boxes[0] 65 | conf = 0 66 | for b in boxes: 67 | iou = bb_intersection_over_union(b[2:], best_box[2:]) 68 | weight = b[1] * iou 69 | box[2:] += (weight * b[2:]) 70 | conf += weight 71 | box[0] = best_box[0] 72 | box[1] = best_box[1] 73 | box[2:] /= conf 74 | return box 75 | 76 | 77 | def find_matching_box(boxes_list, new_box, match_iou): 78 | best_iou = match_iou 79 | best_index = -1 80 | for i in range(len(boxes_list)): 81 | box = boxes_list[i] 82 | if box[0] != new_box[0]: 83 | continue 84 | iou = bb_intersection_over_union(box[2:], new_box[2:]) 85 | if iou > best_iou: 86 | best_index = i 87 | best_iou = iou 88 | 89 | return best_index, best_iou 90 | 91 | 92 | def non_maximum_weighted(boxes_list, scores_list, labels_list, weights=None, iou_thr=0.55, skip_box_thr=0.0): 93 | ''' 94 | :param boxes_list: list of boxes predictions from each model, each box is 4 numbers. 95 | It has 3 dimensions (models_number, model_preds, 4) 96 | Order of boxes: x1, y1, x2, y2. We expect float normalized coordinates [0; 1] 97 | :param scores_list: list of scores for each model 98 | :param labels_list: list of labels for each model 99 | :param weights: list of weights for each model. Default: None, which means weight == 1 for each model 100 | :param intersection_thr: IoU value for boxes to be a match 101 | :param skip_box_thr: exclude boxes with score lower than this variable 102 | 103 | :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2). 104 | :return: scores: confidence scores 105 | :return: labels: boxes labels 106 | ''' 107 | 108 | if weights is None: 109 | weights = np.ones(len(boxes_list)) 110 | if len(weights) != len(boxes_list): 111 | print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights), len(boxes_list))) 112 | weights = np.ones(len(boxes_list)) 113 | weights = np.array(weights) 114 | for i in range(len(weights)): 115 | scores_list[i] = (np.array(scores_list[i]) * weights[i]) / weights.sum() 116 | 117 | filtered_boxes = prefilter_boxes(boxes_list, scores_list, labels_list, weights, skip_box_thr) 118 | if len(filtered_boxes) == 0: 119 | return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,)) 120 | 121 | overall_boxes = [] 122 | for label in filtered_boxes: 123 | boxes = filtered_boxes[label] 124 | new_boxes = [] 125 | main_boxes = [] 126 | 127 | # Clusterize boxes 128 | for j in range(0, len(boxes)): 129 | index, best_iou = find_matching_box(main_boxes, boxes[j], iou_thr) 130 | if index != -1: 131 | new_boxes[index].append(boxes[j].copy()) 132 | else: 133 | new_boxes.append([boxes[j].copy()]) 134 | main_boxes.append(boxes[j].copy()) 135 | 136 | weighted_boxes = [] 137 | for j in range(0, len(new_boxes)): 138 | box = get_weighted_box(new_boxes[j]) 139 | weighted_boxes.append(box.copy()) 140 | 141 | overall_boxes.append(np.array(weighted_boxes)) 142 | 143 | overall_boxes = np.concatenate(overall_boxes, axis=0) 144 | overall_boxes = overall_boxes[overall_boxes[:, 1].argsort()[::-1]] 145 | boxes = overall_boxes[:, 2:] 146 | scores = overall_boxes[:, 1] 147 | labels = overall_boxes[:, 0] 148 | return boxes, scores, labels 149 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/ensemble_boxes_wbf.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | 5 | import numpy as np 6 | 7 | 8 | def bb_intersection_over_union(A, B): 9 | xA = max(A[0], B[0]) 10 | yA = max(A[1], B[1]) 11 | xB = min(A[2], B[2]) 12 | yB = min(A[3], B[3]) 13 | 14 | # compute the area of intersection rectangle 15 | interArea = max(0, xB - xA) * max(0, yB - yA) 16 | 17 | if interArea == 0: 18 | return 0.0 19 | 20 | # compute the area of both the prediction and ground-truth rectangles 21 | boxAArea = (A[2] - A[0]) * (A[3] - A[1]) 22 | boxBArea = (B[2] - B[0]) * (B[3] - B[1]) 23 | 24 | iou = interArea / float(boxAArea + boxBArea - interArea) 25 | return iou 26 | 27 | 28 | def prefilter_boxes(boxes, scores, labels, weights, thr): 29 | # Create dict with boxes stored by its label 30 | new_boxes = dict() 31 | for t in range(len(boxes)): 32 | for j in range(len(boxes[t])): 33 | label = int(labels[t][j]) 34 | score = scores[t][j] 35 | if score < thr: 36 | break 37 | box_part = boxes[t][j] 38 | b = [int(label), float(score) * weights[t], float(box_part[0]), float(box_part[1]), float(box_part[2]), float(box_part[3])] 39 | if label not in new_boxes: 40 | new_boxes[label] = [] 41 | new_boxes[label].append(b) 42 | 43 | # Sort each list in dict and transform it to numpy array 44 | for k in new_boxes: 45 | current_boxes = np.array(new_boxes[k]) 46 | new_boxes[k] = current_boxes[current_boxes[:, 1].argsort()[::-1]] 47 | 48 | return new_boxes 49 | 50 | 51 | def get_weighted_box(boxes, conf_type='avg'): 52 | """ 53 | Create weighted box for set of boxes 54 | :param boxes: set of boxes to fuse 55 | :param conf_type: type of confidence one of 'avg' or 'max' 56 | :return: weighted box 57 | """ 58 | 59 | box = np.zeros(6, dtype=np.float32) 60 | conf = 0 61 | conf_list = [] 62 | for b in boxes: 63 | box[2:] += (b[1] * b[2:]) 64 | conf += b[1] 65 | conf_list.append(b[1]) 66 | box[0] = boxes[0][0] 67 | if conf_type == 'avg': 68 | box[1] = conf / len(boxes) 69 | elif conf_type == 'max': 70 | box[1] = np.array(conf_list).max() 71 | box[2:] /= conf 72 | return box 73 | 74 | 75 | def find_matching_box(boxes_list, new_box, match_iou): 76 | best_iou = match_iou 77 | best_index = -1 78 | for i in range(len(boxes_list)): 79 | box = boxes_list[i] 80 | if box[0] != new_box[0]: 81 | continue 82 | iou = bb_intersection_over_union(box[2:], new_box[2:]) 83 | if iou > best_iou: 84 | best_index = i 85 | best_iou = iou 86 | 87 | return best_index, best_iou 88 | 89 | 90 | def weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=None, iou_thr=0.55, skip_box_thr=0.0, conf_type='avg', allows_overflow=False): 91 | ''' 92 | :param boxes_list: list of boxes predictions from each model, each box is 4 numbers. 93 | It has 3 dimensions (models_number, model_preds, 4) 94 | Order of boxes: x1, y1, x2, y2. We expect float normalized coordinates [0; 1] 95 | :param scores_list: list of scores for each model 96 | :param labels_list: list of labels for each model 97 | :param weights: list of weights for each model. Default: None, which means weight == 1 for each model 98 | :param intersection_thr: IoU value for boxes to be a match 99 | :param skip_box_thr: exclude boxes with score lower than this variable 100 | :param conf_type: how to calculate confidence in weighted boxes. 'avg': average value, 'max': maximum value 101 | :param allows_overflow: false if we want confidence score not exceed 1.0 102 | 103 | :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2). 104 | :return: scores: confidence scores 105 | :return: labels: boxes labels 106 | ''' 107 | 108 | if weights is None: 109 | weights = np.ones(len(boxes_list)) 110 | if len(weights) != len(boxes_list): 111 | print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights), len(boxes_list))) 112 | weights = np.ones(len(boxes_list)) 113 | weights = np.array(weights) 114 | 115 | if conf_type not in ['avg', 'max']: 116 | print('Unknown conf_type: {}. Must be "avg" or "max"'.format(conf_type)) 117 | exit() 118 | 119 | filtered_boxes = prefilter_boxes(boxes_list, scores_list, labels_list, weights, skip_box_thr) 120 | if len(filtered_boxes) == 0: 121 | return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,)) 122 | 123 | overall_boxes = [] 124 | for label in filtered_boxes: 125 | boxes = filtered_boxes[label] 126 | new_boxes = [] 127 | weighted_boxes = [] 128 | 129 | # Clusterize boxes 130 | for j in range(0, len(boxes)): 131 | index, best_iou = find_matching_box(weighted_boxes, boxes[j], iou_thr) 132 | if index != -1: 133 | new_boxes[index].append(boxes[j]) 134 | weighted_boxes[index] = get_weighted_box(new_boxes[index], conf_type) 135 | else: 136 | new_boxes.append([boxes[j].copy()]) 137 | weighted_boxes.append(boxes[j].copy()) 138 | 139 | # Rescale confidence based on number of models and boxes 140 | for i in range(len(new_boxes)): 141 | if not allows_overflow: 142 | weighted_boxes[i][1] = weighted_boxes[i][1] * min(weights.sum(), len(new_boxes[i])) / weights.sum() 143 | else: 144 | weighted_boxes[i][1] = weighted_boxes[i][1] * len(new_boxes[i]) / weights.sum() 145 | overall_boxes.append(np.array(weighted_boxes)) 146 | 147 | overall_boxes = np.concatenate(overall_boxes, axis=0) 148 | overall_boxes = overall_boxes[overall_boxes[:, 1].argsort()[::-1]] 149 | boxes = overall_boxes[:, 2:] 150 | scores = overall_boxes[:, 1] 151 | labels = overall_boxes[:, 0] 152 | return boxes, scores, labels 153 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/example.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | 5 | import cv2 6 | import numpy as np 7 | from ensemble_boxes import * 8 | 9 | 10 | def show_image(im, name='image'): 11 | cv2.imshow(name, im.astype(np.uint8)) 12 | cv2.waitKey(0) 13 | cv2.destroyAllWindows() 14 | 15 | 16 | def gen_color_list(model_num, labels_num): 17 | color_list = np.zeros((model_num, labels_num, 3)) 18 | colors_to_use = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (0, 255, 255), (255, 0, 255), (255, 255, 0), (0, 0, 0)] 19 | total = 0 20 | for i in range(model_num): 21 | for j in range(labels_num): 22 | color_list[i, j, :] = colors_to_use[total] 23 | total = (total + 1) % len(colors_to_use) 24 | return color_list 25 | 26 | 27 | def show_boxes(boxes_list, scores_list, labels_list, image_size=800): 28 | thickness = 5 29 | color_list = gen_color_list(len(boxes_list), len(np.unique(labels_list))) 30 | image = np.zeros((image_size, image_size, 3), dtype=np.uint8) 31 | image[...] = 255 32 | for i in range(len(boxes_list)): 33 | for j in range(len(boxes_list[i])): 34 | x1 = int(image_size * boxes_list[i][j][0]) 35 | y1 = int(image_size * boxes_list[i][j][1]) 36 | x2 = int(image_size * boxes_list[i][j][2]) 37 | y2 = int(image_size * boxes_list[i][j][3]) 38 | lbl = labels_list[i][j] 39 | cv2.rectangle(image, (x1, y1), (x2, y2), color_list[i][lbl], int(thickness * scores_list[i][j])) 40 | show_image(image) 41 | 42 | 43 | def example_wbf_2_models(iou_thr=0.55, draw_image=True): 44 | """ 45 | This example shows how to ensemble boxes from 2 models using WBF method 46 | :return: 47 | """ 48 | 49 | boxes_list = [ 50 | [ 51 | [0.00, 0.51, 0.81, 0.91], 52 | [0.10, 0.31, 0.71, 0.61], 53 | [0.01, 0.32, 0.83, 0.93], 54 | [0.02, 0.53, 0.11, 0.94], 55 | [0.03, 0.24, 0.12, 0.35], 56 | ], 57 | [ 58 | [0.04, 0.56, 0.84, 0.92], 59 | [0.12, 0.33, 0.72, 0.64], 60 | [0.38, 0.66, 0.79, 0.95], 61 | [0.08, 0.49, 0.21, 0.89], 62 | ], 63 | ] 64 | scores_list = [ 65 | [ 66 | 0.9, 67 | 0.8, 68 | 0.2, 69 | 0.4, 70 | 0.7, 71 | ], 72 | [ 73 | 0.5, 74 | 0.8, 75 | 0.7, 76 | 0.3, 77 | ] 78 | ] 79 | labels_list = [ 80 | [ 81 | 0, 82 | 1, 83 | 0, 84 | 1, 85 | 1, 86 | ], 87 | [ 88 | 1, 89 | 1, 90 | 1, 91 | 0, 92 | ] 93 | ] 94 | weights = [2, 1] 95 | if draw_image: 96 | show_boxes(boxes_list, scores_list, labels_list) 97 | 98 | boxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr, skip_box_thr=0.0) 99 | 100 | if draw_image: 101 | show_boxes([boxes], [scores], [labels.astype(np.int32)]) 102 | 103 | print(len(boxes)) 104 | print(boxes) 105 | 106 | 107 | def example_wbf_1_model(iou_thr=0.55, draw_image=True): 108 | """ 109 | This example shows how to ensemble boxes from single model using WBF method 110 | :return: 111 | """ 112 | 113 | boxes_list = [ 114 | [0.00, 0.51, 0.81, 0.91], 115 | [0.10, 0.31, 0.71, 0.61], 116 | [0.01, 0.32, 0.83, 0.93], 117 | [0.02, 0.53, 0.11, 0.94], 118 | [0.03, 0.24, 0.12, 0.35], 119 | [0.04, 0.56, 0.84, 0.92], 120 | [0.12, 0.33, 0.72, 0.64], 121 | [0.38, 0.66, 0.79, 0.95], 122 | [0.08, 0.49, 0.21, 0.89], 123 | ] 124 | scores_list = [0.9, 0.8, 0.2, 0.4, 0.7, 0.5, 0.8, 0.7, 0.3] 125 | labels_list = [0, 1, 0, 1, 1, 1, 1, 1, 0] 126 | 127 | if draw_image: 128 | show_boxes([boxes_list], [scores_list], [labels_list]) 129 | 130 | boxes, scores, labels = weighted_boxes_fusion([boxes_list], [scores_list], [labels_list], weights=None, iou_thr=iou_thr, skip_box_thr=0.0) 131 | 132 | if draw_image: 133 | show_boxes([boxes], [scores], [labels.astype(np.int32)]) 134 | 135 | print(len(boxes)) 136 | print(boxes) 137 | 138 | 139 | def example_nms_2_models(method, iou_thr=0.5, sigma=0.5, thresh=0.001, draw_image=True): 140 | """ 141 | This example shows how to ensemble boxes from 2 models using NMS method 142 | :return: 143 | """ 144 | 145 | boxes_list = [ 146 | [ 147 | [0.00, 0.51, 0.81, 0.91], 148 | [0.10, 0.31, 0.71, 0.61], 149 | [0.01, 0.32, 0.83, 0.93], 150 | [0.02, 0.53, 0.11, 0.94], 151 | [0.03, 0.24, 0.12, 0.35], 152 | ], 153 | [ 154 | [0.04, 0.56, 0.84, 0.92], 155 | [0.12, 0.33, 0.72, 0.64], 156 | [0.38, 0.66, 0.79, 0.95], 157 | [0.08, 0.49, 0.21, 0.89], 158 | ], 159 | ] 160 | scores_list = [ 161 | [ 162 | 0.9, 163 | 0.8, 164 | 0.2, 165 | 0.4, 166 | 0.7, 167 | ], 168 | [ 169 | 0.5, 170 | 0.8, 171 | 0.7, 172 | 0.3, 173 | ] 174 | ] 175 | labels_list = [ 176 | [ 177 | 0, 178 | 1, 179 | 0, 180 | 1, 181 | 1, 182 | ], 183 | [ 184 | 1, 185 | 1, 186 | 1, 187 | 0, 188 | ] 189 | ] 190 | weights = [2, 1] 191 | 192 | if draw_image: 193 | show_boxes(boxes_list, scores_list, labels_list) 194 | 195 | boxes, scores, labels = nms_method(boxes_list, scores_list, labels_list, method=method, weights=weights, iou_thr=iou_thr, sigma=sigma, thresh=thresh) 196 | 197 | if draw_image: 198 | show_boxes([boxes], [scores], [labels.astype(np.int32)]) 199 | 200 | print(len(boxes)) 201 | print(boxes) 202 | 203 | 204 | if __name__ == '__main__': 205 | draw_image = True 206 | example_wbf_2_models(draw_image=draw_image) 207 | example_wbf_1_model(draw_image=draw_image) 208 | example_nms_2_models(draw_image=draw_image, method=3, iou_thr=0.5, thresh=0.0) 209 | example_nms_2_models(draw_image=draw_image, method=2, iou_thr=0.3, sigma=0.05, thresh=0.001) 210 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/ensemble_boxes/ensemble_boxes_nms.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import numpy as np 5 | 6 | 7 | def cpu_soft_nms_float(dets, sc, Nt, sigma, thresh, method): 8 | """ 9 | Based on: https://github.com/DocF/Soft-NMS/blob/master/soft_nms.py 10 | It's different from original soft-NMS because we have float coordinates on range [0; 1] 11 | 12 | :param dets: boxes format [x1, y1, x2, y2] 13 | :param sc: scores for boxes 14 | :param Nt: required iou 15 | :param sigma: 16 | :param thresh: 17 | :param method: 1 - linear soft-NMS, 2 - gaussian soft-NMS, 3 - standard NMS 18 | :return: index of boxes to keep 19 | """ 20 | 21 | # indexes concatenate boxes with the last column 22 | N = dets.shape[0] 23 | indexes = np.array([np.arange(N)]) 24 | dets = np.concatenate((dets, indexes.T), axis=1) 25 | 26 | # the order of boxes coordinate is [y1, x1, y2, x2] 27 | y1 = dets[:, 1] 28 | x1 = dets[:, 0] 29 | y2 = dets[:, 3] 30 | x2 = dets[:, 2] 31 | scores = sc 32 | areas = (x2 - x1) * (y2 - y1) 33 | 34 | for i in range(N): 35 | # intermediate parameters for later parameters exchange 36 | tBD = dets[i, :].copy() 37 | tscore = scores[i].copy() 38 | tarea = areas[i].copy() 39 | pos = i + 1 40 | 41 | # 42 | if i != N - 1: 43 | maxscore = np.max(scores[pos:], axis=0) 44 | maxpos = np.argmax(scores[pos:], axis=0) 45 | else: 46 | maxscore = scores[-1] 47 | maxpos = 0 48 | if tscore < maxscore: 49 | dets[i, :] = dets[maxpos + i + 1, :] 50 | dets[maxpos + i + 1, :] = tBD 51 | tBD = dets[i, :] 52 | 53 | scores[i] = scores[maxpos + i + 1] 54 | scores[maxpos + i + 1] = tscore 55 | tscore = scores[i] 56 | 57 | areas[i] = areas[maxpos + i + 1] 58 | areas[maxpos + i + 1] = tarea 59 | tarea = areas[i] 60 | 61 | # IoU calculate 62 | xx1 = np.maximum(dets[i, 1], dets[pos:, 1]) 63 | yy1 = np.maximum(dets[i, 0], dets[pos:, 0]) 64 | xx2 = np.minimum(dets[i, 3], dets[pos:, 3]) 65 | yy2 = np.minimum(dets[i, 2], dets[pos:, 2]) 66 | 67 | w = np.maximum(0.0, xx2 - xx1) 68 | h = np.maximum(0.0, yy2 - yy1) 69 | inter = w * h 70 | ovr = inter / (areas[i] + areas[pos:] - inter) 71 | 72 | # Three methods: 1.linear 2.gaussian 3.original NMS 73 | if method == 1: # linear 74 | weight = np.ones(ovr.shape) 75 | weight[ovr > Nt] = weight[ovr > Nt] - ovr[ovr > Nt] 76 | elif method == 2: # gaussian 77 | weight = np.exp(-(ovr * ovr) / sigma) 78 | else: # original NMS 79 | weight = np.ones(ovr.shape) 80 | weight[ovr > Nt] = 0 81 | 82 | scores[pos:] = weight * scores[pos:] 83 | 84 | # select the boxes and keep the corresponding indexes 85 | inds = dets[:, 4][scores > thresh] 86 | keep = inds.astype(int) 87 | return keep 88 | 89 | 90 | def nms_float_fast(dets, scores, thresh): 91 | """ 92 | # It's different from original nms because we have float coordinates on range [0; 1] 93 | :param dets: numpy array of boxes with shape: (N, 5). Order: x1, y1, x2, y2, score. All variables in range [0; 1] 94 | :param thresh: IoU value for boxes 95 | :return: 96 | """ 97 | x1 = dets[:, 0] 98 | y1 = dets[:, 1] 99 | x2 = dets[:, 2] 100 | y2 = dets[:, 3] 101 | 102 | areas = (x2 - x1) * (y2 - y1) 103 | order = scores.argsort()[::-1] 104 | 105 | keep = [] 106 | while order.size > 0: 107 | i = order[0] 108 | keep.append(i) 109 | xx1 = np.maximum(x1[i], x1[order[1:]]) 110 | yy1 = np.maximum(y1[i], y1[order[1:]]) 111 | xx2 = np.minimum(x2[i], x2[order[1:]]) 112 | yy2 = np.minimum(y2[i], y2[order[1:]]) 113 | 114 | w = np.maximum(0.0, xx2 - xx1) 115 | h = np.maximum(0.0, yy2 - yy1) 116 | inter = w * h 117 | ovr = inter / (areas[i] + areas[order[1:]] - inter) 118 | inds = np.where(ovr <= thresh)[0] 119 | order = order[inds + 1] 120 | 121 | return keep 122 | 123 | 124 | def nms_method(boxes, scores, labels, method=3, iou_thr=0.5, sigma=0.5, thresh=0.001, weights=None): 125 | """ 126 | :param boxes: list of boxes predictions from each model, each box is 4 numbers. 127 | It has 3 dimensions (models_number, model_preds, 4) 128 | Order of boxes: x1, y1, x2, y2. We expect float normalized coordinates [0; 1] 129 | :param scores: list of scores for each model 130 | :param labels: list of labels for each model 131 | :param method: 1 - linear soft-NMS, 2 - gaussian soft-NMS, 3 - standard NMS 132 | :param iou_thr: IoU value for boxes to be a match 133 | :param sigma: Sigma value for SoftNMS 134 | :param thresh: threshold for boxes to keep (important for SoftNMS) 135 | :param weights: list of weights for each model. Default: None, which means weight == 1 for each model 136 | 137 | :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2). 138 | :return: scores: confidence scores 139 | :return: labels: boxes labels 140 | """ 141 | 142 | # If weights are specified 143 | if weights is not None: 144 | if len(boxes) != len(weights): 145 | print('Incorrect number of weights: {}. Must be: {}. Skip it'.format(len(weights), len(boxes))) 146 | else: 147 | weights = np.array(weights) 148 | for i in range(len(weights)): 149 | scores[i] = (np.array(scores[i]) * weights[i]) / weights.sum() 150 | 151 | # We concatenate everything 152 | boxes = np.concatenate(boxes) 153 | scores = np.concatenate(scores) 154 | labels = np.concatenate(labels) 155 | 156 | # Run NMS independently for each label 157 | unique_labels = np.unique(labels) 158 | final_boxes = [] 159 | final_scores = [] 160 | final_labels = [] 161 | for l in unique_labels: 162 | condition = (labels == l) 163 | boxes_by_label = boxes[condition] 164 | scores_by_label = scores[condition] 165 | labels_by_label = np.array([l] * len(boxes_by_label)) 166 | 167 | if method != 3: 168 | keep = cpu_soft_nms_float(boxes_by_label.copy(), scores_by_label.copy(), Nt=iou_thr, sigma=sigma, thresh=thresh, method=method) 169 | else: 170 | # Use faster function 171 | keep = nms_float_fast(boxes_by_label, scores_by_label, thresh=iou_thr) 172 | 173 | final_boxes.append(boxes_by_label[keep]) 174 | final_scores.append(scores_by_label[keep]) 175 | final_labels.append(labels_by_label[keep]) 176 | final_boxes = np.concatenate(final_boxes) 177 | final_scores = np.concatenate(final_scores) 178 | final_labels = np.concatenate(final_labels) 179 | 180 | return final_boxes, final_scores, final_labels 181 | 182 | 183 | def nms(boxes, scores, labels, iou_thr=0.5, weights=None): 184 | """ 185 | Short call for standard NMS 186 | 187 | :param boxes: 188 | :param scores: 189 | :param labels: 190 | :param iou_thr: 191 | :param weights: 192 | :return: 193 | """ 194 | return nms_method(boxes, scores, labels, method=3, iou_thr=iou_thr, weights=weights) 195 | 196 | 197 | def soft_nms(boxes, scores, labels, method=2, iou_thr=0.5, sigma=0.5, thresh=0.001, weights=None): 198 | """ 199 | Short call for Soft-NMS 200 | 201 | :param boxes: 202 | :param scores: 203 | :param labels: 204 | :param method: 205 | :param iou_thr: 206 | :param sigma: 207 | :param thresh: 208 | :param weights: 209 | :return: 210 | """ 211 | return nms_method(boxes, scores, labels, method=method, iou_thr=iou_thr, sigma=sigma, thresh=thresh, weights=weights) -------------------------------------------------------------------------------- /code/Retinex.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import math 4 | import argparse 5 | import os 6 | from tqdm import tqdm 7 | import time 8 | 9 | 10 | class GaussianBlurConv(): 11 | ''' 12 | 高斯滤波 13 | 依据图像金字塔和高斯可分离滤波器思路加速 14 | ''' 15 | def FilterGaussian(self, img, sigma): 16 | ''' 17 | 高斯分离卷积，按照x轴y轴拆分运算，再合并，加速运算 18 | ''' 19 | # reject unreasonable demands 20 | if sigma > 300: 21 | sigma = 300 22 | # 获取滤波器尺寸且强制为奇数 23 | kernel_size = round(sigma * 3 * 2 +1) | 1 # 当图像类型为CV_8U的时候能量集中区域为3 * sigma, 24 | # 创建内核 25 | kernel = cv2.getGaussianKernel(ksize=kernel_size, sigma=sigma, ktype=cv2.CV_32F) 26 | # 初始化图像 27 | temp = np.zeros_like(img) 28 | # x轴滤波 29 | for j in range(temp.shape[0]): 30 | for i in range(temp.shape[1]): 31 | # 内层循环展开 32 | v1 = v2 = v3 = 0 33 | for k in range(kernel_size): 34 | source = math.floor(i+ kernel_size/2 -k) # 把第i个坐标和kernel的中心对齐 -k是从右往左遍历kernel对应的图像，得到与kernel的第k个元素相乘的图像坐标 35 | if source < 0: 36 | source = source * -1 # 如果图像超出左边缘，就反向，对称填充 37 | if source > img.shape[1]: 38 | source = math.floor(2 * (img.shape[1] - 1) - source) # 图像如果超出右边缘，就用左边从头数着补 39 | v1 += kernel[k] * img[j, source, 0] 40 | if temp.shape[2] == 1: continue 41 | v2 += kernel[k] * img[j, source, 1] 42 | v3 += kernel[k] * img[j, source, 2] 43 | temp[j, i, 0] = v1 44 | if temp.shape[2] == 1: continue 45 | temp[j, i, 1] = v2 46 | temp[j, i, 2] = v3 47 | # 分离滤波，先在原图用x轴的滤波器滤波，得到temp图，再用y轴滤波在temp图上滤波，结果一致 48 | # y轴滤波 49 | for i in range(img.shape[1]): # height 50 | for j in range(img.shape[0]): 51 | v1 = v2 = v3 = 0 52 | for k in range(kernel_size): 53 | source = math.floor(j + kernel_size/2 - k) 54 | if source < 0: 55 | source = source * -1 56 | if source > temp.shape[0]: 57 | source = math.floor(2 * (img.shape[0] - 1) - source) # 上下对称 58 | v1 += kernel[k] * temp[source, i, 0] 59 | if temp.shape[2] == 1: continue 60 | v2 += kernel[k] * temp[source, i, 1] 61 | v3 += kernel[k] * temp[source, i, 2] 62 | img[j, i, 0] = v1 63 | if img.shape[2] == 1: continue 64 | img[j, i, 1] = v2 65 | img[j, i, 2] = v3 66 | return img 67 | 68 | def FastFilter(self, img, sigma): 69 | ''' 70 | 快速滤波，按照图像金字塔，逐级降低图像分辨率，对应降低高斯核的sigma， 71 | 当sigma转换成高斯核size小于10，再进行滤波，后逐级resize 72 | 递归思路 73 | ''' 74 | # reject unreasonable demands 75 | if sigma > 300: 76 | sigma = 300 77 | # 获取滤波尺寸，且强制为奇数 78 | kernel_size = round(sigma * 3 * 2 + 1) | 1 # 当图像类型为CV_8U的时候能量集中区域为3 * sigma, 79 | # 如果s*sigma小于一个像素，则直接退出 80 | if kernel_size < 3: 81 | return 82 | # 处理方式(1) 滤波 (2) 高斯光滑处理 (3) 递归处理滤波器大小 83 | if kernel_size < 10: 84 | # img = self.FilterGaussian(img, sigma) 85 | img = cv2.GaussianBlur(img, (kernel_size, kernel_size), 0) # 官方函数 86 | return img 87 | else: 88 | # 若降采样到最小，直接退出 89 | if img.shape[1] < 2 or img.shape[0] < 2: 90 | return img 91 | sub_img = np.zeros_like(img) # 初始化降采样图像 92 | sub_img = cv2.pyrDown(img, sub_img) # 使用gaussian滤波对输入图像向下采样，缩放二分之一，仅支持CV_GAUSSIAN_5x5 93 | sub_img = self.FastFilter(sub_img, sigma/2.0) 94 | img = cv2.resize(sub_img, (img.shape[1], img.shape[0])) # resize到原图大小 95 | return img 96 | 97 | def __call__(self, x, sigma): 98 | x = self.FastFilter(img, sigma) 99 | return x 100 | 101 | 102 | class Retinex(object): 103 | """ 104 | SSR: baseline 105 | MSR: keep the high fidelity and the dynamic range as well as compressing img 106 | MSRCR_GIMP: 107 | Adapt the dynamics of the colors according to the statistics of the first and second order. 108 | The use of the variance makes it possible to control the degree of saturation of the colors. 109 | """ 110 | def __init__(self, model='MSR', sigma=[30, 150, 300], restore_factor=2.0, color_gain=10.0, gain=270.0, offset=128.0): 111 | self.model_list = ['SSR','MSR'] 112 | if model in self.model_list: 113 | self.model = model 114 | else: 115 | raise ValueError 116 | self.sigma = sigma # 高斯核的方差 117 | # 颜色恢复 118 | self.restore_factor = restore_factor # 控制颜色修复的非线性 119 | self.color_gain = color_gain # 控制颜色修复增益 120 | # 图像恢复 121 | self.gain = gain # 图像像素值改变范围的增益 122 | self.offset = offset # 图像像素值改变范围的偏移量 123 | self.gaussian_conv = GaussianBlurConv() # 实例化高斯算子 124 | 125 | def _SSR(self, img, sigma): 126 | filter_img = self.gaussian_conv(img, sigma) # [h,w,c] 127 | retinex = np.log10(img) - np.log10(filter_img) 128 | return retinex 129 | 130 | def _MSR(self, img, simga): 131 | retinex = np.zeros_like(img) 132 | for sig in simga: 133 | retinex += self._SSR(img, sig) 134 | retinex = retinex / float(len(self.sigma)) 135 | return retinex 136 | 137 | def _colorRestoration(self, img, retinex): 138 | img_sum = np.sum(img, axis=2, keepdims=True) # 在通道层面求和 139 | # 颜色恢复 140 | # 权重矩阵归一化 并求对数，得到颜色增益 141 | color_restoration = np.log10((img * self.restore_factor / img_sum) * 1.0 + 1.0) 142 | # 将Retinex做差后的图像，按照权重和颜色增益重新组合 143 | img_merge = retinex * color_restoration * self.color_gain 144 | # 恢复图像 145 | img_restore = img_merge * self.gain + self.offset 146 | return img_restore 147 | 148 | def _simplestColorBalance(self, img, low_clip, high_clip): 149 | total = img.shape[0] * img.shape[1] 150 | for i in range(img.shape[2]): 151 | unique, counts = np.unique(img[:, :, i], return_counts=True) # 返回新列表元素在旧列表中的位置，并以列表形式储存在s中 152 | current = 0 153 | for u, c in zip(unique, counts): 154 | if float(current) / total < low_clip: 155 | low_val = u 156 | if float(current) / total < high_clip: 157 | high_val = u 158 | current += c 159 | 160 | img[:, :, i] = np.maximum(np.minimum(img[:, :, i], high_val), low_val) 161 | 162 | return img 163 | 164 | def _MSRCR_GIMP(self, img): 165 | 166 | # self.img = results['img'] 167 | self.img = np.float32(img) + 1.0 168 | if self.model == 'SSR': 169 | self.retinex = self._SSR(self.img, self.sigma) 170 | elif self.model == 'MSR': 171 | self.retinex = self._MSR(self.img, self.sigma) 172 | # 颜色恢复 图像恢复 173 | self.img_restore = self._colorRestoration(self.img, self.retinex) 174 | 175 | return self.img_restore 176 | 177 | def __call__(self, img): 178 | return self._MSRCR_GIMP(img) 179 | 180 | def __repr__(self): 181 | repr_str = self.__class__.__name__ 182 | repr_str += '{},sigma={},dynamic={}'.format(self.model, self.sigma, self.Dynamic) 183 | return repr_str 184 | 185 | def parse_args(): 186 | parser = argparse.ArgumentParser(description='Train a detector') 187 | parser.add_argument('--configs', default='configs/dcn/10_1.py', help='train config file path') 188 | args = parser.parse_args() 189 | return args 190 | 191 | if __name__ == '__main__': 192 | 193 | img_root =r'F:\study\0_Project\sea_detection\retinex\retinex\test\000239.jpg' 194 | # img_root = r'F:\study\0_Project\sea_detection\MSRCR-Restoration-master\image\1.png' 195 | img = cv2.imread(img_root) 196 | retinex = Retinex() 197 | img = retinex(img) 198 | save_root = r'F:\study\0_Project\sea_detection\retinex\retinex\test\000239_test.jpg' 199 | cv2.imwrite(save_root, img) 200 | # 读数据 201 | # img_root = '../data/seacoco/train/' 202 | # img_list = os.listdir(img_root) 203 | # img_save = img_root.replace('train/', 'train_lable') 204 | # if not os.path.exists(img_save): 205 | # os.makedirs(img_save) 206 | # retinex = Retinex() 207 | # start = time.time() 208 | # 209 | # for img_name in tqdm(img_list): 210 | # read_root = img_root + img_name 211 | # save_root = img_save + '/' + img_name 212 | # img = cv2.imread(read_root) 213 | # img = retinex(img) 214 | # cv2.imwrite(save_root, img) 215 | # end = time.time() 216 | # print('total_time: {}'.format(end - start)) 217 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | fp16 = dict(loss_scale=512.) 2 | # model settings 3 | model = dict( 4 | type='CascadeRCNN', 5 | num_stages=3, 6 | pretrained=None, 7 | backbone=dict( 8 | type='ResNeXt', 9 | depth=101, 10 | groups=64, 11 | base_width=4, 12 | num_stages=4, 13 | out_indices=(0, 1, 2, 3), 14 | frozen_stages=1, 15 | style='pytorch'), 16 | neck=dict( 17 | type='FPN', 18 | in_channels=[256, 512, 1024, 2048], 19 | out_channels=256, 20 | num_outs=5), 21 | rpn_head=dict( 22 | type='RPNHead', 23 | in_channels=256, 24 | feat_channels=256, 25 | anchor_scales=[8], 26 | anchor_ratios=[0.5, 1.0, 2.0], 27 | anchor_strides=[4, 8, 16, 32, 64], 28 | target_means=[.0, .0, .0, .0], 29 | target_stds=[1.0, 1.0, 1.0, 1.0], 30 | loss_cls=dict( 31 | type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), 32 | loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), 33 | bbox_roi_extractor=dict( 34 | type='SingleRoIExtractor', 35 | roi_layer=dict(type='RoIAlign', out_size=7, sample_num=2), 36 | out_channels=256, 37 | featmap_strides=[4, 8, 16, 32]), 38 | bbox_head=[ 39 | dict( 40 | type='SharedFCBBoxHead', 41 | num_fcs=2, 42 | in_channels=256, 43 | fc_out_channels=1024, 44 | roi_feat_size=7, 45 | num_classes=6, 46 | target_means=[0., 0., 0., 0.], 47 | target_stds=[0.1, 0.1, 0.2, 0.2], 48 | reg_class_agnostic=True, 49 | loss_cls=dict( 50 | type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), 51 | loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), 52 | dict( 53 | type='SharedFCBBoxHead', 54 | num_fcs=2, 55 | in_channels=256, 56 | fc_out_channels=1024, 57 | roi_feat_size=7, 58 | num_classes=6, 59 | target_means=[0., 0., 0., 0.], 60 | target_stds=[0.05, 0.05, 0.1, 0.1], 61 | reg_class_agnostic=True, 62 | loss_cls=dict( 63 | type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), 64 | loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), 65 | dict( 66 | type='SharedFCBBoxHead', 67 | num_fcs=2, 68 | in_channels=256, 69 | fc_out_channels=1024, 70 | roi_feat_size=7, 71 | num_classes=6, 72 | target_means=[0., 0., 0., 0.], 73 | target_stds=[0.033, 0.033, 0.067, 0.067], 74 | reg_class_agnostic=True, 75 | loss_cls=dict( 76 | type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), 77 | loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) 78 | ]) 79 | # model training and testing settings 80 | train_cfg = dict( 81 | rpn=dict( 82 | assigner=dict( 83 | type='MaxIoUAssigner', 84 | pos_iou_thr=0.7, 85 | neg_iou_thr=0.3, 86 | min_pos_iou=0.3, 87 | ignore_iof_thr=-1), 88 | sampler=dict( 89 | type='RandomSampler', 90 | num=256, 91 | pos_fraction=0.5, 92 | neg_pos_ub=-1, 93 | add_gt_as_proposals=False), 94 | allowed_border=0, 95 | pos_weight=-1, 96 | debug=False), 97 | rpn_proposal=dict( 98 | nms_across_levels=False, 99 | nms_pre=2000, 100 | nms_post=2000, 101 | max_num=2000, 102 | nms_thr=0.7, 103 | min_bbox_size=0), 104 | rcnn=[ 105 | dict( 106 | assigner=dict( 107 | type='MaxIoUAssigner', 108 | pos_iou_thr=0.5, 109 | neg_iou_thr=0.5, 110 | min_pos_iou=0.5, 111 | ignore_iof_thr=-1), 112 | sampler=dict( 113 | type='RandomSampler', 114 | num=512, 115 | pos_fraction=0.25, 116 | neg_pos_ub=-1, 117 | add_gt_as_proposals=True), 118 | pos_weight=-1, 119 | debug=False), 120 | dict( 121 | assigner=dict( 122 | type='MaxIoUAssigner', 123 | pos_iou_thr=0.6, 124 | neg_iou_thr=0.6, 125 | min_pos_iou=0.6, 126 | ignore_iof_thr=-1), 127 | sampler=dict( 128 | type='RandomSampler', 129 | num=512, 130 | pos_fraction=0.25, 131 | neg_pos_ub=-1, 132 | add_gt_as_proposals=True), 133 | pos_weight=-1, 134 | debug=False), 135 | dict( 136 | assigner=dict( 137 | type='MaxIoUAssigner', 138 | pos_iou_thr=0.7, 139 | neg_iou_thr=0.7, 140 | min_pos_iou=0.7, 141 | ignore_iof_thr=-1), 142 | sampler=dict( 143 | type='RandomSampler', 144 | num=512, 145 | pos_fraction=0.25, 146 | neg_pos_ub=-1, 147 | add_gt_as_proposals=True), 148 | pos_weight=-1, 149 | debug=False) 150 | ], 151 | stage_loss_weights=[1, 0.5, 0.25]) 152 | test_cfg = dict( 153 | rpn=dict( 154 | nms_across_levels=False, 155 | nms_pre=1000, 156 | nms_post=1000, 157 | max_num=1000, 158 | nms_thr=0.7, 159 | min_bbox_size=0), 160 | rcnn=dict( 161 | score_thr=0.0001, nms=dict(type='soft_nms', iou_thr=0.5, min_score=0.0001), max_per_img=5000)) 162 | # dataset settings 163 | albu_train_transforms = [ 164 | # dict( 165 | # type='ShiftScaleRotate', 166 | # shift_limit=0.0625, 167 | # scale_limit=0.0, 168 | # rotate_limit=[-10, 10], 169 | # interpolation=1, 170 | # p=0.5), 171 | # dict( 172 | # type='IAASharpen', 173 | # p=0.5), 174 | # dict( 175 | # type='MotionBlur', 176 | # blur_limit=(3,7), 177 | # p=0.5), 178 | # dict( 179 | # type='RandomSizedBBoxSafeCrop', 180 | # height=1778, 181 | # width=800, 182 | # interpolation=1, 183 | # p=0.5), 184 | # dict( 185 | # type='RandomBrightnessContrast', 186 | # brightness_limit=[0.1, 0.3], 187 | # contrast_limit=[0.1, 0.3], 188 | # p=0.2), 189 | # dict( 190 | # type='OneOf', 191 | # transforms=[ 192 | # dict( 193 | # type='RGBShift', 194 | # r_shift_limit=10, 195 | # g_shift_limit=10, 196 | # b_shift_limit=10, 197 | # p=1.0), 198 | # dict( 199 | # type='HueSaturationValue', 200 | # hue_shift_limit=20, 201 | # sat_shift_limit=30, 202 | # val_shift_limit=20, 203 | # p=1.0) 204 | # ], 205 | # p=0.1), 206 | # dict(type='ChannelShuffle', p=0.1), 207 | dict( 208 | type='OneOf', 209 | transforms=[ 210 | dict(type='CLAHE', clip_limit=2), 211 | dict(type='IAASharpen'), 212 | dict(type='IAAEmboss'), 213 | dict(type='RandomBrightnessContrast'), 214 | ], 215 | p=0.3), 216 | # dict( 217 | # type='OneOf', 218 | # transforms=[ 219 | # dict(type='Blur', blur_limit=3, p=1.0), 220 | # dict(type='MedianBlur', blur_limit=3, p=1.0) 221 | # ], 222 | # p=0.1), 223 | 224 | ] 225 | dataset_type = 'Underwater' 226 | data_root = 'data/' 227 | img_norm_cfg = dict( 228 | mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) 229 | train_pipeline = [ 230 | dict(type='LoadImageFromFile'), 231 | dict(type='LoadAnnotations', with_bbox=True), 232 | dict(type='Resize', img_scale=[(4096, 600), (4096, 1400)], 233 | multiscale_mode='range', keep_ratio=True), 234 | dict(type='RandomFlip', flip_ratio=0.5), 235 | dict(type='Pad', size_divisor=32), 236 | dict( 237 | type='Albu', 238 | transforms=albu_train_transforms, 239 | bbox_params=dict( 240 | type='BboxParams', 241 | format='pascal_voc', 242 | label_fields=['gt_labels'], 243 | min_visibility=0.0, 244 | filter_lost_elements=True), 245 | keymap={ 246 | 'img': 'image', 247 | 'gt_bboxes': 'bboxes' 248 | }, 249 | update_pad_shape=False, 250 | skip_img_without_anno=True), 251 | dict(type='Normalize', **img_norm_cfg), 252 | dict(type='DefaultFormatBundle'), 253 | dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), 254 | ] 255 | 256 | test_pipeline = [ 257 | dict(type='LoadImageFromFile'), 258 | dict( 259 | type='MultiScaleFlipAug', 260 | img_scale=[(4096, 600), (4096, 800), (4096, 1000)], 261 | flip=True, 262 | transforms=[ 263 | dict(type='Resize', keep_ratio=True), 264 | dict(type='RandomFlip'), 265 | dict(type='Normalize', **img_norm_cfg), 266 | dict(type='Pad', size_divisor=32), 267 | dict(type='ImageToTensor', keys=['img']), 268 | dict(type='Collect', keys=['img']), 269 | ]) 270 | ] 271 | data = dict( 272 | imgs_per_gpu=1, 273 | workers_per_gpu=2, 274 | train=dict( 275 | type=dataset_type, 276 | ann_file=data_root + 'all_waterweeds.json', 277 | img_prefix=data_root + 'train_val/', 278 | pipeline=train_pipeline), 279 | val=dict( 280 | type=dataset_type, 281 | ann_file=data_root + 'val_waterweeds.json', 282 | img_prefix=data_root + 'val/', 283 | pipeline=test_pipeline), 284 | test=dict( 285 | type=dataset_type, 286 | ann_file=data_root + 'val_waterweeds.json', 287 | img_prefix=data_root + 'val/', 288 | pipeline=test_pipeline)) 289 | # optimizer 290 | optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001) # (base_lr0.02 / 8) x num_gpus4 x (img_per_gpu1/2) 291 | optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) 292 | # learning policy 293 | lr_config = dict( 294 | policy='step', 295 | warmup='linear', 296 | warmup_iters=500, 297 | warmup_ratio=1.0 / 3, 298 | step=[8, 11]) 299 | checkpoint_config = dict(interval=1) 300 | # yapf:disable 301 | log_config = dict( 302 | interval=50, 303 | hooks=[ 304 | dict(type='TextLoggerHook'), 305 | # dict(type='TensorboardLoggerHook') 306 | ]) 307 | # yapf:enable 308 | # runtime settings 309 | total_epochs = 15 310 | dist_params = dict(backend='nccl') 311 | log_level = 'INFO' 312 | import datetime 313 | exp_time = datetime.datetime.now().strftime('%Y%m%d%H%M') 314 | 315 | work_dir = 'tools/work_dirs/p_{}'.format(exp_time) 316 | load_from = 'model/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth' 317 | 318 | # debug 319 | # work_dir = 'work_dirs/p_{}'.format(exp_time) 320 | # load_from = '../model/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth' 321 | 322 | resume_from = None 323 | workflow = [('train', 1)] 324 | -------------------------------------------------------------------------------- /code/Weighted-Boxes-Fusion/example_oid.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | 5 | import os 6 | import time 7 | import pickle 8 | import numpy as np 9 | import pandas as pd 10 | from multiprocessing import Pool, cpu_count 11 | from itertools import repeat 12 | from ensemble_boxes import * 13 | from map_boxes import * 14 | 15 | 16 | def save_in_file_fast(arr, file_name): 17 | pickle.dump(arr, open(file_name, 'wb')) 18 | 19 | 20 | def load_from_file_fast(file_name): 21 | return pickle.load(open(file_name, 'rb')) 22 | 23 | 24 | def get_detections(path): 25 | preds = pd.read_csv(path) 26 | ids = preds['ImageId'].values 27 | preds_strings = preds['PredictionString'].values 28 | 29 | ImageID = [] 30 | LabelName = [] 31 | Conf = [] 32 | XMin = [] 33 | XMax = [] 34 | YMin = [] 35 | YMax = [] 36 | for j in range(len(ids)): 37 | # print('Go for {}'.format(ids[j])) 38 | id = ids[j] 39 | if str(preds_strings[j]) == 'nan': 40 | continue 41 | arr = preds_strings[j].strip().split(' ') 42 | if len(arr) % 6 != 0: 43 | print('Some problem here! {}'.format(id)) 44 | exit() 45 | for i in range(0, len(arr), 6): 46 | ImageID.append(id) 47 | LabelName.append(arr[i]) 48 | Conf.append(float(arr[i + 1])) 49 | XMin.append(float(arr[i + 2])) 50 | XMax.append(float(arr[i + 4])) 51 | YMin.append(float(arr[i + 3])) 52 | YMax.append(float(arr[i + 5])) 53 | 54 | res = pd.DataFrame(ImageID, columns=['ImageId']) 55 | res['LabelName'] = LabelName 56 | res['Conf'] = Conf 57 | res['XMin'] = XMin 58 | res['XMax'] = XMax 59 | res['YMin'] = YMin 60 | res['YMax'] = YMax 61 | 62 | return res 63 | 64 | 65 | def process_single_id(id, res, weights, params): 66 | run_type = params['run_type'] 67 | verbose = params['verbose'] 68 | 69 | if verbose: 70 | print('Go for ID: {}'.format(id)) 71 | boxes_list = [] 72 | scores_list = [] 73 | labels_list = [] 74 | labels_to_use_forward = dict() 75 | labels_to_use_backward = dict() 76 | 77 | for i in range(len(res[id])): 78 | boxes = [] 79 | scores = [] 80 | labels = [] 81 | 82 | dt = res[id][i] 83 | if str(dt) == 'nan': 84 | boxes = np.zeros((0, 4), dtype=np.float32) 85 | scores = np.zeros((0, ), dtype=np.float32) 86 | labels = np.zeros((0, ), dtype=np.int32) 87 | boxes_list.append(boxes) 88 | scores_list.append(scores) 89 | labels_list.append(labels) 90 | continue 91 | 92 | pred = dt.strip().split(' ') 93 | 94 | # Empty preds 95 | if len(pred) <= 1: 96 | boxes = np.zeros((0, 4), dtype=np.float32) 97 | scores = np.zeros((0,), dtype=np.float32) 98 | labels = np.zeros((0,), dtype=np.int32) 99 | boxes_list.append(boxes) 100 | scores_list.append(scores) 101 | labels_list.append(labels) 102 | continue 103 | 104 | # Check correctness 105 | if len(pred) % 6 != 0: 106 | print('Erorr % 6 {}'.format(len(pred))) 107 | print(dt) 108 | exit() 109 | 110 | for j in range(0, len(pred), 6): 111 | lbl = pred[j] 112 | scr = float(pred[j + 1]) 113 | box_x1 = float(pred[j + 2]) 114 | box_y1 = float(pred[j + 3]) 115 | box_x2 = float(pred[j + 4]) 116 | box_y2 = float(pred[j + 5]) 117 | 118 | if box_x1 >= box_x2: 119 | if verbose: 120 | print('Problem with box x1 and x2: {}. Skip it'.format(pred[j:j+6])) 121 | continue 122 | if box_y1 >= box_y2: 123 | if verbose: 124 | print('Problem with box y1 and y2: {}. Skip it'.format(pred[j:j+6])) 125 | continue 126 | if scr <= 0: 127 | if verbose: 128 | print('Problem with box score: {}. Skip it'.format(pred[j:j+6])) 129 | continue 130 | 131 | boxes.append([box_x1, box_y1, box_x2, box_y2]) 132 | scores.append(scr) 133 | if lbl not in labels_to_use_forward: 134 | cur_point = len(labels_to_use_forward) 135 | labels_to_use_forward[lbl] = cur_point 136 | labels_to_use_backward[cur_point] = lbl 137 | labels.append(labels_to_use_forward[lbl]) 138 | 139 | boxes = np.array(boxes, dtype=np.float32) 140 | scores = np.array(scores, dtype=np.float32) 141 | labels = np.array(labels, dtype=np.int32) 142 | 143 | boxes_list.append(boxes) 144 | scores_list.append(scores) 145 | labels_list.append(labels) 146 | 147 | # Empty predictions for all models 148 | if len(boxes_list) == 0: 149 | return np.array([]), np.array([]), np.array([]) 150 | 151 | if run_type == 'wbf': 152 | merged_boxes, merged_scores, merged_labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, 153 | weights=weights, iou_thr=params['intersection_thr'], 154 | skip_box_thr=params['skip_box_thr'], 155 | conf_type=params['conf_type']) 156 | elif run_type == 'nms': 157 | iou_thr = params['iou_thr'] 158 | merged_boxes, merged_scores, merged_labels = nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr) 159 | elif run_type == 'soft-nms': 160 | iou_thr = params['iou_thr'] 161 | sigma = params['sigma'] 162 | thresh = params['thresh'] 163 | merged_boxes, merged_scores, merged_labels = soft_nms(boxes_list, scores_list, labels_list, 164 | weights=weights, iou_thr=iou_thr, sigma=sigma, thresh=thresh) 165 | elif run_type == 'nmw': 166 | merged_boxes, merged_scores, merged_labels = non_maximum_weighted(boxes_list, scores_list, labels_list, 167 | weights=weights, iou_thr=params['intersection_thr'], 168 | skip_box_thr=params['skip_box_thr']) 169 | 170 | if verbose: 171 | print(len(boxes_list), len(merged_boxes)) 172 | if 'limit_boxes' in params: 173 | limit_boxes = params['limit_boxes'] 174 | if len(merged_boxes) > limit_boxes: 175 | merged_boxes = merged_boxes[:limit_boxes] 176 | merged_scores = merged_scores[:limit_boxes] 177 | merged_labels = merged_labels[:limit_boxes] 178 | 179 | # Rename labels back 180 | merged_labels_string = [] 181 | for m in merged_labels: 182 | merged_labels_string.append(labels_to_use_backward[m]) 183 | merged_labels = np.array(merged_labels_string, dtype=np.str) 184 | 185 | # Create IDs array 186 | ids_list = [id] * len(merged_labels) 187 | 188 | return merged_boxes, merged_scores, merged_labels, ids_list 189 | 190 | 191 | def ensemble_predictions(pred_filenames, weights, params): 192 | verbose = False 193 | if 'verbose' in params: 194 | verbose = params['verbose'] 195 | 196 | start_time = time.time() 197 | procs_to_use = max(cpu_count() // 2, 1) 198 | # procs_to_use = 1 199 | if verbose: 200 | print('Use processes: {}'.format(procs_to_use)) 201 | 202 | res = dict() 203 | ref_ids = None 204 | for j in range(len(pred_filenames)): 205 | s = pd.read_csv(pred_filenames[j]) 206 | try: 207 | s.sort_values('ImageId', inplace=True) 208 | except: 209 | s.sort_values('ImageID', inplace=True) 210 | s.reset_index(drop=True, inplace=True) 211 | try: 212 | ids = s['ImageId'].values 213 | except: 214 | ids = s['ImageID'].values 215 | preds = s['PredictionString'].values 216 | if ref_ids is None: 217 | ref_ids = tuple(ids) 218 | else: 219 | if ref_ids != tuple(ids): 220 | print('Different IDs in ensembled CSVs!') 221 | exit() 222 | for i in range(len(ids)): 223 | id = ids[i] 224 | if id not in res: 225 | res[id] = [] 226 | res[id].append(preds[i]) 227 | 228 | p = Pool(processes=procs_to_use) 229 | ids_to_use = sorted(list(res.keys())) 230 | results = p.starmap(process_single_id, zip(ids_to_use, repeat(res), repeat(weights), repeat(params))) 231 | 232 | all_ids = [] 233 | all_boxes = [] 234 | all_scores = [] 235 | all_labels = [] 236 | for boxes, scores, labels, ids_list in results: 237 | if boxes is None: 238 | continue 239 | all_boxes.append(boxes) 240 | all_scores.append(scores) 241 | all_labels.append(labels) 242 | all_ids.append(ids_list) 243 | 244 | all_ids = np.concatenate(all_ids) 245 | all_boxes = np.concatenate(all_boxes) 246 | all_scores = np.concatenate(all_scores) 247 | all_labels = np.concatenate(all_labels) 248 | if verbose: 249 | print(all_ids.shape, all_boxes.shape, all_scores.shape, all_labels.shape) 250 | 251 | res = pd.DataFrame(all_ids, columns=['ImageId']) 252 | res['LabelName'] = all_labels 253 | res['Conf'] = all_scores 254 | res['XMin'] = all_boxes[:, 0] 255 | res['XMax'] = all_boxes[:, 2] 256 | res['YMin'] = all_boxes[:, 1] 257 | res['YMax'] = all_boxes[:, 3] 258 | if verbose: 259 | print('Run time: {:.2f}'.format(time.time() - start_time)) 260 | return res 261 | 262 | 263 | if __name__ == '__main__': 264 | if 1: 265 | params = { 266 | 'run_type': 'nms', 267 | 'iou_thr': 0.5, 268 | 'verbose': True, 269 | } 270 | if 1: 271 | params = { 272 | 'run_type': 'soft-nms', 273 | 'iou_thr': 0.5, 274 | 'thresh': 0.0001, 275 | 'sigma': 0.1, 276 | 'verbose': True, 277 | } 278 | if 1: 279 | params = { 280 | 'run_type': 'nmw', 281 | 'skip_box_thr': 0.000000001, 282 | 'intersection_thr': 0.5, 283 | 'limit_boxes': 30000, 284 | 'verbose': True, 285 | } 286 | if 1: 287 | params = { 288 | 'run_type': 'wbf', 289 | 'skip_box_thr': 0.0000001, 290 | 'intersection_thr': 0.6, 291 | 'conf_type': 'avg', 292 | 'limit_boxes': 30000, 293 | 'verbose': True, 294 | } 295 | 296 | # Files available here: https://github.com/ZFTurbo/Weighted-Boxes-Fusion/releases/download/v1.0/test_data.zip 297 | annotations_path = 'test_data/challenge-2019-validation-detection-bbox-expand_3520.csv' 298 | pred_list = [ 299 | 'test_data/0.46450_TF_IRV2_atrous_3520.csv', 300 | 'test_data/0.52319_mmdet_3520.csv', 301 | 'test_data/0.52918_tensorpack1_3520.csv', 302 | 'test_data/0.53775_tensorpack2_3520.csv', 303 | 'test_data/0.51145_retinanet_3520.csv', 304 | ] 305 | weights = [1, 1, 1, 1, 1] 306 | 307 | ann = pd.read_csv(annotations_path) 308 | ann = ann[['ImageId', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values 309 | 310 | # Find initial scores 311 | for i in range(len(pred_list)): 312 | det = get_detections(pred_list[i]) 313 | det = det[['ImageId', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values 314 | mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det, verbose=False) 315 | print("File: {} mAP: {:.6f}".format(os.path.basename(pred_list[i]), mean_ap)) 316 | 317 | ensemble_preds = ensemble_predictions(pred_list, weights, params) 318 | ensemble_preds.to_csv("test_data/debug.csv", index=False) 319 | ensemble_preds = ensemble_preds[['ImageId', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values 320 | mean_ap, average_precisions = mean_average_precision_for_boxes(ann, ensemble_preds, verbose=True) 321 | print("Ensemble [{}] Weights: {} Params: {} mAP: {:.6f}".format(len(weights), weights, params, mean_ap)) 322 | -------------------------------------------------------------------------------- /logs/Logs.md: -------------------------------------------------------------------------------- 1 | # 模型修改日志 2 | 3 | [toc] 4 | 5 | ## Faster Rcnn overview 6 | 7 | | date | MAP | MAP50 | MAP75 | s | m | l | score | loss | 8 | | ---------- | --------- | --------- | --------- | --------- | --------- | --------- | -------------- | -------- | 9 | | 3/10_3 | 0.499 | 0.853 | 0.533 | 0.253 | 0.463 | 0.553 | 0.45975257 | 0.39 | 10 | | **3/10_1** | **0.500** | **0.851** | **0.536** | **0.248** | **0.470** | **0.553** | **0.46365661** | **0.24** | 11 | | 3/9_2 | 0.490 | 0.842 | 0.524 | 0.242 | 0.463 | 0.540 | 0.44945764 | 0.21 | 12 | | 3/8_3 | 0.474 | 0.839 | 0.486 | 0.232 | 0.446 | 0.527 | 0.43323416 | 0.2 | 13 | | 3/8_2 | 0.462 | 0.832 | 0.463 | 0.223 | 0.441 | 0.508 | 0.42316451 | 0.2 | 14 | | 3/8_1 | 0.467 | 0.834 | 0.475 | 0.218 | 0.446 | 0.516 | 0.42315355 | 0.21 | 15 | | 3/6 | 0.465 | 0.833 | 0.465 | 0.229 | 0.44 | 0.513 | 0.42745937 | 0.22 | 16 | 17 | | data | bb | lr | ep | anchro_ratio | 多尺度训练 | nms_thr | OHEM | 18 | | ---------- | ------- | -------- | ------ | --------------------- | ----------------------------- | ------------------- | ----- | 19 | | 3/10_3 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (4096, 800), (4096, 1200) | 0.001(soft_nms) | √ | 20 | | **3/10_1** | **r50** | **0.02** | **12** | **[.2, .5, 1, 2, 5]** | **(4096, 800), (4096, 1200)** | **0.001(soft_nms)** | **×** | 21 | | 3/9_2 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1333, 800)(1333, 1200) | 0.001(soft_nms) | × | 22 | | 3/8_3 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1333, 800)(1333, 1200) | 0.05 | × | 23 | | 3/8_2 | r50 | 0.02 | 12 | [ .5, 1, 2] | (1080, 920) | 0.05 | × | 24 | | 3/8_1 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1080, 920)改为dcn权重 | 0.05 | × | 25 | | 3/6 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1080, 920) | 0.05 | × | 26 | 27 | ## Cascade overview 28 | 29 | | date | MAP | MAP50 | MAP75 | s | m | l | score | loss | 30 | | ---------- | --------- | --------- | --------- | --------- | --------- | --------- | -------------- | -------- | 31 | | **3/10_2** | **0.498** | **0.836** | **0.540** | **0.242** | **0.464** | **0.553** | **0.45107298** | **0.65** | 32 | | 3/9_1 | 0.481 | 0.821 | 0.518 | 0.237 | 0.457 | 0.528 | 0.43794168 | 0.68 | 33 | | 3/7 | 0.413 | 0.722 | 0.425 | 0.179 | 0.395 | 0.452 | 0.36510317 | 0.66 | 34 | | 3/5 | 0.401 | 0.689 | 0.433 | 0.149 | 0.367 | 0.462 | 0.34733043 | 0.89 | 35 | | 3/4 | 0.405 | 0.702 | 0.429 | 0.157 | 0.372 | 0.468 | 0.35986083 | 1.0 | 36 | | 3/3 | 0.475 | 0.832 | 0.497 | 0.224 | 0.448 | 0.528 | 0.39238524 | 0.6 | 37 | 38 | | data | bb | lr | ep | anchro_ratio | 多尺度训练 | nms_thr | dcn | OHEM | 39 | | ---------- | ------- | -------- | ------ | ----------------- | ---------------------------- | ------------------ | ----- | ----- | 40 | | **3/10_2** | **r50** | **0.02** | **12** | **[.5, 1, 2]** | **(4096, 800), (4096,1200)** | **.001(soft_nms)** | **×** | **×** | 41 | | 3/9_1 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (4096, 800), (4096,1200) | .001(soft_nms) | √ | × | 42 | | 3/7 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1920, 1080), (720, 405) | 0.5 | √ | × | 43 | | 3/5 | r101 | 0.04 | 22 | [.2, .5, 1, 2, 5] | (1080, 920) | 0.5 | √ | √ | 44 | | 3/4 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1080, 920) | 0.5 | √ | √ | 45 | | 3/3 | r50 | 0.02 | 12 | [.2, .5, 1, 2, 5] | (1080, 920) | 0.5 | √ | × | 46 | 47 | + 前期受到了nms的影响，提高了预测的nms的阈值，导致coco方法计算map时普遍降低 48 | + 确实尺度越大准确率越高 49 | 50 | ## 20200303 51 | 52 | **Config** 53 | 54 | | baseline | lr | step | anchor_ratios | ima_scale | 55 | | ------------------------ | ---- | ------- | ------------------------- | ----------- | 56 | | cascade_rcnn_dcn_r50_fpn | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1080, 920) | 57 | 58 | + 观测数据，发现数据集多呈序列分布，若用随机抽样可能会因场景分布产生误差，故采用间隔抽样，按照0.85：0.15的比例切分训练集和验证集 59 | 60 | + 验证序列抽样后，训练集、验证集、原始数据集各目标分布基本保持不变 61 | 62 | | dataset | holothurian | echinus | scallop | starfish | 63 | | ------- | ----------- | ------- | ------- | -------- | 64 | | org | 5537 | 22343 | 6720 | 6814 | 65 | | train | 4574 | 18676 | 5554 | 5704 | 66 | | val | 963 | 3667 | 1166 | 1137 | 67 | 68 | + 针对图像大小分布情况，当时选取了靠近(720,405)，且略大的分辨率，mmdet保持比例不变 69 | 70 | ![1583591299153](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583591299153.png) 71 | 72 | 这里疑似会出问题，若图像大小为(3840,2160)，resize到(1080, 920)，图像缩小3.5倍，可能会删除较小标注框 73 | 74 | 结果图 75 | 76 | ![1583592005705](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583592005705.png) 77 | 78 | **结论：** 79 | 80 | 1. 修改学习率意义不大，模型拟合较好，map和损失都较早趋于稳定 81 | 82 | ## 20200304 83 | 84 | **Config** 85 | 86 | | baseline | lr | step | anchor_ratios | ima_scale | 87 | | ------------------------ | ---- | ------- | ------------------------- | ----------- | 88 | | cascade_rcnn_dcn_r50_fpn | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1080, 920) | 89 | 90 | + 引入了OHEM，在线难样本挖掘，在cascade的三个阶段。本想使用focal loss，但focal loss的根本目的是解决one-stage中，正负样本分布不均衡，简单负样本过多，导致模型收敛更快，但却没有很好拟合困难正样本的知识。使用的cascade网络，本身是two-stage，已经解决了正负样本分布不均衡的问题 91 | + 其他什么都没有改，模型在验证集下降了0.07个点，在平台得分下降了0.04个点，**OHEM不是直接改了就能用，需要读一下原文，了解适用环境** 92 | 93 | ![1583592621770](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583592621770.png) 94 | 95 | **结论：** 96 | 97 | 1. 看loss曲线，模型欠拟合，且下降幅度较小，感觉可以略微提高学习率，增加训练epoch 98 | 2. 引入OHEM，损失先上升，后下降。map的起点精度均降低 (mAP: 0.34 ->0.22) 99 | 100 | ## 20200305 101 | 102 | **Config** 103 | 104 | | baseline | lr | step | anchor_ratios | ima_scale | 105 | | ------------------------ | ---- | -------- | ------------------------- | ----------- | 106 | | cascade_rcnn_dcn_r50_fpn | 0.04 | [16, 19] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1080, 920) | 107 | 108 | + backbone换成了 ResNet101 109 | + 采样器依然是OHEMSampler 110 | + 提高学习率0.02->0.04 111 | + 增加了训练周期到22个epoch 112 | 113 | ![1583593178075](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583593178075.png) 114 | 115 | **总结：** 116 | 117 | 1. 1-5个epoch，使用四个GPU，一个epoch迭代285次 118 | 6-22个epoch，使用两个GPU，一个epoch迭代570次 119 | 2. 增大学习率，算法在1-5个epoch损失下降于4日相比没有明显变化，均为1.2左右。提高了学习率网络并非收敛更快，反而更早的趋于稳定，感觉学习率给大了，学习率降低后，网络还是可以降到0.9的损失，**且mAP与4日差别不大**，**ResNet从50->101，没有发挥作用，OHEM没有配合默契，限制了模型表达** 120 | 3. mAP起点更低，在第16个epoch更新学习率，损失有明显下降，MAP有明显提升，但随后趋于稳定，在第19个epoch更新学习率，从损失上看略早，**还可以延后**，学习率降低后，网络趋于稳定，可能是train不动了 121 | 122 | ## 20200306 123 | 124 | **Config** 125 | 126 | | baseline | lr | step | anchor_ratios | ima_scale | 127 | | ----------------------- | ---- | ------- | ------------------------- | ----------- | 128 | | faster_rcnn_dcn_r50_fpn | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1080, 920) | 129 | 130 | + 原基础上加了anchor_ratios = [0.2, 0.5, 1.0, 2.0, 5.0] 131 | + 使用RandomSampler作为抽样器，未修改 132 | + 训练尺度，使用当前最优尺度(1080, 920) 133 | + 预训练权重使用的faster_rcnn_r50_fpn_1x_cls_24.pth，并非dcn的权重 134 | 135 | ![1583595721774](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583595721774.png) 136 | 137 | **总结：** 138 | 139 | 1. faster-rcnn损失起点更低，但是这与cascade的损失没有对比意义 140 | 2. 在第8个epoch调整学习率（4600次iter左右），损失有略微下降，第11次修改学习率，损失基本不变，模型拟合较好 141 | 3. 在第8个epoch调整学习率，map有略微升高，后趋于稳定 142 | 4. 这是目前最优baseline，考虑在此基础上做两个试验 143 | + **使用dcn的预训练权重，其他参数不变 --1** 基本曲线没变化 144 | + **使用原始ratio，其他参数不变 --2** 145 | + **使用多尺度训练，只修改短边，多尺度使用推荐尺度(1300, 1200) (1300, 800) 测试使用(1300,1000) --3** 146 | 147 | ## 20200307 148 | 149 | **Config** 150 | 151 | | baseline | lr | step | anchor_ratios | ima_scale | 152 | | ------------------------ | ---- | ------- | ------------------------- | ------------------------ | 153 | | cascade_rcnn_dcn_r50_fpn | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1920, 1080), (720, 405) | 154 | 155 | + 使用初始的cascade_r50训练 156 | + 使用RandomSampler作为抽样器，未修改 157 | + 添加了多尺度训练，使用两个尺度img_scale=[(1920, 1080), (720, 405)]， 测试的是scale使用的(1920, 1080) 158 | + 加载的预训练文件出现问题，加载的r101的预训练文件cascade_rcnn_dconv_c3-c5_r101_fpn_1x 159 | 160 | ![1583596919791](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583596919791.png) 161 | 162 | **总结：** 163 | 164 | 1. 经过eda， (720, 405)的图像更多，使用img_scale=[(1920, 1080), (720, 405)]，会导致部分图像缩放 165 | 2. 与3月3日对比，多尺度下，网络的损失起点更低 166 | 3. 在第七次更新学习率时，网络的损失还在下降，网络还没稳定，应延后调整学习率 167 | 4. 但从map来看，网络最终趋于稳定 168 | 169 | ## 20200308对照试验 170 | 171 | **Config** 172 | 173 | | baseline | lr | step | anchor_ratios | ima_scale | 174 | | ---------- | ---- | ------- | ------------------------- | ----------- | 175 | | FasterRCNN | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1080, 920) | 176 | 177 | + 20200306的对照组试验，使用了dcn训练权重faster_rcnn_dconv_c3-c5_r50_fpn_1x 178 | 179 | ![1583675087773](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583675087773.png) 180 | 181 | **总结：** 182 | 183 | + 与使用faster_rcnn_r50_fpn_1x_cls_24.pth对比，loss下降曲线基本完全一致 184 | + 使用dcn，更契合的权重后，Map起始有些下降，其他曲线基本与之前一致 185 | + **结果差距不大**，解释：其实有一个较好的权重初始就够了，后面有多轮的训练，都会让网络趋向于收敛 186 | 187 | --- 188 | 189 | **Config** 190 | 191 | | baseline | lr | step | anchor_ratios | ima_scale | 192 | | ---------- | ---- | ------- | --------------- | ----------- | 193 | | FasterRCNN | 0.02 | [8, 11] | [0.5, 1.0, 2.0] | (1080, 920) | 194 | 195 | + 使用原始ratio，其他参数不变 196 | 197 | ![1583679520091](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583679520091.png) 198 | 199 | **总结：** 200 | 201 | + 改为初始ration[0.5, 1, 2]，MAP,MAP50,MAP75,S,M,L，六项指标均有所下降，总得分下降 202 | + anchor多尺度还是有意义的 203 | 204 | --- 205 | 206 | **Config** 207 | 208 | | baseline | lr | step | anchor_ratios | ima_scale | 209 | | ---------- | ---- | ------- | ------------------------- | ----------------------- | 210 | | FasterRCNN | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1333, 800)(1333, 1200) | 211 | 212 | + 使用官方标准多尺度训练，训练尺度(1333,1200)(1333,800)，测试尺度取短边中值(1333,1000)其他参数不变 213 | 214 | ![1583679878973](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583679878973.png) 215 | 216 | + MAP75 MAP_S(<32*32),MAP_M(32 * 32 < s< 96*96),MAP_I(<96 *96)均上升0.01-0.02个点，说明多尺度训练，让网络对不同尺度的目标检测效果更好 217 | 218 | **Case** 219 | 220 | 221 | 222 | ## 20200309 223 | 224 | **Config** 225 | 226 | | baseline | lr | step | anchor_ratios | ima_scale | 227 | | ------------------------ | ---- | ------- | ------------------------- | ------------------------- | 228 | | cascade_rcnn_dcn_r50_fpn | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (4096, 800), (4096, 1200) | 229 | 230 | + 使用nms_soft修改预测阈值，提高对于遮挡的鲁棒性 231 | + 提高尺度长边上限。选取样本集样本最大长宽比，将短边限制为800-1200，将长边按照样本集最大比例缩放。再GPU容量足够，且未达到速度容忍度时，短边可提高限制 232 | + score_thr 从0.5改为0.001 233 | 234 | ![1583757891581](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583757891581.png) 235 | 236 | --- 237 | 238 | **Config** 239 | 240 | | baseline | lr | step | anchor_ratios | ima_scale | 241 | | -------- | ---- | ------- | ------------------------- | ------------------------- | 242 | | faster | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (1333, 1200), (1333, 800) | 243 | 244 | + 使用nms_soft修改预测阈值，提高对于遮挡的鲁棒性 245 | 246 | ![1583758287656](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583758287656.png) 247 | 248 | ## 20200310 249 | 250 | **Config** 251 | 252 | | baseline | lr | step | anchor_ratios | ima_scale | 253 | | -------- | ---- | ------- | ------------------------- | ------------------------- | 254 | | faster | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (4096, 800), (4096, 1200) | 255 | 256 | + 修改了多尺度训练的参数 257 | 258 | ![1583772207178](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583772207178.png) 259 | 260 | --- 261 | 262 | **Config** 263 | 264 | | baseline | lr | step | anchor_ratios | ima_scale | 265 | | ----------- | ---- | ------- | --------------- | ------------------------- | 266 | | CascadeRCNN | 0.02 | [8, 11] | [0.5, 1.0, 2.0] | (4096, 800), (4096, 1200) | 267 | 268 | + 复现群内大神结果 269 | + 删除FCN 270 | + anchor_ratios=[0.5, 1.0, 2.0] 271 | + 测试配置中加入了随机旋转 272 | 273 | ![1583817792570](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583817792570.png) 274 | 275 | --- 276 | 277 | **Config** 278 | 279 | | baseline | lr | step | anchor_ratios | ima_scale | 280 | | -------- | ---- | ------- | ------------------------- | ------------------------- | 281 | | faster | 0.02 | [8, 11] | [0.2, 0.5, 1.0, 2.0, 5.0] | (4096, 800), (4096, 1200) | 282 | 283 | + 在0310_1的基础上添加了OHEM 284 | 285 | ![1583819552453](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1583819552453.png) 286 | 287 | ## 20200312 288 | 289 | 针对（1317,1079,3）的图， 290 | 291 | 服务器跑cv2的高斯滤波，第一张图需要20.4s 292 | 293 | 笔记本跑需要近50s 294 | 295 | 使用图像金字塔滤波，多尺度滤波只需要0.18199s 296 | 297 | ![1584368673598](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1584368673598.png) 298 | 299 | 使用图像金字塔滤波，单尺度只需要0.05s 300 | 301 | ![1584368760018](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1584368760018.png) 302 | 303 | 必须用cython或者用torch的卷积改写 304 | 305 | ```python 306 | cv2.GaussianBlur(img, (0, 0), self.sigma) 307 | ``` 308 | 309 | 310 | 311 | 单尺度 对于double型图像 size为100 处理时间为0.39s 针对1317,1079的图 312 | 313 | ![1584100226457](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1584100226457.png) 314 | 315 | 单尺度 对于double型图像 size为200 处理时间为7s 针对1317,1079的图 316 | 317 | ![1584102699456](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1584102699456.png) 318 | 319 | 单尺度 对于double型图像 size为500 处理时间为7s 针对1317,1079的图 320 | 321 | ![1584102900836](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1584102900836.png) 322 | 323 | 不同的高斯kernel对图像的影响。sigma = 300，使用GIMP的MSRCR算法，单尺度，Dynamic = 2 324 | 325 | 高斯模糊在这里主要起到模拟光照成像，认为光照的成像是稳定的，变化是缓慢的，是低频的，对应背景和轮廓。所以用低通滤波器 326 | 327 | sigma确定时，kernel的size越大，生成的图像越模糊，频带更窄，阻碍了高频，原图-生成的图(留下低频背景轮廓) = 那么相减，得到的就是滤过的高频细节，剪掉轮廓，图像更清晰 328 | 329 | sigma确定时，kernel的size越小，生成的图像越清晰，频带更宽，通过了高频，原图-生成的图(滤掉少量的高频) = 那么相减，得到的就是滤过的少量的高频细节和噪声 相当于高通 330 | 331 | 在这个问题中，size越大，得到的低频更纯净（高频阻碍的越多）-> 模拟的低频稳定光照成像更完美 332 | size越小，得到的低频混杂更多高频（高频阻碍的少）—> 模拟的低频稳定光照杂质细节更多，相减就生下了少量的细节，如图2 333 | 334 | ![1584192507227](C:\Users\60155\AppData\Roaming\Typora\typora-user-images\1584192507227.png) 335 | 336 | > 高斯滤波： 337 | > 338 | > kernel_size固定的情况下， 339 | > 频域：时域sigma越大，变换后频域sigma越小，频带越窄，阻碍更多高频细节噪声，图像越模糊 340 | > 图像：权值分布越平缓，频带越宽，领域各个点的值对输出值影响越大，通过保留的频率越多，图像越模糊 341 | > 342 | > 频域：时域sigma越小，变换后频域sigma越大，频带越宽，通过更多的高频细节噪声，图像相对尖锐 343 | > 图像：权值分布越突起，频带越窄，领域各个点的值对输出值影响越小，滤除的高频成分越多，图像变换较小 344 | > 345 | > sigma固定时， 346 | > 核越大图像越模糊， 347 | > 核越小图像变化越小，相对更尖锐 348 | > 349 | > 高斯函数的实现 350 | > 351 | > 由于高斯函数的可分离性，较大尺寸的高斯滤波器可以得以有效地实现．二维高斯函数卷积可以分两步来进行，首先将图像与一维高斯函数进行卷积，然后将卷积结果与方向垂直的相同一维高斯函数卷积．因此，二维高斯滤波的计算量随滤波模板宽度成线性增长而不是成平方增长． 352 | > 353 | > 核大对应频率低 核小对应频率高 354 | 355 | 因为当高斯函数的sigma等于300的时候，滤波器的大小大概在1801左右，直接对图像滤波，速度会很慢，最重要的是，需要补大量的padding，使图像失真 356 | 357 | 方法：给出任意大小的sigma值，都可以通过使用**图像金字塔**与**可分离滤波器**计算高斯卷积； 358 | 359 | （1）使用图像金字塔，使用5*5的高斯滤波器，逐层对高斯金字塔滤波，并下采样，下采样后，sigma减半，用于下一层滤波 360 | 361 | （2）不断递归，直到由sigma算出来的滤波器的size小于10 362 | 363 | （3）当递归到，滤波器的大小小于10后，分x轴y轴对对应的下采样图进行高斯运算，这个图是经过一次次高斯模糊降采样得到的。 364 | 365 | （4）反向逐层上采样，得到与输入原图同样分辨率，滤波结束 366 | 367 | 需要用多尺度的MSR，因为当图很小的时候，大的sigma产生的滤波器无法对图像滤波 -------------------------------------------------------------------------------- /logs/AHF_EXPERIMENT.md: -------------------------------------------------------------------------------- 1 | ## 实验记录 2 | #### 艾宏峰 3 | 4 | 当前baseline模型最优配置如下： 5 | 6 | |配置|设置| 7 | |:---:|:---:| 8 | |模型|Faster R-CNN + r50 + FPN + DCN| 9 | |anchor_ratio|[0.2, 0.5, 1.0, 2.0, 5.0]| 10 | |训练多尺度|[(4096, 800), (4096, 1200)]| 11 | |测试尺度|(4096, 1000)| 12 | |NMS|soft_nms (min_score=0.001, max_per_img = 100)| 13 | |epoch|1x schedule (12 epochs)| 14 | |steps|[8, 11]| 15 | |fp16|未开启| 16 | |score|0.46353357| 17 | |MAP/MAP50/MAP75|0.500/0.851/0.536| 18 | **** 19 | ### 2020年3月11日 20 | 1. 实验1主要是研究**libra**： 21 | 22 | |实验|libra|MAP|MAP50|MAP75|score|loss| 23 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:| 24 | |0|无|0.500|0.851|0.536|**0.4635**|0.24| 25 | |1|有|0.509|0.855|0.558|0.4616|0.46| 26 | 27 | **总结**：Libra的初衷是解决训练中样本层，特征层和目标层的不平衡问题。分数略低一点的原因可能是当前数据量和数据的现有特征决定了模型的上限，虽然Libra在验证集上表现好，但可能存在验证集过拟合嫌疑。建议后续先在数据上下工夫，再考虑提高模型的复杂度。 28 | ### 2020年4月3日 29 | 30 | 1.实验27基于实验19训练尺度修改为（4096，800），（4096，1400）。 31 | 32 | | 实验 | 测试尺度 | 线上分数 | 33 | | :--: | --------------------------------------- | :------------: | 34 | | 19 | (4096, 600), (4096, 800), (4096, 1000) | **0.49229116** | 35 | | 27 | (4096, 600), (4096, 1000), (4096, 1400) | **0.49281524** | 36 | 37 | 总结：增加短边尺寸能提高分数，猜测为小目标结果提升，后续将对测试尺度进行筛选。 38 | 39 | ### 2020年3月12日 40 | 1. 实验2主要是研究**allowed_border和neg_pos_ub**： 41 | 42 | |实验|allowed_border|neg_pos_ub|MAP|MAP50|MAP75|score|loss| 43 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 44 | |0|0|-1|0.500|0.851|0.536|**0.4635**|0.24| 45 | |2|-1|3|0.503|0.854|0.537|0.4629|0.26| 46 | 47 | 注：neg_pos_ub=-1是指RPN是不对‘负样本/正样本’比率作限制，而neg_pos_ub=3是指负样本数最多是正样本数3倍。allowed_border=0是指超过图片边界的anchor将会被忽略，allowed_border=-1是不忽略。 48 | 49 | **总结**：按mmdetection论文说法实验2会提高了AR，而且论文提到在靠近边界的GT目标能够在训练中有更多匹配正样本，此外，数据集上有些海产就在图片边缘附近，但最后这种配置并没有带来提升，猜想原因是之前加入了更小的anchor ratio，这样边缘目标也能被更好的召回，所以这么没有提升。 50 | 51 | ### 2020年3月13日 52 | 1. 实验3主要是研究**群内选手提供的方案**: 53 | 54 | |配置|设置| 55 | |:---:|:---:| 56 | |模型|Cascade R-CNN + ResNeXt101 + FPN | 57 | |anchor_ratio|[0.5, 1.0, 2.0]| 58 | |训练多尺度|[(4096, 600), (4096, 1000)]| 59 | |测试多尺度|[(4096, 600), (4096, 800), (4096, 1000)]| 60 | |NMS|soft_nms (min_score=0.0001, max_per_img = 200)| 61 | |epoch|1x schedule (12 epochs)| 62 | |steps|[8, 11]| 63 | |fp16|开启| 64 | |预训练模型|HTC| 65 | |score|0.4841776| 66 | |MAP50|0.867| 67 | 68 | |实验|baseline|MAP50|score|loss| 69 | |:---:|:---:|:---:|:---:|:---:| 70 | |0|我们|0.851|0.4635|0.24| 71 | |3|郑烨|0.867|**0.4836**|0.54| 72 | 73 | **总结**：郑烨大大提高了模型复杂度。相比我们，他提供的方案中模型更复杂，图像尺度更大更多变，NMS更宽松，也因为如此运行时间更长，为此他开启了混合精度训练的方法（通过16位浮点数（FP16）进行深度学习模型训练，从而减少了训练深度学习模型所需的内存，同时由于FP16的运算比FP32运算更快，从而也进一步提高了硬件效率），后续实验暂时在该基础上扩展实验内容。 74 | 75 | ### 2020年3月15日 76 | 1. 实验4主要是研究**训练开启的翻转增强**: 77 | 78 | |实验|水平翻转|垂直翻转|MAP|MAP50|MAP75|score|loss| 79 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 80 | |3|开启|未开启|0.526|0.863|0.585|**0.4836**|0.54| 81 | |4|开启|开启|0.514|0.855|0.565|0.4780|0.46| 82 | 83 | 注：翻转概率均为0.5。 84 | 85 | **总结**：追加垂直翻转不能带来表现提升，可能是因为加入垂直翻转的训练集中场景与测试集正常采集场景有些不一致引起的。暂时后续**不考虑垂直翻转**。 86 | 87 | ### 2020年3月16日 88 | 1. 实验5主要是研究**实例平衡增强 (Instance-Balanced Augmentation)**: 89 | 90 | |实验|实例平衡增强|MAP|MAP50|MAP75|score|loss| 91 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:| 92 | |3|未开启|0.526|0.863|0.585|**0.4836**|0.54| 93 | |5|开启|0.516|0.856|0.570|0.4562|0.2070| 94 | 95 | **总结**：由于原数据集存在类别不平衡问题（实例数量：海参4574，海胆18676，扇贝5554和海星5704），所以打算使用阿里之前提出的一个实例平衡增强方法去增强数据解决不平衡问题，首先先把原图放大1.5倍，然后用原图原始尺寸大小作为滑窗大小，以滑窗形式水平平均地移动三次，垂直平均地移动三次，最后1张图会得到9张相当于shift和scale后的增强图片。在滑动中发现如果不对滑动窗口做限制，会加重类别不平衡，因为海胆数量太多，且滑动结果会有很多单张只有一个扇贝的情况从而导致扇贝很多。因此对滑动窗口进行限制：滑动窗口内含海胆就不要该窗口，滑动窗口内仅有一个扇贝的不要要。最后增强数据和原数据合并后的实例数量是：海参13016，海胆18676，扇贝13287和海星12322。**虽然类别平衡许多了且该增强模仿了水下手持拍摄设备从远到近的拍摄过程，但结果反而下降，原因是单纯的实例平衡增强其实有点像重复数据集操作**，这一点在‘第一个epoch下验证集比之前实验都高，在第9个epoch验证集表现最好’能反映出来，但是图片并没有很大的变化，最多是解决了模型平移不变性的问题。阿里其实后面还对这些增强图片还加入了一种自动并行增强（Auto Affine Augmentation）方法，即旋转边界框，白平衡，按照x轴或y轴截断等。这块后续有时间可以尝试下。 96 | 97 | 2. 今天对数据再次深入研究了下，有以下发现： 98 | - 宽高比1.22的2种采集图片像素低，放大易失真，且部分图片中目标及其密集，海产重叠严重。 99 | - 宽高比1.77的3中采集图片中，部分图片含有大量密集的扇贝目标或海胆目标（猜测是不同海产放养区所致）。 100 | - 图片有很多漏标，少扇贝区域的地方即使出现扇贝，也不太会被认真标注，而且因为被沙掩盖部分外壳，标注存在不确定性。而海参由于肉眼本身因海水可见度，辨别难度导致存在一定的漏标。一般从上拍下的海星不会被漏标，但是海底平行拍摄时容易被漏标。海胆漏标不严重，一般都是远处有小黑团的这种情况，可能不会被标注。 101 | - 数据集中无标注的图片是拍摄结束回到水面过程中记录的视频切片图。 102 | - 当前最优模型下，AP表现从大到小是：**海胆0.92 > 海星0.89 > 扇贝0.85 > 海参 0.80**。海胆检测好是因为标注数量多，特征（带刺）明显。海星第二好是因为海星大部分就是一个样，蓝绿角中间橘红色的特征，且一般属于大目标好检测。扇贝检测难度是沙土遮掩，标注不确定性大，小目标。海参难检测原因在于由于海水可见度低导致海参身上的触点较难观察到，而且颜色深沉，在海底环境中不突出，而且容易与条状石头，海洋生物排泄物或断落珊瑚，海草等混淆。 103 | 104 | ### 2020年3月17日 105 | 1. 实验6主要是研究**训练开启的模糊处理（MedianBlur或者Blur）**: 106 | 107 | |实验|水平翻转|垂直翻转|MAP|MAP50|MAP75|score|loss| 108 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 109 | |3|开启|未开启|0.526|0.863|0.585|**0.4836**|0.54| 110 | |6|开启|未开启|0.524|0.862|0.584|**0.4837**|0.6386| 111 | 112 | 注：模糊概率均为0.1。 113 | 114 | **总结**：模糊处理提升效果不大，可能是因为引入噪声引起的。后续**考虑增加模糊概率**。 115 | 116 | ### 2020年3月18日 117 | 1. 实验7主要是研究**训练使用retinex数据增强**: 118 | 119 | |实验|retinex|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 120 | |:---:|:---:|:---:|:---:|:---:|:---:|-----|-----|-----|-----|-----| 121 | |3|未开启|0.526|0.863|0.585|46.842|**52.548**|**55.064**|**55.929**|**0.4836**|0.54| 122 | |7|开启|0.522|0.859|0.581|**47.105**|52.547|53.914|55.389|0.4830|0.5483| 123 | 124 | 注：dict(type='Retinex', model='MSR', sigma=[30, 150, 300], restore_factor=2.0, color_gain=6.0, gain=128.0, offset=128.0) 125 | **总结**：增强算法也有参数要调，对海参有所加强 126 | 127 | ### 2020年3月18日 128 | 1. 实验8主要是研究**训练开启one of(CLAHE+IAASharpen+IAAEmboss+RandomBrightnessContrast)**: 129 | 130 | |实验|C|IAAS|IAAE|RBC|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 131 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 132 | |3|没|没|没|没|0.526|0.863|0.585|46.842|**52.548**|**55.064**|**55.929**|0.4836|0.54| 133 | |8|有|有|有|有|0.526|0.866|0.586|**47.193**|52.331|54.785|55.887|**0.4853**|0.5617| 134 | 135 | 注：'CLAHE'对比度受限的自适应直方图均衡，clip_limit=2（对比度限制的上阈值）其他均为默认参数.。'IAAEmboss'，压印输入图像，并将结果与原始图像重叠。'IAASharpen'锐化输入图像，并将结果与原始图像重叠。其他均为默认参数.one of概率为0.3，内部概率方法均为0.5。 136 | 137 | **总结**：将图像直方均匀和重叠处理对暗部目标有一定效果。后续**分别对四个方法进行深入分析**，经过各类的对比，可以发现通过处理的验证集只在海参的表现上高于baseline，故猜测海参这类对最终结果的影响较大，后续将针对海参这类的增强进行改进。 138 | 139 | 140 | ### 2020年3月19日 141 | 1. 实验9主要研究**泊松融合Poisson Blending数据增强**： 142 | 143 | |实验|泊松融合|MAP|MAP50|MAP75|score|loss| 144 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:| 145 | |3|未开启|0.526|0.863|0.585|**0.4836**|0.54| 146 | |9|开启|0.508|0.851|0.558|未提交|0.52| 147 | 148 | 注：该对比实验组没加其他Albu手段。 149 | **总结**：在之前实验中发现海参检测效果差，受论文[UDD: An Underwater Open-sea Farm Object Detection Dataset for Underwater Robot Picking](https://arxiv.org/abs/2003.01446)启发：“海参和扇贝因为训练样本不足和数据类别不平衡问题导致很多模型表现不好”。因此实验7针对宽高为720和405的图片进行海参目标的数据增强工作（采用该尺寸的图片有两点原因：一是图片第二大，处理速度快不像第一大尺寸的图片，处理慢且耗内存。二是单独对这类图片融合后背景差异不会太大，避免融合突兀）。由于海参问题严重，所以只针对抠取了900多个海参，然后选取单图标注数量不超过4的图片作为融合海参的背景图片（单图过多标注可能会影响放置粘贴目标和带来冗余的目标加重类别不平衡问题）。之后根据融合后可视化结果将2616张增强图人工剔除剩1441张融合较好的图加入到原始数据集中进行训练。最后增强后各类别数据是：海参（4574变到6991），海胆（18676到20818），扇贝（5554到5653）和海星（5704到6326）。模型最后的表现并没有提升，即使是海参的AP也没增加，猜测原因是：融合的还是有些许不自然，容易使模型过拟合，而且这里并没有使用新的背景图片，而是原数据的图片，这也导致一些背景和目标重复出现，影响模型通用性的提升。 150 | 151 | ### 2020年3月20日 152 | 1. 实验10主要研究是**去除长短比为1.22的数据集**: 153 | 154 | |实验|长短边1.22图|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 155 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 156 | |3|有|0.526|0.863|0.585|46.842|**52.548**|**55.064**|55.929|**0.4836**|0.52| 157 | |10|无|0.526|0.863|0.590|**47.379**|52.199|54.587|**56.127**|未提交|0.49| 158 | 159 | 注：去除长短比为1.22的数据集是指不将（704,576）和（586,480）这两种尺寸图片（后续简称122图片）的加入到模型训练中。 160 | 161 | **总结**: 经过进一步EDA发现：122图片与177图片是不同类设备采集得到的，而且训练集中122图片（共有82张）的各类别数量是：海参1个，海胆632个，扇贝和海星都是20个。而测试集图片尺寸情况如下： 162 | 163 | |测试集图片尺寸（h,w）|长短边比|图片数量| 164 | |:---:|:---:|:---:| 165 | |(1080,1920)|1.77|42| 166 | |(1440,2560)|1.77|32| 167 | |**(1536,2048)**|**1.33**|**21**| 168 | |(2160,3840)|1.77|653| 169 | |(405,720)|1.77|52| 170 | 171 | 后面特意抽取看了长短边比为1.33的1536x2048图，发现其跟训练集122图片有些类似，猜测是同一类的设备，但是测试集中133图片（均为2018年拍摄）目测比训练集中122图（均为2017年拍摄）有更多的扇贝，且图片像素更大了，猜测是设备升级了。因为没有提交结果暂不下结论。 172 | 173 | ## 2020年3月20日 174 | 175 | 实验11主要研究是**在3月18日的最佳得分上进行了全数据集训练**： 176 | 177 | |实验|数据集|MAP|MAP50|MAP75|score|loss| 178 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:| 179 | |8|训练集模型|0.526|0.866|0.586|**0.4853**|0.5617| 180 | |11|全数据集模型|0.600|0.922|0.708|**0.4856**|0.54| 181 | 182 | 注：提升不大，说明训练集的分布基本囊括了验证集的分布 183 | 184 | ### 2020年3月21日 185 | 1. 实验12主要研究是**anchor_ratio增加0.2和allowed_border=-1**： 186 | 187 | |实验|achor_ratio|allowed_border|MAP|MAP50|MAP75|score|loss| 188 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 189 | |3|无0.2|0|0.526|0.863|0.585|**0.4836**|0.54| 190 | |12|加0.2|-1|0.527|0.866|0.593|0.4819|0.62| 191 | 192 | 注：allowed_border=-1允许模型RPN在图片边缘提出超出边缘的框。 193 | 194 | **总结**：虽然增强模型在边缘目标的表现能力和小目标检测能力在验证集上有提升，但提交结果反而下降了，猜测原因是由于测试集存在部分数据不属于训练集分布下，所以有可能实验10过拟合了验证集，后续实验应该注意提高模型通用性。 195 | 196 | ### 2020年3月22日 197 | 1. 实验13主要研究是**Motion Blurring**: 198 | 199 | |MB|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 200 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 201 | |0|0.526|0.863|0.585|46.842|**52.548**|**55.064**|55.929|**0.4836**|0.54| 202 | |0.3|0.522|0.863|0.575|**46.988**|52.177|53.780|**55.945**|未提交|0.56| 203 | 204 | 注：‘MB=0.3’是指训练时开启动态模糊Motion Blurring的概率为0.3。 205 | 206 | **总结**：由于数据是视频拍摄某些连续帧下取得，因此拍摄设备的水下移动带来了动态模糊，为了模型在这种数据下获得更好的鲁棒性，在``mmdet/pipelines``下的``transforms.py``和``__init__.py``下添加了MotionBlur的数据增强手段。在验证集上，实验11在海参和海星上有所提高，但是海胆和扇贝反而下降，这个可能涉及到不同海产在这种画面模糊的在分布情况不一致，原因也暂不明晰。参考：2020年3月17日的MedianBlur（p=0.1）虽然验证集上表现不好，但最终结果有提升，需要注意的是实验11的动态模糊发生概率越大，验证集表现越差，所以实验11只是设置为0.3而不是常见的0.5。另外，实验11动态模糊充分考虑到不同尺寸的数据集原图而单独赋予不同的模糊核大小（(1) w<=750, 核大小为10-20; (2) 750=2000, 核大小为100-140）。 207 | 208 | ### 2020年3月22日 209 | 210 | 211 | 212 | 2. 实验14主要研究的是**将虚警概率大的水草作为正样本，但不输出** 213 | 214 | |实验|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 215 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 216 | |8| 0.526 | 0.866 | 0.586 | **47.193** | 52.331 | 54.785 | 55.887 | 0.4853 | 0.5617 | 217 | |14| 0.570 | 0.879 | 0.673 | | | | | **0.4922** | 0.5397 | 218 | 219 | 注：该实验使用的是**全数据集** 220 | 221 | 222 | 223 | ### 2020年3月23日 224 | 225 | 1. 实验15主要研究是**IAAsharpen**: 226 | 227 | |实验|S|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 228 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 229 | |3|0|0.526|0.863|0.585|46.842|**52.548**|**55.064**|55.929|**0.4836**|0.54| 230 | |15|0.5|0.525|0.865|0.583|**47.182**|52.343|54.517|**55.959**|未提交|0.5551| 231 | 232 | 注：‘S=0.5’是指训练时开启IAAsharpen的概率为0.5。 233 | **总结**：锐化重叠对海参和海星类有提升，但对海胆和扇贝却降低，具体原因还得深入分析。 234 | 235 | 236 | 237 | ### 2020年3月24日 238 | 239 | 实验16主要在实验14的基础上，**将验证集和训练集分开了** 240 | 241 | |实验|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 242 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 243 | |8| **0.526** | **0.866** | **0.586** | 47.193 | 52.331 | **54.785** | **55.887** | 0.4853 | 0.5617 | 244 | |11| 0.600 | 0.922 | 0.708 | | | | | **0.4856** | 0.54 | 245 | |16| 0.460 | 0.783 | 0.491 | **47.557** | **52.404** | 54.412 | 55.844 | 0.4857 |0.5483 | 246 | |14| 0.570 | 0.879 | 0.673 | | | | | **0.4922** | 0.5397 | 247 | 248 | 实验16仅仅拆开了验证和训练，但是降低了很多点，主要怀疑是因为实验16每张图的测试max_per_img=200，实验14每张图的max=300，实验14提高了测试时的查全率，其实模型的性能本质没有改变，map表现提高。 249 | 250 | | 类别 | 验证集ann数量 | 251 | | ---- | ------------- | 252 | | 海参 | 963 | 253 | | 海胆 | 3667 | 254 | | 扇贝 | 1166 | 255 | | 海星 | 1137 | 256 | | 水草 | 12 | 257 | 258 | | 类别 | 训练集ann数量 | 259 | | ---- | ------------- | 260 | | 海参 | 4574 | 261 | | 海胆 | 3667 | 262 | | 扇贝 | 5554 | 263 | | 海星 | 5704 | 264 | | 水草 | 70 | 265 | 266 | - 实验8、11的未加水草的数据集与实验16、14加水草的数据集，除水草外，其余数据完全一致。 267 | 268 | - 一些黑乎乎的阴影，实验11检测为海胆，实验14正确检测。 269 | 270 | - 实验11检测不到的扇贝，实验14正确检测。 271 | 272 | - 实验14将水草认成海胆海参机率提高。 273 | 274 | - 实验14对扇贝提升效果很高。 275 | 276 | ### 2020年3月25日 277 | 1. 实验17主要研究是**dcn**的表现 278 | 实验18主要研究是**seresnext101**的表现: 279 | 280 | | 实验 | MAP | MAP50 | MAP75 | 海参 | 海胆 | 扇贝 | 海星 | score | loss | 281 | | :--: | :---: | :---: | :---: | :--------: | :--------: | :----: | :--------: | :----: | :----: | 282 | | 16 | 0.460 | 0.783 | 0.491 | **47.557** | 52.404 | **54.412** | 55.844 | 0.4857 | 0.5397 | 283 | | 17 | 0.500 | 0.807 | 0.536 | 47.190 | **53.773** | 53.696 | 56.017 | 0.4776 | 0.3818 | 284 | | 18 | 0.483 | 0.816 | 0.504 | 46.967 | 53.537 | 53.716 | **56.085** | 未提交 | 0.4690 | 285 | 286 | 287 | **总结**：猜测dcn使模型过拟合，seresnext101的速度比baseline快2-3倍，最终的模型选取，可以在速度上权衡使用seresnext101。 288 | 289 | ### 2020年3月25日 290 | 1. 实验18主要研究的是在实验14的基础上添加mixup 291 | 292 | |实验|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 293 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 294 | |18| 0.507 | 0.796 | 0.580| | | | | 0.4819 | 0.7826 | 295 | |14| 0.570 | 0.879 | 0.673 | | | | | **0.4922** | 0.5397 | 296 | 297 | ### 2020年3月26日 298 | 1.实验19，主要研究在实验14的基础上放开max_per_num = 1000 299 | |实验|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 300 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 301 | |19| 0.570 | 0.879 | 0.673 | | | | | **0.49229116** | 0.5397 | 302 | |14| 0.570 | 0.879 | 0.673 | | | | | 0.49222164 | 0.5397 | 303 | 304 | - 实验19max_per_img=5000，有26w个标注结果，去掉水草2万个标注结果，还剩24万个标注结果，'240720/800=301框/图， 301/4=75.25框/类。 305 | - 实验14max_per_img=300，有20W个标注结果。 306 | - 实验16max_per_img=200，有14W个标注结果，单图每类平均有50个框。 307 | 308 | ### 2020年3月27日 309 | 1. 实验20主要研究的是**追加海草难样本标注**下模型的表现： 310 | 311 | |实验|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 312 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 313 | |19| 0.570 | 0.879 | 0.673 | | | | | **0.49229116** | 0.5397 | 314 | |20| | | | | | | | 0.48268375 | 0.5097 | 315 | 316 | 注：实验19和20均为带海草的全数据集训练得到的结果，且max_per_img=5000。 317 | 318 | **总结**：在实验14的结果可视化发现2020年3月24日几个问题，由于海草在全数据集上只有82个标注且很多海草并没有被认真标注，且存在海草被误认做海胆和海参，对此的猜测是海草标注不清楚带来了模型辨认上的困难。为了清除这个遗漏，我们先将当前最优模型预测训练集并可视化框在图上，之后将训练集中被误检成其他类且无标注的海草目标进行框的绘制，之后补充到原数据集中，5千多张图最后补充了额外800多个海草标注，提交结果比实验19少了5w的标注。但最后模型提交结果并不是很好，可能是因为在减少误检和提高查全率（即检测出更多框）之间，mAP在此数据上更加偏向于后者。 319 | 320 | 2. @郑烨选手更新了Github，有几点内容需要留意下： 321 | - 最近很多选手分突然上了0.49，应该是得益于选手分享的：**res50和se50均可以达到线上testA 46-47 mAP, 经过spytensor试验进行模型集成可以达到49+。** 在论文《Weighted Boxes Fusion: ensembling boxes for object detection models》里提到两种模型融合思路，一种是**同一模型不同backbone的融合（也是其他选手@spytensor的推荐，且建议使用WBF（IOU=0.7））**， 第二种是**不同结构的模型进行融合，根据论文所述这种方法比前者提升更大**。WBF的代码开源地址：[https://github.com/ZFTurbo/Weighted-Boxes-Fusion](https://github.com/ZFTurbo/Weighted-Boxes-Fusion)。建议后面实验使用看看。 322 | - @郑烨有一些不work的内容，大部分我们都试过，但有几点需要后面留意下：**引入往年数据**(2020年3月20日的实验10我们就有做过，但是没提交所以也不确定结果会怎样)，**se154与x101接近**(那能不能考虑进模型融合当中？)，**高分的fp： 相当多一部分是标注漏标错标造成，例如训练集存在相邻帧图片同一目标前一帧标注，后一帧不标情况，例如对于较为模糊的目标是否标注也很不统一**(之前试过通过人工标注解决这个问题，但标注量是在太大了，且画面移动后模糊情况不一样，很难采用统一的标注标准)，**低分的tp： 这一类主要集中在模糊目标上，模型整体对模糊目标预测的score较低，但是数据的标注对于模糊对象标准并不一致**(与前面一点也有谈及这个问题，之前有做过MedianBlur和MotionBlur的实验，前者提升很小很小，后者因验证集无明显提升所以没提交，怎么解决模糊对象标注标准不一致的问题也是该留意的问题)。 323 | 324 | ### 2020年3月29日 325 | 1. 实验21和22主要研究的是**Weighted Boxes Fusion (WBF)**: 326 | 327 | |实验|模型|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 328 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 329 | |19|x101| 0.570 | 0.879 | 0.673 | | | | | **0.49229116** | 0.5397 | 330 | |21|x101+s50|| | | | | | | 0.47551257|| 331 | |22|x101+r50|| | | | | | | 0.46915054|| 332 | 333 | 注：这里三个实验均使用baseline模型，只是backbone不同，实验21融合的是senet50，实验22是融合resnet50。在模型结果融合中：当前最优下x101模型给予权重2，而s50和r50的权重为1，框匹配判定的IOU是0.7（作者是0.55）。 334 | 335 | **总结**：结果都不好，猜测有以下几个变量需要在验证集上进行测试后再提交比较稳当： 336 | 337 | |测试编号|模型(w,albu) |次模型(w,albu) |主模型带海草|次模型带海草|IOU | nms |验证集mAP|海参 |海胆 |扇贝 |海星 | 338 | |:-----:|:-------------:|:---------------:|:--------:|:--------:|:--------:|:------:|:------:|:----:|:----:|:----:|:----:| 339 | | 1 | x101(1,✓) | s50(1,✗) | ✓ | ✗ | 0.7 | nms+avg| 0.517 |46.855|51.130|53.772|54.859| 340 | | 2 | x101(1,✓) | s50(1,✗) | ✓ | ✗ | 0.5 | nms+avg| 0.513 |46.294|50.821|53.477|54.686| 341 | | 3 | x101(1,✓) | s50(1,✗) | ✓ | ✗ | 0.7 |soft+max| 0.518 |46.925|51.622|53.857|54.937| 342 | | 4 | x101(1,✓) | s50(1,✗) | ✓ | ✗ | 0.5 |soft+max| 0.513 |46.327|50.686|53.549|54.669| 343 | 344 | 345 | 对于WBF的一些发现和思考： 346 | - WBF融合分为两块：框和置信度。各自聚类匹配框和其置信度的融合权重是score × weight。 347 | - 如果WBF是在softNMS的结果上做融合，会因为某目标上**存在冗余低分框拉低融合后的框分数，而框的位置也是平均了冗余框得到**。解决方法有两个： 348 | （1）使用论文的方法，**使用NMS输出的结果上实施WBF,@spytensor推荐NMS的score_thr=0.001**，这样NMS只是保留当前模型下唯一的高分框，这样最后平均融合后不会受冗余低分框影响。 349 | （2）**在softNMS下，改置信度融合方法conf_type为max**，这样在两个模型预测结果中选择其中最大的置信度作为平均融合框的置信度。 350 | - 以上两种方法中，框坐标的融合都是采用score x weight然后平均的方式，只是置信度不一样，前者使用avg(NMS)，后者max(SoftNMS)。需要注意的是以上两种方法不建议使用weights=[2,1]，因为如果一个框0.9,一个框1.0，最后融合conf=(0.9x2+1.0)=1.4>1.0，分数会不正常。因此后续**推荐使用weights=[1,1]**。 351 | 352 | 2. 实验23主要研究的是**softNMS + WBF**: 353 | 354 | |实验|模型|conf_type|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 355 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 356 | |19|x101|-| 0.570 | 0.879 | 0.673 | | | | | **0.49229116** | 0.5397 | 357 | |21|x101+s50|avg|| | | | | | | 0.47551257|| 358 | |23|x101+s50|max|| | | | | | | 0.49188273|| 359 | 360 | 注：实验23是采用基于SoftNMS结果的基础上，conf_type='max', weights=[1,1]和iou_thr = 0.7的配置。而实验21是SoftNMS+conf_type='avg', weights=[2,1]和iou_thr = 0.7。 361 | 362 | **总结**：在‘对于WBF的一些发现和思考’中的发现是正确的，后期建议可以采用解决方法（1）试试。 363 | 364 | ### 2020年3月31日 365 | 1. 实验24主要研究的是**建海参专家模型并WBF结果融合** 366 | 367 | |实验|模型|conf_type|MAP|MAP50|MAP75|海参|海胆|扇贝|海星|score|loss| 368 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| 369 | |19|x101|-| 0.570 | 0.879 | 0.673 | | | | | **0.49229116** | 0.5397 | 370 | |24|x101+r101|max| - | - | - | | | | | 0.49051643 |- | 371 | 372 | 注：实验24最后的结果是‘x101其他类别结果保留 + x101模型下海参结果和r101海参专家模型下海参结果进行Softnms max的WBF’。 373 | 374 | **总结**:x101模型下海参框的数量是120596个，而x101模型下海参数量是55041，融合后海参数量为120847。最后结果并不好。 375 | ### 2020年4月1日 376 | 1.实验25，26均使用了albu增强。 377 | |实验|模型|conf_type|MAP|MAP50|MAP75| 378 | |:---:|:---:|:---:|:---:|:---:|:---:| 379 | |25|x101|-| 0.4560 | 0.7800 | 0.481 | 380 | |26|fasterrcnn|-| 0.461 | 0.785 | 0.498 | 381 | |27|r101|-| 0.467 | 0.791 | 0.509 | 382 | 383 | ### 2020年4月3日 384 | 385 | 1.实验27基于实验19训练尺度修改为（4096，800），（4096，1400）。 386 | 387 | | 实验 | 测试尺度 | 线上分数 | 388 | | :--: | --------------------------------------- | :------------: | 389 | | 19 | (4096, 600), (4096, 800), (4096, 1000) | 0.49229116| 390 | | 27 | (4096, 600), (4096, 1000), (4096, 1400) | **0.49281524** | 391 | 392 | **总结**：猜测增加训练尺度能提高小目标的检测精度，后期将对测试尺度再深入的分析实验一下。 393 | 394 | ### 2020年4月4日 395 | 396 | 1.实验28基于实验27添加了guided-anchoring模块。 397 | 在实验27的第12个epoch上，使用学习率0.002,0.0002，0.00002进行finetune了3个epoch 398 | 399 | |实验|MAP|MAP50|MAP75| 测试尺度 |score|loss| 400 | |:---:|:---:|:---:|:---:|:---:|:---:|:---:| 401 | |27| 0.544 | 0.847 | 0.647| (4096, 600),(4096, 1400) | 0.49281524 | 0.5147 | 402 | |28| 0.551 | 0.859 | 0.632 | (4096, 600),(4096, 1200) | 0.48914663 | 0.8240 | 403 | 404 | ga模块需要占用现存，训练缩放到1400，GPU显存会炸，只能缩放到1200 405 | 406 | **总结**：损失一直没有降下来，训练尺度小，可能ga的损失本来就略高，finetune了3个epoch并没有效果 407 | 408 | ### 2020年4月8日 409 | 1. 实验29主要研究的是**cosine lr decay**： 410 | 411 | |实验|lr_decay_type|score| 412 | |:---:|:---:|:---:| 413 | |19|step|**0.49229116**| 414 | |29|cosine|0.48785642| 415 | 416 | **总结**：从训练日志的损失图来看，收敛地更好，但也可能因此导致了过拟合使得模型在测试集上表现下降。 417 | 418 | ### 2020年4月9-10日 419 | 1. 实验30和31主要研究的是**label smoothing标签平滑**： 420 | 421 | |实验|lr_decay|label_smoothing|score| 422 | |:---:|:---:|:---:|:---:| 423 | |27|step|无|0.49281524| 424 | |30|cosine|0.005|0.49232983| 425 | |31|step|0.001|0.49232983| 426 | 427 | 注：label_smoothing下的数是指平滑指数，它越高，平滑效果越大。 428 | 429 | ### 2020年4月11日 430 | 1. 实验32主要研究的是**albu**： 431 | 432 | |实验|albu概率|score| 433 | |:---:|:---:|:---:| 434 | |27|0.3|0.4700782| 435 | |30|0.5|0.47165024| 436 | 437 | 注：这里的ablu是使用的one of(CLAHE+IAASharpen+IAAEmboss+RandomBrightnessContrast)，而score为B榜测试集数据上的表现。 438 | -------------------------------------------------------------------------------- /code/draw_bbox.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Draw BBox\n", 8 | "According to the given dataset, draw bboxes on the images and save them. Also, at the end, the class comparision is available" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import json\n", 18 | "import cv2\n", 19 | "from tqdm import tqdm" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "ann_path = 'seacoco/all_waterweeds.json' # annotation json\n", 29 | "img_path = 'seacoco/trainval/'\n", 30 | "save_path = 'seacoco/trainval_with_bbox/' # the path of saveing image with annotated bboxes" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 3, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "with open(ann_path,'r') as f:\n", 40 | " ann = json.load(f)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 4, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stderr", 50 | "output_type": "stream", 51 | "text": [ 52 | "100%|██████████| 5543/5543 [08:55<00:00, 10.36it/s]\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "for ann_img in tqdm(ann['images']):\n", 58 | " img = cv2.imread(img_path + ann_img['file_name'])\n", 59 | " img_id = ann_img['id']\n", 60 | " for ann_ann in ann['annotations']:\n", 61 | " if ann_ann['image_id'] == img_id:\n", 62 | " x1 = ann_ann['bbox'][0]\n", 63 | " y1 = ann_ann['bbox'][1]\n", 64 | " x2 = ann_ann['bbox'][0] + ann_ann['bbox'][2]\n", 65 | " y2 = ann_ann['bbox'][1] + ann_ann['bbox'][3]\n", 66 | " img = cv2.rectangle(img, (x1,y1), (x2,y2), (255,0,0), 8)\n", 67 | " cv2.imwrite(save_path + ann_img['file_name'], img)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 19, 73 | "metadata": {}, 74 | "outputs": [ 75 | { 76 | "name": "stdout", 77 | "output_type": "stream", 78 | "text": [ 79 | "The augmentation image number: 6060\n", 80 | "The augmentation annotation number: 39788\n", 81 | "\n", 82 | "The instance number of each class:\n", 83 | "holothurian : 6991\n", 84 | "echinus : 20818\n", 85 | "scallop : 5653\n", 86 | "starfish : 6326\n" 87 | ] 88 | } 89 | ], 90 | "source": [ 91 | "aug_anns = ann\n", 92 | "print(\"The augmentation image number: %d\" % len(aug_anns['images']))\n", 93 | "print(\"The augmentation annotation number: %d\" % len(aug_anns['annotations']))\n", 94 | "print(\"\")\n", 95 | "class_freq_dict = {}\n", 96 | "\n", 97 | "# init class_fre_dict\n", 98 | "for cls in aug_anns['categories']:\n", 99 | " class_freq_dict[cls['id']] = 0\n", 100 | "\n", 101 | "# count the instance number of each class\n", 102 | "for ann in aug_anns['annotations']:\n", 103 | " class_freq_dict[ann['category_id']] += 1\n", 104 | "\n", 105 | "# print out class frequency\n", 106 | "print(\"The instance number of each class:\")\n", 107 | "for cls_id in list(class_freq_dict.keys()):\n", 108 | " for cat in aug_anns['categories']:\n", 109 | " if cat['id'] == cls_id:\n", 110 | " print(cat['name'], ': ', class_freq_dict[cls_id])" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 21, 116 | "metadata": {}, 117 | "outputs": [ 118 | { 119 | "data": { 120 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEYCAYAAAATRII7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXwV1f3/8debfRFkERCMCCKbIkZBoCpKtSpqBfd9A6m/am2xdfliqYpaLa1WXOsKBTdQqwgqVlFBrIIIGkR2BCqRlE1kEQUCn98fcxIm4Sa5hCw35PN8PO7j3nvmzJkz2/3cOXNmRmaGc845V96qlHcFnHPOOfCA5JxzLkV4QHLOOZcSPCA555xLCR6QnHPOpQQPSM4551JCpQhIkoZIer6861HeJD0h6bbyrseeKO11KWmkpD8Xc9zakt6QtF7SKyVdt9K0J8tVUjNJUyRtlPT3kq5bAdPsJSmzLKbldpL0tqQrS6v8aqVVcFmStCn2tQ6wBdgevv+/sq9RyZM0Esg0sz8lmf8qYICZHZeTZma/Lp3aueA8oBnQ2Myyy7syZegaYA1Q38xM0hDgEDO7rHyrFUm0L1RGkiYDz5vZM0nmH0K+9Whmp5VO7SJ7xRGSme2T8wK+Ac6Mpb1Q3vVzqUlSSf8hOwhYWJxgVAp1KUsHAXOthK6yr+DLwu0JM9urXsAy4Bf50oYALwPPAhuBOUDX2PAWwKvAamAp8LtCyj8D+ALYACwHhsSGtQIMuJIoMK4BBu9GPToCk4Hvw7A+If0aYBuwFdgEvBHSBwFfh7LmAmfHyvmJ6ChxE/B9SB8J/Dk2vV8Bi4HvgPFAi9gwA34NLALWAY8BKmCZFDVfRvRPi/z1AHoBmcAtwCogCzgLOB1YGOr2x3zT+hfwUpjW58ARyazL2LjPh/U3IMG8jASeACaG8j8EDooN7xCGfQcsAC4I6XeG9bMtLPOrif7w/Qn4b5i3Z4F9820rVxNtK1NCeg/gk7ANzAJ6FbItJlz/YdhVwH+A+8P6WwqcFhveOszbxjA/jxL9e040nYbAm2GZrguf02LLK75t/jLfcpgV8u0LDA/r91vgz0DVWF0/BoaF5frnBHWoHaa1LszrzUQtBoUuCwreFwrcj3dn/hP95hBtZ8/Hvl8RtoG1wG3x/CHvK0Tb5EZgNtAOuDVsM8uBU2JlFbUcE65z4J6wDH4Ky+HRkP5QmMYGYCbQM6T3LmA9TibsNyS3fSf8LSxwWZdGUCjPV/6NI7bSfyL6kasK/AWYFluoM4HbgRrAwcAS4NQCyu8FHB7G6wysBM7KtxKeJtqBjiBqPuyYRD2qEwWHP4Z6nBg20PaxHf/P+epyPtEPcBXgQuAHoHl848yXP7eMUP4a4CigJvAI4UcxDDeiHa8B0JJoZ+xdwDIpcL5iZRUWkLLD8q9OFCRXAy8C9YDDQtkHx6a1jah5rDpwE9GOV72odRkb96yQt3aCeRkZlvvxYbk8lLMcgbpEO28/oubuo8IyPKyAH6L+YZ0eDOwDvAY8l29beTaUWxs4gOhH6/RQv5PD9yYFLPei1v+2sDyrAtcCKwh/KoCpwANhHo8P81xQQGoMnEvUHF6P6Af09UTrM9FyCGmvA0+GeW0KTAf+X6yu2cBvw3JNtF6GAh8BjYADga/IG5B2d1/oRQH7cTHmfxkFBCTgUKIf9OOItsn7w3qJB6SfgFPDvD9LtD0PZuf+sHQ3lmNh63wy+f6EAZeF+asG3Aj8D6hVyHrMLYPktu+Ev4UF/n6XVmAor1f+jSO2YN+LfT8U+DF87g58ky//rcA/k5zeg8CwfCsh/u9pOnBREvXoGTaGKrHhown/3EgQkBLUJQPoW8hOmFsG0b+sv8WG7RM25lbhuwHHxYa/DAwqYLoFzlesrMIC0o/s/JdXL+TvHss/k51Bfwh5g10Von+LPYtal2HcKYnmIV/dxuRbLtuJfgQvBD7Kl/9J4I5Y+fGA9D5wXex7+7CMq8W2lYNjw/+PsEPH0t4BrkxyW8y//hfHhtUJ09uf6A9GNlA3NvxFCghICaaTDqxLtD4LWA7NiH6MasfSLgYmxer6TRHTXELsDxFRq0FmIfkL3RcS5M/dj4sx/8soOCDdDozOtx62kjcgTYwNP5MogOXfHxokuRwTrvPwfTIJWgXyzds6QotD/vWYvwyS274T/hYW9KpMbbX/i33eDNQKbdUHAS0kfR8bXpXo39guJHUn+rfWiegfT02if0yFTWufJOrRAlhuZjtiw/9L9K85IUlXAH8gWvmE6exXUP58WhA1dwFgZpskrQ3TW5bEfOSXcL4sufMpa80spxPKj+F9ZWz4j/mmvTxW7x2ht1ULoh2gqHW5nKLFy98k6btQ/kFA93zlVwOeK6CcFkTrMMd/Q/5mBdTnIOB8SWfG0qoDkxIVnsT6z10nZrZZUjzPOjP7IV/dDixgOnWImtN6EzVfAdSTVDW23gpzUJiPrFAHiP5IxOe9qPXSIl+e+HLd7X0hyf04J++ezH+eeof1sDZfnvzb+poE+8M+oayilmNB6zwhSTcCA9i5/9Rn935Ditq+d+c3pFIFpIIsJzokbptk/heJ2ttPM7OfJD1I8iuwMCuAAyVViQWllkTnUSDaWHJJOojocPgkYKqZbZeUAShR/gKmd1CsvLpEh+7f7tFcJLaZ6N9ajv2JzhsVV+4Pp6QqQBrR/GRT9LosarnkL38fomaiFUTbyodmdnKS9cyzjNl5ZLIy1Dl/fZYTHSH9qqiCk1j/hckCGkqqGwtKLSl42dxI9O+3u5n9T1I60fmXgqaVv5zlRP/s9yvkD0pR6yWLaL3MidUXKPa+sDv7cVHz/wO7bt/xereP1bU20X5WHMksx8Lk/w3pSXRUfhIwJ/y5W0cxf0NIvH3vlr2il90emg5skPR/4TqSqpI6STq6gPz1gO/CRtwNuKSE6vEp0YZ9i6TqknoRHb6PCcNXErXV5qhLtMGsBpDUj+jfHrH8aZJqFDC9F4F+ktIl1QTuBT41s2UlMzt5ZACXhGXbGzhhD8vrIumccGR5A9FOOo3dX5cFOV3ScWHZ3U20XJYTnVNrJ+nysI6qSzpaUscCyhkN/F5S6xDY7gVeKuTH5HngTEmnhrrXCtfbJNq5i1r/BTKz/wIzgDsl1ZB0HNG2VpB6RP/Uv5fUCLijiEmsBFqFPwuYWRbwLvB3SfUlVZHURtLubAcvA7dKahiWx29jw4qzL+zOflzU/GcAF4XtoSvR+c0c/yJap8eE6d9Jcn8adlECyzH/b0g9ogCyGqgm6XaiI6R4/tz1mMDubt9FqvQBKRwan0nULryU6CT1M0S9WRK5DrhL0kai9uGXS6geW4E+wGmhDv8ArjCz+SHLcOBQSd9Let3M5gJ/Jzo5vZLoBO3HsSI/IPo3+T9JaxJM732iHj+vEv2LawNcVBLzksBAomX8PXAp0YnZPTGO6HzOOuBy4Bwz21aMdVmQF4l+dL4DuoQ6Y2YbgVOIltMKouaIvxI19yQygqg5b0qoz0/k/SHNIwS9vkQdW1YT/SO+mQT7aRLrvyiXEJ1z+y7M67OF5H2Q6MT0GqLA/+8iys5p+lorKadZ+AqiprG5ROvtX0Dz3ajvnURNQkuJfpRzm0mLuS/szn5c1PzfRrT/rAv1fDFWtzlE63wM0X62kahH2pbkZnsXe7IcHwLOk7RO0sNE5yffJmqF+S/R9hlv/ku0HuN2a/tORk7vC+ecc6UsHEl8D7Q1s6XlXZ9UU+mPkJxzrjRJOlNSnXCe9n6ia42WlW+tUpMHJOecK119iZp4VwBtibo+e9NUAt5k55xzLiX4EZKrdCQdKGmSpHmS5kgaGNIbSZooaVF4bxjS91V0F+9ZIX+/WFn/Dh1N3sw3jZMkfS4pQ9J/JB1StnPpXMVT6Y6Q9ttvP2vVqlV5V8OVo23btrFt2zbq1KnD9u3bmTdvHm3atGHt2rVUq1aN/fffn//9739kZ2eTlpZGVlYW27dvJy0tjW3btjFnzhw6d+5MlSpV2LBhAzt27GDNmjUccsjOmPPVV1/Rpk0bateuzapVq9i8eTO+3bm92cyZM9eYWZM9KiSZW2XsTa8uXbqYc3F9+vSxd99919q1a2crVqwwM7MVK1ZYu3btzMzs3nvvtWuvvdZ27NhhS5YssTZt2tj27dtzx580aZKdccYZecps166dTZs2LXf8W2+9tYzmxrnyAcywPfx99js1uEpt2bJlfPHFF3Tv3p2VK1fSvHl0SUfz5s1ZtWoVANdffz19+vShRYsWbNy4kZdeeokqVQpv7X7mmWc4/fTTqV27NvXr12fatGmlPi/OVXR+DslVWps2beLcc8/lwQcfpH79+gXme+edd0hPT2fFihVkZGRw/fXXs2HDhkLLHjZsGBMmTCAzM5N+/frxhz/8oaSr79xexwOSq5S2bdvGueeey6WXXso555wDQLNmzcjKygIgKyuLpk2bAvDPf/6Tc845B0kccsghtG7dmvnz5xdY9urVq5k1axbdu3cH4MILL+STTz4p5TlyruLzJjtX6ZgZV199NR07dsxz5NKnTx9GjRrFoEGDGDVqFH379gWgZcuWvP/++/Ts2ZOVK1eyYMECDj744IKKp2HDhqxfv56FCxfSrl07Jk6cSMeOBd3uzpWVbdu2kZmZyU8//VTeVanQatWqRVpaGtWrVy/xsitdL7uuXbvajBkzyrsarhz95z//oWfPnhx++OG554LuvfdeunfvzgUXXMA333xDy5YteeWVV2jUqBErVqzgqquuIisrCzNj0KBBXHbZZQD07NmT+fPns2nTJho3bszw4cM59dRTGTt2LLfffjtVqlShYcOGjBgxotAg5krf0qVLqVevHo0bNyb2+Aa3G8yMtWvXsnHjRlq3bp1nmKSZZtZ1T8r3gOScqxTmzZtHhw4dPBjtITNj/vz5uxz1l0RA8nNIzrlKw4PRnivNZegByTnnXErwTg3OuUqp1aC3SrS8ZUPPKHT4999/z4svvsh11123W+WefvrpvPjiizRo0GBPqlch+BGSc86Vge+//55//OMfu6Rv37690PEmTJhQKYIR+BGSq4yG7O4DZEtquuvLZ7ouJQwaNIivv/6a9PR0qlevzj777EPz5s3JyMhg7ty5nHXWWSxfvpyffvqJgQMHcs011wDQqlUrZsyYwaZNmzjttNM47rjj+OSTTzjggAMYN24ctWvXLuc5Kzl+hOScc2Vg6NChtGnThoyMDO677z6mT5/OPffcw9y5cwEYMWIEM2fOZMaMGTz88MOsXbt2lzIWLVrEb37zG+bMmUODBg149dVXy3o2SpUfITnnXDno1q1bnmt5Hn74YcaOHQvA8uXLWbRoEY0bN84zTuvWrUlPTwegS5cuLFu2rMzqWxY8IDnnXDmoW7du7ufJkyfz3nvvMXXqVOrUqUOvXr0S3lGiZs2auZ+rVq3Kjz/+WCZ1LSveZOecc2WgXr16bNy4MeGw9evX07BhQ+rUqcP8+fMr7d3hS+0ISdKBwLPA/sAO4Ckze0hSI+AloBWwDLjAzNYputrqIeB0YDNwlZl9Hsq6EvhTKPrPZjYqpHcBRgK1gQnAQKtst55wzhVLUd20S1rjxo059thj6dSpE7Vr16ZZs2a5w3r37s0TTzxB586dad++PT169CjTuqWKUrt1kKTmQHMz+1xSPWAmcBZwFfCdmQ2VNAhoaGb/J+l04LdEAak78JCZdQ8BbAbQFbBQTpcQxKYDA4FpRAHpYTN7u7B6+a2DnPeyq5zmzZvnN7ktIYmWZUrfOsjMsnKOcMxsIzAPOADoC4wK2UYRBSlC+rPh4YPTgAYhqJ0KTDSz78xsHTAR6B2G1TezqeGo6NlYWc455yqYMjmHJKkVcCTwKdDMzLIgClpA05DtAGB5bLTMkFZYemaC9ETTv0bSDEkzVq9evaez45xzrhSUekCStA/wKnCDmRX2mM1Ed+yzYqTvmmj2lJl1NbOuTZo0KarKzjnnykGpBiRJ1YmC0Qtm9lpIXhma23LOM60K6ZnAgbHR04AVRaSnJUh3FUj//v1p2rQpnTp1yk3LyMigR48epKen07VrV6ZPn547bPLkyaSnp3PYYYdxwgknALBgwQLS09NzX/Xr1+fBBx/MHeeRRx6hffv2HHbYYdxyyy1lN3POud1SagEp9JobDswzswdig8YDV4bPVwLjYulXKNIDWB+a9N4BTpHUUFJD4BTgnTBso6QeYVpXxMpyFcRVV13Fv//97zxpt9xyC3fccQcZGRncdddduUHk+++/57rrrmP8+PHMmTOHV155BYD27duTkZFBRkYGM2fOpE6dOpx99tkATJo0iXHjxvHll18yZ84cbrrpprKdQedc0krzwthjgcuB2ZIyQtofgaHAy5KuBr4Bzg/DJhD1sFtM1O27H4CZfSfpbuCzkO8uM/sufL6Wnd2+3w4vV4Ecf/zxu1xtLokNG6LW3fXr19OiRQsAXnzxRc455xxatmwJQNOmTcnv/fffp02bNhx00EEAPP744wwaNCj3gsJE4zjnUkOpBSQz+w+Jz/MAnJQgvwG/KaCsEcCIBOkzgE67juEqsgcffJBTTz2Vm266iR07dvDJJ58AsHDhQrZt20avXr3YuHEjAwcO5Iorrsgz7pgxY7j44otzvy9cuJCPPvqIwYMHU6tWLe6//36OLtO5cSmrpLv/e7f+PeZ3anAp5/HHH2fYsGEsX76cYcOGcfXVVwOQnZ3NzJkzeeutt3jnnXe4++67WbhwYe54W7duZfz48Zx//vm5adnZ2axbt45p06Zx3333ccEFF+DXTru9zbJly3jxxReLNe4xxxxTwrUpPg9ILuWMGjWKc845B4Dzzz8/t1NDWloavXv3pm7duuy3334cf/zxzJo1K3e8t99+m6OOOirPFfBpaWmcc845SKJbt25UqVKFNZs9ILm9S2EBKTs7u9Bxc1ogUoEHJJdyWrRowYcffgjABx98QNu2bQHo27cvH330EdnZ2WzevJlPP/00z9Xio0ePztNcB3DWWWfxwQcfAFHz3datW9mvTkEtyc6VrrPOOosuXbpw2GGH8dRTTwGwzz775A7/17/+xVVXXQXA119/TY8ePTj66KO5/fbb8+TLb9CgQXz00Uekp6czbNgwRo4cyfnnn8+ZZ57JKaecwqZNmzjppJM46qijOPzwwxk3bmf/r5xyJ0+eTK9evTjvvPPo0KEDl156aZm3Jvjdvl25uvjii5k8eTJr1qwhLS2NO++8k6effpqBAweSnZ1NrVq1cnfcjh070rt3bzp37kyVKlUYMGBAbnfxzZs3M3HiRJ588sk85ffv35/+/fvTqVMnatSowahRo9CUs8t8Pp2D6JlHjRo14scff+Too4/m3HPPLTDvwIEDGThwIBdffDFPPPFEoeUOHTqU+++/nzfffBOAkSNHMnXqVL788ksaNWpEdnY2Y8eOpX79+qxZs4YePXrQp08fog7KO33xxRfMmTOHFi1acOyxx/Lxxx9z3HHH7fmMJ8kDkitXo0ePTpg+c+bMhOk333wzN9988y7pderUSfhAsxo1avD888/nTZyy+/V0riQkeuZRQaZOncrrr78OwCWXXLLblyycfPLJNGrUCAAz449//CNTpkyhSpUqfPvtt6xcuZL9998/zzjdunUjLS26vDM9PZ1ly5Z5QHLOub1NQc88ih+lJHoGUnHFn7f0wgsvsHr1ambOnEn16tVp1apVUs9bKur8U0nzgOScq5zKuJt2Qc88atasGfPmzaN9+/aMHTuWevXqAdCjRw9effVVLrzwQsaMGVNo2YU9ayln2k2bNqV69epMmjSJ//73vyU3YyXIOzU451wZ6N27N9nZ2XTu3Jnbbrst95lHQ4cO5Ze//CUnnngizZs3z83/4IMP8sADD9CtWzeysrLYd9+Cr5vq3Lkz1apV44gjjmDYsGG7DL/00kuZMWMGXbt25YUXXqBDhw4lP4MloNSeh5Sq/HlIqaXVoLfKfJrLal1S5tME/MLJclbRnoe0efNmateujSTGjBnD6NGj8/SOK0+l9Twkb7JzzrkUNHPmTK6//nrMjAYNGjBixC43q9nreEByzrkU1LNnzzwXfgPMnj2byy+/PE9azZo1+fTTT8uyaqXGA5JzrtIws12uvalIDj/8cDIyMorOWIpK8zSPd2pwzlUKtWrVYu3atX4vwz1gZqxdu5ZatWqVSvl+hOScqxTS0tLIzMxk9erV5V2VCq1WrVq5F8+WNA9IzrlKoXr16rRu3bq8q+EK4U12zjnnUoIHJOeccynBA5JzzrmUUGoBSdIISaskfRVLe0lSRngtk5QR0ltJ+jE27InYOF0kzZa0WNLDCn02JTWSNFHSovDesLTmxTnnXOkrzSOkkUDveIKZXWhm6WaWDrwKvBYb/HXOMDP7dSz9ceAaoG145ZQ5CHjfzNoC74fvzjnnKqhSC0hmNgX4LtGwcJRzAZD4YTg78zUH6pvZVIsuHngWOCsM7guMCp9HxdKdc85VQOV1DqknsNLM4k+nai3pC0kfSuoZ0g4AMmN5MkMaQDMzywII700LmpikayTNkDTDr0FwzrnUVF4B6WLyHh1lAS3N7EjgD8CLkuoDie7xsduXWZvZU2bW1cy6NmnSpFgVds45V7rK/MJYSdWAc4AuOWlmtgXYEj7PlPQ10I7oiCh+SXAasCJ8XimpuZllhaa9VWVRf+ecc6WjPI6QfgHMN7PcpjhJTSRVDZ8PJuq8sCQ0xW2U1COcd7oCyHkgyHjgyvD5yli6c865Cqg0u32PBqYC7SVlSro6DLqIXTszHA98KWkW8C/g12aW0yHiWuAZYDHwNfB2SB8KnCxpEXBy+O6cc66CKrUmOzO7uID0qxKkvUrUDTxR/hlApwTpa4GT9qyWzjnnUoXfqcE551xK8IDknHMuJXhAcs45lxI8IDnnnEsJHpCcc86lBA9IzjnnUoIHJOeccynBA5JzzrmU4AHJOedcSvCA5JxzLiV4QHLOOZcSPCA555xLCR6QnHPOpQQPSM4551KCByTnnHMpwQOSc865lOAByTnnXEooNCBJqirpveIULGmEpFWSvoqlDZH0raSM8Do9NuxWSYslLZB0aiy9d0hbLGlQLL21pE8lLZL0kqQaxamnc8651FBoQDKz7cBmSfsWo+yRQO8E6cPMLD28JgBIOhS4CDgsjPOPEAyrAo8BpwGHAheHvAB/DWW1BdYBVxejjs4551JEtSTy/ATMljQR+CEn0cx+V9hIZjZFUqsk69EXGGNmW4ClkhYD3cKwxWa2BEDSGKCvpHnAicAlIc8oYAjweJLTc845l2KSCUhvhVdJuV7SFcAM4EYzWwccAEyL5ckMaQDL86V3BxoD35tZdoL8zjnnKqAiA5KZjZJUG2hpZgv2cHqPA3cDFt7/DvQHlGjSJG5StELyJyTpGuAagJYtW+5ejZ1zzpWJInvZSToTyAD+Hb6nSxpfnImZ2Uoz225mO4Cn2dkslwkcGMuaBqwoJH0N0EBStXzpBU33KTPramZdmzRpUpyqO+ecK2XJdPseQhQ4vgcwswygdXEmJql57OvZQE4PvPHARZJqSmoNtAWmA58BbUOPuhpEHR/Gm5kBk4DzwvhXAuOKUyfnnHOpIZlzSNlmtl7K00pWYPNYDkmjgV7AfpIygTuAXpLSw/jLgP8HYGZzJL0MzAWygd+EHn5Iuh54B6gKjDCzOWES/weMkfRn4AtgeBLz4pxzLkUlE5C+knQJUFVSW+B3wCdFjWRmFydILjBomNk9wD0J0icAExKkL2Fnk59zzrkKLpkmu98SXR+0BRgNbABuKM1KOeecq3yS6WW3GRgs6a/RV9tY+tVyzjlX2STTy+5oSbOBL4kukJ0lqUvpV80551xlksw5pOHAdWb2EYCk44B/Ap1Ls2LOOecql2TOIW3MCUYAZvYfwJvtnHPOlagCj5AkHRU+Tpf0JFGHBgMuBCaXftWcc85VJoU12f093/c7Yp+LvA7JOeec2x0FBiQz+3lZVsQ551zlVmSnBkkNgCuAVvH8RT1+wjnnnNsdyfSym0D0aIjZwI7SrY5zzrnKKpmAVMvM/lDqNXHOOVepJdPt+zlJv5LUXFKjnFep18w551ylkswR0lbgPmAwO3vXGXBwaVXKOedc5ZNMQPoDcIiZrSntyjjnnKu8kmmymwNsLu2KOOecq9ySOULaDmRImkT0CArAu30755wrWckEpNfDyznnnCs1yTwPaVRZVMQ551zllszzkJZKWpL/lcR4IyStkvRVLO0+SfMlfSlpbLgLBJJaSfpRUkZ4PREbp4uk2ZIWS3pYkkJ6I0kTJS0K7w2Ltwicc86lgmQ6NXQFjg6vnsDDwPNJjDcS6J0vbSLQycw6AwuBW2PDvjaz9PD6dSz9ceAaoG145ZQ5CHjfzNoC74fvzjnnKqgiA5KZrY29vjWzB4ETkxhvCvBdvrR3zSw7fJ0GpBVWhqTmQH0zm2pmBjwLnBUG9wVymhNHxdKdc85VQMncXPWo2NcqREdM9Upg2v2Bl2LfW0v6AtgA/Ck8FPAAIDOWJzOkATQzsywAM8uS1LSgCUm6hugoi5YtW5ZA1Z1zzpW0ZHrZxZ+LlA0sAy7Yk4lKGhzKeiEkZQEtzWytpC7A65IOA5Rg9N1+FpOZPQU8BdC1a1d/lpNzzqWgZHrZlehzkSRdCfwSOCk0w2FmWwjXOJnZTElfA+2IjojizXppwIrweaWk5uHoqDmwqiTr6ZxzrmwV9gjzKwob0cye3d2JSeoN/B9wgpltjqU3Ab4zs+2SDibqvLDEzL6TtFFSD+BToucyPRJGGw9cCQwN7+N2tz7OOedSR2FHSEcnSBNwJtF5nEIDkqTRQC9gP0mZRI9AvxWoCUwMvbenhR51xwN3ScomujPEr80sp0PEtUQ99moDb4cXRIHoZUlXA98A5xdWH+ecc6mtsEeY/zbnc7j251Kio5tpwD1FFWxmFydIHl5A3leBVwsYNgPolCB9LXBSUfVwzjlXMRR6DklSNeAq4EaiJrPzzGxBGdTLOedcJVPYOaTfAAOJLjrtbWb/LbNaOeecq3QKO0J6hORrIS4AAB0VSURBVKjn2nHAG+GcD0TnkSzcbcE555wrEYUFpNZlVgvnnHOVXmGdGryJzjnnXJlJ5uaqzjnnXKnzgOSccy4leEByzjmXEpK52/exwBDgoJA/p5fdwaVbNeecc5VJMnf7Hg78HphJdFsf55xzrsQlE5DWm9nbRWdzzjnnii+ZgDRJ0n3Aa4RHRACY2eelVivnnHOVTjIBqXt47xpLM5J4jLlzzjmXrDJ/QJ9zzjmXSJHdviXtK+kBSTPC6++S9i2LyjnnnKs8krkOaQSwEbggvDYA/yzNSjnnnKt8kjmH1MbMzo19v1NSRmlVyDnnXOWUzBHSj5KOy/kSLpT9sfSq5JxzrjJKJiBdCzwmaZmk/wKPAr9OpnBJIyStkvRVLK2RpImSFoX3hiFdkh6WtFjSl5KOio1zZci/SNKVsfQukmaHcR5W7KFNzjnnKpYiA5KZZZjZEUBn4HAzO9LMZiVZ/kigd760QcD7ZtaW6Gm0g0L6aUDb8LoGeByiAAbcQdT9vBtwR04QC3muiY2Xf1rOOecqiMIeYX6ZmT0v6Q/50gEwsweKKtzMpkhqlS+5L9ArfB4FTAb+L6Q/a2YGTJPUQFLzkHeimX0Xpj8R6C1pMlDfzKaG9GeBswC/q4RzzlVAhXVqqBve6yUYZnswzWZmlgVgZlmSmob0A4DlsXyZIa2w9MwE6buQdA3RkRQtW7bcg6o755wrLYU9MfbJ8PE9M/s4Pix0bChpic7/WDHSd000ewp4CqBr1657Ekydc86VkmQ6NTySZFqyVoamOML7qpCeCRwYy5cGrCgiPS1BunPOuQqosHNIPwOOAZrkO49UH6i6B9McD1wJDA3v42Lp10saQ9SBYX1o0nsHuDfWkeEU4FYz+07SRkk9gE+BK9izQOmcc64cFXYOqQawT8gTP4+0ATgvmcIljSbqlLCfpEyi3nJDgZclXQ18A5wfsk8ATgcWA5uBfgAh8NwNfBby3ZXTwYGoS/pIoDZRZwbv0OCccxVUYeeQPgQ+lDTSzP5bnMLN7OICBp2UIK8BvymgnBFEtzDKnz4D6FScujnnnEstydw6aHN4HtJhQK2cRDPzx08455wrMcl0angBmA+0Bu4ElrGz+cw555wrEckEpMZmNhzYZmYfmll/oEcp18s551wlk0yT3bbwniXpDKKu1WmF5HfOOed2WzIB6c/hgXw3EnWrrg/cUKq1cs45V+kkE5DWmdl6YD3wcyi1OzU455yrxMrjTg3OOefcLsrjTg3OOefcLkr1Tg3OOedcspK+U4OkelGybSq76jnnnKsskunUUE/SF0AjAElrgCvN7KvCR3POOeeSl0ynhqeAP5jZQWZ2EFH376dKt1rOOecqm2QCUl0zm5Tzxcwms/Npss4551yJSKbJbomk24DnwvfLgKWlVyXnnHOVUTJHSP2BJsBrwNjwuV9pVsoV7vvvv+e8886jQ4cOdOzYkalTpzJr1ix+9rOfcfjhh3PmmWeyYcMGALZu3Uq/fv04/PDDOeKII5g8eXJuOYMHD+bAAw9kn332Kac5cc65nYoMSGa2zsx+Z2ZHmdmRZjbQzNaVReVcYgMHDqR3797Mnz+fWbNm0bFjRwYMGMDQoUOZPXs2Z599Nvfddx8ATz/9NACzZ89m4sSJ3HjjjezYsQOAM888k+nTp5fbfDjnXFyRAUlSO0lPSXpX0gc5r7KonNvVhg0bmDJlCldffTUANWrUoEGDBixYsIDjjz8egJNPPplXX30VgLlz53LSSdHzEJs2bUqDBg2YMWMGAD169KB58+blMBfOOberZJrsXgG+AP4E3Bx7uXKwZMkSmjRpQr9+/TjyyCMZMGAAP/zwA506dWL8+PEAvPLKKyxfvhyAI444gnHjxpGdnc3SpUuZOXNm7jDnnEslyQSkbDN73Mymm9nMnFdxJyipvaSM2GuDpBskDZH0bSz99Ng4t0paLGmBpFNj6b1D2mJJg4pbp4okOzubzz//nGuvvZYvvviCunXrMnToUEaMGMFjjz1Gly5d2LhxIzVq1ACgf//+pKWl0bVrV2644QaOOeYYqlVLpi+Lc86VrWR+md6QdB1Rh4YtOYlm9l1xJmhmC4B0AElVgW9D2f2AYWZ2fzy/pEOBi4geod4CeE9SuzD4MeBkIBP4TNJ4M5tbnHpVFGlpaaSlpdG9e3cAzjvvPIYOHcrdd9/Nu+++C8DChQt56623AKhWrRrDhg3LHf+YY46hbdu2ZV9x55wrQjIB6crwHm+mM+DgEpj+ScDX4dZEBeXpC4wxsy3AUkmLgW5h2GIzWwIgaUzIu1cHpP33358DDzyQBQsW0L59e95//30OPfRQVq1aRdOmTdmxYwd//vOf+fWvfw3A5s2bMTPq1q3LxIkTqVatGoceemg5z4Vzzu0qmV52rRO8SiIYQXTkMzr2/XpJX0oaIalhSDsAiJ/0yAxpBaXvQtI1kmZImrF69eoSqnr5eeSRR7j00kvp3LkzGRkZ/PGPf2T06NG0a9eODh060KJFC/r1i3rmr1q1iqOOOoqOHTvy17/+leeeey63nFtuuYW0tDQ2b95MWloaQ4YMKac5cs4VJtGlHrfddhudO3cmPT2dU045hRUrVgDwwgsv0LlzZzp37swxxxzDrFmzCi0nlcjMis4kHQO0InZEZWbP7tGEpRpEj0M/zMxWSmoGrCE6+robaG5m/SU9Bkw1s+fDeMOBCUTB9FQzGxDSLwe6mdlvC5tu165dLaeXmSt/rQa9VebTXFbrkjKfJgBD1pfPdF2Fd+WVV9KzZ08GDBjA1q1b2bx5M1WqVKF+/foAPPzww8ydO5cnnniCTz75hI4dO9KwYUPefvtthgwZwqefflpgOQ0aNCiROkqaaWZd96SMIpvsJD0HtAEygO0h2YA9CkjAacDnZrYSIOc9TPNp4M3wNRM4MDZeGlEgo5B055zbK+Rc6jFy5EggutQjp9NSjh9++IGc0x7HHHNMbnqPHj3IzMxMupzylsw5pK7AoZbModTuuZhYc52k5maWFb6eDeTcTXw88KKkB4g6NbQFpgMC2kpqTdQx4iKgnP76Oudc6Yhf6jFr1iy6dOnCQw89RN26dRk8eDDPPvss++67L5MmTdpl3OHDh3PaaacVWU6qSKbb91fA/iU5UUl1iHrHvRZL/puk2ZK+BH4O/B7AzOYALxN1Vvg38Bsz225m2cD1wDvAPODlkNc55/YaBV3qAXDPPfewfPlyLr30Uh599NE8402aNInhw4fz17/+tchyUkUyR0j7AXMlTSdvt+8+xZ2omW0GGudLu7yQ/PcA9yRIn0B0PmnvNmTfcpimn+9wLhUUdKlH3CWXXMIZZ5zBnXfeCcCXX37JgAEDePvtt2ncuHHS5ZS3ZALSkNKuhHPOucQKutRj0aJFudcUjh8/ng4dOgDwzTffcM455/Dcc8/Rrl27IstJJUUGpPAo81ySjiU6V/Nh4jGcc5VZq1atqFevHlWrVqVatWq590585JFHePTRR6lWrRpnnHEGf/vb31i2bBkdO3akffv2QHQS/oknngCgd+/eZGVlkZ2dTc+ePXnssceoWrVquc0XJJ63Cy+8kAULFgBRt+oGDRqQkZEBwF/+8heGDx9O1apVefjhhzn11NwbzbB9+3a6du3KAQccwJtvvplwejlyLvXYunUrBx98MP/85z8ZMGAACxYsoEqVKhx00EG5y+2uu+5i7dq1XHfddQC7rIP85aSSpO4hIymdKAhdQPQspFdLs1LOuZKR6Ad0yJAhPP300zRp0gSAe++9l9NPP73Q4JCjT58+LFmyhK+++mqXacVNmjSJ/fbbL8/3cePG8eWXX1KzZk1WrVqVO6xNmza5P+BxL7/8MvXr18fMOO+883jllVe46KKLir0sSkr+eXvppZdyP994443su2/UxD537lzGjBnDnDlzWLFiBb/4xS9YuHBhblB96KGH6NixY+6jYgqTnp5O/stVcm6gnN8zzzzDM888k3Q5qaTAgBRuz3MRUW+4tcBLRNct/byM6uacKwH5f0ABfv/733PTTTftkreg4ADw2muvFfvZWY8//jiDBg2iZs2aQHTn+aLkXGOTnZ3N1q1bKeRuLinBzHj55Zf54IPoYQjjxo3joosuombNmrRu3ZpDDjmE6dOn87Of/YzMzEzeeustBg8ezAMPPFDONU8dhfWym090a58zzew4M3uEndchOecqkU2bNvHAAw/wpz/9qci8kjjllFPo0qULTz31FBDdX/Gjjz6ie/funHDCCXz22We5+ZcuXcqRRx7JCSecwEcffZSnrFNPPZWmTZtSr149zjvvvJKdqWJING85PvroI5o1a5Z7Xufbb7/lwAN3XiqZlpbGt99+C8ANN9zA3/72N6pUSaajc+VR2NI4F/gfMEnS05JOIrr2xzlXQRT0A/roo4/SuXNn+vfvz7p1O5+3WVBwuO2227jxxhupU6dOkdP8+OOP+fzzz3n77bd57LHHmDJlCtnZ2axbt45p06Zx3333ccEFF2BmNG/enG+++YYvvviCBx54gEsuuSRPE9Y777xDVlYWW7ZsyT3yKE+J5i3H6NGjufjii3O/J7p0UxJvvvkmTZs2pUuXLmVS54qkwCY7MxsLjJVUFziL6LqgZpIeB8aa2btlVEfnXDF9/PHHtGjRglWrVnHyySfToUMHrr32Wm677TYk5QaaESNG5AaHxo0bM3PmTM466yzmzJnDkiVLWLx4McOGDWPZsmVFTrNFixZA1Cx39tlnM336dNLS0jjnnHOQRLdu3ahSpQpr1qyhSZMmuc14Xbp0oU2bNixcuJCuXXfegaZWrVr06dOHcePGcfLJJ5fKckpWonk7/vjjyc7O5rXXXmPmzJ1P5klLS8vz7LHMzExatGjB+PHjGT9+PBMmTOCnn35iw4YNXHbZZTz//PM7J1RJL/VI5uaqP5jZC2b2S6Lb82QAleLZQ85VdIl+QJs1a0bVqlWpUqUKv/rVr3IfY1+zZs3ca1biwWHq1KnMnDmTVq1acdxxx7Fw4UJ69eqVcHo//PADGzduzP387rvv0qlTJ84666zcI5yFCxeydetW9ttvP1avXs327dGZgCVLlrBo0SIOPvhgNm3aRFZWdOOW7OxsJkyYkNutubwUNG8A7733Hh06dCAtLS03f58+fRgzZgxbtmxh6dKlLFq0iG7duvGXv/yFzMxMli1bxpgxYzjxxBPzBqNKbLee1BaegfRkeDnnUtgPP/zAjh07qFevXu4P6O23305WVlbuo+vHjh2b+6O6evVqGjVqRNWqVfMEh65du3LttdcCsGzZMn75y18yefLkhNNcuXIlZ599NhAFkksuuYTevXuzdetW+vfvT6dOnahRowajRo1CElOmTOH222+nWrVqVK1alSeeeIJGjRqxcuVK+vTpw5YtW9i+fTsnnnhi7iNVyktB8wYwZsyYPM11AIcddhgXXHABhx56KNWqVUuJbuupLqm7fe9NKuTdvvfiw3e/23fpWbJkyS4/oIMHD+byyy8nIyMDSbRq1Yonn3yS5s2b8+qrr+YJDnfeeSdnnnlmnjJzAlJR3b7dHqqA+3yZ3O3bOVcxHXzwwXmehZMj/kysuHPPPZdzzz230DJbtWrlwciVGu9z6JxzLiV4QHLOOZcSvMluD+W/H9VVV13Fhx9+mHv7kJEjR5Kens59993HCy+8AETt+fPmzcs9iZyoHOcqjPI43wFlck6uPM5xAiyrVS6TLXcekPZQovtR3XfffbtcVX7zzTdz8803A/DGG28wbNiw3GBUUDnO7a7y6SRS5pN0eylvstsDOfejGjBgwG6Nl/+K7uKW45xzexMPSHugoPtRDR48mM6dO/P73/+eLVu25Bm2efNm/v3vf+fpzeT3tXLOuXIMSJKWhUeWZ0iaEdIaSZooaVF4bxjSJelhSYslfSnpqFg5V4b8iyRdWVb1L+h+VH/5y1+YP38+n332Gd99913u44NzvPHGGxx77LG5zXV+XyvnnIuU91/yn5tZeuxiqkHA+2bWFnifnbcoOg1oG17XAI9DFMCAO4DuQDfgjpwgVto+/vhjxo8fT6tWrbjooov44IMPuOyyy2jevDmSqFmzJv369cu9LUuO/Fd0F1SOc85VNuUdkPLrC4wKn0cR3dQ1J/1Zi0wDGkhqDpwKTDSz78xsHTAR6F0WFS3oflQ5998yM15//fXc27IArF+/ng8//JC+ffsWWY5zzlU25dnLzoB3JRnwpJk9BTQzsywAM8uSlPMUrwOA5bFxM0NaQel5SLqG6MiKli1blvR85HHppZeyevVqzIz09PQ8T9wcO3Ysp5xyCnXr1i3VOjjnXEVUngHpWDNbEYLOREnzC8mb6DlMVkh63oQo2D0F0b3silPZwvTq1Sv37seFPbPlqquu4qqrrkqqHOecq2zKrcnOzFaE91XAWKJzQCtDUxzhfVXIngkcGBs9DVhRSLpzzrkKplwCkqS6kurlfAZOAb4CxgM5PeWuBMaFz+OBK0Jvux7A+tC09w5wiqSGoTPDKSHNOedcBVNeTXbNiJ5Gm1OHF83s35I+A16WdDXwDXB+yD8BOB1YDGwG+kH0fCZJdwOfhXx3hWc2Oeecq2DKJSCZ2RLgiATpa4GTEqQb8JsCyhoBjCjpOibi97VyzrnSk2rdvp1zzlVSHpCcc86lBA9IzjnnUoIHJOeccynBA5JzzrmU4AHJOedcSvCA5JxzLiV4QHLOOZcSPCA555xLCR6QnHPOpQQPSM4551KCByTnnHMpwQOSc865lOAByTnnXErwgOSccy4leEByzjmXEjwgOeecSwkekJxzzqWEMg9Ikg6UNEnSPElzJA0M6UMkfSspI7xOj41zq6TFkhZIOjWW3jukLZY0qKznxTnnXMmpVg7TzAZuNLPPJdUDZkqaGIYNM7P745klHQpcBBwGtADek9QuDH4MOBnIBD6TNN7M5pbJXDjnnCtRZR6QzCwLyAqfN0qaBxxQyCh9gTFmtgVYKmkx0C0MW2xmSwAkjQl5PSA551wFVK7nkCS1Ao4EPg1J10v6UtIISQ1D2gHA8thomSGtoPRE07lG0gxJM1avXl2Cc+Ccc66klFtAkrQP8Cpwg5ltAB4H2gDpREdQf8/JmmB0KyR910Szp8ysq5l1bdKkyR7X3TnnXMkrj3NISKpOFIxeMLPXAMxsZWz408Cb4WsmcGBs9DRgRfhcULpzzrkKpjx62QkYDswzswdi6c1j2c4GvgqfxwMXSaopqTXQFpgOfAa0ldRaUg2ijg/jy2IenHPOlbzyOEI6FrgcmC0pI6T9EbhYUjpRs9sy4P8BmNkcSS8TdVbIBn5jZtsBJF0PvANUBUaY2ZyynBHnnHMlpzx62f2HxOd/JhQyzj3APQnSJxQ2nnPOuYrD79TgnHMuJXhAcs45lxI8IDnnnEsJHpCcc86lBA9IzjnnUoIHJOeccynBA5JzzrmU4AHJOedcSvCA5JxzLiV4QHLOOZcSPCA555xLCR6QnHPOpQQPSM4551KCByTnnHMpwQOSc865lOAByTnnXErwgOSccy4leEByzjmXEip8QJLUW9ICSYslDSrv+jjnnCueCh2QJFUFHgNOAw4FLpZ0aPnWyjnnXHFU6IAEdAMWm9kSM9sKjAH6lnOdnHPOFYPMrLzrUGySzgN6m9mA8P1yoLuZXZ8v3zXANeFre2BBmVZ0z+0HrCnvSuxFfHmWLF+eJa8iLtODzKzJnhRQraRqUk6UIG2XCGtmTwFPlX51SoekGWbWtbzrsbfw5VmyfHmWvMq6TCt6k10mcGDsexqwopzq4pxzbg9U9ID0GdBWUmtJNYCLgPHlXCfnnHPFUKGb7MwsW9L1wDtAVWCEmc0p52qVhgrb3JiifHmWLF+eJa9SLtMK3anBOefc3qOiN9k555zbS3hAcs45lxI8IBWDpFaSvtqN/CPDNVOF5blBUp3Y9017UsdQxid7WkZFJGmIpJsKGFYpl0lpiu8PknpJerO865RK8u/buzFeB0kZkr6Q1KaobbckfjPKmwek1HEDsNsbbSLhlkqY2TElUd7exJeJKwe7vW+HffgsYJyZHWlmX1eGbdcDUvFVlfS0pDmS3pVUW1K6pGmSvpQ0VlLD/CNJOin845ktaYSkmpJ+B7QAJkmaFMt7j6RZocxmIS3P0VbOv6Lwz3SSpBeB2fmG7SPpfUmfh+n2DemtJM3LPx+luMz2iKTLJE0P/xqflFQ13Fz387Cc3o9lP1TSZElLwvLNKSO+vCZL+pek+ZJekKQwbJmk/cLnrpImh88nhGnn/GutV3ZzX3ok1ZX0VliGX0m6UNLRkj4JadMl1Qvby0dheX8uqdAfSEmNJL0e9odpkjqH9CGSnpP0gaRFkn5VNnNa+hIsyzvIt29LelzSjLDP3Rkbd5mk2yX9B7iQKJANiI2Xs+02lzQlbIdfSeoZK2OX34wKxcz8tZsvoBWQDaSH7y8DlwFfAieEtLuAB8PnkcB5QC1gOdAupD8L3BA+LwP2i03DgDPD578Bf4qXFcu3Kbz3An4AWicYVg2oHz7vBywmustFwvko7+VbwDLvCLwBVA/f/wFcGZZn65DWKLwPAT4Baob5XRsbL7681hNdTF0FmAocl39dAF2ByeHzG8Cx4fM+QLXyXi4ltGzPBZ6Ofd8XWAIcHb7XD9tQHaBWSGsLzIjtD1/Fluub4fMjwB3h84lARmz9zAJqh/WzHGhR3suhFJdl/n07ZzutCkwGOse2u1ti+YYAN8W+52y7NwKDY2XUC58T/mZUpJcfIRXfUjPLCJ9nAm2ABmb2YUgbBRyfb5z2YbyFheTJsRXIaYufSbTTF2W6mS1NkC7gXklfAu8BBwA5/57yz0cy0ykPJwFdgM8kZYTvvwOm5MyzmX0Xy/+WmW0xszXAKnbOb9x0M8s0sx1ABkXP+8fAA+GIq4GZZe/RHKWO2cAvJP01/NtuCWSZ2WcAZrYhzGt14GlJs4FXiO6wX5jjgOdCGR8AjSXtG4aNM7Mfw/qZRHSj5L1BnmVpZusT5LlA0ufAF8Bh5F2OLyUxjc+AfpKGAIeb2caQXpzfjJTiAan4tsQ+bwcaJDFOonvvFWSbhb86ofyci5izCestNDHViI3zQwFlXQo0AbqYWTqwkuhoDXadj1S9WFrAKDNLD6/2wJ0kuHdhkMx8FZQndxmzczlhZkOBAUT/7KdJ6rDbc5GCwh+kLkQ/pn8Bzibxcv090bZzBNGRY40EeeIKu9dk/vL3igsi8y9LSbfHh0tqDdwEnGRmnYG3iG1jFLwPx6cxheiP7LfAc5KuCIMK+s2oMDwglZz1wLpYe+7lwIf58swHWkk6JEGejUAy5ySWEW3wED1qo3oS4+wLrDKzbZJ+DhyUxDip5n3gPElNITo/QdTsc0LYyXPSSsIydi7jc3MSJbUxs9lm9ldgBrBXBCRJLYDNZvY8cD/QA2gh6egwvJ6kakTbUVY4orycqLmoMFOI/gwhqRewxsw2hGF9JdWS1Jiome+zkp2r8pFgWR5F3n27PlHQWR/O8ZxWjGkcRLQ/Pw0MD9PYK1S4CJrirgSeUNTFcwnQLz7QzH6S1A94JezgnwFPhMFPAW9LyjKznxcyjaeBcZKmE/1IF/mPCngBeEPSDKKmqfm7M1OpwMzmSvoT8K6kKsA24DdEjxV5LaStAk4ugcndCQyX9Efg01j6DSGgbwfmAm+XwLRSweHAfZJ2EC3Xa4mObh4JnVx+BH5BdN7uVUnnEzWzFbXtDQH+GZqKNxPtHzmmEx0dtATuNrO95abIiZblz4jt25K+AOYQ/UZ8XIxp9AJulrQN2ARcUXj2isNvHeScK1Ph3McmM7u/vOviUos32TnnnEsJfoTknHMuJfgRknPOuZTgAck551xK8IDknHMuJXhAcs45lxI8IDnnnEsJ/x+tG7eKR0af8wAAAABJRU5ErkJggg==\n", 121 | "text/plain": [ 122 | "
" 123 | ] 124 | }, 125 | "metadata": { 126 | "needs_background": "light" 127 | }, 128 | "output_type": "display_data" 129 | } 130 | ], 131 | "source": [ 132 | "import matplotlib\n", 133 | "import matplotlib.pyplot as plt\n", 134 | "import numpy as np\n", 135 | "\n", 136 | "\n", 137 | "labels = ['holothurian', 'echinus', 'scallop', 'starfish']\n", 138 | "train_means = [4574, 18676, 5554, 5704]\n", 139 | "aug_train_means = [6991, 20818, 5653, 6326]\n", 140 | "\n", 141 | "x = np.arange(len(labels)) # the label locations\n", 142 | "width = 0.35 # the width of the bars\n", 143 | "\n", 144 | "fig, ax = plt.subplots()\n", 145 | "rects1 = ax.bar(x - width/2, train_means, width, label='train')\n", 146 | "rects2 = ax.bar(x + width/2, aug_train_means, width, label='aug_train')\n", 147 | "\n", 148 | "# Add some text for labels, title and custom x-axis tick labels, etc.\n", 149 | "ax.set_ylabel('Annotation Number')\n", 150 | "ax.set_title('The annotation number before and after data augmentation')\n", 151 | "ax.set_xticks(x)\n", 152 | "ax.set_xticklabels(labels)\n", 153 | "ax.legend()\n", 154 | "\n", 155 | "\n", 156 | "def autolabel(rects):\n", 157 | " \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n", 158 | " for rect in rects:\n", 159 | " height = rect.get_height()\n", 160 | " ax.annotate('{}'.format(height),\n", 161 | " xy=(rect.get_x() + rect.get_width() / 2, height),\n", 162 | " xytext=(0, 3), # 3 points vertical offset\n", 163 | " textcoords=\"offset points\",\n", 164 | " ha='center', va='bottom')\n", 165 | "\n", 166 | "\n", 167 | "autolabel(rects1)\n", 168 | "autolabel(rects2)\n", 169 | "\n", 170 | "fig.tight_layout()\n", 171 | "\n", 172 | "plt.show()" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [] 181 | } 182 | ], 183 | "metadata": { 184 | "kernelspec": { 185 | "display_name": "Python 3", 186 | "language": "python", 187 | "name": "python3" 188 | }, 189 | "language_info": { 190 | "codemirror_mode": { 191 | "name": "ipython", 192 | "version": 3 193 | }, 194 | "file_extension": ".py", 195 | "mimetype": "text/x-python", 196 | "name": "python", 197 | "nbconvert_exporter": "python", 198 | "pygments_lexer": "ipython3", 199 | "version": "3.7.3" 200 | } 201 | }, 202 | "nbformat": 4, 203 | "nbformat_minor": 2 204 | } 205 | -------------------------------------------------------------------------------- /code/mmdet/datasets/pipelines/transforms.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | import mmcv 7 | import numpy as np 8 | from numpy import random 9 | 10 | from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps 11 | from ..registry import PIPELINES 12 | 13 | try: 14 | from imagecorruptions import corrupt 15 | except ImportError: 16 | corrupt = None 17 | 18 | try: 19 | import albumentations 20 | from albumentations import Compose 21 | except ImportError: 22 | albumentations = None 23 | Compose = None 24 | import random 25 | import json 26 | import cv2 27 | import time 28 | import math 29 | 30 | @PIPELINES.register_module 31 | class MotionBlur(object): 32 | """Motion blurring images 33 | Args: 34 | p (float): the motion blurring probability. 35 | 36 | """ 37 | def __init__(self, p=None): 38 | self.p = p 39 | if p is not None: 40 | assert p >= 0 and p <= 1 41 | 42 | def motion_blur(self, img, img_shape): 43 | """Motion Blurring image. 44 | Args: 45 | img_shape(tuple): (height, width) 46 | """ 47 | if img_shape[1] <= 750: 48 | kernel_size = int(random.sample(range(10,20),1)[0]) 49 | elif img_shape[1] < 2000 and img_shape[1] > 750: 50 | kernel_size = int(random.sample(range(20,60),1)[0]) 51 | elif img_shape[1] >= 2000: 52 | kernel_size = int(random.sample(range(100,140),1)[0]) 53 | kernel_motion_blur = np.zeros((kernel_size, kernel_size)) 54 | kernel_motion_blur[int((kernel_size - 1) / 2), :] = np.ones(kernel_size) 55 | kernel_motion_blur = kernel_motion_blur / kernel_size 56 | img = cv2.filter2D(img, -1, kernel_motion_blur) 57 | return img 58 | 59 | def __call__(self, results): 60 | if 'motion_blur' not in results: 61 | motion_blur = True if np.random.rand() < self.p else False 62 | results['motion_blur'] = motion_blur 63 | if results['motion_blur']: 64 | # motion blurring image 65 | results['img'] = self.motion_blur(results['img'], results['img_shape']) 66 | return results 67 | 68 | def __repr__(self): 69 | return self.__class__.__name__ + '(p={})'.format( 70 | self.p) 71 | 72 | @PIPELINES.register_module 73 | class Mixup(object): 74 | """Mixup images & bbox 75 | Args: 76 | prob (float): the probability of carrying out mixup process. 77 | lambd (float): the parameter for mixup. 78 | mixup (bool): mixup switch. 79 | json_path (string): the path to dataset json file. 80 | """ 81 | 82 | def __init__(self, prob=0.5, lambd=0.5, mixup=False, 83 | json_path='data/coco/annotations/pgtrainval2017.json', 84 | img_path='data/coco/images/'): 85 | self.lambd = lambd 86 | self.prob = prob 87 | self.mixup = mixup 88 | self.json_path = json_path 89 | self.img_path = img_path 90 | with open(json_path, 'r') as json_file: 91 | all_labels = json.load(json_file) 92 | self.all_labels = all_labels 93 | 94 | def get_img2(self): 95 | # random get image2 for mixup 96 | idx2 = np.random.choice(np.arange(len(self.all_labels['images']))) 97 | img2_fn = self.all_labels['images'][idx2]['file_name'] 98 | img2_id = self.all_labels['images'][idx2]['id'] 99 | img2_path = self.img_path + img2_fn 100 | img2 = cv2.imread(img2_path) 101 | 102 | # get image2 label 103 | labels2 = [] 104 | boxes2 = [] 105 | for annt in self.all_labels['annotations']: 106 | if annt['image_id'] == img2_id: 107 | labels2.append(np.int64(annt['category_id'])) 108 | boxes2.append([np.float32(annt['bbox'][0]), 109 | np.float32(annt['bbox'][1]), 110 | np.float32(annt['bbox'][0] + annt['bbox'][2] - 1), 111 | np.float32(annt['bbox'][1] + annt['bbox'][3] - 1)]) 112 | return img2, labels2, boxes2 113 | 114 | def __call__(self, results): 115 | if self.mixup == True: 116 | if random.uniform(0, 1) > self.prob: 117 | img1 = results['img'] 118 | labels1 = results['gt_labels'] 119 | 120 | img2, labels2, boxes2 = self.get_img2() 121 | # if labels2 != []: 122 | # break 123 | 124 | height = max(img1.shape[0], img2.shape[0]) 125 | width = max(img1.shape[1], img2.shape[1]) 126 | 127 | if labels2 == []: 128 | self.lambd = 0.9 # float(round(random.uniform(0.5,0.9),1)) 129 | # mix image 130 | mixup_image = np.zeros([height, width, 3], dtype='float32') 131 | mixup_image[:img1.shape[0], :img1.shape[1], :] = img1.astype('float32') * self.lambd 132 | mixup_image[:img2.shape[0], :img2.shape[1], :] += img2.astype('float32') * (1. - self.lambd) 133 | mixup_image = mixup_image.astype('uint8') 134 | else: 135 | # mix image 136 | mixup_image = np.zeros([height, width, 3], dtype='float32') 137 | mixup_image[:img1.shape[0], :img1.shape[1], :] = img1.astype('float32') * self.lambd 138 | mixup_image[:img2.shape[0], :img2.shape[1], :] += img2.astype('float32') * (1. - self.lambd) 139 | mixup_image = mixup_image.astype('uint8') 140 | 141 | # mix labels 142 | results['gt_labels'] = np.hstack((labels1, np.array(labels2))) 143 | results['gt_bboxes'] = np.vstack((list(results['gt_bboxes']), boxes2)) 144 | 145 | results['img'] = mixup_image 146 | 147 | # if the image2 has not bboxes, the 'gt_labels' and 'gt_bboxes' need to be doubled 148 | # so at the end the half of loss weight can be added as 1 instead of 0.5 149 | # if boxes2 == []: 150 | # results['gt_labels'] = np.hstack((labels1, labels1)) 151 | # results['gt_bboxes'] = np.vstack((list(results['gt_bboxes']), list(results['gt_bboxes']))) 152 | # else: 153 | 154 | return results 155 | else: 156 | return results 157 | 158 | def __repr__(self): 159 | return self.__class__.__name__ + '(prob={}, lambd={}, mixup={}, json_path={}, img_path={})'.format(self.prob, 160 | self.lambd, 161 | self.mixup, 162 | self.json_path, 163 | self.img_path) 164 | 165 | 166 | class GaussianBlurConv(): 167 | ''' 168 | 高斯滤波 169 | 依据图像金字塔和高斯可分离滤波器思路加速 170 | ''' 171 | def FilterGaussian(self, img, sigma): 172 | ''' 173 | 高斯分离卷积，按照x轴y轴拆分运算，再合并，加速运算 174 | ''' 175 | # reject unreasonable demands 176 | if sigma > 300: 177 | sigma = 300 178 | # 获取滤波器尺寸且强制为奇数 179 | kernel_size = round(sigma * 3 * 2 +1) | 1 # 当图像类型为CV_8U的时候能量集中区域为3 * sigma, 180 | # 创建内核 181 | kernel = cv2.getGaussianKernel(ksize=kernel_size, sigma=sigma, ktype=cv2.CV_32F) 182 | # 初始化图像 183 | temp = np.zeros_like(img) 184 | # x轴滤波 185 | for j in range(temp.shape[0]): 186 | for i in range(temp.shape[1]): 187 | # 内层循环展开 188 | v1 = v2 = v3 = 0 189 | for k in range(kernel_size): 190 | source = math.floor(i+ kernel_size/2 -k) # 把第i个坐标和kernel的中心对齐 -k是从右往左遍历kernel对应的图像，得到与kernel的第k个元素相乘的图像坐标 191 | if source < 0: 192 | source = source * -1 # 如果图像超出左边缘，就反向，对称填充 193 | if source > img.shape[1]: 194 | source = math.floor(2 * (img.shape[1] - 1) - source) # 图像如果超出右边缘，就用左边从头数着补 195 | v1 += kernel[k] * img[j, source, 0] 196 | if temp.shape[2] == 1: continue 197 | v2 += kernel[k] * img[j, source, 1] 198 | v3 += kernel[k] * img[j, source, 2] 199 | temp[j, i, 0] = v1 200 | if temp.shape[2] == 1: continue 201 | temp[j, i, 1] = v2 202 | temp[j, i, 2] = v3 203 | # 分离滤波，先在原图用x轴的滤波器滤波，得到temp图，再用y轴滤波在temp图上滤波，结果一致 204 | # y轴滤波 205 | for i in range(img.shape[1]): # height 206 | for j in range(img.shape[0]): 207 | v1 = v2 = v3 = 0 208 | for k in range(kernel_size): 209 | source = math.floor(j + kernel_size/2 - k) 210 | if source < 0: 211 | source = source * -1 212 | if source > temp.shape[0]: 213 | source = math.floor(2 * (img.shape[0] - 1) - source) # 上下对称 214 | v1 += kernel[k] * temp[source, i, 0] 215 | if temp.shape[2] == 1: continue 216 | v2 += kernel[k] * temp[source, i, 1] 217 | v3 += kernel[k] * temp[source, i, 2] 218 | img[j, i, 0] = v1 219 | if img.shape[2] == 1: continue 220 | img[j, i, 1] = v2 221 | img[j, i, 2] = v3 222 | return img 223 | 224 | def FastFilter(self, img, sigma): 225 | ''' 226 | 快速滤波，按照图像金字塔，逐级降低图像分辨率，对应降低高斯核的sigma， 227 | 当sigma转换成高斯核size小于10，再进行滤波，后逐级resize 228 | 递归思路 229 | ''' 230 | # reject unreasonable demands 231 | if sigma > 300: 232 | sigma = 300 233 | # 获取滤波尺寸，且强制为奇数 234 | kernel_size = round(sigma * 3 * 2 + 1) | 1 # 当图像类型为CV_8U的时候能量集中区域为3 * sigma, 235 | # 如果s*sigma小于一个像素，则直接退出 236 | if kernel_size < 3: 237 | return 238 | # 处理方式(1) 滤波 (2) 高斯光滑处理 (3) 递归处理滤波器大小 239 | if kernel_size < 10: 240 | # img = self.FilterGaussian(img, sigma) 241 | img = cv2.GaussianBlur(img, (kernel_size, kernel_size), 0) # 官方函数 242 | return img 243 | else: 244 | # 若降采样到最小，直接退出 245 | if img.shape[1] < 2 or img.shape[0] < 2: 246 | return img 247 | sub_img = np.zeros_like(img) # 初始化降采样图像 248 | sub_img = cv2.pyrDown(img, sub_img) # 使用gaussian滤波对输入图像向下采样，缩放二分之一，仅支持CV_GAUSSIAN_5x5 249 | sub_img = self.FastFilter(sub_img, sigma/2.0) 250 | img = cv2.resize(sub_img, (img.shape[1], img.shape[0])) # resize到原图大小 251 | return img 252 | 253 | def __call__(self, x, sigma): 254 | x = self.FastFilter(x, sigma) 255 | return x 256 | 257 | 258 | 259 | @PIPELINES.register_module 260 | class Retinex(object): 261 | """ 262 | SSR: baseline 263 | MSR: keep the high fidelity and the dynamic range as well as compressing img 264 | MSRCR_GIMP: 265 | Adapt the dynamics of the colors according to the statistics of the first and second order. 266 | The use of the variance makes it possible to control the degree of saturation of the colors. 267 | """ 268 | def __init__(self, model='MSR', sigma=[30, 150, 300], restore_factor=2.0, color_gain=10.0, gain=270.0, offset=128.0): 269 | self.model_list = ['SSR','MSR'] 270 | if model in self.model_list: 271 | self.model = model 272 | else: 273 | raise ValueError 274 | self.sigma = sigma # 高斯核的方差 275 | # 颜色恢复 276 | self.restore_factor = restore_factor # 控制颜色修复的非线性 277 | self.color_gain = color_gain # 控制颜色修复增益 278 | # 图像恢复 279 | self.gain = gain # 图像像素值改变范围的增益 280 | self.offset = offset # 图像像素值改变范围的偏移量 281 | self.gaussian_conv = GaussianBlurConv() # 实例化高斯算子 282 | 283 | def _SSR(self, img, sigma): 284 | filter_img = self.gaussian_conv(img, sigma) # [h,w,c] 285 | retinex = np.log10(img) - np.log10(filter_img) 286 | return retinex 287 | 288 | def _MSR(self, img, simga): 289 | retinex = np.zeros_like(img) 290 | for sig in simga: 291 | retinex += self._SSR(img, sig) 292 | retinex = retinex / float(len(self.sigma)) 293 | return retinex 294 | 295 | def _colorRestoration(self, img, retinex): 296 | img_sum = np.sum(img, axis=2, keepdims=True) # 在通道层面求和 297 | # 颜色恢复 298 | # 权重矩阵归一化 并求对数，得到颜色增益 299 | color_restoration = np.log10((img * self.restore_factor / img_sum) * 1.0 + 1.0) 300 | # 将Retinex做差后的图像，按照权重和颜色增益重新组合 301 | img_merge = retinex * color_restoration * self.color_gain 302 | # 恢复图像 303 | img_restore = img_merge * self.gain + self.offset 304 | return img_restore 305 | 306 | def _simplestColorBalance(self, img, low_clip, high_clip): 307 | total = img.shape[0] * img.shape[1] 308 | for i in range(img.shape[2]): 309 | unique, counts = np.unique(img[:, :, i], return_counts=True) # 返回新列表元素在旧列表中的位置，并以列表形式储存在s中 310 | current = 0 311 | for u, c in zip(unique, counts): 312 | if float(current) / total < low_clip: 313 | low_val = u 314 | if float(current) / total < high_clip: 315 | high_val = u 316 | current += c 317 | img[:, :, i] = np.maximum(np.minimum(img[:, :, i], high_val), low_val) 318 | return img 319 | 320 | def _MSRCR_GIMP(self, results): 321 | 322 | self.img = results['img'] 323 | self.img = np.float32(self.img) + 1.0 324 | if self.model == 'SSR': 325 | self.retinex = self._SSR(self.img, self.sigma) 326 | elif self.model == 'MSR': 327 | self.retinex = self._MSR(self.img, self.sigma) 328 | # 颜色恢复 图像恢复 329 | self.img_restore = self._colorRestoration(self.img, self.retinex) 330 | results['img'] = self.img_restore 331 | 332 | def __call__(self, results): 333 | self._MSRCR_GIMP(results) 334 | return results 335 | 336 | def __repr__(self): 337 | repr_str = self.__class__.__name__ 338 | repr_str += '{},sigma={},dynamic={}'.format(self.model, self.sigma) 339 | return repr_str 340 | 341 | 342 | @PIPELINES.register_module 343 | class Resize(object): 344 | """Resize images & bbox & mask. 345 | 346 | This transform resizes the input image to some scale. Bboxes and masks are 347 | then resized with the same scale factor. If the input dict contains the key 348 | "scale", then the scale in the input dict is used, otherwise the specified 349 | scale in the init method is used. 350 | 351 | `img_scale` can either be a tuple (single-scale) or a list of tuple 352 | (multi-scale). There are 3 multiscale modes: 353 | - `ratio_range` is not None: randomly sample a ratio from the ratio range 354 | and multiply it with the image scale. 355 | - `ratio_range` is None and `multiscale_mode` == "range": randomly sample a 356 | scale from the a range. 357 | - `ratio_range` is None and `multiscale_mode` == "value": randomly sample a 358 | scale from multiple scales. 359 | 360 | Args: 361 | img_scale (tuple or list[tuple]): Images scales for resizing. 362 | multiscale_mode (str): Either "range" or "value". 363 | ratio_range (tuple[float]): (min_ratio, max_ratio) 364 | keep_ratio (bool): Whether to keep the aspect ratio when resizing the 365 | image. 366 | """ 367 | 368 | def __init__(self, 369 | img_scale=None, 370 | multiscale_mode='range', 371 | ratio_range=None, 372 | keep_ratio=True): 373 | if img_scale is None: 374 | self.img_scale = None 375 | else: 376 | if isinstance(img_scale, list): 377 | self.img_scale = img_scale 378 | else: 379 | self.img_scale = [img_scale] 380 | assert mmcv.is_list_of(self.img_scale, tuple) 381 | 382 | if ratio_range is not None: 383 | # mode 1: given a scale and a range of image ratio 384 | assert len(self.img_scale) == 1 385 | else: 386 | # mode 2: given multiple scales or a range of scales 387 | assert multiscale_mode in ['value', 'range'] 388 | 389 | self.multiscale_mode = multiscale_mode 390 | self.ratio_range = ratio_range 391 | self.keep_ratio = keep_ratio 392 | 393 | @staticmethod 394 | def random_select(img_scales): 395 | assert mmcv.is_list_of(img_scales, tuple) 396 | scale_idx = np.random.randint(len(img_scales)) 397 | img_scale = img_scales[scale_idx] 398 | return img_scale, scale_idx 399 | 400 | @staticmethod 401 | def random_sample(img_scales): 402 | assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 403 | img_scale_long = [max(s) for s in img_scales] 404 | img_scale_short = [min(s) for s in img_scales] 405 | long_edge = np.random.randint( 406 | min(img_scale_long), 407 | max(img_scale_long) + 1) 408 | short_edge = np.random.randint( 409 | min(img_scale_short), 410 | max(img_scale_short) + 1) 411 | img_scale = (long_edge, short_edge) 412 | return img_scale, None 413 | 414 | @staticmethod 415 | def random_sample_ratio(img_scale, ratio_range): 416 | assert isinstance(img_scale, tuple) and len(img_scale) == 2 417 | min_ratio, max_ratio = ratio_range 418 | assert min_ratio <= max_ratio 419 | ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio 420 | scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) 421 | return scale, None 422 | 423 | def _random_scale(self, results): 424 | if self.ratio_range is not None: 425 | scale, scale_idx = self.random_sample_ratio( 426 | self.img_scale[0], self.ratio_range) 427 | elif len(self.img_scale) == 1: 428 | scale, scale_idx = self.img_scale[0], 0 429 | elif self.multiscale_mode == 'range': 430 | scale, scale_idx = self.random_sample(self.img_scale) 431 | elif self.multiscale_mode == 'value': 432 | scale, scale_idx = self.random_select(self.img_scale) 433 | else: 434 | raise NotImplementedError 435 | 436 | results['scale'] = scale 437 | results['scale_idx'] = scale_idx 438 | 439 | def _resize_img(self, results): 440 | if self.keep_ratio: 441 | img, scale_factor = mmcv.imrescale( 442 | results['img'], results['scale'], return_scale=True) 443 | else: 444 | img, w_scale, h_scale = mmcv.imresize( 445 | results['img'], results['scale'], return_scale=True) 446 | scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], 447 | dtype=np.float32) 448 | results['img'] = img 449 | results['img_shape'] = img.shape 450 | results['pad_shape'] = img.shape # in case that there is no padding 451 | results['scale_factor'] = scale_factor 452 | results['keep_ratio'] = self.keep_ratio 453 | 454 | def _resize_bboxes(self, results): 455 | img_shape = results['img_shape'] 456 | for key in results.get('bbox_fields', []): 457 | bboxes = results[key] * results['scale_factor'] 458 | bboxes[:, 0::2] = np.clip(bboxes[:, 0::2], 0, img_shape[1] - 1) 459 | bboxes[:, 1::2] = np.clip(bboxes[:, 1::2], 0, img_shape[0] - 1) 460 | results[key] = bboxes 461 | 462 | def _resize_masks(self, results): 463 | for key in results.get('mask_fields', []): 464 | if results[key] is None: 465 | continue 466 | if self.keep_ratio: 467 | masks = [ 468 | mmcv.imrescale( 469 | mask, results['scale_factor'], interpolation='nearest') 470 | for mask in results[key] 471 | ] 472 | else: 473 | mask_size = (results['img_shape'][1], results['img_shape'][0]) 474 | masks = [ 475 | mmcv.imresize(mask, mask_size, interpolation='nearest') 476 | for mask in results[key] 477 | ] 478 | results[key] = masks 479 | 480 | def _resize_seg(self, results): 481 | for key in results.get('seg_fields', []): 482 | if self.keep_ratio: 483 | gt_seg = mmcv.imrescale( 484 | results[key], results['scale'], interpolation='nearest') 485 | else: 486 | gt_seg = mmcv.imresize( 487 | results[key], results['scale'], interpolation='nearest') 488 | results['gt_semantic_seg'] = gt_seg 489 | 490 | def __call__(self, results): 491 | if 'scale' not in results: 492 | self._random_scale(results) 493 | self._resize_img(results) 494 | self._resize_bboxes(results) 495 | self._resize_masks(results) 496 | self._resize_seg(results) 497 | return results 498 | 499 | def __repr__(self): 500 | repr_str = self.__class__.__name__ 501 | repr_str += ('(img_scale={}, multiscale_mode={}, ratio_range={}, ' 502 | 'keep_ratio={})').format(self.img_scale, 503 | self.multiscale_mode, 504 | self.ratio_range, 505 | self.keep_ratio) 506 | return repr_str 507 | 508 | 509 | @PIPELINES.register_module 510 | class RandomFlip(object): 511 | """Flip the image & bbox & mask. 512 | 513 | If the input dict contains the key "flip", then the flag will be used, 514 | otherwise it will be randomly decided by a ratio specified in the init 515 | method. 516 | 517 | Args: 518 | flip_ratio (float, optional): The flipping probability. 519 | """ 520 | 521 | def __init__(self, flip_ratio=None, direction='horizontal'): 522 | self.flip_ratio = flip_ratio 523 | self.direction = direction 524 | if flip_ratio is not None: 525 | assert flip_ratio >= 0 and flip_ratio <= 1 526 | assert direction in ['horizontal', 'vertical'] 527 | 528 | def bbox_flip(self, bboxes, img_shape, direction): 529 | """Flip bboxes horizontally. 530 | 531 | Args: 532 | bboxes(ndarray): shape (..., 4*k) 533 | img_shape(tuple): (height, width) 534 | """ 535 | assert bboxes.shape[-1] % 4 == 0 536 | flipped = bboxes.copy() 537 | if direction == 'horizontal': 538 | w = img_shape[1] 539 | flipped[..., 0::4] = w - bboxes[..., 2::4] - 1 540 | flipped[..., 2::4] = w - bboxes[..., 0::4] - 1 541 | elif direction == 'vertical': 542 | h = img_shape[0] 543 | flipped[..., 1::4] = h - bboxes[..., 3::4] - 1 544 | flipped[..., 3::4] = h - bboxes[..., 1::4] - 1 545 | else: 546 | raise ValueError( 547 | 'Invalid flipping direction "{}"'.format(direction)) 548 | return flipped 549 | 550 | def __call__(self, results): 551 | if 'flip' not in results: 552 | flip = True if np.random.rand() < self.flip_ratio else False 553 | results['flip'] = flip 554 | if 'flip_direction' not in results: 555 | results['flip_direction'] = self.direction 556 | if results['flip']: 557 | # flip image 558 | results['img'] = mmcv.imflip( 559 | results['img'], direction=results['flip_direction']) 560 | # flip bboxes 561 | for key in results.get('bbox_fields', []): 562 | results[key] = self.bbox_flip(results[key], 563 | results['img_shape'], 564 | results['flip_direction']) 565 | # flip masks 566 | for key in results.get('mask_fields', []): 567 | results[key] = [ 568 | mmcv.imflip(mask, direction=results['flip_direction']) 569 | for mask in results[key] 570 | ] 571 | 572 | # flip segs 573 | for key in results.get('seg_fields', []): 574 | results[key] = mmcv.imflip( 575 | results[key], direction=results['flip_direction']) 576 | return results 577 | 578 | def __repr__(self): 579 | return self.__class__.__name__ + '(flip_ratio={})'.format( 580 | self.flip_ratio) 581 | 582 | 583 | @PIPELINES.register_module 584 | class Pad(object): 585 | """Pad the image & mask. 586 | 587 | There are two padding modes: (1) pad to a fixed size and (2) pad to the 588 | minimum size that is divisible by some number. 589 | 590 | Args: 591 | size (tuple, optional): Fixed padding size. 592 | size_divisor (int, optional): The divisor of padded size. 593 | pad_val (float, optional): Padding value, 0 by default. 594 | """ 595 | 596 | def __init__(self, size=None, size_divisor=None, pad_val=0): 597 | self.size = size 598 | self.size_divisor = size_divisor 599 | self.pad_val = pad_val 600 | # only one of size and size_divisor should be valid 601 | assert size is not None or size_divisor is not None 602 | assert size is None or size_divisor is None 603 | 604 | def _pad_img(self, results): 605 | if self.size is not None: 606 | padded_img = mmcv.impad(results['img'], self.size) 607 | elif self.size_divisor is not None: 608 | padded_img = mmcv.impad_to_multiple( 609 | results['img'], self.size_divisor, pad_val=self.pad_val) 610 | results['img'] = padded_img 611 | results['pad_shape'] = padded_img.shape 612 | results['pad_fixed_size'] = self.size 613 | results['pad_size_divisor'] = self.size_divisor 614 | 615 | def _pad_masks(self, results): 616 | pad_shape = results['pad_shape'][:2] 617 | for key in results.get('mask_fields', []): 618 | padded_masks = [ 619 | mmcv.impad(mask, pad_shape, pad_val=self.pad_val) 620 | for mask in results[key] 621 | ] 622 | if padded_masks: 623 | results[key] = np.stack(padded_masks, axis=0) 624 | else: 625 | results[key] = np.empty((0,) + pad_shape, dtype=np.uint8) 626 | 627 | def _pad_seg(self, results): 628 | for key in results.get('seg_fields', []): 629 | results[key] = mmcv.impad(results[key], results['pad_shape'][:2]) 630 | 631 | def __call__(self, results): 632 | self._pad_img(results) 633 | self._pad_masks(results) 634 | self._pad_seg(results) 635 | return results 636 | 637 | def __repr__(self): 638 | repr_str = self.__class__.__name__ 639 | repr_str += '(size={}, size_divisor={}, pad_val={})'.format( 640 | self.size, self.size_divisor, self.pad_val) 641 | return repr_str 642 | 643 | 644 | @PIPELINES.register_module 645 | class Normalize(object): 646 | """Normalize the image. 647 | 648 | Args: 649 | mean (sequence): Mean values of 3 channels. 650 | std (sequence): Std values of 3 channels. 651 | to_rgb (bool): Whether to convert the image from BGR to RGB, 652 | default is true. 653 | """ 654 | 655 | def __init__(self, mean, std, to_rgb=True): 656 | self.mean = np.array(mean, dtype=np.float32) 657 | self.std = np.array(std, dtype=np.float32) 658 | self.to_rgb = to_rgb 659 | 660 | def __call__(self, results): 661 | results['img'] = mmcv.imnormalize(results['img'], self.mean, self.std, 662 | self.to_rgb) 663 | results['img_norm_cfg'] = dict( 664 | mean=self.mean, std=self.std, to_rgb=self.to_rgb) 665 | return results 666 | 667 | def __repr__(self): 668 | repr_str = self.__class__.__name__ 669 | repr_str += '(mean={}, std={}, to_rgb={})'.format( 670 | self.mean, self.std, self.to_rgb) 671 | return repr_str 672 | 673 | 674 | @PIPELINES.register_module 675 | class RandomCrop(object): 676 | """Random crop the image & bboxes & masks. 677 | 678 | Args: 679 | crop_size (tuple): Expected size after cropping, (h, w). 680 | """ 681 | 682 | def __init__(self, crop_size): 683 | self.crop_size = crop_size 684 | 685 | def __call__(self, results): 686 | img = results['img'] 687 | margin_h = max(img.shape[0] - self.crop_size[0], 0) 688 | margin_w = max(img.shape[1] - self.crop_size[1], 0) 689 | offset_h = np.random.randint(0, margin_h + 1) 690 | offset_w = np.random.randint(0, margin_w + 1) 691 | crop_y1, crop_y2 = offset_h, offset_h + self.crop_size[0] 692 | crop_x1, crop_x2 = offset_w, offset_w + self.crop_size[1] 693 | 694 | # crop the image 695 | img = img[crop_y1:crop_y2, crop_x1:crop_x2, ...] 696 | img_shape = img.shape 697 | results['img'] = img 698 | results['img_shape'] = img_shape 699 | 700 | # crop bboxes accordingly and clip to the image boundary 701 | for key in results.get('bbox_fields', []): 702 | bbox_offset = np.array([offset_w, offset_h, offset_w, offset_h], 703 | dtype=np.float32) 704 | bboxes = results[key] - bbox_offset 705 | bboxes[:, 0::2] = np.clip(bboxes[:, 0::2], 0, img_shape[1] - 1) 706 | bboxes[:, 1::2] = np.clip(bboxes[:, 1::2], 0, img_shape[0] - 1) 707 | results[key] = bboxes 708 | 709 | # crop semantic seg 710 | for key in results.get('seg_fields', []): 711 | results[key] = results[key][crop_y1:crop_y2, crop_x1:crop_x2] 712 | 713 | # filter out the gt bboxes that are completely cropped 714 | if 'gt_bboxes' in results: 715 | gt_bboxes = results['gt_bboxes'] 716 | valid_inds = (gt_bboxes[:, 2] > gt_bboxes[:, 0]) & ( 717 | gt_bboxes[:, 3] > gt_bboxes[:, 1]) 718 | # if no gt bbox remains after cropping, just skip this image 719 | if not np.any(valid_inds): 720 | return None 721 | results['gt_bboxes'] = gt_bboxes[valid_inds, :] 722 | if 'gt_labels' in results: 723 | results['gt_labels'] = results['gt_labels'][valid_inds] 724 | 725 | # filter and crop the masks 726 | if 'gt_masks' in results: 727 | valid_gt_masks = [] 728 | for i in np.where(valid_inds)[0]: 729 | gt_mask = results['gt_masks'][i][crop_y1:crop_y2, 730 | crop_x1:crop_x2] 731 | valid_gt_masks.append(gt_mask) 732 | results['gt_masks'] = valid_gt_masks 733 | 734 | return results 735 | 736 | def __repr__(self): 737 | return self.__class__.__name__ + '(crop_size={})'.format( 738 | self.crop_size) 739 | 740 | 741 | @PIPELINES.register_module 742 | class SegRescale(object): 743 | """Rescale semantic segmentation maps. 744 | 745 | Args: 746 | scale_factor (float): The scale factor of the final output. 747 | """ 748 | 749 | def __init__(self, scale_factor=1): 750 | self.scale_factor = scale_factor 751 | 752 | def __call__(self, results): 753 | for key in results.get('seg_fields', []): 754 | if self.scale_factor != 1: 755 | results[key] = mmcv.imrescale( 756 | results[key], self.scale_factor, interpolation='nearest') 757 | return results 758 | 759 | def __repr__(self): 760 | return self.__class__.__name__ + '(scale_factor={})'.format( 761 | self.scale_factor) 762 | 763 | 764 | @PIPELINES.register_module 765 | class PhotoMetricDistortion(object): 766 | """Apply photometric distortion to image sequentially, every transformation 767 | is applied with a probability of 0.5. The position of random contrast is in 768 | second or second to last. 769 | 770 | 1. random brightness 771 | 2. random contrast (mode 0) 772 | 3. convert color from BGR to HSV 773 | 4. random saturation 774 | 5. random hue 775 | 6. convert color from HSV to BGR 776 | 7. random contrast (mode 1) 777 | 8. randomly swap channels 778 | 779 | Args: 780 | brightness_delta (int): delta of brightness. 781 | contrast_range (tuple): range of contrast. 782 | saturation_range (tuple): range of saturation. 783 | hue_delta (int): delta of hue. 784 | """ 785 | 786 | def __init__(self, 787 | brightness_delta=32, 788 | contrast_range=(0.5, 1.5), 789 | saturation_range=(0.5, 1.5), 790 | hue_delta=18): 791 | self.brightness_delta = brightness_delta 792 | self.contrast_lower, self.contrast_upper = contrast_range 793 | self.saturation_lower, self.saturation_upper = saturation_range 794 | self.hue_delta = hue_delta 795 | 796 | def __call__(self, results): 797 | img = results['img'] 798 | # random brightness 799 | if random.randint(2): 800 | delta = random.uniform(-self.brightness_delta, 801 | self.brightness_delta) 802 | img += delta 803 | 804 | # mode == 0 --> do random contrast first 805 | # mode == 1 --> do random contrast last 806 | mode = random.randint(2) 807 | if mode == 1: 808 | if random.randint(2): 809 | alpha = random.uniform(self.contrast_lower, 810 | self.contrast_upper) 811 | img *= alpha 812 | 813 | # convert color from BGR to HSV 814 | img = mmcv.bgr2hsv(img) 815 | 816 | # random saturation 817 | if random.randint(2): 818 | img[..., 1] *= random.uniform(self.saturation_lower, 819 | self.saturation_upper) 820 | 821 | # random hue 822 | if random.randint(2): 823 | img[..., 0] += random.uniform(-self.hue_delta, self.hue_delta) 824 | img[..., 0][img[..., 0] > 360] -= 360 825 | img[..., 0][img[..., 0] < 0] += 360 826 | 827 | # convert color from HSV to BGR 828 | img = mmcv.hsv2bgr(img) 829 | 830 | # random contrast 831 | if mode == 0: 832 | if random.randint(2): 833 | alpha = random.uniform(self.contrast_lower, 834 | self.contrast_upper) 835 | img *= alpha 836 | 837 | # randomly swap channels 838 | if random.randint(2): 839 | img = img[..., random.permutation(3)] 840 | 841 | results['img'] = img 842 | return results 843 | 844 | def __repr__(self): 845 | repr_str = self.__class__.__name__ 846 | repr_str += ('(brightness_delta={}, contrast_range={}, ' 847 | 'saturation_range={}, hue_delta={})').format( 848 | self.brightness_delta, self.contrast_range, 849 | self.saturation_range, self.hue_delta) 850 | return repr_str 851 | 852 | 853 | @PIPELINES.register_module 854 | class Expand(object): 855 | """Random expand the image & bboxes. 856 | 857 | Randomly place the original image on a canvas of 'ratio' x original image 858 | size filled with mean values. The ratio is in the range of ratio_range. 859 | 860 | Args: 861 | mean (tuple): mean value of dataset. 862 | to_rgb (bool): if need to convert the order of mean to align with RGB. 863 | ratio_range (tuple): range of expand ratio. 864 | prob (float): probability of applying this transformation 865 | """ 866 | 867 | def __init__(self, 868 | mean=(0, 0, 0), 869 | to_rgb=True, 870 | ratio_range=(1, 4), 871 | seg_ignore_label=None, 872 | prob=0.5): 873 | self.to_rgb = to_rgb 874 | self.ratio_range = ratio_range 875 | if to_rgb: 876 | self.mean = mean[::-1] 877 | else: 878 | self.mean = mean 879 | self.min_ratio, self.max_ratio = ratio_range 880 | self.seg_ignore_label = seg_ignore_label 881 | self.prob = prob 882 | 883 | def __call__(self, results): 884 | if random.uniform(0, 1) > self.prob: 885 | return results 886 | 887 | img, boxes = [results[k] for k in ('img', 'gt_bboxes')] 888 | 889 | h, w, c = img.shape 890 | ratio = random.uniform(self.min_ratio, self.max_ratio) 891 | expand_img = np.full((int(h * ratio), int(w * ratio), c), 892 | self.mean).astype(img.dtype) 893 | left = int(random.uniform(0, w * ratio - w)) 894 | top = int(random.uniform(0, h * ratio - h)) 895 | expand_img[top:top + h, left:left + w] = img 896 | boxes = boxes + np.tile((left, top), 2).astype(boxes.dtype) 897 | 898 | results['img'] = expand_img 899 | results['gt_bboxes'] = boxes 900 | 901 | if 'gt_masks' in results: 902 | expand_gt_masks = [] 903 | for mask in results['gt_masks']: 904 | expand_mask = np.full((int(h * ratio), int(w * ratio)), 905 | 0).astype(mask.dtype) 906 | expand_mask[top:top + h, left:left + w] = mask 907 | expand_gt_masks.append(expand_mask) 908 | results['gt_masks'] = expand_gt_masks 909 | 910 | # not tested 911 | if 'gt_semantic_seg' in results: 912 | assert self.seg_ignore_label is not None 913 | gt_seg = results['gt_semantic_seg'] 914 | expand_gt_seg = np.full((int(h * ratio), int(w * ratio)), 915 | self.seg_ignore_label).astype(gt_seg.dtype) 916 | expand_gt_seg[top:top + h, left:left + w] = gt_seg 917 | results['gt_semantic_seg'] = expand_gt_seg 918 | return results 919 | 920 | def __repr__(self): 921 | repr_str = self.__class__.__name__ 922 | repr_str += '(mean={}, to_rgb={}, ratio_range={}, ' \ 923 | 'seg_ignore_label={})'.format( 924 | self.mean, self.to_rgb, self.ratio_range, 925 | self.seg_ignore_label) 926 | return repr_str 927 | 928 | 929 | @PIPELINES.register_module 930 | class MinIoURandomCrop(object): 931 | """Random crop the image & bboxes, the cropped patches have minimum IoU 932 | requirement with original image & bboxes, the IoU threshold is randomly 933 | selected from min_ious. 934 | 935 | Args: 936 | min_ious (tuple): minimum IoU threshold for all intersections with 937 | bounding boxes 938 | min_crop_size (float): minimum crop's size (i.e. h,w := a*h, a*w, 939 | where a >= min_crop_size). 940 | """ 941 | 942 | def __init__(self, min_ious=(0.1, 0.3, 0.5, 0.7, 0.9), min_crop_size=0.3): 943 | # 1: return ori img 944 | self.sample_mode = (1, *min_ious, 0) 945 | self.min_crop_size = min_crop_size 946 | 947 | def __call__(self, results): 948 | img, boxes, labels = [ 949 | results[k] for k in ('img', 'gt_bboxes', 'gt_labels') 950 | ] 951 | h, w, c = img.shape 952 | while True: 953 | mode = random.choice(self.sample_mode) 954 | if mode == 1: 955 | return results 956 | 957 | min_iou = mode 958 | for i in range(50): 959 | new_w = random.uniform(self.min_crop_size * w, w) 960 | new_h = random.uniform(self.min_crop_size * h, h) 961 | 962 | # h / w in [0.5, 2] 963 | if new_h / new_w < 0.5 or new_h / new_w > 2: 964 | continue 965 | 966 | left = random.uniform(w - new_w) 967 | top = random.uniform(h - new_h) 968 | 969 | patch = np.array( 970 | (int(left), int(top), int(left + new_w), int(top + new_h))) 971 | overlaps = bbox_overlaps( 972 | patch.reshape(-1, 4), boxes.reshape(-1, 4)).reshape(-1) 973 | if overlaps.min() < min_iou: 974 | continue 975 | 976 | # center of boxes should inside the crop img 977 | center = (boxes[:, :2] + boxes[:, 2:]) / 2 978 | mask = ((center[:, 0] > patch[0]) * (center[:, 1] > patch[1]) * 979 | (center[:, 0] < patch[2]) * (center[:, 1] < patch[3])) 980 | if not mask.any(): 981 | continue 982 | boxes = boxes[mask] 983 | labels = labels[mask] 984 | 985 | # adjust boxes 986 | img = img[patch[1]:patch[3], patch[0]:patch[2]] 987 | boxes[:, 2:] = boxes[:, 2:].clip(max=patch[2:]) 988 | boxes[:, :2] = boxes[:, :2].clip(min=patch[:2]) 989 | boxes -= np.tile(patch[:2], 2) 990 | 991 | results['img'] = img 992 | results['gt_bboxes'] = boxes 993 | results['gt_labels'] = labels 994 | 995 | if 'gt_masks' in results: 996 | valid_masks = [ 997 | results['gt_masks'][i] for i in range(len(mask)) 998 | if mask[i] 999 | ] 1000 | results['gt_masks'] = [ 1001 | gt_mask[patch[1]:patch[3], patch[0]:patch[2]] 1002 | for gt_mask in valid_masks 1003 | ] 1004 | 1005 | # not tested 1006 | if 'gt_semantic_seg' in results: 1007 | results['gt_semantic_seg'] = results['gt_semantic_seg'][ 1008 | patch[1]:patch[3], patch[0]:patch[2]] 1009 | return results 1010 | 1011 | def __repr__(self): 1012 | repr_str = self.__class__.__name__ 1013 | repr_str += '(min_ious={}, min_crop_size={})'.format( 1014 | self.min_ious, self.min_crop_size) 1015 | return repr_str 1016 | 1017 | 1018 | @PIPELINES.register_module 1019 | class Corrupt(object): 1020 | 1021 | def __init__(self, corruption, severity=1): 1022 | self.corruption = corruption 1023 | self.severity = severity 1024 | 1025 | def __call__(self, results): 1026 | if corrupt is None: 1027 | raise RuntimeError('imagecorruptions is not installed') 1028 | results['img'] = corrupt( 1029 | results['img'].astype(np.uint8), 1030 | corruption_name=self.corruption, 1031 | severity=self.severity) 1032 | return results 1033 | 1034 | def __repr__(self): 1035 | repr_str = self.__class__.__name__ 1036 | repr_str += '(corruption={}, severity={})'.format( 1037 | self.corruption, self.severity) 1038 | return repr_str 1039 | 1040 | 1041 | @PIPELINES.register_module 1042 | class Albu(object): 1043 | 1044 | def __init__(self, 1045 | transforms, 1046 | bbox_params=None, 1047 | keymap=None, 1048 | update_pad_shape=False, 1049 | skip_img_without_anno=False): 1050 | """ 1051 | Adds custom transformations from Albumentations lib. 1052 | Please, visit `https://albumentations.readthedocs.io` 1053 | to get more information. 1054 | 1055 | transforms (list): list of albu transformations 1056 | bbox_params (dict): bbox_params for albumentation `Compose` 1057 | keymap (dict): contains {'input key':'albumentation-style key'} 1058 | skip_img_without_anno (bool): whether to skip the image 1059 | if no ann left after aug 1060 | """ 1061 | if Compose is None: 1062 | raise RuntimeError('albumentations is not installed') 1063 | 1064 | self.transforms = transforms 1065 | self.filter_lost_elements = False 1066 | self.update_pad_shape = update_pad_shape 1067 | self.skip_img_without_anno = skip_img_without_anno 1068 | 1069 | # A simple workaround to remove masks without boxes 1070 | if (isinstance(bbox_params, dict) and 'label_fields' in bbox_params 1071 | and 'filter_lost_elements' in bbox_params): 1072 | self.filter_lost_elements = True 1073 | self.origin_label_fields = bbox_params['label_fields'] 1074 | bbox_params['label_fields'] = ['idx_mapper'] 1075 | del bbox_params['filter_lost_elements'] 1076 | 1077 | self.bbox_params = ( 1078 | self.albu_builder(bbox_params) if bbox_params else None) 1079 | self.aug = Compose([self.albu_builder(t) for t in self.transforms], 1080 | bbox_params=self.bbox_params) 1081 | 1082 | if not keymap: 1083 | self.keymap_to_albu = { 1084 | 'img': 'image', 1085 | 'gt_masks': 'masks', 1086 | 'gt_bboxes': 'bboxes' 1087 | } 1088 | else: 1089 | self.keymap_to_albu = keymap 1090 | self.keymap_back = {v: k for k, v in self.keymap_to_albu.items()} 1091 | 1092 | def albu_builder(self, cfg): 1093 | """Import a module from albumentations. 1094 | Inherits some of `build_from_cfg` logic. 1095 | 1096 | Args: 1097 | cfg (dict): Config dict. It should at least contain the key "type". 1098 | Returns: 1099 | obj: The constructed object. 1100 | """ 1101 | assert isinstance(cfg, dict) and "type" in cfg 1102 | args = cfg.copy() 1103 | 1104 | obj_type = args.pop("type") 1105 | if mmcv.is_str(obj_type): 1106 | if albumentations is None: 1107 | raise RuntimeError('albumentations is not installed') 1108 | obj_cls = getattr(albumentations, obj_type) 1109 | elif inspect.isclass(obj_type): 1110 | obj_cls = obj_type 1111 | else: 1112 | raise TypeError( 1113 | 'type must be a str or valid type, but got {}'.format( 1114 | type(obj_type))) 1115 | 1116 | if 'transforms' in args: 1117 | args['transforms'] = [ 1118 | self.albu_builder(transform) 1119 | for transform in args['transforms'] 1120 | ] 1121 | 1122 | return obj_cls(**args) 1123 | 1124 | @staticmethod 1125 | def mapper(d, keymap): 1126 | """ 1127 | Dictionary mapper. 1128 | Renames keys according to keymap provided. 1129 | 1130 | Args: 1131 | d (dict): old dict 1132 | keymap (dict): {'old_key':'new_key'} 1133 | Returns: 1134 | dict: new dict. 1135 | """ 1136 | updated_dict = {} 1137 | for k, v in zip(d.keys(), d.values()): 1138 | new_k = keymap.get(k, k) 1139 | updated_dict[new_k] = d[k] 1140 | return updated_dict 1141 | 1142 | def __call__(self, results): 1143 | # dict to albumentations format 1144 | results = self.mapper(results, self.keymap_to_albu) 1145 | 1146 | if 'bboxes' in results: 1147 | # to list of boxes 1148 | if isinstance(results['bboxes'], np.ndarray): 1149 | results['bboxes'] = [x for x in results['bboxes']] 1150 | # add pseudo-field for filtration 1151 | if self.filter_lost_elements: 1152 | results['idx_mapper'] = np.arange(len(results['bboxes'])) 1153 | 1154 | results = self.aug(**results) 1155 | 1156 | if 'bboxes' in results: 1157 | if isinstance(results['bboxes'], list): 1158 | results['bboxes'] = np.array( 1159 | results['bboxes'], dtype=np.float32) 1160 | 1161 | # filter label_fields 1162 | if self.filter_lost_elements: 1163 | 1164 | results['idx_mapper'] = np.arange(len(results['bboxes'])) 1165 | 1166 | for label in self.origin_label_fields: 1167 | results[label] = np.array( 1168 | [results[label][i] for i in results['idx_mapper']]) 1169 | if 'masks' in results: 1170 | results['masks'] = np.array( 1171 | [results['masks'][i] for i in results['idx_mapper']]) 1172 | 1173 | if (not len(results['idx_mapper']) 1174 | and self.skip_img_without_anno): 1175 | return None 1176 | 1177 | if 'gt_labels' in results: 1178 | if isinstance(results['gt_labels'], list): 1179 | results['gt_labels'] = np.array(results['gt_labels']) 1180 | 1181 | # back to the original format 1182 | results = self.mapper(results, self.keymap_back) 1183 | 1184 | # update final shape 1185 | if self.update_pad_shape: 1186 | results['pad_shape'] = results['img'].shape 1187 | 1188 | return results 1189 | 1190 | def __repr__(self): 1191 | repr_str = self.__class__.__name__ 1192 | repr_str += '(transformations={})'.format(self.transformations) 1193 | return repr_str 1194 | --------------------------------------------------------------------------------