├── .envrc ├── src ├── __init__.py ├── extensions │ ├── __init__.py │ ├── dataset │ │ ├── coco_custom.py │ │ └── coco_otc.py │ └── metrics │ │ └── ot_cost.py ├── utils │ └── neptune_utils.py └── tools │ ├── tune_hparams.py │ ├── run_evaluation.py │ └── evaluate_bootstrap.py ├── pyproject.toml ├── data └── conf │ └── model_cfg.py ├── .gitignore ├── README.md └── notebooks └── interactive_oc_demo.ipynb /.envrc: -------------------------------------------------------------------------------- 1 | dotenv 2 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/extensions/dataset/coco_custom.py: -------------------------------------------------------------------------------- 1 | from .coco_otc import CocoOtcDataset 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "eval-detection" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["mayu-ot"] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | imageio = "^2.19.1" 10 | matplotlib = "^3.5.2" 11 | jupyter-bbox-widget = "^0.4.0" 12 | jupyterlab-widgets = "^1.1.0" 13 | notebook = "^6.4.11" 14 | openmim = "^0.1.5" 15 | pandas = "1.2.5" 16 | POT = "^0.8.2" 17 | prettytable = "^3.3.0" 18 | pycocotools = "^2.0.4" 19 | seaborn = "^0.11.2" 20 | krippendorff = "^0.5.1" 21 | pytest = "^7.1.2" 22 | neptune-client = "^0.16.2" 23 | torch = "^1.11.0" 24 | torchvision = "^0.12.0" 25 | black = "21.7b0" 26 | tqdm = "^4.64.0" 27 | 28 | [tool.poetry.dev-dependencies] 29 | 30 | [build-system] 31 | requires = ["poetry-core>=1.0.0"] 32 | build-backend = "poetry.core.masonry.api" 33 | -------------------------------------------------------------------------------- /data/conf/model_cfg.py: -------------------------------------------------------------------------------- 1 | MODEL_CFGS = { 2 | "retinanet_r50_fpn_2x_coco": "RetinaNet", 3 | "faster_rcnn_r50_fpn_2x_coco": "Faster-RCNN", 4 | "yolof_r50_c5_8x8_1x_coco": "YOLOF", 5 | "detr_r50_8x2_150e_coco": "DETR", 6 | "vfnet_r50_fpn_mstrain_2x_coco": "VFNet", 7 | } 8 | 9 | HPARAM_RUNS = { 10 | "alpha=0.5,beta=0.6": { 11 | "retinanet_r50_fpn_2x_coco": "EV-110", 12 | "faster_rcnn_r50_fpn_2x_coco": "EV-111", 13 | "yolof_r50_c5_8x8_1x_coco": "EV-112", 14 | "vfnet_r50_fpn_mstrain_2x_coco": "EV-113", 15 | }, 16 | "mAP": { 17 | "retinanet_r50_fpn_2x_coco": "EV-85", 18 | "faster_rcnn_r50_fpn_2x_coco": "EV-86", 19 | "yolof_r50_c5_8x8_1x_coco": "EV-89", 20 | "vfnet_r50_fpn_mstrain_2x_coco": "EV-90", 21 | }, 22 | } 23 | 24 | HPARAMS = { 25 | "alpha=0.5,beta=0.6": { 26 | "retinanet_r50_fpn_2x_coco": { 27 | "score_thr": 0.435989, 28 | "iou_threshold": 0.412228, 29 | }, # "EV-110" 30 | "faster_rcnn_r50_fpn_2x_coco": { 31 | "score_thr": 0.500332, 32 | "iou_threshold": 0.415385, 33 | }, # "EV-111" 34 | "yolof_r50_c5_8x8_1x_coco": { 35 | "score_thr": 0.378166, 36 | "iou_threshold": 0.53194, 37 | }, # "EV-112", 38 | "vfnet_r50_fpn_mstrain_2x_coco": { 39 | "score_thr": 0.533642, 40 | "iou_threshold": 0.618896, 41 | }, # "EV-113", 42 | }, 43 | "mAP": { 44 | "retinanet_r50_fpn_2x_coco": { 45 | "score_thr": 0.0839853, 46 | "iou_threshold": 0.405454, 47 | }, # "EV-85", 48 | "faster_rcnn_r50_fpn_2x_coco": { 49 | "score_thr": 0.697227, 50 | "iou_threshold": 0.500017, 51 | }, # "EV-86", 52 | "yolof_r50_c5_8x8_1x_coco": { 53 | "score_thr": 0.0822186, 54 | "iou_threshold": 0.569493, 55 | }, # "EV-89", 56 | "vfnet_r50_fpn_mstrain_2x_coco": { 57 | "score_thr": 0.100739, 58 | "iou_threshold": 0.714145, 59 | }, # "EV-90", 60 | }, 61 | } 62 | -------------------------------------------------------------------------------- /src/utils/neptune_utils.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | 3 | try: 4 | import neptune.new as neptune 5 | except ImportError: 6 | raise ImportWarning("neptune client is not installed") 7 | 8 | import os 9 | from data.conf.model_cfg import HPARAM_RUNS, HPARAMS 10 | import mmcv 11 | from typing import Tuple 12 | 13 | 14 | def load_hparam_cfg(model_cfg: str, key: str) -> Tuple[str]: 15 | """Load hyperparameter settings from config file 16 | 17 | Args: 18 | model_cfg (str): Model config name. 19 | key (str): Hyperparameter config name. 20 | 21 | Returns: 22 | Tuple[str]: Config strings 23 | """ 24 | if model_cfg not in HPARAMS[key]: 25 | return [] 26 | hparams = HPARAMS[key][model_cfg] 27 | 28 | score_thr = hparams["score_thr"] 29 | iou_threshold = hparams["iou_threshold"] 30 | hparam_options = ( 31 | f"model.test_cfg.score_thr={score_thr}", 32 | f"model.test_cfg.nms.iou_threshold={iou_threshold}", 33 | ) 34 | return hparam_options 35 | 36 | 37 | def load_hparam_neptune(model_cfg: str, key: str) -> Tuple[str]: 38 | """Download hyperparameter settings from neptune 39 | 40 | Args: 41 | model_cfg (str): Model config name. 42 | key (str): Hyperparameter config name. 43 | 44 | Returns: 45 | Tuple[str]: Config strings 46 | """ 47 | if model_cfg not in HPARAM_RUNS[key]: 48 | return [] 49 | run_id = HPARAM_RUNS[key][model_cfg] 50 | run = neptune.init( 51 | project=os.environ["NEPTUNE_PROJECT"], run=run_id, mode="read-only" 52 | ) 53 | hparams = run["best/params"].fetch() 54 | 55 | score_thr = hparams["best"]["params"]["score_thr"] 56 | iou_threshold = hparams["best"]["params"]["iou_threshold"] 57 | hparam_options = ( 58 | f"model.test_cfg.score_thr={score_thr}", 59 | f"model.test_cfg.nms.iou_threshold={iou_threshold}", 60 | ) 61 | run.stop() 62 | return hparam_options 63 | 64 | 65 | def load_results(run_id, model_name): 66 | run = neptune.init( 67 | project=os.environ["NEPTUNE_PROJECT"], run=run_id, mode="read-only" 68 | ) 69 | with tempfile.TemporaryDirectory() as tmpdirname: 70 | run[f"results/{model_name}"].download(tmpdirname) 71 | results = mmcv.load(os.path.join(tmpdirname, f"{model_name}.pkl")) 72 | run.stop() 73 | return results 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # pytype static type analyzer 135 | .pytype/ 136 | 137 | # Cython debug symbols 138 | cython_debug/ 139 | 140 | # my settings 141 | data/coco 142 | notebooks/exploratory 143 | notebooks/reports 144 | outputs/ 145 | .vscode/ 146 | .dist_test/ 147 | tmp/ 148 | .neptune/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Optimal Correction Cost for Object Detection Evaluation 2 | 3 | This repository is the official implementation of [Optimal Correction Cost for Object Detection Evaluation](https://arxiv.org/abs/2203.14438). 4 | 5 | ## Links 6 | - [Paper](https://openaccess.thecvf.com/content/CVPR2022/html/Otani_Optimal_Correction_Cost_for_Object_Detection_Evaluation_CVPR_2022_paper.html) 7 | - [Video](https://www.dropbox.com/s/0ckbxd2odzv2znf/oc-cost_1.mp4?dl=0) 8 | - [Poster](https://www.dropbox.com/s/w9duuk9q6wsi1do/Poster4.2-233b.pdf?dl=0) 9 | - [Blog (Japanese)](https://cyberagent.ai/blog/research/computervision/16366/) 10 | - [日経ロボティクス(Japanese)](https://xtech.nikkei.com/atcl/nxt/mag/rob/18/012600001/00101/) 11 | - A third-party [implementation](https://github.com/Solafune-Inc/OC-cost) by Solafune-Inc. 12 | 13 | ## Requirements 14 | 15 | To install requirements: 16 | 17 | ```setup 18 | poetry install 19 | mim install mmcv-full 20 | mim install mmdet 21 | ``` 22 | 23 | This code is tested on mmcv-full==1.3.10 and mmdet==2.15.0. 24 | 25 | ## Quick demo 26 | 27 | You can try OC-cost on a notebook `notebooks/interactive_oc_demo.ipynb`. 28 | 29 | ## Data 30 | 31 | If you want to test OC-cost on COCO, download [coco2017](https://cocodataset.org/#download) in `data` folder 32 | 33 | ``` 34 | data 35 | ├── annotations 36 | └── val2017 37 | ``` 38 | 39 | ## Evaluation 40 | 41 | To evaluate detectors on COCO, run: 42 | 43 | ```eval 44 | python src/tools/run_evaluation.py evaluate outputs/run_evaluation/ N_GPUs -s --use-tuned-hparam alpha=0.5,beta=0.6 45 | ``` 46 | 47 | The scirpt will download detectors from MMDetection and compute mAP and OC-cost on COCO validation 2017. 48 | 49 | ## Results 50 | OC-cost and mAP of the detectors on MMDetection on COCO validation 2017 are as follows : 51 | 52 | ### OC-cost and mAP on COCO validation 2017 53 | 54 | | Model name | mAP ↑ | OC-cost ↓ | 55 | | ------------------ |---------------- | -------------- | 56 | | Faseter-RCNN [[config](https://github.com/open-mmlab/mmdetection/blob/master/configs/fast_rcnn/fast_rcnn_r50_fpn_2x_coco.py)] | 0.38 | 0.45 | 57 | |RetinaNet [[config](https://github.com/open-mmlab/mmdetection/blob/master/configs/retinanet/retinanet_r50_fpn_2x_coco.py)] | 0.32 | 0.28 | 58 | |DETR [[config](https://github.com/open-mmlab/mmdetection/blob/master/configs/detr/detr_r50_8x2_150e_coco.py)] | 0.40 | 0.57 | 59 | |YOLOF [[config](https://github.com/open-mmlab/mmdetection/blob/master/configs/yolof/yolof_r50_c5_8x8_1x_coco.py)] | 0.32 | 0.30 | 60 | |VFNet [[config](https://github.com/open-mmlab/mmdetection/blob/master/configs/vfnet/vfnet_r50_fpn_mstrain_2x_coco.py)] | 0.37 | 0.26 | 61 | 62 | NMS parameters are tuned on OC-cost. 63 | 64 | ## Citation 65 | 66 | If this work helps your research, please cite: 67 | 68 | ``` 69 | @InProceedings{Otani_2022_CVPR, 70 | author = {Otani, Mayu and Togashi, Riku and Nakashima, Yuta and Rahtu, Esa and Heikkil\"a, Janne and Satoh, Shin'ichi}, 71 | title = {Optimal Correction Cost for Object Detection Evaluation}, 72 | booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 73 | month = {June}, 74 | year = {2022}, 75 | pages = {21107-21115} 76 | } 77 | ``` 78 | -------------------------------------------------------------------------------- /src/tools/tune_hparams.py: -------------------------------------------------------------------------------- 1 | from mim import test, download, get_model_info 2 | from mim.utils import DEFAULT_CACHE_DIR 3 | import os 4 | import json 5 | import click 6 | import time 7 | import glob 8 | import optuna 9 | import neptune.new as neptune 10 | import neptune.new.integrations.optuna as optuna_utils 11 | 12 | MODEL_CFGS = { 13 | "retinanet_r50_fpn_2x_coco": "RetinaNet", 14 | "faster_rcnn_r50_fpn_2x_coco": "Faster-RCNN", 15 | "yolof_r50_c5_8x8_1x_coco": "YOLOF", 16 | "vfnet_r50_fpn_mstrain_2x_coco": "VFNet", 17 | } 18 | 19 | 20 | @click.group() 21 | def cli(): 22 | pass 23 | 24 | 25 | @cli.command() 26 | @click.argument("dataset") 27 | @click.argument("out_dir", type=click.Path(file_okay=False, dir_okay=True)) 28 | @click.option("--measure", default="mOTC") 29 | @click.option("--alpha", default=0.5) 30 | @click.option("--beta", default=0.5) 31 | @click.option("--eval-options", type=str, multiple=True) 32 | def hptune(dataset, out_dir, measure, alpha, beta, eval_options): 33 | args = locals() 34 | timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) 35 | out_dir = os.path.join(out_dir, timestamp) 36 | 37 | if not os.path.exists(out_dir): 38 | os.makedirs(out_dir) 39 | 40 | model_infos = get_model_info("mmdet") 41 | 42 | if len(eval_options): 43 | eval_options = ( 44 | ["--eval-options"] 45 | + [ 46 | f"otc_params=[(alpha, {alpha}), (beta, {beta}), (use_dummy, True)]" 47 | ] 48 | + [x for x in eval_options] 49 | ) 50 | 51 | for model_cfg in MODEL_CFGS.keys(): 52 | run = neptune.init( 53 | project=os.environ["NEPTUNE_PROJECT"], 54 | name="tune_hparams", 55 | tags=["optuna", "hptune", MODEL_CFGS[model_cfg], measure], 56 | ) 57 | run["params"] = args 58 | neptune_callback = optuna_utils.NeptuneCallback( 59 | run, log_plot_param_importances=False 60 | ) 61 | 62 | if not os.path.exists( 63 | os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py") 64 | ): 65 | download("mmdet", [model_cfg]) 66 | 67 | model_info = model_infos.loc[model_cfg] 68 | checkpoint_name = os.path.basename(model_info.weight) 69 | 70 | def objective(trial): 71 | score_thr = trial.suggest_float("score_thr", 0.01, 0.9, log=False) 72 | iou_threshold = trial.suggest_float( 73 | "iou_threshold", 0.1, 0.9, log=False 74 | ) 75 | 76 | is_success, _ = test( 77 | package="mmdet", 78 | config=os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py"), 79 | checkpoint=os.path.join(DEFAULT_CACHE_DIR, checkpoint_name), 80 | gpus=2, 81 | launcher="pytorch", 82 | other_args=( 83 | "--eval", 84 | "bbox", 85 | "--work-dir", 86 | f"{out_dir}", 87 | "--cfg-options", 88 | f"data.test.type={dataset}", 89 | f"data.test.img_prefix=data/coco/train2017/", 90 | "data.test.ann_file=data/coco/annotations/instances_train2017_subset.json", # subset of train for hptube 91 | "data.workers_per_gpu=1", 92 | "custom_imports.imports=[src.extensions.dataset.coco_custom]", 93 | "custom_imports.allow_failed_imports=False", 94 | f"model.test_cfg.score_thr={score_thr}", 95 | f"model.test_cfg.nms.iou_threshold={iou_threshold}", 96 | *eval_options, 97 | ), 98 | ) 99 | 100 | if is_success: 101 | files = glob.glob(os.path.join(out_dir, "*.json")) 102 | latest_file = max(files, key=os.path.getctime) 103 | res = json.load(open(latest_file)) 104 | cost = res["metric"][measure] 105 | else: 106 | cost = None 107 | 108 | return cost 109 | 110 | if measure == "mOTC": 111 | direction = "minimize" 112 | else: 113 | direction = "maximize" 114 | 115 | study = optuna.create_study( 116 | study_name=f"{MODEL_CFGS[model_cfg]}", 117 | direction=direction, 118 | ) # Create a new study. 119 | study.optimize(objective, n_trials=30, callbacks=[neptune_callback]) 120 | 121 | run.stop() 122 | 123 | 124 | if __name__ == "__main__": 125 | cli() 126 | -------------------------------------------------------------------------------- /notebooks/interactive_oc_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%cd ../" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from jupyter_bbox_widget import BBoxWidget\n", 19 | "import numpy as np\n", 20 | "from src.extensions.metrics.ot_cost import get_ot_cost, get_cmap\n", 21 | "import ipywidgets as widgets\n", 22 | "%matplotlib inline" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "SAMPLE_IMG_URL = \"http://farm8.staticflickr.com/7162/6767429191_69b495e08c_z.jpg\"" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "from IPython.display import display\n", 41 | "\n", 42 | "\n", 43 | "CLASS_LABELS = [\"apple\", \"banana\", \"orange\", \"cup\"]\n", 44 | "n_class = len(CLASS_LABELS)\n", 45 | "\n", 46 | "\n", 47 | "bbox_widget = BBoxWidget(\n", 48 | " image=SAMPLE_IMG_URL,\n", 49 | " classes= CLASS_LABELS + [\" \".join([\"GT\", l]) for l in CLASS_LABELS],\n", 50 | " colors=[\"green\"] * n_class + [\"orange\"] * n_class,\n", 51 | " hide_buttons=True\n", 52 | " )\n", 53 | "\n", 54 | "w_conf = widgets.FloatSlider(value=0.5, min=0, max=1., description='Confidence')\n", 55 | "bbox_widget.attach(w_conf, name=\"confidence\")\n", 56 | "\n", 57 | "def format_bboxes(bboxes, classes, return_orders=False):\n", 58 | " orders = []\n", 59 | " formatted_bboxes = []\n", 60 | " for label in classes:\n", 61 | " formatted_bboxes.append([])\n", 62 | " for i, bbox in enumerate(bboxes):\n", 63 | " if label in bbox[\"label\"]:\n", 64 | " if bbox[\"label\"].startswith(\"GT\"):\n", 65 | " conf = 1 \n", 66 | " else:\n", 67 | " conf = bbox[\"confidence\"]\n", 68 | " formatted_bboxes[-1].append([bbox[\"x\"], bbox[\"y\"], bbox[\"x\"]+bbox[\"width\"], bbox[\"y\"]+bbox[\"height\"], conf])\n", 69 | " orders.append(i)\n", 70 | " formatted_bboxes[-1] = np.asarray(formatted_bboxes[-1], dtype=np.float32).reshape(-1, 5)\n", 71 | " if return_orders:\n", 72 | " return formatted_bboxes, orders\n", 73 | " return formatted_bboxes\n", 74 | " \n", 75 | "def evaluate_bboxes():\n", 76 | " gt_bboxes = [b for b in bbox_widget.bboxes if b[\"label\"].startswith(\"GT\")]\n", 77 | " gt_bboxes = format_bboxes(gt_bboxes, CLASS_LABELS)\n", 78 | " bboxes = [b for b in bbox_widget.bboxes if not b[\"label\"].startswith(\"GT\")]\n", 79 | " bboxes = format_bboxes(bboxes, CLASS_LABELS)\n", 80 | " cmap_func = lambda x, y: get_cmap(x, y, alpha=0.5, beta=0.6,)\n", 81 | " otc, log = get_ot_cost(gt_bboxes, bboxes, cmap_func, return_matrix=True)\n", 82 | " return otc, log\n", 83 | "\n", 84 | "w_out = widgets.Output()\n", 85 | "\n", 86 | "def update_label_conf():\n", 87 | " idx = bbox_widget.selected_index\n", 88 | " cur_label = bbox_widget.bboxes[idx][\"label\"]\n", 89 | "\n", 90 | " if cur_label.startswith(\"GT\"):\n", 91 | " return\n", 92 | " \n", 93 | " for c_name in CLASS_LABELS:\n", 94 | " if c_name in cur_label:\n", 95 | " break\n", 96 | "\n", 97 | " # re-label bboxes of c_name class\n", 98 | " for idx, b in enumerate(bbox_widget.bboxes):\n", 99 | " if b[\"label\"].startswith(\"GT\"):\n", 100 | " continue\n", 101 | " if c_name in b[\"label\"]:\n", 102 | " conf = b[\"confidence\"]\n", 103 | " new_label = f\"{c_name}|{conf}\"\n", 104 | " bbox_widget._set_bbox_property(idx, \"label\", new_label)\n", 105 | "\n", 106 | "def on_bbox_change(change):\n", 107 | " update_label_conf()\n", 108 | " w_out.clear_output(wait=True)\n", 109 | " otc, _ = evaluate_bboxes()\n", 110 | " with w_out:\n", 111 | " print(f\"OC-cost: {otc:.3f}\")\n", 112 | " \n", 113 | "bbox_widget.observe(on_bbox_change, names=['bboxes'])\n", 114 | "\n", 115 | "w_container = widgets.VBox([\n", 116 | " bbox_widget,\n", 117 | " w_conf,\n", 118 | " w_out,\n", 119 | "])\n", 120 | "display(w_container)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [] 129 | } 130 | ], 131 | "metadata": { 132 | "interpreter": { 133 | "hash": "0f16fb013a11341aac360d319dcd451a965e953453d16294bbcc10151dd53404" 134 | }, 135 | "kernelspec": { 136 | "display_name": "Python 3 (ipykernel)", 137 | "language": "python", 138 | "name": "python3" 139 | }, 140 | "language_info": { 141 | "codemirror_mode": { 142 | "name": "ipython", 143 | "version": 3 144 | }, 145 | "file_extension": ".py", 146 | "mimetype": "text/x-python", 147 | "name": "python", 148 | "nbconvert_exporter": "python", 149 | "pygments_lexer": "ipython3", 150 | "version": "3.8.2" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 4 155 | } 156 | -------------------------------------------------------------------------------- /src/extensions/metrics/ot_cost.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Sequence, Tuple, Callable, Union 2 | from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps 3 | import ot 4 | import numpy as np 5 | import numpy.typing as npt 6 | from scipy.spatial.distance import cdist 7 | 8 | 9 | def bbox_gious( 10 | bboxes1: npt.ArrayLike, 11 | bboxes2: npt.ArrayLike, 12 | eps: float = 1e-6, 13 | use_legacy_coordinate: bool = False, 14 | ) -> npt.ArrayLike: 15 | """Calculate the generalized ious between each bbox of bboxes1 and bboxes2. 16 | Args: 17 | bboxes1 (ndarray): Shape (n, 4) # [[x1, y1, x2, y2], ...] 18 | bboxes2 (ndarray): Shape (k, 4) # [[x1, y1, x2, y2], ...] 19 | use_legacy_coordinate (bool): Whether to use coordinate system in 20 | mmdet v1.x. which means width, height should be 21 | calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively. 22 | Note when function is used in `VOCDataset`, it should be 23 | True to align with the official implementation 24 | `http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCdevkit_18-May-2011.tar` 25 | Default: False. 26 | Returns: 27 | gious (ndarray): Shape (n, k) 28 | """ 29 | 30 | if not use_legacy_coordinate: 31 | extra_length = 0.0 32 | else: 33 | extra_length = 1.0 34 | 35 | bboxes1 = bboxes1.astype(np.float32) 36 | bboxes2 = bboxes2.astype(np.float32) 37 | rows = bboxes1.shape[0] 38 | cols = bboxes2.shape[0] 39 | gious = np.zeros((rows, cols), dtype=np.float32) 40 | if rows * cols == 0: 41 | return gious 42 | exchange = False 43 | if bboxes1.shape[0] > bboxes2.shape[0]: 44 | bboxes1, bboxes2 = bboxes2, bboxes1 45 | gious = np.zeros((cols, rows), dtype=np.float32) 46 | exchange = True 47 | area1 = (bboxes1[:, 2] - bboxes1[:, 0] + extra_length) * ( 48 | bboxes1[:, 3] - bboxes1[:, 1] + extra_length 49 | ) 50 | area2 = (bboxes2[:, 2] - bboxes2[:, 0] + extra_length) * ( 51 | bboxes2[:, 3] - bboxes2[:, 1] + extra_length 52 | ) 53 | for i in range(bboxes1.shape[0]): 54 | x_start = np.maximum(bboxes1[i, 0], bboxes2[:, 0]) 55 | y_start = np.maximum(bboxes1[i, 1], bboxes2[:, 1]) 56 | x_end = np.minimum(bboxes1[i, 2], bboxes2[:, 2]) 57 | y_end = np.minimum(bboxes1[i, 3], bboxes2[:, 3]) 58 | overlap = np.maximum(x_end - x_start + extra_length, 0) * np.maximum( 59 | y_end - y_start + extra_length, 0 60 | ) 61 | 62 | union = area1[i] + area2 - overlap 63 | union = np.maximum(union, eps) 64 | ious = overlap / union 65 | 66 | # Finding the coordinate of smallest enclosing box 67 | x_min = np.minimum(bboxes1[i, 0], bboxes2[:, 0]) 68 | y_min = np.minimum(bboxes1[i, 1], bboxes2[:, 1]) 69 | x_max = np.maximum(bboxes1[i, 2], bboxes2[:, 2]) 70 | y_max = np.maximum(bboxes1[i, 3], bboxes2[:, 3]) 71 | hull = (x_max - x_min + extra_length) * (y_max - y_min + extra_length) 72 | 73 | gious[i, :] = ious - (hull - union) / hull 74 | 75 | if exchange: 76 | gious = gious.T 77 | 78 | return gious 79 | 80 | 81 | def add_label(result: Sequence[Sequence]) -> npt.ArrayLike: 82 | labels = [[i] * len(r) for i, r in enumerate(result)] 83 | labels = np.hstack(labels) 84 | return np.hstack([np.vstack(result), labels[:, None]]) 85 | 86 | 87 | def cost_func(x, y, mode: str = "giou", alpha: float = 0.8): 88 | """Calculate a unit cost 89 | 90 | Args: 91 | x (np.ndarray): a detection [x1, y1, x2, y2, s, l]. s is a confidence value, and l is a classification label. 92 | y (np.ndarray): a detection [x1, y1, x2, y2, s, l]. s is a confidence value, and l is a classification label. 93 | mode (str, optional): Type of IoUs. Defaults to "giou" (Generalized IoU). 94 | alpha (float, optional): weights to balance localization and classification errors. Defaults to 0.8. 95 | 96 | Returns: 97 | float: a unit cost 98 | """ 99 | giou_val = bbox_gious(x[:4][None, :], y[:4][None, :]) # range [-1, 1] 100 | loc_cost = 1 - (giou_val + 1) * 0.5 # normalized to [0, 1] 101 | l_x, l_y = x[-1], y[-1] 102 | if l_x == l_y: 103 | cls_cost = np.abs(x[-2] - y[-2]) 104 | else: 105 | cls_cost = x[-2] + y[-2] 106 | cls_cost *= 0.5 # normalized to [0, 1] 107 | 108 | return alpha * loc_cost + (1 - alpha) * cls_cost 109 | 110 | 111 | def get_cmap( 112 | a_result: Sequence[npt.ArrayLike], 113 | b_result: Sequence[npt.ArrayLike], 114 | alpha: float = 0.8, 115 | beta: float = 0.4, 116 | mode="giou", 117 | ) -> Tuple[npt.ArrayLike]: 118 | """Calculate cost matrix 119 | 120 | Args: 121 | a_result ([type]): detections 122 | b_result ([type]): detections 123 | mode (str, optional): [description]. Defaults to "giou". 124 | 125 | Returns: 126 | dist_a (np.array): (N+1,) array. distribution over detections. 127 | dist_b (np.array): (M+1,) array. distribution over detections. 128 | cost_map: 129 | """ 130 | a_result = add_label(a_result) 131 | b_result = add_label(b_result) 132 | n = len(a_result) 133 | m = len(b_result) 134 | 135 | cost_map = np.zeros((n + 1, m + 1)) 136 | 137 | metric = lambda x, y: cost_func(x, y, alpha=alpha, mode=mode) 138 | cost_map[:n, :m] = cdist(a_result, b_result, metric) 139 | 140 | dist_a = np.ones(n + 1) 141 | dist_b = np.ones(m + 1) 142 | 143 | # cost for dummy demander / supplier 144 | cost_map[-1, :] = beta 145 | cost_map[:, -1] = beta 146 | dist_a[-1] = m 147 | dist_b[-1] = n 148 | 149 | return dist_a, dist_b, cost_map 150 | 151 | 152 | def postprocess(M: npt.ArrayLike, P: npt.ArrayLike) -> float: 153 | """drop dummy to dummy costs, normalize the transportation plan, and return total cost 154 | 155 | Args: 156 | M (npt.ArrayLike): correction cost matrix 157 | P (npt.ArrayLike)): optimal transportation plan matrix 158 | 159 | Returns: 160 | float: _description_ 161 | """ 162 | P[-1, -1] = 0 163 | P /= P.sum() 164 | total_cost = (M * P).sum() 165 | return total_cost 166 | 167 | 168 | def get_ot_cost( 169 | a_detection: list, 170 | b_detection: list, 171 | costmap_func: Callable, 172 | return_matrix: bool = False, 173 | ) -> Union[float, Tuple[float, dict]]: 174 | """[summary] 175 | 176 | Args: 177 | a_detection (list): list of detection results. a_detection[i] contains bounding boxes for i-th class. 178 | Each element is numpy array whose shape is N x 5. [[x1, y1, x2, y2, s], ...] 179 | b_detection (list): ditto 180 | costmap_func (callable): a function that takes a_detection and b_detection as input and returns a unit cost matrix 181 | Returns: 182 | [float]: optimal transportation cost 183 | """ 184 | 185 | if sum(map(len, a_detection)) == 0: 186 | if sum(map(len, b_detection)) == 0: 187 | return 0 188 | 189 | a, b, M = costmap_func(a_detection, b_detection) 190 | P = ot.emd(a, b, M) 191 | total_cost = postprocess(M, P) 192 | 193 | if return_matrix: 194 | log = {"M": M, "a": a, "b": b} 195 | return total_cost, log 196 | else: 197 | return total_cost 198 | -------------------------------------------------------------------------------- /src/tools/run_evaluation.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from mim import test, download, get_model_info 3 | from mim.utils import DEFAULT_CACHE_DIR 4 | import os 5 | import json 6 | import seaborn as sns 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | import click 10 | import time 11 | from data.conf.model_cfg import MODEL_CFGS 12 | from src.utils.neptune_utils import load_hparam_cfg 13 | 14 | try: 15 | import neptune.new as neptune 16 | from neptune.new.types import File 17 | except ImportError: 18 | raise ImportWarning("neptune client is not installed") 19 | 20 | 21 | @click.group() 22 | def cli(): 23 | pass 24 | 25 | 26 | def stylize_bars(bars, ax, txt_color="w"): 27 | ranks = np.argsort([b.get_height() for b in bars]) 28 | cmap = sns.light_palette("seagreen", reverse=True) 29 | for _, bar in enumerate(bars): 30 | ax.text( 31 | bar.get_x() + bar.get_width() * 0.5, 32 | bar.get_height() - 0.05, 33 | f"{bar.get_height():.2f}", 34 | horizontalalignment="center", 35 | fontsize=9, 36 | color=txt_color, 37 | ) 38 | 39 | for r in ranks: 40 | bars[r].set_color(cmap.pop()) 41 | 42 | 43 | def generate_reports( 44 | out_dir: str, 45 | ncols: int, 46 | neptune_run_id: str, 47 | metrics: List[str] = ["bbox_mAP", "bbox_mAP_50", "bbox_mAP_75", "mOTC"], 48 | ): 49 | sns.set_style("white") 50 | work_dir = out_dir 51 | results = [] 52 | files = os.listdir(work_dir) 53 | files.sort() 54 | for fn in files: 55 | if os.path.splitext(fn)[-1] == ".json": 56 | res = json.load(open(os.path.join(work_dir, fn))) 57 | results.append(res) 58 | 59 | models = [] 60 | for res in results: 61 | cfg_name = os.path.splitext(os.path.basename(res["config"]))[0] 62 | models.append(MODEL_CFGS[cfg_name]) 63 | 64 | data = {} 65 | data["model"] = models 66 | 67 | for metric in metrics: 68 | vals = [res["metric"][metric] for res in results] 69 | data[metric] = vals 70 | 71 | nrows = len(metrics) // ncols 72 | fig, axes = plt.subplots(nrows, ncols, figsize=(4 * ncols, 4 * nrows)) 73 | axes = axes.ravel() 74 | 75 | for i, metric in enumerate(metrics): 76 | bars = axes[i].bar( 77 | np.arange(len(models)), 78 | data[metric], 79 | width=0.3, 80 | ) 81 | axes[i].set_ylim(0, 1.0) 82 | axes[i].set_title(metric) 83 | stylize_bars(bars, axes[i], "k") 84 | 85 | axes[i].set_xticks(np.arange(len(models))) 86 | axes[i].set_xticklabels(models, rotation=45) 87 | 88 | if neptune_run_id: 89 | proj_name = os.environ["NEPTUNE_PROJECT"] 90 | nptn_run = neptune.init( 91 | project=proj_name, 92 | run=neptune_run_id, 93 | capture_hardware_metrics=False, 94 | ) 95 | nptn_run["evaluation/figs/summary"].upload(File.as_image(fig)) 96 | nptn_run.stop() 97 | 98 | fig.savefig(os.path.join(work_dir, "metric.pdf"), bbox_inches="tight") 99 | 100 | 101 | @cli.command() 102 | @click.argument( 103 | "out_dir", type=click.Path(exists=True, file_okay=False, dir_okay=True) 104 | ) 105 | @click.option("--ncols", type=int, default=4) 106 | @click.option("--neptune-run-id", type=str, default="") 107 | def generate_reports_cmd( 108 | out_dir: str, 109 | ncols: int, 110 | neptune_run_id: str, 111 | metrics: List[str] = ["bbox_mAP", "bbox_mAP_50", "bbox_mAP_75", "mOTC"], 112 | ): 113 | return generate_reports(out_dir, ncols, neptune_run_id, metrics) 114 | 115 | 116 | @cli.command() 117 | @click.argument("out_dir", type=click.Path(file_okay=False, dir_okay=True)) 118 | @click.argument("n_gpus", default=None) 119 | @click.option("--neptune-on", is_flag=True) 120 | @click.option("--use-tuned-hparam", default="") 121 | @click.option("--alpha", default=0.5) 122 | @click.option("--beta", default=0.6) 123 | @click.option("--show-on", is_flag=True) 124 | @click.option("--eval-options", type=str, multiple=True) 125 | @click.option("-s", "--run-subset", is_flag=True) 126 | def evaluate( 127 | out_dir, 128 | n_gpus, 129 | neptune_on, 130 | use_tuned_hparam, 131 | alpha, 132 | beta, 133 | show_on, 134 | eval_options, 135 | run_subset, 136 | ): 137 | """Evaluate OC-cost for detectors. 138 | 139 | Args: 140 | out_dir (str): output directory. All result files will be stored in a sub-directory with timestamp (out_dir/%Y%m%d_%H%M%S/). 141 | neptune_on (bool): When neptune_on is True, the experiment will be uploaded to the neptune server. 142 | use_tuned_hparam (str): download tuned parameters from neptune server. Set values either ["alpha=0.5,beta=0.6", "mAP"] 143 | alpha (float): OC-cost parameter. This parameter is called lambda in the paper. 144 | beta (float): _description_ 145 | show_on (flag): write detection result images in output directory 146 | eval_options (_type_): _description_ 147 | run_subset (flag): for test run. The evaluation is done on a small subset of validation set. 148 | """ 149 | 150 | args = locals() 151 | 152 | tags = [f"alpha={alpha}", f"beta={beta}"] 153 | if len(use_tuned_hparam): 154 | tags.append("use-tuned-param") 155 | if run_subset: 156 | tags.append("run-subset") 157 | 158 | nptn_cfg = [] 159 | nptn_run_id = "" 160 | if neptune_on: 161 | proj_name = os.environ["NEPTUNE_PROJECT"] 162 | run = neptune.init( 163 | proj_name, 164 | name="run_evaluation", 165 | mode="sync", 166 | capture_hardware_metrics=False, 167 | tags=tags, 168 | ) 169 | nptn_run_id = run._short_id 170 | nptn_cfg = [ 171 | f"data.test.nptn_project_id={proj_name}", 172 | f"data.test.nptn_run_id={nptn_run_id}", 173 | "", 174 | ] 175 | run["params"] = args 176 | if run_subset: 177 | data_cfg = [ 178 | "data.test.ann_file=data/coco/annotations/instances_val2017_subset.json" 179 | ] 180 | else: 181 | data_cfg = [] 182 | 183 | eval_options = ( 184 | ["--eval-options"] 185 | + [f"otc_params=[(alpha, {alpha}), (beta, {beta})]"] 186 | + [x for x in eval_options] 187 | ) 188 | 189 | timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) 190 | out_dir = os.path.join(out_dir, timestamp) 191 | 192 | if not os.path.exists(out_dir): 193 | os.makedirs(out_dir) 194 | 195 | model_infos = get_model_info("mmdet") 196 | 197 | for model_cfg in MODEL_CFGS.keys(): 198 | 199 | if not os.path.exists( 200 | os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py") 201 | ): 202 | download("mmdet", [model_cfg]) 203 | 204 | model_info = model_infos.loc[model_cfg] 205 | checkpoint_name = os.path.basename(model_info.weight) 206 | 207 | # test hyperparameters 208 | hparam_options = () 209 | if len(use_tuned_hparam): 210 | hparam_options = load_hparam_cfg(model_cfg, use_tuned_hparam) 211 | 212 | if len(nptn_cfg): 213 | nptn_cfg[ 214 | -1 215 | ] = f"data.test.nptn_metadata_suffix={MODEL_CFGS[model_cfg]}" 216 | 217 | out_pkl = f"{os.path.join(out_dir, MODEL_CFGS[model_cfg]+'.pkl')}" 218 | other_args = [ 219 | "--out", 220 | out_pkl, 221 | "--eval", 222 | "bbox", 223 | "--work-dir", 224 | f"{out_dir}", 225 | "--cfg-options", 226 | f"data.test.type=CocoOtcDataset", 227 | *nptn_cfg, 228 | *data_cfg, # to run evaluation on a small subset 229 | "custom_imports.imports=[src.extensions.dataset.coco_custom, src.utils.matplotlib_settings]", 230 | "custom_imports.imports=[src.extensions.dataset.coco_custom]", 231 | "custom_imports.allow_failed_imports=False", 232 | *hparam_options, 233 | *eval_options, 234 | ] 235 | if show_on: 236 | show_dir = os.path.join(out_dir, MODEL_CFGS[model_cfg]) 237 | if not os.path.exists(show_dir): 238 | os.makedirs(show_dir) 239 | other_args = [ 240 | "--show", 241 | "--show-score-thr", 242 | "0.0", 243 | "--show-dir", 244 | show_dir, 245 | ] + other_args 246 | 247 | _ = test( 248 | package="mmdet", 249 | config=os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py"), 250 | checkpoint=os.path.join(DEFAULT_CACHE_DIR, checkpoint_name), 251 | other_args=other_args, 252 | ) 253 | else: 254 | _ = test( 255 | package="mmdet", 256 | config=os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py"), 257 | checkpoint=os.path.join(DEFAULT_CACHE_DIR, checkpoint_name), 258 | gpus=n_gpus, 259 | launcher="pytorch", 260 | other_args=other_args, 261 | ) 262 | 263 | if neptune_on: 264 | run[f"other_args/{MODEL_CFGS[model_cfg]}"] = json.dumps(other_args) 265 | run[f"results/{MODEL_CFGS[model_cfg]}"].upload(out_pkl) 266 | 267 | generate_reports(out_dir, 4, nptn_run_id) 268 | 269 | if neptune_on: 270 | run.stop() 271 | 272 | 273 | if __name__ == "__main__": 274 | cli() 275 | -------------------------------------------------------------------------------- /src/tools/evaluate_bootstrap.py: -------------------------------------------------------------------------------- 1 | from mim import test, download, get_model_info 2 | from mim.utils import DEFAULT_CACHE_DIR 3 | import os 4 | import json 5 | import mmcv 6 | from mmcv.utils.config import Config 7 | import seaborn as sns 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | import click 11 | import time 12 | import pdb 13 | import tempfile 14 | 15 | from src.extensions.dataset.coco_custom import CocoOtcDataset 16 | from src.utils.neptune_utils import load_results 17 | from concurrent.futures import ProcessPoolExecutor, as_completed 18 | from tqdm import tqdm 19 | import glob 20 | import pandas as pd 21 | from tabulate import tabulate 22 | import neptune.new as neptune 23 | from neptune.new.types import File 24 | from data.conf.model_cfg import MODEL_CFGS 25 | from src.utils.neptune_utils import load_hparam_cfg 26 | from mmdet.datasets import build_dataset, get_loading_pipeline 27 | from random import choices 28 | 29 | 30 | @click.group() 31 | def cli(): 32 | pass 33 | 34 | 35 | def get_overall_measures(out_dir): 36 | coco = CocoOtcDataset( 37 | "data/coco/annotations/instances_val2017.json", 38 | [], 39 | test_mode=True, 40 | ) 41 | measures = {} 42 | for _, model_name in MODEL_CFGS.items(): 43 | results = mmcv.load(os.path.join(out_dir, f"{model_name}.pkl")) 44 | measures[model_name] = coco.evaluate(results) 45 | return measures 46 | 47 | 48 | @cli.command() 49 | @click.argument( 50 | "out_dir", type=click.Path(exists=True, file_okay=False, dir_okay=True) 51 | ) 52 | @click.argument("ratio", default=0.8) 53 | def generate_reports_cmd( 54 | out_dir, ratio, measures=["bbox_mAP", "bbox_mAP_50", "bbox_mAP_75", "mOTC"] 55 | ): 56 | return generate_reports(out_dir, ratio, measures) 57 | 58 | 59 | def generate_reports( 60 | out_dir, 61 | ratio, 62 | measures=["bbox_mAP", "bbox_mAP_50", "bbox_mAP_75", "mOTC"], 63 | nptn_run=None, 64 | ): 65 | sns.set_style("white") 66 | work_dir = out_dir 67 | files = glob.glob(os.path.join(work_dir, f"*.{ratio}.measures.json")) 68 | 69 | ncols = len(measures) 70 | 71 | f, axes = plt.subplots(1, ncols, figsize=(4 * ncols, 6)) 72 | 73 | data_frame = {measure: [] for measure in measures} 74 | data_frame["model"] = [] 75 | 76 | for file in files: 77 | print(file) 78 | data = json.load(open(file)) 79 | model_name = os.path.basename(file).split(".")[0] 80 | data_frame["model"] += [model_name] * len(data) 81 | for measure in measures: 82 | data_frame[measure] += [x[measure] for x in data] 83 | 84 | data_frame = pd.DataFrame(data_frame) 85 | for i, measure in enumerate(measures): 86 | ax = axes[i] 87 | sns.violinplot(x="model", y=measure, ax=ax, data=data_frame) 88 | ax.set_title(measure) 89 | y_min, _ = ax.get_ylim() 90 | ax.set_ylim(y_min, y_min + 0.15) 91 | ax.set_ylabel("") 92 | ax.set_xlabel("") 93 | ax.set_xticklabels(ax.get_xticklabels(), rotation=45) 94 | 95 | if nptn_run is not None: 96 | nptn_run["figs/summary"].upload(File.as_image(f)) 97 | 98 | f.savefig( 99 | os.path.join(work_dir, f"{ratio}_measures_dist.pdf"), 100 | bbox_inches="tight", 101 | ) 102 | 103 | display_bias(out_dir, measures, data_frame, ratio) 104 | 105 | 106 | def display_bias(out_dir, measures, data_frame, ratio): 107 | measures_overall_file = os.path.join(out_dir, "all_measure.json") 108 | if os.path.exists(measures_overall_file): 109 | measures_overall = json.load(open(measures_overall_file)) 110 | else: 111 | measures_overall = get_overall_measures(out_dir) 112 | json.dump(measures_overall, open(measures_overall_file, "w")) 113 | 114 | bias_table = [] 115 | for model_name in MODEL_CFGS.values(): 116 | entry = [] 117 | for measure in measures: 118 | all_score = measures_overall[model_name][measure] 119 | sub_mean = data_frame[data_frame.model == model_name][ 120 | measure 121 | ].mean() 122 | entry.append(f"{all_score:.4f}") 123 | entry.append(f"{sub_mean:.4f} ({sub_mean-all_score:.5f})") 124 | bias_table.append(entry) 125 | 126 | columns = np.ravel( 127 | [[measure, f"{measure} ({ratio*100}%)"] for measure in measures] 128 | ) 129 | columns = columns.tolist() 130 | table = pd.DataFrame(np.asarray(bias_table), columns=columns) 131 | table["model"] = MODEL_CFGS.values() 132 | table.set_index("model") 133 | table_str = tabulate(table, headers="keys", tablefmt="psql") 134 | print(table_str) 135 | table.to_csv(os.path.join(out_dir, f"{ratio}_bias.csv")) 136 | 137 | 138 | def prepare_dataset(model_cfg, data_file): 139 | cfg = Config.fromfile(os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py")) 140 | cfg.data.test.type = "CocoOtcDataset" 141 | cfg.data.test.ann_file = data_file 142 | cfg.data.test.test_mode = True 143 | cfg.data.test.pop("samples_per_gpu", 0) 144 | cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.pipeline) 145 | dataset = build_dataset(cfg.data.test) 146 | return dataset 147 | 148 | 149 | def eval_on_subset(out_pkl, model_cfg, ratio=0.8): 150 | results = mmcv.load(out_pkl) 151 | # write subset json file to tmp file 152 | dataset = json.load( 153 | open("data/coco/annotations/instances_val2017.json", "r") 154 | ) 155 | n_dataset = len(dataset["images"]) 156 | n_sub = int(n_dataset * ratio) 157 | 158 | sub_idx = choices(range(n_dataset), k=n_sub) 159 | new_ids = range(len(sub_idx)) 160 | 161 | sub_results = [results[i] for i in sub_idx] 162 | 163 | sub_dataset = { 164 | "images": [], 165 | "annotations": [], 166 | "categories": dataset["categories"].copy(), 167 | } 168 | for i, n_id in zip(sub_idx, new_ids): 169 | sub_dataset["images"].append(dataset["images"][i].copy()) 170 | c_id = sub_dataset["images"][-1]["id"] 171 | sub_dataset["images"][-1]["id"] = n_id 172 | anns = [ 173 | x.copy() for x in dataset["annotations"] if x["image_id"] == c_id 174 | ] 175 | for ann in anns: 176 | ann["image_id"] = n_id 177 | ann["id"] = len(sub_dataset["annotations"]) 178 | sub_dataset["annotations"].append(ann) 179 | 180 | tmp_dir = tempfile.TemporaryDirectory() 181 | tmp_file = os.path.join(tmp_dir.name, "sub_dataset.json") 182 | json.dump(sub_dataset, open(tmp_file, "w")) 183 | 184 | dataset = prepare_dataset(model_cfg, tmp_file) 185 | measure = dataset.evaluate(sub_results) 186 | return measure 187 | 188 | 189 | @cli.command() 190 | @click.option( 191 | "--out-dir", default=None, type=click.Path(file_okay=False, dir_okay=True) 192 | ) 193 | @click.option( 194 | "--load-dir", default=None, type=click.Path(file_okay=False, dir_okay=True) 195 | ) 196 | @click.option("--ratio", default=0.8) 197 | @click.option("--download-res", default="") 198 | @click.option("--use-tuned-hparam", default="") 199 | @click.option("--alpha", default=0.5) 200 | @click.option("--beta", default=0.4) 201 | @click.option("--neptune-on", is_flag=True) 202 | def evaluate( 203 | out_dir, 204 | load_dir, 205 | ratio, 206 | download_res, 207 | use_tuned_hparam, 208 | alpha, 209 | beta, 210 | neptune_on, 211 | ): 212 | args = locals() 213 | if neptune_on: 214 | proj_name = os.environ["NEPTUNE_PROJECT"] 215 | run = neptune.init( 216 | proj_name, 217 | name="evaluate_bootstrap", 218 | capture_hardware_metrics=False, 219 | tags=["bootstrap"], 220 | ) 221 | run["params"] = args 222 | 223 | if load_dir is None: 224 | timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) 225 | out_dir = os.path.join(out_dir, timestamp) 226 | 227 | if not os.path.exists(out_dir): 228 | os.makedirs(out_dir) 229 | else: 230 | out_dir = load_dir 231 | 232 | model_infos = get_model_info("mmdet") 233 | 234 | eval_options = ["--eval-options"] + [ 235 | f"otc_params=[(alpha, {alpha}), (beta, {beta}), (use_dummy, True)]" 236 | ] 237 | 238 | for model_cfg in MODEL_CFGS.keys(): 239 | 240 | if not os.path.exists( 241 | os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py") 242 | ): 243 | download("mmdet", [model_cfg]) 244 | 245 | model_info = model_infos.loc[model_cfg] 246 | checkpoint_name = os.path.basename(model_info.weight) 247 | 248 | hparam_options = () 249 | if len(use_tuned_hparam): 250 | hparam_options = load_hparam_cfg(model_cfg, use_tuned_hparam) 251 | 252 | out_pkl = f"{os.path.join(out_dir, MODEL_CFGS[model_cfg]+'.pkl')}" 253 | 254 | if len(download_res): 255 | results = load_results(download_res, MODEL_CFGS[model_cfg]) 256 | mmcv.dump(results, out_pkl) 257 | 258 | if not os.path.exists(out_pkl): 259 | _ = test( 260 | package="mmdet", 261 | config=os.path.join(DEFAULT_CACHE_DIR, model_cfg + ".py"), 262 | checkpoint=os.path.join(DEFAULT_CACHE_DIR, checkpoint_name), 263 | gpus=2, 264 | launcher="pytorch", 265 | other_args=( 266 | "--out", 267 | out_pkl, 268 | "--work-dir", 269 | f"{out_dir}", 270 | "--cfg-options", 271 | f"data.test.type=CocoOtcDataset", 272 | "custom_imports.imports=[src.extensions.dataset.coco_custom]", 273 | "custom_imports.allow_failed_imports=False", 274 | *hparam_options, 275 | *eval_options, 276 | ), 277 | ) 278 | 279 | n_trials = 100 280 | print(f"run {n_trials} trials") 281 | measures = [] 282 | 283 | progress = tqdm(total=n_trials) 284 | 285 | with ProcessPoolExecutor(10) as pool: 286 | futures = [] 287 | 288 | for _ in range(n_trials): 289 | future = pool.submit( 290 | eval_on_subset, out_pkl, model_cfg, ratio=ratio 291 | ) 292 | futures.append(future) 293 | 294 | for future in as_completed(futures): 295 | progress.update(1) 296 | measures.append(future.result()) 297 | 298 | out_file = os.path.join( 299 | out_dir, MODEL_CFGS[model_cfg] + f".{ratio}.measures.json" 300 | ) 301 | json.dump(measures, open(out_file, "w")) 302 | if neptune_on: 303 | run[f"measures/{MODEL_CFGS[model_cfg]}"].upload(out_file) 304 | 305 | if neptune_on: 306 | generate_reports(out_dir, ratio, nptn_run=run) 307 | run.stop() 308 | else: 309 | generate_reports(out_dir, ratio) 310 | 311 | 312 | if __name__ == "__main__": 313 | cli() 314 | -------------------------------------------------------------------------------- /src/extensions/dataset/coco_otc.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from mmdet.datasets.coco import CocoDataset 3 | from mmdet.datasets.builder import DATASETS 4 | import numpy as np 5 | from src.extensions.metrics.ot_cost import get_ot_cost, get_cmap 6 | from copy import deepcopy 7 | import pdb 8 | import json 9 | import time 10 | import os.path as osp 11 | from mmcv.utils import get_logger 12 | import matplotlib.pyplot as plt 13 | import seaborn as sns 14 | import neptune.new as neptune 15 | from neptune.new.types import File 16 | from mmcv.runner.dist_utils import master_only 17 | 18 | N_COCOCLASSES = 80 19 | 20 | 21 | def count_items(items): 22 | ns = [] 23 | for x in items: 24 | if x is None: 25 | n = 0 26 | else: 27 | n = sum(map(len, x)) 28 | ns.append(n) 29 | return ns 30 | 31 | 32 | def get_stats(ot_costs, gts, results): 33 | mean = np.mean(ot_costs) 34 | std = np.std(ot_costs) 35 | 36 | n_gts = count_items(gts) 37 | n_preds = count_items(results) 38 | 39 | cov_gts = np.cov(ot_costs, n_gts)[0, 1] 40 | cov_preds = np.cov(ot_costs, n_preds)[0, 1] 41 | 42 | return { 43 | "mean": mean, 44 | "std": std, 45 | "cov_n-gts": cov_gts, 46 | "cov_n-preds": cov_preds, 47 | } 48 | 49 | 50 | def draw_stats(ot_costs, gts, results): 51 | n_gts = count_items(gts) 52 | n_preds = count_items(results) 53 | figures = {} 54 | 55 | fig, axes = plt.subplots(1, 2, figsize=(10, 5)) 56 | sns.kdeplot(x=n_gts, y=ot_costs, fill=True, cmap="rocket", ax=axes[0]) 57 | # axes[0].scatter(n_gts, ot_costs) 58 | axes[0].set_title("otc vs # GTs") 59 | sns.kdeplot(x=n_preds, y=ot_costs, fill=True, cmap="rocket", ax=axes[1]) 60 | # axes[1].scatter(n_preds, ot_costs) 61 | axes[1].set_title("otc vs # Preds") 62 | figures["otc_vs_num_bb"] = fig 63 | 64 | fig, axes = plt.subplots(1, 2, sharex=True, sharey=True) 65 | axes[0].hist(n_gts, bins=10) 66 | axes[0].set_title("# Ground truth boudning boxes") 67 | axes[1].hist(n_preds, bins=10) 68 | axes[1].set_title("# Prediction boudning boxes") 69 | figures["dist_n_bb"] = fig 70 | 71 | fig = plt.figure() 72 | plt.hist(ot_costs, bins=10) 73 | plt.title("OTC Distribution") 74 | figures["dist_otc"] = fig 75 | 76 | fig_src = {"ot_costs": ot_costs, "n_gts": n_gts, "n_preds": n_preds} 77 | 78 | return figures, fig_src 79 | 80 | 81 | def write2json(ot_costs, file_names): 82 | timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) 83 | json_file = osp.join(f"tmp/otc_{timestamp}.json") 84 | data = [(f_name, c) for f_name, c in zip(file_names, ot_costs)] 85 | json.dump(data, open(json_file, "w")) 86 | 87 | 88 | def eval_ot_costs(gts, results, cmap_func): 89 | return [get_ot_cost(x, y, cmap_func) for x, y in zip(gts, results)] 90 | 91 | 92 | @DATASETS.register_module() 93 | class CocoOtcDataset(CocoDataset): 94 | def __init__( 95 | self, 96 | ann_file, 97 | pipeline, 98 | classes=None, 99 | data_root=None, 100 | img_prefix="", 101 | seg_prefix=None, 102 | proposal_file=None, 103 | test_mode=False, 104 | filter_empty_gt=True, 105 | nptn_project_id="", 106 | nptn_run_id="", 107 | nptn_metadata_suffix="", 108 | ): 109 | 110 | super().__init__( 111 | ann_file, 112 | pipeline, 113 | classes=classes, 114 | data_root=data_root, 115 | img_prefix=img_prefix, 116 | seg_prefix=seg_prefix, 117 | proposal_file=proposal_file, 118 | test_mode=test_mode, 119 | filter_empty_gt=filter_empty_gt, 120 | ) 121 | 122 | self.nptn_project_id = nptn_project_id 123 | self.nptn_run_id = nptn_run_id 124 | self.nptn_on = False 125 | 126 | if (nptn_project_id != "") and (nptn_run_id != ""): 127 | self.nptn_metadata_suffix = nptn_metadata_suffix 128 | self.nptn_on = True 129 | 130 | def evaluate( 131 | self, 132 | results, 133 | metric="bbox", 134 | logger=None, 135 | jsonfile_prefix=None, 136 | classwise=False, 137 | proposal_nums=(100, 300, 1000), 138 | iou_thrs=None, 139 | metric_items=None, 140 | eval_map=True, 141 | otc_params=[("alpha", 0.5), ("beta", 0.6)], 142 | ): 143 | """Evaluate predicted bboxes. Overide this method for your measure. 144 | 145 | Args: 146 | results ([type]): outputs of a detector 147 | metric (str, optional): [description]. Defaults to "bbox". 148 | logger ([type], optional): [description]. Defaults to None. 149 | jsonfile_prefix ([type], optional): [description]. Defaults to None. 150 | classwise (bool, optional): [description]. Defaults to False. 151 | proposal_nums (tuple, optional): [description]. Defaults to (100, 300, 1000). 152 | iou_thrs ([type], optional): [description]. Defaults to None. 153 | metric_items ([type], optional): [description]. Defaults to None. 154 | eval_map (bool): Whether to evaluating mAP 155 | otc_params (list): OC-cost parameters. 156 | alpha (lambda in the paper): balancing localization and classification costs. 157 | beta: cost of extra / missing detections. 158 | Defaults to [("alpha", 0.5), ("beta", 0.6)] 159 | 160 | Returns: 161 | dict[str, float]: {metric_name: metric_value} 162 | """ 163 | if eval_map: 164 | eval_results = super().evaluate( 165 | results, 166 | metric=metric, 167 | logger=logger, 168 | jsonfile_prefix=jsonfile_prefix, 169 | classwise=classwise, 170 | proposal_nums=proposal_nums, 171 | iou_thrs=iou_thrs, 172 | metric_items=metric_items, 173 | ) 174 | else: 175 | eval_results = {} 176 | 177 | otc_params = {k: v for k, v in otc_params} 178 | mean_otc = self.eval_OTC(results, **otc_params) 179 | eval_results["mOTC"] = mean_otc 180 | 181 | if self.nptn_on: 182 | self.upload_eval_results(eval_results) 183 | 184 | return eval_results 185 | 186 | def get_gts(self): 187 | gts = [] 188 | for i in range(len(self.img_ids)): 189 | ann_ids = self.coco.get_ann_ids(img_ids=self.img_ids[i]) 190 | ann_info = self.coco.load_anns(ann_ids) 191 | gt = self._ann2detformat(ann_info) 192 | if gt is None: 193 | gt = [ 194 | np.asarray([]).reshape(0, 5) 195 | for _ in range(len(self.CLASSES)) 196 | ] 197 | gts.append(gt) 198 | return gts 199 | 200 | @master_only 201 | def upload_eval_results(self, eval_results): 202 | nptn_run = neptune.init( 203 | project=self.nptn_project_id, 204 | run=self.nptn_run_id, 205 | mode="sync", 206 | capture_hardware_metrics=False, 207 | ) 208 | for k, v in eval_results.items(): 209 | nptn_run[f"evaluation/summary/{k}/{self.nptn_metadata_suffix}"] = v 210 | nptn_run.stop() 211 | 212 | @master_only 213 | def upload_otc_results(self, ot_costs, gts, results): 214 | nptn_run = neptune.init( 215 | project=self.nptn_project_id, 216 | run=self.nptn_run_id, 217 | mode="sync", 218 | capture_hardware_metrics=False, 219 | ) 220 | 221 | file_names = [x["file_name"] for x in self.data_infos] 222 | otc_per_img = json.dumps(list(zip(file_names, ot_costs))) 223 | nptn_run[f"evaluation/otc/per_img/{self.nptn_metadata_suffix}"].upload( 224 | File.from_content(otc_per_img, extension="json") 225 | ) 226 | 227 | for k, v in get_stats(ot_costs, gts, results).items(): 228 | nptn_run[ 229 | f"evaluation/otc/stats/{k}/{self.nptn_metadata_suffix}" 230 | ] = v 231 | 232 | figs, fig_src = draw_stats(ot_costs, gts, results) 233 | for fig_name, fig in figs.items(): 234 | nptn_run[ 235 | f"evaluation/figs/{fig_name}/{self.nptn_metadata_suffix}" 236 | ].upload(File.as_image(fig)) 237 | fig.savefig(f"tmp/{fig_name}.pdf", bbox_inches="tight") 238 | nptn_run[ 239 | f"evaluation/figs/pdfs/{fig_name}/{self.nptn_metadata_suffix}" 240 | ].upload(f"tmp/{fig_name}.pdf") 241 | 242 | nptn_run.stop() 243 | 244 | def eval_OTC( 245 | self, 246 | results, 247 | alpha=0.8, 248 | beta=0.4, 249 | get_average=True, 250 | ): 251 | gts = self.get_gts() 252 | cmap_func = lambda x, y: get_cmap( 253 | x, y, alpha=alpha, beta=beta, mode="giou" 254 | ) 255 | tic = time.time() 256 | ot_costs = eval_ot_costs(gts, results, cmap_func) 257 | toc = time.time() 258 | print("OTC DONE (t={:0.2f}s).".format(toc - tic)) 259 | 260 | if self.nptn_on: 261 | self.upload_otc_results(ot_costs, gts, results) 262 | 263 | if get_average: 264 | mean_ot_costs = np.mean(ot_costs) 265 | return mean_ot_costs 266 | else: 267 | return ot_costs 268 | 269 | def evaluate_gt( 270 | self, 271 | bbox_noise_level=None, 272 | **kwargs, 273 | ): 274 | 275 | gts = self.get_gts() 276 | n = len(gts) 277 | for i in range(n): 278 | gt = gts[i] 279 | if gt is None: 280 | gts[i] = [np.asarray([]).reshape(0, 5) for _ in self.CLASSES] 281 | continue 282 | for bbox in gt: 283 | if len(bbox) == 0: 284 | continue 285 | 286 | w = bbox[:, 2] - bbox[:, 0] 287 | h = bbox[:, 3] - bbox[:, 1] 288 | shift_x = ( 289 | w * bbox_noise_level * np.random.choice((-1, 1), w.shape) 290 | ) 291 | shift_y = ( 292 | h * bbox_noise_level * np.random.choice((-1, 1), h.shape) 293 | ) 294 | bbox[:, 0] += shift_x 295 | bbox[:, 2] += shift_x 296 | bbox[:, 1] += shift_y 297 | bbox[:, 3] += shift_y 298 | return self.evaluate(gts, **kwargs) 299 | 300 | def _ann2detformat(self, ann_info): 301 | """convert annotation info of CocoDataset into detection output format. 302 | 303 | Parameters 304 | ---------- 305 | ann : list[dict] 306 | ground truth annotation. each item in the list correnponds to an instance. 307 | >>> ann_info[i].keys() 308 | dict_keys(['segmentation', 'area', 'iscrowd', 'image_id', 'bbox', 'category_id', 'id']) 309 | Returns 310 | ------- 311 | bboxes : list[numpy] 312 | list of bounding boxes with confidence score. 313 | bboxes[i] contains bounding boxes of instances of class i. 314 | """ 315 | if len(ann_info) == 0: 316 | return None 317 | 318 | bboxes = [[] for _ in range(len(self.cat2label))] 319 | 320 | for ann in ann_info: 321 | if ann.get("ignore", False) or ann["iscrowd"]: 322 | continue 323 | c_id = ann["category_id"] 324 | x1, y1, w, h = ann["bbox"] 325 | 326 | bboxes[self.cat2label[c_id]].append([x1, y1, x1 + w, y1 + h, 1.0]) 327 | 328 | np_bboxes = [] 329 | for x in bboxes: 330 | if len(x): 331 | np_bboxes.append(np.asarray(x, dtype=np.float32)) 332 | else: 333 | np_bboxes.append( 334 | np.asarray([], dtype=np.float32).reshape(0, 5) 335 | ) 336 | return np_bboxes 337 | --------------------------------------------------------------------------------