├── .coveragerc ├── .github └── workflows │ └── pytest.yml ├── .gitignore ├── LICENSE ├── README.md ├── deprecated ├── pascal2json.py ├── sample_2 │ ├── detections.json │ └── groundtruths.json ├── sample_3 │ ├── detections.json │ └── groundtruths.json ├── test_utils.py └── utils.py ├── pyproject.toml ├── requirements.txt ├── requirements_windows.txt ├── setup.cfg ├── src └── podm │ ├── __init__.py │ ├── box.py │ ├── coco.py │ ├── coco2lableme.py │ ├── coco_decoder.py │ ├── coco_encoder.py │ ├── metrics.py │ ├── pascal2coco.py │ └── visualize.py └── tests ├── conftest.py ├── helpers ├── __init__.py └── utils.py ├── sample ├── detections.zip ├── detections_coco.json ├── expected0_5.json ├── groundtruths.zip └── groundtruths_coco.json ├── sample_3 ├── detections.zip ├── detections_coco.json ├── expected0_5.json ├── groundtruths.zip └── groundtruths_coco.json ├── test_box.py ├── test_coco.py ├── test_coco_decoder.py ├── test_coco_encoder.py ├── test_metrics.py ├── test_pascal2coco.py ├── test_pascal_voc_metrics.py ├── test_plot.py └── test_pycocotools.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | def __repr__ 5 | def __str__ 6 | if self.debug: 7 | if settings.DEBUG 8 | raise AssertionError 9 | raise NotImplementedError 10 | if 0: 11 | if __name__ == .__main__.: 12 | 13 | [html] 14 | directory = coverage_html_report -------------------------------------------------------------------------------- /.github/workflows/pytest.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: build and test 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | test: 14 | name: Test 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: [3.7, 3.8, 3.9] 19 | env: 20 | PYTHONPATH: src 21 | 22 | steps: 23 | - name: Checkout repository code 24 | uses: actions/checkout@v2 25 | - name: Set up Python ${{ matrix.python-version }} 26 | uses: actions/setup-python@v2 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install -r requirements.txt 34 | 35 | - name: Test with pytest and generate coverage report 36 | run: | 37 | pip install pytest pytest-cov 38 | pytest --cov=./ --cov-report=xml tests 39 | 40 | - name: Upload coverage to Codecov 41 | uses: codecov/codecov-action@v2 42 | with: 43 | token: ${{ secrets.CODECOV_TOKEN }} 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage_html_report 2 | .idea 3 | output 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | .pytest_cache/ 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule.* 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # Environments 83 | .env 84 | .venv 85 | env/ 86 | venv/ 87 | ENV/ 88 | env.bak/ 89 | venv.bak/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | ### VisualStudioCode ### 105 | .vscode/ 106 | .vscode/* 107 | !.vscode/settings.json 108 | !.vscode/tasks.json 109 | !.vscode/launch.json 110 | !.vscode/extensions.json 111 | .history 112 | 113 | # My stuff 114 | ToDo.txt 115 | test.py 116 | references/ 117 | aux_images/older_version/ 118 | 3rd_party/ 119 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 BioNLP Lab at Weill Cornell Medicine 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build status](https://github.com/yfpeng/object_detection_metrics/actions/workflows/pytest.yml/badge.svg)](https://github.com/yfpeng/object_detection_metrics/) 2 | [![Latest version on PyPI](https://img.shields.io/pypi/v/object_detection_metrics.svg)](https://pypi.python.org/pypi/object_detection_metrics) 3 | [![License](https://img.shields.io/pypi/l/object_detection_metrics.svg)](https://opensource.org/licenses/MIT) 4 | [![Downloads](https://img.shields.io/pypi/dm/object_detection_metrics.svg)](https://pypi.python.org/pypi/object_detection_metrics) 5 | [![Pythong version](https://img.shields.io/pypi/pyversions/object_detection_metrics)](https://pypi.python.org/pypi/object_detection_metrics) 6 | [![codecov](https://codecov.io/gh/yfpeng/object_detection_metrics/branch/master/graph/badge.svg?token=m4mJ9fD88s)](https://codecov.io/gh/yfpeng/object_detection_metrics) 7 | [![Hits](https://hits.dwyl.com/yfpeng/object_detection_metrics.svg)](https://hits.dwyl.com/yfpeng/object_detection_metrics) 8 | 9 | 10 | This project was forked from [rafaelpadilla/Object-Detection-Metrics](https://github.com/rafaelpadilla/Object-Detection-Metrics). 11 | 12 | Development of `object_detection_metrics` happens on GitHub: https://github.com/yfpeng/object_detection_metrics 13 | 14 | The latest `object_detection_metrics releases` are available over [pypi](https://pypi.org/project/object-detection-metrics/). 15 | 16 | ## Getting started 17 | 18 | Installing `object_detection_metrics` 19 | 20 | ```shell 21 | $ pip install object_detection_metrics 22 | ``` 23 | 24 | Reading COCO file 25 | 26 | ```python 27 | from podm import coco_decoder 28 | with open('tests/sample/groundtruths_coco.json') as fp: 29 | gold_dataset = coco_decoder.load_true_object_detection_dataset(fp) 30 | ``` 31 | 32 | PASCAL VOC Metrics 33 | 34 | ```python 35 | from podm import coco_decoder 36 | from podm.metrics import get_pascal_voc_metrics, MetricPerClass, get_bounding_boxes 37 | 38 | with open('tests/sample/groundtruths_coco.json') as fp: 39 | gold_dataset = coco_decoder.load_true_object_detection_dataset(fp) 40 | with open('tests/sample/detections_coco.json') as fp: 41 | pred_dataset = coco_decoder.load_pred_object_detection_dataset(fp, gold_dataset) 42 | 43 | gt_BoundingBoxes = get_bounding_boxes(gold_dataset) 44 | pd_BoundingBoxes = get_bounding_boxes(pred_dataset) 45 | results = get_pascal_voc_metrics(gt_BoundingBoxes, pd_BoundingBoxes, .5) 46 | ``` 47 | 48 | ap, precision, recall, tp, fp, etc 49 | 50 | ```python 51 | for cls, metric in results.items(): 52 | label = metric.label 53 | print('ap', metric.ap) 54 | print('precision', metric.precision) 55 | print('interpolated_recall', metric.interpolated_recall) 56 | print('interpolated_precision', metric.interpolated_precision) 57 | print('tp', metric.tp) 58 | print('fp', metric.fp) 59 | print('num_groundtruth', metric.num_groundtruth) 60 | print('num_detection', metric.num_detection) 61 | ``` 62 | 63 | mAP 64 | 65 | ```python 66 | from podm.metrics import MetricPerClass 67 | mAP = MetricPerClass.mAP(results) 68 | ``` 69 | 70 | IoU 71 | 72 | ```python 73 | from podm.box import Box, intersection_over_union 74 | 75 | box1 = Box.of_box(0., 0., 10., 10.) 76 | box2 = Box.of_box(1., 1., 11., 11.) 77 | intersection_over_union(box1, box2) 78 | ``` 79 | 80 | Official COCO Eval 81 | 82 | ```python 83 | from pycocotools.coco import COCO 84 | from pycocotools.cocoeval import COCOeval 85 | 86 | coco_gld = COCO('tests/sample/groundtruths_coco.json') 87 | coco_rst = coco_gld.loadRes('tests/sample/detections_coco.json') 88 | cocoEval = COCOeval(coco_gld, coco_rst, iouType='bbox') 89 | cocoEval.evaluate() 90 | cocoEval.accumulate() 91 | cocoEval.summarize() 92 | ``` 93 | 94 | ## Implemented metrics 95 | 96 | [Tutorial](https://medium.com/@jonathan_hui/map-mean-average-precision-for-object-detection-45c121a31173) 97 | 98 | - Intersection Over Union (IOU) 99 | - TP and FP 100 | - True Positive (TP): IOU ≥ *IOU threshold* (default: 0.5) 101 | - False Positive (FP): IOU \< *IOU threshold* (default: 0.5) 102 | - Precision and Recall 103 | - Average Precision 104 | - 11-point AP 105 | - all-point AP 106 | - Official COCO Eval 107 | 108 | ## License 109 | 110 | Copyright BioNLP Lab at Weill Cornell Medicine, 2022. 111 | 112 | Distributed under the terms of the [MIT](https://github.com/yfpeng/object_detection_metrics/blob/master/LICENSE) 113 | license, this is free and open source software. 114 | -------------------------------------------------------------------------------- /deprecated/pascal2json.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert from a PASCAL VOC zip file to one json file. 3 | 4 | Usage: 5 | pascal2json --input= --output= 6 | 7 | Options: 8 | --input= PASCAL VOC zip file 9 | --output= JSON file 10 | """ 11 | import io 12 | import zipfile 13 | from enum import Enum 14 | from pathlib import Path 15 | import os 16 | import json 17 | from typing import Iterator 18 | 19 | import docopt 20 | import tqdm 21 | 22 | 23 | class BBFormat(Enum): 24 | """ 25 | Class representing the format of a bounding box. 26 | It can be (X,Y,width,height) => XYWH 27 | or (X1,Y1,X2,Y2) => XYX2Y2 28 | 29 | Developed by: Rafael Padilla 30 | Last modification: May 24 2018 31 | """ 32 | XYWH = 1 33 | X1Y1X2Y2 = 2 34 | 35 | 36 | def parse_single_file(items_file: Iterator[str], format: BBFormat): 37 | boxes = [] 38 | for i, line in enumerate(items_file): 39 | tok = line.strip().split(' ') 40 | if len(tok) == 6: 41 | b = { 42 | 'xtl': int(tok[2]), 43 | 'ytl': int(tok[3]), 44 | 'xbr': int(tok[4]), 45 | 'ybr': int(tok[5]), 46 | 'label': tok[0], 47 | 'score': float(tok[1]) 48 | } 49 | elif len(tok) == 5: 50 | b = { 51 | 'xtl': int(tok[1]), 52 | 'ytl': int(tok[2]), 53 | 'xbr': int(tok[3]), 54 | 'ybr': int(tok[4]), 55 | 'label': tok[0], 56 | } 57 | else: 58 | raise ValueError('%s: ill-formatted. Should have 5 or 6 field. %s' % i, line) 59 | if format == BBFormat.XYWH: 60 | b['xbr'] += b['xtl'] 61 | b['ybr'] += b['ytl'] 62 | boxes.append(b) 63 | return boxes 64 | 65 | 66 | def convert_pascal_voc_to_json_zip(src, dest, format=BBFormat.X1Y1X2Y2): 67 | data = [] 68 | with zipfile.ZipFile(src, 'r') as myzip: 69 | namelist = myzip.namelist() 70 | for name in tqdm.tqdm(namelist): 71 | if not name.endswith('.txt'): 72 | continue 73 | x = { 74 | 'name': name, 75 | 'boxes': [] 76 | } 77 | with myzip.open(name, 'r') as fp: 78 | items_file = io.TextIOWrapper(fp) 79 | boxes = parse_single_file(items_file, format) 80 | x['boxes'].extend(boxes) 81 | data.append(x) 82 | 83 | with open(dest, 'w') as fp: 84 | json.dump(data, fp, indent=2) 85 | 86 | 87 | def convert_pascal_voc_to_json(dirname, dest, format: BBFormat = BBFormat.X1Y1X2Y2): 88 | data = [] 89 | for entry in os.scandir(dirname): 90 | x = { 91 | 'name': Path(entry.path).stem, 92 | 'boxes': [] 93 | } 94 | with open(entry.path) as fp: 95 | boxes = parse_single_file(fp, format) 96 | x['boxes'].extend(boxes) 97 | data.append(x) 98 | 99 | with open(dest, 'w') as fp: 100 | json.dump(data, fp, indent=2) 101 | 102 | 103 | def main(): 104 | argv = docopt.docopt(__doc__) 105 | convert_pascal_voc_to_json_zip(argv['--input'], argv['--output'], BBFormat.X1Y1X2Y2) 106 | 107 | 108 | if __name__ == '__main__': 109 | main() 110 | -------------------------------------------------------------------------------- /deprecated/sample_3/detections.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "00001", 4 | "boxes": [ 5 | { 6 | "xtl": 5, 7 | "ytl": 67, 8 | "xbr": 36, 9 | "ybr": 115, 10 | "label": "person", 11 | "score": 0.88 12 | }, 13 | { 14 | "xtl": 119, 15 | "ytl": 111, 16 | "xbr": 159, 17 | "ybr": 178, 18 | "label": "person", 19 | "score": 0.7 20 | }, 21 | { 22 | "xtl": 124, 23 | "ytl": 9, 24 | "xbr": 173, 25 | "ybr": 76, 26 | "label": "person", 27 | "score": 0.8 28 | } 29 | ] 30 | }, 31 | { 32 | "name": "00002", 33 | "boxes": [ 34 | { 35 | "xtl": 64, 36 | "ytl": 111, 37 | "xbr": 128, 38 | "ybr": 169, 39 | "label": "person", 40 | "score": 0.71 41 | }, 42 | { 43 | "xtl": 26, 44 | "ytl": 140, 45 | "xbr": 86, 46 | "ybr": 187, 47 | "label": "person", 48 | "score": 0.54 49 | }, 50 | { 51 | "xtl": 19, 52 | "ytl": 18, 53 | "xbr": 62, 54 | "ybr": 53, 55 | "label": "person", 56 | "score": 0.74 57 | } 58 | ] 59 | }, 60 | { 61 | "name": "00003", 62 | "boxes": [ 63 | { 64 | "xtl": 109, 65 | "ytl": 15, 66 | "xbr": 186, 67 | "ybr": 54, 68 | "label": "person", 69 | "score": 0.18 70 | }, 71 | { 72 | "xtl": 86, 73 | "ytl": 63, 74 | "xbr": 132, 75 | "ybr": 108, 76 | "label": "person", 77 | "score": 0.67 78 | }, 79 | { 80 | "xtl": 160, 81 | "ytl": 62, 82 | "xbr": 196, 83 | "ybr": 115, 84 | "label": "person", 85 | "score": 0.38 86 | }, 87 | { 88 | "xtl": 105, 89 | "ytl": 131, 90 | "xbr": 152, 91 | "ybr": 178, 92 | "label": "person", 93 | "score": 0.91 94 | }, 95 | { 96 | "xtl": 18, 97 | "ytl": 148, 98 | "xbr": 58, 99 | "ybr": 192, 100 | "label": "person", 101 | "score": 0.44 102 | } 103 | ] 104 | }, 105 | { 106 | "name": "00004", 107 | "boxes": [ 108 | { 109 | "xtl": 83, 110 | "ytl": 28, 111 | "xbr": 111, 112 | "ybr": 54, 113 | "label": "person", 114 | "score": 0.35 115 | }, 116 | { 117 | "xtl": 28, 118 | "ytl": 68, 119 | "xbr": 70, 120 | "ybr": 135, 121 | "label": "person", 122 | "score": 0.78 123 | }, 124 | { 125 | "xtl": 87, 126 | "ytl": 89, 127 | "xbr": 112, 128 | "ybr": 128, 129 | "label": "person", 130 | "score": 0.45 131 | }, 132 | { 133 | "xtl": 10, 134 | "ytl": 155, 135 | "xbr": 70, 136 | "ybr": 181, 137 | "label": "person", 138 | "score": 0.14 139 | } 140 | ] 141 | }, 142 | { 143 | "name": "00005", 144 | "boxes": [ 145 | { 146 | "xtl": 50, 147 | "ytl": 38, 148 | "xbr": 78, 149 | "ybr": 84, 150 | "label": "person", 151 | "score": 0.62 152 | }, 153 | { 154 | "xtl": 95, 155 | "ytl": 11, 156 | "xbr": 148, 157 | "ybr": 39, 158 | "label": "person", 159 | "score": 0.44 160 | }, 161 | { 162 | "xtl": 29, 163 | "ytl": 131, 164 | "xbr": 101, 165 | "ybr": 160, 166 | "label": "person", 167 | "score": 0.95 168 | }, 169 | { 170 | "xtl": 29, 171 | "ytl": 163, 172 | "xbr": 101, 173 | "ybr": 192, 174 | "label": "person", 175 | "score": 0.23 176 | } 177 | ] 178 | }, 179 | { 180 | "name": "00006", 181 | "boxes": [ 182 | { 183 | "xtl": 43, 184 | "ytl": 48, 185 | "xbr": 117, 186 | "ybr": 86, 187 | "label": "person", 188 | "score": 0.45 189 | }, 190 | { 191 | "xtl": 17, 192 | "ytl": 155, 193 | "xbr": 46, 194 | "ybr": 190, 195 | "label": "person", 196 | "score": 0.84 197 | }, 198 | { 199 | "xtl": 95, 200 | "ytl": 110, 201 | "xbr": 120, 202 | "ybr": 152, 203 | "label": "person", 204 | "score": 0.43 205 | } 206 | ] 207 | }, 208 | { 209 | "name": "00007", 210 | "boxes": [ 211 | { 212 | "xtl": 16, 213 | "ytl": 20, 214 | "xbr": 117, 215 | "ybr": 108, 216 | "label": "person", 217 | "score": 0.48 218 | }, 219 | { 220 | "xtl": 33, 221 | "ytl": 116, 222 | "xbr": 70, 223 | "ybr": 165, 224 | "label": "person", 225 | "score": 0.95 226 | } 227 | ] 228 | } 229 | ] -------------------------------------------------------------------------------- /deprecated/sample_3/groundtruths.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "00001", 4 | "boxes": [ 5 | { 6 | "xtl": 25, 7 | "ytl": 16, 8 | "xbr": 63, 9 | "ybr": 72, 10 | "label": "person" 11 | }, 12 | { 13 | "xtl": 129, 14 | "ytl": 123, 15 | "xbr": 170, 16 | "ybr": 185, 17 | "label": "person" 18 | } 19 | ] 20 | }, 21 | { 22 | "name": "00002", 23 | "boxes": [ 24 | { 25 | "xtl": 123, 26 | "ytl": 11, 27 | "xbr": 166, 28 | "ybr": 66, 29 | "label": "person" 30 | }, 31 | { 32 | "xtl": 38, 33 | "ytl": 132, 34 | "xbr": 97, 35 | "ybr": 177, 36 | "label": "person" 37 | } 38 | ] 39 | }, 40 | { 41 | "name": "00003", 42 | "boxes": [ 43 | { 44 | "xtl": 16, 45 | "ytl": 14, 46 | "xbr": 51, 47 | "ybr": 62, 48 | "label": "person" 49 | }, 50 | { 51 | "xtl": 123, 52 | "ytl": 30, 53 | "xbr": 172, 54 | "ybr": 74, 55 | "label": "person" 56 | }, 57 | { 58 | "xtl": 99, 59 | "ytl": 139, 60 | "xbr": 146, 61 | "ybr": 186, 62 | "label": "person" 63 | } 64 | ] 65 | }, 66 | { 67 | "name": "00004", 68 | "boxes": [ 69 | { 70 | "xtl": 53, 71 | "ytl": 42, 72 | "xbr": 93, 73 | "ybr": 94, 74 | "label": "person" 75 | }, 76 | { 77 | "xtl": 154, 78 | "ytl": 43, 79 | "xbr": 185, 80 | "ybr": 77, 81 | "label": "person" 82 | } 83 | ] 84 | }, 85 | { 86 | "name": "00005", 87 | "boxes": [ 88 | { 89 | "xtl": 59, 90 | "ytl": 31, 91 | "xbr": 103, 92 | "ybr": 82, 93 | "label": "person" 94 | }, 95 | { 96 | "xtl": 48, 97 | "ytl": 128, 98 | "xbr": 82, 99 | "ybr": 180, 100 | "label": "person" 101 | } 102 | ] 103 | }, 104 | { 105 | "name": "00006", 106 | "boxes": [ 107 | { 108 | "xtl": 36, 109 | "ytl": 89, 110 | "xbr": 88, 111 | "ybr": 165, 112 | "label": "person" 113 | }, 114 | { 115 | "xtl": 62, 116 | "ytl": 58, 117 | "xbr": 106, 118 | "ybr": 125, 119 | "label": "person" 120 | } 121 | ] 122 | }, 123 | { 124 | "name": "00007", 125 | "boxes": [ 126 | { 127 | "xtl": 28, 128 | "ytl": 31, 129 | "xbr": 83, 130 | "ybr": 94, 131 | "label": "person" 132 | }, 133 | { 134 | "xtl": 58, 135 | "ytl": 67, 136 | "xbr": 108, 137 | "ybr": 125, 138 | "label": "person" 139 | } 140 | ] 141 | } 142 | ] -------------------------------------------------------------------------------- /deprecated/test_utils.py: -------------------------------------------------------------------------------- 1 | from deprecated.utils import load_data, load_data_coco 2 | import math 3 | 4 | 5 | def test_load_data(tests_dir): 6 | dir = tests_dir / 'sample_2' 7 | gt_bb = load_data(dir / 'groundtruths.json') 8 | pd_bb = load_data(dir / 'detections.json') 9 | 10 | assert len(gt_bb) == 686 11 | assert gt_bb[-1].image == '2007_001416' 12 | assert gt_bb[-1].category == 'cup' 13 | assert gt_bb[-1].score == 1 14 | 15 | assert len(pd_bb) == 494 16 | assert pd_bb[-1].image == '2007_001416' 17 | assert pd_bb[-1].category == 'chair' 18 | assert pd_bb[-1].score == 0.644847 19 | assert math.isclose(pd_bb[-1].score, 0.644847, rel_tol=1e-3) 20 | 21 | 22 | def test_load_data_coco(tests_dir): 23 | dir = tests_dir / 'sample_2' 24 | gt_bb, pd_bb = load_data_coco(dir / 'groundtruths_coco.json', 25 | dir / 'detections_coco.json') 26 | 27 | assert len(gt_bb) == 686 28 | assert gt_bb[-1].image == '2007_001416.txt.jpg' 29 | assert gt_bb[-1].category == 'cup' 30 | assert gt_bb[-1].score is None 31 | 32 | assert len(pd_bb) == 494 33 | assert pd_bb[-1].image == '2007_001416.txt.jpg' 34 | assert pd_bb[-1].category == 'chair' 35 | assert pd_bb[-1].score == 0.644847 36 | assert math.isclose(pd_bb[-1].score, 0.644847, rel_tol=1e-3) 37 | -------------------------------------------------------------------------------- /deprecated/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List, Tuple 3 | 4 | from podm.box import BoundingBox 5 | 6 | 7 | def load_data(pathname) -> List[BoundingBox]: 8 | with open(pathname, 'r') as fp: 9 | objs = json.load(fp) 10 | 11 | boxes = [] 12 | for fig in objs: 13 | for box in fig['boxes']: 14 | if 'score' in box: 15 | score = box['score'] 16 | else: 17 | score = 1 18 | bb = BoundingBox( 19 | fig['name'], 20 | box['label'], 21 | box['xtl'], 22 | box['ytl'], 23 | box['xbr'], 24 | box['ybr'], 25 | score) 26 | boxes.append(bb) 27 | return boxes 28 | 29 | 30 | def load_data_coco(gd_pathname, rt_pathname=None) -> Tuple[List[BoundingBox], List[BoundingBox]]: 31 | with open(gd_pathname) as fp: 32 | objs = json.load(fp) 33 | 34 | # categroies 35 | label_map = {c['id']: c['name'] for c in objs['categories']} 36 | # images 37 | image_map = {c['id']: c['file_name'] for c in objs['images']} 38 | 39 | gt_boxes = [] 40 | for box in objs['annotations']: 41 | bb = BoundingBox( 42 | image_map[box['image_id']], 43 | label_map[box['category_id']], 44 | box['bbox'][0], 45 | box['bbox'][1], 46 | box['bbox'][0] + box['bbox'][2], 47 | box['bbox'][1] + box['bbox'][3]) 48 | gt_boxes.append(bb) 49 | 50 | rt_boxes = [] 51 | if rt_pathname is not None: 52 | with open(rt_pathname) as fp: 53 | objs = json.load(fp) 54 | 55 | for box in objs: 56 | bb = BoundingBox( 57 | image_map[box['image_id']], 58 | label_map[box['category_id']], 59 | box['bbox'][0], 60 | box['bbox'][1], 61 | box['bbox'][0] + box['bbox'][2], 62 | box['bbox'][1] + box['bbox'][3], 63 | box['score'] 64 | ) 65 | rt_boxes.append(bb) 66 | 67 | return gt_boxes, rt_boxes -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | numpy 3 | pycocotools 4 | scikit-image 5 | tqdm 6 | Pillow 7 | pandas 8 | docopt 9 | shapely -------------------------------------------------------------------------------- /requirements_windows.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | numpy 3 | pycocotools-windows 4 | scikit-image 5 | Pillow 6 | tqdm 7 | pandas 8 | docopt -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = object-detection-metrics 3 | # Versions should comply with PEP440. For a discussion on single-sourcing 4 | # the version across setup.py and the project code, see 5 | # https://packaging.python.org/en/latest/guides/single-sourcing-package-version/ 6 | version = 0.4.post1 7 | # Author details 8 | author = Yifan Peng 9 | author_email = yip4002@med.cornell.edu 10 | description = Object Detection Metrics 11 | long_description = file: README.md 12 | long_description_content_type = text/markdown 13 | # The project's main homepage. 14 | url = https://github.com/yfpeng/object_detection_metrics 15 | license = MIT License 16 | keywords = object detection metrics 17 | # See https://pypi.org/classifiers/ 18 | classifiers = 19 | Development Status :: 2 - Pre-Alpha 20 | Intended Audience :: Developers 21 | Intended Audience :: Science/Research 22 | License :: OSI Approved :: MIT License 23 | Operating System :: POSIX :: Linux 24 | Operating System :: MacOS 25 | Topic :: Text Processing 26 | Topic :: Software Development 27 | Topic :: Scientific/Engineering :: Artificial Intelligence 28 | Topic :: Scientific/Engineering :: Information Analysis 29 | Programming Language :: Python :: 3.7 30 | Programming Language :: Python :: 3.8 31 | Programming Language :: Python :: 3.9 32 | 33 | [options] 34 | python_requires = >=3.7 35 | package_dir= 36 | =src 37 | packages = find: 38 | install_requires = 39 | matplotlib 40 | numpy 41 | pycocotools 42 | scikit-image 43 | tqdm 44 | Pillow 45 | pandas 46 | docopt 47 | 48 | [options.packages.find] 49 | where=src 50 | exclude = 51 | tests.* 52 | tests 53 | deprecated/* 54 | 55 | [options.package_data] 56 | * = 57 | resources/* 58 | 59 | [options.extras_require] 60 | rest = docutils>=0.15.2 61 | 62 | [tool:pytest] 63 | norecursedirs=tests/helpers 64 | 65 | [options.entry_points] 66 | console_scripts = 67 | pascal2coco = podm.pascal2coco:main 68 | -------------------------------------------------------------------------------- /src/podm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfpeng/object_detection_metrics/3801acc32d052d1fbf1565f96a9f8d390701f911/src/podm/__init__.py -------------------------------------------------------------------------------- /src/podm/box.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Tuple 3 | 4 | 5 | class Box: 6 | """ 7 | 0,0 ------> x (width) 8 | | 9 | | (Left,Top) 10 | | *_________ 11 | | | | 12 | | | 13 | y |_________| 14 | (height) * 15 | (Right,Bottom) 16 | 17 | xtl: the X top-left coordinate of the bounding box. 18 | ytl: the Y top-left coordinate of the bounding box. 19 | xbr: the X bottom-right coordinate of the bounding box. 20 | ybr: the Y bottom-right coordinate of the bounding box. 21 | """ 22 | def __init__(self): 23 | self.xtl = None # type: float or None 24 | self.ytl = None # type: float or None 25 | self.xbr = None # type: float or None 26 | self.ybr = None # type: float or None 27 | 28 | @classmethod 29 | def of_box(cls, xtl: float, ytl: float, xbr: float, ybr: float) -> 'Box': 30 | """ 31 | :param xtl: the X top-left coordinate of the bounding box. 32 | :param ytl: the Y top-left coordinate of the bounding box. 33 | :param xbr: the X bottom-right coordinate of the bounding box. 34 | :param ybr: the Y bottom-right coordinate of the bounding box. 35 | """ 36 | box = Box() 37 | box.xtl = xtl 38 | box.ytl = ytl 39 | box.xbr = xbr 40 | box.ybr = ybr 41 | box.verify() 42 | return box 43 | 44 | def set_box(self, box: 'Box'): 45 | self.xtl = box.xtl 46 | self.ytl = box.ytl 47 | self.xbr = box.xbr 48 | self.ybr = box.ybr 49 | 50 | def verify(self): 51 | assert self.xtl <= self.xbr, f'xtl < xbr: xtl:{self.xtl}, xbr:{self.xbr}' 52 | assert self.ytl <= self.ybr, f'ytl < ybr: ytl:{self.ytl}, xbr:{self.ybr}' 53 | 54 | @property 55 | def segment(self): 56 | return [self.xtl, self.ytl, self.xtl, self.ybr, self.xbr, self.ybr, self.xbr, self.ytl] 57 | 58 | @property 59 | def width(self) -> float: 60 | return self.xbr - self.xtl 61 | 62 | @property 63 | def height(self) -> float: 64 | return self.ybr - self.ytl 65 | 66 | @property 67 | def area(self) -> float: 68 | return (self.xbr - self.xtl) * (self.ybr - self.ytl) 69 | 70 | @property 71 | def center(self) -> Tuple[float, float]: 72 | return (self.xbr + self.xtl) / 2, (self.ybr + self.ytl) / 2 73 | 74 | def __contains__(self, item): 75 | if not type(item) == list and not type(item) == tuple: 76 | raise TypeError('Has to be a list or a tuple: %s' % type(item)) 77 | if len(item) == 2: 78 | return self.xtl <= item[0] < self.xbr and self.ytl <= item[1] < self.ybr 79 | else: 80 | raise ValueError('Only support a point') 81 | 82 | def __str__(self): 83 | return 'Box[xtl={},ytl={},xbr={},ybr={}]'.format(self.xtl, self.ytl, self.xbr, self.ybr) 84 | 85 | def __eq__(self, other): 86 | if not isinstance(other, Box): 87 | return False 88 | return self.xtl == other.xtl and self.ytl == other.ytl and self.xbr == other.xbr and self.ybr == other.ybr 89 | 90 | 91 | def intersection_over_union(box1: 'Box', box2: 'Box') -> float: 92 | """ 93 | Intersection Over Union (IOU) is measure based on Jaccard Index that evaluates the overlap between 94 | two bounding boxes. 95 | """ 96 | # if boxes dont intersect 97 | if not is_intersecting(box1, box2): 98 | return 0 99 | intersection_area = intersection(box1, box2).area 100 | union = union_areas(box1, box2, intersection_area=intersection_area) 101 | # intersection over union 102 | iou = intersection_area / union 103 | assert iou >= 0, '{} = {} / {}, box1={}, box2={}'.format(iou, intersection, union, box1, box2) 104 | return iou 105 | 106 | 107 | def is_intersecting(box1: 'Box', box2: 'Box') -> bool: 108 | if box1.xtl > box2.xbr: 109 | return False # boxA is right of boxB 110 | if box2.xtl > box1.xbr: 111 | return False # boxA is left of boxB 112 | if box1.ybr < box2.ytl: 113 | return False # boxA is above boxB 114 | if box1.ytl > box2.ybr: 115 | return False # boxA is below boxB 116 | return True 117 | 118 | 119 | def union_areas(box1: 'Box', box2: 'Box', intersection_area: float = None) -> float: 120 | if intersection_area is None: 121 | intersection_area = intersection(box1, box2).area 122 | return box1.area + box2.area - intersection_area 123 | 124 | 125 | def union(box1: 'Box', box2: 'Box'): 126 | xtl = min(box1.xtl, box2.xtl) 127 | ytl = min(box1.ytl, box2.ytl) 128 | xbr = max(box1.xbr, box2.xbr) 129 | ybr = max(box1.ybr, box2.ybr) 130 | return Box.of_box(xtl, ytl, xbr, ybr) 131 | 132 | 133 | def intersection(box1: 'Box', box2: 'Box'): 134 | xtl = max(box1.xtl, box2.xtl) 135 | ytl = max(box1.ytl, box2.ytl) 136 | xbr = min(box1.xbr, box2.xbr) 137 | ybr = min(box1.ybr, box2.ybr) 138 | return Box.of_box(xtl, ytl, xbr, ybr) 139 | 140 | 141 | class BBFormat(Enum): 142 | """ 143 | Class representing the format of a bounding box. 144 | It can be (X,Y,width,height) => XYWH 145 | or (X1,Y1,X2,Y2) => XYX2Y2 146 | 147 | Developed by: Rafael Padilla 148 | Last modification: May 24 2018 149 | """ 150 | XYWH = 1 151 | X1Y1X2Y2 = 2 152 | 153 | 154 | # class BoundingBox(Box): 155 | # def __init__(self): 156 | # """Constructor. 157 | # Args: 158 | # image: image. 159 | # category: category. 160 | # xtl: the X top-left coordinate of the bounding box. 161 | # ytl: the Y top-left coordinate of the bounding box. 162 | # xbr: the X bottom-right coordinate of the bounding box. 163 | # ybr: the Y bottom-right coordinate of the bounding box. 164 | # score: (optional) the confidence of the detected class. 165 | # """ 166 | # super(BoundingBox, self).__init__() 167 | # self.image = None 168 | # self.category = None 169 | # self.score = None # type: float or None 170 | # 171 | # @classmethod 172 | # def of_bbox(cls, image, category, xtl: float, ytl: float, xbr: float, ybr: float, score: float = None) \ 173 | # -> 'BoundingBox': 174 | # bbox = BoundingBox() 175 | # bbox.xtl = xtl 176 | # bbox.ytl = ytl 177 | # bbox.xbr = xbr 178 | # bbox.ybr = ybr 179 | # bbox.image = image 180 | # bbox.score = score 181 | # bbox.category = category 182 | # return bbox -------------------------------------------------------------------------------- /src/podm/coco.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from abc import ABC 3 | from typing import List, Tuple, Set, Collection 4 | from datetime import date, datetime 5 | from shapely.geometry import Polygon, Point 6 | from podm import box 7 | 8 | 9 | class PCOCOInfo: 10 | def __init__(self): 11 | self.year = date.today().year # type:int 12 | self.version = '' # type: str 13 | self.description = '' # type: str 14 | self.contributor = '' # type: str 15 | self.url = '' # type: str 16 | self.date_created = datetime.now().strftime('%m/%d/%Y') # type:str 17 | 18 | 19 | class PCOCOAnnotation(ABC): 20 | def __init__(self): 21 | self.id = None # type:int or None 22 | self.image_id = None # type:int or None 23 | self.score = None # type:float or None 24 | self.contributor = '' # type: str 25 | self.attributes = {} # type: dict 26 | 27 | 28 | class PCOCOImage: 29 | def __init__(self): 30 | self.id = None # type:int or None 31 | self.width = 0 # type:int 32 | self.height = 0 # type:int 33 | self.file_name = '' # type:str 34 | self.license = None # type:int or None 35 | self.flickr_url = '' # type:str 36 | self.coco_url = '' # type:str 37 | self.date_captured = datetime.now().strftime('%m/%d/%Y') # type:str 38 | 39 | 40 | class PCOCOLicense: 41 | def __init__(self): 42 | self.id = None # type:int or None 43 | self.name = '' # type:str 44 | self.url = '' # type:str 45 | 46 | 47 | class PCOCOCategory: 48 | def __init__(self): 49 | self.id = None # type:int or None 50 | self.name = '' # type:str 51 | self.supercategory = '' # type:str 52 | 53 | 54 | class PCOCODataset(ABC): 55 | def __init__(self): 56 | self.info = PCOCOInfo() # type: PCOCOInfo or None 57 | self.images = [] # type: List[PCOCOImage] 58 | self.licenses = [] # type: List[PCOCOLicense] 59 | 60 | def add_license(self, license: PCOCOLicense): 61 | for lic in self.licenses: 62 | if lic.id == license.id or lic.name == license.name: 63 | raise KeyError('%s: License exists' % lic.id) 64 | self.licenses.append(license) 65 | 66 | def add_image(self, image: PCOCOImage): 67 | for img in self.images: 68 | if img.id == image.id or img.file_name == image.file_name: 69 | raise KeyError('%s: Image exists' % img.id) 70 | self.images.append(image) 71 | 72 | def get_image(self, id: int = None, file_name: str = None, default=None) -> PCOCOImage: 73 | if id is None and file_name is None: 74 | raise KeyError('%s %s: Cannot set both to None' % (id, file_name)) 75 | if id is not None and file_name is not None: 76 | raise KeyError('%s %s: Cannot set both' % (id, file_name)) 77 | 78 | imgs = self.images 79 | if id is not None: 80 | imgs = [img for img in imgs if img.id == id] 81 | if len(imgs) == 0: 82 | return default 83 | elif len(imgs) == 1: 84 | return next(iter(imgs)) 85 | else: 86 | raise KeyError('%s: more than one image with the same id' % id) 87 | 88 | if file_name is not None: 89 | imgs = [img for img in imgs if img.file_name == file_name] 90 | if len(imgs) == 0: 91 | return default 92 | elif len(imgs) == 1: 93 | return next(iter(imgs)) 94 | else: 95 | raise KeyError('%s: more than one image with the same name' % file_name) 96 | 97 | raise Exception('Should not be here') 98 | 99 | def get_images(self, ids: Collection[int] = None) -> List[PCOCOImage]: 100 | """ 101 | Load anns with the specified ids. 102 | :param ids: integer ids specifying img 103 | :return: imgs: loaded img objects 104 | """ 105 | return [img for img in self.images if img.id in ids] 106 | 107 | 108 | ############################################################################## 109 | # object detection 110 | ############################################################################## 111 | 112 | 113 | class PCOCOBoundingBox(PCOCOAnnotation, box.Box): 114 | def __init__(self): 115 | super(PCOCOBoundingBox, self).__init__() 116 | self.category_id = None # type:int or None 117 | 118 | 119 | class PCOCOSegments(PCOCOAnnotation): 120 | def __init__(self): 121 | super(PCOCOSegments, self).__init__() 122 | self.category_id = None # type:int or None 123 | self.segmentation = [] # type: List[List[float]] 124 | self.iscrowd = False # type:bool 125 | 126 | def add_box(self, box: box.Box): 127 | self.add_segmentation(box.segment) 128 | 129 | def add_segmentation(self, segmentation: List[float]): 130 | self.segmentation.append(segmentation) 131 | 132 | def __contains__(self, item): 133 | if not type(item) == list and not type(item) == tuple: 134 | raise TypeError('Has to be a list or a tuple: %s' % type(item)) 135 | if len(item) == 2: 136 | point = Point(item[0], item[1]) 137 | for p in self.polygons: 138 | if p.contains(point): 139 | return True 140 | return False 141 | else: 142 | raise ValueError('Only support a point') 143 | 144 | @property 145 | def polygons(self) -> List[Polygon]: 146 | return [Polygon([(seg[i], seg[i+1]) for i in range(0, len(seg), 2)]) for seg in self.segmentation] 147 | 148 | @property 149 | def bbox(self) -> 'box.Box' or None: 150 | if len(self.segmentation) == 0: 151 | return None 152 | else: 153 | b = self.box_polygon(self.segmentation[0]) 154 | for polygon in self.segmentation[1:]: 155 | b = box.union(b, self.box_polygon(polygon)) 156 | return b 157 | 158 | @classmethod 159 | def box_polygon(cls, polygon: List[float]) -> 'box.Box': 160 | xtl = min(polygon[i] for i in range(0, len(polygon), 2)) 161 | ytl = min(polygon[i] for i in range(1, len(polygon), 2)) 162 | xbr = max(polygon[i] for i in range(0, len(polygon), 2)) 163 | ybr = max(polygon[i] for i in range(1, len(polygon), 2)) 164 | return box.Box.of_box(xtl, ytl, xbr, ybr) 165 | 166 | 167 | class PCOCOImageCaptioning(PCOCOAnnotation): 168 | def __init__(self): 169 | super(PCOCOImageCaptioning, self).__init__() 170 | self.caption = None # type:str or None 171 | 172 | 173 | class PCOCOObjectDetectionDataset(PCOCODataset): 174 | def __init__(self): 175 | super(PCOCOObjectDetectionDataset, self).__init__() 176 | self.annotations = [] # type: List[PCOCOBoundingBox or PCOCOSegments] 177 | self.categories = [] # type: List[PCOCOCategory] 178 | 179 | def add_annotation(self, annotation: 'PCOCOBoundingBox' or 'PCOCOSegments'): 180 | for ann in self.annotations: 181 | if ann.id == annotation.id: 182 | raise KeyError('%s: Annotation exists' % ann.id) 183 | self.annotations.append(annotation) 184 | 185 | def add_category(self, category: PCOCOCategory): 186 | for cat in self.categories: 187 | if cat.id == category.id or cat.name == category.name: 188 | raise KeyError('%s: Category exists' % cat.id) 189 | self.categories.append(category) 190 | 191 | def get_max_category_id(self): 192 | return max(cat.id for cat in self.categories) 193 | 194 | def get_category(self, id: int = None, name: str = None, default=None) -> PCOCOCategory: 195 | if id is None and name is None: 196 | raise KeyError('%s %s: Cannot set both to None' % (id, name)) 197 | if id is not None and name is not None: 198 | raise KeyError('%s %s: Cannot set both' % (id, name)) 199 | 200 | cats = self.categories 201 | if id is not None: 202 | cats = [cat for cat in cats if cat.id == id] 203 | if len(cats) == 0: 204 | return default 205 | elif len(cats) == 1: 206 | return next(iter(cats)) 207 | else: 208 | raise KeyError('%s: more than one category with the same id' % id) 209 | 210 | if name is not None: 211 | cats = [cat for cat in cats if cat.name == name] 212 | if len(cats) == 0: 213 | return default 214 | elif len(cats) == 1: 215 | return next(iter(cats)) 216 | else: 217 | raise KeyError('%s: more than one category with the same name' % name) 218 | 219 | raise Exception('Should not be here') 220 | 221 | def get_annotation(self, id: int, default=None) -> PCOCOAnnotation: 222 | anns = [ann for ann in self.annotations if ann.id == id] 223 | if len(anns) == 0: 224 | return default 225 | elif len(anns) == 1: 226 | return next(iter(anns)) 227 | else: 228 | raise KeyError('%s: more than one annotation' % id) 229 | 230 | def get_new_dataset(self, annotations: Collection[PCOCOBoundingBox or PCOCOSegments]): 231 | new_dataset = PCOCOObjectDetectionDataset() 232 | new_dataset.info = copy.deepcopy(self.info) 233 | new_dataset.licenses = copy.deepcopy(self.licenses) 234 | new_dataset.images = copy.deepcopy(self.images) 235 | new_dataset.categories = copy.deepcopy(self.categories) 236 | for ann in annotations: 237 | new_dataset.add_annotation(ann) 238 | return new_dataset 239 | 240 | def get_category_ids(self, category_names: Collection[str] = None, 241 | supercategory_names: Collection[str] = None) -> Collection[int]: 242 | """ 243 | filtering parameters. default skips that filter. 244 | :param category_names: get cats for given cat names 245 | :param supercategory_names: get cats for given supercategory names 246 | :return: integer array of cat ids 247 | """ 248 | itr = iter(self.categories) 249 | if category_names is not None: 250 | itr = filter(lambda x: x.name in category_names, itr) 251 | if supercategory_names is not None: 252 | itr = filter(lambda x: x.supercategory in supercategory_names, itr) 253 | return [cat.id for cat in itr] 254 | 255 | def get_annotation_ids(self, image_ids: Collection[int] = None, 256 | category_ids: Collection[int] = None, 257 | area_range: Tuple[float, float] = None) -> Collection[int]: 258 | """ 259 | Get ann ids that satisfy given filter conditions. default skips that filter 260 | :param image_ids: get anns for given imgs 261 | :param category_ids: get anns for given cats 262 | :param area_range: get anns for given area range (e.g. [0 inf]) 263 | :return: integer array of ann ids 264 | """ 265 | # image_ids = convert_array_argument(image_ids) 266 | # category_ids = convert_array_argument(category_ids) 267 | # area_range = convert_array_argument(area_range) 268 | 269 | itr = iter(self.annotations) 270 | if image_ids is not None: 271 | itr = filter(lambda x: x.image_id in image_ids, itr) 272 | if category_ids is not None: 273 | itr = filter(lambda x: x.category_id in category_ids, itr) 274 | if area_range is not None: 275 | itr = filter(lambda x: area_range[0] <= x.area <= area_range[1], itr) 276 | return [ann.id for ann in itr] 277 | 278 | def get_image_ids(self, category_ids: Collection[int] = None) -> Collection[int]: 279 | """ 280 | Get img ids that satisfy given filter conditions. 281 | :param category_ids: get imgs with all given cats 282 | :return: ids: integer array of img ids 283 | """ 284 | ids = set(img.id for img in self.images) 285 | for i, cat_id in enumerate(category_ids): 286 | if i == 0 and len(ids) == 0: 287 | ids = set(ann.image_id for ann in self.annotations if ann.category_id == cat_id) 288 | else: 289 | ids &= set(ann.image_id for ann in self.annotations if ann.category_id == cat_id) 290 | return list(ids) 291 | 292 | def get_annotations(self, ids: Collection[int] = None) -> Collection[PCOCOAnnotation]: 293 | """ 294 | Load anns with the specified ids. 295 | :param ids: integer ids specifying anns 296 | :return: anns: loaded ann objects 297 | """ 298 | return [ann for ann in self.annotations if ann.id in ids] 299 | 300 | def get_categories(self, ids: Collection[int] = None) -> Collection[PCOCOCategory]: 301 | """ 302 | Load cats with the specified ids. 303 | :param ids: integer ids specifying cats 304 | :return: cats: loaded cat objects 305 | """ 306 | return [cat for cat in self.categories if cat.id in ids] 307 | -------------------------------------------------------------------------------- /src/podm/coco2lableme.py: -------------------------------------------------------------------------------- 1 | from podm.coco import PCOCOObjectDetectionDataset, PCOCOBoundingBox, PCOCOSegments 2 | 3 | 4 | def coco2labelme(cocodataset: PCOCOObjectDetectionDataset): 5 | objs = [] 6 | for img in cocodataset.images: 7 | obj = { 8 | "version": "5.0.1", 9 | "flags": {}, 10 | "imagePath": img.file_name, 11 | "imageData": None, 12 | "imageHeight": img.height, 13 | "imageWidth": img.width, 14 | "shapes": [] 15 | } 16 | for annid in cocodataset.get_annotation_ids(image_ids=[img.id]): 17 | ann = cocodataset.get_annotation(annid) 18 | if isinstance(ann, PCOCOBoundingBox): 19 | shape = { 20 | "label": ann.attributes['ID'], 21 | "points": [[ann.xtl, ann.ytl], [ann.xbr, ann.ybr]], 22 | "group_id": None, 23 | "shape_type": "rectangle", 24 | "flags": {} 25 | } 26 | elif isinstance(ann, PCOCOSegments): 27 | shape = { 28 | "label": ann.attributes['ID'], 29 | "points": [[ann.segmentation[0][i], ann.segmentation[0][i+1]] 30 | for i in range(0, len(ann.segmentation[0]), 2)], 31 | "group_id": None, 32 | "shape_type": "polygon", 33 | "flags": {} 34 | } 35 | else: 36 | raise TypeError 37 | obj['shapes'].append(shape) 38 | 39 | objs.append(obj) 40 | 41 | return objs -------------------------------------------------------------------------------- /src/podm/coco_decoder.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import json 3 | from typing import Dict 4 | 5 | from podm.coco import PCOCOLicense, PCOCOInfo, PCOCOImage, PCOCOCategory, PCOCOBoundingBox, PCOCOSegments, \ 6 | PCOCOObjectDetectionDataset 7 | 8 | 9 | def parse_infon(obj: Dict) -> PCOCOInfo: 10 | info = PCOCOInfo() 11 | info.contributor = obj['contributor'] 12 | info.description = obj['description'] 13 | info.url = obj['url'] 14 | info.date_created = obj['date_created'] 15 | info.version = obj['version'] 16 | info.year = obj['year'] 17 | return info 18 | 19 | 20 | def parse_license(obj: Dict) -> PCOCOLicense: 21 | lic = PCOCOLicense() 22 | lic.id = obj['id'] 23 | lic.name = obj['name'] 24 | lic.url = obj['url'] 25 | return lic 26 | 27 | 28 | def parse_image(obj: Dict) -> PCOCOImage: 29 | img = PCOCOImage() 30 | img.id = obj['id'] 31 | img.height = obj['height'] 32 | img.width = obj['width'] 33 | img.file_name = obj['file_name'] 34 | img.flickr_url = obj['flickr_url'] 35 | img.coco_url = obj['coco_url'] 36 | img.date_captured = obj['date_captured'] 37 | img.license = obj['license'] 38 | return img 39 | 40 | 41 | def parse_bounding_box(obj: Dict) -> PCOCOBoundingBox: 42 | ann = PCOCOBoundingBox() 43 | ann.id = obj['id'] 44 | ann.category_id = obj['category_id'] 45 | ann.image_id = obj['image_id'] 46 | ann.xtl = obj['bbox'][0] 47 | ann.ytl = obj['bbox'][1] 48 | ann.xbr = ann.xtl + obj['bbox'][2] 49 | ann.ybr = ann.ytl + obj['bbox'][3] 50 | if 'contributor' in obj: 51 | ann.contributor = obj['contributor'] 52 | if 'score' in obj: 53 | ann.score = obj['score'] 54 | if 'attributes' in obj: 55 | ann.attributes = obj['attributes'] 56 | return ann 57 | 58 | 59 | def parse_segments(obj: Dict) -> PCOCOSegments: 60 | ann = PCOCOSegments() 61 | ann.id = obj['id'] 62 | ann.category_id = obj['category_id'] 63 | ann.image_id = obj['image_id'] 64 | ann.iscrowd = obj['iscrowd'] 65 | ann.segmentation = obj['segmentation'] 66 | if 'score' in obj: 67 | ann.score = obj['score'] 68 | if 'contributor' in obj: 69 | ann.contributor = obj['contributor'] 70 | if 'attributes' in obj: 71 | ann.attributes = obj['attributes'] 72 | return ann 73 | 74 | 75 | def parse_category(obj: Dict) -> PCOCOCategory: 76 | cat = PCOCOCategory() 77 | cat.id = obj['id'] 78 | cat.name = obj['name'] 79 | cat.supercategory = obj['supercategory'] 80 | return cat 81 | 82 | 83 | def parse_object_detection_dataset(coco_obj: Dict) -> PCOCOObjectDetectionDataset: 84 | dataset = PCOCOObjectDetectionDataset() 85 | dataset.info = parse_infon(coco_obj['info']) 86 | 87 | for lic_obj in coco_obj['licenses']: 88 | lic = parse_license(lic_obj) 89 | dataset.licenses.append(lic) 90 | 91 | for img_obj in coco_obj['images']: 92 | img = parse_image(img_obj) 93 | dataset.images.append(img) 94 | 95 | for ann_obj in coco_obj['annotations']: 96 | if 'segmentation' in ann_obj and len(ann_obj['segmentation']) > 0: 97 | ann = parse_segments(ann_obj) 98 | else: 99 | ann = parse_bounding_box(ann_obj) 100 | dataset.add_annotation(ann) 101 | 102 | for cat_obj in coco_obj['categories']: 103 | cat = parse_category(cat_obj) 104 | dataset.categories.append(cat) 105 | 106 | return dataset 107 | 108 | 109 | def load_true_object_detection_dataset(fp, **kwargs) -> PCOCOObjectDetectionDataset: 110 | coco_obj = json.load(fp, **kwargs) 111 | return parse_object_detection_dataset(coco_obj) 112 | 113 | 114 | def load_pred_object_detection_dataset(fp, dataset: PCOCOObjectDetectionDataset, **kwargs) \ 115 | -> PCOCOObjectDetectionDataset: 116 | new_dataset = PCOCOObjectDetectionDataset() 117 | new_dataset.info = copy.deepcopy(dataset.info) 118 | new_dataset.licenses = copy.deepcopy(dataset.licenses) 119 | new_dataset.images = copy.deepcopy(dataset.images) 120 | new_dataset.categories = copy.deepcopy(dataset.categories) 121 | # check annotation 122 | coco_obj = json.load(fp, **kwargs) 123 | annotations = [] 124 | for obj in coco_obj: 125 | ann = parse_bounding_box(obj) 126 | if new_dataset.get_image(id=ann.image_id) is None: 127 | print('%s: Cannot find image' % ann.image_id) 128 | if new_dataset.get_category(id=ann.category_id) is None: 129 | print('%s: Cannot find category' % ann.category_id) 130 | annotations.append(ann) 131 | new_dataset.annotations = annotations 132 | return new_dataset 133 | -------------------------------------------------------------------------------- /src/podm/coco_encoder.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Union, TextIO, Dict, List 3 | 4 | from podm.coco import PCOCOImage, PCOCOLicense, PCOCOInfo, \ 5 | PCOCOCategory, PCOCOBoundingBox, \ 6 | PCOCOSegments, PCOCOObjectDetectionDataset 7 | 8 | PCOCO_OBJ = Union[ 9 | PCOCOImage, PCOCOLicense, PCOCOInfo, 10 | PCOCOCategory, 11 | PCOCOObjectDetectionDataset, 12 | List[PCOCOBoundingBox], 13 | ] 14 | 15 | 16 | class PCOCOJSONEncoder(json.JSONEncoder): 17 | """ 18 | Extensible BioC JSON encoder for BioC data structures. 19 | """ 20 | 21 | def default(self, o): 22 | # print('xxxxxxxxxxxxxxxxxxxx') 23 | # print(type(o)) 24 | # print(isinstance(o, PCOCOObjectDetectionDataset)) 25 | # print(repr(o.__class__), repr(PCOCOObjectDetectionResult)) 26 | if isinstance(o, PCOCOImage): 27 | return { 28 | "width": o.width, 29 | "height": o.height, 30 | "flickr_url": o.flickr_url, 31 | "coco_url": o.coco_url, 32 | "file_name": o.file_name, 33 | "date_captured": o.date_captured, 34 | "license": o.license, 35 | "id": o.id, 36 | } 37 | if isinstance(o, PCOCOLicense): 38 | return { 39 | "id": o.id, 40 | "name": o.name, 41 | "url": o.url, 42 | } 43 | if isinstance(o, PCOCOInfo): 44 | return { 45 | "year": o.year, 46 | "version": o.version, 47 | "description": o.description, 48 | "contributor": o.contributor, 49 | "url": o.url, 50 | "date_created": o.date_created, 51 | } 52 | if isinstance(o, PCOCOCategory): 53 | return { 54 | "id": o.id, 55 | "name": o.name, 56 | "supercategory": o.supercategory, 57 | } 58 | if isinstance(o, PCOCOBoundingBox): 59 | return { 60 | "id": o.id, 61 | "image_id": o.image_id, 62 | "category_id": o.category_id, 63 | "bbox": [o.xtl, o.ytl, o.width, o.height], 64 | "score": o.score, 65 | "contributor": o.contributor, 66 | "attributes": json.dumps(o.attributes) 67 | } 68 | if isinstance(o, PCOCOSegments): 69 | bb = o.bbox 70 | return { 71 | "id": o.id, 72 | "image_id": o.image_id, 73 | "category_id": o.category_id, 74 | "segmentation": o.segmentation, 75 | "bbox": [bb.xtl, bb.ytl, bb.width, bb.height], 76 | "area": bb.area, 77 | "iscrowd": o.iscrowd, 78 | "score": o.score, 79 | "contributor": o.contributor, 80 | "attributes": json.dumps(o.attributes) 81 | } 82 | if isinstance(o, PCOCOObjectDetectionDataset): 83 | return { 84 | "info": self.default(o.info), 85 | 'images': [self.default(img) for img in o.images], 86 | "licenses": [self.default(l) for l in o.licenses], 87 | 'annotations': [self.default(ann) for ann in o.annotations], 88 | 'categories': [self.default(cat) for cat in o.categories], 89 | } 90 | # Let the base class default method raise the TypeError 91 | return json.JSONEncoder.default(self, o) 92 | 93 | 94 | def toJSON(o) -> Dict: 95 | """ 96 | Convert a pcoco obj to a Python `dict` 97 | """ 98 | return PCOCOJSONEncoder().default(o) 99 | 100 | 101 | def dumps(obj: PCOCO_OBJ, **kwargs) -> str: 102 | """ 103 | Serialize a BioC ``obj`` to a JSON formatted ``str``. kwargs are passed to json. 104 | """ 105 | return json.dumps(obj, cls=PCOCOJSONEncoder, indent=2, **kwargs) 106 | 107 | 108 | def dump(obj: PCOCO_OBJ, fp: TextIO, **kwargs): 109 | """ 110 | Serialize ``obj`` as a JSON formatted stream to ``fp`` 111 | (a ``.write()``-supporting file-like object). kwargs are passed to json. 112 | """ 113 | return json.dump(obj, fp, cls=PCOCOJSONEncoder, indent=2, **kwargs) -------------------------------------------------------------------------------- /src/podm/metrics.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from collections import Counter, defaultdict 3 | from enum import Enum 4 | from typing import List, Dict, Any, Tuple 5 | 6 | import numpy as np 7 | 8 | from podm import box 9 | from podm.coco import PCOCOObjectDetectionDataset, PCOCOBoundingBox, PCOCOSegments 10 | 11 | 12 | class BoundingBox(box.Box): 13 | """ 14 | image: image. 15 | category: category. 16 | xtl: the X top-left coordinate of the bounding box. 17 | ytl: the Y top-left coordinate of the bounding box. 18 | xbr: the X bottom-right coordinate of the bounding box. 19 | ybr: the Y bottom-right coordinate of the bounding box. 20 | score: (optional) the confidence of the detected class. 21 | """ 22 | def __init__(self): 23 | super(BoundingBox, self).__init__() 24 | self.image = None 25 | self.category = None 26 | self.score = None # type: float or None 27 | 28 | @classmethod 29 | def of_bbox(cls, image, category, xtl: float, ytl: float, xbr: float, ybr: float, score: float = None) \ 30 | -> 'BoundingBox': 31 | bbox = BoundingBox() 32 | bbox.xtl = xtl 33 | bbox.ytl = ytl 34 | bbox.xbr = xbr 35 | bbox.ybr = ybr 36 | bbox.image = image 37 | bbox.score = score 38 | bbox.category = category 39 | return bbox 40 | 41 | 42 | def get_bounding_boxes(dataset: PCOCOObjectDetectionDataset, use_name: bool = True) -> List[BoundingBox]: 43 | bboxes = [] 44 | for ann in dataset.annotations: 45 | if isinstance(ann, PCOCOBoundingBox): 46 | bb = BoundingBox.of_bbox(ann.image_id, ann.category_id, ann.xtl, ann.ytl, ann.xbr, ann.ybr, ann.score) 47 | elif isinstance(ann, PCOCOSegments): 48 | bb = BoundingBox.of_bbox(ann.image_id, ann.category_id, 49 | ann.bbox.xtl, ann.bbox.ytl, ann.bbox.xbr, ann.bbox.ybr, ann.score) 50 | else: 51 | raise TypeError 52 | if use_name: 53 | bb.image = dataset.get_image(id=ann.image_id).file_name 54 | bb.category = dataset.get_category(id=ann.category_id).name 55 | bboxes.append(bb) 56 | return bboxes 57 | 58 | 59 | class MethodAveragePrecision(Enum): 60 | """ 61 | Class representing if the coordinates are relative to the 62 | image size or are absolute values. 63 | 64 | Developed by: Rafael Padilla 65 | Last modification: Apr 28 2018 66 | """ 67 | AllPointsInterpolation = 1 68 | ElevenPointsInterpolation = 2 69 | 70 | 71 | class MetricPerClass: 72 | def __init__(self): 73 | self.label = None 74 | self.precision = None 75 | self.recall = None 76 | self.ap = None 77 | self.interpolated_precision = None # type: None or np.ndarray 78 | self.interpolated_recall = None # type: None or np.ndarray 79 | self.num_groundtruth = None 80 | self.num_detection = None 81 | self.tp = None 82 | self.fp = None 83 | 84 | @staticmethod 85 | def mAP(results: Dict[Any, 'MetricPerClass']): 86 | return np.average([m.ap for m in results.values() if m.num_groundtruth > 0]) 87 | 88 | 89 | def get_pascal_voc_metrics(gold_standard: List[BoundingBox], 90 | predictions: List[BoundingBox], 91 | iou_threshold: float = 0.5, 92 | method: MethodAveragePrecision = MethodAveragePrecision.AllPointsInterpolation 93 | ) -> Dict[str, MetricPerClass]: 94 | """Get the metrics used by the VOC Pascal 2012 challenge. 95 | 96 | Args: 97 | gold_standard: ground truth bounding boxes; 98 | predictions: detected bounding boxes; 99 | iou_threshold: IOU threshold indicating which detections will be considered TP or FP (default value = 0.5); 100 | method: It can be calculated as the implementation in the official PASCAL VOC toolkit (EveryPointInterpolation), 101 | or applying the 11-point interpolation as described in the paper "The PASCAL Visual Object Classes(VOC) 102 | Challenge" or AllPointsInterpolation" (ElevenPointInterpolation); 103 | Returns: 104 | A dictionary containing metrics of each class. 105 | """ 106 | ret = {} # list containing metrics (precision, recall, average precision) of each class 107 | 108 | # Get all classes 109 | categories = sorted(set(b.category for b in gold_standard + predictions)) 110 | 111 | # Precision x Recall is obtained individually by each class 112 | # Loop through by classes 113 | for category in categories: 114 | preds = [b for b in predictions if b.category == category] # type: List[BoundingBox] 115 | golds = [b for b in gold_standard if b.category == category] # type: List[BoundingBox] 116 | npos = len(golds) 117 | 118 | # sort detections by decreasing confidence 119 | preds = sorted(preds, key=lambda b: b.score, reverse=True) 120 | tps = np.zeros(len(preds)) 121 | fps = np.zeros(len(preds)) 122 | 123 | # create dictionary with amount of gts for each image 124 | counter = Counter([cc.image for cc in golds]) 125 | for key, val in counter.items(): 126 | counter[key] = np.zeros(val) 127 | 128 | # Pre-processing groundtruths of the some image 129 | image_name2gt = defaultdict(list) 130 | for b in golds: 131 | image_name2gt[b.image].append(b) 132 | 133 | # Loop through detections 134 | for i in range(len(preds)): 135 | # Find ground truth image 136 | gt = image_name2gt[preds[i].image] 137 | max_iou = sys.float_info.min 138 | mas_idx = -1 139 | for j in range(len(gt)): 140 | iou = box.intersection_over_union(preds[i], gt[j]) 141 | if iou > max_iou: 142 | max_iou = iou 143 | mas_idx = j 144 | # Assign detection as true positive/don't care/false positive 145 | if max_iou >= iou_threshold: 146 | if counter[preds[i].image][mas_idx] == 0: 147 | tps[i] = 1 # count as true positive 148 | counter[preds[i].image][mas_idx] = 1 # flag as already 'seen' 149 | else: 150 | # - A detected "cat" is overlaped with a GT "cat" with IOU >= IOUThreshold. 151 | fps[i] = 1 # count as false positive 152 | else: 153 | fps[i] = 1 # count as false positive 154 | # compute precision, recall and average precision 155 | cumulative_fps = np.cumsum(fps) 156 | cumulative_tps = np.cumsum(tps) 157 | recalls = np.divide(cumulative_tps, npos, out=np.full_like(cumulative_tps, np.nan), where=npos != 0) 158 | precisions = np.divide(cumulative_tps, (cumulative_fps + cumulative_tps)) 159 | # Depending on the method, call the right implementation 160 | if method == MethodAveragePrecision.AllPointsInterpolation: 161 | ap, mrec, mpre, _ = calculate_all_points_average_precision(recalls, precisions) 162 | else: 163 | ap, mrec, mpre = calculate_11_points_average_precision(recalls, precisions) 164 | # add class result in the dictionary to be returned 165 | r = MetricPerClass() 166 | r.label = category 167 | r.precision = precisions 168 | r.recall = recalls 169 | r.ap = ap 170 | r.interpolated_recall = np.array(mrec) 171 | r.interpolated_precision = np.array(mpre) 172 | r.tp = np.sum(tps) 173 | r.fp = np.sum(fps) 174 | r.num_groundtruth = len(golds) 175 | r.num_detection = len(preds) 176 | ret[category] = r 177 | return ret 178 | 179 | 180 | def calculate_all_points_average_precision(recall: List[float], precision: List[float]) \ 181 | -> Tuple[float, List[float], List[float], List[int]]: 182 | """ 183 | All-point interpolated average precision 184 | 185 | Returns: 186 | average precision 187 | interpolated recall 188 | interpolated precision 189 | interpolated points 190 | """ 191 | mrec = [0.0] + [e for e in recall] + [1.0] 192 | mpre = [0.0] + [e for e in precision] + [0] 193 | for i in range(len(mpre) - 1, 0, -1): 194 | mpre[i - 1] = max(mpre[i - 1], mpre[i]) 195 | ii = [] 196 | for i in range(len(mrec) - 1): 197 | if mrec[i + 1] != mrec[i]: 198 | ii.append(i + 1) 199 | ap = 0 200 | for i in ii: 201 | ap = ap + np.sum((mrec[i] - mrec[i - 1]) * mpre[i]) 202 | return ap, mrec[0:len(mpre) - 1], mpre[0:len(mpre) - 1], ii 203 | 204 | 205 | def calculate_11_points_average_precision(recall: List[float], precision: List[float]) -> Tuple[float, List[float], List[float]]: 206 | """ 207 | 11-point interpolated average precision. This is done by segmenting the recalls evenly into 11 parts: 208 | {0,0.1,0.2,...,0.9,1}. 209 | 210 | Args: 211 | recall: recall list 212 | precision: precision list 213 | 214 | Returns: 215 | average precision, interpolated recall, interpolated precision 216 | 217 | """ 218 | mrec = [e for e in recall] 219 | mpre = [e for e in precision] 220 | recall_values = np.linspace(0, 1, 11) 221 | recall_values = list(recall_values[::-1]) 222 | rho_interp = [] 223 | recall_valid = [] 224 | # For each recallValues (0, 0.1, 0.2, ... , 1) 225 | for r in recall_values: 226 | # Obtain all recall values higher or equal than r 227 | arg_greater_recalls = np.argwhere(mrec[:] >= r) 228 | pmax = 0 229 | # If there are recalls above r 230 | if arg_greater_recalls.size != 0: 231 | pmax = max(mpre[arg_greater_recalls.min():]) 232 | recall_valid.append(r) 233 | rho_interp.append(pmax) 234 | # By definition AP = sum(max(precision whose recall is above r))/11 235 | ap = sum(rho_interp) / 11 236 | # Generating values for the plot 237 | rvals = [recall_valid[0]] + [e for e in recall_valid] + [0] 238 | pvals = [0] + [e for e in rho_interp] + [0] 239 | # rhoInterp = rhoInterp[::-1] 240 | cc = [] 241 | for i in range(len(rvals)): 242 | p = (rvals[i], pvals[i - 1]) 243 | if p not in cc: 244 | cc.append(p) 245 | p = (rvals[i], pvals[i]) 246 | if p not in cc: 247 | cc.append(p) 248 | recall_values = [i[0] for i in reversed(cc)] 249 | precision_values = [i[1] for i in reversed(cc)] 250 | return ap, recall_values, precision_values 251 | -------------------------------------------------------------------------------- /src/podm/pascal2coco.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert from a PASCAL VOC zip file to a COCO file. 3 | 4 | Usage: 5 | pascal2coco gold --gold= --output-gold= 6 | pascal2coco pred --gold= --pred= --output-gold= --output-pred= 7 | 8 | Options: 9 | --gold= PASCAL VOC groundtruths zip file 10 | --pred= PASCAL VOC predictions zip file 11 | --output-gold= Groundtruths JSON file 12 | --output-pred= Predictions JSON file 13 | """ 14 | import json 15 | import zipfile 16 | import io 17 | import warnings 18 | 19 | import docopt 20 | import tqdm 21 | import pandas as pd 22 | from podm import coco_encoder 23 | from podm.box import BBFormat, Box 24 | from podm.coco import PCOCOObjectDetectionDataset, PCOCOImage, PCOCOCategory, PCOCOBoundingBox 25 | 26 | 27 | def convert_pascal_to_df(src): 28 | rows = [] 29 | with zipfile.ZipFile(src, 'r') as myzip: 30 | namelist = myzip.namelist() 31 | for name in tqdm.tqdm(namelist): 32 | if not name.endswith('.txt'): 33 | continue 34 | with myzip.open(name, 'r') as fp: 35 | name = name[name.find('/') + 1:] 36 | items_file = io.TextIOWrapper(fp) 37 | for line in items_file: 38 | toks = line.strip().split(' ') 39 | if len(toks) == 5: 40 | row = { 41 | 'name': name, 42 | 'label': toks[0], 43 | 'xtl': int(toks[1]), 44 | 'ytl': int(toks[2]), 45 | 'xbr': int(toks[3]), 46 | 'ybr': int(toks[4]) 47 | } 48 | elif len(toks) == 6: 49 | row = { 50 | 'name': name, 51 | 'label': toks[0], 52 | 'score': float(toks[1]), 53 | 'xtl': int(toks[2]), 54 | 'ytl': int(toks[3]), 55 | 'xbr': int(toks[4]), 56 | 'ybr': int(toks[5]) 57 | } 58 | else: 59 | raise ValueError 60 | rows.append(row) 61 | return pd.DataFrame(rows) 62 | 63 | 64 | class PascalVoc2COCO: 65 | def __init__(self, format: BBFormat = BBFormat.X1Y1X2Y2): 66 | self.format = format 67 | 68 | def convert_gold(self, src) -> PCOCOObjectDetectionDataset: 69 | df = convert_pascal_to_df(src) 70 | 71 | dataset = PCOCOObjectDetectionDataset() 72 | # add image 73 | for i, name in enumerate(df['name'].unique()): 74 | img = PCOCOImage() 75 | img.id = i 76 | img.file_name = name 77 | dataset.add_image(img) 78 | # add category 79 | for i, label in enumerate(df['label'].unique()): 80 | cat = PCOCOCategory() 81 | cat.id = i 82 | cat.name = label 83 | dataset.add_category(cat) 84 | # add annotation 85 | for i, row in tqdm.tqdm(df.iterrows(), total=len(df)): 86 | box = Box.of_box(row['xtl'], row['ytl'], row['xbr'], row['ybr']) 87 | if self.format == BBFormat.XYWH: 88 | box.xbr += box.xtl 89 | box.ybr += box.ytl 90 | ann = PCOCOBoundingBox() 91 | ann.image_id = dataset.get_image(file_name=row['name']).id 92 | ann.id = i 93 | ann.category_id = dataset.get_category(name=row['label']).id 94 | ann.set_box(box) 95 | dataset.add_annotation(ann) 96 | return dataset 97 | 98 | def convert_gold_file(self, src, dest): 99 | dataset = self.convert_gold(src) 100 | with open(dest, 'w') as fp: 101 | coco_encoder.dump(dataset, fp) 102 | 103 | def convert_gold_pred(self, src_gold, src_pred): 104 | gold_dataset = self.convert_gold(src_gold) 105 | 106 | df = convert_pascal_to_df(src_pred) 107 | # check cat 108 | subrows = [] 109 | for i, row in tqdm.tqdm(df.iterrows(), total=len(df)): 110 | if gold_dataset.get_category(name=row['label']) is None: 111 | warnings.warn('%s: Category does not exist' % row['label']) 112 | continue 113 | if gold_dataset.get_image(file_name=row['name']) is None: 114 | warnings.warn('%s: Image does not exist' % row['name']) 115 | continue 116 | subrows.append(row) 117 | if len(subrows) < len(df): 118 | warnings.warn('Remove %s rows' % (len(df) - len(subrows))) 119 | 120 | annotations = [] 121 | for i, row in tqdm.tqdm(enumerate(subrows), total=len(subrows)): 122 | box = Box.of_box(row['xtl'], row['ytl'], row['xbr'], row['ybr']) 123 | if self.format == BBFormat.XYWH: 124 | box.xbr += box.xtl 125 | box.ybr += box.ytl 126 | ann = PCOCOBoundingBox() 127 | ann.image_id = gold_dataset.get_image(file_name=row['name']).id 128 | ann.id = i 129 | ann.category_id = gold_dataset.get_category(name=row['label']).id 130 | ann.score = row['score'] 131 | ann.set_box(box) 132 | annotations.append(ann) 133 | 134 | pred_dataset = gold_dataset.get_new_dataset(annotations) 135 | return gold_dataset, pred_dataset 136 | 137 | def convert_gold_pred_file(self, src_gold, src_pred, dest_gold, dest_pred): 138 | gold_dataset, pred_dataset = self.convert_gold_pred(src_gold, src_pred) 139 | with open(dest_gold, 'w') as fp: 140 | coco_encoder.dump(gold_dataset, fp) 141 | 142 | with open(dest_pred, 'w') as fp: 143 | json.dump(pred_dataset.annotations, fp, cls=coco_encoder.PCOCOJSONEncoder, indent=2) 144 | 145 | 146 | def main(): 147 | argv = docopt.docopt(__doc__) 148 | converter = PascalVoc2COCO(BBFormat.X1Y1X2Y2) 149 | if argv['gold']: 150 | converter.convert_gold_file(argv['--gold'], argv['--output-gold']) 151 | if argv['pred']: 152 | converter.convert_gold_pred_file(argv['--gold'], argv['--pred'], argv['--output-gold'], argv['--output-pred']) 153 | 154 | 155 | if __name__ == '__main__': 156 | main() 157 | -------------------------------------------------------------------------------- /src/podm/visualize.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | from podm.metrics import MethodAveragePrecision, MetricPerClass 7 | 8 | 9 | def plot_precision_recall_curve(result: MetricPerClass, 10 | dest, 11 | method: MethodAveragePrecision = MethodAveragePrecision.AllPointsInterpolation, 12 | show_ap: bool=False, 13 | show_interpolated_precision: bool=False): 14 | """PlotPrecisionRecallCurve 15 | Plot the Precision x Recall curve for a given class. 16 | Args: 17 | result: metric per class 18 | dest: the plot will be saved as an image in this path 19 | method: method for interpolation 20 | show_ap: if True, the average precision value will be shown in the title of 21 | the graph (default = False); 22 | show_interpolated_precision (optional): if True, it will show in the plot the interpolated 23 | precision (default = False); 24 | """ 25 | mpre = result.interpolated_precision 26 | mrec = result.interpolated_recall 27 | 28 | plt.close() 29 | if show_interpolated_precision: 30 | if method == MethodAveragePrecision.AllPointsInterpolation: 31 | plt.plot(mrec, mpre, '--r', label='Interpolated precision (every point)') 32 | elif method == MethodAveragePrecision.ElevenPointsInterpolation: 33 | # Uncomment the line below if you want to plot the area 34 | # plt.plot(mrec, mpre, 'or', label='11-point interpolated precision') 35 | # Remove duplicates, getting only the highest precision of each recall value 36 | nrec = [] 37 | nprec = [] 38 | for idx in range(len(mrec)): 39 | r = mrec[idx] 40 | if r not in nrec: 41 | idxEq = np.argwhere(mrec == r) 42 | nrec.append(r) 43 | nprec.append(max([mpre[int(id)] for id in idxEq])) 44 | plt.plot(nrec, nprec, 'or', label='11-point interpolated precision') 45 | plt.plot(result.recall, result.precision, '-o') 46 | # add a new penultimate point to the list (mrec[-2], 0.0) 47 | # since the last line segment (and respective area) do not affect the AP value 48 | area_under_curve_x = mrec[:-1].tolist() + [mrec[-2]] + [mrec[-1]] 49 | area_under_curve_y = mpre[:-1].tolist() + [0.0] + [mpre[-1]] 50 | plt.fill_between(area_under_curve_x, 0, area_under_curve_y, alpha=0.2, edgecolor='r') 51 | 52 | plt.xlabel('Recall') 53 | plt.ylabel('Precision') 54 | if show_ap: 55 | ap_str = "{0:.2f}%".format(result.ap * 100) 56 | plt.title('Precision x Recall curve \nClass: %s, AP: %s' % (result.label, ap_str)) 57 | else: 58 | plt.title('Precision x Recall curve \nClass: %s' % result.label) 59 | # plt.legend(shadow=True) 60 | plt.grid() 61 | plt.savefig(str(dest)) 62 | 63 | 64 | def plot_precision_recall_curve_all(results: Dict[Any, MetricPerClass], 65 | dest_dir, 66 | method: MethodAveragePrecision = MethodAveragePrecision.AllPointsInterpolation, 67 | show_ap: bool=False, 68 | show_interpolated_precision: bool=False): 69 | """ 70 | Plot the Precision x Recall curve for a given class. 71 | 72 | Args: 73 | results: metric per class 74 | dest_dir: the plot will be saved as an image in this path 75 | method: method for interpolation 76 | show_ap: if True, the average precision value will be shown in the title of 77 | the graph (default = False); 78 | show_interpolated_precision (optional): if True, it will show in the plot the interpolated 79 | precision (default = False); 80 | """ 81 | for label, result in results.items(): 82 | dest = str(dest_dir / (label + '_pr.png')) 83 | try: 84 | plot_precision_recall_curve(result, dest, method, show_ap, show_interpolated_precision) 85 | except: 86 | print(f'{label}: Cannot plot') 87 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(scope="module") 7 | def sample_dir(): 8 | return Path(__file__).parent / 'sample' 9 | 10 | 11 | @pytest.fixture(scope="module") 12 | def tests_dir(): 13 | return Path(__file__).parent 14 | -------------------------------------------------------------------------------- /tests/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfpeng/object_detection_metrics/3801acc32d052d1fbf1565f96a9f8d390701f911/tests/helpers/__init__.py -------------------------------------------------------------------------------- /tests/helpers/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | from podm.metrics import MetricPerClass 4 | import numpy as np 5 | 6 | 7 | class bcolors: 8 | HEADER = '\033[95m' 9 | OKBLUE = '\033[94m' 10 | OKGREEN = '\033[92m' 11 | WARNING = '\033[93m' 12 | FAIL = '\033[91m' 13 | ENDC = '\033[0m' 14 | BOLD = '\033[1m' 15 | UNDERLINE = '\033[4m' 16 | 17 | 18 | # def load_data(pathname) -> List[BoundingBox]: 19 | # with open(pathname, 'r') as fp: 20 | # objs = json.load(fp) 21 | # 22 | # boxes = [] 23 | # for fig in objs: 24 | # for box in fig['boxes']: 25 | # if 'score' in box: 26 | # score = box['score'] 27 | # else: 28 | # score = 1 29 | # bb = BoundingBox( 30 | # fig['name'], 31 | # box['label'], 32 | # box['xtl'], 33 | # box['ytl'], 34 | # box['xbr'], 35 | # box['ybr'], 36 | # score) 37 | # boxes.append(bb) 38 | # return boxes 39 | 40 | 41 | # def assert_results(actuals: Dict[str, MetricPerClass], expecteds, key, classes=None): 42 | # if classes is None: 43 | # classes = set(m.label for m in actuals.values()) 44 | # 45 | # for m in actuals.values(): 46 | # label = m.label 47 | # if label in classes: 48 | # print(f'{label}, {key}: ', end='') 49 | # actual = m.__dict__[key] 50 | # expected = expecteds[label][key] 51 | # try: 52 | # if np.allclose(actual, expected, rtol=1e-1, equal_nan=True): 53 | # print(f'{bcolors.OKGREEN}Passed{bcolors.ENDC}') 54 | # else: 55 | # print(f'{bcolors.FAIL}Failed. Expected:{expected}, Actual:{actual}{bcolors.ENDC}') 56 | # exit(1) 57 | # except Exception as e: 58 | # print(f'{bcolors.FAIL}Failed. Expected:{expected}, Actual:{actual}{bcolors.ENDC}') 59 | # exit(1) 60 | 61 | 62 | def assert_results(actuals: Dict[Any, MetricPerClass], expecteds, key, classes=None): 63 | if classes is None: 64 | classes = set(m.label for m in actuals.values()) 65 | 66 | for m in actuals.values(): 67 | label = m.label 68 | if label in classes: 69 | actual = m.__dict__[key] 70 | expected = expecteds[label][key] 71 | assert np.allclose(actual, expected, rtol=1e-1, equal_nan=True),\ 72 | f'Failed. Expected:{expected}, Actual:{actual}' 73 | -------------------------------------------------------------------------------- /tests/sample/detections.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfpeng/object_detection_metrics/3801acc32d052d1fbf1565f96a9f8d390701f911/tests/sample/detections.zip -------------------------------------------------------------------------------- /tests/sample/expected0_5.json: -------------------------------------------------------------------------------- 1 | { 2 | "backpack": { 3 | "class_name": "backpack", 4 | "ap": 0.22727272727272724, 5 | "precision": [ 6 | 1.0, 7 | 0.5, 8 | 0.6666666666666666, 9 | 0.75, 10 | 0.6 11 | ], 12 | "recall": [ 13 | 0.09090909090909091, 14 | 0.09090909090909091, 15 | 0.18181818181818182, 16 | 0.2727272727272727, 17 | 0.2727272727272727 18 | ], 19 | "num_groundtruth": 11, 20 | "num_detection": 5, 21 | "tp": 3, 22 | "fp": 2 23 | }, 24 | "bed": { 25 | "class_name": "bed", 26 | "ap": 0.859375, 27 | "precision": [ 28 | 1.0, 29 | 1.0, 30 | 1.0, 31 | 1.0, 32 | 1.0, 33 | 1.0, 34 | 0.8571428571428571, 35 | 0.875 36 | ], 37 | "recall": [ 38 | 0.125, 39 | 0.25, 40 | 0.375, 41 | 0.5, 42 | 0.625, 43 | 0.75, 44 | 0.75, 45 | 0.875 46 | ], 47 | "num_groundtruth": 8, 48 | "num_detection": 8, 49 | "tp": 7, 50 | "fp": 1 51 | }, 52 | "book": { 53 | "class_name": "book", 54 | "ap": 0.1752305665349143, 55 | "precision": [ 56 | 1.0, 57 | 0.5, 58 | 0.3333333333333333, 59 | 0.25, 60 | 0.4, 61 | 0.3333333333333333, 62 | 0.2857142857142857, 63 | 0.25, 64 | 0.2222222222222222, 65 | 0.3, 66 | 0.2727272727272727, 67 | 0.3333333333333333, 68 | 0.38461538461538464, 69 | 0.35714285714285715, 70 | 0.3333333333333333, 71 | 0.375, 72 | 0.4117647058823529, 73 | 0.4444444444444444, 74 | 0.42105263157894735, 75 | 0.4, 76 | 0.42857142857142855, 77 | 0.45454545454545453, 78 | 0.4782608695652174, 79 | 0.4583333333333333, 80 | 0.44 81 | ], 82 | "recall": [ 83 | 0.030303030303030304, 84 | 0.030303030303030304, 85 | 0.030303030303030304, 86 | 0.030303030303030304, 87 | 0.06060606060606061, 88 | 0.06060606060606061, 89 | 0.06060606060606061, 90 | 0.06060606060606061, 91 | 0.06060606060606061, 92 | 0.09090909090909091, 93 | 0.09090909090909091, 94 | 0.12121212121212122, 95 | 0.15151515151515152, 96 | 0.15151515151515152, 97 | 0.15151515151515152, 98 | 0.18181818181818182, 99 | 0.21212121212121213, 100 | 0.24242424242424243, 101 | 0.24242424242424243, 102 | 0.24242424242424243, 103 | 0.2727272727272727, 104 | 0.30303030303030304, 105 | 0.3333333333333333, 106 | 0.3333333333333333, 107 | 0.3333333333333333 108 | ], 109 | "num_groundtruth": 33, 110 | "num_detection": 25, 111 | "tp": 11, 112 | "fp": 14 113 | }, 114 | "bookcase": { 115 | "class_name": "bookcase", 116 | "ap": 0.14285714285714285, 117 | "precision": [ 118 | 1.0 119 | ], 120 | "recall": [ 121 | 0.14285714285714285 122 | ], 123 | "num_groundtruth": 7, 124 | "num_detection": 1, 125 | "tp": 1, 126 | "fp": 0 127 | }, 128 | "bottle": { 129 | "class_name": "bottle", 130 | "ap": 0.23484848484848486, 131 | "precision": [ 132 | 0.0, 133 | 0.5, 134 | 0.6666666666666666, 135 | 0.5, 136 | 0.4, 137 | 0.3333333333333333, 138 | 0.2857142857142857, 139 | 0.375, 140 | 0.3333333333333333, 141 | 0.3, 142 | 0.36363636363636365, 143 | 0.4166666666666667, 144 | 0.38461538461538464, 145 | 0.35714285714285715, 146 | 0.3333333333333333, 147 | 0.3125, 148 | 0.29411764705882354, 149 | 0.2777777777777778, 150 | 0.2631578947368421, 151 | 0.25 152 | ], 153 | "recall": [ 154 | 0.0, 155 | 0.09090909090909091, 156 | 0.18181818181818182, 157 | 0.18181818181818182, 158 | 0.18181818181818182, 159 | 0.18181818181818182, 160 | 0.18181818181818182, 161 | 0.2727272727272727, 162 | 0.2727272727272727, 163 | 0.2727272727272727, 164 | 0.36363636363636365, 165 | 0.45454545454545453, 166 | 0.45454545454545453, 167 | 0.45454545454545453, 168 | 0.45454545454545453, 169 | 0.45454545454545453, 170 | 0.45454545454545453, 171 | 0.45454545454545453, 172 | 0.45454545454545453, 173 | 0.45454545454545453 174 | ], 175 | "num_groundtruth": 11, 176 | "num_detection": 20, 177 | "tp": 5, 178 | "fp": 15 179 | }, 180 | "bowl": { 181 | "class_name": "bowl", 182 | "ap": 0.3185714285714286, 183 | "precision": [ 184 | 1.0, 185 | 1.0, 186 | 0.6666666666666666, 187 | 0.75, 188 | 0.6, 189 | 0.6666666666666666, 190 | 0.7142857142857143, 191 | 0.625, 192 | 0.5555555555555556, 193 | 0.6 194 | ], 195 | "recall": [ 196 | 0.06666666666666667, 197 | 0.13333333333333333, 198 | 0.13333333333333333, 199 | 0.2, 200 | 0.2, 201 | 0.26666666666666666, 202 | 0.3333333333333333, 203 | 0.3333333333333333, 204 | 0.3333333333333333, 205 | 0.4 206 | ], 207 | "num_groundtruth": 15, 208 | "num_detection": 10, 209 | "tp": 6, 210 | "fp": 4 211 | }, 212 | "cabinetry": { 213 | "class_name": "cabinetry", 214 | "ap": 0.07932692307692307, 215 | "precision": [ 216 | 0.0, 217 | 0.0, 218 | 0.0, 219 | 0.25, 220 | 0.4, 221 | 0.5, 222 | 0.5714285714285714, 223 | 0.625, 224 | 0.5555555555555556, 225 | 0.5, 226 | 0.45454545454545453, 227 | 0.5, 228 | 0.46153846153846156, 229 | 0.5 230 | ], 231 | "recall": [ 232 | 0.0, 233 | 0.0, 234 | 0.0, 235 | 0.019230769230769232, 236 | 0.038461538461538464, 237 | 0.057692307692307696, 238 | 0.07692307692307693, 239 | 0.09615384615384616, 240 | 0.09615384615384616, 241 | 0.09615384615384616, 242 | 0.09615384615384616, 243 | 0.11538461538461539, 244 | 0.11538461538461539, 245 | 0.1346153846153846 246 | ], 247 | "num_groundtruth": 52, 248 | "num_detection": 14, 249 | "tp": 7, 250 | "fp": 7 251 | }, 252 | "chair": { 253 | "class_name": "chair", 254 | "ap": 0.5384346220032401, 255 | "precision": [ 256 | 1.0, 257 | 1.0, 258 | 1.0, 259 | 1.0, 260 | 1.0, 261 | 1.0, 262 | 1.0, 263 | 1.0, 264 | 1.0, 265 | 0.9, 266 | 0.9090909090909091, 267 | 0.9166666666666666, 268 | 0.9230769230769231, 269 | 0.8571428571428571, 270 | 0.8, 271 | 0.8125, 272 | 0.8235294117647058, 273 | 0.8333333333333334, 274 | 0.8421052631578947, 275 | 0.85, 276 | 0.8571428571428571, 277 | 0.8636363636363636, 278 | 0.8695652173913043, 279 | 0.8333333333333334, 280 | 0.84, 281 | 0.8076923076923077, 282 | 0.8148148148148148, 283 | 0.8214285714285714, 284 | 0.8275862068965517, 285 | 0.8333333333333334, 286 | 0.8387096774193549, 287 | 0.8125, 288 | 0.8181818181818182, 289 | 0.7941176470588235, 290 | 0.8, 291 | 0.7777777777777778, 292 | 0.7567567567567568, 293 | 0.7631578947368421, 294 | 0.7692307692307693, 295 | 0.775, 296 | 0.7560975609756098, 297 | 0.7619047619047619, 298 | 0.7674418604651163, 299 | 0.7727272727272727, 300 | 0.7555555555555555, 301 | 0.7391304347826086, 302 | 0.7446808510638298, 303 | 0.75, 304 | 0.7551020408163265, 305 | 0.76, 306 | 0.7647058823529411, 307 | 0.75, 308 | 0.7547169811320755, 309 | 0.7592592592592593, 310 | 0.7454545454545455, 311 | 0.75, 312 | 0.7543859649122807, 313 | 0.7413793103448276, 314 | 0.7457627118644068, 315 | 0.75, 316 | 0.7377049180327869, 317 | 0.7419354838709677, 318 | 0.746031746031746, 319 | 0.75, 320 | 0.7538461538461538, 321 | 0.7575757575757576, 322 | 0.7611940298507462, 323 | 0.75, 324 | 0.7391304347826086, 325 | 0.7285714285714285, 326 | 0.7323943661971831, 327 | 0.7361111111111112, 328 | 0.726027397260274, 329 | 0.7297297297297297, 330 | 0.7333333333333333, 331 | 0.7236842105263158, 332 | 0.7142857142857143, 333 | 0.717948717948718, 334 | 0.7215189873417721, 335 | 0.7125, 336 | 0.7037037037037037, 337 | 0.7073170731707317, 338 | 0.6987951807228916, 339 | 0.6904761904761905, 340 | 0.6823529411764706, 341 | 0.686046511627907, 342 | 0.6896551724137931, 343 | 0.6818181818181818, 344 | 0.6741573033707865, 345 | 0.6666666666666666, 346 | 0.6593406593406593, 347 | 0.6630434782608695, 348 | 0.6559139784946236, 349 | 0.648936170212766, 350 | 0.6421052631578947, 351 | 0.6458333333333334, 352 | 0.6391752577319587, 353 | 0.6326530612244898, 354 | 0.6262626262626263, 355 | 0.62, 356 | 0.6138613861386139, 357 | 0.6078431372549019, 358 | 0.6116504854368932, 359 | 0.6057692307692307, 360 | 0.6, 361 | 0.6037735849056604, 362 | 0.6074766355140186, 363 | 0.6018518518518519, 364 | 0.5963302752293578, 365 | 0.5909090909090909, 366 | 0.5855855855855856, 367 | 0.5803571428571429, 368 | 0.5752212389380531, 369 | 0.5701754385964912, 370 | 0.5652173913043478, 371 | 0.5603448275862069, 372 | 0.5641025641025641, 373 | 0.5677966101694916, 374 | 0.5714285714285714, 375 | 0.5666666666666667, 376 | 0.5619834710743802, 377 | 0.5573770491803278, 378 | 0.5609756097560976, 379 | 0.5564516129032258, 380 | 0.552, 381 | 0.5476190476190477, 382 | 0.5511811023622047, 383 | 0.5546875, 384 | 0.5503875968992248, 385 | 0.5538461538461539, 386 | 0.549618320610687, 387 | 0.553030303030303, 388 | 0.5488721804511278, 389 | 0.5447761194029851, 390 | 0.5407407407407407 391 | ], 392 | "recall": [ 393 | 0.009433962264150943, 394 | 0.018867924528301886, 395 | 0.02830188679245283, 396 | 0.03773584905660377, 397 | 0.04716981132075472, 398 | 0.05660377358490566, 399 | 0.0660377358490566, 400 | 0.07547169811320754, 401 | 0.08490566037735849, 402 | 0.08490566037735849, 403 | 0.09433962264150944, 404 | 0.10377358490566038, 405 | 0.11320754716981132, 406 | 0.11320754716981132, 407 | 0.11320754716981132, 408 | 0.12264150943396226, 409 | 0.1320754716981132, 410 | 0.14150943396226415, 411 | 0.1509433962264151, 412 | 0.16037735849056603, 413 | 0.16981132075471697, 414 | 0.1792452830188679, 415 | 0.18867924528301888, 416 | 0.18867924528301888, 417 | 0.19811320754716982, 418 | 0.19811320754716982, 419 | 0.20754716981132076, 420 | 0.2169811320754717, 421 | 0.22641509433962265, 422 | 0.2358490566037736, 423 | 0.24528301886792453, 424 | 0.24528301886792453, 425 | 0.25471698113207547, 426 | 0.25471698113207547, 427 | 0.2641509433962264, 428 | 0.2641509433962264, 429 | 0.2641509433962264, 430 | 0.27358490566037735, 431 | 0.2830188679245283, 432 | 0.29245283018867924, 433 | 0.29245283018867924, 434 | 0.3018867924528302, 435 | 0.3113207547169811, 436 | 0.32075471698113206, 437 | 0.32075471698113206, 438 | 0.32075471698113206, 439 | 0.330188679245283, 440 | 0.33962264150943394, 441 | 0.3490566037735849, 442 | 0.3584905660377358, 443 | 0.36792452830188677, 444 | 0.36792452830188677, 445 | 0.37735849056603776, 446 | 0.3867924528301887, 447 | 0.3867924528301887, 448 | 0.39622641509433965, 449 | 0.4056603773584906, 450 | 0.4056603773584906, 451 | 0.41509433962264153, 452 | 0.42452830188679247, 453 | 0.42452830188679247, 454 | 0.4339622641509434, 455 | 0.44339622641509435, 456 | 0.4528301886792453, 457 | 0.46226415094339623, 458 | 0.4716981132075472, 459 | 0.4811320754716981, 460 | 0.4811320754716981, 461 | 0.4811320754716981, 462 | 0.4811320754716981, 463 | 0.49056603773584906, 464 | 0.5, 465 | 0.5, 466 | 0.5094339622641509, 467 | 0.5188679245283019, 468 | 0.5188679245283019, 469 | 0.5188679245283019, 470 | 0.5283018867924528, 471 | 0.5377358490566038, 472 | 0.5377358490566038, 473 | 0.5377358490566038, 474 | 0.5471698113207547, 475 | 0.5471698113207547, 476 | 0.5471698113207547, 477 | 0.5471698113207547, 478 | 0.5566037735849056, 479 | 0.5660377358490566, 480 | 0.5660377358490566, 481 | 0.5660377358490566, 482 | 0.5660377358490566, 483 | 0.5660377358490566, 484 | 0.5754716981132075, 485 | 0.5754716981132075, 486 | 0.5754716981132075, 487 | 0.5754716981132075, 488 | 0.5849056603773585, 489 | 0.5849056603773585, 490 | 0.5849056603773585, 491 | 0.5849056603773585, 492 | 0.5849056603773585, 493 | 0.5849056603773585, 494 | 0.5849056603773585, 495 | 0.5943396226415094, 496 | 0.5943396226415094, 497 | 0.5943396226415094, 498 | 0.6037735849056604, 499 | 0.6132075471698113, 500 | 0.6132075471698113, 501 | 0.6132075471698113, 502 | 0.6132075471698113, 503 | 0.6132075471698113, 504 | 0.6132075471698113, 505 | 0.6132075471698113, 506 | 0.6132075471698113, 507 | 0.6132075471698113, 508 | 0.6132075471698113, 509 | 0.6226415094339622, 510 | 0.6320754716981132, 511 | 0.6415094339622641, 512 | 0.6415094339622641, 513 | 0.6415094339622641, 514 | 0.6415094339622641, 515 | 0.6509433962264151, 516 | 0.6509433962264151, 517 | 0.6509433962264151, 518 | 0.6509433962264151, 519 | 0.660377358490566, 520 | 0.6698113207547169, 521 | 0.6698113207547169, 522 | 0.6792452830188679, 523 | 0.6792452830188679, 524 | 0.6886792452830188, 525 | 0.6886792452830188, 526 | 0.6886792452830188, 527 | 0.6886792452830188 528 | ], 529 | "num_groundtruth": 106, 530 | "num_detection": 135, 531 | "tp": 73, 532 | "fp": 62 533 | }, 534 | "coffeetable": { 535 | "class_name": "coffeetable", 536 | "ap": 0.045454545454545456, 537 | "precision": [ 538 | 0.0, 539 | 0.0, 540 | 0.3333333333333333, 541 | 0.5 542 | ], 543 | "recall": [ 544 | 0.0, 545 | 0.0, 546 | 0.045454545454545456, 547 | 0.09090909090909091 548 | ], 549 | "num_groundtruth": 22, 550 | "num_detection": 4, 551 | "tp": 2, 552 | "fp": 2 553 | }, 554 | "countertop": { 555 | "class_name": "countertop", 556 | "ap": 0.19047619047619047, 557 | "precision": [ 558 | 1.0, 559 | 1.0, 560 | 1.0, 561 | 1.0 562 | ], 563 | "recall": [ 564 | 0.047619047619047616, 565 | 0.09523809523809523, 566 | 0.14285714285714285, 567 | 0.19047619047619047 568 | ], 569 | "num_groundtruth": 21, 570 | "num_detection": 4, 571 | "tp": 4, 572 | "fp": 0 573 | }, 574 | "cup": { 575 | "class_name": "cup", 576 | "ap": 0.42500329735623854, 577 | "precision": [ 578 | 1.0, 579 | 1.0, 580 | 1.0, 581 | 1.0, 582 | 1.0, 583 | 1.0, 584 | 1.0, 585 | 1.0, 586 | 1.0, 587 | 0.9, 588 | 0.9090909090909091, 589 | 0.8333333333333334, 590 | 0.8461538461538461, 591 | 0.7857142857142857, 592 | 0.8, 593 | 0.8125, 594 | 0.8235294117647058, 595 | 0.7777777777777778, 596 | 0.7368421052631579, 597 | 0.7, 598 | 0.7142857142857143, 599 | 0.6818181818181818, 600 | 0.6521739130434783, 601 | 0.6666666666666666, 602 | 0.68, 603 | 0.6538461538461539, 604 | 0.6296296296296297 605 | ], 606 | "recall": [ 607 | 0.027777777777777776, 608 | 0.05555555555555555, 609 | 0.08333333333333333, 610 | 0.1111111111111111, 611 | 0.1388888888888889, 612 | 0.16666666666666666, 613 | 0.19444444444444445, 614 | 0.2222222222222222, 615 | 0.25, 616 | 0.25, 617 | 0.2777777777777778, 618 | 0.2777777777777778, 619 | 0.3055555555555556, 620 | 0.3055555555555556, 621 | 0.3333333333333333, 622 | 0.3611111111111111, 623 | 0.3888888888888889, 624 | 0.3888888888888889, 625 | 0.3888888888888889, 626 | 0.3888888888888889, 627 | 0.4166666666666667, 628 | 0.4166666666666667, 629 | 0.4166666666666667, 630 | 0.4444444444444444, 631 | 0.4722222222222222, 632 | 0.4722222222222222, 633 | 0.4722222222222222 634 | ], 635 | "num_groundtruth": 36, 636 | "num_detection": 27, 637 | "tp": 17, 638 | "fp": 10 639 | }, 640 | "diningtable": { 641 | "class_name": "diningtable", 642 | "ap": 0.39655709330302574, 643 | "precision": [ 644 | 1.0, 645 | 1.0, 646 | 1.0, 647 | 1.0, 648 | 1.0, 649 | 0.8333333333333334, 650 | 0.7142857142857143, 651 | 0.625, 652 | 0.6666666666666666, 653 | 0.7, 654 | 0.7272727272727273, 655 | 0.75, 656 | 0.7692307692307693, 657 | 0.7142857142857143, 658 | 0.7333333333333333, 659 | 0.6875, 660 | 0.7058823529411765, 661 | 0.6666666666666666, 662 | 0.631578947368421, 663 | 0.6, 664 | 0.6190476190476191, 665 | 0.5909090909090909, 666 | 0.5652173913043478, 667 | 0.5416666666666666, 668 | 0.52, 669 | 0.5, 670 | 0.5185185185185185, 671 | 0.5, 672 | 0.5172413793103449, 673 | 0.5, 674 | 0.4838709677419355, 675 | 0.5, 676 | 0.5151515151515151, 677 | 0.5294117647058824, 678 | 0.5428571428571428, 679 | 0.5555555555555556, 680 | 0.5405405405405406, 681 | 0.5526315789473685, 682 | 0.5641025641025641, 683 | 0.575, 684 | 0.5853658536585366, 685 | 0.5952380952380952, 686 | 0.5813953488372093, 687 | 0.5909090909090909, 688 | 0.5777777777777777 689 | ], 690 | "recall": [ 691 | 0.02127659574468085, 692 | 0.0425531914893617, 693 | 0.06382978723404255, 694 | 0.0851063829787234, 695 | 0.10638297872340426, 696 | 0.10638297872340426, 697 | 0.10638297872340426, 698 | 0.10638297872340426, 699 | 0.1276595744680851, 700 | 0.14893617021276595, 701 | 0.1702127659574468, 702 | 0.19148936170212766, 703 | 0.2127659574468085, 704 | 0.2127659574468085, 705 | 0.23404255319148937, 706 | 0.23404255319148937, 707 | 0.2553191489361702, 708 | 0.2553191489361702, 709 | 0.2553191489361702, 710 | 0.2553191489361702, 711 | 0.2765957446808511, 712 | 0.2765957446808511, 713 | 0.2765957446808511, 714 | 0.2765957446808511, 715 | 0.2765957446808511, 716 | 0.2765957446808511, 717 | 0.2978723404255319, 718 | 0.2978723404255319, 719 | 0.3191489361702128, 720 | 0.3191489361702128, 721 | 0.3191489361702128, 722 | 0.3404255319148936, 723 | 0.3617021276595745, 724 | 0.3829787234042553, 725 | 0.40425531914893614, 726 | 0.425531914893617, 727 | 0.425531914893617, 728 | 0.44680851063829785, 729 | 0.46808510638297873, 730 | 0.48936170212765956, 731 | 0.5106382978723404, 732 | 0.5319148936170213, 733 | 0.5319148936170213, 734 | 0.5531914893617021, 735 | 0.5531914893617021 736 | ], 737 | "num_groundtruth": 47, 738 | "num_detection": 45, 739 | "tp": 26, 740 | "fp": 19 741 | }, 742 | "doll": { 743 | "class_name": "doll", 744 | "ap": 0.0, 745 | "precision": [], 746 | "recall": [], 747 | "num_groundtruth": 8, 748 | "num_detection": 0, 749 | "tp": 0, 750 | "fp": 0 751 | }, 752 | "door": { 753 | "class_name": "door", 754 | "ap": 0.20689655172413793, 755 | "precision": [ 756 | 1.0, 757 | 1.0, 758 | 1.0, 759 | 1.0, 760 | 1.0, 761 | 1.0 762 | ], 763 | "recall": [ 764 | 0.034482758620689655, 765 | 0.06896551724137931, 766 | 0.10344827586206896, 767 | 0.13793103448275862, 768 | 0.1724137931034483, 769 | 0.20689655172413793 770 | ], 771 | "num_groundtruth": 29, 772 | "num_detection": 6, 773 | "tp": 6, 774 | "fp": 0 775 | }, 776 | "heater": { 777 | "class_name": "heater", 778 | "ap": 0.07692307692307693, 779 | "precision": [ 780 | 1.0, 781 | 0.5 782 | ], 783 | "recall": [ 784 | 0.07692307692307693, 785 | 0.07692307692307693 786 | ], 787 | "num_groundtruth": 13, 788 | "num_detection": 2, 789 | "tp": 1, 790 | "fp": 1 791 | }, 792 | "nightstand": { 793 | "class_name": "nightstand", 794 | "ap": 0.7142857142857143, 795 | "precision": [ 796 | 1.0, 797 | 1.0, 798 | 1.0, 799 | 1.0, 800 | 1.0 801 | ], 802 | "recall": [ 803 | 0.14285714285714285, 804 | 0.2857142857142857, 805 | 0.42857142857142855, 806 | 0.5714285714285714, 807 | 0.7142857142857143 808 | ], 809 | "num_groundtruth": 7, 810 | "num_detection": 5, 811 | "tp": 5, 812 | "fp": 0 813 | }, 814 | "person": { 815 | "class_name": "person", 816 | "ap": 0.42857142857142855, 817 | "precision": [ 818 | 1.0, 819 | 1.0, 820 | 1.0 821 | ], 822 | "recall": [ 823 | 0.14285714285714285, 824 | 0.2857142857142857, 825 | 0.42857142857142855 826 | ], 827 | "num_groundtruth": 7, 828 | "num_detection": 3, 829 | "tp": 3, 830 | "fp": 0 831 | }, 832 | "pictureframe": { 833 | "class_name": "pictureframe", 834 | "ap": 0.17708333333333331, 835 | "precision": [ 836 | 0.0, 837 | 0.5, 838 | 0.6666666666666666, 839 | 0.5, 840 | 0.4, 841 | 0.5, 842 | 0.5714285714285714, 843 | 0.5, 844 | 0.5555555555555556, 845 | 0.5, 846 | 0.5454545454545454, 847 | 0.5833333333333334, 848 | 0.5384615384615384 849 | ], 850 | "recall": [ 851 | 0.0, 852 | 0.041666666666666664, 853 | 0.08333333333333333, 854 | 0.08333333333333333, 855 | 0.08333333333333333, 856 | 0.125, 857 | 0.16666666666666666, 858 | 0.16666666666666666, 859 | 0.20833333333333334, 860 | 0.20833333333333334, 861 | 0.25, 862 | 0.2916666666666667, 863 | 0.2916666666666667 864 | ], 865 | "num_groundtruth": 24, 866 | "num_detection": 13, 867 | "tp": 7, 868 | "fp": 6 869 | }, 870 | "pillow": { 871 | "class_name": "pillow", 872 | "ap": 0.13012345679012347, 873 | "precision": [ 874 | 1.0, 875 | 1.0, 876 | 1.0, 877 | 0.75, 878 | 0.8, 879 | 0.6666666666666666, 880 | 0.5714285714285714, 881 | 0.5, 882 | 0.5555555555555556, 883 | 0.5, 884 | 0.45454545454545453, 885 | 0.4166666666666667, 886 | 0.46153846153846156, 887 | 0.42857142857142855, 888 | 0.4666666666666667, 889 | 0.5 890 | ], 891 | "recall": [ 892 | 0.022222222222222223, 893 | 0.044444444444444446, 894 | 0.06666666666666667, 895 | 0.06666666666666667, 896 | 0.08888888888888889, 897 | 0.08888888888888889, 898 | 0.08888888888888889, 899 | 0.08888888888888889, 900 | 0.1111111111111111, 901 | 0.1111111111111111, 902 | 0.1111111111111111, 903 | 0.1111111111111111, 904 | 0.13333333333333333, 905 | 0.13333333333333333, 906 | 0.15555555555555556, 907 | 0.17777777777777778 908 | ], 909 | "num_groundtruth": 45, 910 | "num_detection": 16, 911 | "tp": 8, 912 | "fp": 8 913 | }, 914 | "pottedplant": { 915 | "class_name": "pottedplant", 916 | "ap": 0.6231254377806101, 917 | "precision": [ 918 | 1.0, 919 | 1.0, 920 | 1.0, 921 | 1.0, 922 | 1.0, 923 | 1.0, 924 | 1.0, 925 | 0.875, 926 | 0.8888888888888888, 927 | 0.9, 928 | 0.9090909090909091, 929 | 0.8333333333333334, 930 | 0.8461538461538461, 931 | 0.8571428571428571, 932 | 0.8, 933 | 0.8125, 934 | 0.8235294117647058, 935 | 0.8333333333333334, 936 | 0.8421052631578947, 937 | 0.85, 938 | 0.8095238095238095, 939 | 0.8181818181818182, 940 | 0.782608695652174, 941 | 0.7916666666666666, 942 | 0.76, 943 | 0.7692307692307693, 944 | 0.7407407407407407, 945 | 0.7142857142857143, 946 | 0.6896551724137931, 947 | 0.6666666666666666 948 | ], 949 | "recall": [ 950 | 0.034482758620689655, 951 | 0.06896551724137931, 952 | 0.10344827586206896, 953 | 0.13793103448275862, 954 | 0.1724137931034483, 955 | 0.20689655172413793, 956 | 0.2413793103448276, 957 | 0.2413793103448276, 958 | 0.27586206896551724, 959 | 0.3103448275862069, 960 | 0.3448275862068966, 961 | 0.3448275862068966, 962 | 0.3793103448275862, 963 | 0.41379310344827586, 964 | 0.41379310344827586, 965 | 0.4482758620689655, 966 | 0.4827586206896552, 967 | 0.5172413793103449, 968 | 0.5517241379310345, 969 | 0.5862068965517241, 970 | 0.5862068965517241, 971 | 0.6206896551724138, 972 | 0.6206896551724138, 973 | 0.6551724137931034, 974 | 0.6551724137931034, 975 | 0.6896551724137931, 976 | 0.6896551724137931, 977 | 0.6896551724137931, 978 | 0.6896551724137931, 979 | 0.6896551724137931 980 | ], 981 | "num_groundtruth": 29, 982 | "num_detection": 30, 983 | "tp": 20, 984 | "fp": 10 985 | }, 986 | "remote": { 987 | "class_name": "remote", 988 | "ap": 0.7321428571428571, 989 | "precision": [ 990 | 1.0, 991 | 1.0, 992 | 1.0, 993 | 1.0, 994 | 1.0, 995 | 0.8333333333333334, 996 | 0.8571428571428571 997 | ], 998 | "recall": [ 999 | 0.125, 1000 | 0.25, 1001 | 0.375, 1002 | 0.5, 1003 | 0.625, 1004 | 0.625, 1005 | 0.75 1006 | ], 1007 | "num_groundtruth": 8, 1008 | "num_detection": 7, 1009 | "tp": 6, 1010 | "fp": 1 1011 | }, 1012 | "shelf": { 1013 | "class_name": "shelf", 1014 | "ap": 0.0, 1015 | "precision": [], 1016 | "recall": [], 1017 | "num_groundtruth": 6, 1018 | "num_detection": 0, 1019 | "tp": 0, 1020 | "fp": 0 1021 | }, 1022 | "sink": { 1023 | "class_name": "sink", 1024 | "ap": 0.16326530612244897, 1025 | "precision": [ 1026 | 0.0, 1027 | 0.5, 1028 | 0.3333333333333333, 1029 | 0.5, 1030 | 0.4, 1031 | 0.5, 1032 | 0.5714285714285714, 1033 | 0.5 1034 | ], 1035 | "recall": [ 1036 | 0.0, 1037 | 0.07142857142857142, 1038 | 0.07142857142857142, 1039 | 0.14285714285714285, 1040 | 0.14285714285714285, 1041 | 0.21428571428571427, 1042 | 0.2857142857142857, 1043 | 0.2857142857142857 1044 | ], 1045 | "num_groundtruth": 14, 1046 | "num_detection": 8, 1047 | "tp": 4, 1048 | "fp": 4 1049 | }, 1050 | "sofa": { 1051 | "class_name": "sofa", 1052 | "ap": 0.9047619047619048, 1053 | "precision": [ 1054 | 1.0, 1055 | 1.0, 1056 | 1.0, 1057 | 1.0, 1058 | 1.0, 1059 | 1.0, 1060 | 1.0, 1061 | 1.0, 1062 | 1.0, 1063 | 1.0, 1064 | 1.0, 1065 | 1.0, 1066 | 1.0, 1067 | 1.0, 1068 | 1.0, 1069 | 1.0, 1070 | 1.0, 1071 | 1.0, 1072 | 1.0, 1073 | 0.95, 1074 | 0.9047619047619048, 1075 | 0.8636363636363636 1076 | ], 1077 | "recall": [ 1078 | 0.047619047619047616, 1079 | 0.09523809523809523, 1080 | 0.14285714285714285, 1081 | 0.19047619047619047, 1082 | 0.23809523809523808, 1083 | 0.2857142857142857, 1084 | 0.3333333333333333, 1085 | 0.38095238095238093, 1086 | 0.42857142857142855, 1087 | 0.47619047619047616, 1088 | 0.5238095238095238, 1089 | 0.5714285714285714, 1090 | 0.6190476190476191, 1091 | 0.6666666666666666, 1092 | 0.7142857142857143, 1093 | 0.7619047619047619, 1094 | 0.8095238095238095, 1095 | 0.8571428571428571, 1096 | 0.9047619047619048, 1097 | 0.9047619047619048, 1098 | 0.9047619047619048, 1099 | 0.9047619047619048 1100 | ], 1101 | "num_groundtruth": 21, 1102 | "num_detection": 22, 1103 | "tp": 19, 1104 | "fp": 3 1105 | }, 1106 | "tap": { 1107 | "class_name": "tap", 1108 | "ap": 0.013888888888888888, 1109 | "precision": [ 1110 | 0.0, 1111 | 0.0, 1112 | 0.0, 1113 | 0.25 1114 | ], 1115 | "recall": [ 1116 | 0.0, 1117 | 0.0, 1118 | 0.0, 1119 | 0.05555555555555555 1120 | ], 1121 | "num_groundtruth": 18, 1122 | "num_detection": 4, 1123 | "tp": 1, 1124 | "fp": 3 1125 | }, 1126 | "tincan": { 1127 | "class_name": "tincan", 1128 | "ap": 0.0, 1129 | "precision": [ 1130 | 0.0 1131 | ], 1132 | "recall": [ 1133 | 0.0 1134 | ], 1135 | "num_groundtruth": 28, 1136 | "num_detection": 1, 1137 | "tp": 0, 1138 | "fp": 1 1139 | }, 1140 | "tvmonitor": { 1141 | "class_name": "tvmonitor", 1142 | "ap": 0.6325, 1143 | "precision": [ 1144 | 1.0, 1145 | 1.0, 1146 | 1.0, 1147 | 1.0, 1148 | 1.0, 1149 | 1.0, 1150 | 1.0, 1151 | 1.0, 1152 | 1.0, 1153 | 1.0, 1154 | 0.9090909090909091, 1155 | 0.9166666666666666, 1156 | 0.8461538461538461, 1157 | 0.8571428571428571, 1158 | 0.8666666666666667, 1159 | 0.8125, 1160 | 0.7647058823529411, 1161 | 0.7222222222222222 1162 | ], 1163 | "recall": [ 1164 | 0.05, 1165 | 0.1, 1166 | 0.15, 1167 | 0.2, 1168 | 0.25, 1169 | 0.3, 1170 | 0.35, 1171 | 0.4, 1172 | 0.45, 1173 | 0.5, 1174 | 0.5, 1175 | 0.55, 1176 | 0.55, 1177 | 0.6, 1178 | 0.65, 1179 | 0.65, 1180 | 0.65, 1181 | 0.65 1182 | ], 1183 | "num_groundtruth": 20, 1184 | "num_detection": 18, 1185 | "tp": 13, 1186 | "fp": 5 1187 | }, 1188 | "vase": { 1189 | "class_name": "vase", 1190 | "ap": 0.1875, 1191 | "precision": [ 1192 | 0.0, 1193 | 0.5, 1194 | 0.6666666666666666, 1195 | 0.75, 1196 | 0.6, 1197 | 0.5, 1198 | 0.42857142857142855, 1199 | 0.375 1200 | ], 1201 | "recall": [ 1202 | 0.0, 1203 | 0.08333333333333333, 1204 | 0.16666666666666666, 1205 | 0.25, 1206 | 0.25, 1207 | 0.25, 1208 | 0.25, 1209 | 0.25 1210 | ], 1211 | "num_groundtruth": 12, 1212 | "num_detection": 8, 1213 | "tp": 3, 1214 | "fp": 5 1215 | }, 1216 | "wastecontainer": { 1217 | "class_name": "wastecontainer", 1218 | "ap": 0.45454545454545453, 1219 | "precision": [ 1220 | 1.0, 1221 | 1.0, 1222 | 1.0, 1223 | 1.0, 1224 | 1.0 1225 | ], 1226 | "recall": [ 1227 | 0.09090909090909091, 1228 | 0.18181818181818182, 1229 | 0.2727272727272727, 1230 | 0.36363636363636365, 1231 | 0.45454545454545453 1232 | ], 1233 | "num_groundtruth": 11, 1234 | "num_detection": 5, 1235 | "tp": 5, 1236 | "fp": 0 1237 | }, 1238 | "windowblind": { 1239 | "class_name": "windowblind", 1240 | "ap": 0.23529411764705882, 1241 | "precision": [ 1242 | 1.0, 1243 | 1.0, 1244 | 1.0, 1245 | 1.0 1246 | ], 1247 | "recall": [ 1248 | 0.058823529411764705, 1249 | 0.11764705882352941, 1250 | 0.17647058823529413, 1251 | 0.23529411764705882 1252 | ], 1253 | "num_groundtruth": 17, 1254 | "num_detection": 4, 1255 | "tp": 4, 1256 | "fp": 0 1257 | }, 1258 | "keyboard": { 1259 | "class_name": "keyboard", 1260 | "ap": NaN, 1261 | "precision": [ 1262 | 0 1263 | ], 1264 | "recall": [ 1265 | NaN 1266 | ], 1267 | "num_groundtruth": 0, 1268 | "num_detection": 1, 1269 | "tp": 0, 1270 | "fp": 1 1271 | }, 1272 | "knife": { 1273 | "class_name": "knife", 1274 | "ap": NaN, 1275 | "precision": [ 1276 | 0 1277 | ], 1278 | "recall": [ 1279 | NaN 1280 | ], 1281 | "num_groundtruth": 0, 1282 | "num_detection": 1, 1283 | "tp": 0, 1284 | "fp": 1 1285 | }, 1286 | "lamp": { 1287 | "class_name": "lamp", 1288 | "ap": NaN, 1289 | "precision": [ 1290 | 0 1291 | ], 1292 | "recall": [ 1293 | NaN 1294 | ], 1295 | "num_groundtruth": 0, 1296 | "num_detection": 1, 1297 | "tp": 0, 1298 | "fp": 1 1299 | }, 1300 | "laptop": { 1301 | "class_name": "laptop", 1302 | "ap": NaN, 1303 | "precision": [ 1304 | 0, 1305 | 0 1306 | ], 1307 | "recall": [ 1308 | NaN, 1309 | NaN 1310 | ], 1311 | "num_groundtruth": 0, 1312 | "num_detection": 2, 1313 | "tp": 0, 1314 | "fp": 2 1315 | }, 1316 | "oven": { 1317 | "class_name": "oven", 1318 | "ap": NaN, 1319 | "precision": [ 1320 | 0, 1321 | 0, 1322 | 0, 1323 | 0 1324 | ], 1325 | "recall": [ 1326 | NaN, 1327 | NaN, 1328 | NaN, 1329 | NaN 1330 | ], 1331 | "num_groundtruth": 0, 1332 | "num_detection": 4, 1333 | "tp": 0, 1334 | "fp": 4 1335 | }, 1336 | "refrigerator": { 1337 | "class_name": "refrigerator", 1338 | "ap": NaN, 1339 | "precision": [ 1340 | 0, 1341 | 0, 1342 | 0, 1343 | 0, 1344 | 0, 1345 | 0, 1346 | 0, 1347 | 0, 1348 | 0, 1349 | 0, 1350 | 0, 1351 | 0, 1352 | 0, 1353 | 0, 1354 | 0, 1355 | 0, 1356 | 0, 1357 | 0, 1358 | 0, 1359 | 0, 1360 | 0, 1361 | 0, 1362 | 0, 1363 | 0, 1364 | 0, 1365 | 0, 1366 | 0, 1367 | 0, 1368 | 0, 1369 | 0, 1370 | 0, 1371 | 0 1372 | ], 1373 | "recall": [ 1374 | NaN, 1375 | NaN, 1376 | NaN, 1377 | NaN, 1378 | NaN, 1379 | NaN, 1380 | NaN, 1381 | NaN, 1382 | NaN, 1383 | NaN, 1384 | NaN, 1385 | NaN, 1386 | NaN, 1387 | NaN, 1388 | NaN, 1389 | NaN, 1390 | NaN, 1391 | NaN, 1392 | NaN, 1393 | NaN, 1394 | NaN, 1395 | NaN, 1396 | NaN, 1397 | NaN, 1398 | NaN, 1399 | NaN, 1400 | NaN, 1401 | NaN, 1402 | NaN, 1403 | NaN, 1404 | NaN, 1405 | NaN 1406 | ], 1407 | "num_groundtruth": 0, 1408 | "num_detection": 32, 1409 | "tp": 0, 1410 | "fp": 32 1411 | }, 1412 | "toilet": { 1413 | "class_name": "toilet", 1414 | "ap": NaN, 1415 | "precision": [ 1416 | 0, 1417 | 0 1418 | ], 1419 | "recall": [ 1420 | NaN, 1421 | NaN 1422 | ], 1423 | "num_groundtruth": 0, 1424 | "num_detection": 2, 1425 | "tp": 0, 1426 | "fp": 2 1427 | }, 1428 | "toothbrush": { 1429 | "class_name": "toothbrush", 1430 | "ap": NaN, 1431 | "precision": [ 1432 | 0 1433 | ], 1434 | "recall": [ 1435 | NaN 1436 | ], 1437 | "num_groundtruth": 0, 1438 | "num_detection": 1, 1439 | "tp": 0, 1440 | "fp": 1 1441 | }, 1442 | "mAP": 0.31047718500906324 1443 | } -------------------------------------------------------------------------------- /tests/sample/groundtruths.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfpeng/object_detection_metrics/3801acc32d052d1fbf1565f96a9f8d390701f911/tests/sample/groundtruths.zip -------------------------------------------------------------------------------- /tests/sample_3/detections.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfpeng/object_detection_metrics/3801acc32d052d1fbf1565f96a9f8d390701f911/tests/sample_3/detections.zip -------------------------------------------------------------------------------- /tests/sample_3/detections_coco.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "image_id": 1, 4 | "category_id": 1, 5 | "bbox": [ 6 | 5, 7 | 67, 8 | 31, 9 | 48 10 | ], 11 | "segmentation": [ 12 | [ 13 | 5, 14 | 67, 15 | 36, 16 | 67, 17 | 36, 18 | 115, 19 | 5, 20 | 115 21 | ] 22 | ], 23 | "iscrowd": 0, 24 | "id": 1, 25 | "score": 0.88 26 | }, 27 | { 28 | "image_id": 1, 29 | "category_id": 1, 30 | "bbox": [ 31 | 119, 32 | 111, 33 | 40, 34 | 67 35 | ], 36 | "segmentation": [ 37 | [ 38 | 119, 39 | 111, 40 | 159, 41 | 111, 42 | 159, 43 | 178, 44 | 119, 45 | 178 46 | ] 47 | ], 48 | "iscrowd": 0, 49 | "id": 2, 50 | "score": 0.7 51 | }, 52 | { 53 | "image_id": 1, 54 | "category_id": 1, 55 | "bbox": [ 56 | 124, 57 | 9, 58 | 49, 59 | 67 60 | ], 61 | "segmentation": [ 62 | [ 63 | 124, 64 | 9, 65 | 173, 66 | 9, 67 | 173, 68 | 76, 69 | 124, 70 | 76 71 | ] 72 | ], 73 | "iscrowd": 0, 74 | "id": 3, 75 | "score": 0.8 76 | }, 77 | { 78 | "image_id": 2, 79 | "category_id": 1, 80 | "bbox": [ 81 | 64, 82 | 111, 83 | 64, 84 | 58 85 | ], 86 | "segmentation": [ 87 | [ 88 | 64, 89 | 111, 90 | 128, 91 | 111, 92 | 128, 93 | 169, 94 | 64, 95 | 169 96 | ] 97 | ], 98 | "iscrowd": 0, 99 | "id": 4, 100 | "score": 0.71 101 | }, 102 | { 103 | "image_id": 2, 104 | "category_id": 1, 105 | "bbox": [ 106 | 26, 107 | 140, 108 | 60, 109 | 47 110 | ], 111 | "segmentation": [ 112 | [ 113 | 26, 114 | 140, 115 | 86, 116 | 140, 117 | 86, 118 | 187, 119 | 26, 120 | 187 121 | ] 122 | ], 123 | "iscrowd": 0, 124 | "id": 5, 125 | "score": 0.54 126 | }, 127 | { 128 | "image_id": 2, 129 | "category_id": 1, 130 | "bbox": [ 131 | 19, 132 | 18, 133 | 43, 134 | 35 135 | ], 136 | "segmentation": [ 137 | [ 138 | 19, 139 | 18, 140 | 62, 141 | 18, 142 | 62, 143 | 53, 144 | 19, 145 | 53 146 | ] 147 | ], 148 | "iscrowd": 0, 149 | "id": 6, 150 | "score": 0.74 151 | }, 152 | { 153 | "image_id": 3, 154 | "category_id": 1, 155 | "bbox": [ 156 | 109, 157 | 15, 158 | 77, 159 | 39 160 | ], 161 | "segmentation": [ 162 | [ 163 | 109, 164 | 15, 165 | 186, 166 | 15, 167 | 186, 168 | 54, 169 | 109, 170 | 54 171 | ] 172 | ], 173 | "iscrowd": 0, 174 | "id": 7, 175 | "score": 0.18 176 | }, 177 | { 178 | "image_id": 3, 179 | "category_id": 1, 180 | "bbox": [ 181 | 86, 182 | 63, 183 | 46, 184 | 45 185 | ], 186 | "segmentation": [ 187 | [ 188 | 86, 189 | 63, 190 | 132, 191 | 63, 192 | 132, 193 | 108, 194 | 86, 195 | 108 196 | ] 197 | ], 198 | "iscrowd": 0, 199 | "id": 8, 200 | "score": 0.67 201 | }, 202 | { 203 | "image_id": 3, 204 | "category_id": 1, 205 | "bbox": [ 206 | 160, 207 | 62, 208 | 36, 209 | 53 210 | ], 211 | "segmentation": [ 212 | [ 213 | 160, 214 | 62, 215 | 196, 216 | 62, 217 | 196, 218 | 115, 219 | 160, 220 | 115 221 | ] 222 | ], 223 | "iscrowd": 0, 224 | "id": 9, 225 | "score": 0.38 226 | }, 227 | { 228 | "image_id": 3, 229 | "category_id": 1, 230 | "bbox": [ 231 | 105, 232 | 131, 233 | 47, 234 | 47 235 | ], 236 | "segmentation": [ 237 | [ 238 | 105, 239 | 131, 240 | 152, 241 | 131, 242 | 152, 243 | 178, 244 | 105, 245 | 178 246 | ] 247 | ], 248 | "iscrowd": 0, 249 | "id": 10, 250 | "score": 0.91 251 | }, 252 | { 253 | "image_id": 3, 254 | "category_id": 1, 255 | "bbox": [ 256 | 18, 257 | 148, 258 | 40, 259 | 44 260 | ], 261 | "segmentation": [ 262 | [ 263 | 18, 264 | 148, 265 | 58, 266 | 148, 267 | 58, 268 | 192, 269 | 18, 270 | 192 271 | ] 272 | ], 273 | "iscrowd": 0, 274 | "id": 11, 275 | "score": 0.44 276 | }, 277 | { 278 | "image_id": 4, 279 | "category_id": 1, 280 | "bbox": [ 281 | 83, 282 | 28, 283 | 28, 284 | 26 285 | ], 286 | "segmentation": [ 287 | [ 288 | 83, 289 | 28, 290 | 111, 291 | 28, 292 | 111, 293 | 54, 294 | 83, 295 | 54 296 | ] 297 | ], 298 | "iscrowd": 0, 299 | "id": 12, 300 | "score": 0.35 301 | }, 302 | { 303 | "image_id": 4, 304 | "category_id": 1, 305 | "bbox": [ 306 | 28, 307 | 68, 308 | 42, 309 | 67 310 | ], 311 | "segmentation": [ 312 | [ 313 | 28, 314 | 68, 315 | 70, 316 | 68, 317 | 70, 318 | 135, 319 | 28, 320 | 135 321 | ] 322 | ], 323 | "iscrowd": 0, 324 | "id": 13, 325 | "score": 0.78 326 | }, 327 | { 328 | "image_id": 4, 329 | "category_id": 1, 330 | "bbox": [ 331 | 87, 332 | 89, 333 | 25, 334 | 39 335 | ], 336 | "segmentation": [ 337 | [ 338 | 87, 339 | 89, 340 | 112, 341 | 89, 342 | 112, 343 | 128, 344 | 87, 345 | 128 346 | ] 347 | ], 348 | "iscrowd": 0, 349 | "id": 14, 350 | "score": 0.45 351 | }, 352 | { 353 | "image_id": 4, 354 | "category_id": 1, 355 | "bbox": [ 356 | 10, 357 | 155, 358 | 60, 359 | 26 360 | ], 361 | "segmentation": [ 362 | [ 363 | 10, 364 | 155, 365 | 70, 366 | 155, 367 | 70, 368 | 181, 369 | 10, 370 | 181 371 | ] 372 | ], 373 | "iscrowd": 0, 374 | "id": 15, 375 | "score": 0.14 376 | }, 377 | { 378 | "image_id": 5, 379 | "category_id": 1, 380 | "bbox": [ 381 | 50, 382 | 38, 383 | 28, 384 | 46 385 | ], 386 | "segmentation": [ 387 | [ 388 | 50, 389 | 38, 390 | 78, 391 | 38, 392 | 78, 393 | 84, 394 | 50, 395 | 84 396 | ] 397 | ], 398 | "iscrowd": 0, 399 | "id": 16, 400 | "score": 0.62 401 | }, 402 | { 403 | "image_id": 5, 404 | "category_id": 1, 405 | "bbox": [ 406 | 95, 407 | 11, 408 | 53, 409 | 28 410 | ], 411 | "segmentation": [ 412 | [ 413 | 95, 414 | 11, 415 | 148, 416 | 11, 417 | 148, 418 | 39, 419 | 95, 420 | 39 421 | ] 422 | ], 423 | "iscrowd": 0, 424 | "id": 17, 425 | "score": 0.44 426 | }, 427 | { 428 | "image_id": 5, 429 | "category_id": 1, 430 | "bbox": [ 431 | 29, 432 | 131, 433 | 72, 434 | 29 435 | ], 436 | "segmentation": [ 437 | [ 438 | 29, 439 | 131, 440 | 101, 441 | 131, 442 | 101, 443 | 160, 444 | 29, 445 | 160 446 | ] 447 | ], 448 | "iscrowd": 0, 449 | "id": 18, 450 | "score": 0.95 451 | }, 452 | { 453 | "image_id": 5, 454 | "category_id": 1, 455 | "bbox": [ 456 | 29, 457 | 163, 458 | 72, 459 | 29 460 | ], 461 | "segmentation": [ 462 | [ 463 | 29, 464 | 163, 465 | 101, 466 | 163, 467 | 101, 468 | 192, 469 | 29, 470 | 192 471 | ] 472 | ], 473 | "iscrowd": 0, 474 | "id": 19, 475 | "score": 0.23 476 | }, 477 | { 478 | "image_id": 6, 479 | "category_id": 1, 480 | "bbox": [ 481 | 43, 482 | 48, 483 | 74, 484 | 38 485 | ], 486 | "segmentation": [ 487 | [ 488 | 43, 489 | 48, 490 | 117, 491 | 48, 492 | 117, 493 | 86, 494 | 43, 495 | 86 496 | ] 497 | ], 498 | "iscrowd": 0, 499 | "id": 20, 500 | "score": 0.45 501 | }, 502 | { 503 | "image_id": 6, 504 | "category_id": 1, 505 | "bbox": [ 506 | 17, 507 | 155, 508 | 29, 509 | 35 510 | ], 511 | "segmentation": [ 512 | [ 513 | 17, 514 | 155, 515 | 46, 516 | 155, 517 | 46, 518 | 190, 519 | 17, 520 | 190 521 | ] 522 | ], 523 | "iscrowd": 0, 524 | "id": 21, 525 | "score": 0.84 526 | }, 527 | { 528 | "image_id": 6, 529 | "category_id": 1, 530 | "bbox": [ 531 | 95, 532 | 110, 533 | 25, 534 | 42 535 | ], 536 | "segmentation": [ 537 | [ 538 | 95, 539 | 110, 540 | 120, 541 | 110, 542 | 120, 543 | 152, 544 | 95, 545 | 152 546 | ] 547 | ], 548 | "iscrowd": 0, 549 | "id": 22, 550 | "score": 0.43 551 | }, 552 | { 553 | "image_id": 7, 554 | "category_id": 1, 555 | "bbox": [ 556 | 16, 557 | 20, 558 | 101, 559 | 88 560 | ], 561 | "segmentation": [ 562 | [ 563 | 16, 564 | 20, 565 | 117, 566 | 20, 567 | 117, 568 | 108, 569 | 16, 570 | 108 571 | ] 572 | ], 573 | "iscrowd": 0, 574 | "id": 23, 575 | "score": 0.48 576 | }, 577 | { 578 | "image_id": 7, 579 | "category_id": 1, 580 | "bbox": [ 581 | 33, 582 | 116, 583 | 37, 584 | 49 585 | ], 586 | "segmentation": [ 587 | [ 588 | 33, 589 | 116, 590 | 70, 591 | 116, 592 | 70, 593 | 165, 594 | 33, 595 | 165 596 | ] 597 | ], 598 | "iscrowd": 0, 599 | "id": 24, 600 | "score": 0.95 601 | } 602 | ] -------------------------------------------------------------------------------- /tests/sample_3/expected0_5.json: -------------------------------------------------------------------------------- 1 | { 2 | "person": { 3 | "ap": 0.0222, 4 | "precision": [ 5 | 0.00, 6 | 0.00, 7 | 0.33, 8 | 0.25, 9 | 0.20, 10 | 0.17, 11 | 0.14, 12 | 0.12, 13 | 0.11, 14 | 0.10, 15 | 0.09, 16 | 0.08, 17 | 0.08, 18 | 0.07, 19 | 0.07, 20 | 0.06, 21 | 0.06, 22 | 0.06, 23 | 0.05, 24 | 0.05, 25 | 0.05, 26 | 0.05, 27 | 0.04, 28 | 0.04 29 | ], 30 | "recall": [ 31 | 0.00, 32 | 0.00, 33 | 0.07, 34 | 0.07, 35 | 0.07, 36 | 0.07, 37 | 0.07, 38 | 0.07, 39 | 0.07, 40 | 0.07, 41 | 0.07, 42 | 0.07, 43 | 0.07, 44 | 0.07, 45 | 0.07, 46 | 0.07, 47 | 0.07, 48 | 0.07, 49 | 0.07, 50 | 0.07, 51 | 0.07, 52 | 0.07, 53 | 0.07, 54 | 0.07 55 | ] 56 | } 57 | } -------------------------------------------------------------------------------- /tests/sample_3/groundtruths.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfpeng/object_detection_metrics/3801acc32d052d1fbf1565f96a9f8d390701f911/tests/sample_3/groundtruths.zip -------------------------------------------------------------------------------- /tests/sample_3/groundtruths_coco.json: -------------------------------------------------------------------------------- 1 | { 2 | "licenses": [ 3 | { 4 | "url": "", 5 | "name": "", 6 | "id": 0 7 | } 8 | ], 9 | "info": { 10 | "contributor": "", 11 | "description": "", 12 | "url": "", 13 | "date_created": "", 14 | "version": "", 15 | "year": "" 16 | }, 17 | "annotations": [ 18 | { 19 | "image_id": 1, 20 | "area": 2128, 21 | "category_id": 1, 22 | "bbox": [ 23 | 25, 24 | 16, 25 | 38, 26 | 56 27 | ], 28 | "segmentation": [ 29 | [ 30 | 25, 31 | 16, 32 | 63, 33 | 16, 34 | 63, 35 | 72, 36 | 25, 37 | 72 38 | ] 39 | ], 40 | "iscrowd": false, 41 | "id": 1 42 | }, 43 | { 44 | "image_id": 1, 45 | "area": 2542, 46 | "category_id": 1, 47 | "bbox": [ 48 | 129, 49 | 123, 50 | 41, 51 | 62 52 | ], 53 | "segmentation": [ 54 | [ 55 | 129, 56 | 123, 57 | 170, 58 | 123, 59 | 170, 60 | 185, 61 | 129, 62 | 185 63 | ] 64 | ], 65 | "iscrowd": false, 66 | "id": 2 67 | }, 68 | { 69 | "image_id": 2, 70 | "area": 2365, 71 | "category_id": 1, 72 | "bbox": [ 73 | 123, 74 | 11, 75 | 43, 76 | 55 77 | ], 78 | "segmentation": [ 79 | [ 80 | 123, 81 | 11, 82 | 166, 83 | 11, 84 | 166, 85 | 66, 86 | 123, 87 | 66 88 | ] 89 | ], 90 | "iscrowd": false, 91 | "id": 3 92 | }, 93 | { 94 | "image_id": 2, 95 | "area": 2655, 96 | "category_id": 1, 97 | "bbox": [ 98 | 38, 99 | 132, 100 | 59, 101 | 45 102 | ], 103 | "segmentation": [ 104 | [ 105 | 38, 106 | 132, 107 | 97, 108 | 132, 109 | 97, 110 | 177, 111 | 38, 112 | 177 113 | ] 114 | ], 115 | "iscrowd": false, 116 | "id": 4 117 | }, 118 | { 119 | "image_id": 3, 120 | "area": 1680, 121 | "category_id": 1, 122 | "bbox": [ 123 | 16, 124 | 14, 125 | 35, 126 | 48 127 | ], 128 | "segmentation": [ 129 | [ 130 | 16, 131 | 14, 132 | 51, 133 | 14, 134 | 51, 135 | 62, 136 | 16, 137 | 62 138 | ] 139 | ], 140 | "iscrowd": false, 141 | "id": 5 142 | }, 143 | { 144 | "image_id": 3, 145 | "area": 2156, 146 | "category_id": 1, 147 | "bbox": [ 148 | 123, 149 | 30, 150 | 49, 151 | 44 152 | ], 153 | "segmentation": [ 154 | [ 155 | 123, 156 | 30, 157 | 172, 158 | 30, 159 | 172, 160 | 74, 161 | 123, 162 | 74 163 | ] 164 | ], 165 | "iscrowd": false, 166 | "id": 6 167 | }, 168 | { 169 | "image_id": 3, 170 | "area": 2209, 171 | "category_id": 1, 172 | "bbox": [ 173 | 99, 174 | 139, 175 | 47, 176 | 47 177 | ], 178 | "segmentation": [ 179 | [ 180 | 99, 181 | 139, 182 | 146, 183 | 139, 184 | 146, 185 | 186, 186 | 99, 187 | 186 188 | ] 189 | ], 190 | "iscrowd": false, 191 | "id": 7 192 | }, 193 | { 194 | "image_id": 4, 195 | "area": 2080, 196 | "category_id": 1, 197 | "bbox": [ 198 | 53, 199 | 42, 200 | 40, 201 | 52 202 | ], 203 | "segmentation": [ 204 | [ 205 | 53, 206 | 42, 207 | 93, 208 | 42, 209 | 93, 210 | 94, 211 | 53, 212 | 94 213 | ] 214 | ], 215 | "iscrowd": false, 216 | "id": 8 217 | }, 218 | { 219 | "image_id": 4, 220 | "area": 1054, 221 | "category_id": 1, 222 | "bbox": [ 223 | 154, 224 | 43, 225 | 31, 226 | 34 227 | ], 228 | "segmentation": [ 229 | [ 230 | 154, 231 | 43, 232 | 185, 233 | 43, 234 | 185, 235 | 77, 236 | 154, 237 | 77 238 | ] 239 | ], 240 | "iscrowd": false, 241 | "id": 9 242 | }, 243 | { 244 | "image_id": 5, 245 | "area": 2244, 246 | "category_id": 1, 247 | "bbox": [ 248 | 59, 249 | 31, 250 | 44, 251 | 51 252 | ], 253 | "segmentation": [ 254 | [ 255 | 59, 256 | 31, 257 | 103, 258 | 31, 259 | 103, 260 | 82, 261 | 59, 262 | 82 263 | ] 264 | ], 265 | "iscrowd": false, 266 | "id": 10 267 | }, 268 | { 269 | "image_id": 5, 270 | "area": 1768, 271 | "category_id": 1, 272 | "bbox": [ 273 | 48, 274 | 128, 275 | 34, 276 | 52 277 | ], 278 | "segmentation": [ 279 | [ 280 | 48, 281 | 128, 282 | 82, 283 | 128, 284 | 82, 285 | 180, 286 | 48, 287 | 180 288 | ] 289 | ], 290 | "iscrowd": false, 291 | "id": 11 292 | }, 293 | { 294 | "image_id": 6, 295 | "area": 3952, 296 | "category_id": 1, 297 | "bbox": [ 298 | 36, 299 | 89, 300 | 52, 301 | 76 302 | ], 303 | "segmentation": [ 304 | [ 305 | 36, 306 | 89, 307 | 88, 308 | 89, 309 | 88, 310 | 165, 311 | 36, 312 | 165 313 | ] 314 | ], 315 | "iscrowd": false, 316 | "id": 12 317 | }, 318 | { 319 | "image_id": 6, 320 | "area": 2948, 321 | "category_id": 1, 322 | "bbox": [ 323 | 62, 324 | 58, 325 | 44, 326 | 67 327 | ], 328 | "segmentation": [ 329 | [ 330 | 62, 331 | 58, 332 | 106, 333 | 58, 334 | 106, 335 | 125, 336 | 62, 337 | 125 338 | ] 339 | ], 340 | "iscrowd": false, 341 | "id": 13 342 | }, 343 | { 344 | "image_id": 7, 345 | "area": 3465, 346 | "category_id": 1, 347 | "bbox": [ 348 | 28, 349 | 31, 350 | 55, 351 | 63 352 | ], 353 | "segmentation": [ 354 | [ 355 | 28, 356 | 31, 357 | 83, 358 | 31, 359 | 83, 360 | 94, 361 | 28, 362 | 94 363 | ] 364 | ], 365 | "iscrowd": false, 366 | "id": 14 367 | }, 368 | { 369 | "image_id": 7, 370 | "area": 2900, 371 | "category_id": 1, 372 | "bbox": [ 373 | 58, 374 | 67, 375 | 50, 376 | 58 377 | ], 378 | "segmentation": [ 379 | [ 380 | 58, 381 | 67, 382 | 108, 383 | 67, 384 | 108, 385 | 125, 386 | 58, 387 | 125 388 | ] 389 | ], 390 | "iscrowd": false, 391 | "id": 15 392 | } 393 | ], 394 | "images": [ 395 | { 396 | "width": 0, 397 | "flickr_url": "", 398 | "coco_url": "", 399 | "file_name": "00001.txt.jpg", 400 | "date_captured": 0, 401 | "license": 0, 402 | "id": 1, 403 | "height": 0 404 | }, 405 | { 406 | "width": 0, 407 | "flickr_url": "", 408 | "coco_url": "", 409 | "file_name": "00002.txt.jpg", 410 | "date_captured": 0, 411 | "license": 0, 412 | "id": 2, 413 | "height": 0 414 | }, 415 | { 416 | "width": 0, 417 | "flickr_url": "", 418 | "coco_url": "", 419 | "file_name": "00003.txt.jpg", 420 | "date_captured": 0, 421 | "license": 0, 422 | "id": 3, 423 | "height": 0 424 | }, 425 | { 426 | "width": 0, 427 | "flickr_url": "", 428 | "coco_url": "", 429 | "file_name": "00004.txt.jpg", 430 | "date_captured": 0, 431 | "license": 0, 432 | "id": 4, 433 | "height": 0 434 | }, 435 | { 436 | "width": 0, 437 | "flickr_url": "", 438 | "coco_url": "", 439 | "file_name": "00005.txt.jpg", 440 | "date_captured": 0, 441 | "license": 0, 442 | "id": 5, 443 | "height": 0 444 | }, 445 | { 446 | "width": 0, 447 | "flickr_url": "", 448 | "coco_url": "", 449 | "file_name": "00006.txt.jpg", 450 | "date_captured": 0, 451 | "license": 0, 452 | "id": 6, 453 | "height": 0 454 | }, 455 | { 456 | "width": 0, 457 | "flickr_url": "", 458 | "coco_url": "", 459 | "file_name": "00007.txt.jpg", 460 | "date_captured": 0, 461 | "license": 0, 462 | "id": 7, 463 | "height": 0 464 | } 465 | ], 466 | "categories": [ 467 | { 468 | "supercategory": "", 469 | "name": "person", 470 | "id": 1 471 | } 472 | ] 473 | } -------------------------------------------------------------------------------- /tests/test_box.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from podm import box 4 | from podm.box import Box 5 | import math 6 | 7 | 8 | def test_iou(): 9 | box1 = Box.of_box(0., 0., 10., 10.) 10 | box2 = Box.of_box(1., 1., 11., 11.) 11 | assert math.isclose(box.intersection_over_union(box1, box2), 0.680672268908, rel_tol=1e-6) 12 | 13 | box1 = Box.of_box(0. / 100, 0. / 100, 10. / 100, 10. / 100) 14 | box2 = Box.of_box(1. / 100, 1. / 100, 11. / 100, 11. / 100) 15 | assert math.isclose(box.intersection_over_union(box1, box2), 0.680672268908, rel_tol=1e-6) 16 | 17 | # no overlap 18 | box1 = Box.of_box(0., 0., 10., 10.) 19 | box2 = Box.of_box(12., 12., 22., 22.) 20 | assert box.intersection_over_union(box1, box2) == 0 21 | 22 | box1 = Box.of_box(0., 0., 2., 2.) 23 | box2 = Box.of_box(1., 1., 3., 3.) 24 | assert math.isclose(box.intersection_over_union(box1, box2), 0.142857142857, rel_tol=1e-6) 25 | 26 | 27 | def test_union(): 28 | box1 = Box.of_box(0., 0., 10., 10.) 29 | box2 = Box.of_box(1., 1., 11., 11.) 30 | assert box.union(box1, box2) == Box.of_box(0, 0, 11, 11) 31 | assert box.union_areas(box1, box2) == 119 32 | 33 | 34 | def test_box(): 35 | box = Box.of_box(0., 0., 10., 10.) 36 | assert box.width == 10 37 | assert box.height == 10 38 | assert box.area == 100 39 | assert box == Box.of_box(0, 0, 10, 10) 40 | assert box != Box.of_box(0, 0, 11, 11) 41 | assert box != 1 42 | 43 | with pytest.raises(AssertionError): 44 | Box.of_box(0., 0., -1, 10.) 45 | -------------------------------------------------------------------------------- /tests/test_coco.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from podm import coco, coco_decoder 4 | from podm.box import Box 5 | 6 | 7 | @pytest.fixture 8 | def dataset(): 9 | dataset = coco.PCOCOObjectDetectionDataset() 10 | 11 | for i in range(0, 10): 12 | cat = coco.PCOCOCategory() 13 | cat.id = i 14 | cat.name = str(i) 15 | cat.supercategory = 's%d' % i 16 | dataset.add_category(cat) 17 | 18 | for i in range(0, 10): 19 | img = coco.PCOCOImage() 20 | img.id = i 21 | img.file_name = str(i) 22 | dataset.add_image(img) 23 | 24 | for i in range(0, 10): 25 | ann = coco.PCOCOBoundingBox() 26 | ann.id = i 27 | ann.image_id = i 28 | ann.category_id = i 29 | ann.set_box(Box.of_box(0, 0, 10, 10)) 30 | dataset.add_annotation(ann) 31 | 32 | return dataset 33 | 34 | 35 | def test_cat(dataset): 36 | for i in range(0, 10): 37 | assert dataset.get_category(id=i).name == str(i) 38 | assert dataset.get_category(name=str(i)).id == i 39 | 40 | assert dataset.get_category(id=-1) is None 41 | assert dataset.get_category(name="-1") is None 42 | 43 | assert dataset.get_max_category_id() == 9 44 | 45 | cat_ids = dataset.get_category_ids(category_names=['1', '2', '-1']) 46 | assert 1 in cat_ids 47 | assert 2 in cat_ids 48 | assert 3 not in cat_ids 49 | 50 | cat_ids = dataset.get_category_ids(supercategory_names=['s1', 's2', '3']) 51 | assert 1 in cat_ids 52 | assert 2 in cat_ids 53 | assert 3 not in cat_ids 54 | 55 | cats = dataset.get_categories([1, 2]) 56 | assert sorted(list(cat.id for cat in cats)) == [1, 2] 57 | 58 | with pytest.raises(KeyError): 59 | cat = coco.PCOCOCategory() 60 | cat.id = 0 61 | dataset.add_category(cat) 62 | 63 | with pytest.raises(KeyError): 64 | dataset.get_category() 65 | 66 | with pytest.raises(KeyError): 67 | dataset.get_category(id=0, name='0') 68 | 69 | 70 | def test_ann(dataset): 71 | for i in range(0, 10): 72 | assert dataset.get_annotation(id=i).id == i 73 | 74 | assert dataset.get_annotation(id=-1) is None 75 | 76 | ann_ids = dataset.get_annotation_ids(image_ids=[1, -1]) 77 | assert 1 in ann_ids 78 | assert 2 not in ann_ids 79 | assert -1 not in ann_ids 80 | 81 | ann_ids = dataset.get_annotation_ids(area_range=(100, 100)) 82 | assert len(ann_ids) == 10 83 | 84 | ann_ids = dataset.get_annotation_ids(area_range=(99, 99)) 85 | assert len(ann_ids) == 0 86 | 87 | ann_ids = dataset.get_annotation_ids(area_range=(0, 110)) 88 | assert len(ann_ids) == 10 89 | 90 | ann_ids = dataset.get_annotation_ids(category_ids=[1, 2, -1]) 91 | assert 1 in ann_ids 92 | assert 2 in ann_ids 93 | assert -1 not in ann_ids 94 | 95 | ann_ids = dataset.get_annotation_ids(image_ids=[1], category_ids=[1, 2]) 96 | assert 1 in ann_ids 97 | assert 2 not in ann_ids 98 | 99 | anns = dataset.get_annotations([1, 2]) 100 | assert sorted(list(ann.id for ann in anns)) == [1, 2] 101 | 102 | with pytest.raises(KeyError): 103 | ann = coco.PCOCOBoundingBox() 104 | ann.id = 0 105 | dataset.add_annotation(ann) 106 | 107 | 108 | def test_image(dataset): 109 | for i in range(0, 10): 110 | assert dataset.get_image(id=i).file_name == str(i) 111 | assert dataset.get_image(file_name=str(i)).id == i 112 | 113 | assert dataset.get_image(id=-1) is None 114 | assert dataset.get_image(file_name='-1') is None 115 | 116 | img_ids = {img.id for img in dataset.get_images([0, 1, -2])} 117 | assert 0 in img_ids 118 | assert 1 in img_ids 119 | assert -2 not in img_ids 120 | 121 | img_ids = dataset.get_image_ids(category_ids=[1]) 122 | assert 1 in img_ids 123 | 124 | img_ids = dataset.get_image_ids(category_ids=[1, 2]) 125 | assert len(img_ids) == 0 126 | 127 | with pytest.raises(KeyError): 128 | img = coco.PCOCOImage() 129 | img.id = 0 130 | dataset.add_image(img) 131 | 132 | with pytest.raises(KeyError): 133 | dataset.get_image() 134 | 135 | with pytest.raises(KeyError): 136 | dataset.get_image(id=0, file_name='0') 137 | 138 | 139 | def test_gets(dataset): 140 | # get all images containing given categories, select one at random 141 | cat_ids = dataset.get_category_ids(category_names=['1', '2']) 142 | assert len(cat_ids) == 2 143 | assert 1 in cat_ids 144 | assert 2 in cat_ids 145 | 146 | ann_ids = dataset.get_annotation_ids(category_ids=cat_ids) 147 | assert len(ann_ids) == 2 148 | assert 1 in ann_ids 149 | assert 2 in ann_ids 150 | 151 | 152 | def test_segments(): 153 | segments = coco.PCOCOSegments() 154 | segments.add_box(Box.of_box(0, 0, 10, 10)) 155 | segments.add_box(Box.of_box(1, 1, 11, 11)) 156 | assert segments.bbox == Box.of_box(0, 0, 11, 11) 157 | 158 | segments.add_segmentation(Box.of_box(2, 2, 12, 12).segment) 159 | assert segments.bbox == Box.of_box(0, 0, 12, 12) 160 | 161 | segments = coco.PCOCOSegments() 162 | assert segments.bbox is None 163 | -------------------------------------------------------------------------------- /tests/test_coco_decoder.py: -------------------------------------------------------------------------------- 1 | from podm import coco_decoder 2 | 3 | 4 | def test_load_true(sample_dir): 5 | with open(sample_dir / 'groundtruths_coco.json') as fp: 6 | dataset = coco_decoder.load_true_object_detection_dataset(fp) 7 | 8 | assert len(dataset.images) == 85 9 | assert len(dataset.categories) == 38 10 | assert len(dataset.annotations) == 686 11 | 12 | 13 | def test_load_pred(sample_dir): 14 | with open(sample_dir / 'groundtruths_coco.json') as fp: 15 | gold_dataset = coco_decoder.load_true_object_detection_dataset(fp) 16 | with open(sample_dir / 'detections_coco.json') as fp: 17 | pred_dataset = coco_decoder.load_pred_object_detection_dataset(fp, gold_dataset) 18 | 19 | assert len(gold_dataset.images) == 85 20 | assert len(gold_dataset.categories) == 38 21 | assert len(gold_dataset.annotations) == 686 22 | 23 | assert len(pred_dataset.images) == 85 24 | assert len(pred_dataset.categories) == 38 25 | assert len(pred_dataset.annotations) == 494 26 | 27 | 28 | # def test_sample3(tests_dir): 29 | # with open(tests_dir / 'sample_3/groundtruths_coco.json') as fp: 30 | # dataset = pcoco_decoder.load_true_bounding_box_dataset(fp) 31 | # 32 | # assert len(dataset.images) == 7 33 | # assert len(dataset.categories) == 1 34 | # assert len(dataset.annotations) == 15 35 | # 36 | # 37 | # def test_sample3_result(tests_dir): 38 | # with open(tests_dir / 'sample_3/groundtruths_coco.json') as fp: 39 | # gold_dataset = pcoco_decoder.load_true_bounding_box_dataset(fp) 40 | # with open(tests_dir / 'sample_3/detections_coco.json') as fp: 41 | # pred_dataset = pcoco_decoder.load_pred_bounding_box_dataset(fp, gold_dataset) 42 | # 43 | # assert len(gold_dataset.images) == 7 44 | # assert len(gold_dataset.categories) == 1 45 | # assert len(gold_dataset.annotations) == 15 46 | # 47 | # assert len(pred_dataset.images) == 7 48 | # assert len(pred_dataset.categories) == 1 49 | # assert len(pred_dataset.annotations) == 24 50 | -------------------------------------------------------------------------------- /tests/test_coco_encoder.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | import pytest 4 | 5 | from podm import coco_decoder, coco_encoder, coco 6 | from podm.box import Box 7 | 8 | 9 | @pytest.fixture 10 | def dataset(): 11 | dataset = coco.PCOCOObjectDetectionDataset() 12 | for i in range(0, 2): 13 | lic = coco.PCOCOLicense() 14 | lic.id = i 15 | lic.name = str(i) 16 | dataset.add_license(lic) 17 | 18 | for i in range(0, 10): 19 | cat = coco.PCOCOCategory() 20 | cat.id = i 21 | cat.name = str(i) 22 | cat.supercategory = 's%d' % i 23 | dataset.add_category(cat) 24 | 25 | for i in range(0, 10): 26 | img = coco.PCOCOImage() 27 | img.id = i 28 | img.file_name = str(i) 29 | dataset.add_image(img) 30 | 31 | for i in range(0, 10): 32 | ann = coco.PCOCOBoundingBox() 33 | ann.id = i 34 | ann.image_id = i 35 | ann.category_id = i 36 | ann.set_box(Box.of_box(0, 0, 10, 10)) 37 | dataset.add_annotation(ann) 38 | 39 | return dataset 40 | 41 | 42 | def test_load(dataset, tmp_path): 43 | tmp_file = tmp_path / 'foo.json' 44 | with open(tmp_file, 'w') as fp: 45 | coco_encoder.dump(dataset, fp) 46 | with open(tmp_file) as fp: 47 | dataset = coco_decoder.load_true_object_detection_dataset(fp) 48 | 49 | assert len(dataset.images) == 10 50 | assert len(dataset.categories) == 10 51 | assert len(dataset.annotations) == 10 52 | assert len(dataset.licenses) == 2 53 | 54 | 55 | def test_loads(dataset): 56 | s = coco_encoder.dumps(dataset) 57 | dataset = coco_decoder.load_true_object_detection_dataset(io.StringIO(s)) 58 | 59 | assert len(dataset.images) == 10 60 | assert len(dataset.categories) == 10 61 | assert len(dataset.annotations) == 10 62 | assert len(dataset.licenses) == 2 63 | 64 | -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from podm.metrics import calculate_11_points_average_precision, calculate_all_points_average_precision 4 | 5 | 6 | def test_calculate_11_points_average_precision(): 7 | recall = 0.5 8 | precision = 1 9 | ap, r_values, p_values = calculate_11_points_average_precision([recall], [precision]) 10 | assert np.allclose(ap, 0.545, rtol=1e-3, equal_nan=True) 11 | assert np.allclose(r_values, [0, 0, .1, .2, .3, .4, .5, .5, .6, .7, .8, .9, 1], rtol=1e-1) 12 | assert np.allclose(p_values, [0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0], rtol=1e-1) 13 | 14 | 15 | def test_calculate_all_points_average_precision(): 16 | recall = 0.5 17 | precision = 1 18 | ap, r_values, p_values, ii = calculate_all_points_average_precision([recall], [precision]) 19 | assert np.allclose(ap, 0.5, rtol=1e-1, equal_nan=True) 20 | assert np.allclose(r_values, [0, 0.5], rtol=1e-1) 21 | assert np.allclose(p_values, [1, 1], rtol=1e-1) -------------------------------------------------------------------------------- /tests/test_pascal2coco.py: -------------------------------------------------------------------------------- 1 | from podm.pascal2coco import PascalVoc2COCO 2 | 3 | 4 | def test_pascal2coco(sample_dir): 5 | converter = PascalVoc2COCO() 6 | dataset = converter.convert_gold(sample_dir / 'groundtruths.zip') 7 | assert len(dataset.images) == 85 8 | 9 | gold_dataset, pred_dataset = converter.convert_gold_pred(sample_dir / 'groundtruths.zip', 10 | sample_dir / 'detections.zip') 11 | assert len(gold_dataset.images) == 85 12 | assert len(gold_dataset.annotations) == 686 13 | 14 | assert len(pred_dataset.images) == 85 15 | assert len(pred_dataset.annotations) == 494 - 44 16 | -------------------------------------------------------------------------------- /tests/test_pascal_voc_metrics.py: -------------------------------------------------------------------------------- 1 | import json 2 | import math 3 | from pathlib import Path 4 | 5 | from helpers.utils import assert_results 6 | from podm import coco_decoder 7 | from podm.metrics import get_pascal_voc_metrics, MetricPerClass, get_bounding_boxes 8 | 9 | 10 | def _get_dataset_helper(sample_dir): 11 | with open(sample_dir / 'groundtruths_coco.json') as fp: 12 | gold_dataset = coco_decoder.load_true_object_detection_dataset(fp) 13 | with open(sample_dir / 'detections_coco.json') as fp: 14 | pred_dataset = coco_decoder.load_pred_object_detection_dataset(fp, gold_dataset) 15 | RESULT0_5 = json.load(open(sample_dir / 'expected0_5.json')) 16 | return get_bounding_boxes(gold_dataset), get_bounding_boxes(pred_dataset), RESULT0_5 17 | 18 | 19 | def test_sample2(sample_dir): 20 | gold_dataset, pred_dataset, expects = _get_dataset_helper(sample_dir) 21 | 22 | results = get_pascal_voc_metrics(gold_dataset, pred_dataset, .5) 23 | assert_results(results, expects, 'ap') 24 | assert_results(results, expects, 'precision') 25 | assert_results(results, expects, 'recall') 26 | assert_results(results, expects, 'tp') 27 | assert_results(results, expects, 'fp') 28 | assert_results(results, expects, 'num_groundtruth') 29 | assert_results(results, expects, 'num_detection') 30 | 31 | mAP = MetricPerClass.mAP(results) 32 | assert math.isclose(expects['mAP'], mAP, rel_tol=1e-3), '{} vs {}'.format(expects['mAP'], mAP) 33 | 34 | 35 | def test_sample3(tests_dir): 36 | dir = tests_dir / 'sample_3' 37 | gold_dataset, pred_dataset, expects = _get_dataset_helper(dir) 38 | 39 | results = get_pascal_voc_metrics(gold_dataset, pred_dataset, .5) 40 | assert_results(results, expects, 'ap') 41 | assert_results(results, expects, 'precision') 42 | assert_results(results, expects, 'recall') 43 | 44 | RESULT0_3 = { 45 | 'person': { 46 | 'ap': 0.245687 47 | } 48 | } 49 | results = get_pascal_voc_metrics(gold_dataset, pred_dataset, .3) 50 | assert_results(results, RESULT0_3, 'ap') 51 | 52 | 53 | if __name__ == '__main__': 54 | test_sample2(Path(__file__).parent) 55 | -------------------------------------------------------------------------------- /tests/test_plot.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from podm import coco_decoder 4 | from podm.metrics import get_pascal_voc_metrics, get_bounding_boxes 5 | from podm.visualize import plot_precision_recall_curve_all 6 | 7 | # @pytest.mark.skip() 8 | def test_plot(sample_dir, tmp_path): 9 | with open(sample_dir / 'groundtruths_coco.json') as fp: 10 | gold_dataset = coco_decoder.load_true_object_detection_dataset(fp) 11 | with open(sample_dir / 'detections_coco.json') as fp: 12 | pred_dataset = coco_decoder.load_pred_object_detection_dataset(fp, gold_dataset) 13 | results = get_pascal_voc_metrics(get_bounding_boxes(gold_dataset), get_bounding_boxes(pred_dataset), .5) 14 | plot_precision_recall_curve_all(results, tmp_path, show_interpolated_precision=True) 15 | 16 | -------------------------------------------------------------------------------- /tests/test_pycocotools.py: -------------------------------------------------------------------------------- 1 | import math 2 | from pycocotools.coco import COCO 3 | from pycocotools.cocoeval import COCOeval 4 | 5 | 6 | def test_pycocotools(sample_dir): 7 | coco = COCO(str(sample_dir / 'groundtruths_coco.json')) 8 | cats = coco.loadCats(coco.getCatIds()) 9 | assert len(cats) == 38 10 | 11 | # get all images containing given categories, select one at random 12 | catIds = coco.getCatIds(catNms=['person']) 13 | assert catIds[0] == 26 14 | 15 | annIds = coco.getAnnIds(catIds=catIds) 16 | assert len(annIds) == 7 17 | 18 | imgIds = coco.getImgIds(catIds=catIds) 19 | assert len(imgIds) == 6 20 | 21 | annIds = coco.getAnnIds(imgIds=imgIds[1], catIds=catIds, iscrowd=None) 22 | assert len(annIds) == 1 23 | 24 | 25 | def test_cocoeval(sample_dir): 26 | coco_gld = COCO(str(sample_dir / 'groundtruths_coco.json')) 27 | coco_rst = coco_gld.loadRes(str(sample_dir / 'detections_coco.json')) 28 | 29 | cocoEval = COCOeval(coco_gld, coco_rst, iouType='bbox') 30 | cocoEval.evaluate() 31 | cocoEval.accumulate() 32 | cocoEval.summarize() 33 | assert math.isclose(cocoEval.stats[1], 0.31195, rel_tol=1e-2) 34 | assert math.isclose(cocoEval.stats[-1], 0.307, rel_tol=1e-2) 35 | --------------------------------------------------------------------------------