├── src
    └── phenobench
    │   ├── evaluation
    │       ├── __init__.py
    │       ├── auxiliary
    │       │   ├── __init__.py
    │       │   ├── common.py
    │       │   ├── filter.py
    │       │   ├── convert.py
    │       │   └── panoptic_eval.py
    │       ├── phenobench_eval.py
    │       ├── evaluate_semantics.py
    │       ├── evaluate_leaf_instance_masks_panoptic.py
    │       ├── evaluate_leaf_bounding_boxes.py
    │       ├── evaluate_plant_bounding_boxes.py
    │       └── evaluate_plant_instance_masks_panoptic.py
    │   ├── __init__.py
    │   ├── tools
    │       └── validator.py
    │   ├── visualization.py
    │   └── phenobench_loader.py
├── .gitattributes
├── CITATION.cff
├── LICENSE
├── pyproject.toml
├── .gitignore
├── README.md
└── phenobench_tutorial.ipynb


/src/phenobench/evaluation/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/auxiliary/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/src/phenobench/__init__.py:
--------------------------------------------------------------------------------
1 | from .phenobench_loader import PhenoBench


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.ipynb filter=nbstripout
2 | *.zpln filter=nbstripout
3 | *.ipynb diff=ipynb
4 | 


--------------------------------------------------------------------------------
/CITATION.cff:
--------------------------------------------------------------------------------
 1 | cff-version: 1.2.0
 2 | preferred-citation:
 3 |   title: "PhenoBench — A Large Dataset and Benchmarks for Semantic Image Interpretation in the Agricultural Domain"
 4 |   year: "2023"
 5 |   type: article
 6 |   journal: "arXiv"
 7 |   url: https://arxiv.org/pdf/2306.04557.pdf
 8 |   codeurl: https://github.com/PRBonn/phenobench-devkit
 9 |   authors:
10 |     - family-names: Weyler
11 |       given-names: Jan
12 |     - family-names: Magistri
13 |       given-names: Federico
14 |     - family-names: Marks
15 |       given-names: Elias
16 |     - family-names: Chong
17 |       given-names: Yue Linn
18 |     - family-names: Sodano
19 |       given-names: Matteo
20 |     - family-names: Roggiolani
21 |       given-names: Gianmarco
22 |     - family-names: Chebrolu
23 |       given-names: Nived
24 |     - family-names: Stachniss
25 |       given-names: Cyrill
26 |     - family-names: Behley
27 |       given-names: Jens


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2023 Jan Weyler, Federico Magistri, Elias Marks, Yue Linn Chong, 
 4 | Matteo Sodano, Gianmarco Roggiolani, Nived Chebrolu, Cyrill Stachniss, 
 5 | Jens Behley.
 6 | 
 7 | Permission is hereby granted, free of charge, to any person obtaining a copy
 8 | of this software and associated documentation files (the "Software"), to deal
 9 | in the Software without restriction, including without limitation the rights
10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 | copies of the Software, and to permit persons to whom the Software is
12 | furnished to do so, subject to the following conditions:
13 | 
14 | The above copyright notice and this permission notice shall be included in all
15 | copies or substantial portions of the Software.
16 | 
17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 | SOFTWARE.
24 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/auxiliary/common.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from pathlib import Path
 3 | from typing import Dict, List, Tuple
 4 | 
 5 | import numpy as np
 6 | import torch
 7 | import torchvision.transforms as T
 8 | from numpy import genfromtxt
 9 | from PIL import Image
10 | 
11 | 
12 | def is_png(filename: str) -> bool:
13 |   return filename.endswith('.png')
14 | 
15 | 
16 | def is_txt(filename: str) -> bool:
17 |   return filename.endswith('.txt')
18 | 
19 | 
20 | def get_png_files_in_dir(path_to_dir: Path) -> List[str]:
21 |   filenames = sorted([filename for filename in os.listdir(path_to_dir) if is_png(filename)])
22 | 
23 |   return filenames
24 | 
25 | 
26 | def get_txt_files_in_dir(path_to_dir: Path) -> List[str]:
27 |   filenames = sorted([filename for filename in os.listdir(path_to_dir) if is_txt(filename)])
28 | 
29 |   return filenames
30 | 
31 | def load_file_as_int_tensor(path_to_file: Path) -> torch.Tensor:
32 | 
33 |   tensor = Image.open(path_to_file)
34 |   tensor = np.asarray(tensor).astype(np.int32)
35 |   tensor = torch.Tensor(tensor).type(torch.int32)
36 | 
37 |   return tensor
38 | 
39 | def load_file_as_tensor(path_to_file: Path) -> torch.Tensor:
40 |   to_tensor = T.ToTensor()
41 | 
42 |   tensor = Image.open(path_to_file)
43 |   tensor = to_tensor(tensor)
44 | 
45 |   return tensor
46 | 
47 | 
48 | def load_yolo_txt_file(path_to_file: Path) -> torch.Tensor:
49 |   annos = genfromtxt(path_to_file)
50 | 
51 |   return torch.Tensor(annos)
52 | 
53 | def centered_text(text: str, line_width: int = 80, fill: str = '=') -> str:
54 |   """ generate string such that text is center in given line width using the fill """
55 |   fill_width = (line_width - len(text)-2)//2
56 |   repeat = fill_width // len(fill)
57 | 
58 |   return f"{fill*repeat} {text} {fill*repeat}"
59 | 


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [project]
 2 | name = "phenobench"
 3 | version = "0.1.0"
 4 | description = "The official development kit of the PhenoBench dataset (www.phenobench.org)."
 5 | readme = "README.md"
 6 | license = {file = "LICENSE"}
 7 | requires-python = ">=3.7"
 8 | 
 9 | authors = [
10 |   {name = "Jens Behley", email = "jens.behley@igg.uni-bonn.de" },
11 |   {name = "Jan Weyler", email = "jan.weyler@igg.uni-bonn.de" },
12 |   {name = "Federico Magistri", email = "federico.magistri@uni-bonn.de" },
13 |   {name = "Matteo Sodano", email = "matteo.sodano@igg.uni-bonn.de" },
14 |   {name = "Gianmarco Roggiolani", email = "groggiol@uni-bonn.de" },
15 |   {name = "Elias Marks", email = "elias.marks@uni-bonn.de" },
16 |   {name = "Yue (Linn) Chong", email = "ychong@uni-bonn.de" },
17 | ]
18 | 
19 | maintainers = [
20 |   {name = "Jens Behley", email = "jens.behley@igg.uni-bonn.de" }
21 | ]
22 | 
23 | 
24 | classifiers = [
25 |   "Development Status :: 4 - Beta",
26 |   "Operating System :: OS Independent",
27 |   "License :: OSI Approved :: MIT License",
28 |   "Programming Language :: Python :: 3",
29 |   "Programming Language :: Python :: 3.7",
30 |   "Programming Language :: Python :: 3.8",
31 |   "Programming Language :: Python :: 3.9",
32 |   "Programming Language :: Python :: 3.10",
33 |   "Programming Language :: Python :: 3.11",
34 |   "Programming Language :: Python :: 3 :: Only",
35 | ]
36 | 
37 | dependencies = [
38 |   "numpy",
39 |   "matplotlib",
40 |   "Pillow>6.2.1"
41 | ]
42 | 
43 | [project.optional-dependencies]
44 | eval = [
45 |   "torch",
46 |   "torchvision",
47 |   "torchmetrics==0.10.3",
48 |   "numpy",
49 |   "pyyaml",
50 |   "tqdm"
51 | ]
52 | 
53 | [project.scripts]
54 | phenobench-validator = "phenobench.tools.validator:run"
55 | phenobench-eval = "phenobench.evaluation.phenobench_eval:run"
56 | 
57 | [project.urls] 
58 | "Homepage" = "https://www.phenobench.org"
59 | "Bug Reports" = "https://github.com/PRBonn/phenobench/issues"
60 | "Source" = "https://github.com/PRBonn/phenobench"
61 | 
62 | [build-system]
63 | requires = ["setuptools>=43.0.0", "wheel"]
64 | build-backend = "setuptools.build_meta"
65 | 
66 | 
67 | # tool configuration
68 | [tool.yapf]
69 | based_on_style = "pep8"
70 | column_limit = "120"
71 | indent_width = "2"
72 | 
73 | [tool.isort]
74 | profile = "black"
75 | 
76 | [tool.pylint.format]
77 | max-line-length = "120"
78 | indent-string = '  '
79 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/auxiliary/filter.py:
--------------------------------------------------------------------------------
 1 | from typing import Dict
 2 | 
 3 | import torch
 4 | 
 5 | from .convert import bounding_box_from_cxcywh_to_xyxy, box_to_mask
 6 | 
 7 | 
 8 | def filter_partial_masks(pred_instance_masks: torch.Tensor, pred_semantics: torch.Tensor, gt_instance_masks: torch.Tensor, gt_semantics: torch.Tensor, gt_visibility: torch.Tensor):
 9 |   assert torch.max(gt_visibility) <= 1.0
10 |   assert torch.min(gt_visibility) >= 0.0
11 | 
12 |   gt_instance_partial_masks = []
13 |   # set semantics of partial masks to 0 and store partial_masks
14 |   for gt_instance_id in torch.unique(gt_instance_masks):
15 |     if gt_instance_id == 0:
16 |       continue
17 |     gt_mask = gt_instance_masks == gt_instance_id
18 |     gt_vis = torch.unique(gt_visibility[gt_mask]) # contain a single value
19 |     if gt_vis > 0.5:
20 |       continue
21 |     gt_semantics[gt_mask] = 0
22 |     gt_instance_partial_masks.append(gt_mask)
23 | 
24 | 
25 |   for pred_instance_id in torch.unique(pred_instance_masks):
26 |     if pred_instance_id == 0:
27 |       continue
28 |     pred_mask = pred_instance_masks == pred_instance_id
29 | 
30 |     for gt_mask in gt_instance_partial_masks:
31 |       # compute how much of the prediction is within the ground truth
32 |       a = torch.sum(gt_mask & pred_mask)
33 |       b = torch.sum(pred_mask)
34 |       score = a / (b + 1e-12)
35 |       assert score <= 1.0
36 | 
37 |       if score > 0.5:
38 |         pred_semantics[pred_mask] = 0
39 | 
40 | def filter_partials_boxes(preds: Dict, targets: Dict):
41 |   n_preds = preds['boxes'].shape[0]
42 |   n_targets = targets['boxes'].shape[0]
43 | 
44 |   remove_pred_indicies = []
45 |   remove_gt_indicies = []
46 | 
47 |   for i in range(n_preds):
48 |     bbox_pred = preds['boxes'][i, :]
49 |     cx, cy, width, height = bbox_pred[0].item(), bbox_pred[1].item(), bbox_pred[2].item(), bbox_pred[3].item()
50 |     xmin, xmax, ymin, ymax = bounding_box_from_cxcywh_to_xyxy(cx, cy, width, height)
51 |     bbox_pred_canvas = box_to_mask(xmin, xmax, ymin, ymax)
52 | 
53 |     for j in range(n_targets):
54 |       vis = targets['visibility'][j]
55 |       if vis > 0.5:
56 |         continue
57 |       remove_gt_indicies.append(j)
58 | 
59 |       bbox_target = targets['boxes'][j, :]
60 |       cx, cy, width, height = bbox_target[0].item(), bbox_target[1].item(), bbox_target[2].item(), bbox_target[3].item()
61 |       xmin, xmax, ymin, ymax = bounding_box_from_cxcywh_to_xyxy(cx, cy, width, height)
62 |       bbox_gt_canvas = box_to_mask(xmin, xmax, ymin, ymax)
63 | 
64 |       # compute how much of the prediction is within the ground truth
65 |       a = torch.sum(bbox_pred_canvas & bbox_gt_canvas)
66 |       b = torch.sum(bbox_pred_canvas)
67 |       score = a / (b + 1e-12)
68 |       assert score <= 1.0
69 | 
70 |       if score > 0.5:
71 |         remove_pred_indicies.append(i)
72 | 
73 |   # filter predictions
74 |   mask_pred = torch.ones_like(preds['labels'], dtype=torch.bool)
75 |   for rm_idx in remove_pred_indicies:
76 |     mask_pred[rm_idx] = False
77 |   preds['labels'] = preds['labels'][mask_pred]
78 |   preds['scores'] = preds['scores'][mask_pred]
79 |   preds['boxes'] = preds['boxes'][mask_pred]
80 | 
81 |   # filter targets
82 |   mask_target = torch.ones_like(targets['labels'], dtype=torch.bool)
83 |   for rm_idx in remove_gt_indicies:
84 |     mask_target[rm_idx] = False
85 |   targets['labels'] = targets['labels'][mask_target]
86 |   targets['boxes'] = targets['boxes'][mask_target]
87 |   targets['visibility'] = targets['visibility'][mask_target]
88 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | 
  2 | # Created by https://www.gitignore.io/api/c++,code,python,virtualenv
  3 | # Edit at https://www.gitignore.io/?templates=c++,code,python,virtualenv
  4 | 
  5 | ### C++ ###
  6 | # Prerequisites
  7 | *.d
  8 | 
  9 | # Compiled Object files
 10 | *.slo
 11 | *.lo
 12 | *.o
 13 | *.obj
 14 | 
 15 | # Precompiled Headers
 16 | *.gch
 17 | *.pch
 18 | 
 19 | # Compiled Dynamic libraries
 20 | *.so
 21 | *.dylib
 22 | *.dll
 23 | 
 24 | # Fortran module files
 25 | *.mod
 26 | *.smod
 27 | 
 28 | # Compiled Static libraries
 29 | *.lai
 30 | *.la
 31 | *.a
 32 | *.lib
 33 | 
 34 | # Executables
 35 | *.exe
 36 | *.out
 37 | *.app
 38 | 
 39 | ### Code ###
 40 | .vscode/*
 41 | !.vscode/settings.json
 42 | !.vscode/tasks.json
 43 | !.vscode/launch.json
 44 | !.vscode/extensions.json
 45 | 
 46 | ### Python ###
 47 | # Byte-compiled / optimized / DLL files
 48 | __pycache__/
 49 | *.py[cod]
 50 | *$py.class
 51 | 
 52 | # C extensions
 53 | 
 54 | # Distribution / packaging
 55 | .Python
 56 | build/
 57 | develop-eggs/
 58 | dist/
 59 | downloads/
 60 | eggs/
 61 | .eggs/
 62 | lib/
 63 | lib64/
 64 | parts/
 65 | sdist/
 66 | var/
 67 | wheels/
 68 | pip-wheel-metadata/
 69 | share/python-wheels/
 70 | *.egg-info/
 71 | .installed.cfg
 72 | *.egg
 73 | MANIFEST
 74 | 
 75 | # PyInstaller
 76 | #  Usually these files are written by a python script from a template
 77 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 78 | *.manifest
 79 | *.spec
 80 | 
 81 | # Installer logs
 82 | pip-log.txt
 83 | pip-delete-this-directory.txt
 84 | 
 85 | # Unit test / coverage reports
 86 | htmlcov/
 87 | .tox/
 88 | .nox/
 89 | .coverage
 90 | .coverage.*
 91 | .cache
 92 | nosetests.xml
 93 | coverage.xml
 94 | *.cover
 95 | .hypothesis/
 96 | .pytest_cache/
 97 | 
 98 | # Translations
 99 | *.mo
100 | *.pot
101 | 
102 | # Scrapy stuff:
103 | .scrapy
104 | 
105 | # Sphinx documentation
106 | docs/_build/
107 | 
108 | # PyBuilder
109 | target/
110 | 
111 | # pyenv
112 | .python-version
113 | 
114 | # pipenv
115 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
116 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
117 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
118 | #   install all needed dependencies.
119 | #Pipfile.lock
120 | 
121 | # celery beat schedule file
122 | celerybeat-schedule
123 | 
124 | # SageMath parsed files
125 | *.sage.py
126 | 
127 | # Spyder project settings
128 | .spyderproject
129 | .spyproject
130 | 
131 | # Rope project settings
132 | .ropeproject
133 | 
134 | # Mr Developer
135 | .mr.developer.cfg
136 | .project
137 | .pydevproject
138 | 
139 | # mkdocs documentation
140 | /site
141 | 
142 | # mypy
143 | .mypy_cache/
144 | .dmypy.json
145 | dmypy.json
146 | 
147 | # Pyre type checker
148 | .pyre/
149 | 
150 | ### VirtualEnv ###
151 | # Virtualenv
152 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
153 | pyvenv.cfg
154 | .env
155 | .venv
156 | env/
157 | venv/
158 | ENV/
159 | env.bak/
160 | venv.bak/
161 | pip-selfcheck.json
162 | 
163 | # End of https://www.gitignore.io/api/c++,code,python,virtualenv
164 | 
165 | ### JupyterNotebooks ###
166 | # gitignore template for Jupyter Notebooks
167 | # website: http://jupyter.org/
168 | 
169 | .ipynb_checkpoints
170 | */.ipynb_checkpoints/*
171 | 
172 | # IPython
173 | profile_default/
174 | ipython_config.py
175 | 
176 | # Remove previous ipynb_checkpoints
177 | #   git rm -r .ipynb_checkpoints/
178 | 
179 | .vscode
180 | image_labeler/widgets/ui_labeler.py
181 | data
182 | *.DS_Store
183 | html
184 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # PhenoBench Development Kit
 2 | 
 3 | ![](https://www.phenobench.org/imgs/devkit_teaser.jpg)
 4 | 
 5 | [PhenoBench](https://www.phenobench.org) is a large dataset and benchmarks for the semantic interpretation of images of real agricultural fields. Together with the dataset, we provide a development kit that provides:
 6 | 
 7 | - a framework-agnostic data loader.
 8 | - visualization functions for drawing our data format.
 9 | - evaluation scripts, `phenobench-eval`, for all tasks (also used on the CodaLab servers).
10 | - validator, called `phenobench-validator`, checking CodaLab submission files for consistency
11 | 
12 | For more information on the dataset, please visit [www.phenobench.org](https://www.phenobench.org).
13 | 
14 | ## Getting started
15 | 
16 | 1. [Download](https://www.phenobench.org/dataset.html) the dataset.
17 | 2. Install the development kit: `pip install phenobench`.
18 | 3. Explore the data with the [tutorial notebook](phenobench_tutorial.ipynb).
19 | 4. See the [code of our baselines](https://github.com/PRBonn/phenobench-baselines) as a starting point or train your own models.
20 | 5. See the [FAQ](#frequently-asked-questions) for common questions and troubleshooting.
21 | 
22 | If you discover a problem or have general questions regarding the dataset, don't hesitate to open an issues. We will try to resolve your issue as quickly as possible.
23 | 
24 | ## Evaluation scripts (`phenobench-eval`)
25 | 
26 | **Important:** Install all dependencies with `pip install "phenobench[eval]"`.
27 | 
28 | For evaluating and computing the metrics for a specific task, you can run the `phenobench-eval` tool as follows:
29 | 
30 | ```bash
31 | $ phenobench-eval --task <task> --phenobench_dir <dir> --prediction_dir <dir> --split <split>
32 | ```
33 |  - `task` is one of the following options: `semantics`, `panoptic`, `leaf_instances`, `plant_detection`, `leaf_detection`, or `hierarchical`.
34 |  - `phenobench_dir` is the root directory of the PhenoBench dataset, where `train`, `val` directories are located.
35 |  - `prediction_dir` is the directory containing the predictions as sub-folders, which depend on the specific tasks.
36 |  - `split` is either `train` or `val`.
37 | 
38 | Note that **all ablation studies of your approach should run on the validation set**. Thus, we also provide a comparably large validation set to enable a solid comparison of different settings of your approach.
39 | 
40 | ## CodaLab Submission Validator (`phenobench-validator`)
41 | 
42 | Before you submit a zip file to our CodaLab competitions, see also our available [benchmarks](https://www.phenobench.org/benchmarks.html), you can use the `phenobench-validator` to check your submission for consistency. The tool is also part of the pip package, therefore after installing the package via pip, you can call the `phenobench-validator` as follows:
43 | 
44 | ```bash 
45 | $ phenobench-validator --task <task> --phenobench_dir <dir> --zipfile <zipfile>
46 | ```
47 | - `task` is one of the following options: `semantics`, `panoptic`, `leaf_instances`, `plant_detection`, `leaf_detection`, or `hierarchical`.
48 | - `phenobench_dir` is the root directory of the PhenoBench dataset, where `train`, `val` directories are located.
49 | - `zipfile` is the zip file that you want to submit to the corresponding benchmark on CodaLab.
50 | 
51 | ## Frequently Asked Questions
52 | 
53 | **Question:**  What are the usage restrictions of the PhenoBench dataset?  
54 | **Answer:** We distribute the dataset using the CC-BY-SA International 4.0 license, which allows research but also commercial usage as long as the dataset is properly attributed (via a citation of the corresponding paper) and distributed with the same license if altered or modified. See also our [dataset overview page](https://www.phenobench.org/dataset.html) for the full license text, etc.
55 | 
56 | 
57 | 
58 | 


--------------------------------------------------------------------------------
/src/phenobench/tools/validator.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | 
  3 | import argparse
  4 | import os
  5 | from zipfile import ZipFile
  6 | 
  7 | import numpy as np
  8 | 
  9 | 
 10 | class ValidationException(Exception):
 11 |   pass
 12 | 
 13 | 
 14 | def run():
 15 |   parser = argparse.ArgumentParser(
 16 |       description=
 17 |       "Validate a submission zip file needed to evaluate on CodaLab competitions of PhenoBench.\n\nThe validator checks:\n  1. correct folder structure,\n  2. existence of label files for each image.\nInvalid labels are ignored by the evaluation script, therefore we don't check\nfor invalid labels.",
 18 |       formatter_class=argparse.RawTextHelpFormatter)
 19 | 
 20 |   parser.add_argument(
 21 |       "--task",
 22 |       type=str,
 23 |       required=True,
 24 |       choices=["semantics", "panoptic", "leaf_instances", "plant_detection", "leaf_detection", "hierarchical"],
 25 |       help='task for which the zip file should be validated.')
 26 | 
 27 |   parser.add_argument(
 28 |       "--zipfile",
 29 |       type=str,
 30 |       required=True,
 31 |       help='zip file that should be validated.',
 32 |   )
 33 | 
 34 |   parser.add_argument('--phenobench_dir',
 35 |                       type=str,
 36 |                       required=True,
 37 |                       help='Root directory containing the folder train, val, test of the PhenoBench.')
 38 | 
 39 |   FLAGS, _ = parser.parse_known_args()
 40 | 
 41 |   checkmark = "\u2713"
 42 | 
 43 |   required_folders = {
 44 |       "semantics": ["semantics"],
 45 |       "panoptic": ["semantics", "plant_instances"],
 46 |       "leaf_instances": ["leaf_instances"],
 47 |       "plant_detection": ["plant_bboxes"],
 48 |       "leaf_detection": ["leaf_bboxes"],
 49 |       "hierarchical": ["semantics", "plant_instances", "leaf_instances"]
 50 |   }
 51 | 
 52 |   try:
 53 | 
 54 |     print('Validating zip archive "{}".\n'.format(FLAGS.zipfile))
 55 | 
 56 |     print(" ========== {:^15} ========== ".format(FLAGS.task))
 57 | 
 58 |     print("  1. Checking filename.............. ", end="", flush=True)
 59 |     if not FLAGS.zipfile.endswith('.zip'):
 60 |       raise ValidationException('Competition submission must end with ".zip"')
 61 |     print(checkmark)
 62 | 
 63 |     with ZipFile(FLAGS.zipfile) as zipfile:
 64 | 
 65 |       print("  2. Checking directory structure... ", end="", flush=True)
 66 | 
 67 |       directories = [folder.filename for folder in zipfile.infolist() if folder.filename.endswith("/")]
 68 |       for expected_folder in required_folders[FLAGS.task]:
 69 |         if expected_folder + "/" not in directories:
 70 |           raise ValidationException(
 71 |               f'Directory "{expected_folder}" missing inside zip file. Your zip file should contain following folders: {", ".join(required_folders[FLAGS.task])}'
 72 |           )
 73 | 
 74 |       print(checkmark)
 75 | 
 76 |       print('  3. Checking files................. ', end='', flush=True)
 77 | 
 78 |       prediction_files = {info.filename: info for info in zipfile.infolist() if not info.filename.endswith("/")}
 79 |       image_files = sorted(os.listdir(os.path.join(FLAGS.phenobench_dir, "test", "images")))
 80 | 
 81 |       for expected_folder in required_folders[FLAGS.task]:
 82 |         if "bboxes" in expected_folder:
 83 |           for image_file in image_files:
 84 |             expected_file = f"{expected_folder}/{os.path.splitext(image_file)[0]}.txt"
 85 |             if expected_file not in prediction_files:
 86 |               raise ValidationException(f'Missing prediction for {image_file} in folder {expected_folder}!')
 87 |         else:
 88 |           for image_file in image_files:
 89 |             expected_file = f"{expected_folder}/{image_file}"
 90 |             if expected_file not in prediction_files:
 91 |               raise ValidationException(f'Missing prediction for {image_file} in folder {expected_folder}!')
 92 | 
 93 |       print(checkmark)
 94 | 
 95 |   except ValidationException as ex:
 96 |     print("\n\n  " + "\u001b[1;31m>>> Error: " + str(ex) + "\u001b[0m")
 97 |     exit(1)
 98 | 
 99 |   print("\n\u001b[1;32mEverything ready for submission!\u001b[0m  \U0001F389")
100 | 
101 | 
102 | if __name__ == "__main__":
103 |   run()
104 | 


--------------------------------------------------------------------------------
/src/phenobench/visualization.py:
--------------------------------------------------------------------------------
  1 | from typing import Dict, List, Tuple
  2 | 
  3 | import matplotlib.patches as patches
  4 | import matplotlib.pyplot as plt
  5 | import numpy as np
  6 | from matplotlib.axes import Axes
  7 | from PIL import Image
  8 | 
  9 | 
 10 | def draw_semantics(axes: Axes,
 11 |                    image: Image,
 12 |                    semantics: np.array,
 13 |                    alpha: float = 0.5,
 14 |                    colors: Dict[int, Tuple[int, int, int]] = {
 15 |                        1: (0, 255, 0),
 16 |                        2: (255, 0, 0),
 17 |                        3: (0, 255, 255),
 18 |                        4: (255, 0, 255)
 19 |                    },
 20 |                    mask_classes: List[int] = [0]):
 21 |   """ draw pixel-wise semantics according to given image and semantics. 
 22 | 
 23 |   Args:
 24 |       axes (matplotlib.axes.Axes): axes object to draw into.
 25 |       image (PIL.Image): image that should be drawn.
 26 |       semantics (np.array): semantics (1=crop, 2=weed, 3=partial crop, 4=partial weed)
 27 |       alpha(float): transparency of semantic mask drawn on the image.
 28 |       colors(Dict[int, tuple]): mapping of class ids to colors. Default: 1(crop) = (0,255,0), 2(weed) = (0,255,0), 3(partial_crop)= (0,255,255), 4(partial_weed) = (255,0,255)
 29 |       mask_classes (List[int]): class indices that are not shown in the final composition.
 30 |   """
 31 | 
 32 |   im_semantics = np.zeros((*semantics.shape, 3), dtype=float)
 33 | 
 34 |   for label, color in colors.items():
 35 |     im_semantics[semantics == label] = np.array(color) / 255.0
 36 | 
 37 |   im_alpha = np.ones((*semantics.shape, 3), dtype=float) * alpha
 38 |   for c in mask_classes:
 39 |     im_alpha[semantics == c] = 0.0
 40 | 
 41 |   composed_image = (1.0 - im_alpha) * (np.array(image) / 255.0) + im_alpha * im_semantics
 42 |   axes.imshow(composed_image)
 43 | 
 44 |   # not sure why, but `ax.im_show(im_semantics, alpha=im_alpha)` doesn't work as expected:
 45 |   # im_alpha = np.ones(semantics.shape, dtype=float) * alpha
 46 |   # for c in mask_classes: im_alpha[semantics == c] = 0.0
 47 |   # axes.imshow(image)
 48 |   # axes.imshow(im_semantics, alpha = im_alpha)
 49 | 
 50 | 
 51 | def draw_instances(ax: Axes, image: Image, instance_mask: np.array, alpha: float = 0.5):
 52 |   """ Draw instance masks from provided data, where individual masks are indicated by random colors.
 53 | 
 54 |   Args:
 55 |       ax (Axes): axes object to draw into.
 56 |       image (Image): image that should be drawn.
 57 |       instance_masks (np.array): _description_
 58 |       alpha (float, optional): _description_. Defaults to 0.5.
 59 |   """
 60 |   random_colors = plt.get_cmap('tab10').resampled(10)
 61 |   unique_ids = np.unique(instance_mask)
 62 |   # unique_ids = unique_ids[unique_ids > 0]
 63 | 
 64 |   im_mask = np.zeros((*instance_mask.shape, 3))
 65 |   for idx, uid in enumerate(unique_ids):
 66 |     im_mask[instance_mask == uid] = random_colors(idx % 10)[:3]
 67 | 
 68 |   im_alpha = np.ones((*instance_mask.shape, 3)) * alpha
 69 |   im_alpha[instance_mask == 0, :] = 0.0
 70 | 
 71 |   float_image = (np.array(image) / 255.0)
 72 |   composed_image = (1.0 - im_alpha) * float_image + im_alpha * im_mask
 73 | 
 74 |   ax.imshow(composed_image)
 75 | 
 76 | 
 77 | def draw_bboxes(ax: Axes,
 78 |                 image: Image,
 79 |                 bboxes: List[Dict],
 80 |                 colors: Dict[int, Tuple[int, int, int]] = {
 81 |                     1: (0, 255, 0),
 82 |                     2: (255, 0, 0),
 83 |                     3: (0, 255, 255),
 84 |                     4: (255, 0, 255)
 85 |                 },
 86 |                 linewidth=2):
 87 |   """ Draw the given bounding boxes on the image.
 88 | 
 89 |   Args:
 90 |       ax (Axes): axes object to draw into.
 91 |       image (Image): image that should be drawn.
 92 |       bboxes (List[Dict): list of bounding boxes in the format {"label":<label>, "center": (x,y), "width": w, "height": h},
 93 |       colors(Dict[int, tuple]): mapping of class ids to colors. Default: 1(crop) = (0,255,0), 2(weed) = (0,255,0), 3(partial_crop)= (0,255,255), 4(partial_weed) = (255,0,255)
 94 |       linewidth(int): thickness of the lines of the bounding box. Default: 2
 95 |   """
 96 | 
 97 |   ax.imshow(image)
 98 | 
 99 |   for bbox in bboxes:
100 |     x, y = bbox["center"][0] - bbox["width"] / 2, bbox["center"][1] - bbox["height"] / 2
101 |     rect = patches.Rectangle((x, y),
102 |                              bbox["width"],
103 |                              bbox["height"],
104 |                              linewidth=linewidth,
105 |                              edgecolor=np.array(colors[bbox["label"]]) / 255.0,
106 |                              facecolor='none')
107 |     ax.add_patch(rect)
108 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/phenobench_eval.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | 
  3 | import argparse
  4 | from pathlib import Path
  5 | 
  6 | from phenobench.evaluation.auxiliary.common import centered_text
  7 | from phenobench.evaluation.evaluate_leaf_bounding_boxes import evaluate_leaf_detection
  8 | from phenobench.evaluation.evaluate_leaf_instance_masks_panoptic import (
  9 |     evaluate_leaf_instances,
 10 | )
 11 | from phenobench.evaluation.evaluate_plant_bounding_boxes import evaluate_plant_detection
 12 | from phenobench.evaluation.evaluate_plant_instance_masks_panoptic import (
 13 |     evaluate_plant_instances,
 14 | )
 15 | from phenobench.evaluation.evaluate_semantics import evaluate_semantics
 16 | 
 17 | 
 18 | def run():
 19 |   parser = argparse.ArgumentParser()
 20 |   parser.add_argument(
 21 |       '--task',
 22 |       choices=['semantics', 'panoptic', 'leaf_instances', 'plant_detection', 'leaf_detection', 'hierarchical'],
 23 |       required=True,
 24 |       type=str,
 25 |       help='task to evaluate.')
 26 |   parser.add_argument('--phenobench_dir', required=True, type=Path, help='Path to the root directory of PhenoBench.')
 27 |   parser.add_argument('--prediction_dir', required=True, type=Path, help='Path to prediction directory.')
 28 |   parser.add_argument('--split',
 29 |                       choices=['train', 'val'],
 30 |                       required=True,
 31 |                       type=str,
 32 |                       default='val',
 33 |                       help='Specify which split to use for evaluation.')
 34 |   args = vars(parser.parse_known_args()[0])
 35 | 
 36 |   if args["task"] == "semantics":
 37 |     print(centered_text("Computing metrics for semantic segmentation"))
 38 | 
 39 |     results = evaluate_semantics(args)
 40 | 
 41 |     print(centered_text(f"Metrics for {args['split']} split of PhenoBench"))
 42 | 
 43 |     print(f"{'IoU (crop)':10}: {results['crop']:4}")
 44 |     print(f"{'IoU (weed)':10}: {results['weed']:4}")
 45 |     print(f"{'mIoU':10}: {results['mIoU']:4}")
 46 | 
 47 |   elif args["task"] == "panoptic":
 48 |     print(centered_text("Computing metrics for plant panoptic segmentation"))
 49 | 
 50 |     semantic_results = evaluate_semantics(args)
 51 |     instance_results = evaluate_plant_instances(args)
 52 | 
 53 |     print(centered_text(f"Metrics for {args['split']} split of PhenoBench"))
 54 | 
 55 |     iou_soil = semantic_results['soil']
 56 |     pq_crop = instance_results['plants_cls'][1]['pq']
 57 |     pq_weed = instance_results['plants_cls'][2]['pq']
 58 | 
 59 |     print(f"{'IoU (soil)':10}: {iou_soil:4}")
 60 |     print(f"{'PQ (crop)':10}: {pq_crop:4}")
 61 |     print(f"{'PQ (weed)':10}: {pq_weed:4}")
 62 |     print(f"{'PQ+':10}: {(iou_soil+pq_crop+pq_weed)/3:4.2f}")
 63 | 
 64 |   elif args["task"] == "leaf_instances":
 65 |     print(centered_text("Computing metrics for leaf instance segmentation"))
 66 | 
 67 |     results = evaluate_leaf_instances(args)
 68 | 
 69 |     print(centered_text(f"Metrics for {args['split']} split of PhenoBench"))
 70 |     pq_leaf = results["leaves_pq"]
 71 |     print(f"{'PQ (leaf)':10}: {pq_leaf:4}")
 72 | 
 73 |   elif args["task"] == "plant_detection":
 74 |     print(centered_text("Computing metrics for plant detection"))
 75 | 
 76 |     results = evaluate_plant_detection(args)
 77 | 
 78 |     print(centered_text(f"Metrics for {args['split']} split of PhenoBench"))
 79 | 
 80 |     print(f"{'mAP':10}: {results['mAP']:4}")
 81 |     print(f"{'mAP_50':10}: {results['mAP_50']:4}")
 82 |     print(f"{'mAP_75':10}: {results['mAP_75']:4}")
 83 |     print(f"{'AP (crop)':10}: {results['mAP_cls'][0]:4}")
 84 |     print(f"{'AP (weed)':10}: {results['mAP_cls'][1]:4}")
 85 | 
 86 |   elif args["task"] == "leaf_detection":
 87 |     print(centered_text("Computing metrics for leaf detection"))
 88 | 
 89 |     results = evaluate_leaf_detection(args)
 90 | 
 91 |     print(centered_text(f"Metrics for {args['split']} split of PhenoBench"))
 92 | 
 93 |     print(f"{'mAP':10}: {results['mAP']:4}")
 94 |     print(f"{'mAP_50':10}: {results['mAP_50']:4}")
 95 |     print(f"{'mAP_75':10}: {results['mAP_75']:4}")
 96 | 
 97 |   elif args["task"] == "hierarchical":
 98 |     print(centered_text("Computing metrics for leaf detection"))
 99 | 
100 |     semantic_results = evaluate_semantics(args)
101 |     instance_results = evaluate_plant_instances(args)
102 |     leaf_results = evaluate_leaf_instances(args)
103 | 
104 |     print(centered_text(f"Metrics for {args['split']} split of PhenoBench"))
105 | 
106 |     iou_soil = semantic_results['soil']
107 |     iou_weed = semantic_results['weed']
108 | 
109 |     pq_leaf = leaf_results["leaves_pq"]
110 |     pq_crop = instance_results['plants_cls'][1]['pq']
111 | 
112 |     print(f"{'IoU (soil)':10}: {iou_soil:4}")
113 |     print(f"{'IoU (weed)':10}: {iou_weed:4}")
114 |     print(f"{'PQ (leaf)':10}: {pq_leaf:4}")
115 |     print(f"{'PQ (crop)':10}: {pq_crop:4}")
116 |     print(f"{'PQ':10}: {(pq_crop+pq_leaf)/2:4.2f}")
117 |     print(f"{'PQ+':10}: {(iou_soil+iou_weed+pq_crop+pq_leaf)/4:4.2f}")
118 | 
119 | 
120 | if __name__ == '__main__':
121 |   run()


--------------------------------------------------------------------------------
/src/phenobench/evaluation/auxiliary/convert.py:
--------------------------------------------------------------------------------
  1 | from typing import Dict, List, Tuple
  2 | 
  3 | import torch
  4 | 
  5 | IMG_WIDTH = 1024
  6 | IMG_HEIGHT = 1024
  7 | 
  8 | def convert_partial_semantics(semantics: torch.Tensor) -> torch.Tensor:
  9 |   semantics_converted = semantics.clone()
 10 | 
 11 |   # convert partial crops to regular crops
 12 |   mask_partial_crops = semantics == 3
 13 |   semantics_converted[mask_partial_crops] = 1
 14 | 
 15 |   # convert partial weeds to regular weeds
 16 |   mask_partial_weeds = semantics == 4
 17 |   semantics_converted[mask_partial_weeds] = 2
 18 | 
 19 |   return semantics_converted
 20 | 
 21 | def bounding_box_from_mask_xyxy(mask: torch.Tensor) -> Tuple[int, int, int, int]:
 22 |   rows, cols = mask.nonzero(as_tuple=True)
 23 | 
 24 |   x_min = int(torch.min(cols))
 25 |   y_min = int(torch.min(rows))
 26 |   x_max = int(torch.max(cols))
 27 |   y_max = int(torch.max(rows))
 28 | 
 29 |   return x_min, y_min, x_max, y_max
 30 | 
 31 | def bounding_box_from_mask_cxcywh(mask: torch.Tensor) -> Tuple[int, int, int, int]:
 32 | 
 33 |   x_min, y_min, x_max, y_max = bounding_box_from_mask_xyxy(mask)
 34 | 
 35 |   cx = int(round((x_min + x_max) / 2))
 36 |   cy = int(round((y_min + y_max) / 2))
 37 |   width = x_max - x_min
 38 |   height = y_max - y_min
 39 | 
 40 |   assert width >= 0
 41 |   assert height >= 0
 42 | 
 43 |   return cx, cy, width, height
 44 | 
 45 | def bounding_box_from_cxcywh_to_xyxy(cx: int, cy: int, width: int, height: int) -> Tuple[int, int, int, int]:
 46 |   x_min = round(cx - (width / 2))
 47 |   x_max = round(cx + (width / 2))
 48 |   y_min = round(cy - (height / 2))
 49 |   y_max = round(cy + (height / 2))
 50 | 
 51 |   return x_min, x_max, y_min, y_max
 52 | 
 53 | def box_to_mask(x_min:int, x_max: int, y_min: int, y_max: int) -> torch.Tensor:
 54 |   canvas = torch.zeros((IMG_HEIGHT, IMG_WIDTH), dtype=torch.bool)
 55 | 
 56 |   canvas[y_min: (y_max + 1), x_min: (x_max + 1)] = True
 57 | 
 58 |   return canvas
 59 | 
 60 | def cvt_yolo_to_bbox_map(yolo_anno: torch.Tensor) -> List[Dict[str, torch.Tensor]]:
 61 |   """ Convert YOLO annotations to format required by MAP of torchmetrics.
 62 | 
 63 |   Please check the following to get more information: https://torchmetrics.readthedocs.io/en/stable/detection/mean_average_precision.html
 64 | 
 65 |   Args:
 66 |       yolo_anno (torch.Tensor): annotations provided by YOLO ... of shape [n_preds x 6]
 67 |         - 1st col -> label id
 68 |         - 2nd col -> center_x in [0,1]
 69 |         - 3rd col -> center_y in [0,1]
 70 |         - 4th col -> width in [0,1]
 71 |         - 5th col -> height in [0,1]
 72 |         - 6th col -> confidence score in [0,1]
 73 | 
 74 |   Returns:
 75 |       List[Dict[str, torch.Tensor]]: format required by MAP of torchmetrics.
 76 |   """
 77 | 
 78 |   if len(yolo_anno.shape) == 1:
 79 |     yolo_anno = yolo_anno.unsqueeze(dim=0)
 80 | 
 81 |   n_predicitions = yolo_anno.shape[0]
 82 |   boxes = []
 83 |   labels = []
 84 |   scores = []
 85 |   for i in range(n_predicitions):
 86 |     # ------- Label -------
 87 |     try:
 88 |       label = int(yolo_anno[i, 0]) # + 1
 89 |     except IndexError:
 90 |       continue
 91 |     if label == 0:
 92 |       continue
 93 | 
 94 |     # ------- Bounding Box -------
 95 |     cx = int(round(float(yolo_anno[i, 1]) * IMG_WIDTH))
 96 |     cy = int(round(float(yolo_anno[i, 2]) * IMG_HEIGHT))
 97 |     width = int(round(float(yolo_anno[i, 3]) * IMG_WIDTH))
 98 |     height = int(round(float(yolo_anno[i, 4]) * IMG_HEIGHT))
 99 | 
100 |     # ------- Confidence Score -------
101 |     score = float(yolo_anno[i, 5])
102 | 
103 |     # ------- Accumulate -------
104 |     labels.append(torch.Tensor([label]))
105 |     boxes.append(torch.Tensor([cx, cy, width, height]))
106 |     scores.append(torch.Tensor([score]))
107 | 
108 |   out = {}
109 |   try:
110 |     out['labels'] = torch.stack(labels).squeeze(1).type(torch.uint8)  # [n_objects]
111 |     out['boxes'] = torch.stack(boxes)  # [n_objects x 4]
112 |     out['scores'] = torch.stack(scores).squeeze(1)  # [n_objects]
113 |   except RuntimeError:
114 |     # i.e., there isn't any prediction
115 |     out['labels'] = torch.tensor([])
116 |     out['boxes'] = torch.tensor([])
117 |     out['scores'] = torch.tensor([])
118 | 
119 | 
120 |   return [out]
121 | 
122 | 
123 | def cvt_gt_to_bbox_map(instance_map: torch.Tensor, semantics: torch.Tensor, visibility: torch.Tensor) -> List[Dict[str, torch.Tensor]]:
124 |   assert torch.max(visibility) <= 1.0
125 |   assert torch.min(visibility) >= 0.0
126 | 
127 |   instance_ids = torch.unique(instance_map)
128 |   instance_ids = instance_ids[instance_ids != 0]  # ignore background
129 | 
130 |   boxes = []
131 |   labels = []
132 |   visibilities = []
133 |   for instance_id in instance_ids:
134 |     # ------- Mask -------
135 |     instance_mask = instance_map == instance_id
136 | 
137 |     # ------- Bounding Box -------
138 |     cx, cy, width, height = bounding_box_from_mask_cxcywh(instance_mask)
139 | 
140 |     # ------- Label -------
141 |     label = torch.unique(semantics[instance_mask])
142 |     try:
143 |       if label == 0:
144 |         continue
145 |       if len(label) != 1:
146 |         assert False, "Single instance has multiple semantic classes"
147 |     except RuntimeError:
148 |       continue
149 | 
150 |     # ------- Visibility -------
151 |     vis = torch.unique(visibility[instance_mask])
152 | 
153 |     # ------- Accumulate -------
154 |     boxes.append(torch.Tensor([cx, cy, width, height]))
155 |     labels.append(torch.Tensor([label]))
156 |     visibilities.append(torch.Tensor(vis))
157 | 
158 |   out = {}
159 |   out['labels'] = torch.stack(labels).squeeze(1).type(torch.uint8)  # [n_objects]
160 |   out['boxes'] = torch.stack(boxes)  # [n_objects x 4]
161 |   out['visibility'] = torch.stack(visibilities)  # [n_objects]
162 | 
163 |   return [out]
164 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/auxiliary/panoptic_eval.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import torch
  3 | 
  4 | 
  5 | class PanopticEvaluator():
  6 |   def __init__(self):
  7 |     self.evaluator = PanopticQuality()
  8 | 
  9 |   def update(self, semantic_pred, semantic_gt, instance_pred, instance_gt):
 10 |     self.evaluator.compute_pq(semantic_pred, semantic_gt, instance_pred, instance_gt)
 11 | 
 12 |   def compute(self):
 13 |     pq = self.evaluator.average_pq(self.evaluator.panoptic_qualities)
 14 | 
 15 |     return pq
 16 | 
 17 |   @property
 18 |   def results_classwise(self):
 19 |     return self.evaluator.panoptic_qualities
 20 | 
 21 | 
 22 | class PanopticQuality:
 23 |   def __init__(self):
 24 |     self.panoptic_qualities = {}
 25 | 
 26 |   def reset(self):
 27 |     self.panoptic_qualities.clear()
 28 | 
 29 |   def compute_pq_single_class(self, prediction, groundtruth):
 30 |     numerator_iou = torch.tensor(0.)
 31 |     class_matches = 0
 32 |     unmatched_class_predictions = 0
 33 | 
 34 |     # take the non-zero INSTANCE labels for both prediction and groundtruth
 35 |     labels = torch.unique(prediction)
 36 |     gt_labels = torch.unique(groundtruth)
 37 |     labels = labels[labels > 0]
 38 |     gt_labels = gt_labels[gt_labels > 0]
 39 | 
 40 |     gt_masks = [groundtruth == gt_label for gt_label in gt_labels]
 41 |     gt_areas = [(groundtruth == gt_label).sum() for gt_label in gt_labels]
 42 | 
 43 |     not_matched = [True] * len(gt_labels)
 44 | 
 45 |     for label in labels: # for each predicted label
 46 |       iou = torch.tensor(0.)
 47 |       best_idx = 0
 48 | 
 49 |       pred_mask = (prediction == label)
 50 |       pred_area = (prediction == label).sum()
 51 | 
 52 |       for idx in np.where(not_matched)[0]:
 53 |         gt_mask = gt_masks[idx]
 54 |         gt_area = gt_areas[idx]
 55 |         # compute iou with all instance gt labels and store the best
 56 |         intersection = ((pred_mask & gt_mask).sum()).float()
 57 |         union = (pred_area + gt_area - intersection).float()
 58 | 
 59 |         iou_tmp = intersection / union
 60 |         if iou_tmp > iou:
 61 |           iou = iou_tmp
 62 |           best_idx = idx
 63 | 
 64 |       # if the best iou is above 0.5, store the match pred_label-gt_label-iou
 65 |       if iou > 0.5:
 66 |         class_matches += 1
 67 |         numerator_iou += iou
 68 |         not_matched[best_idx] = False
 69 |       else:
 70 |         # unmatched_class_predictions.append(label.item())
 71 |         unmatched_class_predictions += 1
 72 | 
 73 |     true_positives = class_matches # len(class_matches)     # TP = number of matches
 74 |     # FP = number of unmatched predictions
 75 |     false_positives = unmatched_class_predictions # len(unmatched_class_predictions)
 76 |     # FN = number of unmatched gt labels
 77 |     false_negatives = len(gt_labels) - class_matches #len(class_matches)
 78 |     panoptic_quality_one_class = numerator_iou / \
 79 |         (true_positives + 0.5 * false_positives + 0.5 * false_negatives)
 80 | 
 81 |     return panoptic_quality_one_class, class_matches
 82 | 
 83 |   def compute_pq(self, semantic_pred, semantic_gt, instance_pred, instance_gt):
 84 |     # take the semantic labels (except 0 = void label)
 85 |     semantic_gt_labels = torch.unique(semantic_gt).int()
 86 |     semantic_gt_labels = semantic_gt_labels[semantic_gt_labels > 0].int()
 87 | 
 88 |     if semantic_gt_labels.sum() == 0:
 89 |       if instance_pred.sum() == 0:
 90 |         return 1.0, {}
 91 |       else:
 92 |         return 0.0 , {}
 93 | 
 94 | 
 95 |     for gt_label in semantic_gt_labels:
 96 |       # take pred and gt where they have sem_class = label
 97 |       semantic_tmp = (semantic_pred == gt_label).int()
 98 |       semantic_gt_tmp = (semantic_gt == gt_label).int()
 99 | 
100 |       # take pred and gt instances corresponding to the considered semantic class
101 |       instance_tmp = (instance_pred * semantic_tmp).int()
102 |       instance_gt_tmp = (instance_gt * semantic_gt_tmp).int()
103 | 
104 |       pq, _ = self.compute_pq_single_class(instance_tmp, instance_gt_tmp)
105 | 
106 |       # update: count = number of images that contains the class, pq is the overall panoptic quality of the images
107 |       if gt_label.item() not in self.panoptic_qualities.keys():
108 |         self.panoptic_qualities[gt_label.item()] = {}
109 |         self.panoptic_qualities[gt_label.item()]['count'] = 1
110 |         self.panoptic_qualities[gt_label.item()]['pq'] = pq.item()
111 |       else:
112 |         self.panoptic_qualities[gt_label.item()]['count'] += 1
113 |         self.panoptic_qualities[gt_label.item()]['pq'] = ((self.panoptic_qualities[gt_label.item()]['pq'] * (
114 |             self.panoptic_qualities[gt_label.item()]['count']-1) + pq) / self.panoptic_qualities[gt_label.item()]['count']).item()
115 | 
116 |     # panoptic_quality = self.average_pq(self.panoptic_qualities)
117 |     # return panoptic_quality, self.panoptic_qualities
118 | 
119 |     return self.panoptic_qualities
120 | 
121 |   def average_pq(self, panoptic_qualities):
122 |     # overall panoptic quality averaged from all classes
123 |     pq = 0.
124 |     n = 0
125 |     for item in panoptic_qualities.keys():
126 |       pq += panoptic_qualities[item]['pq']
127 |       n += 1
128 | 
129 |     if n == 0:
130 |       return -1
131 |     pq /= n
132 |     return pq
133 | 
134 |   def panoptic_quality_forward(self, batch_semantic_pred, batch_semantic_gt, batch_instance_pred, batch_instance_gt):
135 |     batch_size = batch_semantic_pred.shape[0]
136 |     for item in range(batch_size):
137 |       semantic_gt = batch_semantic_gt[item]
138 |       semantic_pred = batch_semantic_pred[item]
139 |       instance_gt = batch_instance_gt[item]
140 |       instance_pred = batch_instance_pred[item]
141 |       # for each batch item
142 |       pq, pqs = self.compute_pq(
143 |           semantic_pred, semantic_gt, instance_pred, instance_gt)
144 |     return pq, pqs


--------------------------------------------------------------------------------
/src/phenobench/evaluation/evaluate_semantics.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import json
  3 | from pathlib import Path
  4 | from typing import Dict
  5 | 
  6 | import torch
  7 | import yaml
  8 | from torchmetrics.classification import MulticlassJaccardIndex  # type: ignore
  9 | from tqdm import tqdm
 10 | 
 11 | from phenobench.evaluation.auxiliary.common import (
 12 |     get_png_files_in_dir,
 13 |     load_file_as_int_tensor,
 14 |     load_file_as_tensor,
 15 | )
 16 | from phenobench.evaluation.auxiliary.convert import convert_partial_semantics
 17 | 
 18 | torch.set_printoptions(precision=4, sci_mode=False)
 19 | 
 20 | 
 21 | def parse_args() -> Dict:
 22 |   parser = argparse.ArgumentParser()
 23 |   parser.add_argument('--phenobench_dir', required=True, type=Path, help='Path to the root directory of PhenoBench.')
 24 |   parser.add_argument('--prediction_dir', required=True, type=Path, help='Path to prediction directory.')
 25 |   parser.add_argument('--export', required=False, type=Path, help='Path to export directory.')
 26 |   parser.add_argument('--split',
 27 |                       choices=['train', 'val', 'test'],
 28 |                       required=True,
 29 |                       type=str,
 30 |                       help='Specify which split to use for evaluation.')
 31 |   parser.add_argument('--mapping', required=False, type=Path, help='Path to mapping.json file (only for test-split)')
 32 | 
 33 |   # Required directory structure
 34 |   # ├── PhenoBench
 35 |   # |   └── <split>
 36 |   # │       └── semantics
 37 |   # └── prediction
 38 |   #     └── semantics
 39 | 
 40 |   args = vars(parser.parse_known_args()[0])
 41 |   if "export" in args: args['export'].mkdir(parents=True, exist_ok=True)
 42 | 
 43 |   if args['split'] == 'test':
 44 |     if (not args['mapping']):
 45 |       raise TypeError(f"For the evaluation on the test-split you need to specify the '--mapping' arguments.")
 46 | 
 47 |   return args
 48 | 
 49 | 
 50 | def evaluate_semantics(args: Dict[str, str]) -> Dict[str, float]:
 51 |   """ Compute semantic segmentation scores: IoU for "crop", "weed", "soil" and "mIoU".
 52 | 
 53 |   Args:
 54 |       args (Dict): command line arguments specifying the required paths
 55 | 
 56 |   Returns:
 57 |       Dict: metrics for classes "soil", "crop", "weed"
 58 |   """
 59 | 
 60 |   # ------- Get all ground truth and prediction files -------
 61 |   gt_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'semantics')
 62 |   pred_fnames = get_png_files_in_dir(args['prediction_dir'] / 'semantics')
 63 | 
 64 |   # Load the mapping from randomized to original fnames
 65 |   if args['split'] == 'test':
 66 |     with open(args['mapping'], 'r') as istream:
 67 |       fname_mapping = json.load(istream)
 68 |     fname_mapping_reverse = {val: key for key, val in fname_mapping.items()}
 69 | 
 70 |     pred_original_fnames = []
 71 |     for pred_fname in pred_fnames:
 72 |       pred_original_fnames.append(fname_mapping[pred_fname])
 73 |     gt_fnames.sort()
 74 |     pred_original_fnames.sort()
 75 |   else:
 76 |     pred_original_fnames = pred_fnames
 77 |     fname_mapping_reverse = {fname: fname for fname in pred_fnames}
 78 | 
 79 |   # ------- Setup evaluator -------
 80 |   evaluator = MulticlassJaccardIndex(num_classes=3, average=None)
 81 |   if args['split'] == 'test':
 82 |     evaluator_2020 = MulticlassJaccardIndex(num_classes=3, average=None)
 83 |     evaluator_2021 = MulticlassJaccardIndex(num_classes=3, average=None)
 84 | 
 85 |   n_total = len(gt_fnames)
 86 |   for gt_fname, pred_fname in tqdm(zip(gt_fnames, pred_original_fnames), total=n_total):
 87 |     assert gt_fname == pred_fname
 88 | 
 89 |     semantics_gt = convert_partial_semantics(
 90 |         load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'semantics' / gt_fname)).squeeze()
 91 |     semantics_pred = convert_partial_semantics(
 92 |         load_file_as_int_tensor(args['prediction_dir'] / 'semantics' / fname_mapping_reverse[pred_fname])).squeeze()
 93 | 
 94 |     if args['split'] == 'test':
 95 |       if '2021' not in gt_fname:
 96 |         evaluator_2020.update(semantics_pred, semantics_gt)
 97 |       if '2021' in gt_fname:
 98 |         evaluator_2021.update(semantics_pred, semantics_gt)
 99 | 
100 |     evaluator.update(semantics_pred, semantics_gt)
101 | 
102 |   metrics = evaluator.compute() * 100.0  # tensor of shape [3]
103 | 
104 |   eval_results = {}
105 |   if args['split'] == 'test':
106 |     ext = "_all"
107 |   else:
108 |     ext = ""
109 |   eval_results[f"soil{ext}"] = round(float(metrics[0]), 2)
110 |   eval_results[f"crop{ext}"] = round(float(metrics[1]), 2)
111 |   eval_results[f"weed{ext}"] = round(float(metrics[2]), 2)
112 |   eval_results[f"mIoU{ext}"] = round(float(metrics.mean()), 2)
113 | 
114 |   if "export" in args:
115 |     fpath_out_all = args['export'] / 'all'
116 |     fpath_out_all.mkdir(parents=True, exist_ok=True)
117 |     with open(fpath_out_all / 'eval_semantics.yaml', 'w') as ostream:
118 |       yaml.dump(eval_results, ostream)
119 | 
120 |   if args['split'] == 'test':
121 |     # 2020
122 |     eval_results_2020 = {}
123 |     metrics_2020 = evaluator_2020.compute() * 100.0
124 |     ext = "_2020"
125 |     eval_results_2020[f"soil{ext}"] = round(float(metrics_2020[0]), 2)
126 |     eval_results_2020[f"crop{ext}"] = round(float(metrics_2020[1]), 2)
127 |     eval_results_2020[f"weed{ext}"] = round(float(metrics_2020[2]), 2)
128 |     eval_results_2020[f"mIoU{ext}"] = round(float(metrics_2020.mean()), 2)
129 | 
130 |     fpath_out_2020 = args['export'] / '2020'
131 |     fpath_out_2020.mkdir(parents=True, exist_ok=True)
132 |     with open(fpath_out_2020 / 'eval_semantics.yaml', 'w') as ostream:
133 |       yaml.dump(eval_results_2020, ostream)
134 | 
135 |     # 2021
136 |     eval_results_2021 = {}
137 |     metrics_2021 = evaluator_2021.compute() * 100.0
138 |     ext = "_2021"
139 |     eval_results_2021[f"soil{ext}"] = round(float(metrics_2021[0]), 2)
140 |     eval_results_2021[f"crop{ext}"] = round(float(metrics_2021[1]), 2)
141 |     eval_results_2021[f"weed{ext}"] = round(float(metrics_2021[2]), 2)
142 |     eval_results_2021[f"mIoU{ext}"] = round(float(metrics_2021.mean()), 2)
143 | 
144 |     if "export" in args:
145 |       fpath_out_2021 = args['export'] / '2021'
146 |       fpath_out_2021.mkdir(parents=True, exist_ok=True)
147 |       with open(fpath_out_2021 / 'eval_semantics.yaml', 'w') as ostream:
148 |         yaml.dump(eval_results_2021, ostream)
149 | 
150 |   return eval_results
151 | 
152 | 
153 | if __name__ == '__main__':
154 |   args = parse_args()
155 |   evaluate_semantics(args)
156 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/evaluate_leaf_instance_masks_panoptic.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import json
  3 | from pathlib import Path
  4 | from typing import Dict
  5 | 
  6 | import torch
  7 | import yaml
  8 | from tqdm import tqdm
  9 | 
 10 | from phenobench.evaluation.auxiliary.common import (
 11 |     get_png_files_in_dir,
 12 |     load_file_as_int_tensor,
 13 |     load_file_as_tensor,
 14 | )
 15 | from phenobench.evaluation.auxiliary.filter import filter_partial_masks
 16 | from phenobench.evaluation.auxiliary.panoptic_eval import PanopticEvaluator
 17 | 
 18 | 
 19 | def parse_args() -> Dict:
 20 |   parser = argparse.ArgumentParser()
 21 |   parser.add_argument('--phenobench_dir', required=True, type=Path, help='Path to ground truth directory.')
 22 |   parser.add_argument('--prediction_dir', required=True, type=Path, help='Path to prediction directory.')
 23 |   parser.add_argument('--export', required=True, type=Path, help='Path to export directory.')
 24 |   parser.add_argument('--split',
 25 |                       choices=['train', 'val', 'test'],
 26 |                       required=True,
 27 |                       type=str,
 28 |                       help='Specify which split to use for evaluation.')
 29 | 
 30 |   parser.add_argument('--mapping', required=False, type=Path, help='Path to mapping.json file (only for test-split)')
 31 |   parser.add_argument('--limit', required=False, type=int, help="Limit evaluation to the first N number of images.")
 32 | 
 33 |   # Required directory structure
 34 |   # ├── ground-truth
 35 |   # |   ├── leaf_visibility
 36 |   # │   └── leaf_instances
 37 |   # └── prediction
 38 |   #     └── leaf_instances
 39 | 
 40 |   args = vars(parser.parse_args())
 41 |   args['export'].mkdir(parents=True, exist_ok=True)
 42 | 
 43 |   if args['split'] == 'test':
 44 |     if (not args['mapping']):
 45 |       raise TypeError(f"For the evaluation on the test-split you need to specify the '--mapping' argument.")
 46 | 
 47 |   return args
 48 | 
 49 | 
 50 | def evaluate_leaf_instances(args) -> Dict[str, float]:
 51 | 
 52 |   # ------- Ground Truth -------
 53 |   gt_instance_fnames = get_png_files_in_dir(args['phenobench_dir'] / args["split"] / 'leaf_instances')
 54 |   gt_visibility_fnames = get_png_files_in_dir(args['phenobench_dir'] / args["split"] / 'leaf_visibility')
 55 | 
 56 |   # ------- Prediction -------
 57 |   pred_fnames = get_png_files_in_dir(args['prediction_dir'] / 'leaf_instances')
 58 | 
 59 |   # ------- Load the mapping from randomized to original fnames -------
 60 |   if args['split'] == 'test':
 61 |     with open(args['mapping'], 'r') as istream:
 62 |       fname_mapping = json.load(istream)
 63 |     fname_mapping_reverse = {val: key for key, val in fname_mapping.items()}
 64 | 
 65 |     pred_original_fnames = []
 66 |     for pred_fname in pred_fnames:
 67 |       pred_original_fnames.append(fname_mapping[pred_fname])
 68 |     gt_instance_fnames.sort()
 69 |     gt_visibility_fnames.sort()
 70 |     pred_original_fnames.sort()
 71 |   else:
 72 |     pred_original_fnames = pred_fnames
 73 |     fname_mapping_reverse = {fname: fname for fname in pred_fnames}
 74 | 
 75 |   # ------- Setup evaluator -------
 76 |   evaluator = PanopticEvaluator()
 77 |   if args['split'] == 'test':
 78 |     evaluator_2020 = PanopticEvaluator()
 79 |     evaluator_2021 = PanopticEvaluator()
 80 | 
 81 |   n_total = len(gt_instance_fnames)
 82 |   for gt_instance_fname, gt_visibility_fname, pred_fname in tqdm(zip(gt_instance_fnames, gt_visibility_fnames,
 83 |                                                                      pred_original_fnames),
 84 |                                                                  total=n_total):
 85 |     assert gt_instance_fname == gt_visibility_fname == pred_fname
 86 | 
 87 |     gt_instance_map = load_file_as_tensor(args['phenobench_dir'] / args["split"] / 'leaf_instances' /
 88 |                                           gt_instance_fname).squeeze()  # [H x W]
 89 |     gt_visibility = load_file_as_tensor(args['phenobench_dir'] / args["split"] / 'leaf_visibility' /
 90 |                                         gt_visibility_fname).squeeze()  # [H x W]
 91 |     gt_semantics = (gt_instance_map > 0).type(torch.uint8)  # derive semantics from instance map
 92 | 
 93 |     pred_instance_map = load_file_as_int_tensor(args['prediction_dir'] / 'leaf_instances' /
 94 |                                                 fname_mapping_reverse[pred_fname]).squeeze()  # [H x W]
 95 |     pred_semantics = (pred_instance_map > 0).type(torch.uint8)  # derive semantics from instance map
 96 | 
 97 |     filter_partial_masks(pred_instance_map, pred_semantics, gt_instance_map, gt_semantics, gt_visibility)
 98 | 
 99 |     if args['split'] == 'test':
100 |       if '2021' not in gt_instance_fname:
101 |         evaluator_2020.update(pred_semantics, gt_semantics, pred_instance_map, gt_instance_map)
102 |       if '2021' in gt_instance_fname:
103 |         evaluator_2021.update(pred_semantics, gt_semantics, pred_instance_map, gt_instance_map)
104 | 
105 |     evaluator.update(pred_semantics, gt_semantics, pred_instance_map, gt_instance_map)
106 | 
107 |   metrics = round(float(evaluator.compute() * 100), 2)
108 | 
109 |   eval_results = {}
110 |   if args['split'] == 'test':
111 |     ext = "_all"
112 |   else:
113 |     ext = ""
114 |   eval_results[f'leaves_pq{ext}'] = metrics
115 | 
116 |   if "export" in args:
117 |     fpath_out_all = args['export'] / 'all'
118 |     fpath_out_all.mkdir(parents=True, exist_ok=True)
119 |     with open(fpath_out_all / 'eval_leaf_pq.yaml', 'w') as ostream:
120 |       yaml.dump(eval_results, ostream)
121 | 
122 |   if args['split'] == 'test':
123 |     # 2020
124 |     metrics_2020 = round(float(evaluator_2020.compute() * 100), 2)
125 |     eval_results_2020 = {}
126 |     ext = "_2020"
127 |     eval_results_2020[f'leaves_pq{ext}'] = metrics_2020
128 | 
129 |     fpath_out_2020 = args['export'] / '2020'
130 |     fpath_out_2020.mkdir(parents=True, exist_ok=True)
131 |     with open(fpath_out_2020 / 'eval_leaf_pq.yaml', 'w') as ostream:
132 |       yaml.dump(eval_results_2020, ostream)
133 | 
134 |     # 2021
135 |     metrics_2021 = round(float(evaluator_2021.compute() * 100), 2)
136 |     eval_results_2021 = {}
137 |     ext = "_2021"
138 |     eval_results_2021[f'leaves_pq{ext}'] = metrics_2021
139 | 
140 |     fpath_out_2021 = args['export'] / '2021'
141 |     fpath_out_2021.mkdir(parents=True, exist_ok=True)
142 |     with open(fpath_out_2021 / 'eval_leaf_pq.yaml', 'w') as ostream:
143 |       yaml.dump(eval_results_2021, ostream)
144 | 
145 |   return eval_results
146 | 
147 | 
148 | if __name__ == '__main__':
149 |   args = parse_args()
150 |   evaluate_leaf_instances(args)
151 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/evaluate_leaf_bounding_boxes.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import json
  3 | from pathlib import Path
  4 | from typing import Dict
  5 | 
  6 | import torch
  7 | import yaml
  8 | from torchmetrics.detection.mean_ap import MeanAveragePrecision  # type: ignore
  9 | from tqdm import tqdm
 10 | 
 11 | from phenobench.evaluation.auxiliary.common import (
 12 |     get_png_files_in_dir,
 13 |     get_txt_files_in_dir,
 14 |     load_file_as_tensor,
 15 |     load_yolo_txt_file,
 16 | )
 17 | from phenobench.evaluation.auxiliary.convert import (
 18 |     cvt_gt_to_bbox_map,
 19 |     cvt_yolo_to_bbox_map,
 20 | )
 21 | from phenobench.evaluation.auxiliary.filter import filter_partials_boxes
 22 | 
 23 | 
 24 | def parse_args() -> Dict:
 25 |   parser = argparse.ArgumentParser()
 26 |   parser.add_argument('--phenobench_dir', required=True, type=Path, help='Path to ground truth directory.')
 27 |   parser.add_argument('--prediction_dir', required=True, type=Path, help='Path to prediction directory.')
 28 |   parser.add_argument('--export', required=True, type=Path, help='Path to export directory.')
 29 |   parser.add_argument('--split', choices=['train', 'val', 'test'], required=True, type=str, help='Specify which split to use for evaluation.')
 30 |   
 31 |   parser.add_argument('--mapping', required=False, type=Path, help='Path to mapping.json file (only for test-split)')
 32 |   # Required directory structure
 33 |   # ├── ground-truth
 34 |   # │   ├── leaf_instances
 35 |   # │   ├── leaf_visibility
 36 |   # └── prediction
 37 |   #     └── leaf_bboxes
 38 | 
 39 |   args = vars(parser.parse_args())
 40 | 
 41 |   args['export'].mkdir(parents=True, exist_ok=True)
 42 | 
 43 |   if args['split'] == 'test':
 44 |     if (not args['mapping']):
 45 |       raise TypeError(f"For the evaluation on the test-split you need to specify the '--date' and '--mapping' arguments.")
 46 | 
 47 |   return args
 48 | 
 49 | 
 50 | def evaluate_leaf_detection(args:Dict[str,str]) -> Dict[str,str]:
 51 | 
 52 |   # ------- Ground Truth -------
 53 |   gt_instance_fnames = get_png_files_in_dir(args['phenobench_dir']  / args["split"] / 'leaf_instances')
 54 |   gt_visibility_fnames = get_png_files_in_dir(args['phenobench_dir']  / args["split"] / 'leaf_visibility')
 55 | 
 56 |   # ------- Predictions -------
 57 |   pred_bboxes_fnames = get_txt_files_in_dir(args['prediction_dir'] / 'leaf_bboxes')
 58 | 
 59 |   # ------- Load the mapping from randomized to original fnames -------
 60 |   if args['split'] == 'test':
 61 |     with open(args['mapping'], 'r') as istream:
 62 |       fname_mapping = json.load(istream)
 63 |     fname_mapping_reverse = {val: key for key, val in fname_mapping.items()}
 64 | 
 65 |     pred_bboxes_original_fnames = []
 66 |     for pred_fname in pred_bboxes_fnames:
 67 |       pred_bboxes_original_fnames.append(fname_mapping[pred_fname])
 68 | 
 69 |     gt_instance_fnames.sort()
 70 |     gt_visibility_fnames.sort()
 71 |     pred_bboxes_original_fnames.sort()
 72 |   else:
 73 |     pred_bboxes_original_fnames = pred_bboxes_fnames
 74 |     fname_mapping_reverse = {fname:fname for fname in pred_bboxes_fnames}
 75 | 
 76 |   # ------- Setup evaluator -------
 77 |   evaluator = MeanAveragePrecision(box_format='cxcywh')
 78 |   if args['split'] == 'test':
 79 |     evaluator_2020 = MeanAveragePrecision(box_format='cxcywh')
 80 |     evaluator_2021 = MeanAveragePrecision(box_format='cxcywh')
 81 | 
 82 |   for gt_instance_fname, gt_visibility_fname, pred_bboxes_fnames in tqdm(zip(gt_instance_fnames, gt_visibility_fnames, pred_bboxes_original_fnames),total=len(gt_instance_fnames)):
 83 |     assert gt_instance_fname.split('.')[0] == gt_visibility_fname.split('.')[0] == pred_bboxes_fnames.split('.')[0]
 84 | 
 85 |     gt_instance_map = load_file_as_tensor(args['phenobench_dir']  / args["split"] / 'leaf_instances' / gt_instance_fname).squeeze()  # [H x W]
 86 |     gt_visibility = load_file_as_tensor(args['phenobench_dir']  / args["split"] / 'leaf_visibility' / gt_visibility_fname).squeeze()  # [H x W]
 87 |     gt_semantics = (gt_instance_map > 0).type(torch.uint8)  # derive semantics from instance map
 88 |     targets = cvt_gt_to_bbox_map(gt_instance_map, gt_semantics, gt_visibility)
 89 | 
 90 |     yolo_pred = load_yolo_txt_file(args['prediction_dir'] / 'leaf_bboxes' / fname_mapping_reverse[pred_bboxes_fnames])
 91 |     preds = cvt_yolo_to_bbox_map(yolo_pred)
 92 | 
 93 |     filter_partials_boxes(preds[0], targets[0])
 94 | 
 95 |     if args['split'] == 'test':
 96 |       if '2021' not in gt_instance_fname:
 97 |         evaluator_2020.update(preds, targets)
 98 |       if '2021' in gt_instance_fname:
 99 |         evaluator_2021.update(preds, targets)
100 | 
101 |     evaluator.update(preds, targets)
102 | 
103 |   metrics = evaluator.compute()
104 |   eval_results = {}
105 |   if args['split'] == 'test':
106 |     ext = "_all"
107 |   else:
108 |     ext = ""
109 |   eval_results[f'mAP{ext}'] = round(float(metrics['map'] * 100),2)
110 |   eval_results[f'mAP_50{ext}'] = round(float(metrics['map_50'] * 100), 2)
111 |   eval_results[f'mAP_75{ext}'] = round(float(metrics['map_75'] * 100), 2)
112 |   
113 |   if "export" in args:
114 |     fpath_out_all = args['export'] / 'all'
115 |     fpath_out_all.mkdir(parents=True, exist_ok=True)
116 |     with open(fpath_out_all / 'eval_leaf_bboxes.yaml', 'w') as ostream:
117 |       yaml.dump(eval_results, ostream)
118 | 
119 |   if args['split'] == 'test':
120 |     metrics_2020 = evaluator_2020.compute()
121 |     eval_results_2020 = {}
122 |     ext = "_2020"
123 |     eval_results_2020[f'mAP{ext}'] = round(float(metrics_2020['map'] * 100), 2)
124 |     eval_results_2020[f'mAP_50{ext}'] = round(float(metrics_2020['map_50'] * 100), 2)
125 |     eval_results_2020[f'mAP_75{ext}'] = round(float(metrics_2020['map_75'] * 100), 2)
126 |     
127 |     fpath_out_2020 = args['export'] / '2020'
128 |     fpath_out_2020.mkdir(parents=True, exist_ok=True)
129 |     with open(fpath_out_2020 / 'eval_leaf_bboxes.yaml', 'w') as ostream:
130 |       yaml.dump(eval_results_2020, ostream)
131 | 
132 |     metrics_2021 = evaluator_2021.compute()
133 |     eval_results_2021 = {}
134 |     ext = "_2021"
135 |     eval_results_2021[f'mAP{ext}'] = round(float(metrics_2021['map'] * 100), 2)
136 |     eval_results_2021[f'mAP_50{ext}'] = round(float(metrics_2021['map_50'] * 100), 2)
137 |     eval_results_2021[f'mAP_75{ext}'] = round(float(metrics_2021['map_75'] * 100), 2)
138 |     
139 |     if "export" in args:
140 |       fpath_out_2021 = args['export'] / '2021'
141 |       fpath_out_2021.mkdir(parents=True, exist_ok=True)
142 |       with open(fpath_out_2021 / 'eval_leaf_bboxes.yaml', 'w') as ostream:
143 |         yaml.dump(eval_results_2021, ostream)
144 | 
145 |   return eval_results
146 | 
147 | if __name__ == '__main__':
148 |   args = parse_args()
149 |   evaluate_leaf_detection(args)
150 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/evaluate_plant_bounding_boxes.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import json
  3 | from pathlib import Path
  4 | from typing import Dict
  5 | 
  6 | import yaml
  7 | from torchmetrics.detection.mean_ap import MeanAveragePrecision  # type: ignore
  8 | from tqdm import tqdm
  9 | 
 10 | from phenobench.evaluation.auxiliary.common import (
 11 |     get_png_files_in_dir,
 12 |     get_txt_files_in_dir,
 13 |     load_file_as_tensor,
 14 |     load_yolo_txt_file,
 15 | )
 16 | from phenobench.evaluation.auxiliary.convert import (
 17 |     cvt_gt_to_bbox_map,
 18 |     cvt_yolo_to_bbox_map,
 19 | )
 20 | from phenobench.evaluation.auxiliary.filter import filter_partials_boxes
 21 | 
 22 | IMG_WIDTH = 1024
 23 | IMG_HEIGHT = 1024
 24 | 
 25 | 
 26 | def parse_args() -> Dict:
 27 |   parser = argparse.ArgumentParser()
 28 |   parser.add_argument('--phenobench_dir', required=True, type=Path, help='Path to ground truth directory.')
 29 |   parser.add_argument('--prediction_dir', required=True, type=Path, help='Path to prediction directory.')
 30 |   parser.add_argument('--export', required=True, type=Path, help='Path to export directory.')
 31 |   parser.add_argument('--split',
 32 |                       choices=['train', 'val', 'test'],
 33 |                       required=True,
 34 |                       type=str,
 35 |                       help='Specify which split to use for evaluation.')
 36 | 
 37 |   parser.add_argument('--mapping', required=False, type=Path, help='Path to mapping.json file (only for test-split)')
 38 |   # Required directory structure
 39 |   # ├── ground-truth
 40 |   # │   ├── plant_instances
 41 |   # │   ├── plant_visibility
 42 |   # │   └── semantics
 43 |   # └── prediction
 44 |   #     └── plant_bboxes
 45 | 
 46 |   args = vars(parser.parse_args())
 47 | 
 48 |   args['export'].mkdir(parents=True, exist_ok=True)
 49 | 
 50 |   if args['split'] == 'test':
 51 |     if (not args['mapping']):
 52 |       raise TypeError(
 53 |           f"For the evaluation on the test-split you need to specify the '--date' and '--mapping' arguments.")
 54 | 
 55 |   return args
 56 | 
 57 | 
 58 | def evaluate_plant_detection(args) -> Dict[str, float]:
 59 | 
 60 |   # ------- Ground Truth -------
 61 |   gt_instance_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'plant_instances')
 62 |   gt_semantic_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'semantics')
 63 |   gt_visibility_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'plant_visibility')
 64 | 
 65 |   # ------- Predictions -------
 66 |   pred_bboxes_fnames = get_txt_files_in_dir(args['prediction_dir'] / 'plant_bboxes')
 67 | 
 68 |   # ------- Load the mapping from randomized to original fnames -------
 69 |   if args['split'] == 'test':
 70 |     with open(args['mapping'], 'r') as istream:
 71 |       fname_mapping = json.load(istream)
 72 |     fname_mapping_reverse = {val: key for key, val in fname_mapping.items()}
 73 | 
 74 |     pred_bboxes_original_fnames = []
 75 |     for pred_fname in pred_bboxes_fnames:
 76 |       pred_bboxes_original_fnames.append(fname_mapping[pred_fname])
 77 | 
 78 |     gt_instance_fnames.sort()
 79 |     gt_semantic_fnames.sort()
 80 |     gt_visibility_fnames.sort()
 81 |     pred_bboxes_original_fnames.sort()
 82 |   else:
 83 |     pred_bboxes_original_fnames = pred_bboxes_fnames
 84 |     fname_mapping_reverse = {fname: fname for fname in pred_bboxes_fnames}
 85 | 
 86 |   # ------- Setup evaluator -------
 87 |   evaluator = MeanAveragePrecision(box_format='cxcywh', class_metrics=True)
 88 |   if args['split'] == 'test':
 89 |     evaluator_2020 = MeanAveragePrecision(box_format='cxcywh', class_metrics=True)
 90 |     evaluator_2021 = MeanAveragePrecision(box_format='cxcywh', class_metrics=True)
 91 | 
 92 |   for gt_instance_fname, gt_semantic_fname, gt_visibility_fname, pred_bboxes_fnames in tqdm(
 93 |       zip(gt_instance_fnames, gt_semantic_fnames, gt_visibility_fnames, pred_bboxes_original_fnames),
 94 |       total=len(gt_instance_fnames)):
 95 |     assert gt_instance_fname.split('.')[0] == gt_semantic_fname.split('.')[0] == gt_visibility_fname.split(
 96 |         '.')[0] == pred_bboxes_fnames.split('.')[0]
 97 | 
 98 |     gt_instance_map = load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'plant_instances' /
 99 |                                           gt_instance_fname).squeeze()  # [H x W]
100 |     gt_semantics = load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'semantics' /
101 |                                        gt_semantic_fname).squeeze()  # [H x W]
102 |     gt_visibility = load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'plant_visibility' /
103 |                                         gt_instance_fname).squeeze()  # [H x W]
104 |     targets = cvt_gt_to_bbox_map(gt_instance_map, gt_semantics, gt_visibility)
105 | 
106 |     yolo_pred = load_yolo_txt_file(args['prediction_dir'] / 'plant_bboxes' / fname_mapping_reverse[pred_bboxes_fnames])
107 |     preds = cvt_yolo_to_bbox_map(yolo_pred)
108 | 
109 |     # Remove predictions and ground truths that belong to partially visible instances
110 |     filter_partials_boxes(preds[0], targets[0])
111 |     if args['split'] == 'test':
112 |       if '2021' not in gt_instance_fname:
113 |         evaluator_2020.update(preds, targets)
114 |       if '2021' in gt_instance_fname:
115 |         evaluator_2021.update(preds, targets)
116 | 
117 |     evaluator.update(preds, targets)
118 | 
119 |   metrics = evaluator.compute()
120 |   eval_results = {}
121 |   if args['split'] == 'test':
122 |     ext = "_all"
123 |   else:
124 |     ext = ""
125 |   eval_results[f'mAP{ext}'] = round(float(metrics['map'] * 100), 2)
126 |   eval_results[f'mAP_50{ext}'] = round(float(metrics['map_50'] * 100), 2)
127 |   eval_results[f'mAP_75{ext}'] = round(float(metrics['map_75'] * 100), 2)
128 |   eval_results[f'mAP_cls{ext}'] = [round(float(val * 100), 2) for val in metrics['map_per_class']]
129 | 
130 |   if "export" in args:
131 |     fpath_out_all = args['export'] / 'all'
132 |     fpath_out_all.mkdir(parents=True, exist_ok=True)
133 |     with open(fpath_out_all / 'eval_plant_bboxes.yaml', 'w') as ostream:
134 |       yaml.dump(eval_results, ostream)
135 | 
136 |   if args['split'] == 'test':
137 |     metrics_2020 = evaluator_2020.compute()
138 |     eval_results_2020 = {}
139 |     ext = "_2020"
140 |     eval_results_2020[f'mAP{ext}'] = round(float(metrics_2020['map'] * 100), 2)
141 |     eval_results_2020[f'mAP_50{ext}'] = round(float(metrics_2020['map_50'] * 100), 2)
142 |     eval_results_2020[f'mAP_75{ext}'] = round(float(metrics_2020['map_75'] * 100), 2)
143 |     eval_results_2020[f'mAP_cls{ext}'] = [round(float(val * 100), 2) for val in metrics_2020['map_per_class']]
144 | 
145 |     if "export" in args:
146 |       fpath_out_2020 = args['export'] / '2020'
147 |       fpath_out_2020.mkdir(parents=True, exist_ok=True)
148 |       with open(fpath_out_2020 / 'eval_plant_bboxes.yaml', 'w') as ostream:
149 |         yaml.dump(eval_results_2020, ostream)
150 | 
151 |     metrics_2021 = evaluator_2021.compute()
152 |     eval_results_2021 = {}
153 |     ext = "_2021"
154 |     eval_results_2021[f'mAP{ext}'] = round(float(metrics_2021['map'] * 100), 2)
155 |     eval_results_2021[f'mAP_50{ext}'] = round(float(metrics_2021['map_50'] * 100), 2)
156 |     eval_results_2021[f'mAP_75{ext}'] = round(float(metrics_2021['map_75'] * 100), 2)
157 |     eval_results_2021[f'mAP_cls{ext}'] = [round(float(val * 100), 2) for val in metrics_2021['map_per_class']]
158 | 
159 |     fpath_out_2021 = args['export'] / '2021'
160 |     fpath_out_2021.mkdir(parents=True, exist_ok=True)
161 |     with open(fpath_out_2021 / 'eval_plant_bboxes.yaml', 'w') as ostream:
162 |       yaml.dump(eval_results_2021, ostream)
163 | 
164 |   return eval_results
165 | 
166 | 
167 | if __name__ == '__main__':
168 |   args = parse_args()
169 |   evaluate_plant_detection(args)
170 | 


--------------------------------------------------------------------------------
/src/phenobench/phenobench_loader.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | from typing import Dict, List
  3 | 
  4 | import numpy as np
  5 | from PIL import Image
  6 | 
  7 | 
  8 | class PhenoBench:
  9 |   """ `PhenoBench <https://www.phenobench.org/>`_ Dataset.
 10 | 
 11 |     This is a very basic dataloader with some utility function that can be easily integrated into a PyTorch, 
 12 |     PyTorch Lightning, etc. data loader. We kept the requirements low profile to ensure extensibility.
 13 |   
 14 |     Args:
 15 |       root (str): Root directory of the PhenoBench dataset, where the folders `train`, `val`, and `test` data is location.
 16 |       split (str, optional): The image split to use, `train`, `test`, or `val`.
 17 |       target_type (List[str], optional): Types of targets to use and provide with each sample, i.e., `semantics`, `plant_instances`, `leaf_instances`, `plant_bboxes`, `leaf_bboxes`, `plant_visibility`, and/or `leaf_visibility`.
 18 |       ignore_partial (bool, optional): Ignore partial annotations? if `True` all partial plants, i.e., will be mask using the `ignore_mask` in `semantics` and mapped to `0` in the instance masks. Default: `False`.
 19 |       make_unique_ids (bool, optional): Make the instance ids of weeds and crops unique. Default: True.
 20 |       ignore_mask(int, optional): ignore_index. Default: 255.
 21 |   """
 22 |   def __init__(self,
 23 |                root: str,
 24 |                split: str = 'train',
 25 |                target_types: List[str] = ['semantics'],
 26 |                ignore_partial: bool = False,
 27 |                make_unique_ids: bool = True,
 28 |                ignore_mask: int = 255) -> None:
 29 | 
 30 |     # sanity checks:
 31 |     root = os.path.expanduser(root)
 32 |     assert os.path.exists(root), f"The path to the dataset does not exist: `{root}`."
 33 |     assert split in ['train', 'val', 'test']
 34 |     for target_type in target_types:
 35 |       assert target_type in [
 36 |           'semantics', 'plant_instances', 'leaf_instances', 'plant_bboxes', 'leaf_bboxes', 'plant_visibility',
 37 |           'leaf_visibility'
 38 |       ]
 39 | 
 40 |     self.root = root
 41 |     self.split = split
 42 |     self.target_types = target_types
 43 |     self.ignore_partial = ignore_partial
 44 |     self.make_unique_ids = make_unique_ids
 45 |     self.ignore_mask = ignore_mask
 46 | 
 47 |     self.filenames = sorted(os.listdir(os.path.join(self.root, split, "images")))
 48 | 
 49 |   def __getitem__(self, index: int) -> Dict[str, np.array]:
 50 |     """
 51 |     Args:
 52 |         index (int): index of the image & annotation
 53 | 
 54 |     Returns:
 55 |         Dict[str, Any]: returns a sample with the specified target_types.
 56 | 
 57 |         A data point is comprised of the following information (at least):
 58 |         ```
 59 |         {
 60 |           "image_name": "
 61 |           "image": PIL Image with size W x H.
 62 |           ...
 63 |           <target>: np.array of shape W x H or a dictionary with the bounding boxes.
 64 |         }
 65 |         ```
 66 |     """
 67 |     sample = {}
 68 | 
 69 |     sample["image_name"] = self.filenames[index]
 70 |     sample["image"] = Image.open(os.path.join(self.root, self.split, "images", self.filenames[index])).convert("RGB")
 71 | 
 72 |     if self.split in ["train", "val"]:
 73 | 
 74 |       for target in self.target_types:
 75 |         if target in ["semantics", "plant_instances", "leaf_instances", "plant_visibility", "leaf_visibility"]:
 76 |           sample[target] = np.array(Image.open(os.path.join(self.root, self.split, target, self.filenames[index])))
 77 | 
 78 |       if self.ignore_partial:
 79 |         semantics = np.array(Image.open(os.path.join(self.root, self.split, "semantics", self.filenames[index])))
 80 |         partial_crops = semantics == 3
 81 |         partial_weeds = semantics == 4
 82 | 
 83 |         if "semantics" in self.target_types:
 84 |           sample["semantics"][partial_crops] = self.ignore_mask
 85 |           sample["semantics"][partial_weeds] = self.ignore_mask
 86 | 
 87 |         for target in self.target_types:
 88 |           if target in ["plant_instances", "leaf_instances"]:
 89 |             sample[target][partial_crops] = 0
 90 |             sample[target][partial_weeds] = 0
 91 |           ## FIXME: do something for the bounding boxes.
 92 |           if target in ["plant_bboxes", "leaf_bboxes"]:
 93 |             pass
 94 | 
 95 |       else:
 96 |         if "semantics" in self.target_types:
 97 |           # remap partial_crop to crop and partial_weed to weed:
 98 |           sample["semantics"][sample["semantics"] == 3] = 1
 99 |           sample["semantics"][sample["semantics"] == 4] = 2
100 | 
101 |         if "plant_bboxes" in self.target_types:
102 |           semantics = np.array(Image.open(os.path.join(self.root, self.split, "semantics", self.filenames[index])))
103 |           plant_instances = np.array(
104 |               Image.open(os.path.join(self.root, self.split, "plant_instances", self.filenames[index])))
105 |           leaf_instances = np.array(
106 |               Image.open(os.path.join(self.root, self.split, "leaf_instances", self.filenames[index])))
107 | 
108 |           sample["plant_bboxes"] = []
109 |           for label in [1, 2]:
110 |             for plant_id in np.unique(plant_instances[semantics == label]):
111 |               ys, xs = np.where((plant_instances == plant_id) & (semantics == label))
112 | 
113 |               width, height = np.max(xs) - np.min(xs), np.max(ys) - np.min(ys)
114 |               center = (np.min(xs) + width // 2, np.min(ys) + height // 2)
115 |               sample["plant_bboxes"].append({
116 |                   "label": label,
117 |                   "corner": (np.min(xs), np.min(ys)),
118 |                   "center": center,
119 |                   "width": width,
120 |                   "height": height
121 |               })
122 | 
123 |         if "leaf_bboxes" in self.target_types:
124 |           semantics = np.array(Image.open(os.path.join(self.root, self.split, "semantics", self.filenames[index])))
125 |           leaf_instances = np.array(
126 |               Image.open(os.path.join(self.root, self.split, "leaf_instances", self.filenames[index])))
127 | 
128 |           sample["leaf_bboxes"] = []
129 |           for leaf_id in np.unique(leaf_instances[leaf_instances > 0]):
130 |             ys, xs = np.where((leaf_instances == leaf_id))
131 | 
132 |             width, height = np.max(xs) - np.min(xs), np.max(ys) - np.min(ys)
133 |             center = (np.min(xs) + width // 2, np.min(ys) + height // 2)
134 |             sample["leaf_bboxes"].append({
135 |                 "label": 1,
136 |                 "corner": (np.min(xs), np.min(ys)),
137 |                 "center": center,
138 |                 "width": width,
139 |                 "height": height
140 |             })
141 | 
142 |     # remapping the instance ids to consecutive ids:
143 |     if self.make_unique_ids:
144 | 
145 |       def replace(array: np.array, values, replacements):
146 |         temp_array = array.copy()
147 | 
148 |         for v, r in zip(values, replacements):
149 |           temp_array[array == v] = r
150 | 
151 |         array = temp_array
152 | 
153 |       if "plant_instances" in self.target_types:
154 |         semantics = np.array(Image.open(os.path.join(self.root, self.split, "semantics", self.filenames[index])))
155 | 
156 |         crop_ids = np.unique(sample["plant_instances"][semantics == 1])
157 |         weed_ids = np.unique(sample["plant_instances"][semantics == 2])
158 | 
159 |         N, M = len(crop_ids), len(weed_ids)
160 | 
161 |         replace(sample["plant_instances"][semantics == 1], crop_ids, np.arange(1, N + 1))
162 |         replace(sample["plant_instances"][semantics == 2], weed_ids, np.arange(N + 1, N + M + 1))
163 | 
164 |       if "leaf_instances" in self.target_types:
165 |         leaf_ids = np.unique(sample["leaf_instances"][sample["leaf_instances"] > 0])
166 | 
167 |         replace(sample["leaf_instances"], leaf_ids, np.arange(1, len(leaf_ids) + 1))
168 | 
169 |     return sample
170 | 
171 |   def __len__(self):
172 |     return len(self.filenames)
173 | 


--------------------------------------------------------------------------------
/src/phenobench/evaluation/evaluate_plant_instance_masks_panoptic.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import json
  3 | from pathlib import Path
  4 | from typing import Dict
  5 | 
  6 | import yaml
  7 | from tqdm import tqdm
  8 | 
  9 | from phenobench.evaluation.auxiliary.common import (
 10 |     get_png_files_in_dir,
 11 |     load_file_as_int_tensor,
 12 |     load_file_as_tensor,
 13 | )
 14 | from phenobench.evaluation.auxiliary.filter import filter_partial_masks
 15 | from phenobench.evaluation.auxiliary.panoptic_eval import PanopticEvaluator
 16 | 
 17 | 
 18 | def parse_args() -> Dict:
 19 |   parser = argparse.ArgumentParser()
 20 |   parser.add_argument('--phenobench_dir', required=True, type=Path, help='Path to ground truth directory.')
 21 |   parser.add_argument('--prediction_dir', required=True, type=Path, help='Path to prediction directory.')
 22 |   parser.add_argument('--export', required=False, type=Path, help='Path to export directory.')
 23 |   parser.add_argument('--split',
 24 |                       choices=['train', 'val', 'test'],
 25 |                       required=True,
 26 |                       type=str,
 27 |                       help='Specify which split to use for evaluation.')
 28 |   parser.add_argument('--mapping', required=False, type=Path, help='Path to mapping.json file (only for test-split)')
 29 | 
 30 |   # Required directory structure
 31 |   # ├── ground-truth
 32 |   # |   └── <split>
 33 |   # │       ├── plant_instances
 34 |   # |       ├── plant_visibility
 35 |   # │       └── semantics
 36 |   # └── prediction
 37 |   #     └── plant_instances
 38 |   #     └── semantics
 39 | 
 40 |   args = vars(parser.parse_args())
 41 |   args['export'].mkdir(parents=True, exist_ok=True)
 42 | 
 43 |   if args['split'] == 'test':
 44 |     if (not args['mapping']):
 45 |       raise TypeError(
 46 |           f"For the evaluation on the test-split you need to specify the '--date' and '--mapping' arguments.")
 47 | 
 48 |   return args
 49 | 
 50 | 
 51 | def evaluate_plant_instances(args: Dict[str, str]) -> Dict[str, float]:
 52 | 
 53 |   # ------- Ground Truth -------
 54 |   gt_instance_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'plant_instances')
 55 |   gt_semantic_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'semantics')
 56 |   gt_visibility_fnames = get_png_files_in_dir(args['phenobench_dir'] / args['split'] / 'plant_visibility')
 57 | 
 58 |   # ------- Predictions -------
 59 |   pred_instance_fnames = get_png_files_in_dir(args['prediction_dir'] / 'plant_instances')
 60 |   pred_semantic_fnames = get_png_files_in_dir(args['prediction_dir'] / 'semantics')
 61 | 
 62 |   # ------- Load the mapping from randomized to original fnames -------
 63 |   if args['split'] == 'test':
 64 |     with open(args['mapping'], 'r') as istream:
 65 |       fname_mapping = json.load(istream)
 66 |     fname_mapping_reverse = {val: key for key, val in fname_mapping.items()}
 67 | 
 68 |     pred_instance_original_fnames = []
 69 |     pred_semantic_original_fnames = []
 70 |     for pred_inst_fname, pred_sem_fname in zip(pred_instance_fnames, pred_semantic_fnames):
 71 |       pred_instance_original_fnames.append(fname_mapping[pred_inst_fname])
 72 |       pred_semantic_original_fnames.append(fname_mapping[pred_sem_fname])
 73 |     gt_instance_fnames.sort()
 74 |     gt_semantic_fnames.sort()
 75 |     gt_visibility_fnames.sort()
 76 |     pred_instance_original_fnames.sort()
 77 |     pred_semantic_original_fnames.sort()
 78 |   else:
 79 |     pred_instance_original_fnames = pred_instance_fnames
 80 |     pred_semantic_original_fnames = pred_semantic_fnames
 81 |     fname_mapping_reverse = {fname: fname for fname in pred_instance_fnames}
 82 | 
 83 |   # ------- Setup evaluator -------
 84 |   evaluator = PanopticEvaluator()
 85 |   if args['split'] == 'test':
 86 |     evaluator_2020 = PanopticEvaluator()
 87 |     evaluator_2021 = PanopticEvaluator()
 88 | 
 89 |   n_total = len(gt_instance_fnames)
 90 |   for gt_instance_fname, gt_semantic_fname, gt_visibility_fname, pred_instance_fname, pred_semantic_fname, in tqdm(
 91 |       zip(gt_instance_fnames, gt_semantic_fnames, gt_visibility_fnames, pred_instance_original_fnames,
 92 |           pred_semantic_original_fnames),
 93 |       total=n_total):
 94 |     assert gt_instance_fname == gt_semantic_fname == gt_visibility_fname == pred_instance_fname == pred_semantic_fname
 95 | 
 96 |     gt_instance_map = load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'plant_instances' /
 97 |                                           gt_instance_fname).squeeze()  # [H x W]
 98 |     gt_semantics = load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'semantics' /
 99 |                                        gt_semantic_fname).squeeze()  # [H x W]
100 |     gt_visibility = load_file_as_tensor(args['phenobench_dir'] / args['split'] / 'plant_visibility' /
101 |                                         gt_instance_fname).squeeze()  # [H x W]
102 | 
103 |     pred_instance_map = load_file_as_int_tensor(args['prediction_dir'] / 'plant_instances' /
104 |                                                 fname_mapping_reverse[pred_instance_fname]).squeeze()  # [H x W]
105 |     pred_semantics = load_file_as_int_tensor(args['prediction_dir'] / 'semantics' /
106 |                                              fname_mapping_reverse[pred_semantic_fname]).squeeze()  # [H x W]
107 | 
108 |     filter_partial_masks(pred_instance_map, pred_semantics, gt_instance_map, gt_semantics, gt_visibility)
109 |     if args['split'] == 'test':
110 |       if '2021' not in gt_instance_fname:
111 |         evaluator_2020.update(pred_semantics, gt_semantics, pred_instance_map, gt_instance_map)
112 |       if '2021' in gt_instance_fname:
113 |         evaluator_2021.update(pred_semantics, gt_semantics, pred_instance_map, gt_instance_map)
114 | 
115 |     evaluator.update(pred_semantics, gt_semantics, pred_instance_map, gt_instance_map)
116 | 
117 |   metrics = round(float(evaluator.compute() * 100), 2)
118 |   metrics_classwise = evaluator.results_classwise
119 |   for class_id in metrics_classwise:
120 |     metrics_classwise[class_id]['pq'] = round(float(metrics_classwise[class_id]['pq'] * 100), 2)
121 | 
122 |   eval_results = {}
123 |   if args['split'] == 'test':
124 |     ext = "_all"
125 |   else:
126 |     ext = ""
127 | 
128 |   eval_results[f"plants_pq{ext}"] = metrics
129 |   eval_results[f"plants_cls{ext}"] = metrics_classwise
130 | 
131 |   if "export" in args:
132 |     fpath_out_all = args['export'] / 'all'
133 |     fpath_out_all.mkdir(parents=True, exist_ok=True)
134 |     with open(fpath_out_all / 'eval_plant_pq.yaml', 'w') as ostream:
135 |       yaml.dump(eval_results, ostream)
136 | 
137 |   if args['split'] == 'test':
138 |     # 2020
139 |     metrics_2020 = round(float(evaluator_2020.compute() * 100), 2)
140 |     metrics_2020_classwise = evaluator_2020.results_classwise
141 |     for class_id in metrics_2020_classwise:
142 |       metrics_2020_classwise[class_id]['pq'] = round(float(metrics_2020_classwise[class_id]['pq'] * 100), 2)
143 | 
144 |     eval_results_2020 = {}
145 |     ext = "_2020"
146 |     eval_results_2020[f"plants_pq{ext}"] = metrics_2020
147 |     eval_results_2020[f"plants_cls{ext}"] = metrics_2020_classwise
148 | 
149 |     fpath_out_2020 = args['export'] / '2020'
150 |     fpath_out_2020.mkdir(parents=True, exist_ok=True)
151 |     with open(fpath_out_2020 / 'eval_plant_pq.yaml', 'w') as ostream:
152 |       yaml.dump(eval_results_2020, ostream)
153 | 
154 |     # 2021
155 |     metrics_2021 = round(float(evaluator_2021.compute() * 100), 2)
156 |     metrics_2021_classwise = evaluator_2021.results_classwise
157 |     for class_id in metrics_2021_classwise:
158 |       metrics_2021_classwise[class_id]['pq'] = round(float(metrics_2021_classwise[class_id]['pq'] * 100), 2)
159 | 
160 |     eval_results_2021 = {}
161 |     ext = "_2021"
162 |     eval_results_2021[f"plants_pq{ext}"] = metrics_2021
163 |     eval_results_2021[f"plants_cls{ext}"] = metrics_2021_classwise
164 |     if "export" in args:
165 |       fpath_out_2021 = args['export'] / '2021'
166 |       fpath_out_2021.mkdir(parents=True, exist_ok=True)
167 |       with open(fpath_out_2021 / 'eval_plant_pq.yaml', 'w') as ostream:
168 |         yaml.dump(eval_results_2021, ostream)
169 | 
170 |   return eval_results
171 | 
172 | if __name__ == '__main__':
173 |   args = parse_args()
174 |   evaluate_plant_instances(args)
175 | 


--------------------------------------------------------------------------------
/phenobench_tutorial.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "attachments": {},
  5 |    "cell_type": "markdown",
  6 |    "metadata": {},
  7 |    "source": [
  8 |     "# Getting Started with PhenoBench \n",
  9 |     "\n",
 10 |     "In this short tutorial, we will show you the main parts of the dataset. We assume that you have downloaded the PhenoBench dataset from the [dataset website](https://www.phenobench.org/dataset.html) and placed it in your home directory at `~/data/PhenoBench`; this folder contains the train, val, and test set of the dataset."
 11 |    ]
 12 |   },
 13 |   {
 14 |    "attachments": {},
 15 |    "cell_type": "markdown",
 16 |    "metadata": {},
 17 |    "source": [
 18 |     "## Installation\n",
 19 |     "\n",
 20 |     "For installing the devkit, you simply have to type:\n",
 21 |     " \n",
 22 |     " ```pip install phenobench```\n",
 23 |     " \n",
 24 |     " This command will install the needed dependencies and provide a simple dataset class, which could also be used to implement your own PyTorch, Tensorflow, or other custom dataloader -- it can also be the basis for your very custom processing pipeline."
 25 |    ]
 26 |   },
 27 |   {
 28 |    "attachments": {},
 29 |    "cell_type": "markdown",
 30 |    "metadata": {},
 31 |    "source": [
 32 |     "## Loading the data\n",
 33 |     "\n",
 34 |     "In the PhenoBench devkit, we have the dataset class called `PhenoBench`, which provides functionally to read the data from a given `root` directory. Here, `root` corresponds to the aforementioned dataset directory, i.e., `~/data/PhenoBench`, which contains the following directories:\n",
 35 |     "\n",
 36 |     "<pre>\n",
 37 |     "PhenoBench\n",
 38 |     "├── test\n",
 39 |     "│   └── images\n",
 40 |     "├── train\n",
 41 |     "│   ├── images\n",
 42 |     "│   ├── leaf_instances\n",
 43 |     "│   ├── leaf_visibility\n",
 44 |     "│   ├── plant_instances\n",
 45 |     "│   ├── plant_visibility\n",
 46 |     "│   └── semantics\n",
 47 |     "└── val\n",
 48 |     "    ├── images\n",
 49 |     "    ├── leaf_instances\n",
 50 |     "    ├── leaf_visibility\n",
 51 |     "    ├── plant_instances\n",
 52 |     "    ├── plant_visibility\n",
 53 |     "    └── semantics\n",
 54 |     "</pre>\n",
 55 |     "\n",
 56 |     "Note that only train and val contain annotations for the specific `target_types`:\n",
 57 |     "\n",
 58 |     "- **semantics**: pixel-wise semantics as integers, such that `1` corresponds to crop, `2` corresponds to `weed` with additional partial labels with class id `3` for partial crops and `4` for partial weeds. Due to the annotation process, where we label complete plants, we can determine if the area of plant pixels in a cropped image is below 50% and mark these areas as partially visible.\n",
 59 |     "- **plant_instances**: pixel-wise instance id as integers. With `make_unique_ids = True` the dataloader will remap all instances to the range `[1, N + M]`, where `N` is the number of crop plants and `M` is the number of weed instances. Without `make_unique_id`, the ids correspond to the arbitrary instance id of the global image. \n",
 60 |     "- **leaf_instances**: pixel-wise instance id as integers. With `make_unique_ids = True` the dataloader will remap all instances to the range `[1, N + M]`, where `N` is the number of crop plants and `M` is the number of weed instances. Without `make_unique_id`, the ids correspond to the arbitrary instance id of the global image. \n",
 61 |     "- **plant_visibility**: pixel-wise visibility mask, where visibility is given in range `[0,1]` encoding the percentage of pixels visible in the image.\n",
 62 |     "- **leaf_visibility**: pixel-wise visibility mask, where visibility is given in range `[0,1]` encoding the percentage of pixels visible in the image.\n",
 63 |     "\n",
 64 |     "Additionally, there are \"meta-targets\" that are generated from the pixel-wise instance annotation:\n",
 65 |     "\n",
 66 |     "- **plant_bboxes**: plant bounding boxes represented as list of tuples containing `{\"label\", \"center\", \"width\", \"height\", \"corner\"}`, where `center` corresponds to the center of the bounding box, `width` and `height` refer tot he width/height of the bounding box, and `corner` to the corner coordinate of the upper-left corner.\n",
 67 |     "- **leaf_bboxes**: leaf bounding boxes represented as list of tuples containing `{\"label\", \"center\", \"width\", \"height\", \"corner\"}`, where `center` corresponds to the center of the bounding box, `width` and `height` refer tot he width/height of the bounding box, and `corner` to the corner coordinate of the upper-left corner.\n",
 68 |     "\n",
 69 |     "Enough theory ... Let's actually visualize some example images and annotations from the dataset."
 70 |    ]
 71 |   },
 72 |   {
 73 |    "cell_type": "code",
 74 |    "execution_count": null,
 75 |    "metadata": {},
 76 |    "outputs": [],
 77 |    "source": [
 78 |     "from phenobench import PhenoBench\n",
 79 |     "from pprint import pprint\n",
 80 |     "\n",
 81 |     "\n",
 82 |     "train_data = PhenoBench(\"~/data/PhenoBench\", \n",
 83 |     "                        target_types=[\"semantics\", \"plant_instances\", \"leaf_instances\", \"plant_bboxes\", \"leaf_bboxes\"])\n",
 84 |     "\n",
 85 |     "\n",
 86 |     "print(\n",
 87 |     "    f\"PhenoBench ({train_data.split} split) contains {len(train_data)} images. We loaded the following targets: {train_data.target_types}.\"\n",
 88 |     ")\n",
 89 |     "print(\"The first entry contains the following fields:\")\n",
 90 |     "pprint([f\"{k} -> {type(v)}\" for k, v in train_data[0].items()])\n"
 91 |    ]
 92 |   },
 93 |   {
 94 |    "attachments": {},
 95 |    "cell_type": "markdown",
 96 |    "metadata": {},
 97 |    "source": [
 98 |     "## Visualizing the Data\n",
 99 |     "\n",
100 |     "Besides the generic loader of the dataset, we also provide some visualization functions that can be used in combination with matplotlib. These drawing functions can be used to visualize the dataset, but also to render predictions.\n",
101 |     "\n",
102 |     "All drawing functions are available in the package `phenobench.visualization` and here we show some examples generated from the dataset itself."
103 |    ]
104 |   },
105 |   {
106 |    "cell_type": "code",
107 |    "execution_count": null,
108 |    "metadata": {},
109 |    "outputs": [],
110 |    "source": [
111 |     "import matplotlib.pyplot as plt\n",
112 |     "import numpy as np\n",
113 |     "import os\n",
114 |     "\n",
115 |     "from phenobench.visualization import draw_semantics, draw_instances, draw_bboxes\n",
116 |     "\n",
117 |     "\n",
118 |     "n_samples = 4\n",
119 |     "n_rows = 4\n",
120 |     "fig, axes = plt.subplots(ncols=n_samples, nrows=n_rows, figsize=(3 * n_samples, 3 * n_rows))\n",
121 |     "\n",
122 |     "indexes = np.random.choice(len(train_data), n_samples)\n",
123 |     "\n",
124 |     "for i in range(n_rows):\n",
125 |     "    for j in range(n_samples):\n",
126 |     "        axes[i, j].set_axis_off()\n",
127 |     "\n",
128 |     "for id, idx in enumerate(indexes):\n",
129 |     "    axes[0, id].set_title(os.path.splitext(train_data[idx][\"image_name\"])[0])\n",
130 |     "\n",
131 |     "    draw_semantics(axes[0, id], train_data[idx][\"image\"], train_data[idx][\"semantics\"], alpha=0.5)\n",
132 |     "    draw_instances(axes[1, id], train_data[idx][\"image\"], train_data[idx][\"plant_instances\"], alpha=0.5)\n",
133 |     "    draw_instances(axes[2, id], train_data[idx][\"image\"], train_data[idx][\"leaf_instances\"], alpha=0.5)\n",
134 |     "    draw_bboxes(axes[3, id], train_data[idx][\"image\"], train_data[idx][\"plant_bboxes\"])\n"
135 |    ]
136 |   },
137 |   {
138 |    "attachments": {},
139 |    "cell_type": "markdown",
140 |    "metadata": {},
141 |    "source": [
142 |     "## Conclusion\n",
143 |     "\n",
144 |     "If you have issues with the usage of the data, but also want to provide feedback regarding the functionality, feel free to open an issues or write us an email.\n",
145 |     "\n",
146 |     "\n",
147 |     "Good luck with the training of your models and we are looking forward to the amazing things you will do with our data."
148 |    ]
149 |   }
150 |  ],
151 |  "metadata": {
152 |   "kernelspec": {
153 |    "display_name": "Python 3",
154 |    "language": "python",
155 |    "name": "python3"
156 |   },
157 |   "language_info": {
158 |    "codemirror_mode": {
159 |     "name": "ipython",
160 |     "version": 3
161 |    },
162 |    "file_extension": ".py",
163 |    "mimetype": "text/x-python",
164 |    "name": "python",
165 |    "nbconvert_exporter": "python",
166 |    "pygments_lexer": "ipython3",
167 |    "version": "3.10.6"
168 |   }
169 |  },
170 |  "nbformat": 4,
171 |  "nbformat_minor": 2
172 | }
173 | 


--------------------------------------------------------------------------------