├── .gitignore ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── docs ├── installation.md └── schema_truckscenes.md ├── pyproject.toml ├── requirements.txt ├── setup.cfg ├── setup.py ├── setup └── requirements │ ├── requirements_base.txt │ ├── requirements_tutorial.txt │ └── requirements_visu.txt ├── src └── truckscenes │ ├── __init__.py │ ├── eval │ ├── __init__.py │ ├── common │ │ ├── __init__.py │ │ ├── config.py │ │ ├── constants.py │ │ ├── data_classes.py │ │ ├── loaders.py │ │ ├── render.py │ │ └── utils.py │ └── detection │ │ ├── README.md │ │ ├── __init__.py │ │ ├── algo.py │ │ ├── config.py │ │ ├── configs │ │ └── detection_cvpr_2024.json │ │ ├── constants.py │ │ ├── data_classes.py │ │ ├── evaluate.py │ │ ├── render.py │ │ └── utils.py │ ├── truckscenes.py │ └── utils │ ├── __init__.py │ ├── colormap.py │ ├── data_classes.py │ ├── geometry_utils.py │ ├── splits.py │ └── visualization_utils.py └── tutorials └── truckscenes_tutorial.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python,c++,visualstudiocode,pycharm,jupyternotebooks 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,c++,visualstudiocode,pycharm,jupyternotebooks 3 | 4 | ### C++ ### 5 | # Prerequisites 6 | *.d 7 | 8 | # Compiled Object files 9 | *.slo 10 | *.lo 11 | *.o 12 | *.obj 13 | 14 | # Precompiled Headers 15 | *.gch 16 | *.pch 17 | 18 | # Compiled Dynamic libraries 19 | *.so 20 | *.dylib 21 | *.dll 22 | 23 | # Fortran module files 24 | *.mod 25 | *.smod 26 | 27 | # Compiled Static libraries 28 | *.lai 29 | *.la 30 | *.a 31 | *.lib 32 | 33 | # Executables 34 | *.exe 35 | *.out 36 | *.app 37 | 38 | ### JupyterNotebooks ### 39 | # gitignore template for Jupyter Notebooks 40 | # website: http://jupyter.org/ 41 | 42 | .ipynb_checkpoints 43 | */.ipynb_checkpoints/* 44 | 45 | # IPython 46 | profile_default/ 47 | ipython_config.py 48 | 49 | # Remove previous ipynb_checkpoints 50 | # git rm -r .ipynb_checkpoints/ 51 | 52 | ### PyCharm ### 53 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 54 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 55 | 56 | # User-specific stuff 57 | .idea/**/workspace.xml 58 | .idea/**/tasks.xml 59 | .idea/**/usage.statistics.xml 60 | .idea/**/dictionaries 61 | .idea/**/shelf 62 | 63 | # AWS User-specific 64 | .idea/**/aws.xml 65 | 66 | # Generated files 67 | .idea/**/contentModel.xml 68 | 69 | # Sensitive or high-churn files 70 | .idea/**/dataSources/ 71 | .idea/**/dataSources.ids 72 | .idea/**/dataSources.local.xml 73 | .idea/**/sqlDataSources.xml 74 | .idea/**/dynamic.xml 75 | .idea/**/uiDesigner.xml 76 | .idea/**/dbnavigator.xml 77 | 78 | # Gradle 79 | .idea/**/gradle.xml 80 | .idea/**/libraries 81 | 82 | # Gradle and Maven with auto-import 83 | # When using Gradle or Maven with auto-import, you should exclude module files, 84 | # since they will be recreated, and may cause churn. Uncomment if using 85 | # auto-import. 86 | # .idea/artifacts 87 | # .idea/compiler.xml 88 | # .idea/jarRepositories.xml 89 | # .idea/modules.xml 90 | # .idea/*.iml 91 | # .idea/modules 92 | # *.iml 93 | # *.ipr 94 | 95 | # CMake 96 | cmake-build-*/ 97 | 98 | # Mongo Explorer plugin 99 | .idea/**/mongoSettings.xml 100 | 101 | # File-based project format 102 | *.iws 103 | 104 | # IntelliJ 105 | out/ 106 | 107 | # mpeltonen/sbt-idea plugin 108 | .idea_modules/ 109 | 110 | # JIRA plugin 111 | atlassian-ide-plugin.xml 112 | 113 | # Cursive Clojure plugin 114 | .idea/replstate.xml 115 | 116 | # SonarLint plugin 117 | .idea/sonarlint/ 118 | 119 | # Crashlytics plugin (for Android Studio and IntelliJ) 120 | com_crashlytics_export_strings.xml 121 | crashlytics.properties 122 | crashlytics-build.properties 123 | fabric.properties 124 | 125 | # Editor-based Rest Client 126 | .idea/httpRequests 127 | 128 | # Android studio 3.1+ serialized cache file 129 | .idea/caches/build_file_checksums.ser 130 | 131 | ### PyCharm Patch ### 132 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 133 | 134 | # *.iml 135 | # modules.xml 136 | # .idea/misc.xml 137 | # *.ipr 138 | 139 | # Sonarlint plugin 140 | # https://plugins.jetbrains.com/plugin/7973-sonarlint 141 | .idea/**/sonarlint/ 142 | 143 | # SonarQube Plugin 144 | # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin 145 | .idea/**/sonarIssues.xml 146 | 147 | # Markdown Navigator plugin 148 | # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced 149 | .idea/**/markdown-navigator.xml 150 | .idea/**/markdown-navigator-enh.xml 151 | .idea/**/markdown-navigator/ 152 | 153 | # Cache file creation bug 154 | # See https://youtrack.jetbrains.com/issue/JBR-2257 155 | .idea/$CACHE_FILE$ 156 | 157 | # CodeStream plugin 158 | # https://plugins.jetbrains.com/plugin/12206-codestream 159 | .idea/codestream.xml 160 | 161 | # Azure Toolkit for IntelliJ plugin 162 | # https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij 163 | .idea/**/azureSettings.xml 164 | 165 | ### Python ### 166 | # Byte-compiled / optimized / DLL files 167 | __pycache__/ 168 | *.py[cod] 169 | *$py.class 170 | 171 | # C extensions 172 | 173 | # Distribution / packaging 174 | .Python 175 | build/ 176 | develop-eggs/ 177 | dist/ 178 | downloads/ 179 | eggs/ 180 | .eggs/ 181 | lib/ 182 | lib64/ 183 | parts/ 184 | sdist/ 185 | var/ 186 | wheels/ 187 | share/python-wheels/ 188 | *.egg-info/ 189 | .installed.cfg 190 | *.egg 191 | MANIFEST 192 | 193 | # PyInstaller 194 | # Usually these files are written by a python script from a template 195 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 196 | *.manifest 197 | *.spec 198 | 199 | # Installer logs 200 | pip-log.txt 201 | pip-delete-this-directory.txt 202 | 203 | # Unit test / coverage reports 204 | htmlcov/ 205 | .tox/ 206 | .nox/ 207 | .coverage 208 | .coverage.* 209 | .cache 210 | nosetests.xml 211 | coverage.xml 212 | *.cover 213 | *.py,cover 214 | .hypothesis/ 215 | .pytest_cache/ 216 | cover/ 217 | 218 | # Translations 219 | *.mo 220 | *.pot 221 | 222 | # Django stuff: 223 | *.log 224 | local_settings.py 225 | db.sqlite3 226 | db.sqlite3-journal 227 | 228 | # Flask stuff: 229 | instance/ 230 | .webassets-cache 231 | 232 | # Scrapy stuff: 233 | .scrapy 234 | 235 | # Sphinx documentation 236 | docs/_build/ 237 | 238 | # PyBuilder 239 | .pybuilder/ 240 | target/ 241 | 242 | # Jupyter Notebook 243 | 244 | # IPython 245 | 246 | # pyenv 247 | # For a library or package, you might want to ignore these files since the code is 248 | # intended to run in multiple environments; otherwise, check them in: 249 | # .python-version 250 | 251 | # pipenv 252 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 253 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 254 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 255 | # install all needed dependencies. 256 | #Pipfile.lock 257 | 258 | # poetry 259 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 260 | # This is especially recommended for binary packages to ensure reproducibility, and is more 261 | # commonly ignored for libraries. 262 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 263 | #poetry.lock 264 | 265 | # pdm 266 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 267 | #pdm.lock 268 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 269 | # in version control. 270 | # https://pdm.fming.dev/#use-with-ide 271 | .pdm.toml 272 | 273 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 274 | __pypackages__/ 275 | 276 | # Celery stuff 277 | celerybeat-schedule 278 | celerybeat.pid 279 | 280 | # SageMath parsed files 281 | *.sage.py 282 | 283 | # Environments 284 | .env 285 | .venv 286 | env/ 287 | venv/ 288 | ENV/ 289 | env.bak/ 290 | venv.bak/ 291 | 292 | # Spyder project settings 293 | .spyderproject 294 | .spyproject 295 | 296 | # Rope project settings 297 | .ropeproject 298 | 299 | # mkdocs documentation 300 | /site 301 | 302 | # mypy 303 | .mypy_cache/ 304 | .dmypy.json 305 | dmypy.json 306 | 307 | # Pyre type checker 308 | .pyre/ 309 | 310 | # pytype static type analyzer 311 | .pytype/ 312 | 313 | # Cython debug symbols 314 | cython_debug/ 315 | 316 | # PyCharm 317 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 318 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 319 | # and can be added to the global gitignore or merged into this file. For a more nuclear 320 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 321 | #.idea/ 322 | 323 | ### Python Patch ### 324 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 325 | poetry.toml 326 | 327 | # ruff 328 | .ruff_cache/ 329 | 330 | # LSP config files 331 | pyrightconfig.json 332 | 333 | ### VisualStudioCode ### 334 | .vscode/ 335 | !.vscode/settings.json 336 | !.vscode/tasks.json 337 | !.vscode/launch.json 338 | !.vscode/extensions.json 339 | !.vscode/*.code-snippets 340 | 341 | # Local History for Visual Studio Code 342 | .history/ 343 | 344 | # Built Visual Studio Code Extensions 345 | *.vsix 346 | 347 | ### VisualStudioCode Patch ### 348 | # Ignore all local history of files 349 | .history 350 | .ionide 351 | 352 | # End of https://www.toptal.com/developers/gitignore/api/python,c++,visualstudiocode,pycharm,jupyternotebooks 353 | images 354 | scripts/test.sh 355 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2024 MAN Truck & Bus SE 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include *.txt 3 | recursive-include docs *.md 4 | recursive-include setup *.txt 5 | recursive-include src *.md 6 | recursive-include src *.py 7 | recursive-include src *.json 8 | recursive-include tutorials *.ipynb 9 | prune venv -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |

MAN TruckScenes devkit

4 | 5 | World's First Public Dataset For Autonomous Trucking 6 | 7 | [![Python](https://img.shields.io/badge/python-3-blue.svg)](https://www.python.org/downloads/) 8 | [![Linux](https://img.shields.io/badge/os-linux-blue.svg)](https://www.linux.org/) 9 | [![Windows](https://img.shields.io/badge/os-windows-blue.svg)](https://www.microsoft.com/windows/) 10 | [![arXiv](https://img.shields.io/badge/arXiv-Paper-blue.svg)](https://arxiv.org/abs/2407.07462) 11 | 12 | [![Watch the video](https://raw.githubusercontent.com/ffent/truckscenes-media/main/thumbnail.jpg)](https://cdn-assets-eu.frontify.com/s3/frontify-enterprise-files-eu/eyJwYXRoIjoibWFuXC9maWxlXC9lb2s3TGF5V1RXMXYxZU1TUk02US5tcDQifQ:man:MuLfMZFfol1xfBIL7rNw0W4SqczZqwTuzhvI-yxJmdY?width={width}&format=mp4) 13 | 14 |
15 | 16 | ## Overview 17 | - [Website](#website) 18 | - [Installation](#installation) 19 | - [Setup](#setup) 20 | - [Usage](#usage) 21 | - [Citation](#citation) 22 | 23 |
24 | 25 | ## 🌐 Website 26 | To read more about the dataset or download it, please visit [https://www.man.eu/truckscenes](https://www.man.eu/truckscenes) 27 | 28 |
29 | 30 | ## 💾 Installation 31 | Our devkit is available and can be installed via pip: 32 | ``` 33 | pip install truckscenes-devkit 34 | ``` 35 | 36 | If you also want to install all the (optional) dependencies for running the visualizations: 37 | ``` 38 | pip install "truckscenes-devkit[all]" 39 | ``` 40 | 41 | For more details on the installation see [installation](./docs/installation.md) 42 | 43 |
44 | 45 | ## 🔨 Setup 46 | Download **all** archives from our [download page](https://www.man.eu/truckscenes/) or the [AWS Open Data Registry](https://registry.opendata.aws/). 47 | 48 | Unpack the archives to the `/data/man-truckscenes` folder **without** overwriting folders that occur in multiple archives. 49 | Eventually you should have the following folder structure: 50 | ``` 51 | /data/man-truckscenes 52 | samples - Sensor data for keyframes. 53 | sweeps - Sensor data for intermediate frames. 54 | v1.0-* - JSON tables that include all the meta data and annotations. Each split (trainval, test, mini) is provided in a separate folder. 55 | ``` 56 | 57 |
58 | 59 | ## 🚀 Usage 60 | Please follow these steps to make yourself familiar with the MAN TruckScenes dataset: 61 | - Read the [dataset description](https://www.man.eu/truckscenes/). 62 | - Explore the dataset [videos](https://cdn-assets-eu.frontify.com/s3/frontify-enterprise-files-eu/eyJwYXRoIjoibWFuXC9maWxlXC9lb2s3TGF5V1RXMXYxZU1TUk02US5tcDQifQ:man:MuLfMZFfol1xfBIL7rNw0W4SqczZqwTuzhvI-yxJmdY?width={width}&format=mp4). 63 | - [Download](https://www.man.eu/truckscenes/) the dataset from our website. 64 | - Make yourself familiar with the [dataset schema](./docs/schema_truckscenes.md) 65 | - Run the [tutorial](./tutorials/truckscenes_tutorial.ipynb) to get started: 66 | - Read the [MAN TruckScenes paper](https://arxiv.org/abs/2407.07462) for a detailed analysis of the dataset. 67 | 68 |
69 | 70 | ## 📄 Citation 71 | ``` 72 | @inproceedings{truckscenes2024, 73 | title = {MAN TruckScenes: A multimodal dataset for autonomous trucking in diverse conditions}, 74 | author = {Fent, Felix and Kuttenreich, Fabian and Ruch, Florian and Rizwin, Farija and Juergens, Stefan and Lechermann, Lorenz and Nissler, Christian and Perl, Andrea and Voll, Ulrich and Yan, Min and Lienkamp, Markus}, 75 | booktitle = {Advances in Neural Information Processing Systems}, 76 | editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang}, 77 | pages = {62062--62082}, 78 | publisher = {Curran Associates, Inc.}, 79 | url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/71ac06f0f8450e7d49063c7bfb3257c2-Paper-Datasets_and_Benchmarks_Track.pdf}, 80 | volume = {37}, 81 | year = {2024} 82 | } 83 | ``` 84 | 85 | _Copied and adapted from [nuscenes-devkit](https://github.com/nutonomy/nuscenes-devkit)_ 86 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Advanced Installation 2 | We provide step-by-step instructions to install our devkit. 3 | - [Download](#download) 4 | - [Install Python](#install-python) 5 | - [Setup a virtual environment](#setup-a-virtual-environment) 6 | - [Setup PYTHONPATH](#setup-pythonpath) 7 | - [Install required packages](#install-required-packages) 8 | - [Setup environment variable](#setup-environment-variable) 9 | - [Setup Matplotlib backend](#setup-matplotlib-backend) 10 | - [Verify install](#verify-install) 11 | 12 | ## Download 13 | 14 | Download the [truckscenes-devkit](https://github.com/TUMFTM/truckscenes-devkit) to your home directory. 15 | 16 | ## Install Python 17 | The devkit is tested for Python 3.8 - 3.11 (recommended Python 3.8). 18 | 19 | **Ubuntu** 20 | ``` 21 | sudo apt update 22 | sudo apt install software-properties-common 23 | sudo add-apt-repository ppa:deadsnakes/ppa 24 | sudo apt update 25 | sudo apt install python3.8 26 | sudo apt install python3.8-dev 27 | ``` 28 | 29 | **Windows** 30 | Install python: [https://www.python.org/downloads/](https://www.python.org/downloads/) 31 | 32 | **Mac OS** 33 | Install python: [https://www.python.org/downloads/](https://www.python.org/downloads/) 34 | 35 | ## Setup a virtual environment 36 | For setting up a virtual environment we use venv. 37 | 38 | #### Install venv 39 | ``` 40 | sudo apt-get install python3-venv 41 | ``` 42 | 43 | #### Create the virtual environment named `venv` 44 | ``` 45 | python3 -m venv venv 46 | ``` 47 | 48 | #### Activate the virtual environment 49 | If you are inside the virtual environment, your shell prompt should look like: `(venv) user@computer:~$` 50 | If that is not the case, you can enable the virtual environment using: 51 | ``` 52 | source venv/bin/activate 53 | ``` 54 | To deactivate the virtual environment, use: 55 | ``` 56 | deactivate 57 | ``` 58 | 59 | ## Setup PYTHONPATH 60 | Add the `src` directory to your `PYTHONPATH` environmental variable: 61 | ``` 62 | export PYTHONPATH="${PYTHONPATH}:$HOME/truckscenes-devkit/src" 63 | ``` 64 | 65 | ## Install required packages 66 | 67 | To install the required packages, run the following command in your favorite virtual environment: 68 | ``` 69 | pip install -r requirements.txt 70 | ``` 71 | **Note:** The requirements file is internally divided into base requirements (`base`) and additional requirements (`tutorial`, `visu`). 72 | 73 | If you want to install these additional requirements, please run: 74 | ``` 75 | pip install -r setup/requirements/requirements_<>.txt 76 | ``` 77 | 78 | ## Setup environment variable 79 | Finally, if you want to run the unit tests you need to point the devkit to the `truckscenes` folder on your disk. 80 | Set the TRUCKSCENES environment variable to point to your data folder: 81 | ``` 82 | export TRUCKSCENES="/data/man-truckscenes" 83 | ``` 84 | 85 | ## Setup Matplotlib backend 86 | When using Matplotlib, it is generally recommended to define the backend used for rendering: 87 | 1) Under Ubuntu the default backend `Agg` results in any plot not being rendered by default. This does not apply inside Jupyter notebooks. 88 | 2) Under MacOSX a call to `plt.plot()` may fail with the following error (see [here](https://github.com/matplotlib/matplotlib/issues/13414) for more details): 89 | ``` 90 | libc++abi.dylib: terminating with uncaught exception of type NSException 91 | ``` 92 | To set the backend, add the following to your `~/.matplotlib/matplotlibrc` file, which needs to be created if it does not exist yet: 93 | ``` 94 | backend: TKAgg 95 | ``` 96 | 97 | ## Verify install 98 | To verify your environment run `python -m unittest` in the `test` folder. 99 | 100 | That's it you should be good to go! 101 |

102 | 103 | 104 | 105 | 106 | Copied and adapted from [nuscenes-devkit](https://github.com/nutonomy/nuscenes-devkit) -------------------------------------------------------------------------------- /docs/schema_truckscenes.md: -------------------------------------------------------------------------------- 1 | TruckScenes schema 2 | ========== 3 | This document describes the database schema used in MAN TruckScenes. 4 | All annotations and meta data (including calibration, taxonomy, vehicle coordinates etc.) are covered in a relational database. 5 | The database tables are listed below. 6 | Every row can be identified by its unique primary key `token`. 7 | Foreign keys such as `sample_token` may be used to link to the `token` of the table `sample`. 8 | 9 | attribute 10 | --------- 11 | An attribute is a property of an instance that can change while the category remains the same. 12 | Example: a vehicle being parked/stopped/moving, and whether or not a bicycle has a rider. 13 | ``` 14 | attribute { 15 | "token": -- Unique record identifier. 16 | "name": -- Attribute name. 17 | "description": -- Attribute description. 18 | } 19 | ``` 20 | 21 | calibrated_sensor 22 | --------- 23 | Definition of a particular sensor (lidar/radar/camera) as calibrated on a particular vehicle. 24 | All extrinsic parameters are given with respect to the ego vehicle body frame. 25 | All camera images come undistorted and rectified. 26 | ``` 27 | calibrated_sensor { 28 | "token": -- Unique record identifier. 29 | "sensor_token": -- Foreign key pointing to the sensor type. 30 | "translation": [3] -- Coordinate system origin in meters: x, y, z. 31 | "rotation": [4] -- Coordinate system orientation as quaternion: w, x, y, z. 32 | "camera_intrinsic": [3, 3] -- Intrinsic camera calibration. Empty for sensors that are not cameras. 33 | } 34 | ``` 35 | 36 | category 37 | --------- 38 | Taxonomy of object categories (e.g. vehicle, human). 39 | Subcategories are delineated by a period (e.g. `human.pedestrian.adult`). 40 | ``` 41 | category { 42 | "token": -- Unique record identifier. 43 | "name": -- Category name. Subcategories indicated by period. 44 | "description": -- Category description. 45 | "index": -- The index of the label used for efficiency reasons. 46 | } 47 | ``` 48 | 49 | ego_motion_cabin 50 | --------- 51 | Ego vehicle cabin motion at a particular timestamp. Given with respect to vehicle coordinate system. 52 | The cabin movement can be different from the chassis movement. 53 | ``` 54 | ego_motion_cabin { 55 | "token": -- Unique record identifier. 56 | "timestamp": -- Unix time stamp. 57 | "vx": -- Velocity in x direction given in meters per second (m/s). 58 | "vy": -- Velocity in y direction given in meters per second (m/s). 59 | "vz": -- Velocity in z direction given in meters per second (m/s). 60 | "ax": -- Acceleration in x direction given in meters per second squared (m/s^2). 61 | "ay": -- Acceleration in y direction given in meters per second squared (m/s^2). 62 | "az": -- Acceleration in z direction given in meters per second squared (m/s^2). 63 | "yaw": -- Yaw angle around the z axis given in rad. 64 | "pitch": -- Pitch angle around the y axis given in rad. 65 | "roll": -- Roll angle around the x axis given in rad. 66 | "yaw_rate": -- Yaw rate around the z axis given in rad per second. 67 | "pitch_rate": -- Pitch rate around the z axis given in rad per second. 68 | "roll_rate": -- Roll rate around the z axis given in rad per second. 69 | } 70 | ``` 71 | 72 | ego_motion_chassis 73 | --------- 74 | Ego vehicle chassis motion at a particular timestamp. Given with respect to vehicle coordinate system. 75 | The cabin movement can be different from the chassis movement. 76 | ``` 77 | ego_motion_chassis { 78 | "token": -- Unique record identifier. 79 | "timestamp": -- Unix time stamp. 80 | "vx": -- Velocity in x direction given in meters per second (m/s). 81 | "vy": -- Velocity in y direction given in meters per second (m/s). 82 | "vz": -- Velocity in z direction given in meters per second (m/s). 83 | "ax": -- Acceleration in x direction given in meters per second squared (m/s^2). 84 | "ay": -- Acceleration in y direction given in meters per second squared (m/s^2). 85 | "az": -- Acceleration in z direction given in meters per second squared (m/s^2). 86 | "yaw": -- Yaw angle around the z axis given in rad. 87 | "pitch": -- Pitch angle around the y axis given in rad. 88 | "roll": -- Roll angle around the x axis given in rad. 89 | "yaw_rate": -- Yaw rate around the z axis given in rad per second. 90 | "pitch_rate": -- Pitch rate around the z axis given in rad per second. 91 | "roll_rate": -- Roll rate around the z axis given in rad per second. 92 | } 93 | ``` 94 | 95 | ego_pose 96 | --------- 97 | Ego vehicle pose at a particular timestamp. Given with respect to global coordinate system in UTM-WGS84 coordinates mapped to cell U32. 98 | ``` 99 | ego_pose { 100 | "token": -- Unique record identifier. 101 | "translation": [3] -- Coordinate system origin in meters: x, y, z. Note that z is always 0. 102 | "rotation": [4] -- Coordinate system orientation as quaternion: w, x, y, z. 103 | "timestamp": -- Unix time stamp. 104 | } 105 | ``` 106 | 107 | instance 108 | --------- 109 | An object instance, e.g. particular vehicle. 110 | This table is an enumeration of all object instances we observed. 111 | Note that instances are not tracked across scenes. 112 | ``` 113 | instance { 114 | "token": -- Unique record identifier. 115 | "category_token": -- Foreign key pointing to the object category. 116 | "nbr_annotations": -- Number of annotations of this instance. 117 | "first_annotation_token": -- Foreign key. Points to the first annotation of this instance. 118 | "last_annotation_token": -- Foreign key. Points to the last annotation of this instance. 119 | } 120 | ``` 121 | 122 | sample 123 | --------- 124 | A sample is an annotated keyframe at 2 Hz. 125 | The data is collected at (approximately) the same timestamp as sample_data marked as keyframes. 126 | ``` 127 | sample { 128 | "token": -- Unique record identifier. 129 | "timestamp": -- Unix time stamp. 130 | "scene_token": -- Foreign key pointing to the scene. 131 | "next": -- Foreign key. Sample that follows this in time. Empty if end of scene. 132 | "prev": -- Foreign key. Sample that precedes this in time. Empty if start of scene. 133 | } 134 | ``` 135 | 136 | sample_annotation 137 | --------- 138 | A bounding box defining the position of an object seen in a sample. 139 | All location data is given with respect to the global coordinate system. 140 | ``` 141 | sample_annotation { 142 | "token": -- Unique record identifier. 143 | "sample_token": -- Foreign key. NOTE: this points to a sample NOT a sample_data since annotations are done on the sample level taking all relevant sample_data into account. 144 | "instance_token": -- Foreign key. Which object instance is this annotating. An instance can have multiple annotations over time. 145 | "attribute_tokens": [n] -- Foreign keys. List of attributes for this annotation. Attributes can change over time, so they belong here, not in the instance table. 146 | "visibility_token": -- Foreign key. Visibility may also change over time. If no visibility is annotated, the token is an empty string. 147 | "translation": [3] -- Bounding box location in meters as center_x, center_y, center_z. 148 | "size": [3] -- Bounding box size in meters as width, length, height. 149 | "rotation": [4] -- Bounding box orientation as quaternion: w, x, y, z. 150 | "num_lidar_pts": -- Number of lidar points in this box. Points are counted during the lidar sweep identified with this sample. 151 | "num_radar_pts": -- Number of radar points in this box. Points are counted during the radar sweep identified with this sample. This number is summed across all radar sensors without any invalid point filtering. 152 | "next": -- Foreign key. Sample annotation from the same object instance that follows this in time. Empty if this is the last annotation for this object. 153 | "prev": -- Foreign key. Sample annotation from the same object instance that precedes this in time. Empty if this is the first annotation for this object. 154 | } 155 | ``` 156 | 157 | sample_data 158 | --------- 159 | A sensor data e.g. image, point cloud or radar return. 160 | For sample_data with is_key_frame=True, the time-stamps should be very close to the sample it points to. 161 | For non key-frames the sample_data points to the sample that follows closest in time. 162 | ``` 163 | sample_data { 164 | "token": -- Unique record identifier. 165 | "sample_token": -- Foreign key. Sample to which this sample_data is associated. 166 | "ego_pose_token": -- Foreign key. 167 | "calibrated_sensor_token": -- Foreign key. 168 | "filename": -- Relative path to data-blob on disk. 169 | "fileformat": -- Data file format. 170 | "width": -- If the sample data is an image, this is the image width in pixels. 171 | "height": -- If the sample data is an image, this is the image height in pixels. 172 | "timestamp": -- Unix time stamp. 173 | "is_key_frame": -- True if sample_data is part of key_frame, else False. 174 | "next": -- Foreign key. Sample data from the same sensor that follows this in time. Empty if end of scene. 175 | "prev": -- Foreign key. Sample data from the same sensor that precedes this in time. Empty if start of scene. 176 | } 177 | ``` 178 | 179 | scene 180 | --------- 181 | A scene is a 20s long sequence of consecutive frames. 182 | Multiple scenes can come from the same measurement drive. 183 | Note that object identities (instance tokens) are not preserved across scenes. 184 | ``` 185 | scene { 186 | "token": -- Unique record identifier. 187 | "name": -- Short string identifier. 188 | "description": -- List of scene tags according to seven distinct categories separated by semicolon. 189 | "log_token": -- Foreign key. Always empty. 190 | "nbr_samples": -- Number of samples in this scene. 191 | "first_sample_token": -- Foreign key. Points to the first sample in scene. 192 | "last_sample_token": -- Foreign key. Points to the last sample in scene. 193 | } 194 | ``` 195 | 196 | sensor 197 | --------- 198 | A specific sensor type. 199 | ``` 200 | sensor { 201 | "token": -- Unique record identifier. 202 | "channel": -- Sensor channel name. 203 | "modality": {camera, lidar, radar} -- Sensor modality. Supports category(ies) in brackets. 204 | } 205 | ``` 206 | 207 | visibility 208 | --------- 209 | The visibility of an instance is the fraction of annotation visible in all 4 images. Binned into 4 bins 0-40%, 40-60%, 60-80% and 80-100%. 210 | ``` 211 | visibility { 212 | "token": -- Unique record identifier. 213 | "level": -- Visibility level. 214 | "description": -- Description of visibility level. 215 | } 216 | ``` 217 | 218 | 219 | 220 | 221 | Copied and adapted from [nuscenes-devkit](https://github.com/nutonomy/nuscenes-devkit) -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=42.0", "wheel"] 3 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r setup/requirements/requirements_base.txt 2 | -r setup/requirements/requirements_tutorial.txt 3 | -r setup/requirements/requirements_visu.txt 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = truckscenes-devkit 3 | version = 1.0.0 4 | author = Felix Fent, Fabian Kuttenreich, Florian Ruch, Farija Rizwin 5 | author_email = truckscenes@man.eu 6 | url = https://github.com/TUMFTM/truckscenes-devkit 7 | description = Official development kit of the MAN TruckScenes dataset (www.man.eu/truckscenes). 8 | long_description = file: README.md 9 | long_description_content_type = text/markdown 10 | keywords = MAN, TruckScenes, dataset, devkit, perception 11 | license = Apache-2.0 12 | license_files = LICENSE 13 | platforms = linux, windows 14 | classifiers = 15 | Intended Audience :: Developers 16 | Intended Audience :: Education 17 | Intended Audience :: Science/Research 18 | License :: OSI Approved :: Apache Software License 19 | Natural Language :: English 20 | Operating System :: OS Independent 21 | Programming Language :: Python :: 3 22 | Programming Language :: Python :: 3 :: Only 23 | Programming Language :: Python :: 3.6 24 | Programming Language :: Python :: 3.7 25 | Programming Language :: Python :: 3.8 26 | Programming Language :: Python :: 3.9 27 | Programming Language :: Python :: 3.10 28 | Programming Language :: Python :: 3.11 29 | Topic :: Scientific/Engineering :: Artificial Intelligence 30 | 31 | [options] 32 | packages = find_namespace: 33 | package_dir = 34 | = src 35 | include_package_data = True 36 | install_requires = 37 | numpy 38 | pyquaternion>=0.9.5 39 | tqdm 40 | pypcd4 41 | python_requires = 42 | >=3.8, <3.12 43 | zip_safe = False 44 | 45 | [options.packages.find] 46 | where = src 47 | exclude = 48 | tutorials 49 | 50 | [options.package_data] 51 | truckscenes.eval.* = 52 | *.json 53 | 54 | [options.extras_require] 55 | all = 56 | matplotlib 57 | jupyter 58 | open3d 59 | opencv-python 60 | Pillow>6.2.1 61 | visu = 62 | matplotlib 63 | open3d 64 | opencv-python 65 | Pillow>6.2.1 66 | 67 | [flake8] 68 | max-line-length = 99 69 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | if __name__ == "__main__": 5 | setup() 6 | -------------------------------------------------------------------------------- /setup/requirements/requirements_base.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pyquaternion>=0.9.5 3 | tqdm 4 | pypcd4 5 | -------------------------------------------------------------------------------- /setup/requirements/requirements_tutorial.txt: -------------------------------------------------------------------------------- 1 | jupyter -------------------------------------------------------------------------------- /setup/requirements/requirements_visu.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | open3d 3 | opencv-python 4 | Pillow>6.2.1 -------------------------------------------------------------------------------- /src/truckscenes/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from .truckscenes import TruckScenes # noqa: E731 5 | 6 | __all__ = ('TruckScenes',) 7 | -------------------------------------------------------------------------------- /src/truckscenes/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TUMFTM/truckscenes-devkit/7c94b2a38492361378a312e85f5f668892265d39/src/truckscenes/eval/__init__.py -------------------------------------------------------------------------------- /src/truckscenes/eval/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TUMFTM/truckscenes-devkit/7c94b2a38492361378a312e85f5f668892265d39/src/truckscenes/eval/common/__init__.py -------------------------------------------------------------------------------- /src/truckscenes/eval/common/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import json 5 | import os 6 | 7 | from truckscenes.eval.detection.data_classes import DetectionConfig 8 | 9 | 10 | def config_factory(configuration_name: str) -> DetectionConfig: 11 | """Creates a DetectionConfig instance that can be used to initialize a DetectionEval instance 12 | 13 | Note that this only works if the config file is located in the 14 | truckscenes/eval/common/configs folder. 15 | 16 | Arguments: 17 | configuration_name: Name of desired configuration in eval_detection_configs. 18 | 19 | Returns: 20 | cfg: A DetectionConfig instance. 21 | """ 22 | # Check prefix 23 | tokens = configuration_name.split('_') 24 | assert len(tokens) > 1, 'Error: Configuration name must have prefix "detection_"' 25 | 26 | # Check if config exists 27 | task = tokens[0] 28 | this_dir = os.path.dirname(os.path.abspath(__file__)) 29 | cfg_path = os.path.join(this_dir, '..', task, 'configs', f'{configuration_name}.json') 30 | assert os.path.exists(cfg_path), \ 31 | f'Requested unknown configuration {configuration_name}' 32 | 33 | # Load config file and deserialize it 34 | with open(cfg_path, 'r') as f: 35 | data = json.load(f) 36 | if task == 'detection': 37 | cfg = DetectionConfig.deserialize(data) 38 | else: 39 | raise Exception('Error: Invalid config file name: %s' % configuration_name) 40 | 41 | return cfg 42 | -------------------------------------------------------------------------------- /src/truckscenes/eval/common/constants.py: -------------------------------------------------------------------------------- 1 | TAG_NAMES = [ 2 | 'weather.clear', 'weather.rain', 'weather.snow', 'weather.fog', 'weather.hail', 3 | 'weather.overcast', 'weather.other_weather', 'area.highway', 'area.rural', 'area.terminal', 4 | 'area.parking', 'area.city', 'area.residential', 'area.other_area', 'daytime.morning', 5 | 'daytime.noon', 'daytime.evening', 'daytime.night', 'season.spring', 'season.summer', 6 | 'season.autumn', 'season.winter', 'lighting.illuminated', 'lighting.glare', 'lighting.dark', 7 | 'lighting.twilight', 'lighting.other_lighting', 'structure.tunnel', 'structure.bridge', 8 | 'structure.underpass', 'structure.overpass', 'structure.regular', 'construction.roadworks', 9 | 'construction.unchanged' 10 | ] 11 | -------------------------------------------------------------------------------- /src/truckscenes/eval/common/data_classes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import abc 5 | from collections import defaultdict 6 | from typing import Dict, List, Tuple 7 | 8 | import numpy as np 9 | 10 | 11 | class EvalBox(abc.ABC): 12 | """ Abstract base class for data classes used during detection evaluation. 13 | 14 | Can be a prediction or ground truth. 15 | 16 | Arguments: 17 | sample_token: 18 | translation: 19 | size: 20 | rotation: 21 | velocity: 22 | ego_translation: Translation to ego vehicle in meters. 23 | num_pts: Nbr. LIDAR or RADAR inside the box. Only for gt boxes. 24 | """ 25 | 26 | def __init__(self, 27 | sample_token: str = "", 28 | translation: Tuple[float, float, float] = (0, 0, 0), 29 | size: Tuple[float, float, float] = (0, 0, 0), 30 | rotation: Tuple[float, float, float, float] = (0, 0, 0, 0), 31 | velocity: Tuple[float, float] = (0, 0), 32 | ego_translation: Tuple[float, float, float] = (0, 0, 0), 33 | num_pts: int = -1): 34 | 35 | # Assert data for shape and NaNs. 36 | assert type(sample_token) == str, 'Error: sample_token must be a string!' 37 | 38 | assert len(translation) == 3, 'Error: Translation must have 3 elements!' 39 | assert not np.any(np.isnan(translation)), 'Error: Translation may not be NaN!' 40 | 41 | assert len(size) == 3, 'Error: Size must have 3 elements!' 42 | assert not np.any(np.isnan(size)), 'Error: Size may not be NaN!' 43 | 44 | assert len(rotation) == 4, 'Error: Rotation must have 4 elements!' 45 | assert not np.any(np.isnan(rotation)), 'Error: Rotation may not be NaN!' 46 | 47 | # Velocity can be NaN from our database for certain annotations. 48 | assert len(velocity) == 2, 'Error: Velocity must have 2 elements!' 49 | 50 | assert len(ego_translation) == 3, 'Error: Translation must have 3 elements!' 51 | assert not np.any(np.isnan(ego_translation)), 'Error: Translation may not be NaN!' 52 | 53 | assert type(num_pts) == int, 'Error: num_pts must be int!' 54 | assert not np.any(np.isnan(num_pts)), 'Error: num_pts may not be NaN!' 55 | 56 | # Assign. 57 | self.sample_token = sample_token 58 | self.translation = translation 59 | self.size = size 60 | self.rotation = rotation 61 | self.velocity = velocity 62 | self.ego_translation = ego_translation 63 | self.num_pts = num_pts 64 | 65 | @property 66 | def ego_dist(self) -> float: 67 | """ Compute the distance from this box to the ego vehicle in 2D. """ 68 | return np.sqrt(np.sum(np.array(self.ego_translation[:2]) ** 2)) 69 | 70 | def __repr__(self): 71 | return str(self.serialize()) 72 | 73 | @abc.abstractmethod 74 | def serialize(self) -> Dict[str, list]: 75 | pass 76 | 77 | @classmethod 78 | @abc.abstractmethod 79 | def deserialize(cls, content: Dict[str, list]): 80 | pass 81 | 82 | 83 | class EvalBoxes: 84 | """ Data class that groups EvalBox instances by sample. """ 85 | 86 | def __init__(self): 87 | """ 88 | Initializes the EvalBoxes for GT or predictions. 89 | """ 90 | self.boxes: Dict[str, List[EvalBox]] = defaultdict(list) 91 | 92 | def __repr__(self): 93 | return f"EvalBoxes with {len(self.all)} boxes across {len(self.sample_tokens)} samples" 94 | 95 | def __getitem__(self, item) -> List[EvalBox]: 96 | return self.boxes[item] 97 | 98 | def __eq__(self, other): 99 | if not set(self.sample_tokens) == set(other.sample_tokens): 100 | return False 101 | for token in self.sample_tokens: 102 | if not len(self[token]) == len(other[token]): 103 | return False 104 | for box1, box2 in zip(self[token], other[token]): 105 | if box1 != box2: 106 | return False 107 | return True 108 | 109 | def __len__(self): 110 | return len(self.boxes) 111 | 112 | @property 113 | def all(self) -> List[EvalBox]: 114 | """ Returns all EvalBoxes in a list. """ 115 | ab = [] 116 | for sample_token in self.sample_tokens: 117 | ab.extend(self[sample_token]) 118 | return ab 119 | 120 | @property 121 | def sample_tokens(self) -> List[str]: 122 | """ Returns a list of all keys. """ 123 | return list(self.boxes.keys()) 124 | 125 | def add_boxes(self, sample_token: str, boxes: Dict[str, List[EvalBox]]) -> None: 126 | """ Adds a list of boxes. """ 127 | self.boxes[sample_token].extend(boxes) 128 | 129 | def serialize(self) -> Dict[str, List[EvalBox]]: 130 | """ Serialize instance into json-friendly format. """ 131 | return {key: [box.serialize() for box in boxes] for key, boxes in self.boxes.items()} 132 | 133 | @classmethod 134 | def deserialize(cls, content: Dict[str, List[EvalBox]], box_cls: EvalBox): 135 | """ 136 | Initialize from serialized content. 137 | :param content: A dictionary with the serialized content of the box. 138 | :param box_cls: The class of the boxes, DetectionBox or TrackingBox. 139 | """ 140 | eb = cls() 141 | for sample_token, boxes in content.items(): 142 | eb.add_boxes(sample_token, [box_cls.deserialize(box) for box in boxes]) 143 | return eb 144 | 145 | 146 | class MetricData(abc.ABC): 147 | """ Abstract base class for the *MetricData classes specific to each task. """ 148 | 149 | @abc.abstractmethod 150 | def serialize(self): 151 | """ Serialize instance into json-friendly format. """ 152 | pass 153 | 154 | @classmethod 155 | @abc.abstractmethod 156 | def deserialize(cls, content: Dict[str, List[EvalBox]]): 157 | """ Initialize from serialized content. """ 158 | pass 159 | -------------------------------------------------------------------------------- /src/truckscenes/eval/common/loaders.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import json 5 | from typing import Dict, Tuple 6 | 7 | import numpy as np 8 | import tqdm 9 | 10 | from collections import defaultdict 11 | from pyquaternion import Quaternion 12 | 13 | from truckscenes import TruckScenes 14 | from truckscenes.eval.common.constants import TAG_NAMES 15 | from truckscenes.eval.common.data_classes import EvalBoxes 16 | from truckscenes.eval.detection.data_classes import DetectionBox 17 | from truckscenes.eval.detection.utils import category_to_detection_name 18 | from truckscenes.utils.data_classes import Box 19 | from truckscenes.utils.geometry_utils import points_in_box 20 | from truckscenes.utils.splits import create_splits_scenes 21 | 22 | 23 | def load_prediction(result_path: str, max_boxes_per_sample: int, box_cls, verbose: bool = False) \ 24 | -> Tuple[EvalBoxes, Dict]: 25 | """ 26 | Loads object predictions from file. 27 | :param result_path: Path to the .json result file provided by the user. 28 | :param max_boxes_per_sample: Maximim number of boxes allowed per sample. 29 | :param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox. 30 | :param verbose: Whether to print messages to stdout. 31 | :return: The deserialized results and meta data. 32 | """ 33 | 34 | # Load from file and check that the format is correct. 35 | with open(result_path) as f: 36 | data = json.load(f) 37 | assert 'results' in data, 'Error: No field `results` in result file.' 38 | 39 | # Deserialize results and get meta data. 40 | all_results = EvalBoxes.deserialize(data['results'], box_cls) 41 | meta = data['meta'] 42 | if verbose: 43 | print("Loaded results from {}. Found detections for {} samples." 44 | .format(result_path, len(all_results.sample_tokens))) 45 | 46 | # Check that each sample has no more than x predicted boxes. 47 | for sample_token in all_results.sample_tokens: 48 | assert len(all_results.boxes[sample_token]) <= max_boxes_per_sample, \ 49 | "Error: Only <= %d boxes per sample allowed!" % max_boxes_per_sample 50 | 51 | return all_results, meta 52 | 53 | 54 | def load_gt(trucksc: TruckScenes, eval_split: str, box_cls, verbose: bool = False) -> EvalBoxes: 55 | """ 56 | Loads ground truth boxes from DB. 57 | :param trucksc: A TruckScenes instance. 58 | :param eval_split: The evaluation split for which we load GT boxes. 59 | :param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox. 60 | :param verbose: Whether to print messages to stdout. 61 | :return: The GT boxes. 62 | """ 63 | # Init. 64 | if box_cls == DetectionBox: 65 | attribute_map = {a['token']: a['name'] for a in trucksc.attribute} 66 | 67 | if verbose: 68 | print(f'Loading annotations for {eval_split} split' 69 | f'from TruckScenes version: {trucksc.version}') 70 | # Read out all sample_tokens in DB. 71 | sample_tokens_all = [s['token'] for s in trucksc.sample] 72 | assert len(sample_tokens_all) > 0, "Error: Database has no samples!" 73 | 74 | # Only keep samples from this split. 75 | splits = create_splits_scenes() 76 | 77 | # Check compatibility of split with trucksc_version. 78 | version = trucksc.version 79 | if eval_split in {'train', 'val', 'train_detect', 'train_track'}: 80 | assert version.endswith('trainval'), \ 81 | f'Error: Requested split {eval_split} ' \ 82 | f'which is not compatible with TruckScenes version {version}' 83 | elif eval_split in {'mini_train', 'mini_val'}: 84 | assert version.endswith('mini'), \ 85 | f'Error: Requested split {eval_split} ' \ 86 | f'which is not compatible with TruckScenes version {version}' 87 | elif eval_split == 'test': 88 | assert version.endswith('test'), \ 89 | f'Error: Requested split {eval_split} ' \ 90 | f'which is not compatible with TruckScenes version {version}' 91 | else: 92 | raise ValueError( 93 | f'Error: Requested split {eval_split} ' 94 | f'which this function cannot map to the correct TruckScenes version.' 95 | ) 96 | 97 | if eval_split == 'test': 98 | # Check that you aren't trying to cheat :). 99 | assert len(trucksc.sample_annotation) > 0, \ 100 | 'Error: You are trying to evaluate on the test set but you dont have the annotations!' 101 | 102 | sample_tokens = [] 103 | for sample_token in sample_tokens_all: 104 | scene_token = trucksc.get('sample', sample_token)['scene_token'] 105 | scene_record = trucksc.get('scene', scene_token) 106 | if scene_record['name'] in splits[eval_split]: 107 | sample_tokens.append(sample_token) 108 | 109 | all_annotations = EvalBoxes() 110 | 111 | # Load annotations and filter predictions and annotations. 112 | for sample_token in tqdm.tqdm(sample_tokens, leave=verbose): 113 | 114 | sample = trucksc.get('sample', sample_token) 115 | sample_annotation_tokens = sample['anns'] 116 | 117 | sample_boxes = [] 118 | for sample_annotation_token in sample_annotation_tokens: 119 | 120 | sample_annotation = trucksc.get('sample_annotation', sample_annotation_token) 121 | if box_cls == DetectionBox: 122 | # Get label name in detection task and filter unused labels. 123 | detection_name = category_to_detection_name(sample_annotation['category_name']) 124 | if detection_name is None: 125 | continue 126 | 127 | # Get attribute_name. 128 | attr_tokens = sample_annotation['attribute_tokens'] 129 | attr_count = len(attr_tokens) 130 | if attr_count == 0: 131 | attribute_name = '' 132 | elif attr_count == 1: 133 | attribute_name = attribute_map[attr_tokens[0]] 134 | else: 135 | raise Exception('Error: GT annotations must not have more than one attribute!') 136 | 137 | num_pts = sample_annotation['num_lidar_pts'] + sample_annotation['num_radar_pts'] 138 | 139 | sample_boxes.append( 140 | box_cls( 141 | sample_token=sample_token, 142 | translation=sample_annotation['translation'], 143 | size=sample_annotation['size'], 144 | rotation=sample_annotation['rotation'], 145 | velocity=trucksc.box_velocity(sample_annotation['token'])[:2], 146 | num_pts=num_pts, 147 | detection_name=detection_name, 148 | detection_score=-1.0, # GT samples do not have a score. 149 | attribute_name=attribute_name 150 | ) 151 | ) 152 | else: 153 | raise NotImplementedError('Error: Invalid box_cls %s!' % box_cls) 154 | 155 | all_annotations.add_boxes(sample_token, sample_boxes) 156 | 157 | if verbose: 158 | print(f"Loaded ground truth annotations for {len(all_annotations.sample_tokens)} samples.") 159 | 160 | return all_annotations 161 | 162 | 163 | def add_center_dist(trucksc: TruckScenes, 164 | eval_boxes: EvalBoxes): 165 | """ 166 | Adds the cylindrical (xy) center distance from ego vehicle to each box. 167 | :param trucksc: The TruckScenes instance. 168 | :param eval_boxes: A set of boxes, either GT or predictions. 169 | :return: eval_boxes augmented with center distances. 170 | """ 171 | for sample_token in eval_boxes.sample_tokens: 172 | sample_rec = trucksc.get('sample', sample_token) 173 | sd_record = trucksc.get('sample_data', sample_rec['data']['LIDAR_LEFT']) 174 | pose_record = trucksc.get('ego_pose', sd_record['ego_pose_token']) 175 | 176 | for box in eval_boxes[sample_token]: 177 | # Both boxes and ego pose are given in global coord system, 178 | # so distance can be calculated directly. 179 | # Note that the z component of the ego pose is 0. 180 | ego_translation = (box.translation[0] - pose_record['translation'][0], 181 | box.translation[1] - pose_record['translation'][1], 182 | box.translation[2] - pose_record['translation'][2]) 183 | if isinstance(box, DetectionBox): 184 | box.ego_translation = ego_translation 185 | else: 186 | raise NotImplementedError 187 | 188 | return eval_boxes 189 | 190 | 191 | def get_scene_tag_masks(trucksc: TruckScenes, 192 | eval_boxes: EvalBoxes): 193 | """ 194 | Adds masks for the individual scene tags. 195 | :param trucksc: The TruckScenes instance. 196 | :param eval_boxes: A set of boxes, either GT or predictions. 197 | :return: eval_boxes augmented with center distances. 198 | """ 199 | masks: Dict[str, Dict[str, bool]] = defaultdict(dict) 200 | 201 | for sample_token in eval_boxes.sample_tokens: 202 | sample_rec = trucksc.get('sample', sample_token) 203 | scene_rec = trucksc.get('scene', sample_rec['scene_token']) 204 | 205 | # Get scene tags from scene description 206 | tags = set(scene_rec['description'].split(';')) 207 | 208 | for tag_name in TAG_NAMES: 209 | if tag_name in tags: 210 | masks[tag_name][sample_token] = True 211 | else: 212 | masks[tag_name][sample_token] = False 213 | 214 | return masks 215 | 216 | 217 | def filter_eval_boxes(trucksc: TruckScenes, 218 | eval_boxes: EvalBoxes, 219 | max_dist: Dict[str, float], 220 | verbose: bool = False) -> EvalBoxes: 221 | """ 222 | Applies filtering to boxes. Distance, bike-racks and points per box. 223 | :param trucksc: An instance of the TruckScenes class. 224 | :param eval_boxes: An instance of the EvalBoxes class. 225 | :param max_dist: Maps the detection name to the eval distance threshold for that class. 226 | :param verbose: Whether to print to stdout. 227 | """ 228 | # Retrieve box type for detectipn/tracking boxes. 229 | class_field = _get_box_class_field(eval_boxes) 230 | 231 | # Accumulators for number of filtered boxes. 232 | total, dist_filter, point_filter, bike_rack_filter = 0, 0, 0, 0 233 | for ind, sample_token in enumerate(eval_boxes.sample_tokens): 234 | 235 | # Filter on distance first. 236 | total += len(eval_boxes[sample_token]) 237 | eval_boxes.boxes[sample_token] = [ 238 | box for box in eval_boxes[sample_token] if 239 | box.ego_dist < max_dist[box.__getattribute__(class_field)] 240 | ] 241 | dist_filter += len(eval_boxes[sample_token]) 242 | 243 | # Then remove boxes with zero points in them. Eval boxes have -1 points by default. 244 | eval_boxes.boxes[sample_token] = [ 245 | box for box in eval_boxes[sample_token] if not box.num_pts == 0 246 | ] 247 | point_filter += len(eval_boxes[sample_token]) 248 | 249 | # Perform bike-rack filtering. 250 | sample_anns = trucksc.get('sample', sample_token)['anns'] 251 | bikerack_recs = [ 252 | trucksc.get('sample_annotation', ann) for ann in sample_anns if 253 | trucksc.get('sample_annotation', ann)['category_name'] == 'static_object.bicycle_rack' 254 | ] 255 | bikerack_boxes = [ 256 | Box(rec['translation'], rec['size'], Quaternion(rec['rotation'])) 257 | for rec in bikerack_recs 258 | ] 259 | filtered_boxes = [] 260 | for box in eval_boxes[sample_token]: 261 | if box.__getattribute__(class_field) in ['bicycle', 'motorcycle']: 262 | in_a_bikerack = False 263 | for bikerack_box in bikerack_boxes: 264 | num_points_in_box = np.sum( 265 | points_in_box( 266 | bikerack_box, 267 | np.expand_dims(np.array(box.translation), axis=1) 268 | ) 269 | ) 270 | if num_points_in_box > 0: 271 | in_a_bikerack = True 272 | if not in_a_bikerack: 273 | filtered_boxes.append(box) 274 | else: 275 | filtered_boxes.append(box) 276 | 277 | eval_boxes.boxes[sample_token] = filtered_boxes 278 | bike_rack_filter += len(eval_boxes.boxes[sample_token]) 279 | 280 | if verbose: 281 | print("=> Original number of boxes: %d" % total) 282 | print("=> After distance based filtering: %d" % dist_filter) 283 | print("=> After LIDAR and RADAR points based filtering: %d" % point_filter) 284 | print("=> After bike rack filtering: %d" % bike_rack_filter) 285 | 286 | return eval_boxes 287 | 288 | 289 | def _get_box_class_field(eval_boxes: EvalBoxes) -> str: 290 | """ 291 | Retrieve the name of the class field in the boxes. 292 | This parses through all boxes until it finds a valid box. 293 | If there are no valid boxes, this function throws an exception. 294 | :param eval_boxes: The EvalBoxes used for evaluation. 295 | :return: The name of the class field in the boxes, e.g. detection_name or tracking_name. 296 | """ 297 | assert len(eval_boxes.boxes) > 0 298 | box = None 299 | for val in eval_boxes.boxes.values(): 300 | if len(val) > 0: 301 | box = val[0] 302 | break 303 | if isinstance(box, DetectionBox): 304 | class_field = 'detection_name' 305 | else: 306 | raise Exception('Error: Invalid box type: %s' % box) 307 | 308 | return class_field 309 | -------------------------------------------------------------------------------- /src/truckscenes/eval/common/render.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | from matplotlib.axes import Axes 7 | 8 | 9 | def setup_axis(xlabel: str = None, 10 | ylabel: str = None, 11 | xlim: int = None, 12 | ylim: int = None, 13 | title: str = None, 14 | min_precision: float = None, 15 | min_recall: float = None, 16 | ax: Axes = None, 17 | show_spines: str = 'none'): 18 | """ 19 | Helper method that sets up the axis for a plot. 20 | :param xlabel: x label text. 21 | :param ylabel: y label text. 22 | :param xlim: Upper limit for x axis. 23 | :param ylim: Upper limit for y axis. 24 | :param title: Axis title. 25 | :param min_precision: Visualize minimum precision as horizontal line. 26 | :param min_recall: Visualize minimum recall as vertical line. 27 | :param ax: (optional) an existing axis to be modified. 28 | :param show_spines: Whether to show axes spines, set to 'none' by default. 29 | :return: The axes object. 30 | """ 31 | if ax is None: 32 | ax = plt.subplot() 33 | 34 | ax.get_xaxis().tick_bottom() 35 | ax.tick_params(labelsize=16) 36 | ax.get_yaxis().tick_left() 37 | 38 | # Hide the selected axes spines. 39 | if show_spines in ['bottomleft', 'none']: 40 | ax.spines['top'].set_visible(False) 41 | ax.spines['right'].set_visible(False) 42 | 43 | if show_spines == 'none': 44 | ax.spines['bottom'].set_visible(False) 45 | ax.spines['left'].set_visible(False) 46 | elif show_spines in ['all']: 47 | pass 48 | else: 49 | raise NotImplementedError 50 | 51 | if title is not None: 52 | ax.set_title(title, size=24) 53 | if xlabel is not None: 54 | ax.set_xlabel(xlabel, size=16) 55 | if ylabel is not None: 56 | ax.set_ylabel(ylabel, size=16) 57 | if xlim is not None: 58 | ax.set_xlim(0, xlim) 59 | if ylim is not None: 60 | ax.set_ylim(0, ylim) 61 | if min_recall is not None: 62 | ax.axvline(x=min_recall, linestyle='--', color=(0, 0, 0, 0.3)) 63 | if min_precision is not None: 64 | ax.axhline(y=min_precision, linestyle='--', color=(0, 0, 0, 0.3)) 65 | 66 | return ax 67 | -------------------------------------------------------------------------------- /src/truckscenes/eval/common/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from typing import List, Dict, Any 5 | 6 | import numpy as np 7 | 8 | from pyquaternion import Quaternion 9 | 10 | from truckscenes.eval.common.data_classes import EvalBox 11 | from truckscenes.utils.data_classes import Box 12 | 13 | DetectionBox = Any # Workaround as direct imports lead to cyclic dependencies. 14 | 15 | 16 | def center_distance(gt_box: EvalBox, pred_box: EvalBox) -> float: 17 | """ 18 | L2 distance between the box centers (xy only). 19 | :param gt_box: GT annotation sample. 20 | :param pred_box: Predicted sample. 21 | :return: L2 distance. 22 | """ 23 | return np.linalg.norm(np.array(pred_box.translation[:2]) - np.array(gt_box.translation[:2])) 24 | 25 | 26 | def velocity_l2(gt_box: EvalBox, pred_box: EvalBox) -> float: 27 | """ 28 | L2 distance between the velocity vectors (xy only). 29 | If the predicted velocities are nan, we return inf, which is subsequently clipped to 1. 30 | :param gt_box: GT annotation sample. 31 | :param pred_box: Predicted sample. 32 | :return: L2 distance. 33 | """ 34 | return np.linalg.norm(np.array(pred_box.velocity) - np.array(gt_box.velocity)) 35 | 36 | 37 | def yaw_diff(gt_box: EvalBox, eval_box: EvalBox, period: float = 2*np.pi) -> float: 38 | """ 39 | Returns the yaw angle difference between the orientation of two boxes. 40 | :param gt_box: Ground truth box. 41 | :param eval_box: Predicted box. 42 | :param period: Periodicity in radians for assessing angle difference. 43 | :return: Yaw angle difference in radians in [0, pi]. 44 | """ 45 | yaw_gt = quaternion_yaw(Quaternion(gt_box.rotation)) 46 | yaw_est = quaternion_yaw(Quaternion(eval_box.rotation)) 47 | 48 | return abs(angle_diff(yaw_gt, yaw_est, period)) 49 | 50 | 51 | def angle_diff(x: float, y: float, period: float) -> float: 52 | """ 53 | Get the smallest angle difference between 2 angles: the angle from y to x. 54 | :param x: To angle. 55 | :param y: From angle. 56 | :param period: Periodicity in radians for assessing angle difference. 57 | :return: . Signed smallest between-angle difference in range (-pi, pi). 58 | """ 59 | 60 | # calculate angle difference, modulo to [0, 2*pi] 61 | diff = (x - y + period / 2) % period - period / 2 62 | if diff > np.pi: 63 | diff = diff - (2 * np.pi) # shift (pi, 2*pi] to (-pi, 0] 64 | 65 | return diff 66 | 67 | 68 | def attr_acc(gt_box: DetectionBox, pred_box: DetectionBox) -> float: 69 | """ 70 | Computes the classification accuracy for the attribute of this class (if any). 71 | If the GT class has no attributes or the annotation is missing attributes, 72 | we assign an accuracy of nan, which is 73 | ignored later on. 74 | 75 | Arguments: 76 | gt_box: GT annotation sample. 77 | pred_box: Predicted sample. 78 | 79 | Returns: 80 | acc: Attribute classification accuracy (0 or 1) or 81 | nan if GT annotation does not have any attributes. 82 | """ 83 | if gt_box.attribute_name == '': 84 | # If the class does not have attributes or this particular 85 | # sample is missing attributes, return nan, which is 86 | # ignored later. Note that about 0.4% of the sample_annotations 87 | # have no attributes, although they should. 88 | acc = np.nan 89 | else: 90 | # Check that label is correct. 91 | acc = float(gt_box.attribute_name == pred_box.attribute_name) 92 | return acc 93 | 94 | 95 | def scale_iou(sample_annotation: EvalBox, sample_result: EvalBox) -> float: 96 | """ 97 | This method compares predictions to the ground truth in terms of scale. 98 | It is equivalent to intersection over union (IOU) between the two boxes in 3D, 99 | if we assume that the boxes are aligned, i.e. translation and 100 | rotation are considered identical. 101 | 102 | Arguments: 103 | sample_annotation: GT annotation sample. 104 | sample_result: Predicted sample. 105 | 106 | Returns: 107 | iou: Scale IOU. 108 | """ 109 | # Validate inputs. 110 | sa_size = np.array(sample_annotation.size) 111 | sr_size = np.array(sample_result.size) 112 | assert all(sa_size > 0), 'Error: sample_annotation sizes must be >0.' 113 | assert all(sr_size > 0), 'Error: sample_result sizes must be >0.' 114 | 115 | # Compute IOU. 116 | min_wlh = np.minimum(sa_size, sr_size) 117 | volume_annotation = np.prod(sa_size) 118 | volume_result = np.prod(sr_size) 119 | intersection = np.prod(min_wlh) # type: float 120 | union = volume_annotation + volume_result - intersection # type: float 121 | iou = intersection / union 122 | 123 | return iou 124 | 125 | 126 | def quaternion_yaw(q: Quaternion) -> float: 127 | """ 128 | Calculate the yaw angle from a quaternion. 129 | Note that this only works for a quaternion that represents a box in 130 | lidar or global coordinate frame. 131 | 132 | It does not work for a box in the camera frame. 133 | 134 | 135 | Arguments: 136 | q: Quaternion of interest. 137 | 138 | Returns: 139 | yaw: Yaw angle in radians. 140 | """ 141 | 142 | # Project into xy plane. 143 | v = np.dot(q.rotation_matrix, np.array([1, 0, 0])) 144 | 145 | # Measure yaw using arctan. 146 | yaw = np.arctan2(v[1], v[0]) 147 | 148 | return yaw 149 | 150 | 151 | def boxes_to_sensor(boxes: List[EvalBox], pose_record: Dict, cs_record: Dict, 152 | use_flat_vehicle_coordinates: bool = False) -> List[EvalBox]: 153 | """ 154 | Map boxes from global coordinates to the vehicle's sensor coordinate system. 155 | :param boxes: The boxes in global coordinates. 156 | :param pose_record: The pose record of the vehicle at the current timestamp. 157 | :param cs_record: The calibrated sensor record of the sensor. 158 | :param use_flat_vehicle_coordinates: Instead of the current sensor's coordinate frame, 159 | use ego frame which is aligned to z-plane in the world. 160 | Note: Previously this method did not use flat vehicle coordinates, which can lead 161 | to small errors when the vertical axis of the global frame and lidar are not 162 | aligned. The new setting is more correct and rotates the plot by ~90 degrees. 163 | :return: The transformed boxes. 164 | """ 165 | boxes_out = [] 166 | for box in boxes: 167 | # Create Box instance. 168 | box = Box(box.translation, box.size, Quaternion(box.rotation)) 169 | 170 | if use_flat_vehicle_coordinates: 171 | # Move box to ego vehicle coord system parallel to world z plane. 172 | sd_yaw = Quaternion(pose_record['rotation']).yaw_pitch_roll[0] 173 | box.translate(-np.array(pose_record['translation'])) 174 | box.rotate(Quaternion(scalar=np.cos(sd_yaw / 2), 175 | vector=[0, 0, np.sin(sd_yaw / 2)]).inverse) 176 | 177 | # Rotate upwards 178 | box.rotate(Quaternion(axis=box.orientation.rotate([0, 0, 1]), angle=np.pi/2)) 179 | else: 180 | # Move box to ego vehicle coord system. 181 | box.translate(-np.array(pose_record['translation'])) 182 | box.rotate(Quaternion(pose_record['rotation']).inverse) 183 | 184 | # Move box to sensor coord system. 185 | box.translate(-np.array(cs_record['translation'])) 186 | box.rotate(Quaternion(cs_record['rotation']).inverse) 187 | 188 | boxes_out.append(box) 189 | 190 | return boxes_out 191 | 192 | 193 | def cummean(x: np.ndarray) -> np.ndarray: 194 | """ 195 | Computes the cumulative mean up to each position in a NaN sensitive way 196 | - If all values are NaN return an array of ones. 197 | - If some values are NaN, accumulate arrays discording those entries. 198 | """ 199 | if sum(np.isnan(x)) == len(x): 200 | # Is all numbers in array are NaN's. 201 | return np.ones(len(x)) # If all errors are NaN set to error to 1 for all operating points. 202 | else: 203 | # Accumulate in a nan-aware manner. 204 | sum_vals = np.nancumsum(x.astype(float)) # Cumulative sum ignoring nans. 205 | count_vals = np.cumsum(~np.isnan(x)) # Number of non-nans up to each position. 206 | return np.divide(sum_vals, count_vals, out=np.zeros_like(sum_vals), where=count_vals != 0) 207 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/README.md: -------------------------------------------------------------------------------- 1 | # MAN TruckScenes detection task 2 | 3 | ## Overview 4 | - [Introduction](#introduction) 5 | - [Participation](#participation) 6 | - [Challenges](#challenges) 7 | - [Submission rules](#submission-rules) 8 | - [Results format](#results-format) 9 | - [Classes and attributes](#classes-attributes-and-detection-ranges) 10 | - [Evaluation metrics](#evaluation-metrics) 11 | - [Leaderboard](#leaderboard) 12 | 13 | ## Introduction 14 | Here we define the 3D object detection task on MAN TruckScenes. 15 | The goal of this task is to place a 3D bounding box around 12 different object categories, 16 | as well as estimating a set of attributes and the current velocity vector. 17 | 18 | ## Participation 19 | The TruckScenes detection [evaluation server](https://huggingface.co/spaces/TruckScenes/MANTruckScenesDetectionChallenge) is open all year round for submission. 20 | To participate in the challenge, please create an account at [Hugging Face](https://huggingface.co/). 21 | Then upload your json result file including all of the required [meta data](#results-format). 22 | After each challenge, the results will be exported to the TruckScenes [leaderboard](https://www.man.eu/truckscenes/). 23 | This is the only way to benchmark your method against the test dataset. 24 | 25 | ## Challenges 26 | To allow users to benchmark the performance of their method against the community, we host a single [leaderboard](https://www.man.eu/truckscenes/) all-year round. 27 | Additionally we organize a number of challenges at leading Computer Vision conference workshops. 28 | Users that submit their results during the challenge period are eligible for awards. 29 | Any user that cannot attend the workshop (direct or via a representative) will be excluded from the challenge, but will still be listed on the leaderboard. 30 | 31 | Click [here](https://huggingface.co/spaces/TruckScenes/MANTruckScenesDetectionChallenge) for the **Hugging Face detection evaluation server**. 32 | 33 | ## Submission rules 34 | ### Detection-specific rules 35 | * The maximum time window of past sensor data and ego poses that may be used at inference time is approximately 0.5s (at most 6 *past* camera images, 11 *past* radar sweeps and 6 *past* lidar sweeps). At training time there are no restrictions. 36 | 37 | ### General rules 38 | * We release annotations for the train and val set, but not for the test set. 39 | * We release sensor data for train, val and test set. 40 | * Users make predictions on the test set and submit the results to our evaluation server, which returns the metrics listed below. 41 | * We do not use strata. Instead, we filter annotations and predictions beyond class specific distances. 42 | * Users must limit the number of submitted boxes per sample to 500. 43 | * Users must limit the number of lines in the submitted json file to less than 1Mio. 44 | * Every submission provides method information. We encourage publishing code, but do not make it a requirement. 45 | * Top leaderboard entries and their papers will be manually reviewed. 46 | * Each user or team can submit at most five results *per day*. These results must come from different models, rather than submitting results from the same model at different training epochs or with slightly different parameters. 47 | * Each user or team can only select 1 result to be ranked on the public leaderboard. 48 | * Any attempt to circumvent these rules will result in a permanent ban of the team or company from all TruckScenes challenges. 49 | 50 | ## Results format 51 | We define a standardized detection result format that serves as an input to the evaluation code. 52 | Results are evaluated for each 2Hz keyframe, also known as `sample`. 53 | The detection results for a particular evaluation set (train/val/test) are stored in a single JSON file. 54 | For the train and val sets the evaluation can be performed by the user on their local machine. 55 | For the test set the user needs to submit the single JSON result file to the official evaluation server. 56 | The JSON file includes meta data `meta` on the type of inputs used for this method. 57 | Furthermore it includes a dictionary `results` that maps each sample_token to a list of `sample_result` entries. 58 | Each `sample_token` from the current evaluation set must be included in `results`, although the list of predictions may be empty if no object is detected. 59 | ``` 60 | submission { 61 | "meta": { 62 | "use_camera": -- True, if camera data was used as an input. Else false. 63 | "use_lidar": -- True, if lidar data was used as an input. Else false. 64 | "use_radar": -- True, if radar data was used as an input. Else false. 65 | "use_map": -- True, if map data was used as an input. Else false. 66 | "use_external": -- True, if external data was used as an input. Else false. 67 | "use_future_frames": -- True, if future frames were used as an input during test. Else false. 68 | "use_tta": -- True, if test time augmentation was applied during test. Else false. 69 | "method_name": -- Name of the used approach. 70 | "authors": -- Authors of the method/paper. Empty string if not available. 71 | "affiliation": -- Company, university etc. Empty string if not available. 72 | "description": -- Short info about method, remarks. Empty string if not available. 73 | "code_url": -- Link to open source code of the method. Empty string if not available. 74 | "paper_url": -- Link to method's paper. Empty string if not available. 75 | }, 76 | "results": { 77 | sample_token : List[sample_result] -- Maps each sample_token to a list of sample_results. 78 | } 79 | } 80 | ``` 81 | For the predictions we create a new database table called `sample_result`. 82 | The `sample_result` table is designed to mirror the `sample_annotation` table. 83 | This allows for processing of results and annotations using the same tools. 84 | A `sample_result` is a dictionary defined as follows: 85 | ``` 86 | sample_result { 87 | "sample_token": -- Foreign key. Identifies the sample/keyframe for which objects are detected. 88 | "translation": [3] -- Estimated bounding box location in m in the global frame: center_x, center_y, center_z. 89 | "size": [3] -- Estimated bounding box size in m: width, length, height. 90 | "rotation": [4] -- Estimated bounding box orientation as quaternion in the global frame: w, x, y, z. 91 | "velocity": [2] -- Estimated bounding box velocity in m/s in the global frame: vx, vy. 92 | "detection_name": -- The predicted class for this sample_result, e.g. car, pedestrian. 93 | "detection_score": -- Object prediction score between 0 and 1 for the class identified by detection_name. 94 | "attribute_name": -- Name of the predicted attribute or empty string for classes without attributes. 95 | See table below for valid attributes for each class, e.g. cycle.with_rider. 96 | } 97 | ``` 98 | Note that the detection classes may differ from the general classes, as detailed below. 99 | 100 | ## Classes, attributes, and detection ranges 101 | The MAN Truckscenes dataset comes with annotations for 27 classes. 102 | Some of these only have a handful of samples. 103 | Hence we merge similar classes and remove rare classes. 104 | This results in 12 classes for the detection challenge. 105 | Below we show the table of detection classes and their counterparts in the TruckScenes dataset. 106 | For more information on the classes and their frequencies, see [this page](https://www.man.eu/truckscenes/). 107 | 108 | | TruckScenes detection class | TruckScenes general class | 109 | |-----------------------------|--------------------------------------| 110 | | void / ignore | human.pedestrian.personal_mobility | 111 | | void / ignore | human.pedestrian.stroller | 112 | | void / ignore | human.pedestrian.wheelchair | 113 | | void / ignore | movable_object.debris | 114 | | void / ignore | movable_object.pushable_pullable | 115 | | void / ignore | static_object.bicycle_rack | 116 | | void / ignore | vehicle.emergency.ambulance | 117 | | void / ignore | vehicle.emergency.police | 118 | | void / ignore | vehicle.train | 119 | | animal | animal | 120 | | barrier | movable_object.barrier | 121 | | bicycle | vehicle.bicycle | 122 | | bus | vehicle.bus.bendy | 123 | | bus | vehicle.bus.rigid | 124 | | car | vehicle.car | 125 | | motorcycle | vehicle.motorcycle | 126 | | other_vehicle | vehicle.construction | 127 | | other_vehicle | vehicle.other | 128 | | pedestrian | human.pedestrian.adult | 129 | | pedestrian | human.pedestrian.child | 130 | | pedestrian | human.pedestrian.construction_worker | 131 | | pedestrian | human.pedestrian.police_officer | 132 | | traffic_cone | movable_object.trafficcone | 133 | | traffic_sign | static_object.traffic_sign | 134 | | trailer | vehicle.ego_trailer | 135 | | trailer | vehicle.trailer | 136 | | truck | vehicle.truck | 137 | 138 | Below we list which TruckScenes classes can have which attributes. 139 | 140 | For each TruckScenes detection class, the number of annotations decreases with increasing range from the ego vehicle, 141 | but the number of annotations per range varies by class. Therefore, each class has its own upper bound on evaluated 142 | detection range, as shown below: 143 | 144 | | TruckScenes detection class | Attributes | Detection range (meters) | 145 | |-----------------------------|-------------------------------------------------------|--------------------------| 146 | | animal | void | 75 | 147 | | barrier | void | 75 | 148 | | traffic_cone | void | 75 | 149 | | bicycle | cycle.{with_rider, without_rider} | 75 | 150 | | motorcycle | cycle.{with_rider, without_rider} | 75 | 151 | | pedestrian | pedestrian.{moving, standing, sitting_lying_down} | 75 | 152 | | traffic_sign | traffic_sign.{pole_mounted, overhanging, temporary} | 75 | 153 | | bus | vehicle.{moving, parked, stopped} | 150 | 154 | | car | vehicle.{moving, parked, stopped} | 150 | 155 | | other_vehicle | vehicle.{moving, parked, stopped} | 150 | 156 | | trailer | vehicle.{moving, parked, stopped} | 150 | 157 | | truck | vehicle.{moving, parked, stopped} | 150 | 158 | 159 | ## Evaluation metrics 160 | Below we define the metrics for the TruckScenes detection task. 161 | Our final score is a weighted sum of mean Average Precision (mAP) and several True Positive (TP) metrics. 162 | 163 | ### Preprocessing 164 | Before running the evaluation code the following pre-processing is done on the data 165 | * All boxes (GT and prediction) are removed if they exceed the class-specific detection range. 166 | * All bikes and motorcycle boxes (GT and prediction) that fall inside a bike-rack are removed. The reason is that we do not annotate bikes inside bike-racks. 167 | * All boxes (GT) without lidar or radar points in them are removed. The reason is that we can not guarantee that they are actually visible in the frame. We do not filter the predicted boxes based on number of points. 168 | 169 | ### Average Precision metric 170 | * **mean Average Precision (mAP)**: 171 | We use the well-known Average Precision metric, 172 | but define a match by considering the 2D center distance on the ground plane rather than intersection over union based affinities. 173 | Specifically, we match predictions with the ground truth objects that have the smallest center-distance up to a certain threshold. 174 | For a given match threshold we calculate average precision (AP) by integrating the recall vs precision curve for recalls and precisions > 0.1. 175 | We finally average over match thresholds of {0.5, 1, 2, 4} meters and compute the mean across classes. 176 | 177 | ### True Positive metrics 178 | Here we define metrics for a set of true positives (TP) that measure translation / scale / orientation / velocity and attribute errors. 179 | All TP metrics are calculated using a threshold of 2m center distance during matching, and they are all designed to be positive scalars. 180 | 181 | Matching and scoring happen independently per class and each metric is the average of the cumulative mean at each achieved recall level above 10%. 182 | If 10% recall is not achieved for a particular class, all TP errors for that class are set to 1. 183 | We define the following TP errors: 184 | * **Average Translation Error (ATE)**: Euclidean center distance in 2D in meters. 185 | * **Average Scale Error (ASE)**: Calculated as *1 - IOU* after aligning centers and orientation. 186 | * **Average Orientation Error (AOE)**: Smallest yaw angle difference between prediction and ground-truth in radians. Orientation error is evaluated at 360 degree for all classes except barriers where it is only evaluated at 180 degrees. Orientation errors for cones are ignored. 187 | * **Average Velocity Error (AVE)**: Absolute velocity error in m/s. Velocity error for barriers and cones are ignored. 188 | * **Average Attribute Error (AAE)**: Calculated as *1 - acc*, where acc is the attribute classification accuracy. Attribute error for barriers and cones are ignored. 189 | 190 | All errors are >= 0, but note that for translation and velocity errors the errors are unbounded, and can be any positive value. 191 | 192 | The TP metrics are defined per class, and we then take a mean over classes to calculate mATE, mASE, mAOE, mAVE and mAAE. 193 | 194 | Below we list which error terms are excluded for which TruckScenes detection classes. This is because some error terms 195 | are ambiguous for specific classes that are rotation invariant, static, or without attributes. 196 | 197 | | TruckScenes detection class | Excluded error terms | 198 | |-------------------------------|----------------------| 199 | | animal | attribute error | 200 | | barrier | attribute error | 201 | | | velocity error | 202 | | traffic_cone | attribute error | 203 | | | velocity error | 204 | | | orientation error | 205 | | bicycle | | 206 | | motorcycle | | 207 | | pedestrian | | 208 | | traffic_sign | velocity error | 209 | | bus | | 210 | | car | | 211 | | other_vehicle | | 212 | | trailer | | 213 | | truck | | 214 | 215 | ### Detection score 216 | We are using the same error definitions and metrics as [nuScenes](https://www.nuscenes.org/object-detection). Thus, 217 | * **nuScenes detection score (NDS)**: 218 | We consolidate the above metrics by computing a weighted sum: mAP, mATE, mASE, mAOE, mAVE and mAAE. 219 | As a first step we convert the TP errors to TP scores as *TP_score = max(1 - TP_error, 0.0)*. 220 | We then assign a weight of *5* to mAP and *1* to each of the 5 TP scores and calculate the normalized sum. 221 | 222 | We deviate at 2 points from the original NDS (nuScenes Detection Score) 223 | * we have additional 2 additional classes: animal + traffic sign 224 | * we take into account higher evaluation ranges 225 | 226 | Despite this similarity, due to the differences in sensor setup, vehicle and scene settings, the NDS calculated on the TruckScenes dataset can't be compared with NDS values achieved using the nuScenes dataset. 227 | 228 | ### Configuration 229 | The default evaluation metrics configurations can be found in `truckscenes\eval\detection\configs\detection_cvpr_2024.json`. 230 | 231 | ## Leaderboard 232 | MAN TruckScenes will maintain a single leaderboard for the detection task. 233 | For each submission the leaderboard will list method aspects and evaluation metrics. 234 | Method aspects include input modalities (lidar, radar, vision), use of map data, external data, future frames (ff), and test time augmentation (tta). 235 | To enable a fair comparison between methods, the user will be able to filter the methods by method aspects. 236 | 237 | 238 | _Copied and adapted from [nuscenes-devkit](https://github.com/nutonomy/nuscenes-devkit/tree/1.0.0/python-sdk/nuscenes/eval/detection)_ -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TUMFTM/truckscenes-devkit/7c94b2a38492361378a312e85f5f668892265d39/src/truckscenes/eval/detection/__init__.py -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/algo.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from typing import Callable, List 5 | 6 | import numpy as np 7 | 8 | from truckscenes.eval.common.data_classes import EvalBoxes 9 | from truckscenes.eval.common.utils import center_distance, scale_iou, yaw_diff, velocity_l2, \ 10 | attr_acc, cummean 11 | from truckscenes.eval.detection.data_classes import DetectionMetricData 12 | 13 | 14 | def accumulate(gt_boxes: EvalBoxes, 15 | pred_boxes: EvalBoxes, 16 | class_name: str, 17 | dist_fcn: Callable, 18 | dist_th: float, 19 | gt_mask: List[bool] = None, 20 | pred_mask: List[bool] = None, 21 | verbose: bool = False) -> DetectionMetricData: 22 | """ 23 | Average Precision over predefined different recall thresholds for a single distance threshold. 24 | The recall/conf thresholds and other raw metrics will be used in secondary metrics. 25 | 26 | Arguments: 27 | gt_boxes: Maps every sample_token to a list of its sample_annotations. 28 | pred_boxes: Maps every sample_token to a list of its sample_results. 29 | class_name: Class to compute AP on. 30 | dist_fcn: Distance function used to match detections and ground truths. 31 | dist_th: Distance threshold for a match. 32 | gt_mask: Mask for ground truth boxes. 33 | pred_mask: Mask for predicted boxes. 34 | verbose: If true, print debug messages. 35 | 36 | Returns: 37 | The average precision value and raw data for a number of metrics (average_prec, metrics). 38 | """ 39 | # --------------------------------------------- 40 | # Organize input and initialize accumulators. 41 | # --------------------------------------------- 42 | 43 | # Count the positives. 44 | npos = len([1 for gt_box in gt_boxes.all if gt_box.detection_name == class_name]) 45 | 46 | # Mask ground truth boxes 47 | if gt_mask is not None: 48 | npos -= len([ 49 | 1 for gt_box in gt_boxes.all 50 | if gt_box.detection_name == class_name and not gt_mask[gt_box.sample_token] 51 | ]) 52 | 53 | if verbose: 54 | print("Found {} GT of class {} out of {} total across {} samples.". 55 | format(npos, class_name, len(gt_boxes.all), len(gt_boxes.sample_tokens))) 56 | 57 | # For missing classes in the GT, return a data structure corresponding to no predictions. 58 | if npos == 0: 59 | return DetectionMetricData.no_predictions() 60 | 61 | # Organize the predictions in a single list. 62 | pred_boxes_list = [box for box in pred_boxes.all if box.detection_name == class_name] 63 | 64 | # Mask predicted boxes 65 | if pred_mask is not None: 66 | pred_boxes_list = [box for box in pred_boxes_list if pred_mask[box.sample_token]] 67 | 68 | pred_confs = [box.detection_score for box in pred_boxes_list] 69 | 70 | if verbose: 71 | print(f"Found {len(pred_confs)} PRED of class {class_name} out of {len(pred_boxes.all)} " 72 | f"total across {len(pred_boxes.sample_tokens)} samples.") 73 | 74 | # Sort by confidence. 75 | sortind = [i for (_, i) in sorted((v, i) for (i, v) in enumerate(pred_confs))][::-1] 76 | 77 | # Do the actual matching. 78 | tp = [] # Accumulator of true positives. 79 | fp = [] # Accumulator of false positives. 80 | conf = [] # Accumulator of confidences. 81 | 82 | # match_data holds the extra metrics we calculate for each match. 83 | match_data = {'trans_err': [], 84 | 'vel_err': [], 85 | 'scale_err': [], 86 | 'orient_err': [], 87 | 'attr_err': [], 88 | 'conf': []} 89 | 90 | # --------------------------------------------- 91 | # Match and accumulate match data. 92 | # --------------------------------------------- 93 | 94 | taken = set() # Initially no gt bounding box is matched. 95 | for ind in sortind: 96 | pred_box = pred_boxes_list[ind] 97 | sample_token = pred_box.sample_token 98 | detection_score = pred_box.detection_score 99 | min_dist = np.inf 100 | match_gt_idx = None 101 | 102 | for gt_idx, gt_box in enumerate(gt_boxes[sample_token]): 103 | 104 | # Find closest match among ground truth boxes 105 | if (gt_box.detection_name == class_name and not (sample_token, gt_idx) in taken): 106 | this_distance = dist_fcn(gt_box, pred_box) 107 | if this_distance < min_dist: 108 | min_dist = this_distance 109 | match_gt_idx = gt_idx 110 | 111 | # If the closest match is close enough according to threshold we have a match! 112 | is_match = min_dist < dist_th 113 | 114 | if is_match: 115 | taken.add((sample_token, match_gt_idx)) 116 | 117 | # Update tp, fp and confs. 118 | tp.append(1) 119 | fp.append(0) 120 | conf.append(detection_score) 121 | 122 | # Since it is a match, update match data also. 123 | gt_box_match = gt_boxes[sample_token][match_gt_idx] 124 | 125 | match_data['trans_err'].append(center_distance(gt_box_match, pred_box)) 126 | match_data['vel_err'].append(velocity_l2(gt_box_match, pred_box)) 127 | match_data['scale_err'].append(1 - scale_iou(gt_box_match, pred_box)) 128 | 129 | # Barrier and traffic_sign orientation is only determined up to 180 degree. 130 | # (For cones orientation is discarded later) 131 | if class_name in {'barrier', 'traffic_sign'}: 132 | period = np.pi 133 | else: 134 | period = 2 * np.pi 135 | match_data['orient_err'].append(yaw_diff(gt_box_match, pred_box, period=period)) 136 | 137 | match_data['attr_err'].append(1 - attr_acc(gt_box_match, pred_box)) 138 | match_data['conf'].append(detection_score) 139 | 140 | else: 141 | # No match. Mark this as a false positive. 142 | tp.append(0) 143 | fp.append(1) 144 | conf.append(detection_score) 145 | 146 | # Check if we have any matches. If not, just return a "no predictions" array. 147 | if len(match_data['trans_err']) == 0: 148 | return DetectionMetricData.no_predictions() 149 | 150 | # --------------------------------------------- 151 | # Calculate and interpolate precision and recall 152 | # --------------------------------------------- 153 | 154 | # Accumulate. 155 | tp = np.cumsum(tp).astype(float) 156 | fp = np.cumsum(fp).astype(float) 157 | conf = np.array(conf) 158 | 159 | # Calculate precision and recall. 160 | prec = tp / (fp + tp) 161 | rec = tp / float(npos) 162 | 163 | # 101 steps, from 0% to 100% recall. 164 | rec_interp = np.linspace(0, 1, DetectionMetricData.nelem) 165 | prec = np.interp(rec_interp, rec, prec, right=0) 166 | conf = np.interp(rec_interp, rec, conf, right=0) 167 | rec = rec_interp 168 | 169 | # --------------------------------------------- 170 | # Re-sample the match-data to match, prec, recall and conf. 171 | # --------------------------------------------- 172 | 173 | for key in match_data.keys(): 174 | if key == "conf": 175 | # Confidence is used as reference to align with fp and tp. So skip in this step. 176 | continue 177 | 178 | else: 179 | # For each match_data, we first calculate the accumulated mean. 180 | tmp = cummean(np.array(match_data[key])) 181 | 182 | # Then interpolate based on the confidences. 183 | # (Note reversing since np.interp needs increasing arrays) 184 | match_data[key] = np.interp(conf[::-1], match_data['conf'][::-1], tmp[::-1])[::-1] 185 | 186 | # --------------------------------------------- 187 | # Done. Instantiate MetricData and return 188 | # --------------------------------------------- 189 | return DetectionMetricData(recall=rec, 190 | precision=prec, 191 | confidence=conf, 192 | trans_err=match_data['trans_err'], 193 | vel_err=match_data['vel_err'], 194 | scale_err=match_data['scale_err'], 195 | orient_err=match_data['orient_err'], 196 | attr_err=match_data['attr_err']) 197 | 198 | 199 | def calc_ap(md: DetectionMetricData, min_recall: float, min_precision: float) -> float: 200 | """ Calculated average precision. """ 201 | 202 | assert 0 <= min_precision < 1 203 | assert 0 <= min_recall <= 1 204 | 205 | prec = np.copy(md.precision) 206 | 207 | # Clip low recalls. +1 to exclude the min recall bin. 208 | prec = prec[round(100 * min_recall) + 1:] 209 | 210 | # Clip low precision 211 | prec -= min_precision 212 | prec[prec < 0] = 0 213 | 214 | return float(np.mean(prec)) / (1.0 - min_precision) 215 | 216 | 217 | def calc_tp(md: DetectionMetricData, min_recall: float, metric_name: str) -> float: 218 | """ Calculates true positive errors. """ 219 | 220 | # +1 to exclude the error at min recall. 221 | first_ind = round(100 * min_recall) + 1 222 | 223 | # First instance of confidence = 0 is index of max achieved recall. 224 | last_ind = md.max_recall_ind 225 | 226 | if last_ind < first_ind: 227 | # Assign 1 here. If this happens for all classes, the score for that TP metric will be 0. 228 | return 1.0 229 | else: 230 | # +1 to include error at max recall. 231 | return float(np.mean(getattr(md, metric_name)[first_ind: last_ind + 1])) 232 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import json 5 | import os 6 | 7 | from truckscenes.eval.detection.data_classes import DetectionConfig 8 | 9 | 10 | def config_factory(configuration_name: str) -> DetectionConfig: 11 | """ 12 | Creates a DetectionConfig instance that can be used to initialize a TruckScenesEval instance. 13 | 14 | Note that this only works if the config file is located in 15 | the truckscenes/eval/detection/configs folder. 16 | 17 | Arguments: 18 | configuration_name: Name of desired configuration in eval_detection_configs. 19 | 20 | Returns: 21 | cfg: DetectionConfig instance. 22 | """ 23 | 24 | # Check if config exists. 25 | this_dir = os.path.dirname(os.path.abspath(__file__)) 26 | cfg_path = os.path.join(this_dir, 'configs', f'{configuration_name}.json') 27 | assert os.path.exists(cfg_path), \ 28 | 'Requested unknown configuration {}'.format(configuration_name) 29 | 30 | # Load config file and deserialize it. 31 | with open(cfg_path, 'r') as f: 32 | data = json.load(f) 33 | cfg = DetectionConfig.deserialize(data) 34 | 35 | return cfg 36 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/configs/detection_cvpr_2024.json: -------------------------------------------------------------------------------- 1 | { 2 | "class_range": { 3 | "car": 150, 4 | "truck": 150, 5 | "bus": 150, 6 | "trailer": 150, 7 | "other_vehicle": 150, 8 | "pedestrian": 75, 9 | "motorcycle": 75, 10 | "bicycle": 75, 11 | "traffic_cone": 75, 12 | "barrier": 75, 13 | "animal": 75, 14 | "traffic_sign": 75 15 | }, 16 | "dist_fcn": "center_distance", 17 | "dist_ths": [0.5, 1.0, 2.0, 4.0], 18 | "dist_th_tp": 2.0, 19 | "min_recall": 0.1, 20 | "min_precision": 0.1, 21 | "max_boxes_per_sample": 500, 22 | "mean_ap_weight": 5 23 | } 24 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | DETECTION_NAMES = ['car', 'truck', 'bus', 'trailer', 'other_vehicle', 'pedestrian', 5 | 'motorcycle', 'bicycle', 'traffic_cone', 'barrier', 'animal', 'traffic_sign'] 6 | 7 | PRETTY_DETECTION_NAMES = {'car': 'Car', 8 | 'truck': 'Truck', 9 | 'bus': 'Bus', 10 | 'trailer': 'Trailer', 11 | 'other_vehicle': 'Other Veh.', 12 | 'pedestrian': 'Pedestrian', 13 | 'motorcycle': 'Motorcycle', 14 | 'bicycle': 'Bicycle', 15 | 'traffic_cone': 'Traffic Cone', 16 | 'barrier': 'Barrier', 17 | 'animal': 'Animal', 18 | 'traffic_sign': 'Traffic Sign'} 19 | 20 | DETECTION_COLORS = {'car': 'C0', 21 | 'truck': 'C1', 22 | 'bus': 'C2', 23 | 'trailer': 'C3', 24 | 'other_vehicle': 'C4', 25 | 'pedestrian': 'C5', 26 | 'motorcycle': 'C6', 27 | 'bicycle': 'C7', 28 | 'traffic_cone': 'C8', 29 | 'barrier': 'C9', 30 | 'animal': 'C10', 31 | 'traffic_sign': 'C11'} 32 | 33 | ATTRIBUTE_NAMES = ['pedestrian.moving', 'pedestrian.sitting_lying_down', 'pedestrian.standing', 34 | 'cycle.with_rider', 'cycle.without_rider', 'vehicle.moving', 'vehicle.parked', 35 | 'vehicle.stopped', 'traffic_sign.pole_mounted', 'traffic_sign.overhanging', 36 | 'traffic_sign.temporary'] 37 | 38 | PRETTY_ATTRIBUTE_NAMES = {'pedestrian.moving': 'Ped. Moving', 39 | 'pedestrian.sitting_lying_down': 'Ped. Sitting', 40 | 'pedestrian.standing': 'Ped. Standing', 41 | 'cycle.with_rider': 'Cycle w/ Rider', 42 | 'cycle.without_rider': 'Cycle w/o Rider', 43 | 'vehicle.moving': 'Veh. Moving', 44 | 'vehicle.parked': 'Veh. Parked', 45 | 'vehicle.stopped': 'Veh. Stopped', 46 | 'traffic_sign.pole_mounted': 'Sign mounted', 47 | 'traffic_sign.overhanging': 'Sign over.', 48 | 'traffic_sign.temporary': 'Sign temp.'} 49 | 50 | TP_METRICS = ['trans_err', 'scale_err', 'orient_err', 'vel_err', 'attr_err'] 51 | 52 | PRETTY_TP_METRICS = {'trans_err': 'Trans.', 'scale_err': 'Scale', 'orient_err': 'Orient.', 53 | 'vel_err': 'Vel.', 'attr_err': 'Attr.'} 54 | 55 | TP_METRICS_UNITS = {'trans_err': 'm', 56 | 'scale_err': '1-IOU', 57 | 'orient_err': 'rad.', 58 | 'vel_err': 'm/s', 59 | 'attr_err': '1-acc.'} 60 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/data_classes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from __future__ import annotations 5 | 6 | from collections import defaultdict 7 | from typing import Any, List, Dict, Tuple 8 | 9 | import numpy as np 10 | 11 | from truckscenes.eval.common.data_classes import MetricData, EvalBox 12 | from truckscenes.eval.common.utils import center_distance 13 | from truckscenes.eval.detection.constants import DETECTION_NAMES, ATTRIBUTE_NAMES, TP_METRICS 14 | 15 | 16 | class DetectionConfig: 17 | """ Data class that specifies the detection evaluation settings. """ 18 | 19 | def __init__(self, 20 | class_range: Dict[str, int], 21 | dist_fcn: str, 22 | dist_ths: List[float], 23 | dist_th_tp: float, 24 | min_recall: float, 25 | min_precision: float, 26 | max_boxes_per_sample: int, 27 | mean_ap_weight: int): 28 | 29 | assert set(class_range.keys()) == set(DETECTION_NAMES), "Class count mismatch." 30 | assert dist_th_tp in dist_ths, "dist_th_tp must be in set of dist_ths." 31 | 32 | self.class_range = class_range 33 | self.dist_fcn = dist_fcn 34 | self.dist_ths = dist_ths 35 | self.dist_th_tp = dist_th_tp 36 | self.min_recall = min_recall 37 | self.min_precision = min_precision 38 | self.max_boxes_per_sample = max_boxes_per_sample 39 | self.mean_ap_weight = mean_ap_weight 40 | 41 | self.class_names = self.class_range.keys() 42 | 43 | def __eq__(self, other): 44 | eq = True 45 | for key in self.serialize().keys(): 46 | eq = eq and np.array_equal(getattr(self, key), getattr(other, key)) 47 | return eq 48 | 49 | def serialize(self) -> dict: 50 | """ Serialize instance into json-friendly format. """ 51 | return { 52 | 'class_range': self.class_range, 53 | 'dist_fcn': self.dist_fcn, 54 | 'dist_ths': self.dist_ths, 55 | 'dist_th_tp': self.dist_th_tp, 56 | 'min_recall': self.min_recall, 57 | 'min_precision': self.min_precision, 58 | 'max_boxes_per_sample': self.max_boxes_per_sample, 59 | 'mean_ap_weight': self.mean_ap_weight 60 | } 61 | 62 | @classmethod 63 | def deserialize(cls, content: dict): 64 | """ Initialize from serialized dictionary. """ 65 | return cls(content['class_range'], 66 | content['dist_fcn'], 67 | content['dist_ths'], 68 | content['dist_th_tp'], 69 | content['min_recall'], 70 | content['min_precision'], 71 | content['max_boxes_per_sample'], 72 | content['mean_ap_weight']) 73 | 74 | @property 75 | def dist_fcn_callable(self): 76 | """ Return the distance function corresponding to the dist_fcn string. """ 77 | if self.dist_fcn == 'center_distance': 78 | return center_distance 79 | else: 80 | raise Exception('Error: Unknown distance function %s!' % self.dist_fcn) 81 | 82 | 83 | class DetectionMetricData(MetricData): 84 | """ This class holds accumulated and interpolated data required to 85 | calculate the detection metrics. 86 | """ 87 | 88 | nelem = 101 89 | 90 | def __init__(self, 91 | recall: np.array, 92 | precision: np.array, 93 | confidence: np.array, 94 | trans_err: np.array, 95 | vel_err: np.array, 96 | scale_err: np.array, 97 | orient_err: np.array, 98 | attr_err: np.array): 99 | 100 | # Assert lengths. 101 | assert len(recall) == self.nelem 102 | assert len(precision) == self.nelem 103 | assert len(confidence) == self.nelem 104 | assert len(trans_err) == self.nelem 105 | assert len(vel_err) == self.nelem 106 | assert len(scale_err) == self.nelem 107 | assert len(orient_err) == self.nelem 108 | assert len(attr_err) == self.nelem 109 | 110 | # Assert ordering. 111 | # Confidences should be descending, Recalls should be ascending. 112 | assert all(confidence == sorted(confidence, reverse=True)) 113 | assert all(recall == sorted(recall)) 114 | 115 | # Set attributes explicitly to help IDEs figure out what is going on. 116 | self.recall = recall 117 | self.precision = precision 118 | self.confidence = confidence 119 | self.trans_err = trans_err 120 | self.vel_err = vel_err 121 | self.scale_err = scale_err 122 | self.orient_err = orient_err 123 | self.attr_err = attr_err 124 | 125 | def __eq__(self, other): 126 | eq = True 127 | for key in self.serialize().keys(): 128 | eq = eq and np.array_equal(getattr(self, key), getattr(other, key)) 129 | return eq 130 | 131 | @property 132 | def max_recall_ind(self): 133 | """ Returns index of max recall achieved. """ 134 | 135 | # Last instance of confidence > 0 is index of max achieved recall. 136 | non_zero = np.nonzero(self.confidence)[0] 137 | if len(non_zero) == 0: # If there are no matches, all the confidence values will be zero. 138 | max_recall_ind = 0 139 | else: 140 | max_recall_ind = non_zero[-1] 141 | 142 | return max_recall_ind 143 | 144 | @property 145 | def max_recall(self): 146 | """ Returns max recall achieved. """ 147 | 148 | return self.recall[self.max_recall_ind] 149 | 150 | def serialize(self): 151 | """ Serialize instance into json-friendly format. """ 152 | return { 153 | 'recall': self.recall.tolist(), 154 | 'precision': self.precision.tolist(), 155 | 'confidence': self.confidence.tolist(), 156 | 'trans_err': self.trans_err.tolist(), 157 | 'vel_err': self.vel_err.tolist(), 158 | 'scale_err': self.scale_err.tolist(), 159 | 'orient_err': self.orient_err.tolist(), 160 | 'attr_err': self.attr_err.tolist(), 161 | } 162 | 163 | @classmethod 164 | def deserialize(cls, content: dict): 165 | """ Initialize from serialized content. """ 166 | return cls(recall=np.array(content['recall']), 167 | precision=np.array(content['precision']), 168 | confidence=np.array(content['confidence']), 169 | trans_err=np.array(content['trans_err']), 170 | vel_err=np.array(content['vel_err']), 171 | scale_err=np.array(content['scale_err']), 172 | orient_err=np.array(content['orient_err']), 173 | attr_err=np.array(content['attr_err'])) 174 | 175 | @classmethod 176 | def no_predictions(cls): 177 | """ Returns a md instance corresponding to having no predictions. """ 178 | return cls(recall=np.linspace(0, 1, cls.nelem), 179 | precision=np.zeros(cls.nelem), 180 | confidence=np.zeros(cls.nelem), 181 | trans_err=np.ones(cls.nelem), 182 | vel_err=np.ones(cls.nelem), 183 | scale_err=np.ones(cls.nelem), 184 | orient_err=np.ones(cls.nelem), 185 | attr_err=np.ones(cls.nelem)) 186 | 187 | @classmethod 188 | def random_md(cls): 189 | """ Returns an md instance corresponding to a random results. """ 190 | return cls(recall=np.linspace(0, 1, cls.nelem), 191 | precision=np.random.random(cls.nelem), 192 | confidence=np.linspace(0, 1, cls.nelem)[::-1], 193 | trans_err=np.random.random(cls.nelem), 194 | vel_err=np.random.random(cls.nelem), 195 | scale_err=np.random.random(cls.nelem), 196 | orient_err=np.random.random(cls.nelem), 197 | attr_err=np.random.random(cls.nelem)) 198 | 199 | 200 | class DetectionMetrics: 201 | """ Stores average precision and true positive metric results. 202 | Provides properties to summarize. 203 | """ 204 | 205 | def __init__(self, cfg: DetectionConfig): 206 | 207 | self.cfg = cfg 208 | self._label_aps = defaultdict(lambda: defaultdict(float)) 209 | self._label_tp_errors = defaultdict(lambda: defaultdict(float)) 210 | self.eval_time = None 211 | 212 | def add_label_ap(self, detection_name: str, dist_th: float, ap: float) -> None: 213 | self._label_aps[detection_name][dist_th] = ap 214 | 215 | def get_label_ap(self, detection_name: str, dist_th: float) -> float: 216 | return self._label_aps[detection_name][dist_th] 217 | 218 | def add_label_tp(self, detection_name: str, metric_name: str, tp: float) -> None: 219 | self._label_tp_errors[detection_name][metric_name] = tp 220 | 221 | def get_label_tp(self, detection_name: str, metric_name: str) -> float: 222 | return self._label_tp_errors[detection_name][metric_name] 223 | 224 | def add_runtime(self, eval_time: float) -> None: 225 | self.eval_time = eval_time 226 | 227 | @property 228 | def mean_dist_aps(self) -> Dict[str, float]: 229 | """ Calculates the mean over distance thresholds for each label. """ 230 | return {class_name: np.mean(list(d.values())) for class_name, d in self._label_aps.items()} 231 | 232 | @property 233 | def mean_ap(self) -> float: 234 | """ Calculates the mean AP by averaging over distance thresholds and classes. """ 235 | return float(np.mean(list(self.mean_dist_aps.values()))) 236 | 237 | @property 238 | def tp_errors(self) -> Dict[str, float]: 239 | """ Calculates the mean true positive error across all classes for each metric. """ 240 | errors = {} 241 | for metric_name in TP_METRICS: 242 | class_errors = [] 243 | for detection_name in self.cfg.class_names: 244 | class_errors.append(self.get_label_tp(detection_name, metric_name)) 245 | 246 | if np.isnan(class_errors).all(): 247 | errors[metric_name] = np.nan 248 | else: 249 | errors[metric_name] = float(np.nanmean(class_errors)) 250 | 251 | return errors 252 | 253 | @property 254 | def tp_scores(self) -> Dict[str, float]: 255 | scores = {} 256 | tp_errors = self.tp_errors 257 | for metric_name in TP_METRICS: 258 | 259 | # We convert the true positive errors to "scores" by 1-error. 260 | score = 1.0 - tp_errors[metric_name] 261 | 262 | # Some of the true positive errors are unbounded, so we bound the scores to min 0. 263 | score = max(0.0, score) 264 | 265 | scores[metric_name] = score 266 | 267 | return scores 268 | 269 | @property 270 | def nd_score(self) -> float: 271 | """ 272 | Compute the nuScenes detection score (NDS, weighted sum of the individual scores). 273 | :return: The NDS. 274 | """ 275 | # Summarize. 276 | total = float( 277 | self.cfg.mean_ap_weight * self.mean_ap + np.sum(list(self.tp_scores.values())) 278 | ) 279 | 280 | # Normalize. 281 | total = total / float(self.cfg.mean_ap_weight + len(self.tp_scores.keys())) 282 | 283 | return total 284 | 285 | def serialize(self) -> Dict[str, Any]: 286 | return { 287 | 'label_aps': self._label_aps, 288 | 'mean_dist_aps': self.mean_dist_aps, 289 | 'mean_ap': self.mean_ap, 290 | 'label_tp_errors': self._label_tp_errors, 291 | 'tp_errors': self.tp_errors, 292 | 'tp_scores': self.tp_scores, 293 | 'nd_score': self.nd_score, 294 | 'eval_time': self.eval_time, 295 | 'cfg': self.cfg.serialize() 296 | } 297 | 298 | @classmethod 299 | def deserialize(cls, content: Dict[str, Any]): 300 | """ Initialize from serialized dictionary. """ 301 | 302 | cfg = DetectionConfig.deserialize(content['cfg']) 303 | 304 | metrics = cls(cfg=cfg) 305 | metrics.add_runtime(content['eval_time']) 306 | 307 | for detection_name, label_aps in content['label_aps'].items(): 308 | for dist_th, ap in label_aps.items(): 309 | metrics.add_label_ap( 310 | detection_name=detection_name, dist_th=float(dist_th), ap=float(ap) 311 | ) 312 | 313 | for detection_name, label_tps in content['label_tp_errors'].items(): 314 | for metric_name, tp in label_tps.items(): 315 | metrics.add_label_tp( 316 | detection_name=detection_name, metric_name=metric_name, tp=float(tp) 317 | ) 318 | 319 | return metrics 320 | 321 | def __eq__(self, other): 322 | eq = True 323 | eq = eq and self._label_aps == other._label_aps 324 | eq = eq and self._label_tp_errors == other._label_tp_errors 325 | eq = eq and self.eval_time == other.eval_time 326 | eq = eq and self.cfg == other.cfg 327 | 328 | return eq 329 | 330 | 331 | class DetectionBox(EvalBox): 332 | """ Data class used during detection evaluation. 333 | 334 | Can be a prediction or ground truth. 335 | 336 | Arguments: 337 | sample_token: 338 | translation: 339 | size: 340 | rotation: 341 | velocity: 342 | ego_translation: Translation to ego vehicle in meters. 343 | num_pts: Nbr. LIDAR or RADAR inside the box. Only for gt boxes. 344 | detection_name: The class name used in the detection challenge. 345 | detection_score: GT samples do not have a score. 346 | attribute_name: Box attribute. Each box can have at most 1 attribute. 347 | """ 348 | 349 | def __init__(self, 350 | sample_token: str = "", 351 | translation: Tuple[float, float, float] = (0, 0, 0), 352 | size: Tuple[float, float, float] = (0, 0, 0), 353 | rotation: Tuple[float, float, float, float] = (0, 0, 0, 0), 354 | velocity: Tuple[float, float] = (0, 0), 355 | ego_translation: Tuple[float, float, float] = (0, 0, 0), 356 | num_pts: int = -1, 357 | detection_name: str = 'car', 358 | detection_score: float = -1.0, 359 | attribute_name: str = ''): 360 | 361 | super().__init__(sample_token, translation, size, 362 | rotation, velocity, ego_translation, num_pts) 363 | 364 | assert detection_name is not None, 'Error: detection_name cannot be empty!' 365 | assert detection_name in DETECTION_NAMES, f'Error: Unknown detection_name {detection_name}' 366 | 367 | assert attribute_name in ATTRIBUTE_NAMES or attribute_name == '', \ 368 | 'Error: Unknown attribute_name %s' % attribute_name 369 | 370 | assert type(detection_score) == float, 'Error: detection_score must be a float!' 371 | assert not np.any(np.isnan(detection_score)), 'Error: detection_score may not be NaN!' 372 | 373 | # Assign. 374 | self.detection_name = detection_name 375 | self.detection_score = detection_score 376 | self.attribute_name = attribute_name 377 | 378 | def __eq__(self, other): 379 | return (self.sample_token == other.sample_token and 380 | self.translation == other.translation and 381 | self.size == other.size and 382 | self.rotation == other.rotation and 383 | self.velocity == other.velocity and 384 | self.ego_translation == other.ego_translation and 385 | self.num_pts == other.num_pts and 386 | self.detection_name == other.detection_name and 387 | self.detection_score == other.detection_score and 388 | self.attribute_name == other.attribute_name) 389 | 390 | def serialize(self) -> Dict[str, Any]: 391 | """ Serialize instance into json-friendly format. """ 392 | return { 393 | 'sample_token': self.sample_token, 394 | 'translation': self.translation, 395 | 'size': self.size, 396 | 'rotation': self.rotation, 397 | 'velocity': self.velocity, 398 | 'ego_translation': self.ego_translation, 399 | 'num_pts': self.num_pts, 400 | 'detection_name': self.detection_name, 401 | 'detection_score': self.detection_score, 402 | 'attribute_name': self.attribute_name 403 | } 404 | 405 | @classmethod 406 | def deserialize(cls, content: Dict[str, Any]): 407 | """ Initialize from serialized content. """ 408 | if 'detection_score' not in content: 409 | detection_score = -1.0 410 | else: 411 | detection_score = float(content['detection_score']) 412 | 413 | return cls(sample_token=content['sample_token'], 414 | translation=tuple(content['translation']), 415 | size=tuple(content['size']), 416 | rotation=tuple(content['rotation']), 417 | velocity=tuple(content['velocity']), 418 | ego_translation=(0.0, 0.0, 0.0) if 'ego_translation' not in content 419 | else tuple(content['ego_translation']), 420 | num_pts=-1 if 'num_pts' not in content else int(content['num_pts']), 421 | detection_name=content['detection_name'], 422 | detection_score=detection_score, 423 | attribute_name=content['attribute_name']) 424 | 425 | 426 | class DetectionMetricDataList: 427 | """ This stores a set of MetricData in a dict indexed by (name, match-distance). """ 428 | 429 | def __init__(self): 430 | self.md = {} 431 | 432 | def __getitem__(self, key): 433 | return self.md[key] 434 | 435 | def __eq__(self, other): 436 | eq = True 437 | for key in self.md.keys(): 438 | eq = eq and self[key] == other[key] 439 | return eq 440 | 441 | def get_class_data(self, detection_name: str, 442 | tag_name: str = None) -> List[Tuple[DetectionMetricData, float]]: 443 | """ Get all the MetricData entries for a certain detection_name. """ 444 | if tag_name is not None: 445 | return [ 446 | (md, dist_th) for (tag, name, dist_th), md in self.md.items() 447 | if name == detection_name and tag == tag_name 448 | ] 449 | 450 | return [ 451 | (md, dist_th) for (_, name, dist_th), md in self.md.items() 452 | if name == detection_name 453 | ] 454 | 455 | def get_dist_data(self, dist_th: float, 456 | tag_name: str = None) -> List[Tuple[DetectionMetricData, str]]: 457 | """ Get all the MetricData entries for a certain match_distance. """ 458 | if tag_name is not None: 459 | return [ 460 | (md, detection_name) 461 | for (tag, detection_name, dist), md in self.md.items() 462 | if dist == dist_th and tag == tag_name 463 | ] 464 | 465 | return [ 466 | (md, detection_name) 467 | for (_, detection_name, dist), md in self.md.items() 468 | if dist == dist_th 469 | ] 470 | 471 | def set(self, tag_name: str, detection_name: str, match_distance: float, 472 | data: DetectionMetricData): 473 | """ Sets the MetricData entry for a certain detection_name and match_distance. """ 474 | self.md[(tag_name, detection_name, match_distance)] = data 475 | 476 | def serialize(self) -> Dict[str, Any]: 477 | return {key[0] + ':' + str(key[1]): value.serialize() for key, value in self.md.items()} 478 | 479 | @classmethod 480 | def deserialize(cls, content: Dict[str, Any]): 481 | mdl = cls() 482 | for key, md in content.items(): 483 | name, distance = key.split(':') 484 | mdl.set(name, float(distance), DetectionMetricData.deserialize(md)) 485 | return mdl 486 | 487 | 488 | class DetectionMetricsList: 489 | """ This stores DetectionMetrics in a dict indexed by (tag-name). """ 490 | def __init__(self): 491 | self.dm: Dict[str, DetectionMetrics] = {} 492 | self.eval_time = None 493 | 494 | def __getitem__(self, key): 495 | return self.dm[key] 496 | 497 | def __eq__(self, other): 498 | eq = True 499 | for key in self.dm.keys(): 500 | eq = eq and self[key] == other[key] 501 | return eq 502 | 503 | def add_detection_metrics(self, tag_name: str, data: DetectionMetrics) -> None: 504 | self.dm[tag_name] = data 505 | 506 | def get_detection_metrics(self, tag_name: str) -> DetectionMetrics: 507 | return self.dm[tag_name] 508 | 509 | def add_label_ap(self, tag_name: str, *args, **kwargs) -> None: 510 | self.dm[tag_name].add_label_ap(*args, **kwargs) 511 | 512 | def get_label_ap(self, tag_name: str, *args, **kwargs) -> float: 513 | return self.dm[tag_name].get_label_ap(*args, **kwargs) 514 | 515 | def add_label_tp(self, tag_name: str, *args, **kwargs) -> None: 516 | self.dm[tag_name].add_label_tp(*args, **kwargs) 517 | 518 | def get_label_tp(self, tag_name: str, *args, **kwargs) -> float: 519 | return self.dm[tag_name].get_label_tp(*args, **kwargs) 520 | 521 | def add_runtime(self, eval_time: float) -> None: 522 | self.eval_time = eval_time 523 | 524 | def serialize(self) -> Dict[str, Any]: 525 | serialized = {k: v.serialize() for k, v in self.dm.items()} 526 | serialized['eval_time'] = self.eval_time 527 | return serialized 528 | 529 | @classmethod 530 | def deserialize(cls, content: Dict[str, Any]): 531 | """ Initialize from serialized dictionary. """ 532 | metrics_list = cls() 533 | metrics_list.eval_time = content.pop('eval_time', None) 534 | metrics_list.dm = {k: DetectionMetrics.deserialize(v) for k, v in content.items()} 535 | return metrics_list 536 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/evaluate.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import argparse 5 | import json 6 | import os 7 | import random 8 | import time 9 | import warnings 10 | 11 | from importlib import import_module 12 | from typing import Tuple, Dict, Any 13 | 14 | import numpy as np 15 | 16 | from truckscenes import TruckScenes 17 | from truckscenes.eval.common.constants import TAG_NAMES 18 | from truckscenes.eval.common.data_classes import EvalBoxes 19 | from truckscenes.eval.common.loaders import load_prediction, load_gt, add_center_dist, \ 20 | get_scene_tag_masks, filter_eval_boxes 21 | from truckscenes.eval.detection.algo import accumulate, calc_ap, calc_tp 22 | from truckscenes.eval.detection.config import config_factory 23 | from truckscenes.eval.detection.constants import TP_METRICS 24 | from truckscenes.eval.detection.data_classes import DetectionConfig, DetectionMetrics, \ 25 | DetectionMetricsList, DetectionBox, DetectionMetricDataList 26 | 27 | 28 | class DetectionEval: 29 | """ 30 | This is the official MAN TruckScenes detection evaluation code. 31 | Results are written to the provided output_dir. 32 | 33 | TruckScenes uses the following detection metrics: 34 | - Mean Average Precision (mAP): Uses center-distance as matching criterion; 35 | averaged over distance thresholds. 36 | - True Positive (TP) metrics: Average of translation, velocity, scale, 37 | orientation and attribute errors. 38 | - nuScenes Detection Score (NDS): The weighted sum of the above. 39 | 40 | Here is an overview of the functions in this method: 41 | - init: Loads GT annotations and predictions stored in JSON format and filters the boxes. 42 | - run: Performs evaluation and dumps the metric data to disk. 43 | - render: Renders various plots and dumps to disk. 44 | 45 | We assume that: 46 | - Every sample_token is given in the results, although there may be not 47 | predictions for that sample. 48 | 49 | Please see https://www.nuscenes.org/object-detection for more details. 50 | """ 51 | def __init__(self, 52 | trucksc: TruckScenes, 53 | config: DetectionConfig, 54 | result_path: str, 55 | eval_set: str, 56 | output_dir: str = None, 57 | verbose: bool = True): 58 | """ 59 | Initialize a DetectionEval object. 60 | :param trucksc: A TruckScenes object. 61 | :param config: A DetectionConfig object. 62 | :param result_path: Path of the TruckScenes JSON result file. 63 | :param eval_set: The dataset split to evaluate on, e.g. train, val or test. 64 | :param output_dir: Folder to save plots and results to. 65 | :param verbose: Whether to print to stdout. 66 | """ 67 | self.trucksc = trucksc 68 | self.result_path = result_path 69 | self.eval_set = eval_set 70 | self.output_dir = output_dir 71 | self.verbose = verbose 72 | self.cfg = config 73 | 74 | # Check result file exists. 75 | assert os.path.exists(result_path), 'Error: The result file does not exist!' 76 | 77 | # Make dirs. 78 | self.plot_dir = os.path.join(self.output_dir, 'plots') 79 | if not os.path.isdir(self.output_dir): 80 | os.makedirs(self.output_dir) 81 | if not os.path.isdir(self.plot_dir): 82 | os.makedirs(self.plot_dir) 83 | 84 | # Load data. 85 | if verbose: 86 | print('Initializing MAN TruckScenes detection evaluation') 87 | self.pred_boxes, self.meta = load_prediction(self.result_path, 88 | self.cfg.max_boxes_per_sample, DetectionBox, 89 | verbose=verbose) 90 | self.gt_boxes = load_gt(self.trucksc, self.eval_set, DetectionBox, verbose=verbose) 91 | 92 | assert set(self.pred_boxes.sample_tokens) == set(self.gt_boxes.sample_tokens), \ 93 | "Samples in split doesn't match samples in predictions." 94 | 95 | # Add center distances. 96 | self.pred_boxes = add_center_dist(trucksc, self.pred_boxes) 97 | self.gt_boxes = add_center_dist(trucksc, self.gt_boxes) 98 | 99 | # Filter boxes (distance, points per box, etc.). 100 | if verbose: 101 | print('Filtering predictions') 102 | self.pred_boxes = filter_eval_boxes(trucksc, self.pred_boxes, self.cfg.class_range, 103 | verbose=verbose) 104 | if verbose: 105 | print('Filtering ground truth annotations') 106 | self.gt_boxes = filter_eval_boxes(trucksc, self.gt_boxes, self.cfg.class_range, 107 | verbose=verbose) 108 | 109 | self.sample_tokens = self.gt_boxes.sample_tokens 110 | 111 | # Add scene tag masks. 112 | self.pred_boxes_masks = get_scene_tag_masks(trucksc, self.pred_boxes) 113 | self.gt_boxes_masks = get_scene_tag_masks(trucksc, self.gt_boxes) 114 | 115 | def evaluate(self, 116 | evaluate_tags: bool = False) -> Tuple[DetectionMetrics, DetectionMetricDataList]: 117 | """ 118 | Performs the actual evaluation. 119 | :param evaluate_tags: Whether to evaluate tag wise. 120 | :return: A tuple of high-level and the raw metric data. 121 | """ 122 | start_time = time.time() 123 | 124 | # Set tags to evaluate 125 | tag_names = ['all'] 126 | if evaluate_tags: 127 | tag_names += TAG_NAMES 128 | 129 | # ----------------------------------- 130 | # Step 1: Accumulate metric data for all classes and distance thresholds. 131 | # ----------------------------------- 132 | if self.verbose: 133 | print('Accumulating metric data...') 134 | metric_data_list = DetectionMetricDataList() 135 | for tag_name in tag_names: 136 | for class_name in self.cfg.class_names: 137 | for dist_th in self.cfg.dist_ths: 138 | md = accumulate(self.gt_boxes, self.pred_boxes, class_name, 139 | self.cfg.dist_fcn_callable, dist_th, 140 | self.gt_boxes_masks.get(tag_name), 141 | self.pred_boxes_masks.get(tag_name)) 142 | metric_data_list.set(tag_name, class_name, dist_th, md) 143 | 144 | # ----------------------------------- 145 | # Step 2: Calculate metrics from the data. 146 | # ----------------------------------- 147 | if self.verbose: 148 | print('Calculating metrics...') 149 | metrics_list = DetectionMetricsList() 150 | for tag_name in tag_names: 151 | metrics = DetectionMetrics(self.cfg) 152 | for class_name in self.cfg.class_names: 153 | # Compute APs. 154 | for dist_th in self.cfg.dist_ths: 155 | metric_data = metric_data_list[(tag_name, class_name, dist_th)] 156 | if tag_name in ['weather.other_weather', 'weather.snow', 'weather.hail', 157 | 'area.parking', 'area.other_area', 'season.spring', 158 | 'lighting.other_lighting']: 159 | ap = np.nan 160 | else: 161 | ap = calc_ap(metric_data, self.cfg.min_recall, self.cfg.min_precision) 162 | metrics.add_label_ap(class_name, dist_th, ap) 163 | 164 | # Compute TP metrics. 165 | for metric_name in TP_METRICS: 166 | metric_data = metric_data_list[(tag_name, class_name, self.cfg.dist_th_tp)] 167 | if tag_name in ['weather.other_weather', 'weather.snow', 'weather.hail', 168 | 'area.parking', 'area.other_area', 'season.spring', 169 | 'lighting.other_lighting']: 170 | tp = np.nan 171 | elif ( 172 | class_name in ['traffic_cone'] and 173 | metric_name in ['attr_err', 'vel_err', 'orient_err'] 174 | ): 175 | tp = np.nan 176 | elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']: 177 | tp = np.nan 178 | elif class_name in ['animal'] and metric_name in ['attr_err']: 179 | tp = np.nan 180 | elif class_name in ['traffic_sign'] and metric_name in ['vel_err']: 181 | tp = np.nan 182 | else: 183 | tp = calc_tp(metric_data, self.cfg.min_recall, metric_name) 184 | metrics.add_label_tp(class_name, metric_name, tp) 185 | 186 | metrics_list.add_detection_metrics(tag_name, metrics) 187 | 188 | # Compute evaluation time. 189 | metrics_list.add_runtime(time.time() - start_time) 190 | 191 | return metrics_list, metric_data_list 192 | 193 | def render(self, metrics: DetectionMetrics, md_list: DetectionMetricDataList) -> None: 194 | """ 195 | Renders various PR and TP curves. 196 | :param metrics: DetectionMetrics instance. 197 | :param md_list: DetectionMetricDataList instance. 198 | """ 199 | # Initialize render module 200 | try: 201 | summary_plot = getattr(import_module("truckscenes.eval.detection.render"), 202 | "summary_plot") 203 | class_pr_curve = getattr(import_module("truckscenes.eval.detection.render"), 204 | "class_pr_curve") 205 | class_tp_curve = getattr(import_module("truckscenes.eval.detection.render"), 206 | "class_tp_curve") 207 | dist_pr_curve = getattr(import_module("truckscenes.eval.detection.render"), 208 | "dist_pr_curve") 209 | except ModuleNotFoundError: 210 | warnings.warn('''The visualization dependencies are not installed on your system! ''' 211 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 212 | else: 213 | # Render curves 214 | if self.verbose: 215 | print('Rendering PR and TP curves') 216 | 217 | def savepath(name): 218 | return os.path.join(self.plot_dir, name + '.pdf') 219 | 220 | summary_plot(md_list, metrics, min_precision=self.cfg.min_precision, 221 | min_recall=self.cfg.min_recall, dist_th_tp=self.cfg.dist_th_tp, 222 | savepath=savepath('summary')) 223 | 224 | for detection_name in self.cfg.class_names: 225 | class_pr_curve(md_list, metrics, detection_name, self.cfg.min_precision, 226 | self.cfg.min_recall, savepath=savepath(detection_name + '_pr')) 227 | 228 | class_tp_curve(md_list, metrics, detection_name, self.cfg.min_recall, 229 | self.cfg.dist_th_tp, savepath=savepath(detection_name + '_tp')) 230 | 231 | for dist_th in self.cfg.dist_ths: 232 | dist_pr_curve(md_list, metrics, dist_th, self.cfg.min_precision, 233 | self.cfg.min_recall, savepath=savepath('dist_pr_' + str(dist_th))) 234 | 235 | def _plot_examples(self, plot_examples: int) -> None: 236 | """ 237 | Plot randomly selected examples. 238 | :param plot_examples: Number of examples to plot. 239 | """ 240 | # Initialize render module 241 | try: 242 | visualize_sample = getattr(import_module("truckscenes.eval.detection.render"), 243 | "visualize_sample") 244 | except ModuleNotFoundError: 245 | warnings.warn('''The visualization dependencies are not installed on your system! ''' 246 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 247 | else: 248 | # Select a random but fixed subset to plot. 249 | random.seed(42) 250 | sample_tokens = list(self.sample_tokens) 251 | random.shuffle(sample_tokens) 252 | sample_tokens = sample_tokens[:plot_examples] 253 | 254 | # Visualize samples. 255 | example_dir = os.path.join(self.output_dir, 'examples') 256 | if not os.path.isdir(example_dir): 257 | os.mkdir(example_dir) 258 | for sample_token in sample_tokens: 259 | visualize_sample(self.trucksc, 260 | sample_token, 261 | self.gt_boxes if self.eval_set != 'test' else EvalBoxes(), 262 | # Don't render test GT. 263 | self.pred_boxes, 264 | eval_range=max(self.cfg.class_range.values()), 265 | savepath=os.path.join(example_dir, '{}.png'.format(sample_token))) 266 | 267 | def main(self, 268 | plot_examples: int = 0, 269 | render_curves: bool = True, 270 | evaluate_tags: bool = False) -> Dict[str, Any]: 271 | """ 272 | Main function that loads the evaluation code, visualizes samples, 273 | runs the evaluation and renders stat plots. 274 | 275 | :param plot_examples: How many example visualizations to write to disk. 276 | :param render_curves: Whether to render PR and TP curves to disk. 277 | :return: A dict that stores the high-level metrics and meta data. 278 | """ 279 | if plot_examples > 0: 280 | self._plot_examples(plot_examples) 281 | 282 | # Run evaluation. 283 | metrics_list, metric_data_list = self.evaluate(evaluate_tags) 284 | 285 | # Render PR and TP curves. 286 | if render_curves: 287 | self.render(metrics_list['all'], metric_data_list) 288 | 289 | # Dump the metric data, meta and metrics to disk. 290 | if self.verbose: 291 | print('Saving metrics to: %s' % self.output_dir) 292 | metrics_summary = metrics_list.serialize() 293 | metrics_summary['meta'] = self.meta.copy() 294 | with open(os.path.join(self.output_dir, 'metrics_summary.json'), 'w') as f: 295 | json.dump(metrics_summary, f, indent=2) 296 | with open(os.path.join(self.output_dir, 'metrics_details.json'), 'w') as f: 297 | json.dump(metric_data_list.serialize(), f, indent=2) 298 | 299 | # Print high-level metrics. 300 | print() 301 | print('High-level results:') 302 | print('mAP: %.4f' % (metrics_summary['all']['mean_ap'])) 303 | err_name_mapping = { 304 | 'trans_err': 'mATE', 305 | 'scale_err': 'mASE', 306 | 'orient_err': 'mAOE', 307 | 'vel_err': 'mAVE', 308 | 'attr_err': 'mAAE' 309 | } 310 | for tp_name, tp_val in metrics_summary['all']['tp_errors'].items(): 311 | print('%s: %.4f' % (err_name_mapping[tp_name], tp_val)) 312 | print('NDS: %.4f' % (metrics_summary['all']['nd_score'])) 313 | print('Eval time: %.1fs' % metrics_summary['eval_time']) 314 | 315 | # Print per-class metrics. 316 | print() 317 | print('Per-class results:') 318 | print('%-22s\t%-6s\t%-6s\t%-6s' % ('Object Class', 'AP', 'ATE', 'ASE') + 319 | '\t%-6s\t%-6s\t%-6s' % ('AOE', 'AVE', 'AAE')) 320 | class_aps = metrics_summary['all']['mean_dist_aps'] 321 | class_tps = metrics_summary['all']['label_tp_errors'] 322 | for class_name in class_aps.keys(): 323 | print('%-22s\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f' % ( 324 | class_name, class_aps[class_name], 325 | class_tps[class_name]['trans_err'], 326 | class_tps[class_name]['scale_err'], 327 | class_tps[class_name]['orient_err'], 328 | class_tps[class_name]['vel_err'], 329 | class_tps[class_name]['attr_err'] 330 | )) 331 | 332 | if not evaluate_tags: 333 | return metrics_summary 334 | 335 | # Print per-tag metrics. 336 | print() 337 | print('Per-tag results:') 338 | print('%-22s\t%-6s\t%-6s\t%-6s' % ('Scene Tag', 'mAP', 'mATE', 'mASE') + 339 | '\t%-6s\t%-6s\t%-6s\t%-6s' % ('mAOE', 'mAVE', 'mAAE', 'NDS')) 340 | for tag_name in TAG_NAMES: 341 | print('%-22s\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f\t%-6.3f' % ( 342 | tag_name, 343 | metrics_summary[tag_name]['mean_ap'], 344 | metrics_summary[tag_name]['tp_errors']['trans_err'], 345 | metrics_summary[tag_name]['tp_errors']['scale_err'], 346 | metrics_summary[tag_name]['tp_errors']['orient_err'], 347 | metrics_summary[tag_name]['tp_errors']['vel_err'], 348 | metrics_summary[tag_name]['tp_errors']['attr_err'], 349 | metrics_summary[tag_name]['nd_score'] 350 | )) 351 | 352 | return metrics_summary 353 | 354 | 355 | class TruckScenesEval(DetectionEval): 356 | """ 357 | Dummy class for backward-compatibility. Same as DetectionEval. 358 | """ 359 | 360 | 361 | if __name__ == "__main__": 362 | 363 | # Settings. 364 | parser = argparse.ArgumentParser(description='Evaluate MAN TruckScenes detection results.', 365 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 366 | parser.add_argument('result_path', type=str, help='The submission as a JSON file.') 367 | parser.add_argument('--output_dir', type=str, default='~/truckscenes-metrics', 368 | help='Folder to store result metrics, graphs and example visualizations.') 369 | parser.add_argument('--eval_set', type=str, default='val', 370 | help='Which dataset split to evaluate on, train, val or test.') 371 | parser.add_argument('--dataroot', type=str, default='/data/man-truckscenes', 372 | help='Default TruckScenes data directory.') 373 | parser.add_argument('--version', type=str, default='v1.0-trainval', 374 | help='Which version of the TruckScenes dataset to evaluate on') 375 | parser.add_argument('--config_path', type=str, default='', 376 | help='Path to the configuration file.' 377 | 'If no path given, the CVPR 2024 configuration will be used.') 378 | parser.add_argument('--evaluate_tags', type=int, default=0, 379 | help='Whether to evaluate tag-wise.') 380 | parser.add_argument('--plot_examples', type=int, default=10, 381 | help='How many example visualizations to write to disk.') 382 | parser.add_argument('--render_curves', type=int, default=1, 383 | help='Whether to render PR and TP curves to disk.') 384 | parser.add_argument('--verbose', type=int, default=1, 385 | help='Whether to print to stdout.') 386 | args = parser.parse_args() 387 | 388 | result_path_ = os.path.expanduser(args.result_path) 389 | output_dir_ = os.path.expanduser(args.output_dir) 390 | eval_set_ = args.eval_set 391 | dataroot_ = args.dataroot 392 | version_ = args.version 393 | config_path = args.config_path 394 | evaluate_tags_ = args.evaluate_tags 395 | plot_examples_ = args.plot_examples 396 | render_curves_ = bool(args.render_curves) 397 | verbose_ = bool(args.verbose) 398 | 399 | if config_path == '': 400 | cfg_ = config_factory('detection_cvpr_2024') 401 | else: 402 | with open(config_path, 'r') as _f: 403 | cfg_ = DetectionConfig.deserialize(json.load(_f)) 404 | 405 | trucksc_ = TruckScenes(version=version_, verbose=verbose_, dataroot=dataroot_) 406 | trucksc_eval = DetectionEval(trucksc_, config=cfg_, result_path=result_path_, 407 | eval_set=eval_set_, output_dir=output_dir_, verbose=verbose_) 408 | trucksc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_, 409 | evaluate_tags=evaluate_tags_) 410 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/render.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from typing import Any 5 | 6 | import numpy as np 7 | 8 | from matplotlib import pyplot as plt 9 | 10 | from truckscenes import TruckScenes 11 | from truckscenes.eval.common.data_classes import EvalBoxes 12 | from truckscenes.eval.common.render import setup_axis 13 | from truckscenes.eval.common.utils import boxes_to_sensor 14 | from truckscenes.eval.detection.constants import TP_METRICS, DETECTION_NAMES, DETECTION_COLORS, \ 15 | TP_METRICS_UNITS, PRETTY_DETECTION_NAMES, PRETTY_TP_METRICS 16 | from truckscenes.eval.detection.data_classes import DetectionMetrics, DetectionMetricData, \ 17 | DetectionMetricDataList 18 | 19 | Axis = Any 20 | 21 | 22 | def visualize_sample(trucksc: TruckScenes, 23 | sample_token: str, 24 | gt_boxes: EvalBoxes, 25 | pred_boxes: EvalBoxes, 26 | nsweeps: int = 1, 27 | conf_th: float = 0.15, 28 | eval_range: float = 50, 29 | verbose: bool = True, 30 | savepath: str = None) -> None: 31 | """ 32 | Visualizes a sample from BEV with annotations and detection results. 33 | :param trucksc: TruckScenes object. 34 | :param sample_token: The TruckScenes sample token. 35 | :param gt_boxes: Ground truth boxes grouped by sample. 36 | :param pred_boxes: Prediction grouped by sample. 37 | :param nsweeps: Number of sweeps used for lidar visualization. 38 | :param conf_th: The confidence threshold used to filter negatives. 39 | :param eval_range: Range in meters beyond which boxes are ignored. 40 | :param verbose: Whether to print to stdout. 41 | :param savepath: If given, saves the the rendering here instead of displaying. 42 | """ 43 | # Retrieve sensor & pose records. 44 | sample_rec = trucksc.get('sample', sample_token) 45 | sd_record = trucksc.get('sample_data', sample_rec['data']['LIDAR_LEFT']) 46 | cs_record = trucksc.get('calibrated_sensor', sd_record['calibrated_sensor_token']) 47 | pose_record = trucksc.get('ego_pose', sd_record['ego_pose_token']) 48 | sample_data_token = [ 49 | sample_rec['data'][sensor] for sensor in sample_rec['data'] if 'lidar' in sensor.lower() 50 | ] 51 | 52 | # Get boxes. 53 | boxes_gt_global = gt_boxes[sample_token] 54 | boxes_est_global = pred_boxes[sample_token] 55 | 56 | # Map GT boxes to lidar. 57 | boxes_gt = boxes_to_sensor(boxes_gt_global, pose_record, cs_record, 58 | use_flat_vehicle_coordinates=True) 59 | 60 | # Map EST boxes to lidar. 61 | boxes_est = boxes_to_sensor(boxes_est_global, pose_record, cs_record, 62 | use_flat_vehicle_coordinates=True) 63 | 64 | # Add scores to EST boxes. 65 | for box_est, box_est_global in zip(boxes_est, boxes_est_global): 66 | box_est.score = box_est_global.detection_score 67 | 68 | # Init axes. 69 | _, ax = plt.subplots(1, 1, figsize=(9, 9)) 70 | axes_limit = eval_range + 3 # Slightly bigger to include boxes that extend beyond the range. 71 | 72 | # Render lidar point cloud 73 | trucksc.render_sample_data(sample_data_token=sample_data_token, with_anns=False, 74 | axes_limit=axes_limit, ax=ax, nsweeps=nsweeps, 75 | use_flat_vehicle_coordinates=True) 76 | 77 | # Show GT boxes. 78 | for box in boxes_gt: 79 | box.render(ax, view=np.eye(4), colors=('g', 'g', 'g'), linewidth=2) 80 | 81 | # Show EST boxes. 82 | for box in boxes_est: 83 | # Show only predictions with a high score. 84 | assert not np.isnan(box.score), 'Error: Box score cannot be NaN!' 85 | if box.score >= conf_th: 86 | box.render(ax, view=np.eye(4), colors=('b', 'b', 'b'), linewidth=1) 87 | 88 | # Limit visible range. 89 | ax.set_xlim(-axes_limit, axes_limit) 90 | ax.set_ylim(-axes_limit, axes_limit) 91 | 92 | # Show / save plot. 93 | if verbose: 94 | print('Rendering sample token %s' % sample_token) 95 | plt.title(sample_token) 96 | if savepath is not None: 97 | plt.savefig(savepath) 98 | plt.close() 99 | else: 100 | plt.show() 101 | 102 | 103 | def class_pr_curve(md_list: DetectionMetricDataList, 104 | metrics: DetectionMetrics, 105 | detection_name: str, 106 | min_precision: float, 107 | min_recall: float, 108 | savepath: str = None, 109 | ax: Axis = None) -> None: 110 | """ 111 | Plot a precision recall curve for the specified class. 112 | :param md_list: DetectionMetricDataList instance. 113 | :param metrics: DetectionMetrics instance. 114 | :param detection_name: The detection class. 115 | :param min_precision: 116 | :param min_recall: Minimum recall value. 117 | :param savepath: If given, saves the the rendering here instead of displaying. 118 | :param ax: Axes onto which to render. 119 | """ 120 | # Prepare axis. 121 | if ax is None: 122 | ax = setup_axis(title=PRETTY_DETECTION_NAMES[detection_name], 123 | xlabel='Recall', ylabel='Precision', xlim=1, ylim=1, 124 | min_precision=min_precision, min_recall=min_recall) 125 | 126 | # Get recall vs precision values of given class for each distance threshold. 127 | data = md_list.get_class_data(detection_name, tag_name='all') 128 | 129 | # Plot the recall vs. precision curve for each distance threshold. 130 | for md, dist_th in data: 131 | md: DetectionMetricData 132 | ap = metrics.get_label_ap(detection_name, dist_th) 133 | ax.plot(md.recall, md.precision, label='Dist. : {}, AP: {:.1f}'.format(dist_th, ap * 100)) 134 | 135 | ax.legend(loc='best') 136 | if savepath is not None: 137 | plt.savefig(savepath) 138 | plt.close() 139 | 140 | 141 | def class_tp_curve(md_list: DetectionMetricDataList, 142 | metrics: DetectionMetrics, 143 | detection_name: str, 144 | min_recall: float, 145 | dist_th_tp: float, 146 | savepath: str = None, 147 | ax: Axis = None) -> None: 148 | """ 149 | Plot the true positive curve for the specified class. 150 | :param md_list: DetectionMetricDataList instance. 151 | :param metrics: DetectionMetrics instance. 152 | :param detection_name: 153 | :param min_recall: Minimum recall value. 154 | :param dist_th_tp: The distance threshold used to determine matches. 155 | :param savepath: If given, saves the the rendering here instead of displaying. 156 | :param ax: Axes onto which to render. 157 | """ 158 | # Get metric data for given detection class with tp distance threshold. 159 | md = md_list[('all', detection_name, dist_th_tp)] 160 | min_recall_ind = round(100 * min_recall) 161 | if min_recall_ind <= md.max_recall_ind: 162 | # For traffic_cone and barrier only a subset of the metrics are plotted. 163 | rel_metrics = [ 164 | m for m in TP_METRICS if not np.isnan(metrics.get_label_tp(detection_name, m)) 165 | ] 166 | ylimit = 1.1 * max( 167 | [ 168 | max(getattr(md, metric)[min_recall_ind:md.max_recall_ind + 1]) 169 | for metric in rel_metrics 170 | ]) 171 | else: 172 | ylimit = 1.0 173 | 174 | # Prepare axis. 175 | if ax is None: 176 | ax = setup_axis(title=PRETTY_DETECTION_NAMES[detection_name], 177 | xlabel='Recall', ylabel='Error', xlim=1, 178 | min_recall=min_recall) 179 | ax.set_ylim(0, ylimit) 180 | 181 | # Plot the recall vs. error curve for each tp metric. 182 | for metric in TP_METRICS: 183 | tp = metrics.get_label_tp(detection_name, metric) 184 | 185 | # Plot only if we have valid data. 186 | if tp is not np.nan and min_recall_ind <= md.max_recall_ind: 187 | recall = md.recall[:md.max_recall_ind + 1] 188 | error = getattr(md, metric)[:md.max_recall_ind + 1] 189 | else: 190 | recall, error = [], [] 191 | 192 | # Change legend based on tp value 193 | if tp is np.nan: 194 | label = '{}: n/a'.format(PRETTY_TP_METRICS[metric]) 195 | elif min_recall_ind > md.max_recall_ind: 196 | label = '{}: nan'.format(PRETTY_TP_METRICS[metric]) 197 | else: 198 | label = \ 199 | '{}: {:.2f} ({})'.format(PRETTY_TP_METRICS[metric], tp, TP_METRICS_UNITS[metric]) 200 | ax.plot(recall, error, label=label) 201 | ax.axvline(x=md.max_recall, linestyle='-.', color=(0, 0, 0, 0.3)) 202 | ax.legend(loc='best') 203 | 204 | if savepath is not None: 205 | plt.savefig(savepath) 206 | plt.close() 207 | 208 | 209 | def dist_pr_curve(md_list: DetectionMetricDataList, 210 | metrics: DetectionMetrics, 211 | dist_th: float, 212 | min_precision: float, 213 | min_recall: float, 214 | savepath: str = None) -> None: 215 | """ 216 | Plot the PR curves for different distance thresholds. 217 | :param md_list: DetectionMetricDataList instance. 218 | :param metrics: DetectionMetrics instance. 219 | :param dist_th: Distance threshold for matching. 220 | :param min_precision: Minimum precision value. 221 | :param min_recall: Minimum recall value. 222 | :param savepath: If given, saves the the rendering here instead of displaying. 223 | """ 224 | # Prepare axis. 225 | fig, (ax, lax) = plt.subplots(ncols=2, gridspec_kw={"width_ratios": [4, 1]}, 226 | figsize=(7.5, 5)) 227 | ax = setup_axis(xlabel='Recall', ylabel='Precision', 228 | xlim=1, ylim=1, min_precision=min_precision, min_recall=min_recall, ax=ax) 229 | 230 | # Plot the recall vs. precision curve for each detection class. 231 | data = md_list.get_dist_data(dist_th, tag_name='all') 232 | for md, detection_name in data: 233 | md = md_list[('all', detection_name, dist_th)] 234 | ap = metrics.get_label_ap(detection_name, dist_th) 235 | ax.plot(md.recall, md.precision, 236 | label='{}: {:.1f}%'.format(PRETTY_DETECTION_NAMES[detection_name], ap * 100), 237 | color=DETECTION_COLORS[detection_name]) 238 | hx, lx = ax.get_legend_handles_labels() 239 | lax.legend(hx, lx, borderaxespad=0) 240 | lax.axis("off") 241 | plt.tight_layout() 242 | if savepath is not None: 243 | plt.savefig(savepath) 244 | plt.close() 245 | 246 | 247 | def summary_plot(md_list: DetectionMetricDataList, 248 | metrics: DetectionMetrics, 249 | min_precision: float, 250 | min_recall: float, 251 | dist_th_tp: float, 252 | savepath: str = None) -> None: 253 | """ 254 | Creates a summary plot with PR and TP curves for each class. 255 | :param md_list: DetectionMetricDataList instance. 256 | :param metrics: DetectionMetrics instance. 257 | :param min_precision: Minimum precision value. 258 | :param min_recall: Minimum recall value. 259 | :param dist_th_tp: The distance threshold used to determine matches. 260 | :param savepath: If given, saves the the rendering here instead of displaying. 261 | """ 262 | n_classes = len(DETECTION_NAMES) 263 | _, axes = plt.subplots(nrows=n_classes, ncols=2, figsize=(15, 5 * n_classes)) 264 | for ind, detection_name in enumerate(DETECTION_NAMES): 265 | title1, title2 = ('Recall vs Precision', 'Recall vs Error') if ind == 0 else (None, None) 266 | 267 | ax1 = setup_axis(xlim=1, ylim=1, title=title1, min_precision=min_precision, 268 | min_recall=min_recall, ax=axes[ind, 0]) 269 | ax1.set_ylabel(f'{PRETTY_DETECTION_NAMES[detection_name]} \n \n Precision', size=20) 270 | 271 | ax2 = setup_axis(xlim=1, title=title2, min_recall=min_recall, ax=axes[ind, 1]) 272 | if ind == n_classes - 1: 273 | ax1.set_xlabel('Recall', size=20) 274 | ax2.set_xlabel('Recall', size=20) 275 | 276 | class_pr_curve(md_list, metrics, detection_name, min_precision, min_recall, ax=ax1) 277 | class_tp_curve(md_list, metrics, detection_name, min_recall, 278 | dist_th_tp=dist_th_tp, ax=ax2) 279 | 280 | plt.tight_layout() 281 | 282 | if savepath is not None: 283 | plt.savefig(savepath) 284 | plt.close() 285 | -------------------------------------------------------------------------------- /src/truckscenes/eval/detection/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | import json 5 | 6 | from typing import List, Optional 7 | 8 | import numpy as np 9 | 10 | from truckscenes.eval.detection.constants import DETECTION_NAMES, \ 11 | TP_METRICS_UNITS, PRETTY_DETECTION_NAMES 12 | 13 | 14 | def category_to_detection_name(category_name: str) -> Optional[str]: 15 | """ 16 | Default label mapping from TruckScenes to TruckScenes detection classes. 17 | Note that pedestrian does not include personal_mobility, stroller and wheelchair. 18 | :param category_name: Generic TruckScenes class. 19 | :return: TruckScenes detection class. 20 | """ 21 | detection_mapping = { 22 | 'animal': 'animal', 23 | 'movable_object.barrier': 'barrier', 24 | 'vehicle.bicycle': 'bicycle', 25 | 'vehicle.bus.bendy': 'bus', 26 | 'vehicle.bus.rigid': 'bus', 27 | 'vehicle.car': 'car', 28 | 'vehicle.motorcycle': 'motorcycle', 29 | 'vehicle.construction': 'other_vehicle', 30 | 'vehicle.other': 'other_vehicle', 31 | 'human.pedestrian.adult': 'pedestrian', 32 | 'human.pedestrian.child': 'pedestrian', 33 | 'human.pedestrian.construction_worker': 'pedestrian', 34 | 'human.pedestrian.police_officer': 'pedestrian', 35 | 'movable_object.trafficcone': 'traffic_cone', 36 | 'static_object.traffic_sign': 'traffic_sign', 37 | 'vehicle.ego_trailer': 'trailer', 38 | 'vehicle.trailer': 'trailer', 39 | 'vehicle.truck': 'truck' 40 | } 41 | 42 | if category_name in detection_mapping: 43 | return detection_mapping[category_name] 44 | else: 45 | return None 46 | 47 | 48 | def detection_name_to_rel_attributes(detection_name: str) -> List[str]: 49 | """ 50 | Returns a list of relevant attributes for a given detection class. 51 | :param detection_name: The detection class. 52 | :return: List of relevant attributes. 53 | """ 54 | if detection_name in ['pedestrian']: 55 | rel_attributes = ['pedestrian.moving', 'pedestrian.sitting_lying_down', 56 | 'pedestrian.standing'] 57 | elif detection_name in ['bicycle', 'motorcycle']: 58 | rel_attributes = ['cycle.with_rider', 'cycle.without_rider'] 59 | elif detection_name in ['car', 'bus', 'other_vehicle', 'trailer', 'truck']: 60 | rel_attributes = ['vehicle.moving', 'vehicle.parked', 'vehicle.stopped'] 61 | elif detection_name in ['traffic_sign']: 62 | rel_attributes = ['traffic_sign.pole_mounted', 'traffic_sign.overhanging', 63 | 'traffic_sign.temporary'] 64 | elif detection_name in ['barrier', 'traffic_cone', 'animal']: 65 | # Classes without attributes: barrier, traffic_cone. 66 | rel_attributes = [] 67 | else: 68 | raise ValueError('Error: %s is not a valid detection class.' % detection_name) 69 | 70 | return rel_attributes 71 | 72 | 73 | def detailed_results_table_tex(metrics_path: str, output_path: str) -> None: 74 | """ 75 | Renders a detailed results table in tex. 76 | :param metrics_path: path to a serialized DetectionMetrics file. 77 | :param output_path: path to the output file. 78 | """ 79 | with open(metrics_path, 'r') as f: 80 | metrics = json.load(f) 81 | 82 | tex = '' 83 | tex += '\\begin{table}[]\n' 84 | tex += '\\small\n' 85 | tex += '\\begin{tabular}{| c | c | c | c | c | c | c |} \\hline\n' 86 | tex += '\\textbf{Class} & \\textbf{AP} & \\textbf{ATE} & \\textbf{ASE} & ' \ 87 | '\\textbf{AOE} & ' \ 88 | '\\textbf{AVE} & ' \ 89 | '\\textbf{AAE} \\\\ \\hline ' \ 90 | '\\hline\n' 91 | for name in DETECTION_NAMES: 92 | ap = np.mean(metrics['label_aps'][name].values()) * 100 93 | ate = metrics['label_tp_errors'][name]['trans_err'] 94 | ase = metrics['label_tp_errors'][name]['scale_err'] 95 | aoe = metrics['label_tp_errors'][name]['orient_err'] 96 | ave = metrics['label_tp_errors'][name]['vel_err'] 97 | aae = metrics['label_tp_errors'][name]['attr_err'] 98 | tex_name = PRETTY_DETECTION_NAMES[name] 99 | if name == 'traffic_cone': 100 | tex += f'{tex_name} & {ap:.1f} & {ate:.2f} & {ase:.2f} & ' \ 101 | f'N/A & N/A & N/A \\\\ \\hline\n' 102 | elif name == 'barrier': 103 | tex += f'{tex_name} & {ap:.1f} & {ate:.2f} & {ase:.2f} & ' \ 104 | f'{aoe:.2f} & N/A & N/A \\\\ \\hline\n' 105 | elif name == 'animal': 106 | tex += f'{tex_name} & {ap:.1f} & {ate:.2f} & {ase:.2f} & ' \ 107 | f'{aoe:.2f} & {ave:.2f} & N/A \\\\ \\hline\n' 108 | elif name == 'traffic_sign': 109 | tex += f'{tex_name} & {ap:.1f} & {ate:.2f} & {ase:.2f} & ' \ 110 | f'{aoe:.2f} & N/A & {aae:.2f} \\\\ \\hline\n' 111 | else: 112 | tex += f'{tex_name} & {ap:.1f} & {ate:.2f} & {ase:.2f} & ' \ 113 | f'{aoe:.2f} & {ave:.2f} & {aae:.2f} \\\\ \\hline\n' 114 | 115 | map_ = metrics['mean_ap'] 116 | mate = metrics['tp_errors']['trans_err'] 117 | mase = metrics['tp_errors']['scale_err'] 118 | maoe = metrics['tp_errors']['orient_err'] 119 | mave = metrics['tp_errors']['vel_err'] 120 | maae = metrics['tp_errors']['attr_err'] 121 | tex += f'\\hline \\textbf{{Mean}} & {map_:.1f} & {mate:.2f} & {mase:.2f} & ' \ 122 | f'{maoe:.2f} & {mave:.2f} & {maae:.2f} \\\\ ' \ 123 | '\\hline\n' 124 | 125 | tex += '\\end{tabular}\n' 126 | 127 | # All one line 128 | tex += '\\caption{Detailed detection performance on the val set. \n' 129 | tex += 'AP: average precision averaged over distance thresholds (%), \n' 130 | tex += 'ATE: average translation error (${}$), \n'.format(TP_METRICS_UNITS['trans_err']) 131 | tex += 'ASE: average scale error (${}$), \n'.format(TP_METRICS_UNITS['scale_err']) 132 | tex += 'AOE: average orientation error (${}$), \n'.format(TP_METRICS_UNITS['orient_err']) 133 | tex += 'AVE: average velocity error (${}$), \n'.format(TP_METRICS_UNITS['vel_err']) 134 | tex += 'AAE: average attribute error (${}$). \n'.format(TP_METRICS_UNITS['attr_err']) 135 | tex += 'nuScenes Detection Score (NDS) = {:.1f} \n'.format(metrics['nd_score'] * 100) 136 | tex += '}\n' 137 | 138 | tex += '\\end{table}\n' 139 | 140 | with open(output_path, 'w') as f: 141 | f.write(tex) 142 | -------------------------------------------------------------------------------- /src/truckscenes/truckscenes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from __future__ import annotations 5 | 6 | import json 7 | import os.path as osp 8 | import sys 9 | import time 10 | import warnings 11 | 12 | from bisect import bisect_left 13 | from collections import OrderedDict 14 | from importlib import import_module 15 | from typing import Dict, List, Tuple, Union 16 | 17 | import numpy as np 18 | 19 | from pyquaternion import Quaternion 20 | 21 | from truckscenes.utils import colormap 22 | from truckscenes.utils.data_classes import Box 23 | from truckscenes.utils.geometry_utils import box_in_image, BoxVisibility 24 | 25 | PYTHON_VERSION = sys.version_info[0] 26 | 27 | if not PYTHON_VERSION == 3: 28 | raise ValueError("truckscenes dev-kit only supports Python version 3.") 29 | 30 | 31 | class TruckScenes: 32 | """ 33 | Database class for truckscenes to help query and retrieve information from the database. 34 | """ 35 | 36 | def __init__(self, 37 | version: str = 'v1.0-mini', 38 | dataroot: str = '/data/man-truckscenes', 39 | verbose: bool = True): 40 | """ 41 | Loads database and creates reverse indexes and shortcuts. 42 | 43 | Arguments: 44 | version: Version to load (e.g. "v1.0-mini", ...). 45 | dataroot: Path to the tables and data. 46 | verbose: Whether to print status messages during load. 47 | """ 48 | self.version = version 49 | self.dataroot = dataroot 50 | self.verbose = verbose 51 | self.table_names = ['attribute', 'calibrated_sensor', 'category', 'ego_motion_cabin', 52 | 'ego_motion_chassis', 'ego_pose', 'instance', 'sample', 53 | 'sample_annotation', 'sample_data', 'scene', 'sensor', 'visibility'] 54 | 55 | assert osp.exists(self.table_root), \ 56 | f'Database version not found: {self.table_root}' 57 | 58 | start_time = time.time() 59 | if verbose: 60 | print(f"======\nLoading truckscenes tables for version {self.version}...") 61 | 62 | # Explicitly assign tables to help the IDE determine valid class members. 63 | self.attribute = self.__load_table__('attribute') 64 | self.calibrated_sensor = self.__load_table__('calibrated_sensor') 65 | self.category = self.__load_table__('category') 66 | self.ego_motion_cabin = self.__load_table__('ego_motion_cabin') 67 | self.ego_motion_chassis = self.__load_table__('ego_motion_chassis') 68 | self.ego_pose = self.__load_table__('ego_pose') 69 | self.instance = self.__load_table__('instance') 70 | self.sample = self.__load_table__('sample') 71 | self.sample_annotation = self.__load_table__('sample_annotation') 72 | self.sample_data = self.__load_table__('sample_data') 73 | self.scene = self.__load_table__('scene') 74 | self.sensor = self.__load_table__('sensor') 75 | self.visibility = self.__load_table__('visibility') 76 | 77 | # Initialize the colormap which maps from class names to RGB values. 78 | self.colormap = colormap.get_colormap() 79 | 80 | if verbose: 81 | for table in self.table_names: 82 | print("{} {},".format(len(getattr(self, table)), table)) 83 | print("Done loading in {:.3f} seconds.\n======".format(time.time() - start_time)) 84 | 85 | # Make reverse indexes for common lookups. 86 | self.__make_reverse_index__(verbose) 87 | 88 | # Initialize TruckScenesExplorer class. 89 | try: 90 | explorer = getattr(import_module("truckscenes.utils.visualization_utils"), 91 | "TruckScenesExplorer") 92 | except ModuleNotFoundError: 93 | warnings.warn('''The visualization dependencies are not installed on your system! ''' 94 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 95 | else: 96 | self.explorer = explorer(self) 97 | 98 | @property 99 | def table_root(self) -> str: 100 | """ Returns the folder where the tables are stored for the relevant version. """ 101 | return osp.join(self.dataroot, self.version) 102 | 103 | def __load_table__(self, table_name) -> dict: 104 | """ Loads a table. """ 105 | with open(osp.join(self.table_root, '{}.json'.format(table_name))) as f: 106 | table = json.load(f) 107 | return table 108 | 109 | def __make_reverse_index__(self, verbose: bool) -> None: 110 | """ 111 | De-normalizes database to create reverse indices for common cases. 112 | 113 | Arguments: 114 | verbose: Whether to print outputs. 115 | """ 116 | start_time = time.time() 117 | if verbose: 118 | print("Reverse indexing ...") 119 | 120 | # Store the mapping from token to table index for each table. 121 | self._token2ind = dict() 122 | for table in self.table_names: 123 | self._token2ind[table] = dict() 124 | 125 | for ind, member in enumerate(getattr(self, table)): 126 | self._token2ind[table][member['token']] = ind 127 | 128 | # Decorate (adds short-cut) sample_annotation table with for category name. 129 | for record in self.sample_annotation: 130 | inst = self.get('instance', record['instance_token']) 131 | record['category_name'] = self.get('category', inst['category_token'])['name'] 132 | 133 | # Decorate (adds short-cut) sample_data with sensor information. 134 | for record in self.sample_data: 135 | cs_record = self.get('calibrated_sensor', record['calibrated_sensor_token']) 136 | sensor_record = self.get('sensor', cs_record['sensor_token']) 137 | record['sensor_modality'] = sensor_record['modality'] 138 | record['channel'] = sensor_record['channel'] 139 | 140 | # Reverse-index samples with sample_data and annotations. 141 | for record in self.sample: 142 | record['data'] = {} 143 | record['anns'] = [] 144 | 145 | for record in self.sample_data: 146 | if record['is_key_frame']: 147 | sample_record = self.get('sample', record['sample_token']) 148 | sample_record['data'][record['channel']] = record['token'] 149 | 150 | for ann_record in self.sample_annotation: 151 | sample_record = self.get('sample', ann_record['sample_token']) 152 | sample_record['anns'].append(ann_record['token']) 153 | 154 | # Reverse-index timestamp to token for fast closest search 155 | self._timestamp2token = dict() 156 | for table in ['ego_pose', 'ego_motion_cabin', 'ego_motion_chassis', 157 | 'sample', 'sample_data']: 158 | tt = [(elem['timestamp'], elem['token']) for elem in getattr(self, table)] 159 | tt = sorted(tt, key=lambda e: e[0]) 160 | self._timestamp2token[table] = OrderedDict(tt) 161 | 162 | if verbose: 163 | print( 164 | "Done reverse indexing in {:.1f} seconds.\n======".format(time.time() - start_time) 165 | ) 166 | 167 | def get(self, table_name: str, token: str) -> dict: 168 | """ 169 | Returns a record from table in constant runtime. 170 | 171 | Arguments: 172 | table_name: Table name. 173 | token: Token of the record. 174 | 175 | Returns: 176 | Table record. See README.md for record details for each table. 177 | """ 178 | assert table_name in self.table_names, "Table {} not found".format(table_name) 179 | return getattr(self, table_name)[self.getind(table_name, token)] 180 | 181 | def getind(self, table_name: str, token: str) -> int: 182 | """ 183 | This returns the index of the record in a table in constant runtime. 184 | 185 | Arguments: 186 | table_name: Table name. 187 | token: Token of the record. 188 | 189 | Returns: 190 | The index of the record in table, table is an array. 191 | """ 192 | return self._token2ind[table_name][token] 193 | 194 | def getclosest(self, table_name: str, timestamp: int) -> dict: 195 | """ 196 | This returns the element with the closest timestamp. 197 | Complexity: O(log n) 198 | 199 | Source: Lauritz V. Thaulow - https://stackoverflow.com/questions/12141150\ 200 | /from-list-of-integers-get-number-closest-to-a-given-value 201 | 202 | Arguments: 203 | table_name: Table name. 204 | timestamp: Timestamp to compare with. 205 | 206 | Returns: 207 | Element of the table with the closest timestamp. 208 | """ 209 | assert table_name in {'ego_pose', 'ego_motion_cabin', 'ego_motion_chassis', 210 | 'sample', 'sample_data'}, \ 211 | f"Table {table_name} has no timestamp" 212 | 213 | # Helper function 214 | def _getclosest(timestamps, t): 215 | """ 216 | Assumes myList is sorted. Returns closest value to myNumber. 217 | If two numbers are equally close, return the smallest number. 218 | """ 219 | pos = bisect_left(timestamps, t) 220 | if pos == 0: 221 | return timestamps[0] 222 | if pos == len(timestamps): 223 | return timestamps[-1] 224 | before = timestamps[pos - 1] 225 | after = timestamps[pos] 226 | if after - t < t - before: 227 | return after 228 | else: 229 | return before 230 | 231 | # Find closest timestamp in given table (name) 232 | closest_timestamp = _getclosest(list(self._timestamp2token[table_name].keys()), timestamp) 233 | 234 | return self.get(table_name, self._timestamp2token[table_name][closest_timestamp]) 235 | 236 | def field2token(self, table_name: str, field: str, query) -> List[str]: 237 | """ 238 | This function queries all records for a certain field value, 239 | and returns the tokens for the matching records. 240 | 241 | Warning: this runs in linear time. 242 | 243 | Arguments: 244 | table_name: Table name. 245 | field: Field name. See README.md for details. 246 | query: Query to match against. Needs to type match the content of the query field. 247 | 248 | Returns: 249 | List of tokens for the matching records. 250 | """ 251 | matches = [] 252 | for member in getattr(self, table_name): 253 | if member[field] == query: 254 | matches.append(member['token']) 255 | return matches 256 | 257 | def get_sample_data_path(self, sample_data_token: str) -> str: 258 | """ Returns the path to a sample_data. """ 259 | 260 | sd_record = self.get('sample_data', sample_data_token) 261 | return osp.join(self.dataroot, sd_record['filename']) 262 | 263 | def get_sample_data(self, sample_data_token: str, 264 | box_vis_level: BoxVisibility = BoxVisibility.ANY, 265 | selected_anntokens: List[str] = None, 266 | use_flat_vehicle_coordinates: bool = False) -> \ 267 | Tuple[str, List[Box], np.ndarray]: 268 | """ 269 | Returns the data path as well as all annotations related to that sample_data. 270 | Note that the boxes are transformed into the current sensor's coordinate frame. 271 | 272 | Arguments: 273 | sample_data_token: Sample_data token. 274 | box_vis_level: If sample_data is an image, this sets required visibility for boxes. 275 | selected_anntokens: If provided only return the selected annotation. 276 | use_flat_vehicle_coordinates: Instead of the current sensor's coordinate frame, 277 | use ego frame which is aligned to z-plane in the world. 278 | 279 | Returns: 280 | (data_path, boxes, camera_intrinsic ) 281 | """ 282 | # Retrieve sensor & pose records 283 | sd_record = self.get('sample_data', sample_data_token) 284 | cs_record = self.get('calibrated_sensor', sd_record['calibrated_sensor_token']) 285 | sensor_record = self.get('sensor', cs_record['sensor_token']) 286 | s_record = self.get('sample', sd_record['sample_token']) 287 | s_pose_record = self.getclosest('ego_pose', s_record['timestamp']) 288 | 289 | data_path = self.get_sample_data_path(sample_data_token) 290 | 291 | if sensor_record['modality'] == 'camera': 292 | cam_intrinsic = np.array(cs_record['camera_intrinsic']) 293 | imsize = (sd_record['width'], sd_record['height']) 294 | else: 295 | cam_intrinsic = None 296 | imsize = None 297 | 298 | # Retrieve all sample annotations and map to sensor coordinate system. 299 | if selected_anntokens is not None: 300 | boxes = list(map(self.get_box, selected_anntokens)) 301 | else: 302 | boxes = self.get_boxes(sample_data_token) 303 | 304 | # Map boxes to sensor coordinate system. 305 | boxes = self.boxes_to_sensor(boxes=boxes, pose_record=s_pose_record, cs_record=cs_record, 306 | use_flat_vehicle_coordinates=use_flat_vehicle_coordinates) 307 | 308 | # Remove boxes that outside the image 309 | if sensor_record['modality'] == 'camera': 310 | boxes = [ 311 | box for box in boxes 312 | if box_in_image(box, cam_intrinsic, imsize, vis_level=box_vis_level) 313 | ] 314 | 315 | return data_path, boxes, cam_intrinsic 316 | 317 | def get_box(self, sample_annotation_token: str) -> Box: 318 | """ 319 | Instantiates a Box class from a sample annotation record. 320 | 321 | Arguments: 322 | sample_annotation_token: Unique sample_annotation identifier. 323 | """ 324 | record = self.get('sample_annotation', sample_annotation_token) 325 | return Box(record['translation'], record['size'], Quaternion(record['rotation']), 326 | name=record['category_name'], token=record['token']) 327 | 328 | def get_boxes(self, sample_data_token: str) -> List[Box]: 329 | """ 330 | Instantiates Boxes for all annotation for a particular sample_data record. 331 | If the sample_data is a keyframe, this returns the annotations for that sample. 332 | But if the sample_data is an intermediate sample_data, a linear interpolation 333 | is applied to estimate the location of the boxes at the time the sample_data was captured. 334 | 335 | Arguments: 336 | sample_data_token: Unique sample_data identifier. 337 | """ 338 | # Retrieve sensor & pose records 339 | sd_record = self.get('sample_data', sample_data_token) 340 | curr_sample_record = self.get('sample', sd_record['sample_token']) 341 | 342 | if curr_sample_record['prev'] == "" or sd_record['is_key_frame']: 343 | # If no previous annotations available, 344 | # or if sample_data is keyframe just return the current ones. 345 | boxes = list(map(self.get_box, curr_sample_record['anns'])) 346 | 347 | else: 348 | prev_sample_record = self.get('sample', curr_sample_record['prev']) 349 | 350 | curr_ann_recs = [ 351 | self.get('sample_annotation', token) for token in curr_sample_record['anns'] 352 | ] 353 | prev_ann_recs = [ 354 | self.get('sample_annotation', token) for token in prev_sample_record['anns'] 355 | ] 356 | 357 | # Maps instance tokens to prev_ann records 358 | prev_inst_map = {entry['instance_token']: entry for entry in prev_ann_recs} 359 | 360 | t0 = prev_sample_record['timestamp'] 361 | t1 = curr_sample_record['timestamp'] 362 | t = sd_record['timestamp'] 363 | 364 | # There are rare situations where the timestamps in the DB are off 365 | # so ensure that t0 < t < t1. 366 | t = max(t0, min(t1, t)) 367 | 368 | boxes = [] 369 | for curr_ann_rec in curr_ann_recs: 370 | 371 | if curr_ann_rec['instance_token'] in prev_inst_map: 372 | # If the annotated instance existed in the previous frame, 373 | # interpolate center & orientation. 374 | prev_ann_rec = prev_inst_map[curr_ann_rec['instance_token']] 375 | 376 | # Interpolate center. 377 | center = [ 378 | np.interp(t, [t0, t1], [c0, c1]) 379 | for c0, c1 in zip(prev_ann_rec['translation'], curr_ann_rec['translation']) 380 | ] 381 | 382 | # Interpolate orientation. 383 | rotation = Quaternion.slerp(q0=Quaternion(prev_ann_rec['rotation']), 384 | q1=Quaternion(curr_ann_rec['rotation']), 385 | amount=(t - t0) / (t1 - t0)) 386 | 387 | box = Box(center, curr_ann_rec['size'], 388 | rotation, name=curr_ann_rec['category_name'], 389 | token=curr_ann_rec['token']) 390 | else: 391 | # If not, simply grab the current annotation. 392 | box = self.get_box(curr_ann_rec['token']) 393 | 394 | boxes.append(box) 395 | return boxes 396 | 397 | def boxes_to_sensor(self, 398 | boxes: List[Box], 399 | pose_record: Dict, 400 | cs_record: Dict, 401 | use_flat_vehicle_coordinates: bool = False) -> List[Box]: 402 | """ 403 | Map boxes from global coordinates to the vehicle's sensor coordinate system. 404 | :param boxes: The boxes in global coordinates. 405 | :param pose_record: The pose record of the vehicle at the current timestamp. 406 | :param cs_record: The calibrated sensor record of the sensor. 407 | :return: The transformed boxes. 408 | """ 409 | boxes_out = [] 410 | for box in boxes: 411 | if use_flat_vehicle_coordinates: 412 | # Move box to ego vehicle coord system parallel to world z plane. 413 | sd_yaw = Quaternion(pose_record['rotation']).yaw_pitch_roll[0] 414 | box.translate(-np.array(pose_record['translation'])) 415 | box.rotate(Quaternion(scalar=np.cos(sd_yaw / 2), 416 | vector=[0, 0, np.sin(sd_yaw / 2)]).inverse) 417 | 418 | # Rotate upwards 419 | box.rotate(Quaternion(axis=box.orientation.rotate([0, 0, 1]), angle=np.pi/2)) 420 | else: 421 | # Move box to ego vehicle coord system. 422 | box.translate(-np.array(pose_record['translation'])) 423 | box.rotate(Quaternion(pose_record['rotation']).inverse) 424 | 425 | # Move box to sensor coord system. 426 | box.translate(-np.array(cs_record['translation'])) 427 | box.rotate(Quaternion(cs_record['rotation']).inverse) 428 | 429 | boxes_out.append(box) 430 | 431 | return boxes_out 432 | 433 | def box_velocity(self, sample_annotation_token: str, max_time_diff: float = 1.5) -> np.ndarray: 434 | """ 435 | Estimate the velocity for an annotation. 436 | If possible, we compute the centered difference between the previous and next frame. 437 | Otherwise we use the difference between the current and previous/next frame. 438 | If the velocity cannot be estimated, values are set to np.nan. 439 | 440 | Arguments: 441 | sample_annotation_token: Unique sample_annotation identifier. 442 | max_time_diff: Max allowed time diff between consecutive samples 443 | that are used to estimate velocities. 444 | 445 | Returns: 446 | . Velocity in x/y/z direction in m/s. 447 | """ 448 | current = self.get('sample_annotation', sample_annotation_token) 449 | has_prev = current['prev'] != '' 450 | has_next = current['next'] != '' 451 | 452 | # Cannot estimate velocity for a single annotation. 453 | if not has_prev and not has_next: 454 | return np.array([np.nan, np.nan, np.nan]) 455 | 456 | if has_prev: 457 | first = self.get('sample_annotation', current['prev']) 458 | else: 459 | first = current 460 | 461 | if has_next: 462 | last = self.get('sample_annotation', current['next']) 463 | else: 464 | last = current 465 | 466 | pos_last = np.array(last['translation']) 467 | pos_first = np.array(first['translation']) 468 | pos_diff = pos_last - pos_first 469 | 470 | time_last = 1e-6 * self.get('sample', last['sample_token'])['timestamp'] 471 | time_first = 1e-6 * self.get('sample', first['sample_token'])['timestamp'] 472 | time_diff = time_last - time_first 473 | 474 | if has_next and has_prev: 475 | # If doing centered difference, allow for up to double the max_time_diff. 476 | max_time_diff *= 2 477 | 478 | if time_diff > max_time_diff: 479 | # If time_diff is too big, don't return an estimate. 480 | return np.array([np.nan, np.nan, np.nan]) 481 | else: 482 | return pos_diff / time_diff 483 | 484 | def list_categories(self) -> None: 485 | self.explorer.list_categories() 486 | 487 | def list_attributes(self) -> None: 488 | self.explorer.list_attributes() 489 | 490 | def list_scenes(self) -> None: 491 | self.explorer.list_scenes() 492 | 493 | def list_sample(self, sample_token: str) -> None: 494 | self.explorer.list_sample(sample_token) 495 | 496 | def render_pointcloud_in_image(self, sample_token: str, dot_size: int = 5, 497 | pointsensor_channel: str = 'LIDAR_LEFT', 498 | camera_channel: str = 'CAMERA_LEFT_FRONT', 499 | out_path: str = None, 500 | render_intensity: bool = False, 501 | cmap: str = 'viridis', 502 | verbose: bool = True) -> None: 503 | self.explorer.render_pointcloud_in_image(sample_token, dot_size, 504 | pointsensor_channel=pointsensor_channel, 505 | camera_channel=camera_channel, 506 | out_path=out_path, 507 | render_intensity=render_intensity, 508 | cmap=cmap, 509 | verbose=verbose) 510 | 511 | def render_sample(self, sample_token: str, 512 | box_vis_level: BoxVisibility = BoxVisibility.ANY, 513 | nsweeps: int = 1, 514 | out_path: str = None, 515 | verbose: bool = True) -> None: 516 | self.explorer.render_sample(token=sample_token, box_vis_level=box_vis_level, 517 | nsweeps=nsweeps, out_path=out_path, verbose=verbose) 518 | 519 | def render_sample_data(self, sample_data_token: str, 520 | with_anns: bool = True, selected_anntokens: List[str] = None, 521 | box_vis_level: BoxVisibility = BoxVisibility.ANY, 522 | axes_limit: float = 40, ax=None, 523 | nsweeps: int = 1, out_path: str = None, 524 | use_flat_vehicle_coordinates: bool = True, 525 | point_scale: float = 1.0, 526 | cmap: str = 'viridis', 527 | cnorm: bool = True) -> None: 528 | self.explorer.render_sample_data(sample_data_token=sample_data_token, 529 | with_anns=with_anns, 530 | selected_anntokens=selected_anntokens, 531 | box_vis_level=box_vis_level, 532 | axes_limit=axes_limit, 533 | ax=ax, 534 | nsweeps=nsweeps, 535 | out_path=out_path, 536 | use_flat_vehicle_coordinates=use_flat_vehicle_coordinates, 537 | point_scale=point_scale, 538 | cmap=cmap, 539 | cnorm=cnorm) 540 | 541 | def render_annotation(self, sample_annotation_token: str, margin: float = 10, 542 | view: np.ndarray = np.eye(4), 543 | use_flat_vehicle_coordinates: bool = True, 544 | box_vis_level: BoxVisibility = BoxVisibility.ANY, 545 | out_path: str = None, 546 | extra_info: bool = False) -> None: 547 | self.explorer.render_annotation(anntoken=sample_annotation_token, 548 | margin=margin, view=view, 549 | use_flat_vehicle_coordinates=use_flat_vehicle_coordinates, 550 | box_vis_level=box_vis_level, out_path=out_path, 551 | extra_info=extra_info) 552 | 553 | def render_instance(self, instance_token: str, margin: float = 10, 554 | view: np.ndarray = np.eye(4), 555 | box_vis_level: BoxVisibility = BoxVisibility.ANY, 556 | out_path: str = None, 557 | extra_info: bool = False) -> None: 558 | self.explorer.render_instance(instance_token=instance_token, margin=margin, view=view, 559 | box_vis_level=box_vis_level, out_path=out_path, 560 | extra_info=extra_info) 561 | 562 | def render_scene(self, scene_token: str, freq: float = 10, 563 | imsize: Tuple[float, float] = (640, 360), 564 | out_path: str = None) -> None: 565 | self.explorer.render_scene(scene_token=scene_token, freq=freq, imsize=imsize, 566 | out_path=out_path) 567 | 568 | def render_scene_channel(self, scene_token: str, channel: str = 'CAMERA_LEFT_FRONT', 569 | freq: float = 10, imsize: Tuple[float, float] = (640, 360), 570 | out_path: str = None) -> None: 571 | self.explorer.render_scene_channel(scene_token, channel=channel, freq=freq, 572 | imsize=imsize, out_path=out_path) 573 | 574 | def render_pointcloud(self, 575 | sample_rec: Dict, 576 | chans: Union[str, List[str]], 577 | ref_chan: str, 578 | with_anns: bool = True, 579 | box_vis_level: BoxVisibility = BoxVisibility.ANY, 580 | nsweeps: int = 1, 581 | min_distance: float = 1.0, 582 | cmap: str = 'viridis', 583 | out_path: str = None) -> None: 584 | self.explorer.render_pointcloud(sample_rec=sample_rec, 585 | chans=chans, 586 | ref_chan=ref_chan, 587 | with_anns=with_anns, 588 | box_vis_level=box_vis_level, 589 | nsweeps=nsweeps, 590 | min_distance=min_distance, 591 | cmap=cmap, 592 | out_path=out_path) 593 | 594 | def render_calibrated_sensor(self, 595 | sample_token: str, 596 | out_path: str = None) -> None: 597 | self.explorer.render_calibrated_sensor(sample_token=sample_token, 598 | out_path=out_path) 599 | -------------------------------------------------------------------------------- /src/truckscenes/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TUMFTM/truckscenes-devkit/7c94b2a38492361378a312e85f5f668892265d39/src/truckscenes/utils/__init__.py -------------------------------------------------------------------------------- /src/truckscenes/utils/colormap.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from typing import Dict, Tuple 5 | 6 | 7 | def get_colormap() -> Dict[str, Tuple[int, int, int]]: 8 | """ 9 | Get the defined colormap. 10 | :return: A mapping from the class names to the respective RGB values. 11 | """ 12 | 13 | classname_to_color = { # RGB. 14 | "animal": (70, 130, 180), # Steelblue 15 | "human.pedestrian.adult": (0, 0, 230), # Blue 16 | "human.pedestrian.child": (135, 206, 235), # Skyblue, 17 | "human.pedestrian.construction_worker": (100, 149, 237), # Cornflowerblue 18 | "human.pedestrian.personal_mobility": (219, 112, 147), # Palevioletred 19 | "human.pedestrian.police_officer": (0, 0, 128), # Navy, 20 | "human.pedestrian.stroller": (240, 128, 128), # Lightcoral 21 | "human.pedestrian.wheelchair": (138, 43, 226), # Blueviolet 22 | "movable_object.barrier": (112, 128, 144), # Slategrey 23 | "movable_object.debris": (210, 105, 30), # Chocolate 24 | "movable_object.pushable_pullable": (105, 105, 105), # Dimgrey 25 | "movable_object.trafficcone": (47, 79, 79), # Darkslategrey 26 | "static_object.bicycle_rack": (188, 143, 143), # Rosybrown 27 | "static_object.traffic_sign": (222, 184, 135), # Burlywood 28 | "vehicle.bicycle": (220, 20, 60), # Crimson 29 | "vehicle.bus.bendy": (255, 127, 80), # Coral 30 | "vehicle.bus.rigid": (255, 69, 0), # Orangered 31 | "vehicle.car": (255, 158, 0), # Orange 32 | "vehicle.construction": (233, 150, 70), # Darksalmon 33 | "vehicle.emergency.ambulance": (255, 83, 0), 34 | "vehicle.emergency.police": (255, 215, 0), # Gold 35 | "vehicle.motorcycle": (255, 61, 99), # Red 36 | "vehicle.trailer": (255, 140, 0), # Darkorange 37 | "vehicle.truck": (255, 99, 71), # Tomato 38 | "vehicle.train": (255, 228, 196), # Bisque 39 | "vehicle.other": (255, 240, 245), 40 | "vehicle.ego_trailer": (0, 0, 0) # Black 41 | } 42 | 43 | return classname_to_color 44 | -------------------------------------------------------------------------------- /src/truckscenes/utils/data_classes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | 5 | from __future__ import annotations 6 | 7 | import copy 8 | import os.path as osp 9 | import warnings 10 | 11 | from abc import ABC, abstractmethod 12 | from functools import reduce 13 | from importlib import import_module 14 | from typing import Tuple, List, Dict 15 | 16 | import numpy as np 17 | import pypcd4 18 | 19 | from pyquaternion import Quaternion 20 | 21 | from truckscenes.utils.geometry_utils import transform_matrix 22 | 23 | 24 | class PointCloud(ABC): 25 | """ 26 | Abstract class for manipulating and viewing point clouds. 27 | Every point cloud (lidar and radar) consists of points where: 28 | - Dimensions 0, 1, 2 represent x, y, z coordinates. 29 | These are modified when the point cloud is rotated or translated. 30 | - All other dimensions are optional. Hence these have to be manually modified 31 | if the reference frame changes. 32 | """ 33 | 34 | def __init__(self, points: np.ndarray, timestamps: np.ndarray = None): 35 | """ 36 | Initialize a point cloud and check it has the correct dimensions. 37 | :param points: . d-dimensional input point cloud matrix. 38 | """ 39 | assert points.shape[0] == self.nbr_dims(), \ 40 | 'Error: Pointcloud points must have format: %d x n' % self.nbr_dims() 41 | self.points = points 42 | self.timestamps = timestamps 43 | 44 | @staticmethod 45 | @abstractmethod 46 | def nbr_dims() -> int: 47 | """ 48 | Returns the number of dimensions. 49 | :return: Number of dimensions. 50 | """ 51 | pass 52 | 53 | @classmethod 54 | @abstractmethod 55 | def from_file(cls, file_name: str) -> PointCloud: 56 | """ 57 | Loads point cloud from disk. 58 | :param file_name: Path of the pointcloud file on disk. 59 | :return: PointCloud instance. 60 | """ 61 | pass 62 | 63 | @classmethod 64 | def from_file_multisweep(cls, 65 | trucksc, 66 | sample_rec: Dict, 67 | chan: str, 68 | ref_chan: str, 69 | nsweeps: int = 5, 70 | min_distance: float = 1.0) -> Tuple[PointCloud, np.ndarray]: 71 | """ 72 | Return a point cloud that aggregates multiple sweeps. 73 | As every sweep is in a different coordinate frame, we need to map the coordinates 74 | to a single reference frame. 75 | As every sweep has a different timestamp, we need to account for that in 76 | the transformations and timestamps. 77 | 78 | Arguments: 79 | trucksc: A TruckScenes instance. 80 | sample_rec: The current sample. 81 | chan: The lidar/radar channel from which we track back n sweeps to aggregate 82 | the point cloud. 83 | ref_chan: The reference channel of the current sample_rec that the 84 | point clouds are mapped to. 85 | nsweeps: Number of sweeps to aggregated. 86 | min_distance: Distance below which points are discarded. 87 | 88 | Returns: 89 | all_pc: The aggregated point cloud. 90 | all_times: The aggregated timestamps. 91 | """ 92 | # Init. 93 | points = np.zeros((cls.nbr_dims(), 0), dtype=np.float64) 94 | timestamps = np.zeros((1, 0), dtype=np.uint64) 95 | all_pc = cls(points, timestamps) 96 | all_times = np.zeros((1, 0)) 97 | 98 | # Get reference pose and timestamp. 99 | ref_sd_token = sample_rec['data'][ref_chan] 100 | ref_sd_rec = trucksc.get('sample_data', ref_sd_token) 101 | ref_pose_rec = trucksc.get('ego_pose', ref_sd_rec['ego_pose_token']) 102 | ref_cs_rec = trucksc.get('calibrated_sensor', ref_sd_rec['calibrated_sensor_token']) 103 | ref_time = 1e-6 * ref_sd_rec['timestamp'] 104 | 105 | # Homogeneous transform from ego car frame to reference frame. 106 | ref_from_car = transform_matrix(ref_cs_rec['translation'], 107 | Quaternion(ref_cs_rec['rotation']), 108 | inverse=True) 109 | 110 | # Homogeneous transformation matrix from global to _current_ ego car frame. 111 | car_from_global = transform_matrix(ref_pose_rec['translation'], 112 | Quaternion(ref_pose_rec['rotation']), 113 | inverse=True) 114 | 115 | # Aggregate current and previous sweeps. 116 | sample_data_token = sample_rec['data'][chan] 117 | current_sd_rec = trucksc.get('sample_data', sample_data_token) 118 | 119 | for _ in range(nsweeps): 120 | # Load up the pointcloud and remove points close to the sensor. 121 | current_pc = cls.from_file(osp.join(trucksc.dataroot, current_sd_rec['filename'])) 122 | current_pc.remove_close(min_distance) 123 | 124 | # Get past pose. 125 | current_pose_rec = trucksc.get('ego_pose', current_sd_rec['ego_pose_token']) 126 | global_from_car = transform_matrix(current_pose_rec['translation'], 127 | Quaternion(current_pose_rec['rotation']), 128 | inverse=False) 129 | 130 | # Homogeneous transformation matrix from sensor coordinate frame to ego car frame. 131 | current_cs_rec = trucksc.get('calibrated_sensor', 132 | current_sd_rec['calibrated_sensor_token']) 133 | car_from_current = transform_matrix(current_cs_rec['translation'], 134 | Quaternion(current_cs_rec['rotation']), 135 | inverse=False) 136 | 137 | # Fuse four transformation matrices into one and perform transform. 138 | trans_matrix = reduce(np.dot, [ref_from_car, car_from_global, 139 | global_from_car, car_from_current]) 140 | current_pc.transform(trans_matrix) 141 | 142 | # Add time vector which can be used as a temporal feature. 143 | if current_pc.timestamps is not None: 144 | # Per point difference 145 | time_lag = ref_time - 1e-6 * current_pc.timestamps 146 | else: 147 | # Difference to sample data 148 | time_lag = ref_time - 1e-6 * current_sd_rec['timestamp'] 149 | time_lag = time_lag * np.ones((1, current_pc.nbr_points())) 150 | all_times = np.hstack((all_times, time_lag)) 151 | 152 | # Merge with key pc. 153 | all_pc.points = np.hstack((all_pc.points, current_pc.points)) 154 | if current_pc.timestamps is not None: 155 | all_pc.timestamps = np.hstack((all_pc.timestamps, current_pc.timestamps)) 156 | 157 | # Abort if there are no previous sweeps. 158 | if current_sd_rec['prev'] == '': 159 | break 160 | else: 161 | current_sd_rec = trucksc.get('sample_data', current_sd_rec['prev']) 162 | 163 | return all_pc, all_times 164 | 165 | def nbr_points(self) -> int: 166 | """ 167 | Returns the number of points. 168 | :return: Number of points. 169 | """ 170 | return self.points.shape[1] 171 | 172 | def subsample(self, ratio: float) -> None: 173 | """ 174 | Sub-samples the pointcloud. 175 | :param ratio: Fraction to keep. 176 | """ 177 | selected_ind = np.random.choice(np.arange(0, self.nbr_points()), 178 | size=int(self.nbr_points() * ratio)) 179 | self.points = self.points[:, selected_ind] 180 | 181 | def remove_close(self, radius: float) -> None: 182 | """ 183 | Removes point too close within a certain radius from origin. 184 | :param radius: Radius below which points are removed. 185 | """ 186 | 187 | x_filt = np.abs(self.points[0, :]) < radius 188 | y_filt = np.abs(self.points[1, :]) < radius 189 | not_close = np.logical_not(np.logical_and(x_filt, y_filt)) 190 | self.points = self.points[:, not_close] 191 | 192 | if self.timestamps is not None: 193 | self.timestamps = self.timestamps[:, not_close] 194 | 195 | def translate(self, x: np.ndarray) -> None: 196 | """ 197 | Applies a translation to the point cloud. 198 | :param x: . Translation in x, y, z. 199 | """ 200 | for i in range(3): 201 | self.points[i, :] = self.points[i, :] + x[i] 202 | 203 | def rotate(self, rot_matrix: np.ndarray) -> None: 204 | """ 205 | Applies a rotation. 206 | :param rot_matrix: . Rotation matrix. 207 | """ 208 | self.points[:3, :] = np.dot(rot_matrix, self.points[:3, :]) 209 | 210 | def transform(self, transf_matrix: np.ndarray) -> None: 211 | """ 212 | Applies a homogeneous transform. 213 | :param transf_matrix: . Homogenous transformation matrix. 214 | """ 215 | self.points[:3, :] = transf_matrix.dot( 216 | np.vstack((self.points[:3, :], np.ones(self.nbr_points()))) 217 | )[:3, :] 218 | 219 | def render_height(self, 220 | ax, 221 | view: np.ndarray = np.eye(4), 222 | x_lim: Tuple[float, float] = (-20, 20), 223 | y_lim: Tuple[float, float] = (-20, 20), 224 | marker_size: float = 1) -> None: 225 | """ 226 | Very simple method that applies a transformation and then scatter plots the points 227 | colored by height (z-value). 228 | 229 | :param ax: Axes on which to render the points. 230 | :param view: . Defines an arbitrary projection (n <= 4). 231 | :param x_lim: (min, max). x range for plotting. 232 | :param y_lim: (min, max). y range for plotting. 233 | :param marker_size: Marker size. 234 | """ 235 | # Initialize visualization methods 236 | try: 237 | _render_helper = getattr(import_module("truckscenes.utils.visualization_utils"), 238 | "_render_pc_helper") 239 | except ModuleNotFoundError: 240 | print('''The visualization dependencies are not installed on your system! ''' 241 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 242 | 243 | # Render point cloud 244 | _render_helper(2, ax, view, x_lim, y_lim, marker_size) 245 | 246 | def render_intensity(self, 247 | ax, 248 | view: np.ndarray = np.eye(4), 249 | x_lim: Tuple[float, float] = (-20, 20), 250 | y_lim: Tuple[float, float] = (-20, 20), 251 | marker_size: float = 1) -> None: 252 | """ 253 | Very simple method that applies a transformation and then scatter plots the points 254 | colored by intensity. 255 | 256 | :param ax: Axes on which to render the points. 257 | :param view: . Defines an arbitrary projection (n <= 4). 258 | :param x_lim: (min, max). 259 | :param y_lim: (min, max). 260 | :param marker_size: Marker size. 261 | """ 262 | # Initialize visualization methods 263 | try: 264 | _render_helper = getattr(import_module("truckscenes.utils.visualization_utils"), 265 | "_render_pc_helper") 266 | except ModuleNotFoundError: 267 | warnings.warn('''The visualization dependencies are not installed on your system! ''' 268 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 269 | 270 | # Render point cloud 271 | _render_helper(3, ax, view, x_lim, y_lim, marker_size) 272 | 273 | 274 | class LidarPointCloud(PointCloud): 275 | 276 | @staticmethod 277 | def nbr_dims() -> int: 278 | """ 279 | Returns the number of dimensions. 280 | :return: Number of dimensions. 281 | """ 282 | return 4 283 | 284 | @classmethod 285 | def from_file(cls, file_name: str) -> LidarPointCloud: 286 | """ 287 | Loads LIDAR data from binary numpy format. Data is stored 288 | as (x, y, z, intensity, ring index). 289 | 290 | :param file_name: Path of the pointcloud file on disk. 291 | :return: LidarPointCloud instance (x, y, z, intensity). 292 | """ 293 | 294 | assert file_name.endswith('.pcd'), 'Unsupported filetype {}'.format(file_name) 295 | 296 | lidar = pypcd4.PointCloud.from_path(file_name) 297 | 298 | lidar_data = lidar.pc_data 299 | points = np.array([lidar_data["x"], lidar_data["y"], lidar_data["z"], 300 | lidar_data["intensity"]], dtype=np.float64) 301 | 302 | return cls(points, np.atleast_2d(lidar_data["timestamp"])) 303 | 304 | 305 | class RadarPointCloud(PointCloud): 306 | 307 | @staticmethod 308 | def nbr_dims() -> int: 309 | """ 310 | Returns the number of dimensions. 311 | :return: Number of dimensions. 312 | """ 313 | return 7 314 | 315 | @classmethod 316 | def from_file(cls, 317 | file_name: str) -> RadarPointCloud: 318 | 319 | assert file_name.endswith('.pcd'), 'Unsupported filetype {}'.format(file_name) 320 | 321 | radar = pypcd4.PointCloud.from_path(file_name) 322 | 323 | radar_data = radar.pc_data 324 | points = np.array([radar_data["x"], radar_data["y"], radar_data["z"], 325 | radar_data["vrel_x"], radar_data["vrel_y"], radar_data["vrel_z"], 326 | radar_data["rcs"]], dtype=np.float64) 327 | 328 | return cls(points) 329 | 330 | def rotate(self, rot_matrix: np.ndarray) -> None: 331 | """ 332 | Applies a rotation. 333 | :param rot_matrix: . Rotation matrix. 334 | """ 335 | # Rotate the point cloud positions 336 | self.points[:3, :] = np.dot(rot_matrix, self.points[:3, :]) 337 | # Rotate the velocity vectors 338 | self.points[3:6, :] = np.dot(rot_matrix, self.points[3:6, :]) 339 | 340 | def transform(self, transf_matrix: np.ndarray) -> None: 341 | """ 342 | Applies a homogeneous transform. 343 | :param transf_matrix: . Homogenous transformation matrix. 344 | """ 345 | # Transform the point cloud positions 346 | self.points[:3, :] = transf_matrix.dot( 347 | np.vstack((self.points[:3, :], np.ones(self.nbr_points()))) 348 | )[:3, :] 349 | # Transform the velocity vectors 350 | self.points[3:6, :] = np.dot(np.hstack((transf_matrix[:, :3], np.identity(4)[:, 3][:, None])), 351 | np.vstack((self.points[3:6, :], np.ones(self.nbr_points()))) 352 | )[:3, :] 353 | 354 | 355 | class Box: 356 | """ Simple data class representing a 3d box including, label, score and velocity. """ 357 | 358 | def __init__(self, 359 | center: List[float], 360 | size: List[float], 361 | orientation: Quaternion, 362 | label: int = np.nan, 363 | score: float = np.nan, 364 | velocity: Tuple = (np.nan, np.nan, np.nan), 365 | name: str = None, 366 | token: str = None): 367 | """ 368 | :param center: Center of box given as x, y, z. 369 | :param size: Size of box in width, length, height. 370 | :param orientation: Box orientation. 371 | :param label: Integer label, optional. 372 | :param score: Classification score, optional. 373 | :param velocity: Box velocity in x, y, z direction. 374 | :param name: Box name, optional. Can be used e.g. for denote category name. 375 | :param token: Unique string identifier from DB. 376 | """ 377 | assert not np.any(np.isnan(center)) 378 | assert not np.any(np.isnan(size)) 379 | assert len(center) == 3 380 | assert len(size) == 3 381 | assert type(orientation) == Quaternion 382 | 383 | self.center = np.array(center) 384 | self.wlh = np.array(size) 385 | self.orientation = orientation 386 | self.label = int(label) if not np.isnan(label) else label 387 | self.score = float(score) if not np.isnan(score) else score 388 | self.velocity = np.array(velocity) 389 | self.name = name 390 | self.token = token 391 | 392 | def __eq__(self, other): 393 | center = np.allclose(self.center, other.center) 394 | wlh = np.allclose(self.wlh, other.wlh) 395 | orientation = np.allclose(self.orientation.elements, other.orientation.elements) 396 | label = (self.label == other.label) or (np.isnan(self.label) and np.isnan(other.label)) 397 | score = (self.score == other.score) or (np.isnan(self.score) and np.isnan(other.score)) 398 | vel = (np.allclose(self.velocity, other.velocity) or 399 | (np.all(np.isnan(self.velocity)) and np.all(np.isnan(other.velocity)))) 400 | 401 | return center and wlh and orientation and label and score and vel 402 | 403 | def __repr__(self): 404 | repr_str = \ 405 | f'label: {self.label}, ' \ 406 | f'score: {self.score:.2f}, ' \ 407 | f'xyz: [{self.center[0]:.2f}, {self.center[1]:.2f}, {self.center[2]:.2f}], ' \ 408 | f'wlh: [{self.wlh[0]:.2f}, {self.wlh[1]:.2f}, {self.wlh[2]:.2f}], ' \ 409 | f'rot axis: [{self.orientation.axis[0]:.2f}, {self.orientation.axis[1]:.2f}, ' \ 410 | f'{self.orientation.axis[2]:.2f}], ' \ 411 | f'ang(degrees): {self.orientation.degrees:.2f}, ' \ 412 | f'ang(rad): {self.orientation.radians:.2f}, ' \ 413 | f'vel: {self.velocity[0]:.2f}, {self.velocity[1]:.2f}, {self.velocity[2]:.2f}, ' \ 414 | f'name: {self.name}, ' \ 415 | f'token: {self.token}' 416 | 417 | return repr_str 418 | 419 | @property 420 | def rotation_matrix(self) -> np.ndarray: 421 | """ 422 | Return a rotation matrix. 423 | :return: . The box's rotation matrix. 424 | """ 425 | return self.orientation.rotation_matrix 426 | 427 | def translate(self, x: np.ndarray) -> None: 428 | """ 429 | Applies a translation. 430 | :param x: . Translation in x, y, z direction. 431 | """ 432 | self.center += x 433 | 434 | def rotate(self, quaternion: Quaternion) -> None: 435 | """ 436 | Rotates box. 437 | :param quaternion: Rotation to apply. 438 | """ 439 | self.center = np.dot(quaternion.rotation_matrix, self.center) 440 | self.orientation = quaternion * self.orientation 441 | self.velocity = np.dot(quaternion.rotation_matrix, self.velocity) 442 | 443 | def corners(self, wlh_factor: float = 1.0) -> np.ndarray: 444 | """ 445 | Returns the bounding box corners. 446 | :param wlh_factor: Multiply w, l, h by a factor to scale the box. 447 | :return: . First four corners are the ones facing forward. 448 | The last four are the ones facing backwards. 449 | """ 450 | w, l, h = self.wlh * wlh_factor 451 | 452 | # 3D bounding box corners. (Convention: x points forward, y to the left, z up.) 453 | x_corners = l / 2 * np.array([1, 1, 1, 1, -1, -1, -1, -1]) 454 | y_corners = w / 2 * np.array([1, -1, -1, 1, 1, -1, -1, 1]) 455 | z_corners = h / 2 * np.array([1, 1, -1, -1, 1, 1, -1, -1]) 456 | corners = np.vstack((x_corners, y_corners, z_corners)) 457 | 458 | # Rotate 459 | corners = np.dot(self.orientation.rotation_matrix, corners) 460 | 461 | # Translate 462 | x, y, z = self.center 463 | corners[0, :] = corners[0, :] + x 464 | corners[1, :] = corners[1, :] + y 465 | corners[2, :] = corners[2, :] + z 466 | 467 | return corners 468 | 469 | def bottom_corners(self) -> np.ndarray: 470 | """ 471 | Returns the four bottom corners. 472 | :return: . Bottom corners. First two face forward, last two face backwards. 473 | """ 474 | return self.corners()[:, [2, 3, 7, 6]] 475 | 476 | def render(self, 477 | axis, 478 | view: np.ndarray = np.eye(3), 479 | normalize: bool = False, 480 | colors: Tuple = ('b', 'r', 'k'), 481 | linewidth: float = 2) -> None: 482 | """ 483 | Renders the box in the provided Matplotlib axis. 484 | :param axis: Axis onto which the box should be drawn. 485 | :param view: . Define a projection in needed 486 | (e.g. for drawing projection in an image). 487 | :param normalize: Whether to normalize the remaining coordinate. 488 | :param colors: (: 3). Valid Matplotlib colors 489 | ( or normalized RGB tuple) for front, back and sides. 490 | :param linewidth: Width in pixel of the box sides. 491 | """ 492 | # Initialize visualization methods 493 | try: 494 | render_box = getattr(import_module("truckscenes.utils.visualization_utils"), 495 | "render_box") 496 | except ModuleNotFoundError: 497 | warnings.warn('''The visualization dependencies are not installed on your system! ''' 498 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 499 | 500 | # Render box 501 | render_box(self, axis, view, normalize, colors, linewidth) 502 | 503 | def render_cv2(self, 504 | im: np.ndarray, 505 | view: np.ndarray = np.eye(3), 506 | normalize: bool = False, 507 | colors: Tuple = ((0, 0, 255), (255, 0, 0), (155, 155, 155)), 508 | linewidth: int = 2) -> None: 509 | """ 510 | Renders box using OpenCV2. 511 | :param im: . Image array. Channels are in BGR order. 512 | :param view: . Define a projection if needed 513 | (e.g. for drawing projection in an image). 514 | :param normalize: Whether to normalize the remaining coordinate. 515 | :param colors: ((R, G, B), (R, G, B), (R, G, B)). Colors for front, side & rear. 516 | :param linewidth: Linewidth for plot. 517 | """ 518 | # Initialize visualization methods 519 | try: 520 | render_box_cv2 = getattr(import_module("truckscenes.utils.visualization_utils"), 521 | "render_box_cv2") 522 | except ModuleNotFoundError: 523 | warnings.warn('''The visualization dependencies are not installed on your system! ''' 524 | '''Run 'pip install "truckscenes-devkit[all]"'.''') 525 | 526 | # Render box 527 | render_box_cv2(self, im, view, normalize, colors, linewidth) 528 | 529 | def copy(self) -> Box: 530 | """ 531 | Create a copy of self. 532 | :return: A copy. 533 | """ 534 | return copy.deepcopy(self) 535 | -------------------------------------------------------------------------------- /src/truckscenes/utils/geometry_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Motional 2 | # Copyright 2024 MAN Truck & Bus SE 3 | 4 | from enum import IntEnum 5 | from typing import Tuple 6 | 7 | import numpy as np 8 | from pyquaternion import Quaternion 9 | 10 | 11 | class BoxVisibility(IntEnum): 12 | """ Enumerates the various level of box visibility in an image """ 13 | ALL = 0 # Requires all corners are inside the image. 14 | ANY = 1 # Requires at least one corner visible in the image. 15 | NONE = 2 # Requires no corners to be inside, i.e. box can be fully outside the image. 16 | 17 | 18 | def view_points(points: np.ndarray, view: np.ndarray, normalize: bool) -> np.ndarray: 19 | """ 20 | This is a helper class that maps 3d points to a 2d plane. It can be used to implement 21 | both perspective and orthographic projections. It first applies the dot product between 22 | the points and the view. By convention, the view should be such that the data is projected 23 | onto the first 2 axis. It then optionally applies a normalization along the third dimension. 24 | 25 | For a perspective projection the view should be a 3x3 camera matrix, and normalize=True 26 | For an orthographic projection with translation the view is a 3x4 matrix and normalize=False 27 | For an orthographic projection without translation the view is a 3x3 matrix 28 | (optionally 3x4 with last columns all zeros) and normalize=False 29 | 30 | :param points: Matrix of points, where each point (x, y, z) 31 | is along each column. 32 | :param view: . Defines an arbitrary projection (n <= 4). 33 | The projection should be such that the corners are projected onto the first 2 axis. 34 | :param normalize: Whether to normalize the remaining coordinate (along the third axis). 35 | :return: . Mapped point. If normalize=False, 36 | the third coordinate is the height. 37 | """ 38 | assert view.shape[0] <= 4 39 | assert view.shape[1] <= 4 40 | assert points.shape[0] == 3 41 | 42 | viewpad = np.eye(4) 43 | viewpad[:view.shape[0], :view.shape[1]] = view 44 | 45 | nbr_points = points.shape[1] 46 | 47 | # Do operation in homogenous coordinates. 48 | points = np.concatenate((points, np.ones((1, nbr_points)))) 49 | points = np.dot(viewpad, points) 50 | points = points[:3, :] 51 | 52 | if normalize: 53 | points = points / points[2:3, :].repeat(3, 0).reshape(3, nbr_points) 54 | 55 | return points 56 | 57 | 58 | def box_in_image(box, intrinsic: np.ndarray, imsize: Tuple[int, int], 59 | vis_level: int = BoxVisibility.ANY) -> bool: 60 | """ 61 | Check if a box is visible inside an image without accounting for occlusions. 62 | :param box: The box to be checked. 63 | :param intrinsic: . Intrinsic camera matrix. 64 | :param imsize: (width, height). 65 | :param vis_level: One of the enumerations of . 66 | :return True if visibility condition is satisfied. 67 | """ 68 | 69 | corners_3d = box.corners() 70 | corners_img = view_points(corners_3d, intrinsic, normalize=True)[:2, :] 71 | 72 | visible = np.logical_and(corners_img[0, :] > 0, corners_img[0, :] < imsize[0]) 73 | visible = np.logical_and(visible, corners_img[1, :] < imsize[1]) 74 | visible = np.logical_and(visible, corners_img[1, :] > 0) 75 | visible = np.logical_and(visible, corners_3d[2, :] > 1) 76 | 77 | # True if a corner is at least 0.1 meter in front of the camera. 78 | in_front = corners_3d[2, :] > 0.1 79 | 80 | if vis_level == BoxVisibility.ALL: 81 | return all(visible) and all(in_front) 82 | elif vis_level == BoxVisibility.ANY: 83 | return any(visible) and all(in_front) 84 | elif vis_level == BoxVisibility.NONE: 85 | return True 86 | else: 87 | raise ValueError("vis_level: {} not valid".format(vis_level)) 88 | 89 | 90 | def transform_matrix(translation: np.ndarray = np.array([0, 0, 0]), 91 | rotation: Quaternion = Quaternion([1, 0, 0, 0]), 92 | inverse: bool = False) -> np.ndarray: 93 | """ 94 | Convert pose to transformation matrix. 95 | :param translation: . Translation in x, y, z. 96 | :param rotation: Rotation in quaternions (w ri rj rk). 97 | :param inverse: Whether to compute inverse transform matrix. 98 | :return: . Transformation matrix. 99 | """ 100 | tm = np.eye(4) 101 | 102 | if inverse: 103 | rot_inv = rotation.rotation_matrix.T 104 | trans = np.transpose(-np.array(translation)) 105 | tm[:3, :3] = rot_inv 106 | tm[:3, 3] = rot_inv.dot(trans) 107 | else: 108 | tm[:3, :3] = rotation.rotation_matrix 109 | tm[:3, 3] = np.transpose(np.array(translation)) 110 | 111 | return tm 112 | 113 | 114 | def points_in_box(box, points: np.ndarray, wlh_factor: float = 1.0): 115 | """ 116 | Checks whether points are inside the box. 117 | 118 | Picks one corner as reference (p1) and computes the vector to a target point (v). 119 | Then for each of the 3 axes, project v onto the axis and compare the length. 120 | Inspired by: https://math.stackexchange.com/a/1552579 121 | :param box: . 122 | :param points: . 123 | :param wlh_factor: Inflates or deflates the box. 124 | :return: . 125 | """ 126 | corners = box.corners(wlh_factor=wlh_factor) 127 | 128 | p1 = corners[:, 0] 129 | p_x = corners[:, 4] 130 | p_y = corners[:, 1] 131 | p_z = corners[:, 3] 132 | 133 | i = p_x - p1 134 | j = p_y - p1 135 | k = p_z - p1 136 | 137 | v = points - p1.reshape((-1, 1)) 138 | 139 | iv = np.dot(i, v) 140 | jv = np.dot(j, v) 141 | kv = np.dot(k, v) 142 | 143 | mask_x = np.logical_and(0 <= iv, iv <= np.dot(i, i)) 144 | mask_y = np.logical_and(0 <= jv, jv <= np.dot(j, j)) 145 | mask_z = np.logical_and(0 <= kv, kv <= np.dot(k, k)) 146 | mask = np.logical_and(np.logical_and(mask_x, mask_y), mask_z) 147 | 148 | return mask 149 | --------------------------------------------------------------------------------