├── .github └── FUNDING.yml ├── LICENSE.txt ├── README.md ├── docs ├── challenge_china3dv.md ├── dataset_stats.md ├── getting_started.md └── toturial.ipynb ├── resources ├── occ_sample_1.jpeg ├── occ_sample_2.jpeg ├── occ_sample_3.jpeg ├── occ_video.gif ├── sample_depth.jpeg └── sample_occ.jpeg └── tools └── ray_iou ├── __init__.py ├── ego_pose_extractor.py ├── lib └── dvr │ ├── dvr.cpp │ └── dvr.cu ├── metric.py └── ray_casting.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [OpenDriveLab] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: # Replace with a single Buy Me a Coffee username 14 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | # LightwheelOcc 4 | **A 3D Occupancy Synthetic Dataset in Autonomous Driving** 5 | 6 | [![LightwheelOcc](https://img.shields.io/badge/LightwheelOcc-v1.0-yellow)](https://lightwheel.ai/) 7 | [![LightwheelAI](https://img.shields.io/badge/Lightwheel_AI-Website-blueviolet)](https://lightwheel.ai/) 8 | 9 | 10 | 11 |
12 | 13 | > - Point of Contact: [Lightwheel AI](mailto:contact@lightwheel.ai) or [Tianyu (李天羽)](mailto:litianyu@pjlab.org.cn) 14 | 15 | ## News 16 | > :fire: China3DV Occupancy and Flow Challenge is ongoing! Check challenge [doc](/docs/challenge_china3dv.md)! 17 | 18 | - **`2024/03/20`** LightwheelOcc dataset `v1.0` released. 19 | - **`2024/03/01`** We are hosting CVPR 2024 Autonomous Grand Challenge and China3DV AD Challenge! 20 | 21 | ## Table of Contents 22 | - [Introduction](#introduction) 23 | - [Highlights](#highlights) 24 | - [Data Overview](#data-overview) 25 | - [Basic Information](#basic-information) 26 | - [Data Sample](#data-sample) 27 | - [Getting Started](#getting-started) 28 | - [Download Data](#download-data) 29 | - [Prepare Dataset](#prepare-dataset) 30 | - [License and Citation](#license-and-citation) 31 | - [Related Resources](#related-resources) 32 | 33 | 34 | ## Introduction 35 | - LightwheelOcc, developed by Lightwheel AI, is a publicly available autonomous driving synthetic dataset. The dataset, which includes 40,000 frames and corresponding ground truth labels for a variety of tasks, is a generalized dataset that navigates a variety of regional terrains, weather patterns, vehicle types, vegetation, and roadway demarcations. 36 | - Lightwheel AI levers generative AI and simulation to deliver 3D, physically realistic and generalizable synthetic data solutions for autonomous driving and embodied AI. By publishing LightwheelOcc, we aim to advance research in the realms of computer vision, autonomous driving and synthetic data. 37 | 38 | ## Highlights 39 | - **Diverse data distributions, including corner cases and hard scenarios** 40 | 41 | - By incorporating complex traffic flows, LightwheelOcc contains diversified simulation of different traffic conditions and driving behaviors. Apart from usual scenarios, the dataset also presents corner cases like small and rare objects on the road, challenging conditions like nighttime and rainy scenes, etc. , enriching real-world data diversity. 42 | 43 |
44 | occ_sample_1 45 | occ_sample_2 46 | occ_sample_3 47 |
48 | 49 | - **Accurate and dense 3D occupancy and depth label** 50 | 51 | - **Realistic sensor configuration simulating nuScenes dataset** 52 | 53 | 54 | ## Data overview 55 | ### Basic Information 56 | - The LightwheelOcc dataset contains 40,000 frames, totaling 240,000 images, of which 28,000 frames are used for training scenarios, 6000 frames are used for validation scenarios, and 6000 frames are used for testing scenarios. 57 | - LightwheelOcc includes 6 camera sensor data, as well as labels for different tasks, including 3D Occupancy, Flow and Depth Map. 58 | 59 | ### Data Sample 60 | | **3D Occupancy** | **Depth Map** | 61 | |---------------------|--------------------------| 62 | | 3D Occupancy | Depth Map | 63 | 64 |

(back to top)

65 | 66 | ## Getting Started 67 | - [Download Data](/docs/getting_started.md#download-data) 68 | - [Prepare Dataset](/docs/getting_started.md#prepare-dataset) 69 | 70 |

(back to top)

71 | 72 | ## License and Citation 73 | 74 | The LightwheelOcc dataset is under [CC BY-NC-ND 4.0](https://creativecommons.org/licenses/by-nc-nd/4.0/) license. 75 | All assets and code within this repository are under the [Apache 2.0](/LICENSE.txt) license unless specified otherwise. 76 | 77 | If this work is helpful for your research, please consider citing the following BibTeX entry. 78 | ``` 79 | @misc{lightwheel2024, 80 | title={LightwheelOcc: A 3D Occupancy Synthetic Dataset in Autonomous Driving} , 81 | author={LightwheelAI and LightwheelOcc contributors}, 82 | howpublished={\url{https://github.com/OpenDriveLab/LightwheelOcc}}, 83 | year={2024} 84 | } 85 | ``` 86 | 87 |

(back to top)

88 | 89 | ## Related Resources 90 | - [DriveAGI](https://github.com/OpenDriveLab/DriveAGI) 91 | - [OccNet](https://github.com/OpenDriveLab/OccNet) | [OpenScene](https://github.com/OpenDriveLab/OpenScene) 92 | 93 |

(back to top)

94 | -------------------------------------------------------------------------------- /docs/challenge_china3dv.md: -------------------------------------------------------------------------------- 1 | # China3DV - 占据栅格与运动估计 2 | > 官方网站: :globe_with_meridians: [China3DV](http://www.csig3dv.net/2024/competition.html) 3 | > 评测服务器: :hugs: [Hugging Face](https://huggingface.co/spaces/China3DV-S/occupancy-and-flow-2024) 4 | 5 | ## 赛道介绍 6 | 7 | 三维框往往不足以描述一般物体，受机器人学概念的启发，可将感知表征描述成对栅格化三维空间的占据情况预测。在纯视觉环视相机输入下，参赛者不仅要给出三维空间的栅格化表示，还须给出栅格的运动预测。 8 | 9 | 本赛道在国际知名数据集 nuScenes 的基础上，引入了行业领先的[光轮智能](http://lightwheel.ai/)自动驾驶模拟器生成的高质量仿真数据。光轮占据栅格仿真数据集（LightwheelOcc）高度重现 nuScenes 的真实传感器布局，提供极度拟真的传感器数据和准确的三维占据与密集的深度图标注，并补充了和 nuScenes 数据集等量的长尾自动驾驶场景作为训练、验证和测试集。 10 | 11 | 本赛道的评测基于 nuScenes OpenOcc 测试集与 LightwheelOcc 测试集，参赛者需要在真实、仿真数据集上同时预测占据栅格与运动估计结果。 12 | 13 | ## 数据集下载 14 | 15 | ### nuScenes OpenOcc 数据集 16 | 17 | nuScenes 数据集下载请详见 [nuScenes 官方主页](https://www.nuscenes.org/nuscenes)。 18 | 19 | nuScenes 占据栅格与运动估计训练标签 OpenOcc 下载请详见 [文档](https://github.com/OpenDriveLab/OccNet?tab=readme-ov-file#data)。 20 | 21 | ### LightwheelOcc 光轮占据栅格仿真数据集 22 | 23 | 请参考本仓库 [Getting Started](/docs/getting_started.md)。 24 | 25 | ## 评测指标 26 | 27 | 本赛道使用指标**占据分数**。该指标包含两部分，使用基于射线投影的 **Ray-based mIoU** 进行占据栅格几何和语义的评测，使用平均速度误差 **mAVE** 进行运动估计的评测。详情请见 [RayIoU 指标文档](https://github.com/OpenDriveLab/OccNet/tree/challenge?tab=readme-ov-file#evaluation-metrics)。 28 | 29 | 本赛道最终占据分数为 nuScenes OpenOcc test 集与 LightwheelOcc test 集上的加权分数。两数据集的加权系数分别为 0.8 和 0.2。 30 | 31 | ## 提交指南 32 | 33 | 参赛者需要按以下步骤将占据栅格预测的结果保存在 `submission.gz` 中。 34 | 35 | 1. 将 `nuScenes OpenOcc val` 与 `Lightwheel val` 上的预测结果保存至本地，格式与占据网络 ground truth 相同。 36 | 2. 在本地进行光线投影，保存投影结果。 37 | 3. 在本地测试 `nuScenes OpenOcc val` 与 `LightwheelOcc val` 的评测是否符合预期。 38 | 4. 将 `nuScenes OpenOcc test` 集与 `Lightwheel test` 的预测结果按 1、 2 两步保存、投影，并上传至竞赛服务器。 39 | 40 | > 光线投影脚本请参照 [ray_casting.py](/tools/ray_iou/ray_casting.py)。 41 | > 生成 `LightwheelOcc val` GT 的命令如下。可以通过修改 `--data-root` 参数来生成预测结果的 `.gz` 文件。 42 | ``` bash 43 | cd tools/ray_iou 44 | python ray_casting.py \ 45 | --dataset-type lightwheelocc \ 46 | --data-root ../../data/lightwheelocc \ 47 | --data-info ../../data/lightwheelocc/lightwheel_occ_infos_val.pkl \ 48 | --output-dir ./output 49 | ``` 50 | > 生成 `nuScenes OpenOcc val` GT 的命令如下。 51 | ``` bash 52 | python ray_casting.py \ 53 | --dataset-type openocc_v2 \ 54 | --data-root ../../data/nuscenes \ 55 | --data-info ../../data/nuscenes/nuscenes_infos_val_occ.pkl \ 56 | --output-dir ./output 57 | ``` 58 | > 对于预测结果，请分别生成两个数据集的预测结果并手动将 `submission['results']` 字典合并。 59 | 60 | 61 | 最终保存的文件结构为： 62 | 63 | ``` 64 | submission = { 65 | 'method': '', -- name of the method 66 | 'team': '', -- name of the team, identical to the Google Form 67 | 'authors': [''] -- list of str, authors 68 | 'e-mail': '', -- e-mail address 69 | 'institution / company': '', -- institution or company 70 | 'country / region': 'zh-CN', -- country or region, checked by iso3166* 71 | 'results': { 72 | [token]: { -- frame (sample) token 73 | 'pcd_cls' [N] -- predicted class ID, np.uint8, 74 | 'pcd_dist' [N] -- predicted depth, np.float16, 75 | 'pcd_flow' [N, 2] -- predicted flow, np.float16, 76 | }, 77 | ... 78 | } 79 | } 80 | ``` 81 | -------------------------------------------------------------------------------- /docs/dataset_stats.md: -------------------------------------------------------------------------------- 1 | # Data Statistics 2 | 3 | ## Basic Information 4 | | **Type** | **Info** | 5 | |-------------------|-------------------------------------| 6 | | Train | 28000 frames | 7 | | Validation | 6000 frames | 8 | | Test | 6000 frames | 9 | | Number of Cameras | 6 | 10 | | Number of Images | 240,000 | 11 | | Image resolution | 1600x900 | 12 | | Frequency | 10 Hz | 13 | | Voxel Size | 0.4 m | 14 | | Range | [-40, -40, -1.0, 40, 40, 5.4] m | 15 | | Classes | 17 | 16 | | Labels | 3D Occupancy, Flow, and Depth Map | 17 | 18 | ### Semantic Classes 19 | 20 | The dataset contains 17 classes. The definition of most of the classes is the same as the [nuScenes-lidarseg](https://github.com/nutonomy/nuscenes-devkit/blob/fcc41628d41060b3c1a86928751e5a571d2fc2fa/python-sdk/nuscenes/eval/lidarseg/README.md#classes) dataset. 21 | The `free` category represents voxels that are not occupied by anything. Voxel semantics for each sample frame is given as `[semantics]` in the `.npz` file. 22 | 23 | | Index | Class Name | 24 | |-------|----------------------| 25 | | 0 | car | 26 | | 1 | truck | 27 | | 2 | trailer | 28 | | 3 | bus | 29 | | 4 | construction_vehicle | 30 | | 5 | bicycle | 31 | | 6 | motorcycle | 32 | | 7 | pedestrian | 33 | | 8 | traffic_cone | 34 | | 9 | barrier | 35 | | 10 | driveable_surface | 36 | | 11 | other_flat | 37 | | 12 | sidewalk | 38 | | 13 | terrain | 39 | | 14 | manmade | 40 | | 15 | vegetation | 41 | | 16 | free | 42 | 43 | ## Data Distribution 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 |
CategoryClassDistribution
By Map TypeUrban68%
Suburbs9%
Freeways23%
By Lighting ConditionsDaytime87%
Nighttime13%
By Weather TypesSunny45%
Overcast38%
Rainy or After Rain17%
86 | 87 | ## Experiment Results 88 | We conduct experiments to evaluate the domain adaption capbilities between nuScene and LightwheelOcc. 89 | We use [OccNet](https://github.com/OpenDriveLab/OccNet) as a baseline. 90 | 91 | | **Train** | **Val** | **mIoU** | 92 | |---------------------------|---------------------------|----------------| 93 | | nuScenes | nuScenes + LightwheelOcc | 15.61 | 94 | | nuScenes + LightwheelOcc | nuScenes + LightwheelOcc | 28.77 | 95 | | nuScenes | nuScenes | 24.84 | 96 | | nuScenes + LightwheelOcc | nuScenes | 26.97 | 97 | 98 | 99 | ## Best Practice 100 | Generally, when training data, it is critical to check three key indicators, so as to determine the training method based on their performance: 101 | 1. Whether the labels of real data and synthetic data are aligned: label alignment of synthetic data usually refers to whether the labels are consistent with those in the real-world dataset. If not aligned, the model may not be able to distinguish each category correctly. 102 | 2. Whether the number of real data and synthetic data is comparable: The volume of data determines the adequacy of model training. Insufficient data volume may lead to overfitting, while a large amount of data can improve the model's generalization ability. 103 | 3. Whether synthetic data is corner case or generic data: The data type (such as corner case or generic data) affects the model's generalization ability and sensitivity to specific situations. 104 | 105 | The above issues must be comprehensively considered to ensure the model performance, when selecting training methods and data sampling strategies. 106 | As for LightwheelOcc, the labels have been aligned, and the amount of real data is greater than that of the synthesized data, aiming at solving the corner case problem. Therefore, we recommend using a hybrid training approach, which involves mixing the synthesized and real data for direct training. 107 | -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | ## Download Data 3 | 4 | The dataset files can be downloaded via [OpenDriveLab](https://openxlab.org.cn/datasets/OpenDriveLab/LightwheelOcc) and :hugs: [Hugging Face](https://huggingface.co/datasets/OpenDriveLab/LightwheelOcc/tree/main/lightwheelocc-v1.0). 5 | 6 | For now, the **depth** label is only available on Hugging Face! 7 | 8 | ## Prepare Dataset 9 | 10 | After download all the files and put them into `lidatwheelocc/`. Please uncompress all the `.tar.gz` files to set up the dataset! 11 | 12 | ### Hierarchy 13 | 14 | The hierarchy of folder `lightwheelocc/` is described below: 15 | 16 | ``` 17 | lightwheelocc 18 | ├── samples 19 | │ ├── CAM_FRONT 20 | │ │ ├── [scene_token] 21 | │ │ │ ├── [timestamp].jpeg 22 | │ │ │ └── ... 23 | │ │ └── ... 24 | │ └── ... 25 | ├── depth 26 | │ ├── CAM_FRONT 27 | │ │ ├── [scene_token] 28 | │ │ │ ├── [timestamp].png 29 | │ │ │ └── ... 30 | │ │ └── ... 31 | │ └── ... 32 | ├── occupancy 33 | │ ├── [scene_token] 34 | │ │ │ ├── [timestamp].npz 35 | │ │ │ └── ... 36 | │ │ └── ... 37 | │ └── ... 38 | ├── lightwheel_occ_infos_train.pkl 39 | ├── lightwheel_occ_infos_val.pkl 40 | └── lightwheel_occ_infos_test.pkl 41 | ``` 42 | 43 | - `[scene_token]` specifies a sequence of frames, and `[timestamp]` specifies a single frame in a sequence. 44 | - `samples/` contains images captured by various cameras. 45 | - `depth/` contains depth map of each sample. 46 | - `occupancy/` contains semantics and flow label for each frame. 47 | - `lightwheel_occ_infos_{train/val/test}.pkl` contains metadata of the dataset. 48 | 49 | ### Meta Data 50 | 51 | The pickle files contain metadata of the dataset. 52 | Each pickle file is formatted as follows: 53 | 54 | ``` 55 | { 56 | "metadata": { -- meta infos of dataset. 57 | "version": -- version of lightwheelocc dataset. 58 | "split": -- split, {train/val/test}. 59 | } 60 | "infos" [ -- meta infos of the scenes 61 | "token": -- token of current frame, unique by sample 62 | "prev": -- frame token of the previous keyframe in the scene 63 | "next": -- frame token of the next keyframe in the scene 64 | "timestamp": -- timestamp, unique by sample 65 | "ego2global_rotation" [4] -- ego to global coordinate system orientation as quaternion 66 | "ego2global_translation" [3] -- ego to global coordinate system translation 67 | "lidar2ego_rotation" [4] -- lidar to ego coordinate system orientation as quaternion 68 | "lidar2ego_translation" [3] -- lidar to ego coordinate system translation 69 | "cams": { -- meta infos of the camera sensor 70 | [cam_type]: { -- type of cameras. 71 | "cam_path": -- corresponding image file path, *.jpeg 72 | "depth_path": -- corresponding deth file path, *.png 73 | "cam_intrinsic": [3, 3] -- intrinsic camera calibration 74 | "sensor2ego_rotation" [4] -- sensor to ego coordinate system orientation as quaternion 75 | "sensor2ego_translation" [3] -- sensor to ego coordinate system translation 76 | "sensor2lidar_rotation" [4] -- sensor to lidar coordinate system orientation as quaternion 77 | "sensor2lidar_translation" [3] -- sensor to lidar coordinate system translation 78 | }, 79 | ... 80 | }, 81 | "occ_path": -- corresponding 3D voxel gt path, *.npz 82 | } 83 | } 84 | ``` 85 | 86 | - The filepath is the relative path to `lightwheelocc`. 87 | - The occupancy label is in `ego` coordinate system. 88 | - The `ego` and `lidar` coordinate system is the same. We keep those keys for better compatibility. 89 | - You can refer to [toturial.ipynb](toturial.ipynb) for the using of depth and occupancy label. 90 | -------------------------------------------------------------------------------- /resources/occ_sample_1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/resources/occ_sample_1.jpeg -------------------------------------------------------------------------------- /resources/occ_sample_2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/resources/occ_sample_2.jpeg -------------------------------------------------------------------------------- /resources/occ_sample_3.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/resources/occ_sample_3.jpeg -------------------------------------------------------------------------------- /resources/occ_video.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/resources/occ_video.gif -------------------------------------------------------------------------------- /resources/sample_depth.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/resources/sample_depth.jpeg -------------------------------------------------------------------------------- /resources/sample_occ.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/resources/sample_occ.jpeg -------------------------------------------------------------------------------- /tools/ray_iou/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenDriveLab/LightwheelOcc/438881b2c7e5e68edbdf79296c2fb1448ea2b162/tools/ray_iou/__init__.py -------------------------------------------------------------------------------- /tools/ray_iou/ego_pose_extractor.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | from pyquaternion import Quaternion 4 | import torch 5 | from torch.utils.data import Dataset 6 | np.set_printoptions(precision=3, suppress=True) 7 | 8 | def trans_matrix(T, R): 9 | tm = np.eye(4) 10 | tm[:3, :3] = R.rotation_matrix 11 | tm[:3, 3] = T 12 | return tm 13 | 14 | class EgoPoseDataset(Dataset): 15 | def __init__(self, data_infos, dataset_type=None): 16 | super(EgoPoseDataset, self).__init__() 17 | 18 | self.data_infos = data_infos 19 | assert dataset_type in ['openocc_v2', 'lightwheelocc'] 20 | self.dataset_type = dataset_type 21 | 22 | if self.dataset_type == 'lightwheelocc': 23 | # lightwheelocc doesn't have lidar now, we use pseudo lidar2ego instead 24 | self.pseudo_lidar2ego = np.array([ 25 | [ 0., 1., 0., 0.94 ], 26 | [-1., 0., 0., 0. ], 27 | [ 0., 0., 1., 1.84 ], 28 | [ 0., 0., 0., 1. ]]) 29 | 30 | self.scene_frames = {} 31 | for info in data_infos: 32 | scene_token = self.get_scene_token(info) 33 | if scene_token not in self.scene_frames: 34 | self.scene_frames[scene_token] = [] 35 | self.scene_frames[scene_token].append(info) 36 | 37 | def __len__(self): 38 | return len(self.data_infos) 39 | 40 | def get_scene_token(self, info): 41 | if self.dataset_type == 'openocc_v2': 42 | # meta info of openocc_v2 don't have scene_token 43 | # extract scene name from 'occ_path' instead 44 | # if the custom data info contains scene_token, we just use it. 45 | if 'scene_token' in info: 46 | scene_name = info['scene_token'] 47 | else: 48 | scene_name = info['occ_path'].split('openocc_v2/')[-1].split('/')[0] 49 | return scene_name 50 | elif self.dataset_type == 'lightwheelocc': 51 | return info['scene_token'] 52 | else: 53 | raise ValueError('Invalid dataset type') 54 | 55 | def get_ego_from_lidar(self, info): 56 | if self.dataset_type == 'openocc_v2': 57 | ego_from_lidar = trans_matrix( 58 | np.array(info['lidar2ego_translation']), 59 | Quaternion(info['lidar2ego_rotation'])) 60 | elif self.dataset_type == 'lightwheelocc': 61 | # lightwheelocc doesn't have lidar2ego, use pseudo lidar2ego instead 62 | ego_from_lidar = self.pseudo_lidar2ego 63 | return ego_from_lidar 64 | 65 | def get_global_pose(self, info, inverse=False): 66 | 67 | global_from_ego = trans_matrix( 68 | np.array(info['ego2global_translation']), 69 | Quaternion(info['ego2global_rotation'])) 70 | if self.dataset_type == 'openocc_v2': 71 | ego_from_lidar = trans_matrix( 72 | np.array(info['lidar2ego_translation']), 73 | Quaternion(info['lidar2ego_rotation'])) 74 | elif self.dataset_type == 'lightwheelocc': 75 | # lightwheelocc doesn't have lidar2ego, use pseudo lidar2ego instead 76 | ego_from_lidar = self.pseudo_lidar2ego 77 | 78 | pose = global_from_ego.dot(ego_from_lidar) 79 | if inverse: 80 | pose = np.linalg.inv(pose) 81 | return pose 82 | 83 | def __getitem__(self, idx): 84 | info = self.data_infos[idx] 85 | 86 | ref_sample_token = info['token'] 87 | ref_lidar_from_global = self.get_global_pose(info, inverse=True) 88 | ref_ego_from_lidar = self.get_ego_from_lidar(info) 89 | 90 | scene_token = self.get_scene_token(info) 91 | scene_frame = self.scene_frames[scene_token] 92 | ref_index = scene_frame.index(info) 93 | 94 | # NOTE: getting output frames 95 | output_origin_list = [] 96 | for curr_index in range(len(scene_frame)): 97 | # if this exists a valid target 98 | if curr_index == ref_index: 99 | origin_tf = np.array([0.0, 0.0, 0.0], dtype=np.float32) 100 | else: 101 | # transform from the current lidar frame to global and then to the reference lidar frame 102 | global_from_curr = self.get_global_pose(scene_frame[curr_index], inverse=False) 103 | ref_from_curr = ref_lidar_from_global.dot(global_from_curr) 104 | origin_tf = np.array(ref_from_curr[:3, 3], dtype=np.float32) 105 | 106 | origin_tf_pad = np.ones([4]) 107 | origin_tf_pad[:3] = origin_tf # pad to [4] 108 | origin_tf = np.dot(ref_ego_from_lidar[:3], origin_tf_pad.T).T # [3] 109 | 110 | # origin 111 | if np.abs(origin_tf[0]) < 39 and np.abs(origin_tf[1]) < 39: 112 | output_origin_list.append(origin_tf) 113 | 114 | # select 8 origins 115 | if len(output_origin_list) > 8: 116 | select_idx = np.round(np.linspace(0, len(output_origin_list) - 1, 8)).astype(np.int64) 117 | output_origin_list = [output_origin_list[i] for i in select_idx] 118 | 119 | output_origin_tensor = torch.from_numpy(np.stack(output_origin_list)) # [T, 3] 120 | 121 | return (ref_sample_token, output_origin_tensor) 122 | -------------------------------------------------------------------------------- /tools/ray_iou/lib/dvr/dvr.cpp: -------------------------------------------------------------------------------- 1 | // Acknowledgments: https://github.com/tarashakhurana/4d-occ-forecasting 2 | // Modified by Haisong Liu 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | /* 9 | * CUDA forward declarations 10 | */ 11 | 12 | std::vector render_forward_cuda(torch::Tensor sigma, 13 | torch::Tensor origin, 14 | torch::Tensor points, 15 | torch::Tensor tindex, 16 | const std::vector grid, 17 | std::string phase_name); 18 | 19 | std::vector 20 | render_cuda(torch::Tensor sigma, torch::Tensor origin, torch::Tensor points, 21 | torch::Tensor tindex, std::string loss_name); 22 | 23 | torch::Tensor init_cuda(torch::Tensor points, torch::Tensor tindex, 24 | const std::vector grid); 25 | 26 | 27 | /* 28 | * C++ interface 29 | */ 30 | 31 | #define CHECK_CUDA(x) \ 32 | TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") 33 | #define CHECK_CONTIGUOUS(x) \ 34 | TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") 35 | #define CHECK_INPUT(x) \ 36 | CHECK_CUDA(x); \ 37 | CHECK_CONTIGUOUS(x) 38 | 39 | std::vector 40 | render_forward(torch::Tensor sigma, torch::Tensor origin, torch::Tensor points, 41 | torch::Tensor tindex, const std::vector grid, 42 | std::string phase_name) { 43 | CHECK_INPUT(sigma); 44 | CHECK_INPUT(origin); 45 | CHECK_INPUT(points); 46 | CHECK_INPUT(tindex); 47 | return render_forward_cuda(sigma, origin, points, tindex, grid, phase_name); 48 | } 49 | 50 | 51 | std::vector render(torch::Tensor sigma, torch::Tensor origin, 52 | torch::Tensor points, torch::Tensor tindex, 53 | std::string loss_name) { 54 | CHECK_INPUT(sigma); 55 | CHECK_INPUT(origin); 56 | CHECK_INPUT(points); 57 | CHECK_INPUT(tindex); 58 | return render_cuda(sigma, origin, points, tindex, loss_name); 59 | } 60 | 61 | torch::Tensor init(torch::Tensor points, torch::Tensor tindex, 62 | const std::vector grid) { 63 | CHECK_INPUT(points); 64 | CHECK_INPUT(tindex); 65 | return init_cuda(points, tindex, grid); 66 | } 67 | 68 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 69 | m.def("init", &init, "Initialize"); 70 | m.def("render", &render, "Render"); 71 | m.def("render_forward", &render_forward, "Render (forward pass only)"); 72 | } 73 | -------------------------------------------------------------------------------- /tools/ray_iou/lib/dvr/dvr.cu: -------------------------------------------------------------------------------- 1 | // Acknowledgments: https://github.com/tarashakhurana/4d-occ-forecasting 2 | // Modified by Haisong Liu 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #define MAX_D 1446 // 700 + 700 + 45 + 1 13 | #define MAX_STEP 1000 14 | 15 | enum LossType {L1, L2, ABSREL}; 16 | enum PhaseName {TEST, TRAIN}; 17 | 18 | template 19 | __global__ void init_cuda_kernel( 20 | const torch::PackedTensorAccessor32 points, 21 | const torch::PackedTensorAccessor32 tindex, 22 | torch::PackedTensorAccessor32 occupancy) { 23 | 24 | // batch index 25 | const auto n = blockIdx.y; 26 | 27 | // ray index 28 | const auto c = blockIdx.x * blockDim.x + threadIdx.x; 29 | 30 | // num of rays 31 | const auto M = points.size(1); 32 | const auto T = occupancy.size(1); 33 | 34 | // we allocated more threads than num_rays 35 | if (c < M) { 36 | // ray end point 37 | const auto t = tindex[n][c]; 38 | 39 | // invalid points 40 | assert(T == 1 || t < T); 41 | 42 | // if t < 0, it is a padded point 43 | if (t < 0) return; 44 | 45 | // time index for sigma 46 | // when T = 1, we have a static sigma 47 | const auto ts = (T == 1) ? 0 : t; 48 | 49 | // grid shape 50 | const int vzsize = occupancy.size(2); 51 | const int vysize = occupancy.size(3); 52 | const int vxsize = occupancy.size(4); 53 | // assert(vzsize + vysize + vxsize <= MAX_D); 54 | 55 | // end point 56 | const int vx = int(points[n][c][0]); 57 | const int vy = int(points[n][c][1]); 58 | const int vz = int(points[n][c][2]); 59 | 60 | // 61 | if (0 <= vx && vx < vxsize && 62 | 0 <= vy && vy < vysize && 63 | 0 <= vz && vz < vzsize) { 64 | occupancy[n][ts][vz][vy][vx] = 1; 65 | } 66 | } 67 | } 68 | 69 | template 70 | __global__ void render_forward_cuda_kernel( 71 | const torch::PackedTensorAccessor32 sigma, 72 | const torch::PackedTensorAccessor32 origin, 73 | const torch::PackedTensorAccessor32 points, 74 | const torch::PackedTensorAccessor32 tindex, 75 | // torch::PackedTensorAccessor32 pog, 76 | torch::PackedTensorAccessor32 pred_dist, 77 | torch::PackedTensorAccessor32 gt_dist, 78 | torch::PackedTensorAccessor32 coord_index, 79 | PhaseName train_phase) { 80 | 81 | // batch index 82 | const auto n = blockIdx.y; 83 | 84 | // ray index 85 | const auto c = blockIdx.x * blockDim.x + threadIdx.x; 86 | 87 | // num of rays 88 | const auto M = points.size(1); 89 | const auto T = sigma.size(1); 90 | 91 | // we allocated more threads than num_rays 92 | if (c < M) { 93 | // ray end point 94 | const auto t = tindex[n][c]; 95 | 96 | // invalid points 97 | // assert(t < T); 98 | assert(T == 1 || t < T); 99 | 100 | // time index for sigma 101 | // when T = 1, we have a static sigma 102 | const auto ts = (T == 1) ? 0 : t; 103 | 104 | // if t < 0, it is a padded point 105 | if (t < 0) return; 106 | 107 | // grid shape 108 | const int vzsize = sigma.size(2); 109 | const int vysize = sigma.size(3); 110 | const int vxsize = sigma.size(4); 111 | // assert(vzsize + vysize + vxsize <= MAX_D); 112 | 113 | // origin 114 | const double xo = origin[n][t][0]; 115 | const double yo = origin[n][t][1]; 116 | const double zo = origin[n][t][2]; 117 | 118 | // end point 119 | const double xe = points[n][c][0]; 120 | const double ye = points[n][c][1]; 121 | const double ze = points[n][c][2]; 122 | 123 | // locate the voxel where the origin resides 124 | const int vxo = int(xo); 125 | const int vyo = int(yo); 126 | const int vzo = int(zo); 127 | 128 | const int vxe = int(xe); 129 | const int vye = int(ye); 130 | const int vze = int(ze); 131 | 132 | // NOTE: new 133 | int vx = vxo; 134 | int vy = vyo; 135 | int vz = vzo; 136 | 137 | // origin to end 138 | const double rx = xe - xo; 139 | const double ry = ye - yo; 140 | const double rz = ze - zo; 141 | double gt_d = sqrt(rx * rx + ry * ry + rz * rz); 142 | 143 | // directional vector 144 | const double dx = rx / gt_d; 145 | const double dy = ry / gt_d; 146 | const double dz = rz / gt_d; 147 | 148 | // In which direction the voxel ids are incremented. 149 | const int stepX = (dx >= 0) ? 1 : -1; 150 | const int stepY = (dy >= 0) ? 1 : -1; 151 | const int stepZ = (dz >= 0) ? 1 : -1; 152 | 153 | // Distance along the ray to the next voxel border from the current position (tMaxX, tMaxY, tMaxZ). 154 | const double next_voxel_boundary_x = vx + (stepX < 0 ? 0 : 1); 155 | const double next_voxel_boundary_y = vy + (stepY < 0 ? 0 : 1); 156 | const double next_voxel_boundary_z = vz + (stepZ < 0 ? 0 : 1); 157 | 158 | // tMaxX, tMaxY, tMaxZ -- distance until next intersection with voxel-border 159 | // the value of t at which the ray crosses the first vertical voxel boundary 160 | double tMaxX = (dx!=0) ? (next_voxel_boundary_x - xo)/dx : DBL_MAX; // 161 | double tMaxY = (dy!=0) ? (next_voxel_boundary_y - yo)/dy : DBL_MAX; // 162 | double tMaxZ = (dz!=0) ? (next_voxel_boundary_z - zo)/dz : DBL_MAX; // 163 | 164 | // tDeltaX, tDeltaY, tDeltaZ -- 165 | // how far along the ray we must move for the horizontal component to equal the width of a voxel 166 | // the direction in which we traverse the grid 167 | // can only be FLT_MAX if we never go in that direction 168 | const double tDeltaX = (dx!=0) ? stepX/dx : DBL_MAX; 169 | const double tDeltaY = (dy!=0) ? stepY/dy : DBL_MAX; 170 | const double tDeltaZ = (dz!=0) ? stepZ/dz : DBL_MAX; 171 | 172 | int3 path[MAX_D]; 173 | double csd[MAX_D]; // cumulative sum of sigma times delta 174 | double p[MAX_D]; // alpha 175 | double d[MAX_D]; 176 | 177 | // forward raymarching with voxel traversal 178 | int step = 0; // total number of voxels traversed 179 | int count = 0; // number of voxels traversed inside the voxel grid 180 | double last_d = 0.0; // correct initialization 181 | 182 | // voxel traversal raycasting 183 | bool was_inside = false; 184 | while (true) { 185 | bool inside = (0 <= vx && vx < vxsize) && 186 | (0 <= vy && vy < vysize) && 187 | (0 <= vz && vz < vzsize); 188 | if (inside) { 189 | was_inside = true; 190 | path[count] = make_int3(vx, vy, vz); 191 | } else if (was_inside) { // was but no longer inside 192 | // we know we are not coming back so terminate 193 | break; 194 | } /*else if (last_d > gt_d) { 195 | break; 196 | } */ 197 | /*else { // has not gone inside yet 198 | // assert(count == 0); 199 | // (1) when we have hit the destination but haven't gone inside the voxel grid 200 | // (2) when we have traveled MAX_D voxels but haven't found one valid voxel 201 | // handle intersection corner cases in case of infinite loop 202 | bool hit = (vx == vxe && vy == vye && vz == vze); // this test seems brittle with corner cases 203 | if (hit || step >= MAX_D) 204 | break; 205 | //if (last_d >= gt_d || step >= MAX_D) break; 206 | } */ 207 | // _d represents the ray distance has traveled before escaping the current voxel cell 208 | double _d = 0.0; 209 | // voxel traversal 210 | if (tMaxX < tMaxY) { 211 | if (tMaxX < tMaxZ) { 212 | _d = tMaxX; 213 | vx += stepX; 214 | tMaxX += tDeltaX; 215 | } else { 216 | _d = tMaxZ; 217 | vz += stepZ; 218 | tMaxZ += tDeltaZ; 219 | } 220 | } else { 221 | if (tMaxY < tMaxZ) { 222 | _d = tMaxY; 223 | vy += stepY; 224 | tMaxY += tDeltaY; 225 | } else { 226 | _d = tMaxZ; 227 | vz += stepZ; 228 | tMaxZ += tDeltaZ; 229 | } 230 | } 231 | if (inside) { 232 | // get sigma at the current voxel 233 | const int3 &v = path[count]; // use the recorded index 234 | const double _sigma = sigma[n][ts][v.z][v.y][v.x]; 235 | const double _delta = max(0.0, _d - last_d); // THIS TURNS OUT IMPORTANT 236 | const double sd = _sigma * _delta; 237 | if (count == 0) { // the first voxel inside 238 | csd[count] = sd; 239 | p[count] = 1 - exp(-sd); 240 | } else { 241 | csd[count] = csd[count-1] + sd; 242 | p[count] = exp(-csd[count-1]) - exp(-csd[count]); 243 | } 244 | // record the traveled distance 245 | d[count] = _d; 246 | // count the number of voxels we have escaped 247 | count ++; 248 | } 249 | last_d = _d; 250 | step ++; 251 | 252 | if (step > MAX_STEP) { 253 | break; 254 | } 255 | } 256 | 257 | // the total number of voxels visited should not exceed this number 258 | assert(count <= MAX_D); 259 | 260 | if (count > 0) { 261 | // compute the expected ray distance 262 | //double exp_d = 0.0; 263 | double exp_d = d[count-1]; 264 | 265 | const int3 &v_init = path[count-1]; 266 | int x = v_init.x; 267 | int y = v_init.y; 268 | int z = v_init.z; 269 | 270 | for (int i = 0; i < count; i++) { 271 | //printf("%f\t%f\n",p[i], d[i]); 272 | //exp_d += p[i] * d[i]; 273 | const int3 &v = path[i]; 274 | const double occ = sigma[n][ts][v.z][v.y][v.x]; 275 | if (occ > 0.5) { 276 | exp_d = d[i]; 277 | 278 | x = v.x; 279 | y = v.y; 280 | z = v.z; 281 | 282 | break; 283 | } 284 | 285 | } 286 | //printf("%f\n",exp_d); 287 | 288 | // add an imaginary sample at the end point should gt_d exceeds max_d 289 | double p_out = exp(-csd[count-1]); 290 | double max_d = d[count-1]; 291 | 292 | // if (gt_d > max_d) 293 | // exp_d += (p_out * gt_d); 294 | 295 | // p_out is the probability the ray escapes the voxel grid 296 | //exp_d += (p_out * max_d); 297 | if (train_phase == 1) { 298 | gt_d = min(gt_d, max_d); 299 | } 300 | 301 | // write the rendered ray distance (max_d) 302 | pred_dist[n][c] = exp_d; 303 | gt_dist[n][c] = gt_d; 304 | 305 | coord_index[n][c][0] = double(x); 306 | coord_index[n][c][1] = double(y); 307 | coord_index[n][c][2] = double(z); 308 | 309 | // // write occupancy 310 | // for (int i = 0; i < count; i ++) { 311 | // const int3 &v = path[i]; 312 | // auto & occ = pog[n][t][v.z][v.y][v.x]; 313 | // if (p[i] >= occ) { 314 | // occ = p[i]; 315 | // } 316 | // } 317 | } 318 | } 319 | } 320 | 321 | /* 322 | * input shape 323 | * sigma : N x T x H x L x W 324 | * origin : N x T x 3 325 | * points : N x M x 4 326 | * output shape 327 | * dist : N x M 328 | */ 329 | std::vector render_forward_cuda( 330 | torch::Tensor sigma, 331 | torch::Tensor origin, 332 | torch::Tensor points, 333 | torch::Tensor tindex, 334 | const std::vector grid, 335 | std::string phase_name) { 336 | 337 | const auto N = points.size(0); // batch size 338 | const auto M = points.size(1); // num of rays 339 | 340 | const auto T = grid[0]; 341 | const auto H = grid[1]; 342 | const auto L = grid[2]; 343 | const auto W = grid[3]; 344 | 345 | const auto device = sigma.device(); 346 | 347 | const int threads = 1024; 348 | const dim3 blocks((M + threads - 1) / threads, N); 349 | 350 | // 351 | // const auto dtype = points.dtype(); 352 | // const auto options = torch::TensorOptions().dtype(dtype).device(device).requires_grad(false); 353 | // auto pog = torch::zeros({N, T, H, L, W}, options); 354 | 355 | // perform rendering 356 | auto gt_dist = -torch::ones({N, M}, device); 357 | auto pred_dist = -torch::ones({N, M}, device); 358 | 359 | auto coord_index = torch::zeros({N, M, 3}, device); 360 | 361 | PhaseName train_phase; 362 | if (phase_name.compare("test") == 0) { 363 | train_phase = TEST; 364 | } else if (phase_name.compare("train") == 0){ 365 | train_phase = TRAIN; 366 | } else { 367 | std::cout << "UNKNOWN PHASE NAME: " << phase_name << std::endl; 368 | exit(1); 369 | } 370 | 371 | AT_DISPATCH_FLOATING_TYPES(sigma.type(), "render_forward_cuda", ([&] { 372 | render_forward_cuda_kernel<<>>( 373 | sigma.packed_accessor32(), 374 | origin.packed_accessor32(), 375 | points.packed_accessor32(), 376 | tindex.packed_accessor32(), 377 | // pog.packed_accessor32(), 378 | pred_dist.packed_accessor32(), 379 | gt_dist.packed_accessor32(), 380 | coord_index.packed_accessor32(), 381 | train_phase); 382 | })); 383 | 384 | cudaDeviceSynchronize(); 385 | 386 | // return {pog, pred_dist, gt_dist}; 387 | return {pred_dist, gt_dist, coord_index}; 388 | } 389 | 390 | template 391 | __global__ void render_cuda_kernel( 392 | const torch::PackedTensorAccessor32 sigma, 393 | const torch::PackedTensorAccessor32 origin, 394 | const torch::PackedTensorAccessor32 points, 395 | const torch::PackedTensorAccessor32 tindex, 396 | // const torch::PackedTensorAccessor32 occupancy, 397 | torch::PackedTensorAccessor32 pred_dist, 398 | torch::PackedTensorAccessor32 gt_dist, 399 | torch::PackedTensorAccessor32 grad_sigma, 400 | // torch::PackedTensorAccessor32 grad_sigma_count, 401 | LossType loss_type) { 402 | 403 | // batch index 404 | const auto n = blockIdx.y; 405 | 406 | // ray index 407 | const auto c = blockIdx.x * blockDim.x + threadIdx.x; 408 | 409 | // num of rays 410 | const auto M = points.size(1); 411 | const auto T = sigma.size(1); 412 | 413 | // we allocated more threads than num_rays 414 | if (c < M) { 415 | // ray end point 416 | const auto t = tindex[n][c]; 417 | 418 | // invalid points 419 | // assert(t < T); 420 | assert(T == 1 || t < T); 421 | 422 | // time index for sigma 423 | // when T = 1, we have a static sigma 424 | const auto ts = (T == 1) ? 0 : t; 425 | 426 | // if t < 0, it is a padded point 427 | if (t < 0) return; 428 | 429 | // grid shape 430 | const int vzsize = sigma.size(2); 431 | const int vysize = sigma.size(3); 432 | const int vxsize = sigma.size(4); 433 | // assert(vzsize + vysize + vxsize <= MAX_D); 434 | 435 | // origin 436 | const double xo = origin[n][t][0]; 437 | const double yo = origin[n][t][1]; 438 | const double zo = origin[n][t][2]; 439 | 440 | // end point 441 | const double xe = points[n][c][0]; 442 | const double ye = points[n][c][1]; 443 | const double ze = points[n][c][2]; 444 | 445 | // locate the voxel where the origin resides 446 | const int vxo = int(xo); 447 | const int vyo = int(yo); 448 | const int vzo = int(zo); 449 | 450 | // 451 | const int vxe = int(xe); 452 | const int vye = int(ye); 453 | const int vze = int(ze); 454 | 455 | // NOTE: new 456 | int vx = vxo; 457 | int vy = vyo; 458 | int vz = vzo; 459 | 460 | // origin to end 461 | const double rx = xe - xo; 462 | const double ry = ye - yo; 463 | const double rz = ze - zo; 464 | double gt_d = sqrt(rx * rx + ry * ry + rz * rz); 465 | 466 | // directional vector 467 | const double dx = rx / gt_d; 468 | const double dy = ry / gt_d; 469 | const double dz = rz / gt_d; 470 | 471 | // In which direction the voxel ids are incremented. 472 | const int stepX = (dx >= 0) ? 1 : -1; 473 | const int stepY = (dy >= 0) ? 1 : -1; 474 | const int stepZ = (dz >= 0) ? 1 : -1; 475 | 476 | // Distance along the ray to the next voxel border from the current position (tMaxX, tMaxY, tMaxZ). 477 | const double next_voxel_boundary_x = vx + (stepX < 0 ? 0 : 1); 478 | const double next_voxel_boundary_y = vy + (stepY < 0 ? 0 : 1); 479 | const double next_voxel_boundary_z = vz + (stepZ < 0 ? 0 : 1); 480 | 481 | // tMaxX, tMaxY, tMaxZ -- distance until next intersection with voxel-border 482 | // the value of t at which the ray crosses the first vertical voxel boundary 483 | double tMaxX = (dx!=0) ? (next_voxel_boundary_x - xo)/dx : DBL_MAX; // 484 | double tMaxY = (dy!=0) ? (next_voxel_boundary_y - yo)/dy : DBL_MAX; // 485 | double tMaxZ = (dz!=0) ? (next_voxel_boundary_z - zo)/dz : DBL_MAX; // 486 | 487 | // tDeltaX, tDeltaY, tDeltaZ -- 488 | // how far along the ray we must move for the horizontal component to equal the width of a voxel 489 | // the direction in which we traverse the grid 490 | // can only be FLT_MAX if we never go in that direction 491 | const double tDeltaX = (dx!=0) ? stepX/dx : DBL_MAX; 492 | const double tDeltaY = (dy!=0) ? stepY/dy : DBL_MAX; 493 | const double tDeltaZ = (dz!=0) ? stepZ/dz : DBL_MAX; 494 | 495 | int3 path[MAX_D]; 496 | double csd[MAX_D]; // cumulative sum of sigma times delta 497 | double p[MAX_D]; // alpha 498 | double d[MAX_D]; 499 | double dt[MAX_D]; 500 | 501 | // forward raymarching with voxel traversal 502 | int step = 0; // total number of voxels traversed 503 | int count = 0; // number of voxels traversed inside the voxel grid 504 | double last_d = 0.0; // correct initialization 505 | 506 | // voxel traversal raycasting 507 | bool was_inside = false; 508 | while (true) { 509 | bool inside = (0 <= vx && vx < vxsize) && 510 | (0 <= vy && vy < vysize) && 511 | (0 <= vz && vz < vzsize); 512 | if (inside) { // now inside 513 | was_inside = true; 514 | path[count] = make_int3(vx, vy, vz); 515 | } else if (was_inside) { // was inside but no longer 516 | // we know we are not coming back so terminate 517 | break; 518 | } else if (last_d > gt_d) { 519 | break; 520 | } /* else { // has not gone inside yet 521 | // assert(count == 0); 522 | // (1) when we have hit the destination but haven't gone inside the voxel grid 523 | // (2) when we have traveled MAX_D voxels but haven't found one valid voxel 524 | // handle intersection corner cases in case of infinite loop 525 | // bool hit = (vx == vxe && vy == vye && vz == vze); 526 | // if (hit || step >= MAX_D) 527 | // break; 528 | if (last_d >= gt_d || step >= MAX_D) break; 529 | } */ 530 | // _d represents the ray distance has traveled before escaping the current voxel cell 531 | double _d = 0.0; 532 | // voxel traversal 533 | if (tMaxX < tMaxY) { 534 | if (tMaxX < tMaxZ) { 535 | _d = tMaxX; 536 | vx += stepX; 537 | tMaxX += tDeltaX; 538 | } else { 539 | _d = tMaxZ; 540 | vz += stepZ; 541 | tMaxZ += tDeltaZ; 542 | } 543 | } else { 544 | if (tMaxY < tMaxZ) { 545 | _d = tMaxY; 546 | vy += stepY; 547 | tMaxY += tDeltaY; 548 | } else { 549 | _d = tMaxZ; 550 | vz += stepZ; 551 | tMaxZ += tDeltaZ; 552 | } 553 | } 554 | if (inside) { 555 | // get sigma at the current voxel 556 | const int3 &v = path[count]; // use the recorded index 557 | const double _sigma = sigma[n][ts][v.z][v.y][v.x]; 558 | const double _delta = max(0.0, _d - last_d); // THIS TURNS OUT IMPORTANT 559 | const double sd = _sigma * _delta; 560 | if (count == 0) { // the first voxel inside 561 | csd[count] = sd; 562 | p[count] = 1 - exp(-sd); 563 | } else { 564 | csd[count] = csd[count-1] + sd; 565 | p[count] = exp(-csd[count-1]) - exp(-csd[count]); 566 | } 567 | // record the traveled distance 568 | d[count] = _d; 569 | dt[count] = _delta; 570 | // count the number of voxels we have escaped 571 | count ++; 572 | } 573 | last_d = _d; 574 | step ++; 575 | 576 | if (step > MAX_STEP) { 577 | break; 578 | } 579 | } 580 | 581 | // the total number of voxels visited should not exceed this number 582 | assert(count <= MAX_D); 583 | 584 | // WHEN THERE IS AN INTERSECTION BETWEEN THE RAY AND THE VOXEL GRID 585 | if (count > 0) { 586 | // compute the expected ray distance 587 | double exp_d = 0.0; 588 | for (int i = 0; i < count; i ++) 589 | exp_d += p[i] * d[i]; 590 | 591 | // add an imaginary sample at the end point should gt_d exceeds max_d 592 | double p_out = exp(-csd[count-1]); 593 | double max_d = d[count-1]; 594 | 595 | exp_d += (p_out * max_d); 596 | gt_d = min(gt_d, max_d); 597 | 598 | // write the rendered ray distance (max_d) 599 | pred_dist[n][c] = exp_d; 600 | gt_dist[n][c] = gt_d; 601 | 602 | /* backward raymarching */ 603 | double dd_dsigma[MAX_D]; 604 | for (int i = count - 1; i >= 0; i --) { 605 | // NOTE: probably need to double check again 606 | if (i == count - 1) 607 | dd_dsigma[i] = p_out * max_d; 608 | else 609 | dd_dsigma[i] = dd_dsigma[i+1] - exp(-csd[i]) * (d[i+1] - d[i]); 610 | } 611 | 612 | for (int i = count - 1; i >= 0; i --) 613 | dd_dsigma[i] *= dt[i]; 614 | 615 | // option 2: cap at the boundary 616 | for (int i = count - 1; i >= 0; i --) 617 | dd_dsigma[i] -= dt[i] * p_out * max_d; 618 | 619 | double dl_dd = 1.0; 620 | if (loss_type == L1) 621 | dl_dd = (exp_d >= gt_d) ? 1 : -1; 622 | else if (loss_type == L2) 623 | dl_dd = (exp_d - gt_d); 624 | else if (loss_type == ABSREL) 625 | dl_dd = (exp_d >= gt_d) ? (1.0/gt_d) : -(1.0/gt_d); 626 | 627 | // apply chain rule 628 | for (int i = 0; i < count; i ++) { 629 | const int3 &v = path[i]; 630 | // NOTE: potential race conditions when writing gradients 631 | grad_sigma[n][ts][v.z][v.y][v.x] += dl_dd * dd_dsigma[i]; 632 | // grad_sigma_count[n][ts][v.z][v.y][v.x] += 1; 633 | } 634 | } 635 | } 636 | } 637 | 638 | /* 639 | * input shape 640 | * sigma : N x T x H x L x W 641 | * origin : N x T x 3 642 | * points : N x M x 4 643 | * output shape 644 | * dist : N x M 645 | * loss : N x M 646 | * grad_sigma : N x T x H x L x W 647 | */ 648 | std::vector render_cuda( 649 | torch::Tensor sigma, 650 | torch::Tensor origin, 651 | torch::Tensor points, 652 | torch::Tensor tindex, 653 | std::string loss_name) { 654 | 655 | const auto N = points.size(0); // batch size 656 | const auto M = points.size(1); // num of rays 657 | 658 | const auto device = sigma.device(); 659 | 660 | const int threads = 1024; 661 | const dim3 blocks((M + threads - 1) / threads, N); 662 | 663 | // perform rendering 664 | auto gt_dist = -torch::ones({N, M}, device); 665 | auto pred_dist = -torch::ones({N, M}, device); 666 | auto grad_sigma = torch::zeros_like(sigma); 667 | // auto grad_sigma_count = torch::zeros_like(sigma); 668 | 669 | LossType loss_type; 670 | if (loss_name.compare("l1") == 0) { 671 | loss_type = L1; 672 | } else if (loss_name.compare("l2") == 0) { 673 | loss_type = L2; 674 | } else if (loss_name.compare("absrel") == 0) { 675 | loss_type = ABSREL; 676 | } else if (loss_name.compare("bce") == 0){ 677 | loss_type = L1; 678 | } else { 679 | std::cout << "UNKNOWN LOSS TYPE: " << loss_name << std::endl; 680 | exit(1); 681 | } 682 | 683 | AT_DISPATCH_FLOATING_TYPES(sigma.type(), "render_cuda", ([&] { 684 | render_cuda_kernel<<>>( 685 | sigma.packed_accessor32(), 686 | origin.packed_accessor32(), 687 | points.packed_accessor32(), 688 | tindex.packed_accessor32(), 689 | // occupancy.packed_accessor32(), 690 | pred_dist.packed_accessor32(), 691 | gt_dist.packed_accessor32(), 692 | grad_sigma.packed_accessor32(), 693 | // grad_sigma_count.packed_accessor32(), 694 | loss_type); 695 | })); 696 | 697 | cudaDeviceSynchronize(); 698 | 699 | // grad_sigma_count += (grad_sigma_count == 0); 700 | // grad_sigma /= grad_sigma_count; 701 | 702 | return {pred_dist, gt_dist, grad_sigma}; 703 | } 704 | 705 | 706 | /* 707 | * input shape 708 | * origin : N x T x 3 709 | * points : N x M x 3 710 | * tindex : N x M 711 | * output shape 712 | * occupancy: N x T x H x L x W 713 | */ 714 | torch::Tensor init_cuda( 715 | torch::Tensor points, 716 | torch::Tensor tindex, 717 | const std::vector grid) { 718 | 719 | const auto N = points.size(0); // batch size 720 | const auto M = points.size(1); // num of rays 721 | 722 | const auto T = grid[0]; 723 | const auto H = grid[1]; 724 | const auto L = grid[2]; 725 | const auto W = grid[3]; 726 | 727 | const auto dtype = points.dtype(); 728 | const auto device = points.device(); 729 | const auto options = torch::TensorOptions().dtype(dtype).device(device).requires_grad(false); 730 | auto occupancy = torch::zeros({N, T, H, L, W}, options); 731 | 732 | const int threads = 1024; 733 | const dim3 blocks((M + threads - 1) / threads, N); 734 | 735 | // initialize occupancy such that every voxel with one or more points is occupied 736 | AT_DISPATCH_FLOATING_TYPES(points.type(), "init_cuda", ([&] { 737 | init_cuda_kernel<<>>( 738 | points.packed_accessor32(), 739 | tindex.packed_accessor32(), 740 | occupancy.packed_accessor32()); 741 | })); 742 | 743 | // synchronize 744 | cudaDeviceSynchronize(); 745 | 746 | return occupancy; 747 | } 748 | -------------------------------------------------------------------------------- /tools/ray_iou/metric.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from tqdm import tqdm 3 | import pickle, gzip 4 | import argparse 5 | 6 | def calc_metrics(pred_cls_list, pred_dist_list, pred_flow_list, gt_cls_list, gt_dist_list, gt_flow_list): 7 | occ_class_names = [ 8 | 'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 9 | 'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone', 'barrier', 10 | 'driveable_surface', 'other_flat', 'sidewalk', 11 | 'terrain', 'manmade', 'vegetation', 'free' 12 | ] 13 | 14 | flow_class_names = [ 15 | 'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 16 | 'bicycle', 'motorcycle', 'pedestrian', 17 | ] 18 | thresholds = [1, 2, 4] 19 | 20 | gt_cnt = np.zeros([len(occ_class_names)]) 21 | pred_cnt = np.zeros([len(occ_class_names)]) 22 | tp_cnt = np.zeros([len(thresholds), len(occ_class_names)]) 23 | 24 | ave = np.zeros([len(thresholds), len(occ_class_names)]) 25 | for i, cls in enumerate(occ_class_names): 26 | if cls not in flow_class_names: 27 | ave[:, i] = np.nan 28 | 29 | ave_count = np.zeros([len(thresholds), len(occ_class_names)]) 30 | 31 | for idx in tqdm(range(len(pred_cls_list))): 32 | for j, threshold in enumerate(thresholds): 33 | pred_cls = pred_cls_list[idx].astype(np.int32) 34 | pred_dist = pred_dist_list[idx].astype(np.float32) 35 | pred_flow = pred_flow_list[idx].astype(np.float32) 36 | 37 | gt_cls = gt_cls_list[idx].astype(np.int32) 38 | gt_dist = gt_dist_list[idx].astype(np.float32) 39 | gt_flow = gt_flow_list[idx].astype(np.float32) 40 | 41 | valid_mask = (gt_cls != len(occ_class_names) - 1) 42 | pred_cls = pred_cls[valid_mask] 43 | pred_dist = pred_dist[valid_mask] 44 | pred_flow = pred_flow[valid_mask] 45 | 46 | gt_cls = gt_cls[valid_mask] 47 | gt_dist = gt_dist[valid_mask] 48 | gt_flow = gt_flow[valid_mask] 49 | 50 | # L1 51 | l1_error = np.abs(pred_dist - gt_dist) 52 | tp_dist_mask = (l1_error < threshold) 53 | 54 | for i, cls in enumerate(occ_class_names): 55 | cls_id = occ_class_names.index(cls) 56 | cls_mask_pred = (pred_cls == cls_id) 57 | cls_mask_gt = (gt_cls == cls_id) 58 | 59 | gt_cnt_i = cls_mask_gt.sum() 60 | pred_cnt_i = cls_mask_pred.sum() 61 | if j == 0: 62 | gt_cnt[i] += gt_cnt_i 63 | pred_cnt[i] += pred_cnt_i 64 | 65 | tp_cls = cls_mask_gt & cls_mask_pred # [N] 66 | tp_mask = np.logical_and(tp_cls, tp_dist_mask) 67 | tp_cnt[j][i] += tp_mask.sum() 68 | 69 | # flow L2 error 70 | if cls in flow_class_names and tp_mask.sum() > 0: 71 | flow_error = np.linalg.norm(gt_flow - pred_flow, axis=1) 72 | ave[j][i] += np.sum(flow_error) 73 | ave_count[j][i] += flow_error.shape[0] 74 | 75 | iou_list = [] 76 | for j, threshold in enumerate(thresholds): 77 | iou_list.append((tp_cnt[j] / (gt_cnt + pred_cnt - tp_cnt[j]))[:-1]) 78 | 79 | ave_list = ave[1][:-1] / ave_count[1][:-1] # use threshold = 2 80 | 81 | return iou_list, ave_list 82 | 83 | def compute(args): 84 | print("Evaluating...") 85 | 86 | with gzip.open(args.submission, 'rb') as f: 87 | pred_file = pickle.load(f) 88 | 89 | with gzip.open(args.lightwheelocc_gt, 'rb') as f: 90 | light_test_file = pickle.load(f) 91 | 92 | with gzip.open(args.openocc_gt, 'rb') as f: 93 | openocc_test_file = pickle.load(f) 94 | 95 | print("Start to evaluate on nuScenes OpenOcc...") 96 | pred_cls_list = [] 97 | pred_dist_list = [] 98 | pred_flow_list = [] 99 | gt_cls_list = [] 100 | gt_dist_list = [] 101 | gt_flow_list = [] 102 | 103 | for gt_token in tqdm(openocc_test_file['results'].keys()): 104 | 105 | if gt_token in pred_file['results'].keys(): # found 106 | pred_data = pred_file['results'][gt_token] 107 | gt_data = openocc_test_file['results'][gt_token] 108 | 109 | pred_cls_list.append(pred_data['pcd_cls']) 110 | pred_dist_list.append(pred_data['pcd_dist']) 111 | pred_flow_list.append(pred_data['pcd_flow']) 112 | 113 | gt_cls_list.append(gt_data['pcd_cls']) 114 | gt_dist_list.append(gt_data['pcd_dist']) 115 | gt_flow_list.append(gt_data['pcd_flow']) 116 | else: 117 | raise RuntimeError(f'OpenOcc: prediction is not found for token: {gt_token}') 118 | 119 | openocc_iou_list, openocc_ave_list = calc_metrics(pred_cls_list, pred_dist_list, pred_flow_list, gt_cls_list, gt_dist_list, gt_flow_list) 120 | 121 | print("Start to evaluate on LightwheelOcc...") 122 | 123 | pred_cls_list = [] 124 | pred_dist_list = [] 125 | pred_flow_list = [] 126 | gt_cls_list = [] 127 | gt_dist_list = [] 128 | gt_flow_list = [] 129 | 130 | for gt_token in tqdm(light_test_file['results'].keys()): 131 | # find the prediction 132 | if gt_token in pred_file['results'].keys(): # found 133 | pred_data = pred_file['results'][gt_token] 134 | gt_data = light_test_file['results'][gt_token] 135 | 136 | pred_cls_list.append(pred_data['pcd_cls']) 137 | pred_dist_list.append(pred_data['pcd_dist']) 138 | pred_flow_list.append(pred_data['pcd_flow']) 139 | 140 | gt_cls_list.append(gt_data['pcd_cls']) 141 | gt_dist_list.append(gt_data['pcd_dist']) 142 | gt_flow_list.append(gt_data['pcd_flow']) 143 | else: 144 | raise RuntimeError(f'LightwheelOcc: prediction is not found for token: {gt_token}') 145 | 146 | light_iou_list, light_ave_list = calc_metrics(pred_cls_list, pred_dist_list, pred_flow_list, gt_cls_list, gt_dist_list, gt_flow_list) 147 | 148 | openocc_miou = np.nanmean(openocc_iou_list) 149 | openocc_mave = np.nanmean(openocc_ave_list) 150 | openocc_occ_score = openocc_miou * 0.9 + max(1 - openocc_mave, 0.0) * 0.1 151 | 152 | light_miou = np.nanmean(light_iou_list) 153 | light_mave = np.nanmean(light_ave_list) 154 | light_occ_score = light_miou * 0.9 + max(1 - light_mave, 0.0) * 0.1 155 | 156 | # final score 157 | occ_score = openocc_occ_score * 0.8 + light_occ_score * 0.2 158 | 159 | output = { 160 | "OpenOcc_RayIoU": openocc_miou, 161 | "OpenOcc_mAVE": openocc_mave, 162 | "OpenOcc_Occ_Score": openocc_occ_score, 163 | "LwOcc_RayIoU": light_miou, 164 | "LwOcc_mAVE": light_mave, 165 | "LwOcc_Occ_Score": light_occ_score, 166 | "final_Occ_Score": occ_score 167 | } 168 | 169 | evaluation = { 170 | "public_score": output, 171 | "private_score": output 172 | } 173 | 174 | print('End of evaluation.') 175 | return evaluation 176 | 177 | if __name__ == "__main__": 178 | parser = argparse.ArgumentParser() 179 | parser.add_argument("--submission", default='submission.gz', choices=['lightwheelocc', 'openocc_v2']) 180 | parser.add_argument("--openocc-gt", default='nuscenes_infos_val_occ_pcd.gz') 181 | parser.add_argument("--lightwheelocc-gt", default='lightwheel_occ_infos_val_pcd.gz') 182 | args = parser.parse_args() 183 | 184 | compute(args) 185 | -------------------------------------------------------------------------------- /tools/ray_iou/ray_casting.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import copy 4 | import math 5 | import gzip 6 | import pickle 7 | import argparse 8 | 9 | import numpy as np 10 | import cv2 11 | import torch 12 | from torch.utils.cpp_extension import load 13 | from torch.utils.data import DataLoader 14 | from tqdm import tqdm 15 | 16 | from ego_pose_extractor import EgoPoseDataset 17 | 18 | color_map = np.array([ 19 | [0, 150, 245, 255], # car blue 20 | [160, 32, 240, 255], # truck purple 21 | [135, 60, 0, 255], # trailer brown 22 | [255, 255, 0, 255], # bus yellow 23 | [0, 255, 255, 255], # construction_vehicle cyan 24 | [255, 192, 203, 255], # bicycle pink 25 | [200, 180, 0, 255], # motorcycle dark orange 26 | [255, 0, 0, 255], # pedestrian red 27 | [255, 240, 150, 255], # traffic_cone light yellow 28 | [255, 120, 50, 255], # barrier orangey 29 | [255, 0, 255, 255], # driveable_surface dark pink 30 | [175, 0, 75, 255], # other_flat dark red 31 | [75, 0, 75, 255], # sidewalk dard purple 32 | [150, 240, 80, 255], # terrain light green 33 | [230, 230, 250, 255], # manmade white 34 | [0, 175, 0, 255], # vegetation green 35 | [255, 255, 255, 255], # free white 36 | ], dtype=np.uint8) 37 | 38 | occ_class_names = [ 39 | 'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 40 | 'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone', 'barrier', 41 | 'driveable_surface', 'other_flat', 'sidewalk', 42 | 'terrain', 'manmade', 'vegetation', 'free' 43 | ] 44 | 45 | VIZ = False 46 | dvr = load("dvr", sources=["lib/dvr/dvr.cpp", "lib/dvr/dvr.cu"], verbose=True, extra_cuda_cflags=['-allow-unsupported-compiler']) 47 | _pc_range = [-40, -40, -1.0, 40, 40, 5.4] 48 | _voxel_size = 0.4 49 | 50 | 51 | def occ2img(semantics): 52 | H, W, D = semantics.shape 53 | 54 | free_id = len(occ_class_names) - 1 55 | semantics_2d = np.ones([H, W], dtype=np.int32) * free_id 56 | 57 | for i in range(D): 58 | semantics_i = semantics[..., i] 59 | non_free_mask = (semantics_i != free_id) 60 | semantics_2d[non_free_mask] = semantics_i[non_free_mask] 61 | 62 | viz = color_map[semantics_2d] 63 | viz = viz[..., :3] 64 | viz = cv2.resize(viz, dsize=(800, 800)) 65 | 66 | return viz 67 | 68 | 69 | def viz_pcd(pcd, cls): 70 | pcd = copy.deepcopy(pcd.astype(np.float32)) 71 | pcd[..., 0] -= _pc_range[0] 72 | pcd[..., 1] -= _pc_range[1] 73 | pcd[..., 2] -= _pc_range[2] 74 | pcd[..., 0:3] /= _voxel_size 75 | pcd = pcd.astype(np.int32) 76 | pcd[..., 0] = np.clip(pcd[..., 0], a_min=0, a_max=200-1) 77 | pcd[..., 1] = np.clip(pcd[..., 1], a_min=0, a_max=200-1) 78 | pcd[..., 2] = np.clip(pcd[..., 2], a_min=0, a_max=16-1) 79 | 80 | free_id = len(occ_class_names) - 1 81 | pcd_dense = np.ones([200, 200, 16], dtype=np.int32) * free_id 82 | 83 | pcd_dense[pcd[..., 0], pcd[..., 1], pcd[..., 2]] = cls.astype(np.int32) 84 | 85 | return occ2img(pcd_dense) 86 | 87 | 88 | # https://github.com/tarashakhurana/4d-occ-forecasting/blob/ff986082cd6ea10e67ab7839bf0e654736b3f4e2/test_fgbg.py#L29C1-L46C16 89 | def get_rendered_pcds(origin, points, tindex, pred_dist): 90 | pcds = [] 91 | for t in range(len(origin)): 92 | mask = (tindex == t) 93 | # skip the ones with no data 94 | if not mask.any(): 95 | continue 96 | _pts = points[mask, :3] 97 | # use ground truth lidar points for the raycasting direction 98 | v = _pts - origin[t][None, :] 99 | d = v / np.sqrt((v ** 2).sum(axis=1, keepdims=True)) 100 | pred_pts = origin[t][None, :] + d * pred_dist[mask][:, None] 101 | pcds.append(torch.from_numpy(pred_pts)) 102 | return pcds 103 | 104 | 105 | def meshgrid3d(occ_size, pc_range): 106 | W, H, D = occ_size 107 | 108 | xs = torch.linspace(0.5, W - 0.5, W).view(W, 1, 1).expand(W, H, D) / W 109 | ys = torch.linspace(0.5, H - 0.5, H).view(1, H, 1).expand(W, H, D) / H 110 | zs = torch.linspace(0.5, D - 0.5, D).view(1, 1, D).expand(W, H, D) / D 111 | xs = xs * (pc_range[3] - pc_range[0]) + pc_range[0] 112 | ys = ys * (pc_range[4] - pc_range[1]) + pc_range[1] 113 | zs = zs * (pc_range[5] - pc_range[2]) + pc_range[2] 114 | xyz = torch.stack((xs, ys, zs), -1) 115 | 116 | return xyz 117 | 118 | 119 | def generate_lidar_rays(): 120 | # prepare lidar ray angles 121 | pitch_angles = [] 122 | for k in range(10): 123 | angle = math.pi / 2 - math.atan(k + 1) 124 | pitch_angles.append(-angle) 125 | 126 | # nuscenes lidar fov: [0.2107773983152201, -0.5439104895672159] (rad) 127 | while pitch_angles[-1] < 0.21: 128 | delta = pitch_angles[-1] - pitch_angles[-2] 129 | pitch_angles.append(pitch_angles[-1] + delta) 130 | 131 | lidar_rays = [] 132 | for pitch_angle in pitch_angles: 133 | for azimuth_angle in np.arange(0, 360, 1): 134 | azimuth_angle = np.deg2rad(azimuth_angle) 135 | 136 | x = np.cos(pitch_angle) * np.cos(azimuth_angle) 137 | y = np.cos(pitch_angle) * np.sin(azimuth_angle) 138 | z = np.sin(pitch_angle) 139 | 140 | lidar_rays.append((x, y, z)) 141 | 142 | return np.array(lidar_rays, dtype=np.float32) 143 | 144 | 145 | def process_one_sample(sem_pred, lidar_rays, output_origin, flow_pred, return_xyz=False): 146 | T = output_origin.shape[1] 147 | pred_pcds_t = [] 148 | 149 | free_id = len(occ_class_names) - 1 150 | occ_pred = copy.deepcopy(sem_pred) 151 | occ_pred[sem_pred < free_id] = 1 152 | occ_pred[sem_pred == free_id] = 0 153 | occ_pred = occ_pred.permute(2, 1, 0) 154 | occ_pred = occ_pred[None, None, :].contiguous().float() 155 | 156 | offset = torch.Tensor(_pc_range[:3])[None, None, :] 157 | scaler = torch.Tensor([_voxel_size] * 3)[None, None, :] 158 | 159 | lidar_tindex = torch.zeros([1, lidar_rays.shape[0]]) 160 | 161 | for t in range(T): 162 | lidar_origin = output_origin[:, t:t+1, :] # [1, 1, 3] 163 | lidar_endpts = lidar_rays[None] + lidar_origin # [1, 15840, 3] 164 | 165 | output_origin_render = ((lidar_origin - offset) / scaler).float() # [1, 1, 3] 166 | output_points_render = ((lidar_endpts - offset) / scaler).float() # [1, N, 3] 167 | output_tindex_render = lidar_tindex # [1, N], all zeros 168 | 169 | with torch.no_grad(): 170 | pred_dist, _, coord_index = dvr.render_forward( 171 | occ_pred.cuda(), 172 | output_origin_render.cuda(), 173 | output_points_render.cuda(), 174 | output_tindex_render.cuda(), 175 | [1, 16, 200, 200], 176 | "test" 177 | ) 178 | pred_dist *= _voxel_size 179 | 180 | pred_pcds = get_rendered_pcds( 181 | lidar_origin[0].cpu().numpy(), 182 | lidar_endpts[0].cpu().numpy(), 183 | lidar_tindex[0].cpu().numpy(), 184 | pred_dist[0].cpu().numpy() 185 | ) 186 | coord_index = coord_index[0, :, :].long().cpu() # [N, 3] 187 | 188 | pred_flow = torch.from_numpy(flow_pred[coord_index[:, 0], coord_index[:, 1], coord_index[:, 2]]) # [N, 2] 189 | pred_label = sem_pred[coord_index[:, 0], coord_index[:, 1], coord_index[:, 2]][:, None] # [N, 1] 190 | pred_dist = pred_dist[0, :, None].cpu() 191 | 192 | if return_xyz: 193 | pred_pcds = torch.cat([pred_label, pred_dist, pred_flow, pred_pcds[0]], dim=-1) # [N, 5] 5: [label, dist, x, y, z] 194 | else: 195 | pred_pcds = torch.cat([pred_label, pred_dist, pred_flow], dim=-1) 196 | 197 | pred_pcds_t.append(pred_pcds) 198 | 199 | pred_pcds_t = torch.cat(pred_pcds_t, dim=0) 200 | 201 | return pred_pcds_t.numpy() 202 | 203 | 204 | def main(args): 205 | token2path = {} 206 | 207 | if args.dataset_type == 'openocc_v2': 208 | data_infos = pickle.load(open(args.data_info, 'rb'))['infos'] 209 | for info in data_infos: 210 | # get reletive path 211 | occ_path = info['occ_path'].split('nuscenes/')[-1] 212 | token2path[info['token']] = os.path.join(args.data_root, occ_path) 213 | 214 | elif args.dataset_type == 'lightwheelocc': 215 | # lightwheelocc is 10Hz, downsample to 1/5 216 | data_infos = pickle.load(open(args.data_info, 'rb'))['infos'][::5] 217 | for info in data_infos: 218 | token2path[info['token']] = os.path.join(args.data_root, info['occ_path']) 219 | 220 | # generate lidar rays 221 | lidar_rays = generate_lidar_rays() 222 | lidar_rays = torch.from_numpy(lidar_rays) 223 | 224 | ego_pose_dataset = EgoPoseDataset(data_infos, dataset_type=args.dataset_type) 225 | data_loader_kwargs={ 226 | "pin_memory": False, 227 | "shuffle": False, 228 | "batch_size": 1, 229 | "num_workers": 0, 230 | } 231 | 232 | data_loader = DataLoader( 233 | ego_pose_dataset, 234 | **data_loader_kwargs, 235 | ) 236 | data_pkl = {} 237 | for batch in tqdm(data_loader, ncols=50): 238 | sample_token = batch[0][0] 239 | output_origin = batch[1].to(torch.float32) 240 | 241 | gt_filepath = token2path[sample_token] 242 | gt_data = np.load(gt_filepath, allow_pickle=True) 243 | sem_gt = gt_data['semantics'] 244 | sem_gt = torch.from_numpy(sem_gt) 245 | 246 | flow_gt = gt_data['flow'] 247 | flow_gt = np.reshape(flow_gt, [200, 200, 16, 2]) 248 | 249 | pcd_gt = process_one_sample(sem_gt, lidar_rays, output_origin, flow_gt, return_xyz=VIZ) 250 | 251 | if VIZ: 252 | pcdimg = viz_pcd(pcd_gt[:, 4:], pcd_gt[:, 0]) 253 | os.makedirs('vis', exist_ok=True) 254 | cv2.imwrite('vis/%s_pcd.jpg' % sample_token, pcdimg[..., ::-1]) 255 | 256 | pcd_cls = pcd_gt[:, 0].astype(np.uint8) 257 | pcd_dist = pcd_gt[:, 1].astype(np.float16) 258 | pcd_flow = pcd_gt[:, 2:4].astype(np.float16) 259 | 260 | data_dict = { 261 | 'pcd_cls': pcd_cls, 262 | 'pcd_dist': pcd_dist, 263 | 'pcd_flow': pcd_flow 264 | } 265 | data_pkl[sample_token] = data_dict 266 | 267 | submission_pkl = { 268 | 'method': 'GT', 269 | 'team': 'OpenDriveLab', 270 | 'authors': 'OpenDriveLab', 271 | 'e-mail': 'contact@opendrivelab.com', 272 | 'institution / company': "OpenDriveLab", 273 | 'country / region': "China", 274 | 'results': data_pkl 275 | } 276 | 277 | os.makedirs(args.output_dir, exist_ok=True) 278 | output_path = os.path.join(args.output_dir, os.path.basename(args.data_info).split('.')[0] + '_pcd.gz') 279 | print("gzip and dumping the data...") 280 | 281 | start = time.time() 282 | with gzip.GzipFile(output_path, 'wb', compresslevel=9) as f: 283 | pickle.dump(submission_pkl, f, protocol=pickle.HIGHEST_PROTOCOL) 284 | 285 | print(f"done in {time.time() - start:.2f}s") 286 | 287 | 288 | if __name__ == "__main__": 289 | parser = argparse.ArgumentParser() 290 | parser.add_argument("--dataset-type", default='lightwheelocc', choices=['lightwheelocc', 'openocc_v2']) 291 | parser.add_argument("--data-root", default='../../data/lightwheelocc') 292 | parser.add_argument("--data-info", default='../../data/lightwheelocc/lightwheel_occ_infos_val.pkl') 293 | parser.add_argument("--output-dir", default='./output/') 294 | args = parser.parse_args() 295 | 296 | torch.random.manual_seed(0) 297 | np.random.seed(0) 298 | 299 | main(args) 300 | --------------------------------------------------------------------------------