├── .gitignore ├── LICENSE ├── README.md ├── assets ├── confusion_matrix.png ├── image1.png ├── image2.png ├── ious.png ├── predictions.png ├── video1.gif └── video2.gif ├── dataset_convert ├── README.md ├── laserscan_nuscenes.py ├── laserscan_semantic_kitti.py ├── nu_dataset.py ├── pcd_dataset.py ├── semantic-kitti.yaml ├── semantic_kitti.py └── semantic_kitti_sequence.py ├── dataset_samples ├── nuscenes │ ├── train │ │ ├── 0.npy │ │ ├── 1.npy │ │ ├── 10.npy │ │ ├── 11.npy │ │ ├── 12.npy │ │ ├── 13.npy │ │ ├── 14.npy │ │ ├── 15.npy │ │ ├── 16.npy │ │ ├── 17.npy │ │ ├── 18.npy │ │ ├── 19.npy │ │ ├── 2.npy │ │ ├── 20.npy │ │ ├── 21.npy │ │ ├── 22.npy │ │ ├── 23.npy │ │ ├── 24.npy │ │ ├── 25.npy │ │ ├── 26.npy │ │ ├── 27.npy │ │ ├── 28.npy │ │ ├── 29.npy │ │ ├── 3.npy │ │ ├── 30.npy │ │ ├── 31.npy │ │ ├── 4.npy │ │ ├── 5.npy │ │ ├── 6.npy │ │ ├── 7.npy │ │ ├── 8.npy │ │ └── 9.npy │ └── val │ │ ├── 25611.npy │ │ ├── 25612.npy │ │ ├── 25613.npy │ │ └── 25614.npy ├── sample_dataset │ ├── train │ │ ├── 1603960637314959526.npy │ │ ├── 1603960637514871120.npy │ │ ├── 1603960637614809990.npy │ │ ├── 1603960637814712524.npy │ │ ├── 1603960637914665699.npy │ │ ├── 1603960638014621019.npy │ │ ├── 1603960638114571095.npy │ │ ├── 1603960638214528561.npy │ │ ├── 1603960638314473152.npy │ │ ├── 1603960638514373779.npy │ │ ├── 1603960638614326477.npy │ │ ├── 1603960638714277029.npy │ │ ├── 1603960638814225197.npy │ │ ├── 1603960638914183378.npy │ │ ├── 1603960639014128208.npy │ │ ├── 1603960639114081383.npy │ │ ├── 1603960639214035511.npy │ │ ├── 1603960639313982010.npy │ │ ├── 1603960639413947344.npy │ │ ├── 1603960639513887405.npy │ │ ├── 1603960639613846302.npy │ │ ├── 1603960639713784933.npy │ │ ├── 1603960639813746929.npy │ │ ├── 1603960639913702011.npy │ │ ├── 1603960640013648272.npy │ │ ├── 1603960640213545799.npy │ │ ├── 1603960640313498735.npy │ │ ├── 1603960640413448095.npy │ │ ├── 1603960640513404846.npy │ │ ├── 1603960640613354445.npy │ │ ├── 1603960640713299513.npy │ │ └── 1603960640813255310.npy │ └── val │ │ ├── 03bag_1565340659535964_labled.npy │ │ ├── 03bag_1565340704534739_labled.npy │ │ └── 04bag_1565009986967384_labled.npy └── semantic_kitti │ ├── train │ ├── 0.npy │ ├── 1.npy │ ├── 10.npy │ ├── 11.npy │ ├── 12.npy │ ├── 13.npy │ ├── 14.npy │ ├── 15.npy │ ├── 16.npy │ ├── 17.npy │ ├── 18.npy │ ├── 19.npy │ ├── 2.npy │ ├── 20.npy │ ├── 3.npy │ ├── 4.npy │ ├── 5.npy │ ├── 6.npy │ ├── 7.npy │ ├── 8.npy │ └── 9.npy │ └── val │ ├── 0.npy │ ├── 1.npy │ ├── 2.npy │ ├── 3.npy │ └── 4.npy ├── docker ├── Dockerfile ├── docker_build.sh ├── docker_eval.sh ├── docker_inference.sh ├── docker_run_all_data.sh ├── docker_train.sh └── requirements.txt ├── pcl_segmentation ├── configs │ ├── Darknet21.py │ ├── Darknet53.py │ ├── Darknet53Kitti.py │ ├── SqueezeSegV2.py │ ├── SqueezeSegV2Kitti.py │ ├── SqueezeSegV2NuScenes.py │ └── __init__.py ├── data_loader │ ├── __init__.py │ └── data_loader.py ├── eval.py ├── inference.py ├── nets │ ├── Darknet.py │ ├── SegmentationNetwork.py │ ├── SqueezeSegV2.py │ └── __init__.py ├── preprocessing │ ├── convert_validation_pcd_to_npy.py │ └── inspect_training_data.py ├── train.py └── utils │ ├── __init__.py │ ├── args_loader.py │ ├── callbacks.py │ └── util.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | venv 3 | 4 | \.idea/ 5 | 6 | data/ 7 | logs/ 8 | models/ 9 | output/ 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Semantic Segmentation of LiDAR Point Clouds in Tensorflow 2.9.1 with SqueezeSeg 2 | 3 | ![](assets/video2.gif) 4 | 5 | This repository contains implementations of SqueezeSegV2 [1], Darknet53 [2] and Darknet21 [2] for semantic point cloud 6 | segmentation implemented in Keras/Tensorflow 2.9.1 The repository contains the model architectures, training, evaluation and 7 | visualisation scripts. We also provide scripts to load and train the public dataset 8 | [Semantic Kitti](http://www.semantic-kitti.org/) and [NuScenes](https://www.nuscenes.org/). 9 | 10 | ## Usage 11 | 12 | #### Installation 13 | All required libraries are listed in the `requirements.txt` file. You may install them within a 14 | [virtual environment](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) 15 | with: 16 | ```bash 17 | pip install -r requirements.txt 18 | ``` 19 | 20 | #### Data Format 21 | This repository relies on the data format as in [1]. A dataset has the following file structure: 22 | ``` 23 | . 24 | ├── train 25 | ├── val 26 | ├── test 27 | ``` 28 | The data samples are located in the directories `train`, `val` and `test`. 29 | 30 | A data sample is stored as a numpy `*.npy` file. Each file contains 31 | a tensor of size `height X width X 6`. The 6 channels correspond to 32 | 33 | 0. X-Coordinate in [m] 34 | 1. Y-Coordinate in [m] 35 | 2. Z-Coordinate in [m] 36 | 3. Intensity (with range [0-255]) 37 | 4. Depth in [m] 38 | 5. Label ID 39 | 40 | For points in the point cloud that are not present (e.g. due to no reflection), the depth will be zero. 41 | A sample dataset can be found in the directory `data`. 42 | 43 | #### Sample Dataset 44 | This repository provides several sample dataset which can be used as a template for your own dataset. The directory 45 | `dataset_samples` contains the directories 46 | ``` 47 | . 48 | ├── nuscenes 49 | ├── sample_dataset 50 | ├── semantic_kitti 51 | ``` 52 | Each directory in turn contains a `train` and `val` split with 32 train samples and 3 validation samples. 53 | 54 | #### Data Normalization 55 | For a proper data normalization it is necessary to iterate over training set and determine the __mean__ and __std__ 56 | values for each of the input fields. The script `preprocessing/inspect_training_data.py` provides such a computation. 57 | 58 | ```bash 59 | # pclsegmentation/pcl_segmentation 60 | $ python3 preprocessing/inspect_training_data.py \ 61 | --input_dir="../dataset_samples/sample_dataset/train/" \ 62 | --output_dir="../dataset_samples/sample_dataset/ImageSet" 63 | ``` 64 | The glob pattern `*.npy` is applied to the `input_dir` path. The script computes and prints the mean and std values 65 | for the five input fields. These values should be set in the configuration files in 66 | [pcl_segmentation/configs](pcl_segmentation/configs) as the arrays `mc.INPUT_MEAN` and `mc.INPUT_STD`. 67 | 68 | #### Training 69 | The training of the segmentation networks can be evoked by using the `train.py` script. It is possible to choose between 70 | three different network architectures: `squeezesegv2` [1], `darknet21` [2] and `darknet53` [2]. 71 | The training script uses the dataset splits `train` and `val`. The metrics for both splits are constantly computed 72 | during training. The Tensorboard callback also uses the `val` split for visualisation of the current model prediction. 73 | ```bash 74 | # pclsegmentation/pcl_segmentation 75 | $ python3 train.py \ 76 | --data_path="../sample_dataset" \ 77 | --train_dir="../output" \ 78 | --epochs=5 \ 79 | --model=squeezesegv2 80 | ``` 81 | 82 | #### Evaluation 83 | For the evaluation the script `eval.py` can be used. 84 | Note that for the evaluation the flag `--image_set` can be set to `val` or `test` according to datasets which are present 85 | at the `data_path`. 86 | ```bash 87 | # pclsegmentation/pcl_segmentation 88 | $ python3 eval.py \ 89 | --data_path="../sample_dataset" \ 90 | --image_set="val" \ 91 | --eval_dir="../eval" \ 92 | --path_to_model="../output/model" \ 93 | --model=squeezesegv2 94 | ``` 95 | 96 | #### Inference 97 | Inference of the model can be performed by loading some data samples and by loading the trained model. The script 98 | includes visualisation methods for the segmented images. The results can be stored by providing 99 | `--output_dir` to the script. 100 | ```bash 101 | # pclsegmentation/pcl_segmentation 102 | $ python3 inference.py \ 103 | --input_path="../sample_dataset/train/*.npy" \ 104 | --output_dir="../output/prediction" \ 105 | --path_to_model="../output/model" \ 106 | --model=squeezesegv2 107 | ``` 108 | 109 | ## Docker 110 | We also provide a docker environment for __training__, __evaluation__ and __inference__. All script can be found in the 111 | directory `docker`. 112 | 113 | First, build the environment with 114 | ```bash 115 | # docker 116 | ./docker_build.sh 117 | ``` 118 | 119 | Then you can execute the sample training with 120 | ```bash 121 | # docker 122 | ./docker_train.sh 123 | ``` 124 | 125 | and you could evaluate the trained model with 126 | ```bash 127 | # docker 128 | ./docker_eval.sh 129 | ``` 130 | 131 | For inference on the sample dataset execute: 132 | ```bash 133 | # docker 134 | ./docker_inference.sh 135 | ``` 136 | 137 | ## Datasets 138 | In the directory [dataset_convert](dataset_convert) you will find conversion scripts to convert following datasets 139 | to a format that can be read by the data pipeline implemented in this repository. 140 | 141 | ### NuScenes 142 | Make sure that you have installed `nuscenes-devkit` and that you downloaded the nuScenes dataset correctly. Then execute 143 | the script `nu_dataset.py` 144 | ```bash 145 | # dataset_convert 146 | $ python3 nu_dataset.py \ 147 | --dataset /root/path/nuscenes \ 148 | --output_dir /root/path/nuscenes/converted 149 | ``` 150 | The script will generate `*.npy` files into the directory `converted`. It will automatically create a train/val split. 151 | Make sure to create two empty directories `train` and `val`. The current implementation will perform a class reduction. 152 | 153 | 154 | ### Semantic Kitti 155 | The [Semantic Kitti](http://www.semantic-kitti.org/) dataset can be converted with the script `semantic_kitti.py`. 156 | ```bash 157 | # dataset_convert 158 | $ python3 semantic_kitti.py \ 159 | --dataset /root/path/semantic_kitti \ 160 | --output_dir /root/path/semantic_kitti/converted 161 | ``` 162 | The script will generate `*.npy` files into the directory `converted`. It will automatically create a train/val split. 163 | Make sure to create two empty directories `train` and `val`. The current implementation will perform a class reduction. 164 | 165 | ### Generic PCD dataset 166 | The script [`pcd_dataset.py`](dataset_convert/pcd_dataset.py) allows the conversion of a labeled `*.pcd` dataset. 167 | As input dataset define the directory that contains all `*.pcd` files. The pcd files need to have the field `label`. 168 | Check the script for more details. 169 | 170 | ```bash 171 | # dataset_convert 172 | $ python3 pcd_dataset.py \ 173 | --dataset /root/path/pcd_dataset \ 174 | --output_dir /root/path/pcd_dataset/converted 175 | ``` 176 | 177 | ## Tensorboard 178 | The implementation also contains a Tensorboard callback which visualizes the most important metrics such as the __confusion 179 | matrix__, __IoUs__, __MIoU__, __Recalls__, __Precisions__, __Learning Rates__, different __losses__ and the current model 180 | __prediction__ on a data sample. The callbacks are evoked by Keras' `model.fit()` function. 181 | 182 | ```bash 183 | # pclsegmentation 184 | $ tensorboard --logdir ../output 185 | ``` 186 | 187 | ![](assets/confusion_matrix.png) 188 | ![](assets/ious.png) 189 | ![](assets/predictions.png) 190 | 191 | 192 | ## More Inference Examples 193 | ![](assets/image2.png) 194 | Left image: Prediction - Right Image: Ground Truth 195 | 196 | ![](assets/image1.png) 197 | ![](assets/video1.gif) 198 | 199 | 200 | ## References 201 | The network architectures are based on 202 | - [1] [SqueezeSegV2: Improved Model Structure and Unsupervised Domain Adaptation for Road-Object Segmentation from a 203 | LiDAR Point Cloud](https://github.com/xuanyuzhou98/SqueezeSegV2) 204 | - [2] [RangeNet++: Fast and Accurate LiDAR Semantic Segmentation](https://github.com/PRBonn/lidar-bonnetal) 205 | - [3] [Semantic Kitti](http://www.semantic-kitti.org/) 206 | - [4] [nuScenes](https://www.nuscenes.org/) 207 | 208 | ### TODO 209 | - [x] Faster input pipeline using TFRecords preprocessing 210 | - [x] Docker support 211 | - [ ] Implement CRF Postprocessing for SqueezeSegV2 212 | - [x] Implement dataloader for Semantic Kitti dataset 213 | - [x] Implement dataloader for nuScenes dataset 214 | - [ ] None class handling 215 | - [ ] Add results for Semantic Kitti and nuScenes 216 | - [x] Update to Tensorflow 2.9 217 | 218 | ### Author of this Repository 219 | [Till Beemelmanns](https://github.com/TillBeemelmanns) 220 | 221 | Mail: `till.beemelmanns (at) ika.rwth-aachen.de` -------------------------------------------------------------------------------- /assets/confusion_matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/confusion_matrix.png -------------------------------------------------------------------------------- /assets/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/image1.png -------------------------------------------------------------------------------- /assets/image2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/image2.png -------------------------------------------------------------------------------- /assets/ious.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/ious.png -------------------------------------------------------------------------------- /assets/predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/predictions.png -------------------------------------------------------------------------------- /assets/video1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/video1.gif -------------------------------------------------------------------------------- /assets/video2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/assets/video2.gif -------------------------------------------------------------------------------- /dataset_convert/README.md: -------------------------------------------------------------------------------- 1 | ## nuScenes dataset 2 | 3 | This directory contains a script `nu_dataset.py` which converts pointcloud samples in [nuScenes dataset](https://www.nuscenes.org/nuscenes#lidarseg) to a data format which can be used to train the neural networks developed for PCL segmentation. Moreover, `laserscan.py` includes definations of some of the classes used in`nu_dataset.py`. 4 | 5 | The directory should have the file structure: 6 | ``` 7 | ├── ImageSet 8 | ├── train.txt 9 | ├── val.txt 10 | ├── test.txt 11 | ├── train 12 | ├── val 13 | ├── test 14 | ├── nu_dataset.py 15 | ├── laserscan.py 16 | 17 | ``` 18 | The data samples are located in the directories `train`, `val` and `test`. The `*.txt` files within the directory `ImageSet` contain the filenames for the corresponding samples in data directories. 19 | 20 | ### Conversion Script 21 | 22 | Conversion script `nu_dataset.py` uses some of the functions and classes defined in [nuScenes devkit](https://github.com/nutonomy/nuscenes-devkit) and in [API for SemanticKITTI](https://github.com/PRBonn/semantic-kitti-api#readme). It opens pointcloud scans, spherically projects the points in these scans into 2D and store these projections as `*.npy` files. Each of these `*.npy` files contains a tensor of size `32 X 1024 X 6`. The 6 channels correspond to 23 | 24 | 0. X-Coordinate in [m] 25 | 1. Y-Coordinate in [m] 26 | 2. Z-Coordinate in [m] 27 | 3. Intensity 28 | 4. Depth in [m] 29 | 5. Label ID 30 | 31 | The script stores the projections in `nuscenes_dataset/train` or `nuscenes_dataset/val` directory of PCL segmentation repository. 32 | 33 | ```bash 34 | ./nu_dataset.py --dataset /path/to/nuScenes/dataset/ --output_dir /path/to/PCLSegmentation/ -v 35 | ``` 36 | where: 37 | - `dataset` is the path to the nuScenes dataset where the `/data/sets/nuscenes` directory is. 38 | - `output_dir` is the output path to the PCL segmentation repository. 39 | - `v`is a flag. If it is used, the projections are stored in validation set, otherwise they are stored in training set. 40 | 41 | To be able to run the script, the instructions explaining how to use nuScenes devkit and how to download the dataset can be found [here](https://github.com/nutonomy/nuscenes-devkit#nuscenes-lidarseg). 42 | 43 | 44 | ## SemanticKITTI dataset 45 | 46 | This directory contains the `semantic_kitti.py` which converts pointcloud samples in [SemanticKITTI dataset](http://www.semantic-kitti.org/) to a data format which can be used to train the neural networks developed for PCL segmentation. It also includes a small `train` and `val` split with 20 samples and 2 samples, respectively. 47 | 48 | ### Conversion Scripts 49 | 50 | The scripts use some of the functions and classes defined in [API for SemanticKITTI](https://github.com/PRBonn/semantic-kitti-api#readme). They open pointcloud scans, project the points in these scans into 2D and store these projections as `*.npy` files. Each of these `*.npy` files contains a tensor of size `64 X 1024 X 6`. The 6 channels correspond to 51 | 52 | 0. X-Coordinate in [m] 53 | 1. Y-Coordinate in [m] 54 | 2. Z-Coordinate in [m] 55 | 3. Intensity (with range [0-255]) 56 | 4. Depth in [m] 57 | 5. Label ID 58 | 59 | #### Downloading SemanticKITTI dataset 60 | 61 | To be able to run the scripts, firstly, SemanticKITTI dataset should be downloaded. Information about files in this dataset and how to download it is provided [here](http://www.semantic-kitti.org/dataset.html) 62 | 63 | SemanticKITTI dataset is organized in the following format: 64 | 65 | ``` 66 | /kitti/dataset/ 67 | └── sequences/ 68 | ├── 00/ 69 | │   ├── poses.txt 70 | │ ├── image_2/ 71 | │   ├── image_3/ 72 | │   ├── labels/ 73 | │ │ ├ 000000.label 74 | │ │ └ 000001.label 75 | | ├── voxels/ 76 | | | ├ 000000.bin 77 | | | ├ 000000.label 78 | | | ├ 000000.occluded 79 | | | ├ 000000.invalid 80 | | | ├ 000001.bin 81 | | | ├ 000001.label 82 | | | ├ 000001.occluded 83 | | | ├ 000001.invalid 84 | │   └── velodyne/ 85 | │ ├ 000000.bin 86 | │ └ 000001.bin 87 | ├── 01/ 88 | ├── 02/ 89 | . 90 | . 91 | . 92 | └── 21/ 93 | ``` 94 | #### Using API for SemanticKITTI 95 | 96 | ##### semantic_kittit_sequence.py 97 | 98 | The script projects the scans in the specified sequence and stores the projections in `semantic_kitti_dataset/train` or `semantic_kitti_dataset/val` directory of PCL segmentation repository. 99 | 100 | ```bash 101 | ./semantic_kitti_sequence.py --sequence 00 --dataset /path/to/kitti/dataset/ --output_dir /path/to/PCLSegmentation/ -v 102 | ``` 103 | where: 104 | - `sequence` is the sequence to be accessed (optional, default value is 00). 105 | - `dataset` is the path to the kitti dataset where the `sequences` directory is. 106 | - `output_dir` is the output path to the PCL segmentation repository. 107 | - `v`is a flag. If it is used, the projections are stored in validation set, otherwise they are stored in training set. 108 | 109 | ##### semantic_kitti.py 110 | 111 | The script randomly picks a specified number of scans from all sequences and stores their projections in `semantic_kitti_dataset/train` and `semantic_kitti_dataset/val` directory of PCL segmentation repository. 112 | 113 | ```bash 114 | ./semantic_kitti.py --dataset /path/to/kitti/dataset/ --output_dir /path/to/PCLSegmentation/ -n 20 -s 0.8 -v 115 | ``` 116 | where: 117 | - `dataset` is the path to the kitti dataset where the `sequences` directory is. 118 | - `output_dir` is the output path to the PCL segmentation repository. 119 | - `n`is the number of training samples (projections) to be used in training and validation sets. Maximum is 23201. Default is 20. 120 | - `s`is the split ratio of samples between training and validation sets. It should be between 0 and 1. Default is 0.9. 121 | - `v` is a flag. If it is used, the projections consist of 32 layers instead of 64. The script extracts 32 specified layers from the SemanticKITTI projections which are 64-layered. 122 | 123 | ### Generalization to VLP-32 Data 124 | 125 | The ultimate goal is to have a network which is trained on higher resolution KITTI dataset and does semantic segmentation on VLP-32 lidar data. KITTI pointcloud projections used in training should be modified in such a way which makes them similar to VLP-32 data. One method is to extract 32 specified layers from the KITTI point cloud projections. However, the network has not been able to generalize to VLP-32 Data well yet. The tested layer configurations are as follows. 126 | 127 | #### Tested Layer Configurations 128 | 129 | `layers` array, which is defined in `conversion_3.py` script, specifies 32 layers which will be extracted from KITTI projections. 130 | 131 | - layers = np.arange(16,48) 132 | - configuration 3 is used, but intensity is not used as a feature in semantic segmentation. 133 | - layers = np.concatenate([np.array([14, 15, 17, 24, 26, 30, 31, 34, 36, 37, 39, 41, 43, 45]), np.arange(46, 64)]) 134 | - layers = [0, 4, 8, 11, 12, 13, 15, 17, 19, 21, 23, 25, 27, 29, 30, 31, 32, 33, 35, 37, 39, 41, 43, 45, 47, 49, 50, 51, 52, 55, 59, 63] 135 | - directly projecting KITTI point clouds into 32-layered projections instead of extracting 32 layers from 64-layered projections. 136 | - layers = np.concatenate([np.array([14, 15, 16, 17, 25, 26, 27, 31, 33, 36, 39, 41, 43, 45]), np.arange(46, 64)]) 137 | - layers = np.arange(1,64,2) 138 | 139 | 140 | -------------------------------------------------------------------------------- /dataset_convert/laserscan_semantic_kitti.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import numpy as np 3 | 4 | 5 | class LaserScan: 6 | """Class that contains LaserScan with x,y,z,r""" 7 | EXTENSIONS_SCAN = ['.bin'] 8 | 9 | def __init__(self, project=False, H=64, W=1024, fov_up=3.0, fov_down=-25.0): 10 | self.project = project 11 | self.proj_H = H 12 | self.proj_W = W 13 | self.proj_fov_up = fov_up 14 | self.proj_fov_down = fov_down 15 | self.reset() 16 | 17 | def reset(self): 18 | """ Reset scan members. """ 19 | self.points = np.zeros((0, 3), dtype=np.float32) # [m, 3]: x, y, z 20 | self.remissions = np.zeros((0, 1), dtype=np.float32) # [m ,1]: remission 21 | 22 | # projected range image - [H,W] range (-1 is no data) 23 | self.proj_range = np.full((self.proj_H, self.proj_W), -1, 24 | dtype=np.float32) 25 | 26 | # unprojected range (list of depths for each point) 27 | self.unproj_range = np.zeros((0, 1), dtype=np.float32) 28 | 29 | # projected point cloud xyz - [H,W,3] xyz coord (-1 is no data) 30 | self.proj_xyz = np.full((self.proj_H, self.proj_W, 3), -1, 31 | dtype=np.float32) 32 | 33 | # projected remission - [H,W] intensity (-1 is no data) 34 | self.proj_remission = np.full((self.proj_H, self.proj_W), -1, 35 | dtype=np.float32) 36 | 37 | # projected index (for each pixel, what I am in the pointcloud) 38 | # [H,W] index (-1 is no data) 39 | self.proj_idx = np.full((self.proj_H, self.proj_W), -1, 40 | dtype=np.int32) 41 | 42 | # for each point, where it is in the range image 43 | self.proj_x = np.zeros((0, 1), dtype=np.float32) # [m, 1]: x 44 | self.proj_y = np.zeros((0, 1), dtype=np.float32) # [m, 1]: y 45 | 46 | # mask containing for each pixel, if it contains a point or not 47 | self.proj_mask = np.zeros((self.proj_H, self.proj_W), 48 | dtype=np.int32) # [H,W] mask 49 | 50 | def size(self): 51 | """ Return the size of the point cloud. """ 52 | return self.points.shape[0] 53 | 54 | def __len__(self): 55 | return self.size() 56 | 57 | def open_scan(self, filename): 58 | """ Open raw scan and fill in attributes 59 | """ 60 | # reset just in case there was an open structure 61 | self.reset() 62 | 63 | # check filename is string 64 | if not isinstance(filename, str): 65 | raise TypeError("Filename should be string type, " 66 | "but was {type}".format(type=str(type(filename)))) 67 | 68 | # check extension is a laserscan 69 | if not any(filename.endswith(ext) for ext in self.EXTENSIONS_SCAN): 70 | raise RuntimeError("Filename extension is not valid scan file.") 71 | 72 | # if all goes well, open pointcloud 73 | scan = np.fromfile(filename, dtype=np.float32) 74 | scan = scan.reshape((-1, 4)) 75 | 76 | # put in attribute 77 | points = scan[:, 0:3] # get xyz 78 | remissions = scan[:, 3] # get remission 79 | self.set_points(points, remissions) 80 | 81 | def set_points(self, points, remissions=None): 82 | """ Set scan attributes (instead of opening from file) 83 | """ 84 | # reset just in case there was an open structure 85 | self.reset() 86 | 87 | # check scan makes sense 88 | if not isinstance(points, np.ndarray): 89 | raise TypeError("Scan should be numpy array") 90 | 91 | # check remission makes sense 92 | if remissions is not None and not isinstance(remissions, np.ndarray): 93 | raise TypeError("Remissions should be numpy array") 94 | 95 | # put in attribute 96 | self.points = points # get xyz 97 | if remissions is not None: 98 | self.remissions = remissions # get remission 99 | else: 100 | self.remissions = np.zeros((points.shape[0]), dtype=np.float32) 101 | 102 | # if projection is wanted, then do it and fill in the structure 103 | if self.project: 104 | self.do_range_projection() 105 | 106 | def do_range_projection(self): 107 | """ Project a pointcloud into a spherical projection image.projection. 108 | Function takes no arguments because it can be also called externally 109 | if the value of the constructor was not set (in case you change your 110 | mind about wanting the projection) 111 | """ 112 | # laser parameters 113 | fov_up = self.proj_fov_up / 180.0 * np.pi # field of view up in rad 114 | fov_down = self.proj_fov_down / 180.0 * np.pi # field of view down in rad 115 | fov = abs(fov_down) + abs(fov_up) # get field of view total in rad 116 | 117 | # get depth of all points 118 | depth = np.linalg.norm(self.points, 2, axis=1) 119 | 120 | # get scan components 121 | scan_x = self.points[:, 0] 122 | scan_y = self.points[:, 1] 123 | scan_z = self.points[:, 2] 124 | 125 | # get angles of all points 126 | yaw = -np.arctan2(scan_y, scan_x) 127 | pitch = np.arcsin(scan_z / depth) 128 | 129 | # get projections in image coords 130 | proj_x = 0.5 * (yaw / np.pi + 1.0) # in [0.0, 1.0] 131 | proj_y = 1.0 - (pitch + abs(fov_down)) / fov # in [0.0, 1.0] 132 | 133 | # scale to image size using angular resolution 134 | proj_x *= self.proj_W # in [0.0, W] 135 | proj_y *= self.proj_H # in [0.0, H] 136 | 137 | # round and clamp for use as index 138 | proj_x = np.floor(proj_x) 139 | proj_x = np.minimum(self.proj_W - 1, proj_x) 140 | proj_x = np.maximum(0, proj_x).astype(np.int32) # in [0,W-1] 141 | self.proj_x = np.copy(proj_x) # store a copy in orig order 142 | 143 | proj_y = np.floor(proj_y) 144 | proj_y = np.minimum(self.proj_H - 1, proj_y) 145 | proj_y = np.maximum(0, proj_y).astype(np.int32) # in [0,H-1] 146 | self.proj_y = np.copy(proj_y) # stope a copy in original order 147 | 148 | # copy of depth in original order 149 | self.unproj_range = np.copy(depth) 150 | 151 | # order in decreasing depth 152 | indices = np.arange(depth.shape[0]) 153 | order = np.argsort(depth)[::-1] 154 | depth = depth[order] 155 | indices = indices[order] 156 | points = self.points[order] 157 | remission = self.remissions[order] 158 | proj_y = proj_y[order] 159 | proj_x = proj_x[order] 160 | 161 | # assing to images 162 | self.proj_range[proj_y, proj_x] = depth 163 | self.proj_xyz[proj_y, proj_x] = points 164 | self.proj_remission[proj_y, proj_x] = remission 165 | self.proj_idx[proj_y, proj_x] = indices 166 | self.proj_mask = (self.proj_idx > 0).astype(np.float32) 167 | 168 | 169 | class SemLaserScan(LaserScan): 170 | """Class that contains LaserScan with x,y,z,r,sem_label,sem_color_label,inst_label,inst_color_label""" 171 | EXTENSIONS_LABEL = ['.label'] 172 | 173 | def __init__(self, nclasses, sem_color_dict=None, project=False, H=64, W=1024, fov_up=3.0, fov_down=-25.0): 174 | super(SemLaserScan, self).__init__(project, H, W, fov_up, fov_down) 175 | self.reset() 176 | self.nclasses = nclasses # number of classes 177 | 178 | # make semantic colors 179 | max_sem_key = 0 180 | for key, data in sem_color_dict.items(): 181 | if key + 1 > max_sem_key: 182 | max_sem_key = key + 1 183 | self.sem_color_lut = np.zeros((max_sem_key + 100, 3), dtype=np.float32) 184 | for key, value in sem_color_dict.items(): 185 | self.sem_color_lut[key] = np.array(value, np.float32) / 255.0 186 | 187 | # make instance colors 188 | max_inst_id = 100000 189 | self.inst_color_lut = np.random.uniform(low=0.0, 190 | high=1.0, 191 | size=(max_inst_id, 3)) 192 | # force zero to a gray-ish color 193 | self.inst_color_lut[0] = np.full((3), 0.1) 194 | 195 | def reset(self): 196 | """ Reset scan members. """ 197 | super(SemLaserScan, self).reset() 198 | 199 | # semantic labels 200 | self.sem_label = np.zeros((0, 1), dtype=np.uint32) # [m, 1]: label 201 | self.sem_label_color = np.zeros((0, 3), dtype=np.float32) # [m ,3]: color 202 | 203 | # instance labels 204 | self.inst_label = np.zeros((0, 1), dtype=np.uint32) # [m, 1]: label 205 | self.inst_label_color = np.zeros((0, 3), dtype=np.float32) # [m ,3]: color 206 | 207 | # projection color with semantic labels 208 | self.proj_sem_label = np.zeros((self.proj_H, self.proj_W), 209 | dtype=np.int32) # [H,W] label 210 | self.proj_sem_color = np.zeros((self.proj_H, self.proj_W, 3), 211 | dtype=np.float) # [H,W,3] color 212 | 213 | # projection color with instance labels 214 | self.proj_inst_label = np.zeros((self.proj_H, self.proj_W), 215 | dtype=np.int32) # [H,W] label 216 | self.proj_inst_color = np.zeros((self.proj_H, self.proj_W, 3), 217 | dtype=np.float) # [H,W,3] color 218 | 219 | def open_label(self, filename): 220 | """ Open raw scan and fill in attributes 221 | """ 222 | # check filename is string 223 | if not isinstance(filename, str): 224 | raise TypeError("Filename should be string type, " 225 | "but was {type}".format(type=str(type(filename)))) 226 | 227 | # check extension is a laserscan 228 | if not any(filename.endswith(ext) for ext in self.EXTENSIONS_LABEL): 229 | raise RuntimeError("Filename extension is not valid label file.") 230 | 231 | # if all goes well, open label 232 | label = np.fromfile(filename, dtype=np.uint32) 233 | label = label.reshape((-1)) 234 | 235 | # set it 236 | self.set_label(label) 237 | 238 | def set_label(self, label): 239 | """ Set points for label not from file but from np 240 | """ 241 | # check label makes sense 242 | if not isinstance(label, np.ndarray): 243 | raise TypeError("Label should be numpy array") 244 | 245 | # only fill in attribute if the right size 246 | if label.shape[0] == self.points.shape[0]: 247 | self.sem_label = label & 0xFFFF # semantic label in lower half 248 | self.inst_label = label >> 16 # instance id in upper half 249 | else: 250 | print("Points shape: ", self.points.shape) 251 | print("Label shape: ", label.shape) 252 | raise ValueError("Scan and Label don't contain same number of points") 253 | 254 | # sanity check 255 | assert((self.sem_label + (self.inst_label << 16) == label).all()) 256 | 257 | if self.project: 258 | self.do_label_projection() 259 | 260 | def colorize(self): 261 | """ Colorize pointcloud with the color of each semantic label 262 | """ 263 | self.sem_label_color = self.sem_color_lut[self.sem_label] 264 | self.sem_label_color = self.sem_label_color.reshape((-1, 3)) 265 | 266 | self.inst_label_color = self.inst_color_lut[self.inst_label] 267 | self.inst_label_color = self.inst_label_color.reshape((-1, 3)) 268 | 269 | def do_label_projection(self): 270 | # only map colors to labels that exist 271 | mask = self.proj_idx >= 0 272 | 273 | # semantics 274 | self.proj_sem_label[mask] = self.sem_label[self.proj_idx[mask]] 275 | self.proj_sem_color[mask] = self.sem_color_lut[self.sem_label[self.proj_idx[mask]]] 276 | 277 | # instances 278 | self.proj_inst_label[mask] = self.inst_label[self.proj_idx[mask]] 279 | self.proj_inst_color[mask] = self.inst_color_lut[self.inst_label[self.proj_idx[mask]]] -------------------------------------------------------------------------------- /dataset_convert/nu_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import numpy as np 4 | 5 | import argparse 6 | import tqdm 7 | 8 | from nuscenes import NuScenes 9 | import matplotlib.pyplot as plt 10 | 11 | from laserscan_nuscenes import SemLaserScan 12 | 13 | classid_to_color = { # RGB. 14 | 0: [0, 0, 0], # Black. 15 | 1: [70, 130, 180], # Steelblue 16 | 2: [0, 0, 230], # Blue 17 | 3: [135, 206, 235], # Skyblue, 18 | 4: [100, 149, 237], # Cornflowerblue 19 | 5: [219, 112, 147], # Palevioletred 20 | 6: [0, 0, 128], # Navy, 21 | 7: [240, 128, 128], # Lightcoral 22 | 8: [138, 43, 226], # Blueviolet 23 | 9: [112, 128, 144], # Slategrey 24 | 10: [210, 105, 30], # Chocolate 25 | 11: [105, 105, 105], # Dimgrey 26 | 12: [47, 79, 79], # Darkslategrey 27 | 13: [188, 143, 143], # Rosybrown 28 | 14: [220, 20, 60], # Crimson 29 | 15: [255, 127, 80], # Coral 30 | 16: [255, 69, 0], # Orangered 31 | 17: [255, 158, 0], # Orange 32 | 18: [233, 150, 70], # Darksalmon 33 | 19: [255, 83, 0], 34 | 20: [255, 215, 0], # Gold 35 | 21: [255, 61, 99], # Red 36 | 22: [255, 140, 0], # Darkorange 37 | 23: [255, 99, 71], # Tomato 38 | 24: [0, 207, 191], # nuTonomy green 39 | 25: [175, 0, 75], 40 | 26: [75, 0, 75], 41 | 27: [112, 180, 60], 42 | 28: [222, 184, 135], # Burlywood 43 | 29: [255, 228, 196], # Bisque 44 | 30: [0, 175, 0], # Green 45 | 31: [255, 240, 245] 46 | } 47 | 48 | label_map = { 49 | # Road 50 | 24: 0, # drivable surface 51 | 52 | # Sidewalk 53 | 25: 1, # flat terrain 54 | 26: 1, # sidewalk 55 | 56 | # Building 57 | 28: 2, # static man made 58 | 59 | # Pole 60 | 9: 3, # road barrier 61 | 12: 3, # traffic cones 62 | 63 | # Vegetation 64 | 30: 4, # vegetation 65 | 27: 4, # flat terrain 66 | 67 | # Person 68 | 2: 5, # adult 69 | 3: 5, # child 70 | 4: 5, # construction worker 71 | 6: 5, # police officer 72 | 73 | # Two-wheeler 74 | 21: 6, # motor cycle 75 | 5: 6, # scooter or different 76 | 13: 6, # bicycles rack including bicycles 77 | 14: 6, # bicycle 78 | 8: 6, # wheel chair 79 | 7: 6, # stroller 80 | 81 | # Car 82 | 20: 7, # police vehicle 83 | 17: 7, # vehicle 84 | 85 | # Truck 86 | 23: 8, # truck 87 | 18: 8, # construction vehicles 88 | 19: 8, # ambulance 89 | 22: 8, # trailer 90 | 91 | # Bus 92 | 15: 9, # bendy bus 93 | 16: 9, # ridgid bus 94 | 95 | # None 96 | 0: 10, # noise 97 | 1: 10, # animal 98 | 10: 10, # debris 99 | 11: 10, # push pull able 100 | 29: 10, # points in the background 101 | 31: 10 # ego vehicle 102 | } 103 | 104 | if __name__ == '__main__': 105 | parser = argparse.ArgumentParser("./nu_dataset.py") 106 | parser.add_argument( 107 | '--dataset', '-d', 108 | type=str, 109 | required=True, 110 | help='path to the nuscenes dataset where the `/data/sets/nuscenes` directory is. No Default', 111 | ) 112 | parser.add_argument( 113 | '--output_dir', '-p', 114 | type=str, 115 | required=True, 116 | help='output path to the PCL segmentation repository. No Default', 117 | ) 118 | parser.add_argument( 119 | '-s', type=float, default=0.75, 120 | help='split percentage of samples for training and validation sets. It should be between 0 and 1. Default is 0.1.' 121 | ) 122 | 123 | FLAGS, unparsed = parser.parse_known_args() 124 | 125 | H = 32 126 | W = 1024 127 | 128 | nusc = NuScenes(version='v1.0-trainval', dataroot=FLAGS.dataset, verbose=True) 129 | nclasses = len(classid_to_color) # number of classes 130 | laser_scan = SemLaserScan(nclasses, 131 | classid_to_color, 132 | project=True, 133 | H=H, W=W, 134 | fov_up=12, fov_down=-30, 135 | use_ring_projection=False) 136 | 137 | # class reduction 138 | vfunc = np.vectorize(label_map.get) 139 | 140 | for index, my_sample in tqdm.tqdm(enumerate(nusc.sample), total=len(nusc.sample)): 141 | sample_data_token = my_sample['data']['LIDAR_TOP'] 142 | sd_record = nusc.get('sample_data', sample_data_token) 143 | # path to the bin file containing the x, y, z and intensity of the points in the point cloud 144 | pcl_path = os.path.join(nusc.dataroot, sd_record['filename']) 145 | # path to the bin file containing the labels of the points in the point cloud 146 | lidarseg_labels_filename = os.path.join(nusc.dataroot, nusc.get('lidarseg', sample_data_token)['filename']) 147 | 148 | laser_scan.open_scan(pcl_path) # spherically project the point cloud scans 149 | laser_scan.open_label(lidarseg_labels_filename) 150 | 151 | if index == 0: 152 | plt.figure(figsize=(30, 3)) 153 | plt.imshow(laser_scan.proj_sem_color) 154 | plt.tight_layout() 155 | plt.show() 156 | 157 | mask = laser_scan.proj_range > 0 # check if the projected depth is positive 158 | laser_scan.proj_range[~mask] = 0.0 159 | laser_scan.proj_xyz[~mask] = 0.0 160 | laser_scan.proj_remission[~mask] = 0.0 161 | laser_scan.proj_sem_label = vfunc(laser_scan.proj_sem_label) # map class labels to values between 0 and 10 162 | 163 | # create the final data sample with shape (H, W, 6) 164 | final_data = np.concatenate([laser_scan.proj_xyz, 165 | laser_scan.proj_remission.reshape((H, W, 1)), 166 | laser_scan.proj_range.reshape((H, W, 1)), 167 | laser_scan.proj_sem_label.reshape((H, W, 1))], 168 | axis=2) 169 | 170 | if index < int(len(nusc.sample) * FLAGS.s): 171 | np.save(os.path.join(FLAGS.output_dir, "train", str(index)), final_data) 172 | else: 173 | np.save(os.path.join(FLAGS.output_dir, "val", str(index)), final_data) 174 | -------------------------------------------------------------------------------- /dataset_convert/pcd_dataset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import glob 4 | import numpy as np 5 | 6 | import argparse 7 | import tqdm 8 | from pyntcloud import PyntCloud 9 | 10 | import matplotlib.pyplot as plt 11 | 12 | from laserscan_nuscenes import SemLaserScan 13 | 14 | classid_to_color = { # RGB. 15 | 0: [0, 0, 0], # Black. 16 | 1: [70, 130, 180], # Steelblue 17 | 2: [0, 0, 230], # Blue 18 | 3: [135, 206, 235], # Skyblue, 19 | 4: [100, 149, 237], # Cornflowerblue 20 | 5: [219, 112, 147], # Palevioletred 21 | 6: [0, 0, 128], # Navy, 22 | 7: [240, 128, 128], # Lightcoral 23 | 8: [138, 43, 226], # Blueviolet 24 | 9: [112, 128, 144], # Slategrey 25 | 10: [210, 105, 30], # Chocolate 26 | 11: [105, 105, 105], # Dimgrey 27 | 12: [47, 79, 79], # Darkslategrey 28 | 13: [188, 143, 143], # Rosybrown 29 | 14: [220, 20, 60], # Crimson 30 | 15: [255, 127, 80], # Coral 31 | 16: [255, 69, 0], # Orangered 32 | 17: [255, 158, 0], # Orange 33 | 18: [233, 150, 70], # Darksalmon 34 | 19: [255, 83, 0], 35 | 20: [255, 215, 0], # Gold 36 | 21: [255, 61, 99], # Red 37 | 22: [255, 140, 0], # Darkorange 38 | 23: [255, 99, 71], # Tomato 39 | 24: [0, 207, 191], # nuTonomy green 40 | 25: [175, 0, 75], 41 | 26: [75, 0, 75], 42 | 27: [112, 180, 60], 43 | 28: [222, 184, 135], # Burlywood 44 | 29: [255, 228, 196], # Bisque 45 | 30: [0, 175, 0], # Green 46 | 31: [255, 240, 245] 47 | } 48 | 49 | label_map = { 50 | # Road 51 | 12: 0, # drivable surface 52 | 53 | # Sidewalk 54 | 13: 1, # sidewalk 55 | 14: 1, # parking 56 | 57 | # Building 58 | 10: 2, # obstacle 59 | 60 | # Pole 61 | 11: 3, # traffic control 62 | 63 | # Vegetation 64 | 15: 4, # vegetation 65 | 16: 4, # flat terrain 66 | 67 | # Person 68 | 7: 5, # person 69 | 70 | # Two-wheeler 71 | 5: 6, # motor cycle 72 | 6: 6, # bicycle 73 | 8: 6, # rider 74 | 75 | # Car 76 | 1: 7, # car 77 | 78 | # Truck 79 | 2: 8, # truck 80 | 4: 8, # trailer 81 | 82 | # Bus 83 | 3: 9, # bus 84 | 85 | # None 86 | 0: 10, # noise 87 | 9: 10 # animal 88 | } 89 | 90 | if __name__ == '__main__': 91 | parser = argparse.ArgumentParser("./pcd_dataset.py") 92 | parser.add_argument( 93 | '--dataset', '-d', 94 | type=str, 95 | required=True, 96 | help='path to the directory which contains the pcd files. No Default', 97 | ) 98 | parser.add_argument( 99 | '--output_dir', '-o', 100 | type=str, 101 | required=True, 102 | help='output path where the npy files should be written to. No Default', 103 | ) 104 | parser.add_argument( 105 | '--plot', '-p', 106 | action='store_true', 107 | help='Plot the semantic image of the PCL projection', 108 | ) 109 | 110 | FLAGS, unparsed = parser.parse_known_args() 111 | 112 | H = 32 113 | W = 1024 114 | 115 | nclasses = len(classid_to_color) # number of classes 116 | laser_scan = SemLaserScan(nclasses, 117 | classid_to_color, 118 | project=True, 119 | H=H, W=W, 120 | fov_up=None, 121 | fov_down=None, 122 | use_ring_projection=True) 123 | 124 | # class reduction 125 | vfunc = np.vectorize(label_map.get) 126 | 127 | list_of_files = sorted(glob.glob(os.path.join(FLAGS.dataset, "*.pcd"))) 128 | 129 | for index, pcd_file in tqdm.tqdm(enumerate(list_of_files)): 130 | cloud = PyntCloud.from_file(pcd_file) 131 | 132 | points = np.array(cloud.points) 133 | 134 | laser_scan.set_points(points=points[:, 0:3], remissions=points[:, 3], ring_index=points[:, 4].astype(np.int32)) 135 | laser_scan.set_label(label=points[:, 6].astype(np.int32)) 136 | 137 | if index == 0 and FLAGS.plot: 138 | plt.figure(figsize=(30, 3)) 139 | plt.imshow(laser_scan.proj_sem_color) 140 | plt.tight_layout() 141 | plt.show() 142 | 143 | mask = laser_scan.proj_range > 0 # check if the projected depth is positive 144 | laser_scan.proj_range[~mask] = 0.0 145 | laser_scan.proj_xyz[~mask] = 0.0 146 | laser_scan.proj_remission[~mask] = 0.0 147 | laser_scan.proj_sem_label = vfunc(laser_scan.proj_sem_label) # apply class ID mapping 148 | 149 | # create the final data sample with shape (32, 1024, 6) 150 | final_data = np.concatenate([laser_scan.proj_xyz, 151 | laser_scan.proj_remission.reshape((H, W, 1)), 152 | laser_scan.proj_range.reshape((H, W, 1)), 153 | laser_scan.proj_sem_label.reshape((H, W, 1))], 154 | axis=2) 155 | 156 | np.save(os.path.join(FLAGS.output_dir, os.path.basename(pcd_file).split(".")[0]), final_data) 157 | -------------------------------------------------------------------------------- /dataset_convert/semantic-kitti.yaml: -------------------------------------------------------------------------------- 1 | # This file is covered by the LICENSE file in the root of this project. 2 | labels: 3 | 0 : "unlabeled" 4 | 1 : "outlier" 5 | 10: "car" 6 | 11: "bicycle" 7 | 13: "bus" 8 | 15: "motorcycle" 9 | 16: "on-rails" 10 | 18: "truck" 11 | 20: "other-vehicle" 12 | 30: "person" 13 | 31: "bicyclist" 14 | 32: "motorcyclist" 15 | 40: "road" 16 | 44: "parking" 17 | 48: "sidewalk" 18 | 49: "other-ground" 19 | 50: "building" 20 | 51: "fence" 21 | 52: "other-structure" 22 | 60: "lane-marking" 23 | 70: "vegetation" 24 | 71: "trunk" 25 | 72: "terrain" 26 | 80: "pole" 27 | 81: "traffic-sign" 28 | 99: "other-object" 29 | 252: "moving-car" 30 | 253: "moving-bicyclist" 31 | 254: "moving-person" 32 | 255: "moving-motorcyclist" 33 | 256: "moving-on-rails" 34 | 257: "moving-bus" 35 | 258: "moving-truck" 36 | 259: "moving-other-vehicle" 37 | color_map: # bgr 38 | 0 : [0, 0, 0] 39 | 1 : [0, 0, 255] 40 | 10: [245, 150, 100] 41 | 11: [245, 230, 100] 42 | 13: [250, 80, 100] 43 | 15: [150, 60, 30] 44 | 16: [255, 0, 0] 45 | 18: [180, 30, 80] 46 | 20: [255, 0, 0] 47 | 30: [30, 30, 255] 48 | 31: [200, 40, 255] 49 | 32: [90, 30, 150] 50 | 40: [255, 0, 255] 51 | 44: [255, 150, 255] 52 | 48: [75, 0, 75] 53 | 49: [75, 0, 175] 54 | 50: [0, 200, 255] 55 | 51: [50, 120, 255] 56 | 52: [0, 150, 255] 57 | 60: [170, 255, 150] 58 | 70: [0, 175, 0] 59 | 71: [0, 60, 135] 60 | 72: [80, 240, 150] 61 | 80: [150, 240, 255] 62 | 81: [0, 0, 255] 63 | 99: [255, 255, 50] 64 | 252: [245, 150, 100] 65 | 256: [255, 0, 0] 66 | 253: [200, 40, 255] 67 | 254: [30, 30, 255] 68 | 255: [90, 30, 150] 69 | 257: [250, 80, 100] 70 | 258: [180, 30, 80] 71 | 259: [255, 0, 0] 72 | content: # as a ratio with the total number of points 73 | 0: 0.018889854628292943 74 | 1: 0.0002937197336781505 75 | 10: 0.040818519255974316 76 | 11: 0.00016609538710764618 77 | 13: 2.7879693665067774e-05 78 | 15: 0.00039838616015114444 79 | 16: 0.0 80 | 18: 0.0020633612104619787 81 | 20: 0.0016218197275284021 82 | 30: 0.00017698551338515307 83 | 31: 1.1065903904919655e-08 84 | 32: 5.532951952459828e-09 85 | 40: 0.1987493871255525 86 | 44: 0.014717169549888214 87 | 48: 0.14392298360372 88 | 49: 0.0039048553037472045 89 | 50: 0.1326861944777486 90 | 51: 0.0723592229456223 91 | 52: 0.002395131480328884 92 | 60: 4.7084144280367186e-05 93 | 70: 0.26681502148037506 94 | 71: 0.006035012012626033 95 | 72: 0.07814222006271769 96 | 80: 0.002855498193863172 97 | 81: 0.0006155958086189918 98 | 99: 0.009923127583046915 99 | 252: 0.001789309418528068 100 | 253: 0.00012709999297008662 101 | 254: 0.00016059776092534436 102 | 255: 3.745553104802113e-05 103 | 256: 0.0 104 | 257: 0.00011351574470342043 105 | 258: 0.00010157861367183268 106 | 259: 4.3840131989471124e-05 107 | # classes that are indistinguishable from single scan or inconsistent in 108 | # ground truth are mapped to their closest equivalent 109 | learning_map: 110 | 0 : 0 # "unlabeled" 111 | 1 : 0 # "outlier" mapped to "unlabeled" --------------------------mapped 112 | 10: 1 # "car" 113 | 11: 2 # "bicycle" 114 | 13: 5 # "bus" mapped to "other-vehicle" --------------------------mapped 115 | 15: 3 # "motorcycle" 116 | 16: 5 # "on-rails" mapped to "other-vehicle" ---------------------mapped 117 | 18: 4 # "truck" 118 | 20: 5 # "other-vehicle" 119 | 30: 6 # "person" 120 | 31: 7 # "bicyclist" 121 | 32: 8 # "motorcyclist" 122 | 40: 9 # "road" 123 | 44: 10 # "parking" 124 | 48: 11 # "sidewalk" 125 | 49: 12 # "other-ground" 126 | 50: 13 # "building" 127 | 51: 14 # "fence" 128 | 52: 0 # "other-structure" mapped to "unlabeled" ------------------mapped 129 | 60: 9 # "lane-marking" to "road" ---------------------------------mapped 130 | 70: 15 # "vegetation" 131 | 71: 16 # "trunk" 132 | 72: 17 # "terrain" 133 | 80: 18 # "pole" 134 | 81: 19 # "traffic-sign" 135 | 99: 0 # "other-object" to "unlabeled" ----------------------------mapped 136 | 252: 1 # "moving-car" to "car" ------------------------------------mapped 137 | 253: 7 # "moving-bicyclist" to "bicyclist" ------------------------mapped 138 | 254: 6 # "moving-person" to "person" ------------------------------mapped 139 | 255: 8 # "moving-motorcyclist" to "motorcyclist" ------------------mapped 140 | 256: 5 # "moving-on-rails" mapped to "other-vehicle" --------------mapped 141 | 257: 5 # "moving-bus" mapped to "other-vehicle" -------------------mapped 142 | 258: 4 # "moving-truck" to "truck" --------------------------------mapped 143 | 259: 5 # "moving-other"-vehicle to "other-vehicle" ----------------mapped 144 | learning_map_inv: # inverse of previous map 145 | 0: 0 # "unlabeled", and others ignored 146 | 1: 10 # "car" 147 | 2: 11 # "bicycle" 148 | 3: 15 # "motorcycle" 149 | 4: 18 # "truck" 150 | 5: 20 # "other-vehicle" 151 | 6: 30 # "person" 152 | 7: 31 # "bicyclist" 153 | 8: 32 # "motorcyclist" 154 | 9: 40 # "road" 155 | 10: 44 # "parking" 156 | 11: 48 # "sidewalk" 157 | 12: 49 # "other-ground" 158 | 13: 50 # "building" 159 | 14: 51 # "fence" 160 | 15: 70 # "vegetation" 161 | 16: 71 # "trunk" 162 | 17: 72 # "terrain" 163 | 18: 80 # "pole" 164 | 19: 81 # "traffic-sign" 165 | learning_ignore: # Ignore classes 166 | 0: True # "unlabeled", and others ignored 167 | 1: False # "car" 168 | 2: False # "bicycle" 169 | 3: False # "motorcycle" 170 | 4: False # "truck" 171 | 5: False # "other-vehicle" 172 | 6: False # "person" 173 | 7: False # "bicyclist" 174 | 8: False # "motorcyclist" 175 | 9: False # "road" 176 | 10: False # "parking" 177 | 11: False # "sidewalk" 178 | 12: False # "other-ground" 179 | 13: False # "building" 180 | 14: False # "fence" 181 | 15: False # "vegetation" 182 | 16: False # "trunk" 183 | 17: False # "terrain" 184 | 18: False # "pole" 185 | 19: False # "traffic-sign" 186 | split: # sequence numbers 187 | train: 188 | - 0 189 | - 1 190 | - 2 191 | - 3 192 | - 4 193 | - 5 194 | - 6 195 | - 7 196 | - 9 197 | - 10 198 | val: 199 | - 8 200 | test: 201 | - 11 202 | - 12 203 | - 13 204 | - 14 205 | - 15 206 | - 16 207 | - 17 208 | - 18 209 | - 19 210 | - 20 211 | - 21 -------------------------------------------------------------------------------- /dataset_convert/semantic_kitti.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | 4 | import argparse 5 | import numpy as np 6 | import tqdm 7 | import yaml 8 | 9 | import matplotlib.pyplot as plt 10 | 11 | from laserscan_semantic_kitti import SemLaserScan 12 | 13 | CONFIG_FILE = "semantic-kitti.yaml" 14 | 15 | 16 | # Layer extraction for VLP32-C 17 | layers = np.arange(16, 48) 18 | 19 | """ 20 | label_map_to_ika = { 21 | # road 22 | 60: 0, 23 | 40: 0, 24 | 44: 0, 25 | 26 | # sidewalk 27 | 48: 1, 28 | 29 | # building 30 | 50: 2, 31 | 51: 2, 32 | 52: 2, 33 | 34 | # pole 35 | 80: 3, 36 | 81: 3, 37 | 38 | # vegetation 39 | 70: 4, 40 | 71: 4, 41 | 72: 4, 42 | 43 | # person 44 | 30: 5, 45 | 254: 5, 46 | 32: 5, 47 | 31: 5, 48 | 49 | # two-wheeler 50 | 11: 6, 51 | 15: 6, 52 | 253: 6, 53 | 255: 6, 54 | 55 | # car 56 | 252: 7, 57 | 20: 7, 58 | 259: 7, 59 | 10: 7, 60 | 61 | # truck 62 | 18: 8, 63 | 258: 8, 64 | 65 | # bus 66 | 13: 9, 67 | 257: 9, 68 | 69 | # none 70 | 0: 10, 71 | 1: 10, 72 | 256: 10, 73 | 16: 10, 74 | 49: 10, 75 | 99: 10 76 | } 77 | """ 78 | 79 | 80 | if __name__ == '__main__': 81 | parser = argparse.ArgumentParser("./semantic_kitti.py") 82 | parser.add_argument( 83 | '--dataset', '-d', 84 | type=str, 85 | required=True, 86 | help='path to the kitti dataset where the `sequences` directory is. No Default', 87 | ) 88 | parser.add_argument( 89 | '--output_dir', '-p', 90 | type=str, 91 | required=True, 92 | help='output path to the PCL segmentation repository. No Default', 93 | ) 94 | parser.add_argument( 95 | '-v', 96 | action='store_true', 97 | help='use only 32 layers of KITTI lidar data in order to match a VLP32 laser scanner ', 98 | ) 99 | 100 | FLAGS, unparsed = parser.parse_known_args() 101 | 102 | # open config file 103 | print("Opening config file %s" % CONFIG_FILE) 104 | config = yaml.safe_load(open(CONFIG_FILE, 'r')) 105 | 106 | for split in ["train", "val", "test"]: 107 | sequences = config["split"][split] 108 | 109 | scans = [] 110 | labels = [] 111 | 112 | for sequence in sequences: 113 | # path which contains the pointclouds for each scan 114 | scan_paths = os.path.join(FLAGS.dataset, "sequences", str(sequence).zfill(2), "velodyne") 115 | 116 | if os.path.isdir(scan_paths): 117 | print("Sequence folder exists! Using sequence from %s" % scan_paths) 118 | else: 119 | print("Sequence folder doesn't exist! Exiting...") 120 | quit() 121 | 122 | scan_names = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(scan_paths)) for f in fn] 123 | scan_names.sort() 124 | scans = scans + scan_names 125 | 126 | for sequence in sequences: 127 | # path which contains the labels for each scan 128 | label_paths = os.path.join(FLAGS.dataset, "sequences", str(sequence).zfill(2), "labels") 129 | 130 | if os.path.isdir(label_paths): 131 | print("Labels folder exists! Using labels from %s" % label_paths) 132 | else: 133 | print("Labels folder doesn't exist! Exiting...") 134 | quit() 135 | 136 | label_names = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(label_paths)) for f in fn] 137 | label_names.sort() 138 | labels = labels + label_names 139 | 140 | print("number of scans : ", len(scans)) 141 | print("number of labels : ", len(labels)) 142 | 143 | # apply learning map 144 | learning_map = config["learning_map"] 145 | vfunc = np.vectorize(learning_map.get) 146 | 147 | color_dict = config["color_map"] # maps numeric labels in .label file to a bgr color 148 | nclasses = len(color_dict) # number of classes 149 | 150 | laser_scan = SemLaserScan(nclasses, color_dict, project=True) # create a scan with all 64 layers of KITTI lidar data 151 | 152 | for index, (scan, label) in tqdm.tqdm(enumerate(zip(scans, labels)), total=len(scans)): 153 | laser_scan.open_scan(scan) 154 | laser_scan.open_label(label) 155 | 156 | if index == 0: 157 | plt.figure(figsize=(30, 3)) 158 | plt.imshow(laser_scan.proj_sem_color) 159 | plt.tight_layout() 160 | plt.show() 161 | 162 | mask = laser_scan.proj_range > 0 # check if the projected depth is positive 163 | laser_scan.proj_range[~mask] = 0.0 164 | laser_scan.proj_xyz[~mask] = 0.0 165 | laser_scan.proj_remission[~mask] = 0.0 166 | laser_scan.proj_sem_label = vfunc(laser_scan.proj_sem_label) # map class labels to values between 0 and 10 167 | 168 | # create the final data sample with shape (64,1024,6) 169 | final_data = np.concatenate([laser_scan.proj_xyz, 170 | laser_scan.proj_remission.reshape((64, 1024, 1)), 171 | laser_scan.proj_range.reshape((64, 1024, 1)), 172 | laser_scan.proj_sem_label.reshape((64, 1024, 1))], 173 | axis=2) 174 | 175 | if FLAGS.v: 176 | vlp_32_data = final_data[layers, :, :] 177 | np.save(os.path.join(FLAGS.output_dir, "converted_dataset", split, str(index)), vlp_32_data) 178 | else: 179 | np.save(os.path.join(FLAGS.output_dir, "converted_dataset", split, str(index)), final_data) 180 | -------------------------------------------------------------------------------- /dataset_convert/semantic_kitti_sequence.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | 4 | import argparse 5 | import numpy as np 6 | import yaml 7 | import tqdm 8 | 9 | import matplotlib.pyplot as plt 10 | 11 | from laserscan_semantic_kitti import SemLaserScan 12 | 13 | label_map_to_ika = { 14 | # road 15 | 60: 0, 16 | 40: 0, 17 | 44: 0, 18 | 19 | # sidewalk 20 | 48: 1, 21 | 22 | # building 23 | 50: 2, 24 | 51: 2, 25 | 52: 2, 26 | 27 | # pole 28 | 80: 3, 29 | 81: 3, 30 | 31 | # vegetation 32 | 70: 4, 33 | 71: 4, 34 | 72: 4, 35 | 36 | # person 37 | 30: 5, 38 | 254: 5, 39 | 32: 5, 40 | 31: 5, 41 | 42 | # two-wheeler 43 | 11: 6, 44 | 15: 6, 45 | 253: 6, 46 | 255: 6, 47 | 48 | # car 49 | 252: 7, 50 | 20: 7, 51 | 259: 7, 52 | 10: 7, 53 | 54 | # truck 55 | 18: 8, 56 | 258: 8, 57 | 58 | # bus 59 | 13: 9, 60 | 257: 9, 61 | 62 | # none 63 | 0: 10, 64 | 1: 10, 65 | 256: 10, 66 | 16: 10, 67 | 49: 10, 68 | 99: 10 69 | } 70 | 71 | label_map = { 72 | 0: 0, # "unlabeled" 73 | 1: 0, # "outlier" mapped to "unlabeled" --------------------------mapped 74 | 10: 1, # "car" 75 | 11: 2, # "bicycle" 76 | 13: 5, # "bus" mapped to "other-vehicle" --------------------------mapped 77 | 15: 3, # "motorcycle" 78 | 16: 5, # "on-rails" mapped to "other-vehicle" ---------------------mapped 79 | 18: 4, # "truck" 80 | 20: 5, # "other-vehicle" 81 | 30: 6, # "person" 82 | 31: 7, # "bicyclist" 83 | 32: 8, # "motorcyclist" 84 | 40: 9, # "road" 85 | 44: 10, # "parking" 86 | 48: 11, # "sidewalk" 87 | 49: 12, # "other-ground" 88 | 50: 13, # "building" 89 | 51: 14, # "fence" 90 | 52: 0, # "other-structure" mapped to "unlabeled" ------------------mapped 91 | 60: 9, # "lane-marking" to "road" ---------------------------------mapped 92 | 70: 15, # "vegetation" 93 | 71: 16, # "trunk" 94 | 72: 17, # "terrain" 95 | 80: 18, # "pole" 96 | 81: 19, # "traffic-sign" 97 | 99: 0, # "other-object" to "unlabeled" ----------------------------mapped 98 | 252: 1, # "moving-car" to "car" ------------------------------------mapped 99 | 253: 7, # "moving-bicyclist" to "bicyclist" ------------------------mapped 100 | 254: 6, # "moving-person" to "person" ------------------------------mapped 101 | 255: 8, # "moving-motorcyclist" to "motorcyclist" ------------------mapped 102 | 256: 5, # "moving-on-rails" mapped to "other-vehicle" --------------mapped 103 | 257: 5, # "moving-bus" mapped to "other-vehicle" -------------------mapped 104 | 258: 4, # "moving-truck" to "truck" --------------------------------mapped 105 | 259: 5, # "moving-other"-vehicle to "other-vehicle" ----------------mapped 106 | } 107 | 108 | if __name__ == '__main__': 109 | parser = argparse.ArgumentParser("./semantic_kitti_sequence.py") 110 | parser.add_argument( 111 | '--dataset', '-d', 112 | type=str, 113 | required=True, 114 | help='path to the kitti dataset where the `sequences` directory is. No Default', 115 | ) 116 | parser.add_argument( 117 | '--sequence', '-s', 118 | type=str, 119 | default="00", 120 | required=False, 121 | help='sequence to project. Defaults to %(default)s', 122 | ) 123 | parser.add_argument( 124 | '--output_dir', '-p', 125 | type=str, 126 | required=True, 127 | help='path to the PCL segmentation repository. No Default', 128 | ) 129 | parser.add_argument( 130 | '-v', 131 | action='store_true', 132 | help='produce data samples for validation set', 133 | ) 134 | 135 | FLAGS, unparsed = parser.parse_known_args() 136 | 137 | config = "semantic-kitti.yaml" # configuration file 138 | CFG = yaml.safe_load(open(config, 'r')) 139 | 140 | scan_paths = os.path.join(FLAGS.dataset, "sequences", FLAGS.sequence, "velodyne") # path which contains the pointclouds for each scan 141 | 142 | if os.path.isdir(scan_paths): 143 | print("Sequence folder exists! Using sequence from %s" % scan_paths) 144 | else: 145 | print("Sequence folder doesn't exist! Exiting...") 146 | quit() 147 | 148 | scan_names = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(scan_paths)) for f in fn] 149 | scan_names.sort() 150 | print(len(scan_names)) 151 | 152 | label_paths = os.path.join(FLAGS.dataset, "sequences", FLAGS.sequence, "labels") # path which contains the labels for each scan 153 | if os.path.isdir(label_paths): 154 | print("Labels folder exists! Using labels from %s" % label_paths) 155 | else: 156 | print("Labels folder doesn't exist! Exiting...") 157 | quit() 158 | 159 | label_names = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(label_paths)) for f in fn] 160 | label_names.sort() 161 | print(len(label_names)) 162 | 163 | color_dict = CFG["color_map"] # maps numeric labels in .label file to a bgr color 164 | nclasses = len(color_dict) # number of classes 165 | laser_scan = SemLaserScan(nclasses, color_dict, project=True) # create a scan 166 | 167 | vfunc = np.vectorize(label_map.get) 168 | 169 | for index, (scan, label) in tqdm.tqdm(enumerate(zip(scan_names, label_names)), total=len(scan_names)): 170 | laser_scan.open_scan(scan) 171 | laser_scan.open_label(label) 172 | 173 | if index == 0: 174 | plt.figure(figsize=(30, 3)) 175 | plt.imshow(laser_scan.proj_sem_color) 176 | plt.tight_layout() 177 | plt.show() 178 | 179 | mask = laser_scan.proj_range > 0 # check if the projected depth is positive 180 | laser_scan.proj_range[~mask] = 0.0 181 | laser_scan.proj_xyz[~mask] = 0.0 182 | laser_scan.proj_remission[~mask] = 0.0 183 | laser_scan.proj_sem_label = vfunc(laser_scan.proj_sem_label) # map class labels to values between 0 and 33 184 | 185 | # create the final data sample with shape (64, 1024, 6) 186 | final_data = np.concatenate([laser_scan.proj_xyz, 187 | laser_scan.proj_remission.reshape((64, 1024, 1)), 188 | laser_scan.proj_range.reshape((64, 1024, 1)), 189 | laser_scan.proj_sem_label.reshape((64, 1024, 1))], axis=2) 190 | 191 | if not FLAGS.v: 192 | # save as npy file to the training set 193 | np.save(os.path.join(FLAGS.output_dir, "converted_dataset/train/" + str(index)), final_data) 194 | else: 195 | # save as npy file to the validation set 196 | np.save(os.path.join(FLAGS.output_dir + "converted_dataset/val/" + str(index)), final_data) 197 | 198 | 199 | -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/0.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/1.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/10.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/10.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/11.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/11.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/12.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/12.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/13.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/13.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/14.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/14.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/15.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/15.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/16.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/17.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/17.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/18.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/18.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/19.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/19.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/2.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/20.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/20.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/21.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/21.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/22.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/22.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/23.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/23.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/24.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/24.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/25.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/25.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/26.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/26.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/27.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/27.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/28.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/28.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/29.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/29.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/3.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/30.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/30.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/31.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/31.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/4.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/5.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/5.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/6.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/6.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/7.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/7.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/8.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/train/9.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/train/9.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/val/25611.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/val/25611.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/val/25612.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/val/25612.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/val/25613.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/val/25613.npy -------------------------------------------------------------------------------- /dataset_samples/nuscenes/val/25614.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/nuscenes/val/25614.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960637314959526.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960637314959526.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960637514871120.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960637514871120.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960637614809990.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960637614809990.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960637814712524.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960637814712524.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960637914665699.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960637914665699.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638014621019.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638014621019.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638114571095.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638114571095.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638214528561.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638214528561.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638314473152.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638314473152.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638514373779.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638514373779.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638614326477.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638614326477.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638714277029.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638714277029.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638814225197.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638814225197.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960638914183378.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960638914183378.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639014128208.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639014128208.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639114081383.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639114081383.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639214035511.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639214035511.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639313982010.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639313982010.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639413947344.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639413947344.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639513887405.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639513887405.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639613846302.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639613846302.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639713784933.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639713784933.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639813746929.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639813746929.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960639913702011.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960639913702011.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640013648272.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640013648272.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640213545799.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640213545799.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640313498735.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640313498735.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640413448095.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640413448095.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640513404846.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640513404846.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640613354445.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640613354445.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640713299513.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640713299513.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/train/1603960640813255310.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/train/1603960640813255310.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/val/03bag_1565340659535964_labled.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/val/03bag_1565340659535964_labled.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/val/03bag_1565340704534739_labled.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/val/03bag_1565340704534739_labled.npy -------------------------------------------------------------------------------- /dataset_samples/sample_dataset/val/04bag_1565009986967384_labled.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/sample_dataset/val/04bag_1565009986967384_labled.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/0.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/1.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/10.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/10.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/11.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/11.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/12.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/12.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/13.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/13.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/14.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/14.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/15.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/15.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/16.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/17.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/17.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/18.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/18.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/19.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/19.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/2.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/20.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/20.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/3.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/4.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/5.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/5.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/6.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/6.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/7.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/7.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/8.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/train/9.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/train/9.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/val/0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/val/0.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/val/1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/val/1.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/val/2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/val/2.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/val/3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/val/3.npy -------------------------------------------------------------------------------- /dataset_samples/semantic_kitti/val/4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/dataset_samples/semantic_kitti/val/4.npy -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:2.9.1-gpu 2 | 3 | # install Python package dependencies 4 | COPY requirements.txt . 5 | RUN pip install -r requirements.txt && \ 6 | rm requirements.txt 7 | -------------------------------------------------------------------------------- /docker/docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker build --tag pcl_segmentation . -------------------------------------------------------------------------------- /docker/docker_eval.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$(cd -P "$(dirname "$0")" && pwd)" 4 | 5 | docker run \ 6 | --gpus all \ 7 | --name='pcl_segmentation' \ 8 | --rm \ 9 | --tty \ 10 | --user "$(id -u):$(id -g)" \ 11 | --volume $DIR/../:/src \ 12 | pcl_segmentation \ 13 | python3 /src/pcl_segmentation/eval.py \ 14 | --data_path="/src/sample_dataset" \ 15 | --eval_dir="/src/output/eval" \ 16 | --path_to_model="/src/output/model" \ 17 | --image_set="val" \ 18 | --model=squeezesegv2 -------------------------------------------------------------------------------- /docker/docker_inference.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$(cd -P "$(dirname "$0")" && pwd)" 4 | 5 | docker run \ 6 | --gpus all \ 7 | --name='pcl_segmentation' \ 8 | --rm \ 9 | --tty \ 10 | --user "$(id -u):$(id -g)" \ 11 | --volume $DIR/../:/src \ 12 | pcl_segmentation \ 13 | python3 /src/pcl_segmentation/inference.py \ 14 | --input_path="/src/sample_dataset/train/*.npy" \ 15 | --output_dir="/src/output/prediction" \ 16 | --path_to_model="/src/output/model" \ 17 | --model=squeezesegv2 -------------------------------------------------------------------------------- /docker/docker_run_all_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$(cd -P "$(dirname "$0")" && pwd)" 4 | 5 | CUDA_VISIBLE_DEVICES=0 6 | 7 | docker run \ 8 | --gpus all \ 9 | --name='pcl_segmentation' \ 10 | --env CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES \ 11 | --rm \ 12 | --tty \ 13 | --user "$(id -u):$(id -g)" \ 14 | --volume $DIR/../:/src \ 15 | pcl_segmentation \ 16 | python3 /src/pcl_segmentation/train.py \ 17 | --data_path="/src/nuscenes_dataset/nuscenes" \ 18 | --train_dir="/src/output" \ 19 | --epochs=500 \ 20 | --model=squeezesegv2 \ 21 | --config=squeezesegv2nuscenes -------------------------------------------------------------------------------- /docker/docker_train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$(cd -P "$(dirname "$0")" && pwd)" 4 | 5 | docker run \ 6 | --gpus all \ 7 | --name='pcl_segmentation' \ 8 | --rm \ 9 | --tty \ 10 | --user "$(id -u):$(id -g)" \ 11 | --volume $DIR/../:/src \ 12 | pcl_segmentation \ 13 | python3 /src/pcl_segmentation/train.py \ 14 | --data_path="/src/sample_dataset" \ 15 | --train_dir="/src/output" \ 16 | --epochs=5 \ 17 | --model=squeezesegv2 -------------------------------------------------------------------------------- /docker/requirements.txt: -------------------------------------------------------------------------------- 1 | scipy 2 | tqdm 3 | easydict 4 | pillow 5 | opencv-python 6 | matplotlib -------------------------------------------------------------------------------- /pcl_segmentation/configs/Darknet21.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | """Model configuration for Darknet21""" 26 | import numpy as np 27 | from easydict import EasyDict 28 | 29 | 30 | def Darknet21(): 31 | mc = EasyDict() 32 | 33 | mc.CLASSES = ['Road', 34 | 'Sidewalk', 35 | 'Building', 36 | 'Pole', 37 | 'Vegetation', 38 | 'Person', 39 | 'TwoWheeler', 40 | 'Car', 41 | 'Truck', 42 | 'Bus', 43 | "None"] 44 | mc.NUM_CLASS = len(mc.CLASSES) 45 | mc.CLS_2_ID = dict(zip(mc.CLASSES, range(len(mc.CLASSES)))) 46 | mc.CLS_LOSS_WEIGHT = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) 47 | mc.CLS_COLOR_MAP = np.array([[128, 64, 128], # Road 48 | [244, 35, 232], # Sidewalk 49 | [ 70, 70, 70], # Building 50 | [153, 153, 153], # Pole 51 | [107, 142, 35], # Vegetation 52 | [220, 20, 60], # Person 53 | [255, 0, 0], # Two Wheeler 54 | [ 0, 0, 142], # Car 55 | [ 0, 0, 70], # Truck 56 | [ 0, 60, 100], # Bus 57 | [ 0, 0, 0] # None 58 | ]) / 255.0 59 | 60 | # Input Shape 61 | mc.BATCH_SIZE = 16 62 | mc.AZIMUTH_LEVEL = 240 63 | mc.ZENITH_LEVEL = 32 64 | mc.NUM_FEATURES = 6 65 | 66 | # Loss 67 | mc.USE_FOCAL_LOSS = False # either use focal loss or sparse categorical cross entropy 68 | mc.FOCAL_GAMMA = 2.0 69 | mc.CLS_LOSS_COEF = 15.0 70 | mc.DENOM_EPSILON = 1e-12 # small value used in denominator to prevent division by 0 71 | 72 | # Gradient Decent 73 | mc.LEARNING_RATE = 0.01 74 | mc.LR_DECAY_STEPS = 500 75 | mc.LR_DECAY_FACTOR = 0.99 76 | mc.MAX_GRAD_NORM = 1.0 77 | 78 | # Network 79 | mc.DROP_RATE = 0.01 80 | mc.BN_MOMENTUM = 0.9 81 | mc.NUM_LAYERS = 21 82 | mc.OUTPUT_STRIDE = 16 # Output stride only horizontally 83 | 84 | # Dataset 85 | mc.DATA_AUGMENTATION = True 86 | mc.RANDOM_FLIPPING = True 87 | mc.SHIFT_UP_DOWN = 0 88 | mc.SHIFT_LEFT_RIGHT = 70 89 | 90 | # x, y, z, intensity, distance 91 | mc.INPUT_MEAN = np.array([[[24.810, 0.819, 0.000, 16.303, 25.436]]]) 92 | mc.INPUT_STD = np.array([[[30.335, 7.807, 2.058, 25.208, 30.897]]]) 93 | 94 | return mc 95 | -------------------------------------------------------------------------------- /pcl_segmentation/configs/Darknet53.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | """Model configuration for Darknet52""" 26 | import numpy as np 27 | from easydict import EasyDict 28 | 29 | 30 | def Darknet53(): 31 | mc = EasyDict() 32 | 33 | mc.CLASSES = ['Road', 34 | 'Sidewalk', 35 | 'Building', 36 | 'Pole', 37 | 'Vegetation', 38 | 'Person', 39 | 'TwoWheeler', 40 | 'Car', 41 | 'Truck', 42 | 'Bus', 43 | "None"] 44 | mc.NUM_CLASS = len(mc.CLASSES) 45 | mc.CLS_2_ID = dict(zip(mc.CLASSES, range(len(mc.CLASSES)))) 46 | mc.CLS_LOSS_WEIGHT = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) 47 | mc.CLS_COLOR_MAP = np.array([[128, 64, 128], # Road 48 | [244, 35, 232], # Sidewalk 49 | [ 70, 70, 70], # Building 50 | [153, 153, 153], # Pole 51 | [107, 142, 35], # Vegetation 52 | [220, 20, 60], # Person 53 | [255, 0, 0], # Two Wheeler 54 | [ 0, 0, 142], # Car 55 | [ 0, 0, 70], # Truck 56 | [ 0, 60, 100], # Bus 57 | [ 0, 0, 0] # None 58 | ]) / 255.0 59 | 60 | # Input Shape 61 | mc.BATCH_SIZE = 16 62 | mc.AZIMUTH_LEVEL = 240 63 | mc.ZENITH_LEVEL = 32 64 | mc.NUM_FEATURES = 6 65 | 66 | # Loss 67 | mc.USE_FOCAL_LOSS = False # either use focal loss or sparse categorical cross entropy 68 | mc.FOCAL_GAMMA = 2.0 69 | mc.CLS_LOSS_COEF = 15.0 70 | mc.DENOM_EPSILON = 1e-12 # small value used in denominator to prevent division by 0 71 | 72 | # Gradient Decent 73 | mc.LEARNING_RATE = 0.005 74 | mc.LR_DECAY_STEPS = 500 75 | mc.LR_DECAY_FACTOR = 0.99 76 | mc.MAX_GRAD_NORM = 1.0 77 | 78 | # Network 79 | mc.DROP_RATE = 0.01 80 | mc.BN_MOMENTUM = 0.9 81 | mc.NUM_LAYERS = 53 82 | mc.OUTPUT_STRIDE = 16 # Output stride only horizontally 83 | 84 | # Dataset 85 | mc.DATA_AUGMENTATION = True 86 | mc.RANDOM_FLIPPING = True 87 | mc.SHIFT_UP_DOWN = 0 88 | mc.SHIFT_LEFT_RIGHT = 70 89 | 90 | # x, y, z, intensity, distance 91 | mc.INPUT_MEAN = np.array([[[24.810, 0.819, 0.000, 16.303, 25.436]]]) 92 | mc.INPUT_STD = np.array([[[30.335, 7.807, 2.058, 25.208, 30.897]]]) 93 | 94 | return mc 95 | -------------------------------------------------------------------------------- /pcl_segmentation/configs/Darknet53Kitti.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | """Model configuration for Darknet52""" 26 | import numpy as np 27 | from easydict import EasyDict 28 | 29 | def rgb(bgr): 30 | return [bgr[2], bgr[1], bgr[0]] 31 | 32 | def Darknet53Kitti(): 33 | mc = EasyDict() 34 | 35 | mc.CLASSES = ["None", 36 | "car", 37 | "bicycle", 38 | "motorcycle", 39 | "truck", 40 | "other-vehicle", 41 | "person", 42 | "bicyclist", 43 | "motorcyclist", 44 | "road", 45 | "parking", 46 | "sidewalk", 47 | "other-ground", 48 | "building", 49 | "fence", 50 | "vegetation", 51 | "trunk", 52 | "terrain", 53 | "pole", 54 | "traffic-sign"] 55 | 56 | 57 | mc.NUM_CLASS = len(mc.CLASSES) 58 | mc.CLS_2_ID = dict(zip(mc.CLASSES, range(len(mc.CLASSES)))) 59 | mc.CLS_LOSS_WEIGHT = np.ones(mc.NUM_CLASS) 60 | 61 | color_map = {0: rgb([0, 0, 0]), 62 | 1: rgb([245, 150, 100]), 63 | 2: rgb([245, 230, 100]), 64 | 3: rgb([150, 60, 30]), 65 | 4: rgb([180, 30, 80]), 66 | 5: rgb([255, 0, 0]), 67 | 6: rgb([30, 30, 255]), 68 | 7: rgb([200, 40, 255]), 69 | 8: rgb([90, 30, 150]), 70 | 9: rgb([255, 0, 255]), 71 | 10: rgb([255, 150, 255]), 72 | 11: rgb([75, 0, 75]), 73 | 12: rgb([75, 0, 175]), 74 | 13: rgb([0, 200, 255]), 75 | 14: rgb([50, 120, 255]), 76 | 15: rgb([0, 175, 0]), 77 | 16: rgb([0, 60, 135]), 78 | 17: rgb([80, 240, 150]), 79 | 18: rgb([150, 240, 255]), 80 | 19: rgb([0, 0, 255]) 81 | } 82 | 83 | mc.CLS_COLOR_MAP = np.zeros((mc.NUM_CLASS, 3), dtype=np.float32) 84 | for key, value in color_map.items(): 85 | mc.CLS_COLOR_MAP[key] = np.array(value, np.float32) / 255.0 86 | 87 | # Input Shape 88 | mc.BATCH_SIZE = 16 89 | mc.AZIMUTH_LEVEL = 1024 90 | mc.ZENITH_LEVEL = 64 91 | mc.NUM_FEATURES = 6 92 | 93 | # Loss 94 | mc.USE_FOCAL_LOSS = False # either use focal loss or sparse categorical cross entropy 95 | mc.FOCAL_GAMMA = 2.0 96 | mc.CLS_LOSS_COEF = 15.0 97 | mc.DENOM_EPSILON = 1e-12 # small value used in denominator to prevent division by 0 98 | 99 | # Gradient Decent 100 | mc.LEARNING_RATE = 0.001 101 | mc.LR_DECAY_STEPS = 500 102 | mc.LR_DECAY_FACTOR = 0.99 103 | mc.MAX_GRAD_NORM = 100.0 104 | 105 | # Network 106 | mc.DROP_RATE = 0.01 107 | mc.BN_MOMENTUM = 0.9 108 | mc.NUM_LAYERS = 53 109 | mc.OUTPUT_STRIDE = 16 # Output stride only horizontally 110 | 111 | # Dataset 112 | mc.DATA_AUGMENTATION = True 113 | mc.RANDOM_FLIPPING = True 114 | mc.SHIFT_UP_DOWN = 0 115 | mc.SHIFT_LEFT_RIGHT = 70 116 | 117 | # x, y, z, intensity, distance 118 | mc.INPUT_MEAN = np.array([[[-0.047, 0.365, -0.855, 0.2198, 8.3568]]]) 119 | mc.INPUT_STD = np.array([[[10.154, 7.627, 0.8651, 0.1764, 9.6474]]]) 120 | 121 | return mc 122 | -------------------------------------------------------------------------------- /pcl_segmentation/configs/SqueezeSegV2.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | """Model configuration for SqueezeSeg""" 26 | import numpy as np 27 | from easydict import EasyDict 28 | 29 | 30 | def SqueezeSegV2Config(): 31 | mc = EasyDict() 32 | 33 | mc.CLASSES = ["Road", 34 | "Sidewalk", 35 | "Building", 36 | "Pole", 37 | "Vegetation", 38 | "Person", 39 | "Two-wheeler", 40 | "Car", 41 | "Truck", 42 | "Bus", 43 | "None"] 44 | mc.NUM_CLASS = len(mc.CLASSES) 45 | mc.CLS_2_ID = dict(zip(mc.CLASSES, range(len(mc.CLASSES)))) 46 | mc.CLS_LOSS_WEIGHT = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) 47 | color_map = {0: [128, 64, 128], # Road 48 | 1: [244, 35, 232], # Sidewalk 49 | 2: [70, 70, 70], # Building 50 | 3: [153, 153, 153], # Pole 51 | 4: [107, 142, 35], # Vegetation 52 | 5: [220, 20, 60], # Person 53 | 6: [255, 0, 0], # Two-Wheeler 54 | 7: [0, 0, 142], # Car 55 | 8: [0, 0, 70], # Truck 56 | 9: [0, 60, 100], # Bus 57 | 10:[0, 0, 0] # None 58 | } 59 | 60 | mc.CLS_COLOR_MAP = np.zeros((mc.NUM_CLASS, 3), dtype=np.float32) 61 | for key, value in color_map.items(): 62 | mc.CLS_COLOR_MAP[key] = np.array(value, np.float32) / 255.0 63 | 64 | # Input Shape 65 | mc.BATCH_SIZE = 32 66 | mc.AZIMUTH_LEVEL = 240 67 | mc.ZENITH_LEVEL = 32 68 | mc.NUM_FEATURES = 6 69 | 70 | # Loss 71 | mc.USE_FOCAL_LOSS = False # either use focal loss or sparse categorical cross entropy 72 | mc.FOCAL_GAMMA = 2.0 73 | mc.CLS_LOSS_COEF = 15.0 74 | mc.DENOM_EPSILON = 1e-12 # small value used in denominator to prevent division by 0 75 | 76 | # Gradient Decent 77 | mc.LEARNING_RATE = 0.003 78 | mc.LR_DECAY_STEPS = 1000 79 | mc.LR_DECAY_FACTOR = 0.97 80 | mc.MAX_GRAD_NORM = 100.0 81 | 82 | # Network 83 | mc.L2_WEIGHT_DECAY = 0.05 84 | mc.DROP_RATE = 0.1 85 | mc.BN_MOMENTUM = 0.99 86 | mc.REDUCTION = 16 87 | 88 | # Dataset 89 | mc.DATA_AUGMENTATION = True 90 | mc.RANDOM_FLIPPING = True 91 | mc.SHIFT_UP_DOWN = 0 92 | mc.SHIFT_LEFT_RIGHT = 70 93 | 94 | # x, y, z, intensity, distance 95 | # ika Dataset 96 | mc.INPUT_MEAN = np.array([[[24.810, 0.819, 0.000, 16.303, 25.436]]]) 97 | mc.INPUT_STD = np.array([[[30.335, 7.807, 2.058, 25.208, 30.897]]]) 98 | 99 | return mc 100 | -------------------------------------------------------------------------------- /pcl_segmentation/configs/SqueezeSegV2Kitti.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | """Model configuration for SqueezeSeg""" 26 | import numpy as np 27 | from easydict import EasyDict 28 | 29 | def rgb(bgr): 30 | return [bgr[2], bgr[1], bgr[0]] 31 | 32 | def SqueezeSegV2KittiConfig(): 33 | mc = EasyDict() 34 | 35 | mc.CLASSES = ["None", 36 | "car", 37 | "bicycle", 38 | "motorcycle", 39 | "truck", 40 | "other-vehicle", 41 | "person", 42 | "bicyclist", 43 | "motorcyclist", 44 | "road", 45 | "parking", 46 | "sidewalk", 47 | "other-ground", 48 | "building", 49 | "fence", 50 | "vegetation", 51 | "trunk", 52 | "terrain", 53 | "pole", 54 | "traffic-sign"] 55 | 56 | mc.NUM_CLASS = len(mc.CLASSES) 57 | mc.CLS_2_ID = dict(zip(mc.CLASSES, range(len(mc.CLASSES)))) 58 | mc.CLS_LOSS_WEIGHT = np.ones(mc.NUM_CLASS) 59 | 60 | color_map = {0: rgb([0, 0, 0]), 61 | 1: rgb([245, 150, 100]), 62 | 2: rgb([245, 230, 100]), 63 | 3: rgb([150, 60, 30]), 64 | 4: rgb([180, 30, 80]), 65 | 5: rgb([255, 0, 0]), 66 | 6: rgb([30, 30, 255]), 67 | 7: rgb([200, 40, 255]), 68 | 8: rgb([90, 30, 150]), 69 | 9: rgb([255, 0, 255]), 70 | 10: rgb([255, 150, 255]), 71 | 11: rgb([75, 0, 75]), 72 | 12: rgb([75, 0, 175]), 73 | 13: rgb([0, 200, 255]), 74 | 14: rgb([50, 120, 255]), 75 | 15: rgb([0, 175, 0]), 76 | 16: rgb([0, 60, 135]), 77 | 17: rgb([80, 240, 150]), 78 | 18: rgb([150, 240, 255]), 79 | 19: rgb([0, 0, 255]) 80 | } 81 | 82 | mc.CLS_COLOR_MAP = np.zeros((mc.NUM_CLASS, 3), dtype=np.float32) 83 | for key, value in color_map.items(): 84 | mc.CLS_COLOR_MAP[key] = np.array(value, np.float32) / 255.0 85 | 86 | # Input Shape 87 | mc.BATCH_SIZE = 64 88 | mc.AZIMUTH_LEVEL = 1024 89 | mc.ZENITH_LEVEL = 64 90 | mc.NUM_FEATURES = 6 91 | 92 | # Loss 93 | mc.USE_FOCAL_LOSS = False # either use focal loss or sparse categorical cross entropy 94 | mc.FOCAL_GAMMA = 2.0 95 | mc.CLS_LOSS_COEF = 15.0 96 | mc.DENOM_EPSILON = 1e-12 # small value used in denominator to prevent division by 0 97 | 98 | # Gradient Decent 99 | mc.LEARNING_RATE = 0.001 100 | mc.LR_DECAY_STEPS = 500 101 | mc.LR_DECAY_FACTOR = 0.99 102 | mc.MAX_GRAD_NORM = 100.0 103 | 104 | # Network 105 | mc.L2_WEIGHT_DECAY = 0.05 106 | mc.DROP_RATE = 0.1 107 | mc.BN_MOMENTUM = 0.9 108 | mc.REDUCTION = 16 109 | 110 | # Dataset 111 | mc.DATA_AUGMENTATION = True 112 | mc.RANDOM_FLIPPING = True 113 | mc.SHIFT_UP_DOWN = 0 114 | mc.SHIFT_LEFT_RIGHT = 70 115 | 116 | # x, y, z, intensity, distance 117 | mc.INPUT_MEAN = np.array([[[-0.047, 0.365, -0.855, 0.2198, 8.3568]]]) 118 | mc.INPUT_STD = np.array([[[10.154, 7.627, 0.8651, 0.1764, 9.6474]]]) 119 | 120 | return mc 121 | -------------------------------------------------------------------------------- /pcl_segmentation/configs/SqueezeSegV2NuScenes.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | """Model configuration for SqueezeSeg""" 26 | import numpy as np 27 | from easydict import EasyDict 28 | 29 | 30 | def SqueezeSegV2ConfigNuScenes(): 31 | mc = EasyDict() 32 | 33 | mc.CLASSES = ["Road", 34 | "Sidewalk", 35 | "Building", 36 | "Pole", 37 | "Vegetation", 38 | "Person", 39 | "Two-wheeler", 40 | "Car", 41 | "Truck", 42 | "Bus", 43 | "None"] 44 | 45 | mc.NUM_CLASS = len(mc.CLASSES) 46 | mc.CLS_2_ID = dict(zip(mc.CLASSES, range(len(mc.CLASSES)))) 47 | 48 | mc.CLS_LOSS_WEIGHT = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0]) 49 | color_map = {0: [128, 64, 128], # Road 50 | 1: [244, 35, 232], # Sidewalk 51 | 2: [70, 70, 70], # Building 52 | 3: [153, 153, 153], # Pole 53 | 4: [107, 142, 35], # Vegetation 54 | 5: [220, 20, 60], # Person 55 | 6: [255, 0, 0], # Two-Wheeler 56 | 7: [0, 0, 142], # Car 57 | 8: [0, 0, 70], # Truck 58 | 9: [0, 60, 100], # Bus 59 | 10:[0, 0, 0] # None 60 | } 61 | 62 | mc.CLS_COLOR_MAP = np.zeros((mc.NUM_CLASS, 3), dtype=np.float32) 63 | for key, value in color_map.items(): 64 | mc.CLS_COLOR_MAP[key] = np.array(value, np.float32) / 255.0 65 | 66 | # Input Shape 67 | mc.BATCH_SIZE = 32 68 | mc.AZIMUTH_LEVEL = 1024 69 | mc.ZENITH_LEVEL = 32 70 | mc.NUM_FEATURES = 6 71 | 72 | # Loss 73 | mc.USE_FOCAL_LOSS = False # either use focal loss or sparse categorical cross entropy 74 | mc.FOCAL_GAMMA = 2.0 75 | mc.CLS_LOSS_COEF = 15.0 76 | mc.DENOM_EPSILON = 1e-12 # small value used in denominator to prevent division by 0 77 | 78 | # Gradient Decent 79 | mc.LEARNING_RATE = 0.003 80 | mc.LR_DECAY_STEPS = 1000 81 | mc.LR_DECAY_FACTOR = 0.99 82 | mc.MAX_GRAD_NORM = 100.0 83 | 84 | # Network 85 | mc.L2_WEIGHT_DECAY = 0.05 86 | mc.DROP_RATE = 0.1 87 | mc.BN_MOMENTUM = 0.99 88 | mc.REDUCTION = 16 89 | 90 | # Dataset 91 | mc.DATA_AUGMENTATION = True 92 | mc.RANDOM_FLIPPING = True 93 | mc.SHIFT_UP_DOWN = 0 94 | mc.SHIFT_LEFT_RIGHT = 70 95 | 96 | # x, y, z, intensity, distance 97 | # ika Dataset 98 | mc.INPUT_MEAN = np.array([[[-0.1090, -0.1645, -0.6275, 17.2574, 11.5727]]]) 99 | mc.INPUT_STD = np.array([[[11.4001, 12.9684, 1.9548, 20.2257, 12.9454]]]) 100 | 101 | return mc 102 | -------------------------------------------------------------------------------- /pcl_segmentation/configs/__init__.py: -------------------------------------------------------------------------------- 1 | from .SqueezeSegV2 import SqueezeSegV2Config 2 | -------------------------------------------------------------------------------- /pcl_segmentation/data_loader/__init__.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | from .data_loader import DataLoader 26 | -------------------------------------------------------------------------------- /pcl_segmentation/data_loader/data_loader.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import os 26 | import random 27 | import numpy as np 28 | import tensorflow as tf 29 | 30 | 31 | def _tensor_feature(value): 32 | """Returns a bytes_list from a string / byte.""" 33 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=[tf.io.serialize_tensor(value).numpy()])) 34 | 35 | 36 | class DataLoader: 37 | def __init__(self, dataset_split, dataset_root_path, mc): 38 | """ 39 | Arguments: 40 | dataset_split -- String containing "train", "val" or "test" 41 | dataset_root_path -- String containing path to the root directory of the splits 42 | """ 43 | self.mc = mc 44 | 45 | validation = True if dataset_split == "val" else False 46 | self._batch_size = mc.BATCH_SIZE if not validation else 1 47 | self._data_augmentation = mc.DATA_AUGMENTATION if not validation else False 48 | 49 | self._dataset_split = dataset_split 50 | self._dataset_root_path = dataset_root_path 51 | self._dataset_path = os.path.join(self._dataset_root_path, dataset_split) 52 | self._sample_pathes = tf.io.gfile.glob(os.path.join(self._dataset_path, "*.npy")) 53 | 54 | @staticmethod 55 | def random_y_flip(lidar, mask, label, weight, prob=0.5): 56 | """ 57 | Randomly flips a Numpy ndArray along the second axis (y-axis) with probability prob 58 | 59 | Arguments: 60 | lidar -- Numpy ndArray of shape [height, width, channels] 61 | mask -- Numpy ndArray of shape [height, width] 62 | label -- Numpy ndArray of shape [height, width] 63 | weight -- Numpy ndArray of shape [height, width] 64 | prob -- Float which describes the probability that the flip is applied on the sample. 65 | 66 | Returns: 67 | lidar -- Numpy ndArray of shape [height, width, channels] 68 | mask -- Numpy ndArray of shape [height, width] 69 | label -- Numpy ndArray of shape [height, width] 70 | weight -- Numpy ndArray of shape [height, width] 71 | """ 72 | # creates a random float between 0 and 1 73 | random_float = tf.random.uniform(shape=[], minval=0, maxval=1, dtype=tf.float16) 74 | 75 | (lidar, mask, label, weight) = tf.cond( 76 | pred=random_float > prob, 77 | true_fn=lambda: (tf.reverse(lidar, axis=[1]), 78 | tf.reverse(mask, axis=[1]), 79 | tf.reverse(label, axis=[1]), 80 | tf.reverse(weight, axis=[1]) 81 | ), 82 | false_fn=lambda: (lidar, mask, label, weight) 83 | ) 84 | 85 | return lidar, mask, label, weight 86 | 87 | @staticmethod 88 | def random_shift_lr(lidar, mask, label, weight, shift): 89 | """ 90 | Randomly shifts a sample left-right 91 | 92 | Arguments: 93 | lidar -- Numpy ndArray of shape [height, width, channels] 94 | mask -- Numpy ndArray of shape [height, width] 95 | label -- Numpy ndArray of shape [height, width] 96 | weight -- Numpy ndArray of shape [height, width] 97 | shift -- Integer which defines the maximal amount of the random horizontal shift 98 | 99 | Returns : 100 | sample -- Numpy ndArray of shape [height, width, channels] 101 | """ 102 | # Generate a random integer between in [-shift, shift] 103 | random = tf.random.uniform(shape=[], minval=-shift, maxval=shift, dtype=tf.int32) 104 | 105 | lidar = tf.roll(lidar, random, axis=1) 106 | mask = tf.roll(mask, random, axis=1) 107 | label = tf.roll(label, random, axis=1) 108 | weight = tf.roll(weight, random, axis=1) 109 | 110 | return lidar, mask, label, weight 111 | 112 | 113 | @staticmethod 114 | def random_shift_up_down(lidar, mask, label, weight, shift): 115 | """ 116 | Randomly shifts a sample up-down 117 | 118 | Arguments: 119 | lidar -- Numpy ndArray of shape [height, width, channels] 120 | mask -- Numpy ndArray of shape [height, width] 121 | label -- Numpy ndArray of shape [height, width] 122 | weight -- Numpy ndArray of shape [height, width] 123 | shift -- Integer which defines the maximal amount of the random horizontal shift 124 | 125 | Returns : 126 | sample -- Numpy ndArray of shape [height, width, channels] 127 | """ 128 | # Generate a random integer between in [-shift, shift] 129 | random = tf.random.uniform(shape=[], minval=-shift, maxval=shift, dtype=tf.int32) 130 | 131 | lidar = tf.roll(lidar, random, axis=0) 132 | mask = tf.roll(mask, random, axis=0) 133 | label = tf.roll(label, random, axis=0) 134 | weight = tf.roll(weight, random, axis=0) 135 | 136 | return lidar, mask, label, weight 137 | 138 | def parse_sample(self, sample_path): 139 | """ 140 | Parses a data sample from a file path an returns a lidar tensor, a mask tensor and a label tensor 141 | 142 | Arguments: 143 | sample_path -- String - File path to a sample *.npy file 144 | 145 | Returns: 146 | lidar -- numpy ndarray of shape [height, width, num_channels] containing the lidar data 147 | mask -- numpy ndarray of shape [height, width] containing a boolean mask 148 | label -- numpy ndarray of shape [height, width] containing the label as segmentation map 149 | weight -- numpy ndarray of shape [height, width] containing the weighting for each class 150 | """ 151 | 152 | # Load numpy sample 153 | sample = np.load(sample_path.numpy()).astype(np.float32, copy=False) 154 | 155 | # Get x, y, z, intensity, depth 156 | lidar = sample[:, :, :5] 157 | 158 | # Compute binary mask: True where the depth is bigger then 0, false in any other case 159 | mask = lidar[:, :, 4] > 0 160 | 161 | # Normalize input data using the mean and standard deviation 162 | lidar = (lidar - self.mc.INPUT_MEAN) / self.mc.INPUT_STD 163 | 164 | # Set lidar on all channels to zero where the mask is False. Ie. where no points are present 165 | lidar[~mask] = 0.0 166 | 167 | # Add Dimension to mask to obtain a tensor of shape [height, width, 1] 168 | mask = np.expand_dims(mask, -1) 169 | 170 | # Append mask to lidar input 171 | lidar = np.append(lidar, mask, axis=2) 172 | 173 | # Squeeze mask 174 | mask = np.squeeze(mask) 175 | 176 | # Get segmentation map from sample 177 | label = sample[:, :, 5] 178 | 179 | # set label to None class where no points are present 180 | label[~mask] = self.mc.CLASSES.index("None") 181 | 182 | # construct class-wise weighting defined in the configuration 183 | weight = np.zeros(label.shape) 184 | for l in range(self.mc.NUM_CLASS): 185 | weight[label == l] = self.mc.CLS_LOSS_WEIGHT[int(l)] 186 | 187 | return lidar.astype('float32'), mask.astype('bool'), label.astype('int32'), weight.astype('float32') 188 | 189 | @staticmethod 190 | def serialize_sample(lidar, mask, label, weight): 191 | """ 192 | Creates a tf.train.Example message ready to be written to a file. 193 | Arguments: 194 | lidar -- numpy ndarray of shape [height, width, num_channels] containing the lidar data 195 | mask -- numpy ndarray of shape [height, width] containing a boolean mask 196 | label -- numpy ndarray of shape [height, width] containing the label as segmentation map 197 | weight -- numpy ndarray of shape [height, width] containing the weighting for each class 198 | 199 | Returns: 200 | example_proto -- Serialized train sample as String 201 | """ 202 | feature = { 203 | 'lidar': _tensor_feature(lidar), 204 | 'mask': _tensor_feature(mask), 205 | 'label': _tensor_feature(label), 206 | 'weight': _tensor_feature(weight) 207 | } 208 | example_proto = tf.train.Example(features=tf.train.Features(feature=feature)) 209 | return example_proto.SerializeToString() 210 | 211 | def tf_serialize_example(self, lidar, mask, label, weight): 212 | """ 213 | Wrapper function for self.serialize_sample 214 | 215 | Arguments: 216 | lidar -- numpy ndarray of shape [height, width, num_channels] containing the lidar data 217 | mask -- numpy ndarray of shape [height, width] containing a boolean mask 218 | label -- numpy ndarray of shape [height, width] containing the label as segmentation map 219 | weight -- numpy ndarray of shape [height, width] containing the weighting for each class 220 | 221 | Returns: 222 | example_proto -- Serialized train sample as String 223 | """ 224 | tf_string = tf.py_function( 225 | self.serialize_sample, 226 | (lidar, mask, label, weight), # Pass these args to the above function. 227 | tf.string) # The return type is `tf.string`. 228 | return tf.reshape(tf_string, ()) # The result is a scalar. 229 | 230 | def serialize_dataset(self, sample_pathes): 231 | """ 232 | Arguments: 233 | sample_pathes -- List of Strings which contain pathes for the training samples 234 | 235 | Returns: 236 | dataset -- tf.data.Dataset with serialized data 237 | """ 238 | random.shuffle(sample_pathes) 239 | 240 | # create a tf.data.Dataset using sample_pathes 241 | dataset = tf.data.Dataset.from_tensor_slices(sample_pathes) 242 | 243 | # Apply parse_sample and read the *.npy file 244 | dataset = dataset.map(lambda sample: 245 | tf.py_function(self.parse_sample, [sample], [tf.float32, tf.bool, tf.int32, tf.float32]), 246 | num_parallel_calls=tf.data.AUTOTUNE) 247 | 248 | # Apply self.tf_serialize_example on the parsed sample 249 | dataset = dataset.map(self.tf_serialize_example, num_parallel_calls=tf.data.AUTOTUNE) 250 | return dataset 251 | 252 | def write_tfrecord_dataset(self): 253 | """ 254 | Write the TFRecord file for the current dataset. The TFRecord file is automatically written to 255 | self._dataset_root_path with the name train.tfrecord or val.tfrecord or test.tfrecord. If the file already exists 256 | this processing part is skipped. 257 | """ 258 | filename = os.path.join(self._dataset_root_path, self._dataset_split + ".tfrecord") 259 | 260 | if os.path.isfile(filename): 261 | print("TFRecord exists at {0}. Skipping TFRecord writing.".format(filename)) 262 | else: 263 | print("Writing TFRecord to {0}".format(filename)) 264 | serialized_dataset = self.serialize_dataset(self._sample_pathes) 265 | writer = tf.data.experimental.TFRecordWriter(filename) 266 | writer.write(serialized_dataset) 267 | return self 268 | 269 | def parse_proto(self, example_proto): 270 | """ 271 | Deserializes the sample proto String encoded with self.serialize_sample back to tf.Tensors. 272 | Also applies augmentation functions to the sample defined in the config. 273 | 274 | Arguments: 275 | example_proto -- Sample serialized as proto String 276 | Returns: 277 | lidar -- numpy ndarray of shape [height, width, num_channels] containing the lidar data 278 | mask -- numpy ndarray of shape [height, width] containing a boolean mask 279 | label -- numpy ndarray of shape [height, width] containing the label as segmentation map 280 | weight -- numpy ndarray of shape [height, width] containing the weighting for each class 281 | """ 282 | 283 | feature_description = { 284 | 'lidar': tf.io.FixedLenFeature([], tf.string), 285 | 'mask': tf.io.FixedLenFeature([], tf.string), 286 | 'label': tf.io.FixedLenFeature([], tf.string), 287 | 'weight': tf.io.FixedLenFeature([], tf.string), 288 | } 289 | # Parse the input `tf.train.Example` proto using the dictionary above. 290 | example = tf.io.parse_single_example(example_proto, feature_description) 291 | 292 | lidar = tf.io.parse_tensor(example["lidar"], out_type=tf.float32) 293 | mask = tf.io.parse_tensor(example["mask"], out_type=tf.bool) 294 | label = tf.io.parse_tensor(example["label"], out_type=tf.int32) 295 | weight = tf.io.parse_tensor(example["weight"], out_type=tf.float32) 296 | 297 | lidar = tf.reshape(lidar, shape=[self.mc.ZENITH_LEVEL, self.mc.AZIMUTH_LEVEL, self.mc.NUM_FEATURES]) 298 | mask = tf.reshape(mask, shape=[self.mc.ZENITH_LEVEL, self.mc.AZIMUTH_LEVEL]) 299 | label = tf.reshape(label, shape=[self.mc.ZENITH_LEVEL, self.mc.AZIMUTH_LEVEL]) 300 | weight = tf.reshape(weight, shape=[self.mc.ZENITH_LEVEL, self.mc.AZIMUTH_LEVEL]) 301 | 302 | if self._data_augmentation: 303 | # Perform the random left-right flip augmentation 304 | if self.mc.RANDOM_FLIPPING: 305 | lidar, mask, label, weight = self.random_y_flip(lidar, mask, label, weight) 306 | 307 | # Perform the random left-right shift augmentation 308 | if self.mc.SHIFT_LEFT_RIGHT > 0: 309 | lidar, mask, label, weight = self.random_shift_lr(lidar, mask, label, weight, self.mc.SHIFT_LEFT_RIGHT) 310 | 311 | # Perform the random up-down shift augmentation 312 | if self.mc.SHIFT_UP_DOWN > 0: 313 | lidar, mask, label, weight = self.random_shift_up_down(lidar, mask, label, weight, self.mc.SHIFT_UP_DOWN) 314 | 315 | return (lidar, mask), label, weight 316 | 317 | def read_tfrecord_dataset(self, buffer_size=1000): 318 | """ 319 | Arguments: 320 | buffer_size -- Shuffle buffer size 321 | Returns: 322 | dataset -- tf.data.Dataset containing the dataset 323 | """ 324 | filename = os.path.join(self._dataset_root_path, self._dataset_split + ".tfrecord") 325 | raw_dataset = tf.data.TFRecordDataset([filename], num_parallel_reads=tf.data.AUTOTUNE) 326 | dataset = raw_dataset.map(self.parse_proto, num_parallel_calls=tf.data.AUTOTUNE) 327 | dataset = dataset.shuffle(buffer_size) 328 | dataset = dataset.batch(self._batch_size, drop_remainder=True) 329 | dataset = dataset.prefetch(tf.data.AUTOTUNE) 330 | return dataset 331 | -------------------------------------------------------------------------------- /pcl_segmentation/eval.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import tensorflow as tf 26 | import argparse 27 | 28 | from data_loader import DataLoader 29 | from utils.util import confusion_matrix_to_iou_recall_precision 30 | from utils.args_loader import load_model_config 31 | 32 | 33 | def evaluation(arg): 34 | config, _ = load_model_config(arg.model, arg.config) 35 | 36 | config.DATA_AUGMENTATION = False 37 | config.BATCH_SIZE = 1 38 | dataset = DataLoader(arg.image_set, arg.data_path, config).write_tfrecord_dataset().read_tfrecord_dataset() 39 | 40 | model = tf.keras.models.load_model(arg.path_to_model) 41 | miou_tracker = tf.metrics.MeanIoU(num_classes=config.NUM_CLASS, name="MeanIoU") 42 | 43 | print("Performing Evaluation") 44 | 45 | for sample in dataset: 46 | (lidar, mask), label, weight = sample 47 | probabilities, predictions = model([lidar, mask]) 48 | miou_tracker.update_state(label, predictions) 49 | 50 | iou, recall, precision = confusion_matrix_to_iou_recall_precision(miou_tracker.total_cm) 51 | 52 | for i, cls in enumerate(config.CLASSES): 53 | print(cls.upper()) 54 | print("IoU: " + str(iou[i].numpy())) 55 | print("Recall: " + str(recall[i].numpy())) 56 | print("Precision: " + str(precision[i].numpy())) 57 | print("") 58 | print("MIoU: {} ".format(miou_tracker.result().numpy())) 59 | 60 | 61 | if __name__ == '__main__': 62 | physical_devices = tf.config.experimental.list_physical_devices('GPU') 63 | if len(physical_devices) > 0: 64 | tf.config.experimental.set_memory_growth(physical_devices[0], True) 65 | 66 | parser = argparse.ArgumentParser(description='Parse Flags for the evaluation script!') 67 | parser.add_argument('-d', '--data_path', type=str, 68 | help='Absolute path to the dataset') 69 | parser.add_argument('-i', '--image_set', type=str, default="val", 70 | help='Default: `val`. But can also be train, val or test') 71 | parser.add_argument('-t', '--eval_dir', type=str, 72 | help="Directory where to write the Tensorboard logs and checkpoints") 73 | parser.add_argument('-p', '--path_to_model', type=str, 74 | help='Path to the model') 75 | parser.add_argument('-m', '--model', type=str, 76 | help='Model name either `squeezesegv2`, `darknet53`, `darknet21`') 77 | parser.add_argument('-n', '--config', type=str, default='squeezesegv2', 78 | help='Which configuration for training `squeezesegv2`, `squeezesegv2kitti`, ' 79 | ' `darknet52`, `darknet21` ') 80 | args = parser.parse_args() 81 | 82 | evaluation(args) 83 | -------------------------------------------------------------------------------- /pcl_segmentation/inference.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import os.path 26 | import glob 27 | 28 | import tqdm 29 | from PIL import Image 30 | import argparse 31 | 32 | from configs import * 33 | from utils.util import * 34 | 35 | 36 | def inference(arg): 37 | config = SqueezeSegV2Config() 38 | 39 | model = tf.keras.models.load_model(arg.path_to_model) 40 | 41 | if not os.path.exists(arg.output_dir): 42 | os.makedirs(arg.output_dir) 43 | 44 | for f in tqdm.tqdm(list(glob.iglob(arg.input_path))): 45 | print("Process: {0}".format(f)) 46 | 47 | sample = np.load(f).astype(np.float32, copy=False) 48 | 49 | # Get x, y, z, intensity, depth 50 | lidar = sample[:, :, :5] 51 | 52 | # Compute binary mask: True where the depth is bigger then 0, false in any other case 53 | mask = lidar[:, :, 4] > 0 54 | 55 | # Normalize input data using the mean and standard deviation 56 | lidar = (lidar - config.INPUT_MEAN) / config.INPUT_STD 57 | 58 | # Set lidar on all channels to zero where the mask is False. Ie. where no points are present 59 | lidar[~mask] = 0.0 60 | 61 | # Append mask to lidar input 62 | lidar = np.append(lidar, np.expand_dims(mask, -1), axis=2) 63 | 64 | # Get segmentation map from sample 65 | label = sample[:, :, 5] 66 | 67 | # set label to None class where no points are present 68 | label[~mask] = config.CLASSES.index("None") 69 | 70 | # add batch dim 71 | lidar = np.expand_dims(lidar, axis=0) 72 | mask = np.expand_dims(mask, axis=0) 73 | 74 | # execute model 75 | probabilities, predictions = model([lidar, mask]) 76 | 77 | # to numpy and remove batch dimension 78 | predictions = predictions.numpy()[0] 79 | 80 | # save the data 81 | file_name = f.strip('.npy').split('/')[-1] 82 | np.save( 83 | os.path.join(arg.output_dir, 'pred_' + file_name + '.npy'), 84 | predictions 85 | ) 86 | 87 | depth_map = Image.fromarray( 88 | (255 * normalize(lidar[0][:, :, 3])).astype(np.uint8)) 89 | label_map = Image.fromarray( 90 | (255 * config.CLS_COLOR_MAP[predictions]).astype(np.uint8)) 91 | 92 | blend_map = Image.blend( 93 | depth_map.convert('RGBA'), 94 | label_map.convert('RGBA'), 95 | alpha=1.0 96 | ) 97 | 98 | blend_map.save( 99 | os.path.join(arg.output_dir, 'plot_' + file_name + '.png')) 100 | 101 | # save the label plot 102 | label_map = Image.fromarray( 103 | (255 * config.CLS_COLOR_MAP[label.astype(np.int32)]).astype(np.uint8) 104 | ) 105 | blend_map = Image.blend( 106 | depth_map.convert('RGBA'), 107 | label_map.convert('RGBA'), 108 | alpha=1.0 109 | ) 110 | blend_map.save( 111 | os.path.join(arg.output_dir, 'plot_gt_' + file_name + '.png') 112 | ) 113 | 114 | 115 | if __name__ == '__main__': 116 | physical_devices = tf.config.experimental.list_physical_devices('GPU') 117 | if len(physical_devices) > 0: 118 | tf.config.experimental.set_memory_growth(physical_devices[0], True) 119 | 120 | parser = argparse.ArgumentParser(description='Parse Flags for the inference script!') 121 | parser.add_argument('-d', '--input_path', type=str, 122 | help='Input LiDAR scans to be detected. Must be a glob pattern input such as' 123 | '`./data/samples/*.npy` !') 124 | parser.add_argument('-m', '--model', type=str, 125 | help='Model name either `squeezesegv2`, `darknet53`, `darknet21`') 126 | parser.add_argument('-t', '--output_dir', type=str, 127 | help="Directory where to write the model predictions and visualizations") 128 | parser.add_argument('-p', '--path_to_model', type=str, 129 | help='Path to the model') 130 | args = parser.parse_args() 131 | 132 | inference(args) 133 | -------------------------------------------------------------------------------- /pcl_segmentation/nets/Darknet.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import tensorflow as tf 26 | 27 | from .SegmentationNetwork import PCLSegmentationNetwork 28 | 29 | class BasicBlock(tf.keras.layers.Layer): 30 | """Basic Block of the Darknet Architecture""" 31 | 32 | def __init__(self, inplanes, planes, bn_momentum=0.9): 33 | super(BasicBlock, self).__init__() 34 | self.conv1 = tf.keras.layers.Conv2D( 35 | filters=planes[0], 36 | kernel_size=1, 37 | strides=1, 38 | padding='VALID', 39 | use_bias=False 40 | ) 41 | self.bn1 = tf.keras.layers.BatchNormalization(momentum=bn_momentum) 42 | self.leaky_relu1 = tf.keras.layers.LeakyReLU(0.1) 43 | 44 | self.conv2 = tf.keras.layers.Conv2D( 45 | filters=planes[1], 46 | kernel_size=3, 47 | strides=1, 48 | padding='SAME', 49 | use_bias=False 50 | ) 51 | self.bn2 = tf.keras.layers.BatchNormalization(momentum=bn_momentum) 52 | self.leaky_relu2 = tf.keras.layers.LeakyReLU(0.1) 53 | 54 | def call(self, inputs, training=False): 55 | residual = inputs 56 | 57 | x = self.conv1(inputs) 58 | x = self.bn1(x) 59 | x = self.leaky_relu1(x) 60 | 61 | x = self.conv2(x) 62 | x = self.bn2(x) 63 | x = self.leaky_relu2(x) 64 | 65 | x += residual 66 | return x 67 | 68 | 69 | class EncoderLayer(tf.keras.layers.Layer): 70 | """Basic Encoder Layer of the Darknet Architecture""" 71 | 72 | def __init__(self, block, planes, num_blocks, stride, bn_momentum=0.9): 73 | super(EncoderLayer, self).__init__() 74 | self.num_blocks = num_blocks 75 | # downsample 76 | self.conv1 = tf.keras.layers.Conv2D( 77 | filters=planes[1], 78 | kernel_size=3, 79 | strides=[1, stride], 80 | dilation_rate=1, 81 | padding='SAME', 82 | use_bias=False 83 | ) 84 | self.bn1 = tf.keras.layers.BatchNormalization(momentum=bn_momentum) 85 | self.leaky_relu1 = tf.keras.layers.LeakyReLU(0.1) 86 | 87 | inplanes = planes[1] 88 | for i in range(0, self.num_blocks): 89 | setattr(self, 90 | "residual_{}".format(i), 91 | block(inplanes=inplanes, 92 | planes=planes, 93 | bn_momentum=bn_momentum) 94 | ) 95 | 96 | def call(self, inputs, training=False): 97 | # downsample 98 | x = self.conv1(inputs) 99 | x = self.bn1(x) 100 | x = self.leaky_relu1(x) 101 | for i in range(0, self.num_blocks): 102 | x = getattr(self, "residual_{}".format(i))(x) 103 | return x 104 | 105 | 106 | class DecoderLayer(tf.keras.layers.Layer): 107 | """Basic Decoder Layer of the Darknet Architecture""" 108 | def __init__(self, block, planes, stride, bn_momentum=0.9): 109 | super(DecoderLayer, self).__init__() 110 | self.stride = stride 111 | if self.stride == 2: 112 | # upsample 113 | self.upconv1 = tf.keras.layers.Conv2DTranspose( 114 | filters=planes[1], 115 | kernel_size=[1, 4], 116 | strides=[1, 2], 117 | padding="SAME" 118 | ) 119 | else: 120 | # keep constant 121 | self.conv1 = tf.keras.layers.Conv2D( 122 | filters=planes[1], 123 | kernel_size=3, 124 | padding="SAME" 125 | ) 126 | self.bn1 = tf.keras.layers.BatchNormalization(momentum=bn_momentum) 127 | self.leaky_relu1 = tf.keras.layers.LeakyReLU(0.1) 128 | self.block = block(planes[1], planes, bn_momentum) 129 | 130 | def call(self, inputs, training=False): 131 | if self.stride == 2: 132 | x = self.upconv1(inputs) 133 | else: 134 | x = self.conv1(inputs) 135 | x = self.bn1(x) 136 | x = self.leaky_relu1(x) 137 | x = self.block(x) 138 | return x 139 | 140 | 141 | # number of layers per model 142 | model_blocks = { 143 | 21: [1, 1, 2, 2, 1], 144 | 53: [1, 2, 8, 8, 4], 145 | } 146 | 147 | class Darknet(PCLSegmentationNetwork): 148 | """Implements the Darknet Segmentation Model""" 149 | def __init__(self, mc): 150 | super(Darknet, self).__init__(mc) 151 | self.mc = mc 152 | self.drop_rate = mc.DROP_RATE 153 | self.bn_momentum = mc.BN_MOMENTUM 154 | self.output_stride = mc.OUTPUT_STRIDE # Output stride only horizontally 155 | self.num_layers = mc.NUM_LAYERS 156 | self.last_channels_encoder = 1024 157 | 158 | # stride play 159 | self.encoder_strides = [2, 2, 2, 2, 2] 160 | 161 | # check current stride 162 | current_os = 1 163 | for s in self.encoder_strides: 164 | current_os *= s 165 | print("Encoder Original OS: ", current_os) 166 | 167 | # make the new stride 168 | if self.output_stride > current_os: 169 | print("Can't do OS, ", self.output_stride, 170 | " because it is bigger than original ", current_os) 171 | else: 172 | # redo strides according to needed stride 173 | for i, stride in enumerate(reversed(self.encoder_strides), 0): 174 | if int(current_os) != self.output_stride: 175 | if stride == 2: 176 | current_os /= 2 177 | self.encoder_strides[-1 - i] = 1 178 | if int(current_os) == self.output_stride: 179 | break 180 | print("Encoder New OS: ", int(current_os)) 181 | print("Encoder Strides: ", self.encoder_strides) 182 | 183 | # generate layers depending on darknet type 184 | self.num_blocks = model_blocks[self.num_layers] 185 | 186 | # input layer 187 | self.conv1 = tf.keras.layers.Conv2D( 188 | input_shape=[self.mc.ZENITH_LEVEL, self.mc.AZIMUTH_LEVEL, self.mc.NUM_FEATURES], 189 | filters=32, 190 | kernel_size=3, 191 | strides=1, 192 | padding='SAME', 193 | use_bias=False 194 | ) 195 | self.bn1 = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 196 | self.leaky_relu1 = tf.keras.layers.LeakyReLU(0.1) 197 | 198 | self.enc1 = EncoderLayer(block=BasicBlock, planes=[32, 64], num_blocks=self.num_blocks[0], 199 | stride=self.encoder_strides[0], bn_momentum=self.bn_momentum) 200 | 201 | self.enc2 = EncoderLayer(block=BasicBlock, planes=[64, 128], num_blocks=self.num_blocks[1], 202 | stride=self.encoder_strides[1], bn_momentum=self.bn_momentum) 203 | 204 | self.enc3 = EncoderLayer(block=BasicBlock, planes=[128, 256], num_blocks=self.num_blocks[2], 205 | stride=self.encoder_strides[2], bn_momentum=self.bn_momentum) 206 | 207 | self.enc4 = EncoderLayer(block=BasicBlock, planes=[256, 512], num_blocks=self.num_blocks[3], 208 | stride=self.encoder_strides[3], bn_momentum=self.bn_momentum) 209 | 210 | self.enc5 = EncoderLayer(block=BasicBlock, planes=[512, self.last_channels_encoder], num_blocks=self.num_blocks[4], 211 | stride=self.encoder_strides[4], bn_momentum=self.bn_momentum) 212 | 213 | self.dropout = tf.keras.layers.Dropout(self.drop_rate) 214 | 215 | # Decoder 216 | self.decoder_strides = [2, 2, 2, 2, 2] 217 | # check current stride 218 | current_os = 1 219 | for s in self.decoder_strides: 220 | current_os *= s 221 | print("Decoder original OS: ", int(current_os)) 222 | # redo strides according to needed stride 223 | for i, stride in enumerate(self.decoder_strides): 224 | if int(current_os) != self.output_stride: 225 | if stride == 2: 226 | current_os /= 2 227 | self.decoder_strides[i] = 1 228 | if int(current_os) == self.output_stride: 229 | break 230 | print("Decoder new OS: ", int(current_os)) 231 | print("Decoder strides: ", self.decoder_strides) 232 | 233 | self.dec5 = DecoderLayer(BasicBlock, 234 | planes=[self.last_channels_encoder, 512], 235 | bn_momentum=self.bn_momentum, 236 | stride=self.decoder_strides[0]) 237 | self.dec4 = DecoderLayer(BasicBlock, 238 | planes=[512, 256], 239 | bn_momentum=self.bn_momentum, 240 | stride=self.decoder_strides[1]) 241 | self.dec3 = DecoderLayer(BasicBlock, 242 | planes=[256, 128], 243 | bn_momentum=self.bn_momentum, 244 | stride=self.decoder_strides[2]) 245 | self.dec2 = DecoderLayer(BasicBlock, 246 | planes=[128, 64], 247 | bn_momentum=self.bn_momentum, 248 | stride=self.decoder_strides[3]) 249 | self.dec1 = DecoderLayer(BasicBlock, 250 | planes=[64, 32], 251 | bn_momentum=self.bn_momentum, 252 | stride=self.decoder_strides[4]) 253 | 254 | # Head 255 | self.head = tf.keras.layers.Conv2D( 256 | filters=mc.NUM_CLASS, 257 | kernel_size=3, 258 | strides=1, 259 | padding="SAME" 260 | ) 261 | 262 | 263 | def run_enc_block(self, x, layer, skips, os): 264 | y = layer(x) 265 | if y.shape[1] < x.shape[1] or y.shape[2] < x.shape[2]: 266 | skips[os] = x 267 | os *= 2 268 | x = y 269 | return x, skips, os 270 | 271 | def run_dec_block(self, x, layer, skips, os): 272 | y = layer(x) # up 273 | if y.shape[2] > x.shape[2]: 274 | os //= 2 # match skip 275 | y = y + skips[os] # add skip 276 | x = y 277 | return x, skips, os 278 | 279 | def call(self, inputs, training=False, mask=None): 280 | lidar_input, lidar_mask = inputs[0], inputs[1] 281 | 282 | # run cnn 283 | # store for skip connections 284 | skips = {} 285 | os = 1 286 | 287 | # first layer 288 | x, skips, os = self.run_enc_block(lidar_input, self.conv1, skips, os) 289 | x = self.bn1(x) 290 | x = self.leaky_relu1(x) 291 | 292 | # all encoder blocks with intermediate dropouts 293 | x, skips, os = self.run_enc_block(x, self.enc1, skips, os) 294 | x = self.dropout(x, training) 295 | x, skips, os = self.run_enc_block(x, self.enc2, skips, os) 296 | x = self.dropout(x, training) 297 | x, skips, os = self.run_enc_block(x, self.enc3, skips, os) 298 | x = self.dropout(x, training) 299 | x, skips, os = self.run_enc_block(x, self.enc4, skips, os) 300 | x = self.dropout(x, training) 301 | x, skips, os = self.run_enc_block(x, self.enc5, skips, os) 302 | x = self.dropout(x, training) 303 | 304 | # run decoder layers 305 | x, skips, os = self.run_dec_block(x, self.dec5, skips, os) 306 | x, skips, os = self.run_dec_block(x, self.dec4, skips, os) 307 | x, skips, os = self.run_dec_block(x, self.dec3, skips, os) 308 | x, skips, os = self.run_dec_block(x, self.dec2, skips, os) 309 | x, skips, os = self.run_dec_block(x, self.dec1, skips, os) 310 | 311 | x = self.dropout(x, training) 312 | logits = self.head(x) 313 | 314 | return self.segmentation_head(logits, lidar_mask) 315 | 316 | -------------------------------------------------------------------------------- /pcl_segmentation/nets/SegmentationNetwork.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import tensorflow as tf 26 | 27 | 28 | class PCLSegmentationNetwork(tf.keras.Model): 29 | """Base Class for segmentation Networks which implements the Loss funcion, train step and tensorboard methods""" 30 | 31 | def __init__(self, mc): 32 | super(PCLSegmentationNetwork, self).__init__() 33 | self.mc = mc 34 | self.NUM_CLASS = mc.NUM_CLASS 35 | self.BATCH_SIZE = mc.BATCH_SIZE 36 | self.ZENITH_LEVEL = mc.ZENITH_LEVEL 37 | self.AZIMUTH_LEVEL = mc.AZIMUTH_LEVEL 38 | self.NUM_FEATURES = mc.NUM_FEATURES 39 | 40 | self.FOCAL_GAMMA = mc.FOCAL_GAMMA 41 | self.CLS_LOSS_COEF = mc.CLS_LOSS_COEF 42 | self.DENOM_EPSILON = mc.DENOM_EPSILON 43 | 44 | self.CLASSES = mc.CLASSES 45 | self.CLS_COLOR_MAP = mc.CLS_COLOR_MAP 46 | 47 | self.softmax = tf.keras.layers.Softmax(axis=-1) 48 | 49 | self.scc_loss = tf.keras.losses.SparseCategoricalCrossentropy() 50 | 51 | # Metrics 52 | self.miou_tracker = tf.keras.metrics.MeanIoU(num_classes=self.NUM_CLASS, name="MeanIoU") 53 | self.loss_tracker = tf.keras.metrics.Mean(name="loss") 54 | 55 | def call(self, inputs, training=False, mask=None): 56 | raise NotImplementedError("Method should be called in child class!") 57 | 58 | def segmentation_head(self, logits, lidar_mask): 59 | with tf.name_scope("segmentation_head") as scope: 60 | probabilities = self.softmax(logits) 61 | 62 | predictions = tf.argmax(probabilities, axis=-1, output_type=tf.int32) 63 | 64 | # set predictions to the "None" class where no points are present 65 | predictions = tf.where(tf.squeeze(lidar_mask), 66 | predictions, 67 | tf.ones_like(predictions) * self.CLASSES.index("None") 68 | ) 69 | return probabilities, predictions 70 | 71 | def focal_loss(self, probabilities, lidar_mask, label, loss_weight): 72 | """Focal Loss""" 73 | with tf.name_scope("focal_loss") as scope: 74 | lidar_mask = tf.cast(lidar_mask, tf.float32) # from bool to float32 75 | 76 | label = tf.reshape(label, (-1,)) # class labels 77 | 78 | prob = tf.reshape(probabilities, (-1, self.NUM_CLASS)) + self.DENOM_EPSILON # output prob 79 | 80 | onehot_labels = tf.one_hot(label, depth=self.NUM_CLASS) # onehot class labels 81 | 82 | cross_entropy = tf.multiply(onehot_labels, -tf.math.log(prob)) * \ 83 | tf.reshape(loss_weight, (-1, 1)) * tf.reshape(lidar_mask, (-1, 1)) # cross entropy 84 | 85 | weight = (1.0 - prob) ** self.FOCAL_GAMMA # weight in the def of focal loss 86 | 87 | fl = weight * cross_entropy # focal loss 88 | 89 | loss = tf.identity(tf.reduce_sum(fl) / tf.reduce_sum(lidar_mask) * self.CLS_LOSS_COEF, name='class_loss') 90 | 91 | return loss 92 | 93 | def train_step(self, data): 94 | (lidar_input, lidar_mask), label, weight = data 95 | 96 | with tf.GradientTape() as tape: 97 | probabilities, predictions = self.call([lidar_input, lidar_mask], training=True) # forward pass 98 | if self.mc.USE_FOCAL_LOSS: 99 | loss = self.focal_loss(probabilities, lidar_mask, label, weight) 100 | else: 101 | loss = self.scc_loss(label, probabilities, weight) 102 | 103 | # Compute gradients 104 | trainable_vars = self.trainable_variables 105 | gradients = tape.gradient(loss, trainable_vars) 106 | 107 | # Update weights 108 | self.optimizer.apply_gradients(zip(gradients, trainable_vars)) 109 | 110 | # Update & Compute Metrics 111 | with tf.name_scope("metrics") as scope: 112 | self.loss_tracker.update_state(loss) 113 | self.miou_tracker.update_state(label, predictions, weight) 114 | loss_result = self.loss_tracker.result() 115 | miou_result = self.miou_tracker.result() 116 | return {'loss': loss_result, 'miou': miou_result} 117 | 118 | def test_step(self, data): 119 | (lidar_input, lidar_mask), label, weight = data 120 | 121 | probabilities, predictions = self.call([lidar_input, lidar_mask], training=False) # forward pass 122 | if self.mc.USE_FOCAL_LOSS: 123 | loss = self.focal_loss(probabilities, lidar_mask, label, weight) 124 | else: 125 | loss = self.scc_loss(label, probabilities, weight) 126 | 127 | # Update Metrics 128 | self.loss_tracker.update_state(loss) 129 | self.miou_tracker.update_state(label, predictions, weight) 130 | 131 | return {'loss': self.loss_tracker.result(), 'miou': self.miou_tracker.result()} 132 | 133 | def predict_step(self, data): 134 | (lidar_input, lidar_mask), _, _ = data 135 | probabilities, predictions = self.call([lidar_input, lidar_mask], training=False) # forward pass 136 | return probabilities, predictions 137 | 138 | @property 139 | def metrics(self): 140 | # We list the `Metric` objects here so that `reset_states()` can be 141 | # called automatically at the start of each epoch 142 | return [self.loss_tracker, self.miou_tracker] 143 | 144 | def get_config(self): 145 | config = super(PCLSegmentationNetwork, self).get_config() 146 | config.update({"mc": self.mc}) 147 | return config 148 | 149 | @classmethod 150 | def from_config(cls, config): 151 | return cls(**config) 152 | -------------------------------------------------------------------------------- /pcl_segmentation/nets/SqueezeSegV2.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import tensorflow as tf 26 | 27 | from .SegmentationNetwork import PCLSegmentationNetwork 28 | 29 | 30 | class CAM(tf.keras.layers.Layer): 31 | """Context Aggregation Module""" 32 | 33 | def __init__(self, in_channels, reduction_factor=16, bn_momentum=0.999, l2=0.0001): 34 | super(CAM, self).__init__() 35 | self.in_channels = in_channels 36 | self.reduction_factor = reduction_factor 37 | self.bn_momentum = bn_momentum 38 | self.l2 = l2 39 | 40 | self.pool = tf.keras.layers.MaxPool2D( 41 | pool_size=7, 42 | strides=1, 43 | padding='SAME' 44 | ) 45 | 46 | self.squeeze = tf.keras.layers.Conv2D( 47 | filters=(self.in_channels // self.reduction_factor), 48 | kernel_size=1, 49 | strides=1, 50 | padding='SAME', 51 | kernel_initializer='glorot_uniform', 52 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 53 | ) 54 | self.squeeze_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 55 | 56 | self.excitation = tf.keras.layers.Conv2D( 57 | filters=self.in_channels, 58 | kernel_size=1, 59 | strides=1, 60 | padding='SAME', 61 | kernel_initializer='glorot_uniform', 62 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 63 | ) 64 | self.excitation_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 65 | 66 | def call(self, inputs, training=False): 67 | pool = self.pool(inputs) 68 | squeeze = tf.nn.relu(self.squeeze_bn(self.squeeze(pool))) 69 | excitation = tf.nn.sigmoid(self.excitation_bn(self.excitation(squeeze))) 70 | return inputs * excitation 71 | 72 | def get_config(self): 73 | config = super(CAM, self).get_config() 74 | config.update({'in_channels': self.in_channels, 75 | 'reduction_factor': self.reduction_factor, 76 | 'bn_momentum': self.bn_momentum, 77 | 'l2': self.l2}) 78 | return config 79 | 80 | @classmethod 81 | def from_config(cls, config): 82 | return cls(**config) 83 | 84 | 85 | class FIRE(tf.keras.layers.Layer): 86 | """FIRE MODULE""" 87 | 88 | def __init__(self, sq1x1_planes, ex1x1_planes, ex3x3_planes, bn_momentum=0.999, l2=0.0001): 89 | super(FIRE, self).__init__() 90 | self.sq1x1_planes = sq1x1_planes 91 | self.ex1x1_planes = ex1x1_planes 92 | self.ex3x3_planes = ex3x3_planes 93 | self.bn_momentum = bn_momentum 94 | self.l2 = l2 95 | 96 | self.squeeze = tf.keras.layers.Conv2D( 97 | filters=self.sq1x1_planes, 98 | kernel_size=1, 99 | strides=1, 100 | padding='SAME', 101 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 102 | ) 103 | self.squeeze_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 104 | 105 | self.expand1x1 = tf.keras.layers.Conv2D( 106 | filters=self.ex1x1_planes, 107 | kernel_size=1, 108 | strides=1, 109 | padding='SAME', 110 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 111 | ) 112 | self.expand1x1_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 113 | 114 | self.expand3x3 = tf.keras.layers.Conv2D( 115 | filters=self.ex3x3_planes, 116 | kernel_size=3, 117 | strides=1, 118 | padding='SAME', 119 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 120 | ) 121 | self.expand3x3_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 122 | 123 | def call(self, inputs, training=False): 124 | squeeze = tf.nn.relu(self.squeeze_bn(self.squeeze(inputs), training)) 125 | expand1x1 = tf.nn.relu(self.expand1x1_bn(self.expand1x1(squeeze), training)) 126 | expand3x3 = tf.nn.relu(self.expand3x3_bn(self.expand3x3(squeeze), training)) 127 | return tf.concat([expand1x1, expand3x3], axis=3) 128 | 129 | def get_config(self): 130 | config = super(FIRE, self).get_config() 131 | config.update({'sq1x1_planes': self.sq1x1_planes, 132 | 'ex1x1_planes': self.ex1x1_planes, 133 | 'ex3x3_planes': self.ex3x3_planes, 134 | 'bn_momentum': self.bn_momentum, 135 | 'l2': self.l2}) 136 | return config 137 | 138 | @classmethod 139 | def from_config(cls, config): 140 | return cls(**config) 141 | 142 | 143 | class FIREUP(tf.keras.layers.Layer): 144 | """FIRE MODULE WITH TRANSPOSE CONVOLUTION""" 145 | 146 | def __init__(self, sq1x1_planes, ex1x1_planes, ex3x3_planes, stride, bn_momentum=0.99, l2=0.0001): 147 | super(FIREUP, self).__init__() 148 | self.sq1x1_planes = sq1x1_planes 149 | self.ex1x1_planes = ex1x1_planes 150 | self.ex3x3_planes = ex3x3_planes 151 | self.stride = stride 152 | self.bn_momentum = bn_momentum 153 | self.l2 = l2 154 | 155 | self.squeeze = tf.keras.layers.Conv2D( 156 | filters=self.sq1x1_planes, 157 | kernel_size=1, 158 | strides=1, 159 | padding='SAME', 160 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 161 | ) 162 | self.squeeze_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 163 | 164 | if self.stride == 2: 165 | self.upconv = tf.keras.layers.Conv2DTranspose( 166 | filters=self.sq1x1_planes, 167 | kernel_size=[1, 4], 168 | strides=[1, 2], 169 | padding='SAME', 170 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 171 | ) 172 | 173 | self.expand1x1 = tf.keras.layers.Conv2D( 174 | filters=self.ex1x1_planes, 175 | kernel_size=1, 176 | strides=1, 177 | padding='SAME', 178 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 179 | ) 180 | self.expand1x1_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 181 | 182 | self.expand3x3 = tf.keras.layers.Conv2D( 183 | filters=self.ex3x3_planes, 184 | kernel_size=3, 185 | strides=1, 186 | padding='SAME', 187 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 188 | ) 189 | self.expand3x3_bn = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 190 | 191 | def call(self, inputs, training=False): 192 | squeeze = tf.nn.relu(self.squeeze_bn(self.squeeze(inputs), training)) 193 | if self.stride == 2: 194 | upconv = tf.nn.relu(self.upconv(squeeze)) 195 | else: 196 | upconv = squeeze 197 | expand1x1 = tf.nn.relu(self.expand1x1_bn(self.expand1x1(upconv), training)) 198 | expand3x3 = tf.nn.relu(self.expand3x3_bn(self.expand3x3(upconv), training)) 199 | return tf.concat([expand1x1, expand3x3], axis=3) 200 | 201 | def get_config(self): 202 | config = super(FIREUP, self).get_config() 203 | config.update({'sq1x1_planes': self.sq1x1_planes, 204 | 'ex1x1_planes': self.ex1x1_planes, 205 | 'ex3x3_planes': self.ex3x3_planes, 206 | 'stride': self.stride, 207 | 'bn_momentum': self.bn_momentum, 208 | 'l2': self.l2}) 209 | return config 210 | 211 | @classmethod 212 | def from_config(cls, config): 213 | return cls(**config) 214 | 215 | 216 | 217 | class SqueezeSegV2(PCLSegmentationNetwork): 218 | """SqueezeSegV2 Model as custom Keras Model in TF 2.4""" 219 | 220 | def __init__(self, mc): 221 | super(SqueezeSegV2, self).__init__(mc) 222 | self.mc = mc 223 | self.ZENITH_LEVEL = mc.ZENITH_LEVEL 224 | self.AZIMUTH_LEVEL = mc.AZIMUTH_LEVEL 225 | self.NUM_FEATURES = mc.NUM_FEATURES 226 | self.NUM_CLASS = mc.NUM_CLASS 227 | self.drop_rate = mc.DROP_RATE 228 | self.l2 = mc.L2_WEIGHT_DECAY 229 | self.bn_momentum = mc.BN_MOMENTUM 230 | 231 | # Layers 232 | self.conv1 = tf.keras.layers.Conv2D( 233 | input_shape=[self.ZENITH_LEVEL, self.AZIMUTH_LEVEL, self.mc.NUM_FEATURES], 234 | filters=64, 235 | kernel_size=3, 236 | strides=[1, 2], 237 | padding='SAME', 238 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 239 | ) 240 | self.bn1 = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 241 | self.cam1 = CAM(in_channels=64, bn_momentum=self.bn_momentum, l2=self.l2) 242 | 243 | self.conv1_skip = tf.keras.layers.Conv2D( 244 | input_shape=[self.ZENITH_LEVEL, self.AZIMUTH_LEVEL, self.NUM_FEATURES], 245 | filters=64, 246 | kernel_size=1, 247 | strides=1, 248 | padding='SAME', 249 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 250 | ) 251 | self.bn1_skip = tf.keras.layers.BatchNormalization(momentum=self.bn_momentum) 252 | 253 | self.fire2 = FIRE(sq1x1_planes=16, ex1x1_planes=64, ex3x3_planes=64, bn_momentum=self.bn_momentum, l2=self.l2) 254 | self.cam2 = CAM(in_channels=128, bn_momentum=self.bn_momentum, l2=self.l2) 255 | self.fire3 = FIRE(sq1x1_planes=16, ex1x1_planes=64, ex3x3_planes=64, bn_momentum=self.bn_momentum, l2=self.l2) 256 | self.cam3 = CAM(in_channels=128, bn_momentum=self.bn_momentum, l2=self.l2) 257 | 258 | self.fire4 = FIRE(sq1x1_planes=32, ex1x1_planes=128, ex3x3_planes=128, bn_momentum=self.bn_momentum, l2=self.l2) 259 | self.fire5 = FIRE(sq1x1_planes=32, ex1x1_planes=128, ex3x3_planes=128, bn_momentum=self.bn_momentum, l2=self.l2) 260 | 261 | self.fire6 = FIRE(sq1x1_planes=48, ex1x1_planes=192, ex3x3_planes=192, bn_momentum=self.bn_momentum, l2=self.l2) 262 | self.fire7 = FIRE(sq1x1_planes=48, ex1x1_planes=192, ex3x3_planes=192, bn_momentum=self.bn_momentum, l2=self.l2) 263 | self.fire8 = FIRE(sq1x1_planes=64, ex1x1_planes=256, ex3x3_planes=256, bn_momentum=self.bn_momentum, l2=self.l2) 264 | self.fire9 = FIRE(sq1x1_planes=64, ex1x1_planes=256, ex3x3_planes=256, bn_momentum=self.bn_momentum, l2=self.l2) 265 | 266 | # Decoder 267 | self.fire10 = FIREUP(sq1x1_planes=64, ex1x1_planes=128, ex3x3_planes=128, stride=2, bn_momentum=self.bn_momentum, 268 | l2=self.l2) 269 | self.fire11 = FIREUP(sq1x1_planes=32, ex1x1_planes=64, ex3x3_planes=64, stride=2, bn_momentum=self.bn_momentum, 270 | l2=self.l2) 271 | self.fire12 = FIREUP(sq1x1_planes=16, ex1x1_planes=32, ex3x3_planes=32, stride=2, bn_momentum=self.bn_momentum, 272 | l2=self.l2) 273 | self.fire13 = FIREUP(sq1x1_planes=16, ex1x1_planes=32, ex3x3_planes=32, stride=2, bn_momentum=self.bn_momentum, 274 | l2=self.l2) 275 | 276 | self.conv14 = tf.keras.layers.Conv2D( 277 | filters=self.NUM_CLASS, 278 | kernel_size=3, 279 | strides=1, 280 | padding='SAME', 281 | kernel_regularizer=tf.keras.regularizers.L2(l2=self.l2) 282 | ) 283 | self.dropout = tf.keras.layers.Dropout(self.drop_rate) 284 | 285 | def call(self, inputs, training=False, mask=None): 286 | lidar_input, lidar_mask = inputs[0], inputs[1] 287 | 288 | # Encoder 289 | x = tf.nn.relu(self.bn1(self.conv1(lidar_input))) 290 | 291 | cam1_output = self.cam1(x) 292 | 293 | conv1_skip = self.bn1_skip(self.conv1_skip(lidar_input)) 294 | 295 | x = tf.nn.max_pool2d(cam1_output, ksize=3, strides=[1, 2], padding='SAME') 296 | x = self.fire2(x) 297 | x = self.cam2(x) 298 | x = self.fire3(x) 299 | cam3_output = self.cam3(x) 300 | 301 | x = tf.nn.max_pool2d(cam3_output, ksize=3, strides=[1, 2], padding='SAME') 302 | x = self.fire4(x) 303 | fire5_output = self.fire5(x) 304 | 305 | x = tf.nn.max_pool2d(fire5_output, ksize=3, strides=[1, 2], padding='SAME') 306 | x = self.fire6(x) 307 | x = self.fire7(x) 308 | x = self.fire8(x) 309 | fire9_output = self.fire9(x) 310 | 311 | # Decoder 312 | x = self.fire10(fire9_output) 313 | x = tf.add(x, fire5_output) 314 | x = self.fire11(x) 315 | x = tf.add(x, cam3_output) 316 | x = self.fire12(x) 317 | x = tf.add(x, cam1_output) 318 | x = self.fire13(x) 319 | x = tf.add(x, conv1_skip) 320 | 321 | x = self.dropout(x, training) 322 | 323 | logits = self.conv14(x) 324 | 325 | return self.segmentation_head(logits, lidar_mask) 326 | 327 | def get_config(self): 328 | config = super(SqueezeSegV2, self).get_config() 329 | config.update({"mc": self.mc}) 330 | return config 331 | 332 | @classmethod 333 | def from_config(cls, config): 334 | return cls(**config) 335 | -------------------------------------------------------------------------------- /pcl_segmentation/nets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/pcl_segmentation/nets/__init__.py -------------------------------------------------------------------------------- /pcl_segmentation/preprocessing/convert_validation_pcd_to_npy.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import os 26 | import glob 27 | import numpy as np 28 | import cv2 29 | import argparse 30 | 31 | from configs import SqueezeSegV2Config 32 | 33 | ignore_id = 10 34 | 35 | cityscapes_trainId_to_trainId = { 36 | # Road 37 | 1: 0, 38 | 39 | # Sidewalk 40 | 2: 1, 41 | 42 | # Building 43 | 15: 2, # Building 44 | 16: 2, # Wall 45 | 19: 2, # Bridge 46 | 20: 2, # Tunnel 47 | 48 | # Pole 49 | 21: 3, # Pole 50 | 23: 3, # Traffic Sign 51 | 24: 3, # Traffic Light 52 | 53 | # Vegetation 54 | 25: 4, # Vegetation 55 | 26: 4, # Terrain 56 | 57 | # Person 58 | 5: 5, 59 | 60 | # Two Wheeler 61 | 6: 6, # Rider 62 | 12: 6, # Bicycle 63 | 11: 6, # Motorcycle 64 | 65 | # Car 66 | 7: 7, # Car 67 | 14: 7, # Trailer 68 | 69 | # Truck 70 | 8: 8, 71 | 72 | # Bus 73 | 9: 9, # Bus 74 | 13: 9, # Caravan 75 | 76 | # None 77 | 27: ignore_id, # Sky class 78 | 0: ignore_id, 79 | 255: ignore_id 80 | } 81 | 82 | 83 | def generateTrainIdMap(length=256): 84 | """ 85 | Create a numpy array of length 'length' which maps the index to the trainID 86 | """ 87 | map = np.ones((length)) * ignore_id 88 | for k, v in cityscapes_trainId_to_trainId.items(): 89 | map[k] = v 90 | return map 91 | 92 | 93 | def normalize(x): 94 | return (x - x.min()) / (x.max() - x.min()) 95 | 96 | 97 | def pcl_xyz_i_r_d_l_to_information_map(pcl, 98 | H=32, W=240, C=7, 99 | leftPhi=np.radians(24.32), 100 | rightPhi=np.radians(22.23)): 101 | """ 102 | Project velodyne points into front view depth map. 103 | :param pcl: velodyne points in shape [:,5] 104 | :param H: the row num of depth map, could be 64(default), 32, 16 105 | :param W: the col num of depth map 106 | :param C: the channel size of depth map 107 | 3 cartesian coordinates (x; y; z), 108 | an intensity measurement and 109 | range r = sqrt(x^2 + y^2 + z^2) 110 | :param dtheta: the delta theta of H, in radian 111 | :param dphi: the delta phi of W, in radian 112 | :param use_ring: Use the ring index as height or projection of theta 113 | :return: `depth_map`: the projected depth map of shape[H,W,C] 114 | """ 115 | 116 | x = pcl[:, 0] 117 | y = pcl[:, 1] 118 | z = pcl[:, 2] 119 | i = pcl[:, 3] 120 | r = pcl[:, 4].astype(int) 121 | d = pcl[:, 5] 122 | l = pcl[:, 6].astype(int) 123 | 124 | # projection on to a 2D plane 125 | deltaPhi_ = (rightPhi + leftPhi) / W 126 | phi = np.arctan2(y, x) 127 | phi_ = ((leftPhi - phi) / deltaPhi_).astype(int) 128 | 129 | # mask 130 | mask = np.zeros_like(l) 131 | mask[d > 0] = 1 132 | 133 | # points that exceed the boundaries must be removed 134 | delete_ids = np.where(np.logical_or(phi_ < 0, phi_ >= W)) 135 | 136 | x = np.delete(x, delete_ids) 137 | y = np.delete(y, delete_ids) 138 | z = np.delete(z, delete_ids) 139 | i = np.delete(i, delete_ids) 140 | r = np.delete(r, delete_ids) 141 | d = np.delete(d, delete_ids) 142 | l = np.delete(l, delete_ids) 143 | phi_ = np.delete(phi_, delete_ids) 144 | mask = np.delete(mask, delete_ids) 145 | 146 | information_map = np.zeros((H, W, C)) 147 | 148 | information_map[(H - 1) - r, phi_, 0] = x 149 | information_map[(H - 1) - r, phi_, 1] = y 150 | information_map[(H - 1) - r, phi_, 2] = z 151 | information_map[(H - 1) - r, phi_, 3] = i 152 | information_map[(H - 1) - r, phi_, 4] = d 153 | information_map[(H - 1) - r, phi_, 5] = l 154 | information_map[(H - 1) - r, phi_, 6] = mask 155 | 156 | return information_map 157 | 158 | 159 | def main(args): 160 | trainIdMap = generateTrainIdMap() 161 | mc = SqueezeSegV2Config() # only for visualisations 162 | 163 | # initialize emtpy dict to count the class distribution 164 | class_distribution = {} 165 | for k, v in cityscapes_trainId_to_trainId.items(): 166 | class_distribution[v] = 0 167 | 168 | # retrieve *.npy files 169 | files = glob.glob(os.path.join(args.input_dir, "*.pcd")) 170 | 171 | for file in files: 172 | with open(file, "r") as pcd_file: 173 | lines = [line.strip().split(" ") for line in pcd_file.readlines()] 174 | 175 | print("Parsing file: ", file) 176 | 177 | pcl = [] 178 | is_data = False 179 | for line in lines: 180 | if line[0] == 'DATA': # skip the header 181 | is_data = True 182 | continue 183 | if is_data: 184 | x = float(line[0]) # x 185 | y = float(line[1]) # y 186 | z = float(line[2]) # z 187 | i = int(line[3]) # intensity 188 | r = float(line[4]) # ring 189 | l = int(line[5]) # label id 190 | o = int(line[6]) # object id 191 | d = np.sqrt(x ** 2 + y ** 2 + z ** 2) 192 | 193 | # not labled points 194 | if o == -1 and l == 0: 195 | l = ignore_id 196 | 197 | # only points in the camera frame 198 | if x > 0: 199 | point = np.array((x, y, z, i, r, d, l)) 200 | pcl.append(point) 201 | 202 | # spherical projection 203 | information_map = pcl_xyz_i_r_d_l_to_information_map(np.asarray(pcl)) 204 | 205 | # apply trainId Mapping 206 | information_map[:, :, 5] = trainIdMap[information_map[:, :, 5].astype(int)] 207 | 208 | # get binary mask 209 | binary_mask = information_map[:, :, 6] 210 | binary_mask_copy = binary_mask.copy() 211 | 212 | # apply mask 213 | information_map[binary_mask == 0] = 0 214 | information_map[:, :, 5][binary_mask == 0] = ignore_id # set label to ignore_id 215 | 216 | # for class distribution statistics 217 | unique, counts = np.unique(information_map[:, :, 5], return_counts=True) 218 | for id, count in zip(unique, counts): 219 | class_distribution[id] += count 220 | 221 | if args.vis_dir: 222 | # x cloud 223 | cloud = (normalize(information_map[:, :, 0].copy()) * 255).astype(np.uint8) 224 | cloud = cv2.applyColorMap(cloud, cv2.COLORMAP_JET) 225 | cloud = cv2.resize(cloud, (0, 0), fx=3, fy=3) 226 | cv2.imshow("spherical x image", cloud) 227 | cv2.imwrite(args.vis_dir + "/" + "x.png", cloud) 228 | cv2.waitKey(1) 229 | 230 | # y cloud 231 | cloud = (normalize(information_map[:, :, 1].copy()) * 255).astype(np.uint8) 232 | cloud = cv2.applyColorMap(cloud, cv2.COLORMAP_JET) 233 | cloud = cv2.resize(cloud, (0, 0), fx=3, fy=3) 234 | cv2.imshow("spherical y image", cloud) 235 | cv2.imwrite(args.vis_dir + "/" + "y.png", cloud) 236 | cv2.waitKey(1) 237 | 238 | # y cloud 239 | cloud = (normalize(information_map[:, :, 2].copy()) * 255).astype(np.uint8) 240 | cloud = cv2.applyColorMap(cloud, cv2.COLORMAP_JET) 241 | cloud = cv2.resize(cloud, (0, 0), fx=3, fy=3) 242 | cv2.imshow("spherical z image", cloud) 243 | cv2.imwrite(args.vis_dir + "/" + "z.png", cloud) 244 | cv2.waitKey(1) 245 | 246 | # intensity cloud 247 | cloud = (normalize(information_map[:, :, 3].copy()) * 255).astype(np.uint8) 248 | cloud = cv2.applyColorMap(cloud, cv2.COLORMAP_JET) 249 | cloud = cv2.resize(cloud, (0, 0), fx=3, fy=3) 250 | cv2.imshow("spherical intensity image", cloud) 251 | cv2.imwrite(args.vis_dir + "/" + "i.png", cloud) 252 | cv2.waitKey(1) 253 | 254 | # depth and normalize 255 | cloud = (normalize(information_map[:, :, 4].copy()) * 255).astype(np.uint8) 256 | cloud = cv2.applyColorMap(cloud, cv2.COLORMAP_JET) 257 | cloud = cv2.resize(cloud, (0, 0), fx=3, fy=3) 258 | cv2.imshow("spherical depth image", cloud) 259 | cv2.imwrite(args.vis_dir + "/" + "d.png", cloud) 260 | cv2.waitKey(1) 261 | 262 | # label cloud 263 | label_cloud = ((255 * mc.CLS_COLOR_MAP[information_map[:, :, 5].astype(np.uint8)]).astype(np.uint8)) 264 | label_cloud = cv2.cvtColor(label_cloud, cv2.COLOR_RGB2BGR) 265 | label_cloud = cv2.resize(label_cloud, (0, 0), fx=3, fy=3) 266 | cv2.imshow("spherical label image", label_cloud) 267 | cv2.imwrite(args.vis_dir + "/" + "l.png", label_cloud) 268 | cv2.waitKey(1) 269 | 270 | # mask 271 | mask = (normalize(binary_mask_copy) * 255).astype(np.uint8) 272 | # mask = cv2.applyColorMap(mask, cv2.COLORMAP_JET) 273 | mask = cv2.resize(mask, (0, 0), fx=3, fy=3) 274 | cv2.imshow("spherical mask image", mask) 275 | cv2.imwrite(args.vis_dir + "/" + "m.png", mask) 276 | cv2.waitKey(1) 277 | 278 | # write numpy tensor to file .npy 279 | if args.output_dir: 280 | filename = os.path.join(args.output_dir, os.path.basename(file).split(".")[0] + ".npy") 281 | print("Write tensor matrix to {0}".format(filename)) 282 | print(np.unique(information_map[:, :, 5])) 283 | np.save(filename, information_map[:, :, :6]) 284 | 285 | 286 | print("Absolute Class Distribution") 287 | print(class_distribution) 288 | sum = np.sum(list(class_distribution.values())) 289 | 290 | for k, v in class_distribution.items(): 291 | class_distribution[k] = v / sum 292 | 293 | print("Normalized Class Distribution") 294 | print(class_distribution) 295 | 296 | 297 | if __name__ == '__main__': 298 | parser = argparse.ArgumentParser(description="Parse flags for the manually annotated PCD to *.NPY conversion") 299 | parser.add_argument('-o', '--output_dir', type=str, help="Output dir for the *.npy files. If flag not defined, then" 300 | "the numpy files are not stored.") 301 | parser.add_argument('-v', '--vis_dir', type=str, help="Output dir for the visualizations. If flag not defined, then" 302 | "visualizations are not stored.") 303 | parser.add_argument('-i', '--input_dir', required=True, type=str, help="Input dir which contains *.pcd files, created" 304 | "by an labeling tool. These are the raw labled" 305 | "point clouds") 306 | args = parser.parse_args() 307 | main(args) 308 | -------------------------------------------------------------------------------- /pcl_segmentation/preprocessing/inspect_training_data.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import os 26 | import sys 27 | import glob 28 | import numpy as np 29 | import matplotlib.pyplot as plt 30 | import tqdm 31 | import cv2 32 | import time 33 | import argparse 34 | 35 | 36 | sys.path.append(".") 37 | from configs import SqueezeSegV2Config 38 | 39 | 40 | class RunningStd(object): 41 | def __init__(self): 42 | self.s0, self.s1, self.s2 = 0.0, 0.0, 0.0 43 | 44 | def include(self, array): 45 | self.s0 += array.size 46 | self.s1 += np.sum(array) 47 | self.s2 += np.sum(np.square(array)) 48 | 49 | @property 50 | def std(self): 51 | return np.sqrt((self.s0 * self.s2 - self.s1 * self.s1) / (self.s0 * (self.s0 - 1))) 52 | 53 | 54 | def main(args): 55 | config = SqueezeSegV2Config() # only for viz 56 | 57 | input_files = sorted(glob.glob(os.path.join(args.input_dir, "*.npy"))) 58 | 59 | if args.output_dir: 60 | outfile_path = os.path.join(args.output_dir, "train.txt") 61 | output_mask_path = os.path.join(args.output_dir, "mask.npy") 62 | else: 63 | outfile_path, output_mask_path = None, None 64 | 65 | target_width = 1024 66 | target_height = 32 67 | 68 | print("Number of Files:", len(input_files)) 69 | 70 | # for means 71 | all_x_means = [] 72 | all_y_means = [] 73 | all_z_means = [] 74 | all_i_means = [] 75 | all_d_means = [] 76 | 77 | all_depths = [] 78 | 79 | # for mask 80 | running_mask = np.zeros((target_height, target_width)) 81 | 82 | # for std 83 | ov_x = RunningStd() 84 | ov_y = RunningStd() 85 | ov_z = RunningStd() 86 | ov_i = RunningStd() 87 | ov_d = RunningStd() 88 | 89 | class_wise_count = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0} 90 | 91 | with open(outfile_path, 'w') as outfile: 92 | for idx, f in tqdm.tqdm(enumerate(input_files), total=len(input_files)): 93 | lidar = np.load(f) 94 | 95 | lidar_mask = lidar[:, :, 4] > 0 96 | 97 | all_x_means.append(np.mean(lidar[:, :, 0][lidar_mask])) 98 | ov_x.include(lidar[:, :, 0][lidar_mask]) 99 | 100 | all_y_means.append(np.mean(lidar[:, :, 1][lidar_mask])) 101 | ov_y.include(lidar[:, :, 1][lidar_mask]) 102 | 103 | all_z_means.append(np.mean(lidar[:, :, 2][lidar_mask])) 104 | ov_z.include(lidar[:, :, 2][lidar_mask]) 105 | 106 | all_i_means.append(np.mean(lidar[:, :, 3][lidar_mask])) 107 | ov_i.include(lidar[:, :, 3][lidar_mask]) 108 | 109 | all_d_means.append(np.mean(lidar[:, :, 4][lidar_mask])) 110 | ov_d.include(lidar[:, :, 4][lidar_mask]) 111 | 112 | if len(all_depths) < 5000: 113 | all_depths.append(lidar[:, :, 4]) 114 | 115 | running_mask += np.reshape((lidar[:, :, 4] > 0), [target_height, target_width]) 116 | """ 117 | unique, count = np.unique(lidar[:, :, 5], return_counts=True) 118 | for train_id, c in zip(unique, count): 119 | try: 120 | class_wise_count[int(train_id)] += c 121 | except KeyError: 122 | print("WARNING INVALID LABEL ID FOUND: ", int(train_id)) 123 | print("FILE: ", f) 124 | print("SKIPPING CURRENT SAMPLE") 125 | continue 126 | 127 | if idx % 100 == 0 and args.do_visualisation: 128 | print("") 129 | print(np.min(lidar[:, :, 3][lidar_mask])) 130 | print(np.max(lidar[:, :, 3][lidar_mask])) 131 | 132 | label_cloud = ((255 * config.CLS_COLOR_MAP[lidar[:, :, 5].astype(np.uint32)]).astype(np.uint8)) 133 | label_cloud = cv2.cvtColor(label_cloud, cv2.COLOR_RGB2BGR) 134 | label_cloud = cv2.resize(label_cloud, (0, 0), fx=3, fy=3) 135 | cv2.imshow("Spherical Label Image", label_cloud) 136 | cv2.waitKey(1) 137 | time.sleep(0.05) 138 | """ 139 | if outfile_path: 140 | outfile.write(f.split("/")[-1].split(".")[0] + os.linesep) 141 | 142 | print("x_mean = ", np.mean(all_x_means)) 143 | print("x_std = ", ov_x.std) 144 | 145 | print("y_mean = ", np.mean(all_y_means)) 146 | print("y_std = ", ov_y.std) 147 | 148 | print("z_mean = ", np.mean(all_z_means)) 149 | print("z_std = ", ov_z.std) 150 | 151 | print("intensities_mean = ", np.mean(all_i_means)) 152 | print("intensities_std = ", ov_i.std) 153 | 154 | print("distances_mean = ", np.mean(all_d_means)) 155 | print("distances_std = ", ov_d.std) 156 | 157 | mask = np.mean(all_depths, axis=0) 158 | 159 | plt.imshow(mask, cmap='gray') 160 | plt.show() 161 | 162 | # to perform prediction 163 | lidar_mask = np.reshape( 164 | (mask > 0), 165 | [target_height, target_width, 1] 166 | ) 167 | print("lidar mask shape: ", np.shape(lidar_mask)) 168 | plt.imshow(lidar_mask[:, :, 0], cmap='gray') 169 | plt.show() 170 | 171 | if args.output_dir: 172 | np.save(output_mask_path, lidar_mask) 173 | 174 | print("number of labels for each class:") 175 | print(class_wise_count) 176 | 177 | all_pixels = sum(list(class_wise_count.values())) 178 | 179 | for k, v in class_wise_count.items(): 180 | print("Class ID: ", k) 181 | print("Ratio: ", v / all_pixels) 182 | print("") 183 | 184 | 185 | if __name__ == '__main__': 186 | parser = argparse.ArgumentParser(description="Parse flags for the manually annotated PCD to *.NPY conversion") 187 | parser.add_argument('-i', '--input_dir', type=str, required=True, help='Input directory which contains the *.npy ' 188 | 'files. A glob patter `*.npy` is applied to ' 189 | 'this path') 190 | parser.add_argument('-o', '--output_dir', type=str, help='Output path to which `train.txt` and also' 191 | '`mask.npy` is applied. If not specified, no files' 192 | 'will be written and a test run is performed.') 193 | parser.add_argument('-v', '--do_visualisation', type=bool, default=False, 194 | help='Visualize the data during the computation') 195 | 196 | args = parser.parse_args() 197 | main(args) 198 | -------------------------------------------------------------------------------- /pcl_segmentation/train.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import os.path 26 | 27 | import argparse 28 | 29 | from data_loader import DataLoader 30 | from utils.callbacks import TensorBoard 31 | from utils.util import * 32 | from utils.args_loader import load_model_config 33 | 34 | 35 | def train(arg): 36 | config, model = load_model_config(args.model, args.config) 37 | 38 | train = DataLoader("train", arg.data_path, config).write_tfrecord_dataset().read_tfrecord_dataset() 39 | val = DataLoader("val", arg.data_path, config).write_tfrecord_dataset().read_tfrecord_dataset() 40 | 41 | tensorboard_callback = TensorBoard(arg.train_dir, val, profile_batch=(200, 202)) 42 | checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(os.path.join(arg.train_dir, "checkpoint")) 43 | 44 | lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( 45 | initial_learning_rate=config.LEARNING_RATE, 46 | decay_steps=config.LR_DECAY_STEPS, 47 | decay_rate=config.LR_DECAY_FACTOR, 48 | staircase=True) 49 | 50 | optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule, clipnorm=config.MAX_GRAD_NORM) 51 | 52 | model.compile(optimizer=optimizer, weighted_metrics=[]) 53 | 54 | model.fit(train, 55 | validation_data=val, 56 | epochs=arg.epochs, 57 | callbacks=[tensorboard_callback, checkpoint_callback], 58 | ) 59 | 60 | model.save(filepath=os.path.join(arg.train_dir, 'model'), save_traces=True) 61 | 62 | 63 | if __name__ == '__main__': 64 | physical_devices = tf.config.experimental.list_physical_devices('GPU') 65 | if len(physical_devices) > 0: 66 | tf.config.experimental.set_memory_growth(physical_devices[0], True) 67 | 68 | parser = argparse.ArgumentParser(description='Parse Flags for the training script!') 69 | parser.add_argument('-d', '--data_path', type=str, 70 | help='Absolute path to the dataset') 71 | parser.add_argument('-e', '--epochs', type=int, default=50, 72 | help='Maximal number of training epochs') 73 | parser.add_argument('-t', '--train_dir', type=str, 74 | help="Directory where to write the Tensorboard logs and checkpoints") 75 | parser.add_argument('-m', '--model', type=str, 76 | help='Model name either `squeezesegv2`, `darknet53`, `darknet21`') 77 | parser.add_argument('-c', '--config', type=str, 78 | help='Which configuration for training `squeezesegv2`, `squeezesegv2kitti`, ' 79 | ' `darknet53`, `darknet21` ') 80 | args = parser.parse_args() 81 | 82 | train(args) 83 | -------------------------------------------------------------------------------- /pcl_segmentation/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ika-rwth-aachen/PCLSegmentation/f308a7e2c90da6d061bed92489c0cd3f0b57a1e1/pcl_segmentation/utils/__init__.py -------------------------------------------------------------------------------- /pcl_segmentation/utils/args_loader.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | from nets.Darknet import Darknet 26 | from nets.SqueezeSegV2 import SqueezeSegV2 27 | 28 | from configs.SqueezeSegV2 import SqueezeSegV2Config 29 | from configs.SqueezeSegV2Kitti import SqueezeSegV2KittiConfig 30 | from configs.SqueezeSegV2NuScenes import SqueezeSegV2ConfigNuScenes 31 | from configs.Darknet53 import Darknet53 32 | from configs.Darknet21 import Darknet21 33 | from configs.Darknet53Kitti import Darknet53Kitti 34 | 35 | 36 | model_map = { 37 | "squeezesegv2": SqueezeSegV2, 38 | "darknet53": Darknet, 39 | "darknet21": Darknet 40 | } 41 | 42 | config_map = { 43 | "squeezesegv2": SqueezeSegV2Config, 44 | "darknet53": Darknet53, 45 | "darknet21": Darknet21, 46 | "darknet53kitti": Darknet53Kitti, 47 | "squeezesegv2kitti": SqueezeSegV2KittiConfig, 48 | "squeezesegv2nuscenes": SqueezeSegV2ConfigNuScenes 49 | } 50 | 51 | 52 | def load_model_config(model_name, config_name): 53 | config = config_map[config_name.lower()]() 54 | model = model_map[model_name.lower()](config) 55 | return config, model 56 | -------------------------------------------------------------------------------- /pcl_segmentation/utils/callbacks.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import tensorflow as tf 26 | 27 | from utils.util import plot_confusion_matrix, confusion_matrix_to_iou_recall_precision, plot_to_image 28 | 29 | 30 | class TensorBoard(tf.keras.callbacks.TensorBoard): 31 | """Callback for storing the intermediate results of the point cloud segmentation networks""" 32 | 33 | def __init__(self, log_dir, dataset, **kwargs): 34 | super().__init__(log_dir, **kwargs) 35 | self.dataset = dataset 36 | self.num_images = 1 37 | self.custom_tb_writer = tf.summary.create_file_writer(self.log_dir + '/validation') 38 | 39 | def on_train_batch_end(self, batch, logs=None): 40 | lr = getattr(self.model.optimizer, 'lr', None) 41 | steps = self.model.optimizer.iterations 42 | with self.custom_tb_writer.as_default(): 43 | tf.summary.scalar('step_learning_rate', lr(steps), steps) 44 | super().on_train_batch_end(batch, logs) 45 | 46 | def on_epoch_end(self, epoch, logs=None): 47 | batch_size = self.model.BATCH_SIZE 48 | class_color_map = self.model.CLS_COLOR_MAP 49 | 50 | # get first batch of dataset 51 | (lidar_input, lidar_mask), label, weight = self.dataset.take(1).get_single_element() 52 | 53 | probabilities, predictions = self.model([lidar_input, lidar_mask]) 54 | 55 | label = label[:self.num_images, :, :] 56 | weight = weight[:self.num_images, :, :].numpy() 57 | predictions = predictions[:self.num_images, :, :].numpy() 58 | 59 | # label and prediction visualizations 60 | label_image = class_color_map[label.numpy().reshape(-1)].reshape([self.num_images, label.shape[1], label.shape[2], 3]) 61 | pred_image = class_color_map[predictions.reshape(-1)].reshape([self.num_images, label.shape[1], label.shape[2], 3]) 62 | weight_image = weight.reshape([self.num_images, weight.shape[1], weight.shape[2], 1]) 63 | depth_image = lidar_input.numpy()[:self.num_images, :, :, [4]] 64 | intensity = lidar_input.numpy()[:self.num_images, :, :, [3]] 65 | 66 | intensity_image = tf.image.resize(intensity, [intensity.shape[1]*3, intensity.shape[2]*3]) 67 | depth_image = tf.image.resize(depth_image, [depth_image.shape[1]*3, depth_image.shape[2]*3]) 68 | weight_image = tf.image.resize(weight_image, [weight.shape[1]*3, weight.shape[2]*3]) 69 | label_image = tf.image.resize(label_image, [label_image.shape[1]*3, label_image.shape[2]*3]) 70 | pred_image = tf.image.resize(pred_image, [pred_image.shape[1]*3, pred_image.shape[2]*3]) 71 | 72 | # confusion matrix visualization 73 | figure = plot_confusion_matrix(self.model.miou_tracker.total_cm.numpy(), 74 | class_names=self.model.mc.CLASSES) 75 | cm_image = plot_to_image(figure) 76 | 77 | with self.custom_tb_writer.as_default(): 78 | tf.summary.image('Images/Depth Image', 79 | depth_image, 80 | max_outputs=batch_size, 81 | step=epoch) 82 | tf.summary.image('Images/Intensity Image', 83 | intensity_image, 84 | max_outputs=batch_size, 85 | step=epoch) 86 | tf.summary.image('Images/Weight Image', 87 | weight_image, 88 | max_outputs=batch_size, 89 | step=epoch) 90 | tf.summary.image('Images/Label Image', 91 | label_image, 92 | max_outputs=batch_size, 93 | step=epoch) 94 | tf.summary.image('Images/Prediction Image', 95 | pred_image, 96 | max_outputs=batch_size, 97 | step=epoch) 98 | tf.summary.image("Confusion Matrix", 99 | cm_image, 100 | step=epoch) 101 | 102 | # Save IoU, Precision, Recall 103 | iou, recall, precision = confusion_matrix_to_iou_recall_precision(self.model.miou_tracker.total_cm) 104 | with self.custom_tb_writer.as_default(): 105 | for i, cls in enumerate(self.model.mc.CLASSES): 106 | tf.summary.scalar('IoU/'+cls, iou[i], step=epoch) 107 | tf.summary.scalar('Recall/'+cls, recall[i], step=epoch) 108 | tf.summary.scalar('Precision/'+cls, precision[i], step=epoch) 109 | 110 | super().on_epoch_end(epoch, logs) 111 | 112 | def on_test_end(self, logs=None): 113 | super().on_test_end(logs) 114 | 115 | def on_train_end(self, logs=None): 116 | super().on_train_end(logs) 117 | self.custom_tb_writer.close() -------------------------------------------------------------------------------- /pcl_segmentation/utils/util.py: -------------------------------------------------------------------------------- 1 | # ============================================================================== 2 | # MIT License 3 | # 4 | # Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University. 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | # ============================================================================== 24 | 25 | import io 26 | import itertools 27 | 28 | import numpy as np 29 | import tensorflow as tf 30 | from matplotlib import pyplot as plt 31 | 32 | 33 | def plot_confusion_matrix(cm, class_names): 34 | """ 35 | Returns a matplotlib figure containing the plotted confusion matrix. 36 | 37 | Args: 38 | cm (array, shape = [n, n]): a confusion matrix of integer classes 39 | class_names (array, shape = [n]): String names of the integer classes 40 | """ 41 | # normalize confusion matrix 42 | cm_normalized = np.around(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis], decimals=2) 43 | 44 | figure = plt.figure(figsize=(8, 8)) 45 | plt.imshow(cm_normalized, interpolation='nearest', cmap=plt.cm.Blues) 46 | plt.title("Normalized Confusion Matrix") 47 | plt.colorbar() 48 | tick_marks = np.arange(len(class_names)) 49 | plt.xticks(tick_marks, class_names, rotation=45) 50 | plt.yticks(tick_marks, class_names) 51 | 52 | # Use white text if squares are dark; otherwise black. 53 | threshold = cm.max() / 2. 54 | for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): 55 | color = "white" if cm[i, j] > threshold else "black" 56 | plt.text(j, i, cm_normalized[i, j], horizontalalignment="center", color=color) 57 | 58 | plt.tight_layout() 59 | plt.ylabel('True label') 60 | plt.xlabel('Predicted label') 61 | return figure 62 | 63 | 64 | def confusion_matrix_to_iou_recall_precision(cm): 65 | """ 66 | Computes the classwise iou, recall and precision from a confusion matrix 67 | cm: Confusion matrix as nxn matrix where n is the number of classes 68 | Confusion matrix has switched axes when taken from *total_cm* !! 69 | """ 70 | with tf.name_scope("compute_iou_recall_precision") as scope: 71 | sum_over_col = tf.reduce_sum(cm, axis=1) # axes are switched for the total_cm within MeanIoU 72 | sum_over_row = tf.reduce_sum(cm, axis=0) # axes are switched for the total_cm within MeanIoU 73 | tp = tf.linalg.diag_part(cm) 74 | fp = sum_over_row - tp 75 | fn = sum_over_col - tp 76 | iou = tf.math.divide_no_nan(tp, tp + fp + fn) 77 | recall = tf.math.divide_no_nan(tp, tp + fn) 78 | precision = tf.math.divide_no_nan(tp, tp + fp) 79 | return iou, recall, precision 80 | 81 | 82 | def plot_to_image(figure): 83 | """Converts the matplotlib plot specified by 'figure' to a PNG image and 84 | returns it. The supplied figure is closed and inaccessible after this call.""" 85 | # Save the plot to a PNG in memory. 86 | buf = io.BytesIO() 87 | plt.savefig(buf, format='png') 88 | # Closing the figure prevents it from being displayed directly inside 89 | # the notebook. 90 | plt.close(figure) 91 | buf.seek(0) 92 | # Convert PNG buffer to TF image 93 | image = tf.image.decode_png(buf.getvalue(), channels=4) 94 | # Add the batch dimension 95 | image = tf.expand_dims(image, 0) 96 | return image 97 | 98 | 99 | def normalize(x): 100 | return (x - x.min()) / (x.max() - x.min()) 101 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow-gpu==2.9.1 2 | scipy 3 | tqdm 4 | easydict 5 | pillow 6 | opencv-python 7 | matplotlib 8 | nuscenes-devkit 9 | pyntcloud 10 | pyyaml --------------------------------------------------------------------------------