├── .gitignore ├── .gitmodules ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── depoco ├── architectures │ ├── loss_handler.py │ ├── network_blocks.py │ └── original_kp_blocks.py ├── config │ ├── depoco.yaml │ └── kitti │ │ ├── e0.yaml │ │ ├── e1.yaml │ │ ├── e2.yaml │ │ └── e3.yaml ├── data_handling │ ├── VoxelGrid.py │ └── train_test_splitter.py ├── datasets │ ├── kitti2voxel.py │ └── submap_handler.py ├── evaluate.py ├── evaluation │ ├── evaluator.py │ └── occupancy_grid.py ├── experiments │ └── results │ │ └── kitti │ │ ├── e0.pkl │ │ ├── e1.pkl │ │ ├── e2.pkl │ │ └── e3.pkl ├── network_files │ ├── e0 │ │ ├── dec.pth │ │ ├── dec_best.pth │ │ ├── e0.yaml │ │ ├── enc.pth │ │ └── enc_best.pth │ ├── e1 │ │ ├── dec.pth │ │ ├── dec_best.pth │ │ ├── e1.yaml │ │ ├── enc.pth │ │ └── enc_best.pth │ ├── e2 │ │ ├── dec.pth │ │ ├── dec_best.pth │ │ ├── e2.yaml │ │ ├── enc.pth │ │ └── enc_best.pth │ └── e3 │ │ ├── dec.pth │ │ ├── dec_best.pth │ │ ├── e3.yaml │ │ ├── enc.pth │ │ └── enc_best.pth ├── notebooks │ └── visualize.ipynb ├── plot_results.py ├── trainer.py ├── utils │ ├── point_cloud_utils.py │ ├── upsampling_rating.py │ └── visualization │ │ ├── visualize_layer.py │ │ └── visualize_pointcloud.py └── visualize.py ├── requirements.txt ├── setup.py └── submodules └── octree_handler ├── .clang-format ├── .gitignore ├── .gitlab-ci.yml ├── CMakeLists.txt ├── setup.py ├── src ├── CMakeLists.txt ├── Octree.hpp ├── OctreeHandler.cpp └── OctreeHandler.h └── tests └── octree_cpp_testing.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,images 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks,images 4 | experiments/ 5 | experiments 6 | network_files/ 7 | data/ 8 | *.json 9 | ### Images ### 10 | # JPEG 11 | *.jpg 12 | *.jpeg 13 | *.jpe 14 | *.jif 15 | *.jfif 16 | *.jfi 17 | 18 | # JPEG 2000 19 | *.jp2 20 | *.j2k 21 | *.jpf 22 | *.jpx 23 | *.jpm 24 | *.mj2 25 | 26 | # JPEG XR 27 | *.jxr 28 | *.hdp 29 | *.wdp 30 | 31 | # Graphics Interchange Format 32 | *.gif 33 | 34 | # RAW 35 | *.raw 36 | 37 | # Web P 38 | *.webp 39 | 40 | # Portable Network Graphics 41 | *.png 42 | 43 | # Animated Portable Network Graphics 44 | *.apng 45 | 46 | # Multiple-image Network Graphics 47 | *.mng 48 | 49 | # Tagged Image File Format 50 | *.tiff 51 | *.tif 52 | 53 | # Scalable Vector Graphics 54 | *.svg 55 | *.svgz 56 | 57 | # Portable Document Format 58 | *.pdf 59 | 60 | # X BitMap 61 | *.xbm 62 | 63 | # BMP 64 | *.bmp 65 | *.dib 66 | 67 | # ICO 68 | *.ico 69 | 70 | # 3D Images 71 | *.3dm 72 | *.max 73 | 74 | ### JupyterNotebooks ### 75 | # gitignore template for Jupyter Notebooks 76 | # website: http://jupyter.org/ 77 | 78 | .ipynb_checkpoints 79 | */.ipynb_checkpoints/* 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # Remove previous ipynb_checkpoints 86 | # git rm -r .ipynb_checkpoints/ 87 | 88 | ### Python ### 89 | # Byte-compiled / optimized / DLL files 90 | __pycache__/ 91 | *.py[cod] 92 | *$py.class 93 | 94 | # C extensions 95 | *.so 96 | 97 | # Distribution / packaging 98 | .Python 99 | build/ 100 | develop-eggs/ 101 | dist/ 102 | downloads/ 103 | eggs/ 104 | .eggs/ 105 | lib/ 106 | lib64/ 107 | parts/ 108 | sdist/ 109 | var/ 110 | wheels/ 111 | pip-wheel-metadata/ 112 | share/python-wheels/ 113 | *.egg-info/ 114 | .installed.cfg 115 | *.egg 116 | MANIFEST 117 | 118 | # PyInstaller 119 | # Usually these files are written by a python script from a template 120 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 121 | *.manifest 122 | *.spec 123 | 124 | # Installer logs 125 | pip-log.txt 126 | pip-delete-this-directory.txt 127 | 128 | # Unit test / coverage reports 129 | htmlcov/ 130 | .tox/ 131 | .nox/ 132 | .coverage 133 | .coverage.* 134 | .cache 135 | nosetests.xml 136 | coverage.xml 137 | *.cover 138 | *.py,cover 139 | .hypothesis/ 140 | .pytest_cache/ 141 | pytestdebug.log 142 | 143 | # Translations 144 | *.mo 145 | *.pot 146 | 147 | # Django stuff: 148 | *.log 149 | local_settings.py 150 | db.sqlite3 151 | db.sqlite3-journal 152 | 153 | # Flask stuff: 154 | instance/ 155 | .webassets-cache 156 | 157 | # Scrapy stuff: 158 | .scrapy 159 | 160 | # Sphinx documentation 161 | docs/_build/ 162 | doc/_build/ 163 | 164 | # PyBuilder 165 | target/ 166 | 167 | # Jupyter Notebook 168 | 169 | # IPython 170 | 171 | # pyenv 172 | .python-version 173 | 174 | # pipenv 175 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 176 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 177 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 178 | # install all needed dependencies. 179 | #Pipfile.lock 180 | 181 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 182 | __pypackages__/ 183 | 184 | # Celery stuff 185 | celerybeat-schedule 186 | celerybeat.pid 187 | 188 | # SageMath parsed files 189 | *.sage.py 190 | 191 | # Environments 192 | .env 193 | .venv 194 | env/ 195 | venv/ 196 | ENV/ 197 | env.bak/ 198 | venv.bak/ 199 | pythonenv* 200 | 201 | # Spyder project settings 202 | .spyderproject 203 | .spyproject 204 | 205 | # Rope project settings 206 | .ropeproject 207 | 208 | # mkdocs documentation 209 | /site 210 | 211 | # mypy 212 | .mypy_cache/ 213 | .dmypy.json 214 | dmypy.json 215 | 216 | # Pyre type checker 217 | .pyre/ 218 | 219 | # pytype static type analyzer 220 | .pytype/ 221 | 222 | # profiling data 223 | .prof 224 | 225 | # End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,images 226 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "submodules/ChamferDistancePytorch"] 2 | path = submodules/ChamferDistancePytorch 3 | url = https://github.com/ThibaultGROUEIX/ChamferDistancePytorch 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Base image 2 | FROM nvidia/cuda:11.0.3-devel-ubuntu20.04 3 | 4 | # setup environment 5 | ENV TERM xterm 6 | ENV DEBIAN_FRONTEND=noninteractive 7 | ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/python3.8/dist-packages/torch/lib/ 8 | ENV PYTHONPATH=/depoco/submodules/ChamferDistancePytorch/ 9 | 10 | # Provide a data directory to share data across docker and the host system 11 | RUN mkdir -p /data 12 | 13 | # Install system packages 14 | RUN apt-get update && apt-get install --no-install-recommends -y \ 15 | build-essential \ 16 | cmake \ 17 | git \ 18 | libeigen3-dev \ 19 | libgl1-mesa-glx \ 20 | libusb-1.0-0-dev \ 21 | ninja-build \ 22 | pybind11-dev \ 23 | python3 \ 24 | python3-dev \ 25 | python3-pip \ 26 | vim \ 27 | && rm -rf /var/lib/apt/lists/* 28 | 29 | # Install Pytorch with CUDA 11 support 30 | RUN pip3 install \ 31 | torch==1.7.1+cu110 \ 32 | torchvision==0.8.2+cu110 \ 33 | torchaudio==0.7.2 \ 34 | -f https://download.pytorch.org/whl/torch_stable.html 35 | 36 | # Install python dependencies 37 | RUN pip3 install \ 38 | open3d \ 39 | tensorboard \ 40 | ruamel.yaml \ 41 | jupyterlab 42 | 43 | # Copy the libary to the docker image 44 | COPY ./ depoco/ 45 | 46 | # Install depoco and 3rdparty dependencies 47 | RUN cd depoco/ && pip3 install -U -e . 48 | RUN cd depoco/submodules/octree_handler && pip3 install -U . 49 | RUN cd depoco/submodules/ChamferDistancePytorch/chamfer3D/ && pip3 install -U . 2>/dev/null 50 | 51 | WORKDIR /depoco/depoco 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Louis Wiesmann & Cyrill Stachniss 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | IMAGE_NAME=depoco 2 | TAG=latest 3 | DATASETS= /media/lwiesmann/WiesmannIPB/data/data_kitti/dataset/submaps/40m_ILEN/ 4 | 5 | build: 6 | @echo Building docker container $(IMAGE_NAME) 7 | nvidia-docker build -t $(IMAGE_NAME):$(TAG) . 8 | 9 | test: 10 | @echo NVIDIA and CUDA setup 11 | @nvidia-docker run --rm $(IMAGE_NAME):$(TAG) nvidia-smi 12 | @echo PytTorch CUDA setup installed? 13 | @nvidia-docker run --rm $(IMAGE_NAME):$(TAG) python3 -c "import torch; print(torch.cuda.is_available())" 14 | 15 | run: 16 | docker run --rm --gpus all -p 8888:8888 -it -v $(DATASETS):/data $(IMAGE_NAME) 17 | 18 | clean: 19 | @echo Removing docker image... 20 | -docker image rm --force $(IMAGE_NAME):$(TAG) 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DEPOCO 2 | 3 | This repository implements the algorithms described in our paper [Deep Compression for Dense Point Cloud Maps](https://www.ipb.uni-bonn.de/wp-content/papercite-data/pdf/wiesmann2021ral.pdf). 4 | 5 | ## How to get started (using Docker) 6 | 7 | ### Dependenices nvida-docker 8 | 9 | Install nvida-docker and follow [these](https://stackoverflow.com/a/61737404) 10 | instructions 11 | 12 | ## Data 13 | You can download the dataset from [here](https://www.ipb.uni-bonn.de/html/projects/depoco/submaps.zip) and link the dataset to the docker container by configuring the Makefile 14 | 15 | ```sh 16 | DATASETS= 17 | ``` 18 | 19 | ## Building the docker container 20 | 21 | For building the Docker Container simply run 22 | 23 | ```sh 24 | make build 25 | ``` 26 | 27 | in the root directory. 28 | 29 | ## Running the Code 30 | 31 | The first step is to run the docker container: 32 | 33 | ```sh 34 | make run 35 | ``` 36 | 37 | The following commands assume to be run inside the docker container. 38 | 39 | ### Training 40 | 41 | For training a network we first have to create the config file with all the parameters. 42 | An example of this can be found in `/depoco/config/depoco.yaml`. 43 | Make sure to give each config file a unique `experiment_id: ...` to not override previous models. 44 | To train the network simply run 45 | 46 | ```sh 47 | python3 trainer -cfg 48 | ``` 49 | 50 | ### Evaluation 51 | 52 | Evaluating the network on the test set can be done by: 53 | 54 | ```sh 55 | python3 evaluate.py -cfg 56 | ``` 57 | 58 | All results will be saved in a dictonary. 59 | 60 | ### Plotting the results 61 | 62 | We can plot the quantitative results e.g. by using Jupyter-Lab. 63 | An example of this is provided in `depoco/notebooks/visualize.ipynb`. 64 | Jupyter-Lab can be started in the Docker container by: 65 | 66 | ```sh 67 | jupyter-lab --ip 0.0.0.0 --no-browser --allow-root 68 | ``` 69 | 70 | The 8888 port is forwarded which allows us to use it as if it would be on the host machine. 71 | 72 | ### Pretrained models 73 | 74 | The config files and the pretrained weights of our models are stored in `depoco/network_files/eX/`. The results can be inspected by the jupyter notebook `depoco/notebooks/visualize.ipynb`. 75 | 76 | ## How to get started (without Docker) 77 | 78 | ### Installation 79 | 80 | A list of all dependencies and install instructions can be derived from the Dockerfile. 81 | 82 | ### Running the code 83 | 84 | After installation the training and evaluation can be run as explained before. 85 | 86 | ### Qualitative Results 87 | 88 | Plotting the point clouds using open3d can be done by 89 | 90 | ```sh 91 | pyhon3 evaluate -cfg 92 | ``` 93 | 94 | This can **not** be done in the docker container and thus requires the installation on the local machine. 95 | 96 | ## Acknowledgements 97 | 98 | Big thanks to [Ignacio Vizzo](https://github.com/nachovizzo) for supporting me with Docker! 99 | 100 | ## Citation 101 | 102 | If you use this library for any academic work, please cite the original paper. 103 | 104 | ```bibtex 105 | @article{wiesmann2021ral, 106 | author = {L. Wiesmann and A. Milioto and X. Chen and C. Stachniss and J. Behley}, 107 | title = {{Deep Compression for Dense Point Cloud Maps}}, 108 | journal = {IEEE Robotics and Automation Letters (RA-L)}, 109 | volume = 6, 110 | issue = 2, 111 | pages = {2060-2067}, 112 | doi = {10.1109/LRA.2021.3059633}, 113 | year = 2021 114 | } 115 | ``` 116 | -------------------------------------------------------------------------------- /depoco/architectures/loss_handler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | # import depoco.sample_net_trainer as snt 4 | import depoco.architectures.original_kp_blocks as okp 5 | import chamfer3D.dist_chamfer_3D 6 | import depoco.architectures.network_blocks as network_blocks 7 | 8 | def linDeconvRegularizer(net, weight,gt_points): 9 | cham_loss = chamfer3D.dist_chamfer_3D.chamfer_3DDist() 10 | loss = torch.tensor( 11 | 0.0, dtype=torch.float32, device=gt_points.device) # init loss 12 | for m in net.modules(): 13 | if (isinstance(m, network_blocks.LinearDeconv) or isinstance(m,network_blocks.AdaptiveDeconv)): 14 | d_map2transf, d_transf2map, idx3, idx4 = cham_loss( 15 | gt_points.unsqueeze(0), m.points.unsqueeze(0)) 16 | loss += (0.5 * d_map2transf.mean() + 17 | 0.5 * d_transf2map.mean()) 18 | return weight * loss 19 | 20 | # From KPCONV 21 | def p2p_fitting_regularizer(net,deform_fitting_power=1.0,repulse_extent=1.2): 22 | l1 = torch.nn.L1Loss() 23 | fitting_loss = 0 24 | repulsive_loss = 0 25 | # print(20*'-') 26 | for m in net.modules(): 27 | if isinstance(m, okp.KPConv) and m.deformable: 28 | # print(m) 29 | 30 | ############## 31 | # Fitting loss 32 | ############## 33 | 34 | # Get the distance to closest input point and normalize to be independant from layers 35 | KP_min_d2 = m.min_d2 / (m.KP_extent ** 2) 36 | 37 | # Loss will be the square distance to closest input point. We use L1 because dist is already squared 38 | fitting_loss += l1(KP_min_d2, torch.zeros_like(KP_min_d2)) 39 | 40 | ################ 41 | # Repulsive loss 42 | ################ 43 | 44 | # Normalized KP locations 45 | KP_locs = m.deformed_KP / m.KP_extent 46 | 47 | # Point should not be close to each other 48 | for i in range(m.K): 49 | other_KP = torch.cat([KP_locs[:, :i, :], KP_locs[:, i + 1:, :]], dim=1).detach() 50 | distances = torch.sqrt(torch.sum((other_KP - KP_locs[:, i:i + 1, :]) ** 2, dim=2)) 51 | rep_loss = torch.sum(torch.clamp_max(distances - repulse_extent, max=0.0) ** 2, dim=1) 52 | repulsive_loss += l1(rep_loss, torch.zeros_like(rep_loss)) / m.K 53 | 54 | return deform_fitting_power * (2 * fitting_loss + repulsive_loss) 55 | -------------------------------------------------------------------------------- /depoco/architectures/network_blocks.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch.nn as nn 4 | import torch 5 | import numpy as np 6 | import octree_handler 7 | import depoco.architectures.original_kp_blocks as o_kp_conv 8 | ################################################## 9 | ############ NETWORK BLOCKS Dictionary############ 10 | ################################################## 11 | 12 | def printNan(bla: torch.tensor, pre=''): 13 | if (bla != bla).any(): 14 | print(pre, 'NAN') 15 | 16 | 17 | class Network(nn.Module): 18 | def __init__(self, config_list: list): 19 | super().__init__() 20 | blocks = [] 21 | for config in config_list: 22 | blocks.append(getBlocks(config)) 23 | self.blocks = nn.Sequential(*blocks) 24 | 25 | def forward(self, input_dict: dict): 26 | return self.blocks(input_dict) 27 | 28 | 29 | def getBlocks(config: dict): 30 | config_it, block_type = blockConfig2Params(config) 31 | blocks = [] 32 | for c in config_it: 33 | blocks.append(eval(block_type)(c)) 34 | if(len(config_it) == 1): 35 | return blocks[0] 36 | return nn.Sequential(*blocks) 37 | 38 | 39 | def blockConfig2Params(config: dict): 40 | """converts the config to a list of dicts 41 | it needs at least the keys 'type', 'parameters' and 'number_blocks' 42 | 43 | Arguments: 44 | config {dict} -- dictionary with the parameters for the block specified in 'type' 45 | 46 | Returns: 47 | {list} -- parameters [param_set1, param_set2, ...] 48 | {string} -- block_type 49 | """ 50 | nr_blocks = config['number_blocks'] 51 | if nr_blocks == 1: 52 | return [config['parameters']], config['type'] 53 | 54 | config_list = [] 55 | for i in range(nr_blocks): 56 | new_config = config["parameters"].copy() 57 | for k, v in zip(config["parameters"].keys(), config["parameters"].values()): 58 | if (type(v) is list): 59 | if(len(v) == nr_blocks): 60 | new_config[k] = v[i] 61 | if k == 'subsampling_ratio': 62 | new_config['cum_subsampling_ratio'] = np.cumprod([1.0]+v)[i] # HACK: to specivic 63 | config_list.append(new_config) 64 | return config_list, config['type'] 65 | 66 | 67 | def dict2initParams(dict_, class_): 68 | init_params = class_.__init__.__code__.co_varnames 69 | print(f'init vars: \n {init_params}') 70 | params = {k: dict_[k] for k in dict_ if k in init_params} 71 | print(params) 72 | return params 73 | 74 | def gridSampling(pcd: torch.tensor, resolution_meter=1.0, map_size=40): 75 | resolution = resolution_meter/map_size 76 | 77 | # v_size = torch.full(size=[3], fill_value=1/resolution, 78 | # dtype=pcd.dtype, device=pcd.device) 79 | 80 | grid = torch.floor(pcd/resolution) 81 | center = (grid+0.5)*resolution 82 | dist = ((pcd-center)**2).sum(dim=1) 83 | dist = dist/dist.max()*0.7 84 | 85 | # grid_idx = grid[:, 0] + grid[:, 1] * \ 86 | # v_size[0] + grid[:, 2] * v_size[0] * v_size[1] 87 | v_size = np.ceil(1/resolution) 88 | grid_idx = grid[:, 0] + grid[:, 1] * \ 89 | v_size + grid[:, 2] * v_size * v_size 90 | grid_d = grid_idx+dist 91 | idx_orig = torch.argsort(grid_d) 92 | 93 | # trick from https://github.com/rusty1s/pytorch_unique 94 | unique, inverse = torch.unique_consecutive( 95 | grid_idx[idx_orig], return_inverse=True) 96 | perm = torch.arange(inverse.size( 97 | 0), dtype=inverse.dtype, device=inverse.device) 98 | inverse, perm = inverse.flip([0]), perm.flip([0]) 99 | 100 | # idx = inverse.new_empty(unique.size(0)).scatter_(0, inverse, perm) 101 | """ 102 | HACK: workaround to get the first item. scatter overwrites indices on gpu not sequentially 103 | -> you get random points in the voxel not the first one 104 | """ 105 | p= perm.cpu() 106 | i=inverse.cpu() 107 | idx = torch.empty(unique.shape,dtype=p.dtype).scatter_(0, i, p) 108 | # idx= torch.empty(unique.shape,dtype=long) 109 | return idx_orig[idx].tolist() 110 | 111 | 112 | class GridSampleConv(nn.Module): 113 | def __init__(self, config: dict): 114 | super().__init__() 115 | self.relu = nn.LeakyReLU() 116 | ### Preactivation #### 117 | in_fdim = config['in_fdim'] 118 | out_fdim = config['out_fdim'] 119 | self.preactivation = nn.Identity() 120 | if in_fdim > 1: 121 | pre_blocks = [ 122 | nn.Linear(in_features=in_fdim, out_features=out_fdim)] 123 | if config['batchnorm']: 124 | pre_blocks.append(nn.BatchNorm1d(out_fdim)) 125 | if config['relu']: 126 | pre_blocks.append(self.relu) 127 | self.preactivation = nn.Sequential(*pre_blocks) 128 | # KP Conv 129 | conf_in_fdim = out_fdim if in_fdim > 1 else in_fdim 130 | self.subsampling_dist = config['subsampling_dist'] * config['subsampling_factor'] 131 | # self.kernel_radius = max(config['kernel_radius'], self.subsampling_dist/40 ) 132 | self.kernel_radius = max(config['min_kernel_radius'],config['kernel_radius']*self.subsampling_dist)/40 133 | KP_extent = self.kernel_radius / \ 134 | (config['num_kernel_points']**(1/3)-1)*1.5 135 | self.kp_conv = o_kp_conv.KPConv(kernel_size=config['num_kernel_points'], 136 | p_dim=3, in_channels=conf_in_fdim, 137 | out_channels=out_fdim, 138 | KP_extent=KP_extent, radius=self.kernel_radius, 139 | deformable=config['deformable']) 140 | self.max_nr_neighbors = config['max_nr_neighbors'] 141 | self.map_size = config['map_size'] 142 | self.octree = octree_handler.Octree() 143 | 144 | print('kernel radius',self.kernel_radius) 145 | # Post linear 146 | post_layer = [] 147 | if config['batchnorm']: 148 | post_layer.append(nn.BatchNorm1d(out_fdim)) 149 | if config['relu']: 150 | post_layer.append(self.relu) 151 | post_layer.append( 152 | nn.Linear(in_features=out_fdim, out_features=out_fdim)) 153 | if config['batchnorm']: 154 | post_layer.append(nn.BatchNorm1d(out_fdim)) 155 | self.post_layer = nn.Sequential(*post_layer) 156 | 157 | # Shortcut 158 | self.shortcut = nn.Identity() 159 | if in_fdim != out_fdim: 160 | sc_blocks = [nn.Linear(in_features=in_fdim, 161 | out_features=out_fdim)] 162 | if config['batchnorm']: 163 | sc_blocks.append(nn.BatchNorm1d(out_fdim)) 164 | self.shortcut = nn.Sequential(*sc_blocks) 165 | 166 | def forward(self, input_dict: dict) -> dict: 167 | source = input_dict['points'] 168 | source_np = source.detach().cpu().numpy() 169 | sample_idx = gridSampling(source,resolution_meter=self.subsampling_dist,map_size=self.map_size) 170 | # get neighbors 171 | self.octree.setInput(source_np) 172 | neighbors_index = self.octree.radiusSearchIndices( 173 | sample_idx, self.max_nr_neighbors, self.kernel_radius) 174 | neighbors_index = torch.from_numpy( 175 | neighbors_index).long().to(source.device) 176 | 177 | features = self.preactivation(input_dict['features']) 178 | features = self.kp_conv.forward(q_pts=input_dict['points'][sample_idx, :], 179 | s_pts=input_dict['points'], 180 | neighb_inds=neighbors_index, 181 | x=features) 182 | 183 | features = self.post_layer(features) 184 | input_dict['features'] = self.relu( 185 | self.shortcut(input_dict['features'][sample_idx, :]) + features) 186 | input_dict['points'] = input_dict['points'][sample_idx, :] 187 | return input_dict 188 | 189 | # https://discuss.pytorch.org/t/apply-mask-softmax/14212/14 190 | 191 | 192 | 193 | class LinearLayer(nn.Module): 194 | def __init__(self, config: dict): 195 | super().__init__() 196 | blocks = [nn.Linear(in_features=config['in_fdim'], 197 | out_features=config['out_fdim'])] 198 | if 'relu' in config: 199 | if config['relu']: 200 | blocks.append(nn.LeakyReLU()) 201 | if 'batchnorm' in config: 202 | if config['batchnorm']: 203 | blocks.append(nn.BatchNorm1d(num_features=config['out_fdim'])) 204 | self.blocks = nn.Sequential(*blocks) 205 | 206 | def forward(self, input_dict: dict): 207 | input_dict['features'] = self.blocks(input_dict['features']) 208 | return input_dict 209 | 210 | 211 | class LinearDeconv(nn.Module): 212 | def __init__(self, config: dict): 213 | super().__init__() 214 | self.config = config 215 | if config['estimate_radius']: 216 | self.kernel_radius = nn.Parameter(torch.tensor( 217 | [config['kernel_radius']]), requires_grad=True).float() 218 | else: 219 | self.kernel_radius = config['kernel_radius'] 220 | # self.kernel_radius = torch.tensor() 221 | 222 | feature_space = config['inter_fdim'] if 'inter_fdim' in config.keys( 223 | ) else 128 224 | 225 | self.upsampling_rate = config['upsampling_rate'] 226 | trans_blocks = [nn.Linear(config['in_fdim'], out_features=feature_space), 227 | nn.LeakyReLU(), 228 | nn.Linear(in_features=feature_space, 229 | out_features=3*self.upsampling_rate), 230 | nn.Tanh()] 231 | self.transl_nn = nn.Sequential(*trans_blocks) 232 | 233 | feature_blocks = [nn.Linear(config['in_fdim'], out_features=feature_space), 234 | nn.LeakyReLU(), 235 | nn.Linear( 236 | in_features=feature_space, out_features=config['out_fdim']*self.upsampling_rate), 237 | nn.LeakyReLU()] 238 | if config['use_batch_norm']: 239 | feature_blocks.append(nn.BatchNorm1d( 240 | config['out_fdim']*self.upsampling_rate)) 241 | self.feature_nn = nn.Sequential(*feature_blocks) 242 | self.tmp_i = 0 243 | 244 | self.points = None 245 | 246 | def forward(self, input_dict: dict): 247 | p = input_dict['points'] 248 | f = input_dict['features'] 249 | # print('p', p, 'f', f) 250 | delta = self.transl_nn(f) 251 | delta = delta.reshape( 252 | (delta.shape[0], self.upsampling_rate, 3))*self.kernel_radius 253 | p_new = (p.unsqueeze(1) + 254 | delta).reshape((delta.shape[0]*self.upsampling_rate, 3)) 255 | f_new = self.feature_nn(f).reshape( 256 | (delta.shape[0]*self.upsampling_rate, self.config['out_fdim'])) 257 | 258 | self.tmp_i += 1 259 | # if(self.tmp_i % 100 == 0) and self.config['estimate_radius']: 260 | # print('learned kernel radius', self.kernel_radius) 261 | self.points = p_new 262 | input_dict['points'] = p_new 263 | input_dict['features'] = f_new 264 | return input_dict 265 | 266 | def getScalingFactor(upsampling_rate, nr_layer,layer=0): 267 | sf = upsampling_rate**(1/nr_layer) 268 | factors = nr_layer*[round(sf)] 269 | # factors[-1]=round(upsampling_rate/np.prod(factors[:-1])) 270 | 271 | sampling_factor = np.prod(factors) 272 | print(f'factors {factors}, upsampling rate {sampling_factor}, should: {upsampling_rate}') 273 | return factors[layer] 274 | 275 | class AdaptiveDeconv(nn.Module): 276 | def __init__(self, config: dict): 277 | super().__init__() 278 | self.config = config 279 | if config['estimate_radius']: 280 | self.kernel_radius = nn.Parameter(torch.tensor( 281 | [config['kernel_radius']]), requires_grad=True).float() 282 | else: 283 | self.kernel_radius = config['kernel_radius'] 284 | 285 | feature_space = config['inter_fdim'] if 'inter_fdim' in config.keys( 286 | ) else 128 287 | 288 | 289 | sub_rate = config['subsampling_fct_p1']*config['subsampling_dist']**(-config['subsampling_fct_p2']) 290 | print('sub rate',sub_rate) 291 | 292 | self.upsampling_rate = getScalingFactor(upsampling_rate=1/sub_rate,nr_layer=config['number_blocks'],layer=config['block_id']) 293 | trans_blocks = [nn.Linear(config['in_fdim'], out_features=feature_space), 294 | nn.LeakyReLU(), 295 | nn.Linear(in_features=feature_space, 296 | out_features=3*self.upsampling_rate), 297 | nn.Tanh()] 298 | self.transl_nn = nn.Sequential(*trans_blocks) 299 | 300 | feature_blocks = [nn.Linear(config['in_fdim'], out_features=feature_space), 301 | nn.LeakyReLU(), 302 | nn.Linear( 303 | in_features=feature_space, out_features=config['out_fdim']*self.upsampling_rate), 304 | nn.LeakyReLU()] 305 | if config['use_batch_norm']: 306 | feature_blocks.append(nn.BatchNorm1d( 307 | config['out_fdim']*self.upsampling_rate)) 308 | self.feature_nn = nn.Sequential(*feature_blocks) 309 | self.tmp_i = 0 310 | 311 | self.points = None 312 | 313 | def forward(self, input_dict: dict): 314 | p = input_dict['points'] 315 | f = input_dict['features'] 316 | delta = self.transl_nn(f) 317 | delta = delta.reshape( 318 | (delta.shape[0], self.upsampling_rate, 3))*self.kernel_radius 319 | p_new = (p.unsqueeze(1) + 320 | delta).reshape((delta.shape[0]*self.upsampling_rate, 3)) 321 | f_new = self.feature_nn(f).reshape( 322 | (delta.shape[0]*self.upsampling_rate, self.config['out_fdim'])) 323 | 324 | self.tmp_i += 1 325 | # if(self.tmp_i % 100 == 0) and self.config['estimate_radius']: 326 | # print('learned kernel radius', self.kernel_radius) 327 | self.points = p_new 328 | input_dict['points'] = p_new 329 | input_dict['features'] = f_new 330 | return input_dict 331 | 332 | -------------------------------------------------------------------------------- /depoco/architectures/original_kp_blocks.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Define network blocks 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 06/03/2020 15 | # 16 | 17 | 18 | import time 19 | import math 20 | import torch 21 | import torch.nn as nn 22 | from torch.nn.parameter import Parameter 23 | from torch.nn.init import kaiming_uniform_ 24 | 25 | # from kernels.kernel_points import load_kernels 26 | 27 | # from utils.ply import write_ply 28 | 29 | # ---------------------------------------------------------------------------------------------------------------------- 30 | # 31 | # Simple functions 32 | # \**********************/ 33 | # 34 | 35 | 36 | def gather(x, idx, method=0): 37 | """ 38 | implementation of a custom gather operation for faster backwards. 39 | :param x: input with shape [N, D_1, ... D_d] 40 | :param idx: indexing with shape [n_1, ..., n_m] 41 | :param method: Choice of the method 42 | :return: x[idx] with shape [n_1, ..., n_m, D_1, ... D_d] 43 | """ 44 | 45 | if method == 0: 46 | return x[idx] 47 | elif method == 1: 48 | x = x.unsqueeze(1) 49 | x = x.expand((-1, idx.shape[-1], -1)) 50 | idx = idx.unsqueeze(2) 51 | idx = idx.expand((-1, -1, x.shape[-1])) 52 | return x.gather(0, idx) 53 | elif method == 2: 54 | for i, ni in enumerate(idx.size()[1:]): 55 | x = x.unsqueeze(i+1) 56 | new_s = list(x.size()) 57 | new_s[i+1] = ni 58 | x = x.expand(new_s) 59 | n = len(idx.size()) 60 | for i, di in enumerate(x.size()[n:]): 61 | idx = idx.unsqueeze(i+n) 62 | new_s = list(idx.size()) 63 | new_s[i+n] = di 64 | idx = idx.expand(new_s) 65 | return x.gather(0, idx) 66 | else: 67 | raise ValueError('Unkown method') 68 | 69 | 70 | def radius_gaussian(sq_r, sig, eps=1e-9): 71 | """ 72 | Compute a radius gaussian (gaussian of distance) 73 | :param sq_r: input radiuses [dn, ..., d1, d0] 74 | :param sig: extents of gaussians [d1, d0] or [d0] or float 75 | :return: gaussian of sq_r [dn, ..., d1, d0] 76 | """ 77 | return torch.exp(-sq_r / (2 * sig**2 + eps)) 78 | 79 | 80 | def closest_pool(x, inds): 81 | """ 82 | Pools features from the closest neighbors. WARNING: this function assumes the neighbors are ordered. 83 | :param x: [n1, d] features matrix 84 | :param inds: [n2, max_num] Only the first column is used for pooling 85 | :return: [n2, d] pooled features matrix 86 | """ 87 | 88 | # Add a last row with minimum features for shadow pools 89 | x = torch.cat((x, torch.zeros_like(x[:1, :])), 0) 90 | 91 | # Get features for each pooling location [n2, d] 92 | return gather(x, inds[:, 0]) 93 | 94 | 95 | def max_pool(x, inds): 96 | """ 97 | Pools features with the maximum values. 98 | :param x: [n1, d] features matrix 99 | :param inds: [n2, max_num] pooling indices 100 | :return: [n2, d] pooled features matrix 101 | """ 102 | 103 | # Add a last row with minimum features for shadow pools 104 | x = torch.cat((x, torch.zeros_like(x[:1, :])), 0) 105 | 106 | # Get all features for each pooling location [n2, max_num, d] 107 | pool_features = gather(x, inds) 108 | 109 | # Pool the maximum [n2, d] 110 | max_features, _ = torch.max(pool_features, 1) 111 | return max_features 112 | 113 | 114 | def global_average(x, batch_lengths): 115 | """ 116 | Block performing a global average over batch pooling 117 | :param x: [N, D] input features 118 | :param batch_lengths: [B] list of batch lengths 119 | :return: [B, D] averaged features 120 | """ 121 | 122 | # Loop over the clouds of the batch 123 | averaged_features = [] 124 | i0 = 0 125 | for b_i, length in enumerate(batch_lengths): 126 | 127 | # Average features for each batch cloud 128 | averaged_features.append(torch.mean(x[i0:i0 + length], dim=0)) 129 | 130 | # Increment for next cloud 131 | i0 += length 132 | 133 | # Average features in each batch 134 | return torch.stack(averaged_features) 135 | 136 | 137 | # ---------------------------------------------------------------------------------------------------------------------- 138 | # 139 | # KPConv class 140 | # \******************/ 141 | # 142 | 143 | 144 | class KPConv(nn.Module): 145 | 146 | def __init__(self, kernel_size, p_dim, in_channels, out_channels, KP_extent, radius, 147 | fixed_kernel_points='center', KP_influence='linear', aggregation_mode='sum', 148 | deformable=False, modulated=False): 149 | """ 150 | Initialize parameters for KPConvDeformable. 151 | :param kernel_size: Number of kernel points. 152 | :param p_dim: dimension of the point space. 153 | :param in_channels: dimension of input features. 154 | :param out_channels: dimension of output features. 155 | :param KP_extent: influence radius of each kernel point. 156 | :param radius: radius used for kernel point init. Even for deformable, use the config.conv_radius 157 | :param fixed_kernel_points: fix position of certain kernel points ('none', 'center' or 'verticals'). 158 | :param KP_influence: influence function of the kernel points ('constant', 'linear', 'gaussian'). 159 | :param aggregation_mode: choose to sum influences, or only keep the closest ('closest', 'sum'). 160 | :param deformable: choose deformable or not 161 | :param modulated: choose if kernel weights are modulated in addition to deformed 162 | """ 163 | super(KPConv, self).__init__() 164 | 165 | # Save parameters 166 | self.K = kernel_size 167 | self.p_dim = p_dim 168 | self.in_channels = in_channels 169 | self.out_channels = out_channels 170 | self.radius = radius 171 | self.KP_extent = KP_extent 172 | self.fixed_kernel_points = fixed_kernel_points 173 | self.KP_influence = KP_influence 174 | self.aggregation_mode = aggregation_mode 175 | self.deformable = deformable 176 | self.modulated = modulated 177 | 178 | # Running variable containing deformed KP distance to input points. (used in regularization loss) 179 | self.min_d2 = None 180 | self.deformed_KP = None 181 | self.offset_features = None 182 | 183 | # Initialize weights 184 | self.weights = Parameter(torch.zeros((self.K, in_channels, out_channels), dtype=torch.float32), 185 | requires_grad=True) 186 | 187 | # Initiate weights for offsets 188 | if deformable: 189 | if modulated: 190 | self.offset_dim = (self.p_dim + 1) * self.K 191 | else: 192 | self.offset_dim = self.p_dim * self.K 193 | self.offset_conv = KPConv(self.K, 194 | self.p_dim, 195 | self.in_channels, 196 | self.offset_dim, 197 | KP_extent, 198 | radius, 199 | fixed_kernel_points=fixed_kernel_points, 200 | KP_influence=KP_influence, 201 | aggregation_mode=aggregation_mode) 202 | self.offset_bias = Parameter(torch.zeros(self.offset_dim, dtype=torch.float32), requires_grad=True) 203 | 204 | else: 205 | self.offset_dim = None 206 | self.offset_conv = None 207 | self.offset_bias = None 208 | 209 | # Reset parameters 210 | self.reset_parameters() 211 | 212 | # Initialize kernel points 213 | self.kernel_points = self.init_KP() 214 | 215 | return 216 | 217 | def reset_parameters(self): 218 | kaiming_uniform_(self.weights, a=math.sqrt(5)) 219 | if self.deformable: 220 | nn.init.zeros_(self.offset_bias) 221 | return 222 | 223 | def init_KP(self): 224 | """ 225 | Initialize the kernel point positions in a sphere 226 | :return: the tensor of kernel points 227 | """ 228 | 229 | # Create one kernel disposition (as numpy array). Choose the KP distance to center thanks to the KP extent 230 | # K_points_numpy = load_kernels(self.radius, 231 | # self.K, 232 | # dimension=self.p_dim, 233 | # fixed=self.fixed_kernel_points) 234 | 235 | # return Parameter(torch.tensor(K_points_numpy, dtype=torch.float32), 236 | # requires_grad=False) 237 | 238 | K_points_numpy = getKernelPoints(self.radius, 239 | self.K) 240 | 241 | return Parameter(torch.tensor(K_points_numpy, dtype=torch.float32), 242 | requires_grad=False) 243 | 244 | 245 | 246 | def forward(self, q_pts, s_pts, neighb_inds, x): 247 | 248 | ################### 249 | # Offset generation 250 | ################### 251 | 252 | if self.deformable: 253 | 254 | # Get offsets with a KPConv that only takes part of the features 255 | self.offset_features = self.offset_conv(q_pts, s_pts, neighb_inds, x) + self.offset_bias 256 | 257 | if self.modulated: 258 | 259 | # Get offset (in normalized scale) from features 260 | unscaled_offsets = self.offset_features[:, :self.p_dim * self.K] 261 | unscaled_offsets = unscaled_offsets.view(-1, self.K, self.p_dim) 262 | 263 | # Get modulations 264 | modulations = 2 * torch.sigmoid(self.offset_features[:, self.p_dim * self.K:]) 265 | 266 | else: 267 | 268 | # Get offset (in normalized scale) from features 269 | unscaled_offsets = self.offset_features.view(-1, self.K, self.p_dim) 270 | 271 | # No modulations 272 | modulations = None 273 | 274 | # Rescale offset for this layer 275 | offsets = unscaled_offsets * self.KP_extent 276 | 277 | else: 278 | offsets = None 279 | modulations = None 280 | 281 | ###################### 282 | # Deformed convolution 283 | ###################### 284 | 285 | # Add a fake point in the last row for shadow neighbors 286 | s_pts = torch.cat((s_pts, torch.zeros_like(s_pts[:1, :]) + 1e6), 0) 287 | 288 | # Get neighbor points [n_points, n_neighbors, dim] 289 | neighbors = s_pts[neighb_inds, :] 290 | 291 | # Center every neighborhood 292 | neighbors = neighbors - q_pts.unsqueeze(1) 293 | 294 | # Apply offsets to kernel points [n_points, n_kpoints, dim] 295 | if self.deformable: 296 | self.deformed_KP = offsets + self.kernel_points 297 | deformed_K_points = self.deformed_KP.unsqueeze(1) 298 | else: 299 | deformed_K_points = self.kernel_points 300 | 301 | # Get all difference matrices [n_points, n_neighbors, n_kpoints, dim] 302 | neighbors.unsqueeze_(2) 303 | differences = neighbors - deformed_K_points 304 | 305 | # Get the square distances [n_points, n_neighbors, n_kpoints] 306 | sq_distances = torch.sum(differences ** 2, dim=3) 307 | # if sq_distances.grad is not None: 308 | # print(sq_distances.grad.max()) 309 | # print(sq_distances.requires_grad) 310 | # print(sq_distances.dtype) 311 | # Optimization by ignoring points outside a deformed KP range 312 | if self.deformable: 313 | 314 | # Save distances for loss 315 | self.min_d2, _ = torch.min(sq_distances, dim=1) 316 | 317 | # Boolean of the neighbors in range of a kernel point [n_points, n_neighbors] 318 | in_range = torch.any(sq_distances < self.KP_extent ** 2, dim=2).type(torch.int32) 319 | 320 | # New value of max neighbors 321 | new_max_neighb = torch.max(torch.sum(in_range, dim=1)) 322 | 323 | # For each row of neighbors, indices of the ones that are in range [n_points, new_max_neighb] 324 | neighb_row_bool, neighb_row_inds = torch.topk(in_range, new_max_neighb.item(), dim=1) 325 | 326 | # Gather new neighbor indices [n_points, new_max_neighb] 327 | new_neighb_inds = neighb_inds.gather(1, neighb_row_inds, sparse_grad=False) 328 | 329 | # Gather new distances to KP [n_points, new_max_neighb, n_kpoints] 330 | neighb_row_inds.unsqueeze_(2) 331 | neighb_row_inds = neighb_row_inds.expand(-1, -1, self.K) 332 | sq_distances = sq_distances.gather(1, neighb_row_inds, sparse_grad=False) 333 | 334 | # New shadow neighbors have to point to the last shadow point 335 | new_neighb_inds *= neighb_row_bool 336 | new_neighb_inds -= (neighb_row_bool.type(torch.int64) - 1) * int(s_pts.shape[0] - 1) 337 | else: 338 | new_neighb_inds = neighb_inds 339 | 340 | # Get Kernel point influences [n_points, n_kpoints, n_neighbors] 341 | if self.KP_influence == 'constant': 342 | # Every point get an influence of 1. 343 | all_weights = torch.ones_like(sq_distances) 344 | all_weights = torch.transpose(all_weights, 1, 2) 345 | 346 | elif self.KP_influence == 'linear': 347 | # Influence decrease linearly with the distance, and get to zero when d = KP_extent. 348 | all_weights = torch.clamp(1 - torch.sqrt(sq_distances) / self.KP_extent, min=0.0) 349 | # all_weights[all_weights==0]=0 350 | all_weights = torch.transpose(all_weights, 1, 2) 351 | elif self.KP_influence == 'gaussian': 352 | # Influence in gaussian of the distance. 353 | sigma = self.KP_extent * 0.3 354 | all_weights = radius_gaussian(sq_distances, sigma) 355 | all_weights = torch.transpose(all_weights, 1, 2) 356 | else: 357 | raise ValueError('Unknown influence function type (config.KP_influence)') 358 | # print('KP extend',self.KP_extent,'KP radius',self.radius) 359 | 360 | # In case of closest mode, only the closest KP can influence each point 361 | if self.aggregation_mode == 'closest': 362 | neighbors_1nn = torch.argmin(sq_distances, dim=2) 363 | all_weights *= torch.transpose(nn.functional.one_hot(neighbors_1nn, self.K), 1, 2) 364 | 365 | elif self.aggregation_mode != 'sum': 366 | raise ValueError("Unknown convolution mode. Should be 'closest' or 'sum'") 367 | 368 | # Add a zero feature for shadow neighbors 369 | x = torch.cat((x, torch.zeros_like(x[:1, :])), 0) 370 | 371 | # Get the features of each neighborhood [n_points, n_neighbors, in_fdim] 372 | neighb_x = gather(x, new_neighb_inds,method=0) 373 | 374 | # Apply distance weights [n_points, n_kpoints, in_fdim] 375 | weighted_features = torch.matmul(all_weights, neighb_x) 376 | 377 | # Apply modulations 378 | if self.deformable and self.modulated: 379 | weighted_features *= modulations.unsqueeze(2) 380 | 381 | # Apply network weights [n_kpoints, n_points, out_fdim] 382 | weighted_features = weighted_features.permute((1, 0, 2)) 383 | kernel_outputs = torch.matmul(weighted_features, self.weights) 384 | 385 | # Convolution sum [n_points, out_fdim] 386 | return torch.sum(kernel_outputs, dim=0) 387 | 388 | def __repr__(self): 389 | return 'KPConv(radius: {:.2f}, in_feat: {:d}, out_feat: {:d})'.format(self.radius, 390 | self.in_channels, 391 | self.out_channels) 392 | 393 | # ---------------------------------------------------------------------------------------------------------------------- 394 | # 395 | # Complex blocks 396 | # \********************/ 397 | # 398 | 399 | def block_decider(block_name, 400 | radius, 401 | in_dim, 402 | out_dim, 403 | layer_ind, 404 | config): 405 | 406 | if block_name == 'unary': 407 | return UnaryBlock(in_dim, out_dim, config.use_batch_norm, config.batch_norm_momentum) 408 | 409 | elif block_name in ['simple', 410 | 'simple_deformable', 411 | 'simple_invariant', 412 | 'simple_equivariant', 413 | 'simple_strided', 414 | 'simple_deformable_strided', 415 | 'simple_invariant_strided', 416 | 'simple_equivariant_strided']: 417 | return SimpleBlock(block_name, in_dim, out_dim, radius, layer_ind, config) 418 | 419 | elif block_name in ['resnetb', 420 | 'resnetb_invariant', 421 | 'resnetb_equivariant', 422 | 'resnetb_deformable', 423 | 'resnetb_strided', 424 | 'resnetb_deformable_strided', 425 | 'resnetb_equivariant_strided', 426 | 'resnetb_invariant_strided']: 427 | return ResnetBottleneckBlock(block_name, in_dim, out_dim, radius, layer_ind, config) 428 | 429 | elif block_name == 'max_pool' or block_name == 'max_pool_wide': 430 | return MaxPoolBlock(layer_ind) 431 | 432 | elif block_name == 'global_average': 433 | return GlobalAverageBlock() 434 | 435 | elif block_name == 'nearest_upsample': 436 | return NearestUpsampleBlock(layer_ind) 437 | 438 | else: 439 | raise ValueError('Unknown block name in the architecture definition : ' + block_name) 440 | 441 | 442 | class BatchNormBlock(nn.Module): 443 | 444 | def __init__(self, in_dim, use_bn, bn_momentum): 445 | """ 446 | Initialize a batch normalization block. If network does not use batch normalization, replace with biases. 447 | :param in_dim: dimension input features 448 | :param use_bn: boolean indicating if we use Batch Norm 449 | :param bn_momentum: Batch norm momentum 450 | """ 451 | super(BatchNormBlock, self).__init__() 452 | self.bn_momentum = bn_momentum 453 | self.use_bn = use_bn 454 | self.in_dim = in_dim 455 | if self.use_bn: 456 | self.batch_norm = nn.BatchNorm1d(in_dim, momentum=bn_momentum) 457 | #self.batch_norm = nn.InstanceNorm1d(in_dim, momentum=bn_momentum) 458 | else: 459 | self.bias = Parameter(torch.zeros(in_dim, dtype=torch.float32), requires_grad=True) 460 | return 461 | 462 | def reset_parameters(self): 463 | nn.init.zeros_(self.bias) 464 | 465 | def forward(self, x): 466 | if self.use_bn: 467 | 468 | x = x.unsqueeze(2) 469 | x = x.transpose(0, 2) 470 | x = self.batch_norm(x) 471 | x = x.transpose(0, 2) 472 | return x.squeeze() 473 | else: 474 | return x + self.bias 475 | 476 | def __repr__(self): 477 | return 'BatchNormBlock(in_feat: {:d}, momentum: {:.3f}, only_bias: {:s})'.format(self.in_dim, 478 | self.bn_momentum, 479 | str(not self.use_bn)) 480 | 481 | 482 | class UnaryBlock(nn.Module): 483 | 484 | def __init__(self, in_dim, out_dim, use_bn, bn_momentum, no_relu=False): 485 | """ 486 | Initialize a standard unary block with its ReLU and BatchNorm. 487 | :param in_dim: dimension input features 488 | :param out_dim: dimension input features 489 | :param use_bn: boolean indicating if we use Batch Norm 490 | :param bn_momentum: Batch norm momentum 491 | """ 492 | 493 | super(UnaryBlock, self).__init__() 494 | self.bn_momentum = bn_momentum 495 | self.use_bn = use_bn 496 | self.no_relu = no_relu 497 | self.in_dim = in_dim 498 | self.out_dim = out_dim 499 | self.mlp = nn.Linear(in_dim, out_dim, bias=False) 500 | self.batch_norm = BatchNormBlock(out_dim, self.use_bn, self.bn_momentum) 501 | if not no_relu: 502 | self.leaky_relu = nn.LeakyReLU(0.1) 503 | return 504 | 505 | def forward(self, x, batch=None): 506 | x = self.mlp(x) 507 | x = self.batch_norm(x) 508 | if not self.no_relu: 509 | x = self.leaky_relu(x) 510 | return x 511 | 512 | def __repr__(self): 513 | return 'UnaryBlock(in_feat: {:d}, out_feat: {:d}, BN: {:s}, ReLU: {:s})'.format(self.in_dim, 514 | self.out_dim, 515 | str(self.use_bn), 516 | str(not self.no_relu)) 517 | 518 | 519 | class SimpleBlock(nn.Module): 520 | 521 | def __init__(self, block_name, in_dim, out_dim, radius, layer_ind, config): 522 | """ 523 | Initialize a simple convolution block with its ReLU and BatchNorm. 524 | :param in_dim: dimension input features 525 | :param out_dim: dimension input features 526 | :param radius: current radius of convolution 527 | :param config: parameters 528 | """ 529 | super(SimpleBlock, self).__init__() 530 | 531 | # get KP_extent from current radius 532 | current_extent = radius * config.KP_extent / config.conv_radius 533 | 534 | # Get other parameters 535 | self.bn_momentum = config.batch_norm_momentum 536 | self.use_bn = config.use_batch_norm 537 | self.layer_ind = layer_ind 538 | self.block_name = block_name 539 | self.in_dim = in_dim 540 | self.out_dim = out_dim 541 | 542 | # Define the KPConv class 543 | self.KPConv = KPConv(config.num_kernel_points, 544 | config.in_points_dim, 545 | in_dim, 546 | out_dim // 2, 547 | current_extent, 548 | radius, 549 | fixed_kernel_points=config.fixed_kernel_points, 550 | KP_influence=config.KP_influence, 551 | aggregation_mode=config.aggregation_mode, 552 | deformable='deform' in block_name, 553 | modulated=config.modulated) 554 | 555 | # Other opperations 556 | self.batch_norm = BatchNormBlock(out_dim // 2, self.use_bn, self.bn_momentum) 557 | self.leaky_relu = nn.LeakyReLU(0.1) 558 | 559 | return 560 | 561 | def forward(self, x, batch): 562 | 563 | if 'strided' in self.block_name: 564 | q_pts = batch.points[self.layer_ind + 1] 565 | s_pts = batch.points[self.layer_ind] 566 | neighb_inds = batch.pools[self.layer_ind] 567 | else: 568 | q_pts = batch.points[self.layer_ind] 569 | s_pts = batch.points[self.layer_ind] 570 | neighb_inds = batch.neighbors[self.layer_ind] 571 | 572 | x = self.KPConv(q_pts, s_pts, neighb_inds, x) 573 | return self.leaky_relu(self.batch_norm(x)) 574 | 575 | 576 | class ResnetBottleneckBlock(nn.Module): 577 | 578 | def __init__(self, block_name, in_dim, out_dim, radius, layer_ind, config): 579 | """ 580 | Initialize a resnet bottleneck block. 581 | :param in_dim: dimension input features 582 | :param out_dim: dimension input features 583 | :param radius: current radius of convolution 584 | :param config: parameters 585 | """ 586 | super(ResnetBottleneckBlock, self).__init__() 587 | 588 | # get KP_extent from current radius 589 | current_extent = radius * config.KP_extent / config.conv_radius 590 | 591 | # Get other parameters 592 | self.bn_momentum = config.batch_norm_momentum 593 | self.use_bn = config.use_batch_norm 594 | self.block_name = block_name 595 | self.layer_ind = layer_ind 596 | self.in_dim = in_dim 597 | self.out_dim = out_dim 598 | 599 | # First downscaling mlp 600 | if in_dim != out_dim // 4: 601 | self.unary1 = UnaryBlock(in_dim, out_dim // 4, self.use_bn, self.bn_momentum) 602 | else: 603 | self.unary1 = nn.Identity() 604 | 605 | # KPConv block 606 | self.KPConv = KPConv(config.num_kernel_points, 607 | config.in_points_dim, 608 | out_dim // 4, 609 | out_dim // 4, 610 | current_extent, 611 | radius, 612 | fixed_kernel_points=config.fixed_kernel_points, 613 | KP_influence=config.KP_influence, 614 | aggregation_mode=config.aggregation_mode, 615 | deformable='deform' in block_name, 616 | modulated=config.modulated) 617 | self.batch_norm_conv = BatchNormBlock(out_dim // 4, self.use_bn, self.bn_momentum) 618 | 619 | # Second upscaling mlp 620 | self.unary2 = UnaryBlock(out_dim // 4, out_dim, self.use_bn, self.bn_momentum, no_relu=True) 621 | 622 | # Shortcut optional mpl 623 | if in_dim != out_dim: 624 | self.unary_shortcut = UnaryBlock(in_dim, out_dim, self.use_bn, self.bn_momentum, no_relu=True) 625 | else: 626 | self.unary_shortcut = nn.Identity() 627 | 628 | # Other operations 629 | self.leaky_relu = nn.LeakyReLU(0.1) 630 | 631 | return 632 | 633 | def forward(self, features, batch): 634 | 635 | if 'strided' in self.block_name: 636 | q_pts = batch.points[self.layer_ind + 1] 637 | s_pts = batch.points[self.layer_ind] 638 | neighb_inds = batch.pools[self.layer_ind] 639 | else: 640 | q_pts = batch.points[self.layer_ind] 641 | s_pts = batch.points[self.layer_ind] 642 | neighb_inds = batch.neighbors[self.layer_ind] 643 | 644 | # First downscaling mlp 645 | x = self.unary1(features) 646 | 647 | # Convolution 648 | x = self.KPConv(q_pts, s_pts, neighb_inds, x) 649 | x = self.leaky_relu(self.batch_norm_conv(x)) 650 | 651 | # Second upscaling mlp 652 | x = self.unary2(x) 653 | 654 | # Shortcut 655 | if 'strided' in self.block_name: 656 | shortcut = max_pool(features, neighb_inds) 657 | else: 658 | shortcut = features 659 | shortcut = self.unary_shortcut(shortcut) 660 | 661 | return self.leaky_relu(x + shortcut) 662 | 663 | #################### SampleNet #################################################### 664 | class SampleResnetBottleneckBlock(nn.Module): 665 | 666 | def __init__(self, config): 667 | """ 668 | Initialize a resnet bottleneck block. 669 | :param in_fdim: dimension input features 670 | :param out_fdim: dimension input features 671 | :param radius: current radius of convolution 672 | :param config: parameters 673 | """ 674 | super(SampleResnetBottleneckBlock, self).__init__() 675 | radius = config['kernel_radius'] 676 | # get KP_extent from current radius 677 | current_extent = radius * config.KP_extent / config.conv_radius 678 | 679 | # Get other parameters 680 | self.bn_momentum = config.batch_norm_momentum 681 | self.use_bn = config.use_batch_norm 682 | self.block_name = block_name 683 | self.layer_ind = layer_ind 684 | self.in_dim = in_dim 685 | self.out_dim = out_dim 686 | 687 | # First downscaling mlp 688 | if in_dim != out_dim // 4: 689 | self.unary1 = UnaryBlock(in_dim, out_dim // 4, self.use_bn, self.bn_momentum) 690 | else: 691 | self.unary1 = nn.Identity() 692 | 693 | # KPConv block 694 | self.KPConv = KPConv(config.num_kernel_points, 695 | config.in_points_dim, 696 | out_dim // 4, 697 | out_dim // 4, 698 | current_extent, 699 | radius, 700 | fixed_kernel_points=config.fixed_kernel_points, 701 | KP_influence=config.KP_influence, 702 | aggregation_mode=config.aggregation_mode, 703 | deformable='deform' in block_name, 704 | modulated=config.modulated) 705 | self.batch_norm_conv = BatchNormBlock(out_dim // 4, self.use_bn, self.bn_momentum) 706 | 707 | # Second upscaling mlp 708 | self.unary2 = UnaryBlock(out_dim // 4, out_dim, self.use_bn, self.bn_momentum, no_relu=True) 709 | 710 | # Shortcut optional mpl 711 | if in_dim != out_dim: 712 | self.unary_shortcut = UnaryBlock(in_dim, out_dim, self.use_bn, self.bn_momentum, no_relu=True) 713 | else: 714 | self.unary_shortcut = nn.Identity() 715 | 716 | # Other operations 717 | self.leaky_relu = nn.LeakyReLU(0.1) 718 | 719 | return 720 | 721 | def forward(self, features, batch): 722 | 723 | if 'strided' in self.block_name: 724 | q_pts = batch.points[self.layer_ind + 1] 725 | s_pts = batch.points[self.layer_ind] 726 | neighb_inds = batch.pools[self.layer_ind] 727 | else: 728 | q_pts = batch.points[self.layer_ind] 729 | s_pts = batch.points[self.layer_ind] 730 | neighb_inds = batch.neighbors[self.layer_ind] 731 | 732 | # First downscaling mlp 733 | x = self.unary1(features) 734 | 735 | # Convolution 736 | x = self.KPConv(q_pts, s_pts, neighb_inds, x) 737 | x = self.leaky_relu(self.batch_norm_conv(x)) 738 | 739 | # Second upscaling mlp 740 | x = self.unary2(x) 741 | 742 | # Shortcut 743 | if 'strided' in self.block_name: 744 | shortcut = max_pool(features, neighb_inds) 745 | else: 746 | shortcut = features 747 | shortcut = self.unary_shortcut(shortcut) 748 | 749 | return self.leaky_relu(x + shortcut) 750 | ################################################################################## 751 | 752 | class GlobalAverageBlock(nn.Module): 753 | 754 | def __init__(self): 755 | """ 756 | Initialize a global average block with its ReLU and BatchNorm. 757 | """ 758 | super(GlobalAverageBlock, self).__init__() 759 | return 760 | 761 | def forward(self, x, batch): 762 | return global_average(x, batch.lengths[-1]) 763 | 764 | 765 | class NearestUpsampleBlock(nn.Module): 766 | 767 | def __init__(self, layer_ind): 768 | """ 769 | Initialize a nearest upsampling block with its ReLU and BatchNorm. 770 | """ 771 | super(NearestUpsampleBlock, self).__init__() 772 | self.layer_ind = layer_ind 773 | return 774 | 775 | def forward(self, x, batch): 776 | return closest_pool(x, batch.upsamples[self.layer_ind - 1]) 777 | 778 | def __repr__(self): 779 | return 'NearestUpsampleBlock(layer: {:d} -> {:d})'.format(self.layer_ind, 780 | self.layer_ind - 1) 781 | 782 | 783 | class MaxPoolBlock(nn.Module): 784 | 785 | def __init__(self, layer_ind): 786 | """ 787 | Initialize a max pooling block with its ReLU and BatchNorm. 788 | """ 789 | super(MaxPoolBlock, self).__init__() 790 | self.layer_ind = layer_ind 791 | return 792 | 793 | def forward(self, x, batch): 794 | return max_pool(x, batch.pools[self.layer_ind + 1]) 795 | 796 | ######################################################################## 797 | # Grid Based Kernel Point initialization 798 | import numpy as np 799 | def getKernelPoints(radius, num_kernel_points): 800 | if round(num_kernel_points**(1/3)) % 1 == 0: 801 | npoints = round(num_kernel_points**(1/3)) 802 | xyz = np.linspace(-1, 1, npoints) 803 | points = np.meshgrid(xyz, xyz, xyz) 804 | points = [p.flatten() for p in points] 805 | points = np.vstack(points).T 806 | points /= 3**(0.5) 807 | return points*radius 808 | else: 809 | assert(False) -------------------------------------------------------------------------------- /depoco/config/depoco.yaml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # training parameters 3 | ################################################################################ 4 | train: 5 | experiment_id: "your_model" 6 | max_epochs: 200 7 | use_adam: True 8 | batch_size: 3 # batch size 9 | max_nr_pts: 30000 # nr points 10 | workers: 0 # number of threads to get data 11 | optimizer: # Adam 12 | start_lr: 0.000001 # start learning rate 13 | max_lr: 0.0001 # max learning rate 14 | end_lr: 0.00001 # end learning rate 15 | pct_incr_cycle: 0.1 # one cycle max at epoch in % 16 | anneal_strategy: "cos" # "linear" or "cos" 17 | momentum: 0.9 # sgd momentum 18 | nr_submaps: 0 # nr of maps using for training, 0 for using all maps 19 | load_pretrained: False 20 | sampling_method: "random" # 'ordered' or 'random' 21 | map_prob_rate: -1 # every x epochs computes the loss for each train map and sets those as sample distribution, -1 for no update 22 | loss_weights: 23 | transf2map: 1.0 # dist(transf, map) transf = samples + t 24 | map2transf: 1.0 # dist(map, transf) 25 | upsampling_reg: 0.2 # chmf dist for intermediate upsampling blocks 26 | validation: 27 | report_rate: 1 # every x epochs on tensorboard 28 | save_result_rate: 500 # every x validations generates one image (doesnt work in docker) 29 | 30 | 31 | ################################################################################ 32 | # Voxel Grid parameter (just for data generation) 33 | ################################################################################ 34 | grid: 35 | pose_distance: 15 36 | size: #x,y,z 37 | - 40.0 38 | - 40.0 39 | - 15.0 # ca. -9 to 4 => center at 2.5 40 | dz: 4.0 # offset to pose center in z direction, = 2.5 - size['z']/2 41 | voxel_size: 0.1 42 | max_range: 20.0 43 | min_range: 2.0 44 | features: ['intensity','label','eigenvalues','normals'] 45 | feature_dim: [1,1,3,3] 46 | normal_eigenvalue_radius: 0.5 # radius for computing normals and the eigenvalues 47 | 48 | evaluation: 49 | float16: True 50 | iou_grid: 51 | resolution: [0.2,0.2,0.1] 52 | f_score_dist: 0.1 # distance for being outlier 53 | out_dir: 'experiments/results/kitti/' 54 | ################################################################################ 55 | # Network parameters 56 | ################################################################################ 57 | network: 58 | # a block needs the attributes: type, number_blocks, parameters 59 | encoder_blocks: # list:* indicates muliple blocks of the same type 60 | - type: "GridSampleConv" #SampleBlock, RandomSampleKPConv 61 | number_blocks: 3 62 | parameters: 63 | in_fdim: [1, 16,32] #input and output dimension 64 | out_fdim: [16,32,32] #input and output dimension 65 | num_kernel_points: 27 66 | max_nr_neighbors: [70,50,25] 67 | relu: True 68 | batchnorm: True 69 | deformable: False 70 | subsampling_dist: 1.7 #min_dist between points: 0: no subsampling 71 | map_size: 40 #to compute normalized radius 72 | subsampling_factor: [0.1,0.5,1.0] 73 | kernel_radius: 1.0 # factor of subsampling dist 74 | min_kernel_radius: 1.5 # 75 | use_dif_sampling: False 76 | - type: "LinearLayer" 77 | number_blocks: 1 78 | parameters: 79 | in_fdim: 32 80 | out_fdim: 3 81 | relu: False 82 | batchnorm: False 83 | decoder_blocks: 84 | - type: "AdaptiveDeconv" 85 | number_blocks: 4 86 | parameters: 87 | number_blocks: 4 88 | block_id: [0,1,2,3] # [0,...,n-1] 89 | subsampling_dist: 1.7 # to compute the subsamping rate, to compute the upsampling rate 90 | in_fdim: [3,32,32,32] 91 | out_fdim: 32 92 | kernel_radius: 0.05 93 | relu: True 94 | use_batch_norm: False 95 | inter_fdim: 128 96 | estimate_radius: False 97 | subsampling_fct_p1: 0.006 # y = p1 * x^(-p2) 98 | subsampling_fct_p2: 1.764 99 | - type: "LinearLayer" 100 | number_blocks: 1 101 | parameters: 102 | in_fdim: 32 103 | out_fdim: 3 104 | relu: False 105 | batchnorm: False 106 | out_dir: "network_files/" 107 | 108 | ################################################################################ 109 | # dataset (to find parser) 110 | ################################################################################ 111 | dataset: 112 | data_folders: 113 | grid_output: "/data/" #path/to/the/submaps/ 114 | prefix: "/path/to/kitti/" #only needed for Kitti2Submap conversion (kitti-format) 115 | train: 116 | - "00" 117 | - "01" 118 | - "02" 119 | - "03" 120 | - "04" 121 | - "05" 122 | - "06" 123 | - "07" 124 | - "09" 125 | - "10" 126 | valid: 127 | - "validation" 128 | test: 129 | - "08" -------------------------------------------------------------------------------- /depoco/config/kitti/e0.yaml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # training parameters 3 | ################################################################################ 4 | train: 5 | experiment_id: "e0" 6 | max_epochs: 200 7 | use_adam: True 8 | batch_size: 3 # batch size 9 | max_nr_pts: 30000 # nr points 10 | workers: 0 # number of threads to get data 11 | optimizer: # Adam 12 | start_lr: 0.000001 # start learning rate 13 | max_lr: 0.0001 # max learning rate 14 | end_lr: 0.00001 # end learning rate 15 | pct_incr_cycle: 0.1 # one cycle max at epoch in % 16 | anneal_strategy: "cos" # "linear" or "cos" 17 | momentum: 0.9 # sgd momentum 18 | nr_submaps: 0 # nr of maps using for training, 0 for using all maps 19 | load_pretrained: False 20 | sampling_method: "random" # 'ordered' or 'random' 21 | map_prob_rate: -1 # every x epochs computes the loss for each train map and sets those as sample distribution, -1 for no update 22 | loss_weights: 23 | transf2map: 1.0 # dist(transf, map) transf = samples + t 24 | map2transf: 1.0 # dist(map, transf) 25 | upsampling_reg: 0.2 # chmf dist for intermediate upsampling blocks 26 | validation: 27 | report_rate: 1 # every x epochs on tensorboard 28 | save_result_rate: 500 # every x validations generates one image (doesnt work in docker) 29 | 30 | 31 | ################################################################################ 32 | # Voxel Grid parameter (just for data generation) 33 | ################################################################################ 34 | grid: 35 | pose_distance: 15 36 | size: #x,y,z 37 | - 40.0 38 | - 40.0 39 | - 15.0 # ca. -9 to 4 => center at 2.5 40 | dz: 4.0 # offset to pose center in z direction, = 2.5 - size['z']/2 41 | voxel_size: 0.1 42 | max_range: 20.0 43 | min_range: 2.0 44 | features: ['intensity','label','eigenvalues','normals'] 45 | feature_dim: [1,1,3,3] 46 | normal_eigenvalue_radius: 0.5 # radius for computing normals and the eigenvalues 47 | 48 | evaluation: 49 | float16: True 50 | iou_grid: 51 | resolution: [0.2,0.2,0.1] 52 | f_score_dist: 0.1 # distance for being outlier 53 | out_dir: 'experiments/results/kitti/' 54 | ################################################################################ 55 | # Network parameters 56 | ################################################################################ 57 | network: 58 | # a block needs the attributes: type, number_blocks, parameters 59 | encoder_blocks: # list:* indicates muliple blocks of the same type 60 | - type: "GridSampleConv" #SampleBlock, RandomSampleKPConv 61 | number_blocks: 3 62 | parameters: 63 | in_fdim: [1, 16,32] #input and output dimension 64 | out_fdim: [16,32,32] #input and output dimension 65 | num_kernel_points: 27 66 | max_nr_neighbors: [70,50,25] 67 | relu: True 68 | batchnorm: True 69 | deformable: False 70 | subsampling_dist: 3 #min_dist between points: 0: no subsampling 71 | map_size: 40 #to compute normalized radius 72 | subsampling_factor: [0.1,0.5,1.0] 73 | kernel_radius: 1.0 # factor of subsampling dist 74 | min_kernel_radius: 1.5 # 75 | use_dif_sampling: False 76 | - type: "LinearLayer" 77 | number_blocks: 1 78 | parameters: 79 | in_fdim: 32 80 | out_fdim: 3 81 | relu: False 82 | batchnorm: False 83 | decoder_blocks: 84 | - type: "AdaptiveDeconv" 85 | number_blocks: 4 86 | parameters: 87 | number_blocks: 4 88 | block_id: [0,1,2,3] # [0,...,n-1] 89 | subsampling_dist: 3 # to compute the subsamping rate, to compute the upsampling rate 90 | in_fdim: [3,32,32,32] 91 | out_fdim: 32 92 | kernel_radius: 0.05 93 | relu: True 94 | use_batch_norm: False 95 | inter_fdim: 128 96 | estimate_radius: False 97 | subsampling_fct_p1: 0.006 # y = p1 * x^(-p2) 98 | subsampling_fct_p2: 1.764 99 | - type: "LinearLayer" 100 | number_blocks: 1 101 | parameters: 102 | in_fdim: 32 103 | out_fdim: 3 104 | relu: False 105 | batchnorm: False 106 | out_dir: "network_files/" 107 | 108 | ################################################################################ 109 | # dataset (to find parser) 110 | ################################################################################ 111 | dataset: 112 | data_folders: 113 | grid_output: "/data/" #path/to/the/submaps/ 114 | prefix: "/path/to/kitti/" #only needed for Kitti2Submap conversion (kitti-format) 115 | train: 116 | - "00" 117 | - "01" 118 | - "02" 119 | - "03" 120 | - "04" 121 | - "05" 122 | - "06" 123 | - "07" 124 | - "09" 125 | - "10" 126 | valid: 127 | - "validation" 128 | test: 129 | - "08" -------------------------------------------------------------------------------- /depoco/config/kitti/e1.yaml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # training parameters 3 | ################################################################################ 4 | train: 5 | experiment_id: "e1" 6 | max_epochs: 200 7 | use_adam: True 8 | batch_size: 3 # batch size 9 | max_nr_pts: 30000 # nr points 10 | workers: 0 # number of threads to get data 11 | optimizer: # Adam 12 | start_lr: 0.000001 # start learning rate 13 | max_lr: 0.0001 # max learning rate 14 | end_lr: 0.00001 # end learning rate 15 | pct_incr_cycle: 0.1 # one cycle max at epoch in % 16 | anneal_strategy: "cos" # "linear" or "cos" 17 | momentum: 0.9 # sgd momentum 18 | nr_submaps: 0 # nr of maps using for training, 0 for using all maps 19 | load_pretrained: False 20 | sampling_method: "random" # 'ordered' or 'random' 21 | map_prob_rate: -1 # every x epochs computes the loss for each train map and sets those as sample distribution, -1 for no update 22 | loss_weights: 23 | transf2map: 1.0 # dist(transf, map) transf = samples + t 24 | map2transf: 1.0 # dist(map, transf) 25 | upsampling_reg: 0.2 # chmf dist for intermediate upsampling blocks 26 | validation: 27 | report_rate: 1 # every x epochs on tensorboard 28 | save_result_rate: 500 # every x validations generates one image (doesnt work in docker) 29 | 30 | 31 | ################################################################################ 32 | # Voxel Grid parameter (just for data generation) 33 | ################################################################################ 34 | grid: 35 | pose_distance: 15 36 | size: #x,y,z 37 | - 40.0 38 | - 40.0 39 | - 15.0 # ca. -9 to 4 => center at 2.5 40 | dz: 4.0 # offset to pose center in z direction, = 2.5 - size['z']/2 41 | voxel_size: 0.1 42 | max_range: 20.0 43 | min_range: 2.0 44 | features: ['intensity','label','eigenvalues','normals'] 45 | feature_dim: [1,1,3,3] 46 | normal_eigenvalue_radius: 0.5 # radius for computing normals and the eigenvalues 47 | 48 | evaluation: 49 | float16: True 50 | iou_grid: 51 | resolution: [0.2,0.2,0.1] 52 | f_score_dist: 0.1 # distance for being outlier 53 | out_dir: 'experiments/results/kitti/' 54 | ################################################################################ 55 | # Network parameters 56 | ################################################################################ 57 | network: 58 | # a block needs the attributes: type, number_blocks, parameters 59 | encoder_blocks: # list:* indicates muliple blocks of the same type 60 | - type: "GridSampleConv" #SampleBlock, RandomSampleKPConv 61 | number_blocks: 3 62 | parameters: 63 | in_fdim: [1, 16,32] #input and output dimension 64 | out_fdim: [16,32,32] #input and output dimension 65 | num_kernel_points: 27 66 | max_nr_neighbors: [70,50,25] 67 | relu: True 68 | batchnorm: True 69 | deformable: False 70 | subsampling_dist: 2 #min_dist between points: 0: no subsampling 71 | map_size: 40 #to compute normalized radius 72 | subsampling_factor: [0.1,0.5,1.0] 73 | kernel_radius: 1.0 # factor of subsampling dist 74 | min_kernel_radius: 1.5 # 75 | use_dif_sampling: False 76 | - type: "LinearLayer" 77 | number_blocks: 1 78 | parameters: 79 | in_fdim: 32 80 | out_fdim: 3 81 | relu: False 82 | batchnorm: False 83 | decoder_blocks: 84 | - type: "AdaptiveDeconv" 85 | number_blocks: 4 86 | parameters: 87 | number_blocks: 4 88 | block_id: [0,1,2,3] # [0,...,n-1] 89 | subsampling_dist: 2 # to compute the subsamping rate, to compute the upsampling rate 90 | in_fdim: [3,32,32,32] 91 | out_fdim: 32 92 | kernel_radius: 0.05 93 | relu: True 94 | use_batch_norm: False 95 | inter_fdim: 128 96 | estimate_radius: False 97 | subsampling_fct_p1: 0.006 # y = p1 * x^(-p2) 98 | subsampling_fct_p2: 1.764 99 | - type: "LinearLayer" 100 | number_blocks: 1 101 | parameters: 102 | in_fdim: 32 103 | out_fdim: 3 104 | relu: False 105 | batchnorm: False 106 | out_dir: "network_files/" 107 | 108 | ################################################################################ 109 | # dataset (to find parser) 110 | ################################################################################ 111 | dataset: 112 | data_folders: 113 | grid_output: "/data/" #path/to/the/submaps/ 114 | prefix: "/path/to/kitti/" #only needed for Kitti2Submap conversion (kitti-format) 115 | train: 116 | - "00" 117 | - "01" 118 | - "02" 119 | - "03" 120 | - "04" 121 | - "05" 122 | - "06" 123 | - "07" 124 | - "09" 125 | - "10" 126 | valid: 127 | - "validation" 128 | test: 129 | - "08" -------------------------------------------------------------------------------- /depoco/config/kitti/e2.yaml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # training parameters 3 | ################################################################################ 4 | train: 5 | experiment_id: "e2" 6 | max_epochs: 200 7 | use_adam: True 8 | batch_size: 3 # batch size 9 | max_nr_pts: 30000 # nr points 10 | workers: 0 # number of threads to get data 11 | optimizer: # Adam 12 | start_lr: 0.000001 # start learning rate 13 | max_lr: 0.0001 # max learning rate 14 | end_lr: 0.00001 # end learning rate 15 | pct_incr_cycle: 0.1 # one cycle max at epoch in % 16 | anneal_strategy: "cos" # "linear" or "cos" 17 | momentum: 0.9 # sgd momentum 18 | nr_submaps: 0 # nr of maps using for training, 0 for using all maps 19 | load_pretrained: False 20 | sampling_method: "random" # 'ordered' or 'random' 21 | map_prob_rate: -1 # every x epochs computes the loss for each train map and sets those as sample distribution, -1 for no update 22 | loss_weights: 23 | transf2map: 1.0 # dist(transf, map) transf = samples + t 24 | map2transf: 1.0 # dist(map, transf) 25 | upsampling_reg: 0.2 # chmf dist for intermediate upsampling blocks 26 | validation: 27 | report_rate: 1 # every x epochs on tensorboard 28 | save_result_rate: 500 # every x validations generates one image (doesnt work in docker) 29 | 30 | 31 | ################################################################################ 32 | # Voxel Grid parameter (just for data generation) 33 | ################################################################################ 34 | grid: 35 | pose_distance: 15 36 | size: #x,y,z 37 | - 40.0 38 | - 40.0 39 | - 15.0 # ca. -9 to 4 => center at 2.5 40 | dz: 4.0 # offset to pose center in z direction, = 2.5 - size['z']/2 41 | voxel_size: 0.1 42 | max_range: 20.0 43 | min_range: 2.0 44 | features: ['intensity','label','eigenvalues','normals'] 45 | feature_dim: [1,1,3,3] 46 | normal_eigenvalue_radius: 0.5 # radius for computing normals and the eigenvalues 47 | 48 | evaluation: 49 | float16: True 50 | iou_grid: 51 | resolution: [0.2,0.2,0.1] 52 | f_score_dist: 0.1 # distance for being outlier 53 | out_dir: 'experiments/results/kitti/' 54 | ################################################################################ 55 | # Network parameters 56 | ################################################################################ 57 | network: 58 | # a block needs the attributes: type, number_blocks, parameters 59 | encoder_blocks: # list:* indicates muliple blocks of the same type 60 | - type: "GridSampleConv" #SampleBlock, RandomSampleKPConv 61 | number_blocks: 3 62 | parameters: 63 | in_fdim: [1, 16,32] #input and output dimension 64 | out_fdim: [16,32,32] #input and output dimension 65 | num_kernel_points: 27 66 | max_nr_neighbors: [70,50,25] 67 | relu: True 68 | batchnorm: True 69 | deformable: False 70 | subsampling_dist: 1.2 #min_dist between points: 0: no subsampling 71 | map_size: 40 #to compute normalized radius 72 | subsampling_factor: [0.1,0.5,1.0] 73 | kernel_radius: 1.0 # factor of subsampling dist 74 | min_kernel_radius: 1.2 # 75 | use_dif_sampling: False 76 | - type: "LinearLayer" 77 | number_blocks: 1 78 | parameters: 79 | in_fdim: 32 80 | out_fdim: 3 81 | relu: False 82 | batchnorm: False 83 | decoder_blocks: 84 | - type: "AdaptiveDeconv" 85 | number_blocks: 4 86 | parameters: 87 | number_blocks: 4 88 | block_id: [0,1,2,3] # [0,...,n-1] 89 | subsampling_dist: 1.2 # to compute the subsamping rate, to compute the upsampling rate 90 | in_fdim: [3,32,32,32] 91 | out_fdim: 32 92 | kernel_radius: 0.03 93 | relu: True 94 | use_batch_norm: False 95 | inter_fdim: 128 96 | estimate_radius: False 97 | subsampling_fct_p1: 0.006 # y = p1 * x^(-p2) 98 | subsampling_fct_p2: 1.764 99 | - type: "LinearLayer" 100 | number_blocks: 1 101 | parameters: 102 | in_fdim: 32 103 | out_fdim: 3 104 | relu: False 105 | batchnorm: False 106 | out_dir: "network_files/" 107 | ################################################################################ 108 | # dataset (to find parser) 109 | ################################################################################ 110 | dataset: 111 | data_folders: 112 | grid_output: "/data/" #path/to/the/submaps/ 113 | prefix: "/path/to/kitti/" #only needed for Kitti2Submap conversion (kitti-format) 114 | train: 115 | - "00" 116 | - "01" 117 | - "02" 118 | - "03" 119 | - "04" 120 | - "05" 121 | - "06" 122 | - "07" 123 | - "09" 124 | - "10" 125 | valid: 126 | - "validation" 127 | test: 128 | - "08" -------------------------------------------------------------------------------- /depoco/config/kitti/e3.yaml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # training parameters 3 | ################################################################################ 4 | train: 5 | experiment_id: "e3" 6 | max_epochs: 200 7 | use_adam: True 8 | batch_size: 3 # batch size 9 | max_nr_pts: 30000 # nr points 10 | workers: 0 # number of threads to get data 11 | optimizer: # Adam 12 | start_lr: 0.000001 # start learning rate 13 | max_lr: 0.0001 # max learning rate 14 | end_lr: 0.00001 # end learning rate 15 | pct_incr_cycle: 0.1 # one cycle max at epoch in % 16 | anneal_strategy: "cos" # "linear" or "cos" 17 | momentum: 0.9 # sgd momentum 18 | nr_submaps: 0 # nr of maps using for training, 0 for using all maps 19 | load_pretrained: False 20 | sampling_method: "random" # 'ordered' or 'random' 21 | map_prob_rate: -1 # every x epochs computes the loss for each train map and sets those as sample distribution, -1 for no update 22 | loss_weights: 23 | transf2map: 1.0 # dist(transf, map) transf = samples + t 24 | map2transf: 1.0 # dist(map, transf) 25 | upsampling_reg: 0.2 # chmf dist for intermediate upsampling blocks 26 | validation: 27 | report_rate: 1 # every x epochs on tensorboard 28 | save_result_rate: 500 # every x validations generates one image (doesnt work in docker) 29 | 30 | 31 | ################################################################################ 32 | # Voxel Grid parameter (just for data generation) 33 | ################################################################################ 34 | grid: 35 | pose_distance: 15 36 | size: #x,y,z 37 | - 40.0 38 | - 40.0 39 | - 15.0 # ca. -9 to 4 => center at 2.5 40 | dz: 4.0 # offset to pose center in z direction, = 2.5 - size['z']/2 41 | voxel_size: 0.1 42 | max_range: 20.0 43 | min_range: 2.0 44 | features: ['intensity','label','eigenvalues','normals'] 45 | feature_dim: [1,1,3,3] 46 | normal_eigenvalue_radius: 0.5 # radius for computing normals and the eigenvalues 47 | 48 | evaluation: 49 | float16: True 50 | iou_grid: 51 | resolution: [0.2,0.2,0.1] 52 | f_score_dist: 0.1 # distance for being outlier 53 | out_dir: 'experiments/results/kitti/' 54 | ################################################################################ 55 | # Network parameters 56 | ################################################################################ 57 | network: 58 | # a block needs the attributes: type, number_blocks, parameters 59 | encoder_blocks: # list:* indicates muliple blocks of the same type 60 | - type: "GridSampleConv" #SampleBlock, RandomSampleKPConv 61 | number_blocks: 3 62 | parameters: 63 | in_fdim: [1, 16,32] #input and output dimension 64 | out_fdim: [16,32,32] #input and output dimension 65 | num_kernel_points: 27 66 | max_nr_neighbors: [70,50,25] 67 | relu: True 68 | batchnorm: True 69 | deformable: False 70 | subsampling_dist: 1.0 #min_dist between points: 0: no subsampling 71 | map_size: 40 #to compute normalized radius 72 | subsampling_factor: [0.1,0.5,1.0] 73 | kernel_radius: 1.0 # factor of subsampling dist 74 | min_kernel_radius: 1.2 # 75 | use_dif_sampling: False 76 | - type: "LinearLayer" 77 | number_blocks: 1 78 | parameters: 79 | in_fdim: 32 80 | out_fdim: 3 81 | relu: False 82 | batchnorm: False 83 | decoder_blocks: 84 | - type: "AdaptiveDeconv" 85 | number_blocks: 4 86 | parameters: 87 | number_blocks: 4 88 | block_id: [0,1,2,3] # [0,...,n-1] 89 | subsampling_dist: 1.0 # to compute the subsamping rate, to compute the upsampling rate 90 | in_fdim: [3,32,32,32] 91 | out_fdim: 32 92 | kernel_radius: 0.05 93 | relu: True 94 | use_batch_norm: False 95 | inter_fdim: 128 96 | estimate_radius: False 97 | subsampling_fct_p1: 0.006 # y = p1 * x^(-p2) 98 | subsampling_fct_p2: 1.764 99 | - type: "LinearLayer" 100 | number_blocks: 1 101 | parameters: 102 | in_fdim: 32 103 | out_fdim: 3 104 | relu: False 105 | batchnorm: False 106 | out_dir: "network_files/" 107 | 108 | ################################################################################ 109 | # dataset (to find parser) 110 | ################################################################################ 111 | dataset: 112 | data_folders: 113 | grid_output: "/data/" #path/to/the/submaps/ 114 | prefix: "/path/to/kitti/" #only needed for Kitti2Submap conversion (kitti-format) 115 | train: 116 | - "00" 117 | - "01" 118 | - "02" 119 | - "03" 120 | - "04" 121 | - "05" 122 | - "06" 123 | - "07" 124 | - "09" 125 | - "10" 126 | valid: 127 | - "validation" 128 | test: 129 | - "08" -------------------------------------------------------------------------------- /depoco/data_handling/VoxelGrid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class VoxelGrid(): 5 | def __init__(self, VOXEL, center, grid_size, voxel_size,point_dim=3): 6 | self.grid_size = grid_size 7 | self.center = center 8 | self.voxel_size = voxel_size 9 | 10 | self.grid_dim = np.ceil(grid_size / voxel_size).astype('int') 11 | self.num_voxel = int(np.prod(self.grid_dim)) 12 | self.origin_offset = center - \ 13 | np.ceil(grid_size / voxel_size) / 2 * self.voxel_size 14 | # print(self.num_voxel) 15 | self.grid = [VOXEL(point_dim) for i in range(self.num_voxel)] 16 | self.used_voxel = [] 17 | 18 | def addPoint(self, point): 19 | idx = self.xyz2index(point) 20 | if(idx is not None): 21 | if (self.grid[idx].isEmpty()): 22 | self.used_voxel.append(idx) 23 | self.grid[idx].addPoint(point) 24 | 25 | def xyz2index(self, point): 26 | rcl = ((point - self.origin_offset)/self.voxel_size).astype("int") 27 | if np.any(rcl < 0) or np.any(rcl >= (self.grid_dim)): 28 | return None 29 | return rcl[0] + rcl[1] * self.grid_dim[0] + rcl[2] * self.grid_dim[0] * self.grid_dim[1] 30 | 31 | def cloud2indices(self, point_cld): 32 | ''' 33 | computes the grid index of each point and a bool if they are inside the grid 34 | point_cld [nx3] 35 | return 36 | ------ 37 | grid_idx [m,], m times 1d indices of the grid 38 | cloud_idx [m,] the indices of the points which are inside the grid 39 | | m: number valid points 40 | ''' 41 | rcl = ((point_cld - self.origin_offset)/self.voxel_size).astype("int") 42 | # print('rcl',rcl) 43 | valid= np.argwhere( np.all(rcl >= 0, axis=1) & np.all(rcl < (self.grid_dim),axis=1)).reshape(-1) 44 | # print('valid',valid) 45 | idx = rcl[valid,0] + rcl[valid,1] * self.grid_dim[0] + rcl[valid,2] * self.grid_dim[0] * self.grid_dim[1] 46 | # print('idx',idx) 47 | 48 | return idx, valid 49 | 50 | def addPointCloud(self, point_cld): 51 | ''' 52 | add all points [d dimensional] to the grid 53 | point_cld [nxd] | first 3 cols need to be xyz 54 | ''' 55 | grid_idx, cloud_idx = self.cloud2indices(point_cld[:, 0:3]) 56 | for g_idx, c_idx in zip(grid_idx, cloud_idx): 57 | if (self.grid[g_idx].isEmpty()): 58 | self.used_voxel.append(g_idx) 59 | self.grid[g_idx].addPoint(point_cld[c_idx, :]) 60 | 61 | def getPointCloud(self): 62 | ''' 63 | returns the voxels which are not empty 64 | ''' 65 | return np.asarray([self.grid[i].getValue() for i in self.used_voxel]) 66 | 67 | class AverageVoxel(): 68 | def __init__(self,point_dim): 69 | self.point = np.zeros((point_dim)) 70 | self.weight = 0 71 | # print('hi') 72 | 73 | def addPoint(self, point): 74 | self.point += point 75 | self.weight += 1 76 | 77 | def getValue(self): 78 | dim = self.point.shape[0] 79 | val = np.ones( (dim+1),dtype= np.float32 ) *self.weight 80 | val[0:dim]= self.point/self.weight 81 | return val 82 | 83 | def isEmpty(self): 84 | return self.weight == 0 85 | 86 | 87 | class AverageGrid(VoxelGrid): 88 | def __init__(self, center, grid_size, voxel_size, point_dim=3): 89 | super().__init__(AverageVoxel, center, grid_size, voxel_size, point_dim=point_dim) 90 | # print(self.grid_dim) 91 | 92 | 93 | 94 | 95 | if __name__ == "__main__": 96 | center = np.array([0.0, 0.0, 0.0]) 97 | grid_size = np.array([10.0, 10.0, 10.0]) 98 | voxel_grid = AverageGrid(center, grid_size, 5) 99 | 100 | p1 = np.array([2.0, -1.0, -1.0]) 101 | p2 = np.array([2.0, 1.0, 1.0]) 102 | p3 = np.array([6.0, 1.2, 1.0]) 103 | voxel_grid.addPoint(p1) 104 | voxel_grid.addPoint(p2) 105 | voxel_grid.addPoint(p3) 106 | voxel_grid.addPoint(p3) 107 | print('v1 used voxel',voxel_grid.used_voxel) 108 | p = voxel_grid.getPointCloud() 109 | print('v1 points',p.shape,p) 110 | 111 | voxel_grid2 = AverageGrid(center, grid_size, 5) 112 | pcs = (p1[np.newaxis,:],p2[np.newaxis,:],p3[np.newaxis,:],p3[np.newaxis,:]) 113 | cld = np.concatenate(pcs) 114 | print('cld',cld.shape) 115 | voxel_grid2.addPointCloud(cld) 116 | p2 = voxel_grid2.getPointCloud() 117 | print('v2 points',p2.shape,p2) 118 | print('diff',p-p2) 119 | 120 | -------------------------------------------------------------------------------- /depoco/data_handling/train_test_splitter.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import depoco.datasets.kitti2voxel as kitti2voxel 4 | import depoco.datasets.submap_handler as submap_handler 5 | import depoco.utils.point_cloud_utils as pcu 6 | import ruamel.yaml as yaml 7 | import argparse 8 | import time 9 | import os 10 | import matplotlib.pyplot as plt 11 | import numpy as np 12 | import glob 13 | import shutil 14 | 15 | 16 | def nearestPoint(points, qx, qy): 17 | dx = points[:, 0]-qx 18 | dy = points[:, 1]-qy 19 | d = (dx**2 + dy**2) 20 | min_idx = np.argmin(d) 21 | return min_idx 22 | 23 | 24 | def drawSubmap(submap: submap_handler.SubMap): 25 | pcu.visPointCloud(submap.getPoints()) 26 | 27 | 28 | def saveFiles(files, source_path): 29 | seq = files[0].split('/')[-2] 30 | print('seq', seq) 31 | filenames = [ 32 | f.split("/")[-1] for f in files] 33 | print(filenames) 34 | myfile = open(source_path+'validation_files.txt', 'w') 35 | 36 | myfile.write('#source_file target_file\n') 37 | # Write a line to the file 38 | for f in filenames: 39 | myfile.write(f+' '+seq+'_'+f+' \n') 40 | 41 | # Close the file 42 | myfile.close() 43 | 44 | 45 | def moveFiles(validation_files_txt, source_path, target_path, undo=False): 46 | f = open(validation_files_txt, "r") 47 | for i, x in enumerate(f): 48 | if i > 0: 49 | print(x) 50 | source = source_path + x.split(' ')[0] 51 | target = target_path + x.split(' ')[1] 52 | if undo: 53 | shutil.move(target, source) 54 | else: 55 | shutil.move(source, target) 56 | f.close() 57 | 58 | 59 | class Splitter(): 60 | def __init__(self, path): 61 | self.path = path 62 | self.submaps = submap_handler.createSubmaps([path]) 63 | print('submaps', len(self.submaps)) 64 | self.poses = np.loadtxt(path+'key_poses.txt') 65 | self.poses = np.reshape(self.poses, (self.poses.shape[0], 4, 4)) 66 | self.xy = self.poses[:, 0:2, -1] 67 | self.current_idx = None 68 | self.validation_idx = [] 69 | print('xy shape', self.xy.shape) 70 | 71 | def onclick(self, event): 72 | x = event.xdata 73 | y = event.ydata 74 | print('x', x, 'y', y) 75 | min_idx = nearestPoint(self.xy, x, y) 76 | print('min_idx', min_idx, 'pt', self.xy[min_idx, :]) 77 | self.ax.plot(self.xy[min_idx, 0], self.xy[min_idx, 1], 'xr') 78 | self.fig.canvas.draw() 79 | drawSubmap(self.submaps[min_idx]) 80 | self.current_idx = min_idx 81 | print('Use in validation set, press y') 82 | 83 | def keyPressed(self, event): 84 | if event.key == 'y': 85 | print('hey you pressed yes') 86 | if self.current_idx not in self.validation_idx: 87 | self.validation_idx.append(int(self.current_idx)) 88 | print('Aded map', self.current_idx, 'to the validation set') 89 | self.ax.plot(self.xy[self.current_idx, 0], 90 | self.xy[self.current_idx, 1], 'og') 91 | self.fig.canvas.draw() 92 | 93 | def draw(self): 94 | self.fig = plt.figure() 95 | self.ax = self.fig.add_subplot(111) 96 | cid = self.fig.canvas.mpl_connect('button_press_event', self.onclick) 97 | bit = self.fig.canvas.mpl_connect('key_press_event', self.keyPressed) 98 | plt.grid() 99 | 100 | self.ax.plot(self.xy[:, 0], self.xy[:, 1], '.b') 101 | plt.axis('equal') 102 | plt.show() 103 | print('the following:', len(self.validation_idx), 104 | 'submaps can be used for validation. For saving the locations press y') 105 | if input() == 'y': 106 | print('saved') 107 | validation_poses = np.reshape( 108 | self.poses[self.validation_idx, :], (len(self.validation_idx), 16)) 109 | np.savetxt(self.path+'validation_poses.txt', validation_poses) 110 | files = [self.submaps[i].file for i in self.validation_idx] 111 | print('files', files) 112 | saveFiles(files, self.path) 113 | else: 114 | print('Not saved.') 115 | 116 | 117 | if __name__ == "__main__": 118 | start_time = time.time() 119 | 120 | parser = argparse.ArgumentParser("./train_test_splitter.py") 121 | parser.add_argument( 122 | '--arch_cfg', '-ac', 123 | type=str, 124 | required=False, 125 | default='../config/arch/depoco.yaml', 126 | help='Architecture yaml cfg file. See /config/arch for sample. No default!', 127 | ) 128 | 129 | FLAGS, unparsed = parser.parse_known_args() 130 | ARCH = yaml.safe_load(open(FLAGS.arch_cfg, 'r')) 131 | 132 | # input_folder = FLAGS.dataset + '/sequences/00/' 133 | # calibration = kitti2voxel.parse_calibration(os.path.join(input_folder, "calib.txt")) 134 | # poses = kitti2voxel.parse_poses(os.path.join(input_folder, "poses.txt"), calibration) 135 | # idx, keypose, d= kitti2voxel.getKeyPoses(poses,delta=ARCH["grid"]["pose_distance"]) 136 | 137 | # xy = np.asarray(poses) 138 | # xy = xy[:,0:2,-1] 139 | # x = xy[:,0] 140 | # y = xy[:,1] 141 | # print(xy.shape) 142 | # plt.figure 143 | # plt.plot(xy[:,0], xy[:,1]) 144 | # plt.plot(xy[idx,0], xy[idx,1],'xr') 145 | # plt.axis('equal') 146 | 147 | ## TODO: change Path 148 | target_path = '/media/lwiesmann/WiesmannIPB/data/data_kitti/dataset/submaps/40m_ILEN/validation/' 149 | # splitter = Splitter(path) 150 | # splitter.draw() 151 | 152 | for i in range(10): 153 | try: 154 | path ='/media/lwiesmann/WiesmannIPB/data/data_kitti/dataset/submaps/40m_ILEN/0'+str(i)+'/' 155 | # print(path) 156 | moveFiles(path+'validation_files.txt', source_path=path, 157 | target_path=target_path, undo=False) 158 | except: 159 | print('Kitti {i} file not found') 160 | 161 | # moveFiles(path+'validation_files.txt', source_path=path, 162 | # target_path=target_path, undo=False) 163 | -------------------------------------------------------------------------------- /depoco/datasets/kitti2voxel.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import argparse 4 | from numpy.linalg import inv 5 | import time 6 | import os 7 | from ruamel import yaml 8 | from matplotlib import pyplot as plt 9 | import open3d as o3d 10 | import depoco.utils.point_cloud_utils as pcu 11 | from pathlib import Path 12 | import octree_handler 13 | 14 | 15 | def open_label(filename): 16 | """ Open raw scan and fill in attributes 17 | """ 18 | # check filename is string 19 | if not isinstance(filename, str): 20 | raise TypeError("Filename should be string type, " 21 | "but was {type}".format(type=str(type(filename)))) 22 | 23 | # check extension is a laserscan 24 | # if not any(filename.endswith(ext) for ext in self.EXTENSIONS_LABEL): 25 | # raise RuntimeError("Filename extension is not valid label file.") 26 | 27 | # if all goes well, open label 28 | label = np.fromfile(filename, dtype=np.int32) 29 | label = label.reshape((-1)) 30 | label = label & 0xFFFF 31 | 32 | # set it 33 | return label 34 | 35 | 36 | def parse_calibration(filename): 37 | """ read calibration file with given filename 38 | Returns 39 | ------- 40 | dict 41 | Calibration matrices as 4x4 numpy arrays. 42 | """ 43 | calib = {} 44 | 45 | calib_file = open(filename) 46 | for line in calib_file: 47 | key, content = line.strip().split(":") 48 | values = [float(v) for v in content.strip().split()] 49 | # print(key,values) 50 | if len(values) == 12: 51 | pose = np.zeros((4, 4)) 52 | pose[0, 0:4] = values[0:4] 53 | pose[1, 0:4] = values[4:8] 54 | pose[2, 0:4] = values[8:12] 55 | pose[3, 3] = 1.0 56 | 57 | calib[key] = pose 58 | 59 | calib_file.close() 60 | print('calibration', calib) 61 | return calib 62 | 63 | 64 | def parse_poses(filename, calibration): 65 | """ read poses file with per-scan poses from given filename 66 | Returns 67 | ------- 68 | list 69 | list of poses as 4x4 numpy arrays. 70 | """ 71 | file = open(filename) 72 | 73 | poses = [] 74 | 75 | Tr = calibration["Tr"] 76 | Tr_inv = inv(Tr) 77 | 78 | for line in file: 79 | values = [float(v) for v in line.strip().split()] 80 | 81 | pose = np.zeros((4, 4)) 82 | pose[0, 0:4] = values[0:4] 83 | pose[1, 0:4] = values[4:8] 84 | pose[2, 0:4] = values[8:12] 85 | pose[3, 3] = 1.0 86 | 87 | poses.append(np.matmul(Tr_inv, np.matmul(pose, Tr))) 88 | 89 | return poses 90 | 91 | 92 | def distanceMatrix(x, y): 93 | """ x[nxd], y[mxd], d = dimensionality (eg. 3) 94 | distance between each point of x to each point of y 95 | Return 96 | ----- 97 | distance matrix [nxm] 98 | 99 | """ 100 | dims = x.shape[1] 101 | dist = np.zeros((x.shape[0], y.shape[0])) 102 | for i in range(dims): 103 | dist += (x[:, i][..., np.newaxis] - y[:, i][np.newaxis, ...])**2 104 | return dist**0.5 105 | 106 | 107 | def getKeyPoses(pose_list, delta=50): 108 | ''' 109 | creates a key pose for every pose which is delta apart (horizontal distance) 110 | returns 111 | ------- 112 | idx, poses, distance_matrix [nxn] 113 | ''' 114 | poses = np.asarray(pose_list) 115 | xy = poses[:, 0:2, -1] 116 | dist = distanceMatrix(xy, xy) 117 | 118 | key_pose_idx = [] 119 | indices = np.arange(poses.shape[0]) 120 | 121 | dist_it = dist.copy() 122 | while (dist_it.shape[0] > 0): 123 | key_pose_idx.append(indices[0]) 124 | valid_idx = dist_it[0, :] > delta 125 | dist_it = dist_it[valid_idx, :] 126 | dist_it = dist_it[:, valid_idx] 127 | indices = indices[valid_idx] 128 | return key_pose_idx, poses[key_pose_idx], dist 129 | 130 | 131 | class Kitti2voxelConverter(): 132 | def __init__(self, config): 133 | self.config = config 134 | self.train_folders = [] 135 | self.valid_folders = [] 136 | self.test_folders = [] 137 | if type(config["dataset"]["data_folders"]["train"]) is list: 138 | self.train_folders = [pcu.path(config["dataset"]["data_folders"]["prefix"])+pcu.path( 139 | fldid) for fldid in config["dataset"]["data_folders"]["train"]] 140 | if type(config["dataset"]["data_folders"]["valid"]) is list: 141 | self.valid_folders = [pcu.path(config["dataset"]["data_folders"]["prefix"])+pcu.path( 142 | fldid) for fldid in config["dataset"]["data_folders"]["valid"]] 143 | if type(config["dataset"]["data_folders"]["test"]) is list: 144 | self.test_folders = [pcu.path(config["dataset"]["data_folders"]["prefix"])+pcu.path( 145 | fldid) for fldid in config["dataset"]["data_folders"]["test"]] 146 | 147 | def getMaxMinHeight(self): 148 | folders = self.train_folders + self.valid_folders + self.test_folders 149 | n_bins = 1000 150 | hist = np.zeros((n_bins)) 151 | time_start = time.time() 152 | 153 | for p in folders: 154 | # calibration = parse_calibration(p+ "calib.txt") 155 | # poses = parse_poses( p+"poses.txt", calibration) 156 | scan_files = [ 157 | f for f in sorted(os.listdir(os.path.join(p, "velodyne"))) 158 | if f.endswith(".bin")] 159 | for i, f in enumerate(scan_files): 160 | scan = np.fromfile(p+"velodyne/"+f, dtype=np.float32) 161 | scan = scan.reshape((-1, 4)) 162 | hist_i, temp_range = np.histogram( 163 | scan[:, 2], bins=n_bins, range=(-30, 30)) 164 | hist += hist_i 165 | 166 | print('min max time', time.time()-time_start) 167 | print('hist_size', hist.shape) 168 | 169 | plt.figure() 170 | plt.plot(temp_range[1:], hist) 171 | np.savetxt('hist', hist) 172 | plt.show() 173 | 174 | def sparsifieO3d(self, poses, key_pose_idx, seq_path, distance_matrix): 175 | grid_size = np.array((self.config['grid']['size'])) 176 | center = poses[key_pose_idx][0:3, -1] + \ 177 | np.array((0, 0, self.config['grid']['dz'])) 178 | upper_bound = center + grid_size/2 179 | lower_bound = center - grid_size/2 180 | valid_scans = np.argwhere( 181 | distance_matrix[key_pose_idx, :] < grid_size[0] + self.config['grid']['max_range']).squeeze() 182 | # print('valid scans', valid_scans,'shape',valid_scans.shape) 183 | point_cld = () 184 | features = () 185 | for i in valid_scans: 186 | sfile = seq_path + "velodyne/" + str(i).zfill(6)+'.bin' 187 | scan = np.fromfile(sfile, dtype = np.float32) if os.path.isfile(sfile) else np.zeros((0, 4)) 188 | scan=scan.reshape((-1, 4)) 189 | dists=np.linalg.norm(scan[:, 0:3], axis=1) 190 | valid_p=(dists > self.config['grid']['min_range']) & ( 191 | dists < self.config['grid']['max_range']) 192 | scan_hom=np.ones_like(scan) 193 | scan_hom[:, 0:3]=scan[:, 0:3] 194 | points=np.matmul(poses[i], scan_hom[valid_p, :].T).T 195 | #### intensity and label ####### 196 | intensity=scan[valid_p, 3:4] 197 | label=np.full((points.shape[0],), 2) 198 | if os.path.isfile(seq_path + "labels/" + 199 | str(i).zfill(6)+'.label'): 200 | label=open_label(filename=seq_path + "labels/" + 201 | str(i).zfill(6)+'.label')[valid_p] 202 | # print('max min label',np.max(label),np.min(label)) 203 | feature=np.hstack((intensity, np.expand_dims( 204 | label.astype('float'), axis=1), np.zeros_like(intensity))) # concat intensity,label and a 0 (0 for havin 3dims) 205 | 206 | points=points[:, 0:3] 207 | # print('features', feature.shape, 'points', points.shape) 208 | valids=np.all(points > lower_bound, axis=1) & np.all( 209 | points < upper_bound, axis=1).reshape(-1) & (label < 200) & (label > 1) # remove moving objects and outlier 210 | point_cld += (points[valids, :],) 211 | features += (feature[valids],) 212 | 213 | # o3d 214 | pcd=o3d.geometry.PointCloud() 215 | cloud=np.concatenate(point_cld) 216 | cloud_clr=np.concatenate(features) 217 | pcd.points=o3d.utility.Vector3dVector(cloud) 218 | pcd.colors=o3d.utility.Vector3dVector(cloud_clr) 219 | downpcd=pcd.voxel_down_sample( 220 | voxel_size=self.config['grid']['voxel_size']) 221 | sparse_points=np.asarray(downpcd.points) 222 | sparse_features=np.asarray(downpcd.colors) 223 | sparse_features=sparse_features[:, :2] 224 | sparse_features[:, 1]=np.around(sparse_features[:, 1]) 225 | sparse_points=np.hstack((sparse_points, sparse_features)) 226 | print('#points', cloud.shape, 'to', sparse_points.shape, 227 | 'features', sparse_features.shape) 228 | 229 | # print('grid time', time.time() - time_start) 230 | return sparse_points 231 | 232 | def convert(self): 233 | time_very_start=time.time() 234 | folders=self.train_folders + self.valid_folders + self.test_folders 235 | for j, p in enumerate(folders): 236 | # print('pcu.path',p.split('/')[-2]) 237 | out_dir=pcu.path( 238 | self.config['dataset']['data_folders']['grid_output'])+pcu.path(p.split('/')[-2]) 239 | if not os.path.exists(out_dir): 240 | os.makedirs(out_dir) 241 | calibration=parse_calibration(p + "calib.txt") 242 | poses=parse_poses(p+"poses.txt", calibration) 243 | scan_files=[ 244 | f for f in sorted(os.listdir(os.path.join(p, "velodyne"))) 245 | if f.endswith(".bin")] 246 | key_poses_idx, key_poses, distance_matrix=getKeyPoses( 247 | poses, self.config['grid']['pose_distance']) 248 | print('kp shape', key_poses.shape) 249 | np.savetxt(out_dir+'key_poses.txt', 250 | np.reshape(key_poses, (key_poses.shape[0], 16))) 251 | for i, idx in enumerate(key_poses_idx): 252 | bla=1 253 | time_start=time.time() 254 | sparse_points_features=self.sparsifieO3d( 255 | poses, idx, p, distance_matrix).astype('float32') 256 | print('seq', j, 'from', len(folders), 257 | 'keypose', i, 'from', len(key_poses_idx)) 258 | print('sparsifie time', time.time() - time_start) 259 | time_start=time.time() 260 | octree=octree_handler.Octree() 261 | points=sparse_points_features[:, :3] 262 | octree.setInput(points) 263 | eig_normals=octree.computeEigenvaluesNormal( 264 | self.config['grid']['normal_eigenvalue_radius']) 265 | sparse_points_features=np.hstack( 266 | (sparse_points_features, eig_normals)) 267 | print('normal and eigenvalues estimation time', 268 | time.time() - time_start) 269 | # print('sparse_points',sparse_points.shape) 270 | pcu.saveCloud2Binary(sparse_points_features, str( 271 | i).zfill(6)+'.bin', out_dir) 272 | 273 | # time_start = time.time() 274 | # sparse_points = self.sparsifieVoxelGrid(poses, idx, p, distance_matrix) 275 | # sparse_points = self.sparsifieO3d(poses, idx, p, distance_matrix) 276 | # print('sparsifie2 time', time.time() - time_start) 277 | # print('sparse_points2',sparse_points.shape) 278 | # pcu.visPointCloud(sparse_points) 279 | # np.savetxt('testgrid.xyz',sparse_points) 280 | # return None 281 | print('convert time', time.time() - time_very_start) 282 | 283 | 284 | if __name__ == "__main__": 285 | start_time=time.time() 286 | 287 | parser=argparse.ArgumentParser("./kitti2voxel.py") 288 | parser.add_argument( 289 | '--dataset', 290 | '-d', 291 | type=str, 292 | required=False, 293 | default="/mnt/91d100fa-d283-4eeb-b68c-e2b4b199d2de/wiesmann/data/data_kitti/dataset", 294 | help='dataset folder containing all sequences in a folder called "sequences".', 295 | ) 296 | parser.add_argument( 297 | '--arch_cfg', '-cfg', 298 | type=str, 299 | required=False, 300 | default='config/arch/sample_net.yaml', 301 | help='Architecture yaml cfg file. See /config/arch for sample. No default!', 302 | ) 303 | 304 | FLAGS, unparsed=parser.parse_known_args() 305 | ARCH=yaml.safe_load(open(FLAGS.arch_cfg, 'r')) 306 | 307 | input_folder=FLAGS.dataset + '/sequences/00/' 308 | calibration=parse_calibration(os.path.join(input_folder, "calib.txt")) 309 | poses=parse_poses(os.path.join(input_folder, "poses.txt"), calibration) 310 | idx, keypose, d=getKeyPoses(poses, delta=ARCH["grid"]["pose_distance"]) 311 | 312 | xy=np.asarray(poses) 313 | xy=xy[:, 0:2, -1] 314 | x=xy[:, 0] 315 | y=xy[:, 1] 316 | print(xy.shape) 317 | plt.figure 318 | plt.plot(xy[:, 0], xy[:, 1]) 319 | plt.plot(xy[idx, 0], xy[idx, 1], 'xr') 320 | plt.axis('equal') 321 | 322 | plt.show() 323 | 324 | converter=Kitti2voxelConverter(ARCH) 325 | # converter.getMaxMinHeight() # -9 to 4 326 | converter.convert() 327 | -------------------------------------------------------------------------------- /depoco/datasets/submap_handler.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import depoco.utils.point_cloud_utils as pcu 4 | # import torch 5 | import ruamel.yaml as yaml 6 | import argparse 7 | import glob 8 | import time 9 | from typing import Tuple, Union 10 | from torch.utils.data import Dataset, Sampler 11 | import torch 12 | import os 13 | ######################################## 14 | # Torch Data loader 15 | ######################################## 16 | 17 | 18 | class SubMapParser(): 19 | def __init__(self, config): 20 | self.config = config 21 | nr_submaps = config['train']['nr_submaps'] 22 | self.grid_size = np.reshape(np.asarray(config['grid']['size']), (1, 3)) 23 | # collect directories of the datasets 24 | out_path = os.environ.get( 25 | 'DATA_SUBMAPS', config["dataset"]["data_folders"]["grid_output"]) 26 | 27 | self.train_folders = [pcu.path(out_path)+pcu.path( 28 | fldid) for fldid in config["dataset"]["data_folders"]["train"]] if config["dataset"]["data_folders"]["train"] else [] 29 | self.valid_folders = [pcu.path(out_path)+pcu.path( 30 | fldid) for fldid in config["dataset"]["data_folders"]["valid"]] if config["dataset"]["data_folders"]["valid"] else [] 31 | # print('valid folders',self.valid_folders) 32 | self.test_folders = [pcu.path(out_path)+pcu.path( 33 | fldid) for fldid in config["dataset"]["data_folders"]["test"]] if config["dataset"]["data_folders"]["test"] else [] 34 | cols = 3+sum(config['grid']['feature_dim']) 35 | # Trainingset 36 | self.train_dataset = SubMapDataSet(data_dirs=self.train_folders, 37 | nr_submaps=nr_submaps, 38 | nr_points=config['train']['max_nr_pts'], cols=cols, on_the_fly=True, 39 | grid_size=np.max(self.grid_size)) 40 | self.train_sampler = SubMapSampler(nr_submaps=len(self.train_dataset), 41 | sampling_method=config['train']['sampling_method']) 42 | self.train_loader = torch.utils.data.DataLoader(dataset=self.train_dataset, 43 | sampler=self.train_sampler, 44 | batch_size=None, 45 | num_workers=0, 46 | ) 47 | self.train_iter = iter(self.train_loader) 48 | 49 | # Ordered Trainingset 50 | self.train_loader_ordered = torch.utils.data.DataLoader(dataset=self.train_dataset, 51 | batch_size=None, 52 | shuffle=False, 53 | num_workers=config['train']['workers']) 54 | self.train_iter_ordered = iter(self.train_loader_ordered) 55 | 56 | # Validationset 57 | self.valid_dataset = SubMapDataSet(data_dirs=self.valid_folders, 58 | nr_submaps=0, 59 | nr_points=config['train']['max_nr_pts'], 60 | cols=cols, on_the_fly=True, 61 | grid_size=np.max(self.grid_size)) 62 | self.valid_sampler = SubMapSampler(nr_submaps=len(self.valid_dataset), 63 | sampling_method='ordered') 64 | self.valid_loader = torch.utils.data.DataLoader(dataset=self.valid_dataset, 65 | batch_size=None, 66 | sampler=self.valid_sampler, 67 | num_workers=config['train']['workers'], 68 | ) 69 | self.valid_iter = iter(self.valid_loader) 70 | 71 | # Testset 72 | self.test_dataset = SubMapDataSet(data_dirs=self.test_folders, 73 | nr_submaps=0, 74 | nr_points=config['train']['max_nr_pts'], 75 | cols=cols, on_the_fly=True, 76 | grid_size=np.max(self.grid_size)) 77 | self.test_sampler = SubMapSampler(nr_submaps=len(self.test_dataset), 78 | sampling_method='ordered') 79 | self.test_loader = torch.utils.data.DataLoader(dataset=self.test_dataset, 80 | batch_size=None, 81 | sampler=self.test_sampler, 82 | num_workers=config['train']['workers'], 83 | ) 84 | self.test_iter = iter(self.test_loader) 85 | # print('test folder:',self.test_folders) 86 | 87 | def getOrderedTrainSet(self): 88 | return self.train_loader_ordered 89 | 90 | def setTrainProbabilities(self, probs): 91 | self.train_loader.sampler.setSampleProbs(probs) 92 | 93 | def getTrainBatch(self): 94 | scans = self.train_iter.next() 95 | return scans 96 | 97 | def getTrainSet(self): 98 | return self.train_loader 99 | 100 | def getValidBatch(self): 101 | scans = self.valid_iter.next() 102 | return scans 103 | 104 | def getValidSet(self): 105 | return self.valid_loader 106 | 107 | def getTestBatch(self): 108 | scans = self.test_iter.next() 109 | return scans 110 | 111 | def getTestSet(self): 112 | return self.test_loader 113 | 114 | def getTrainSize(self): 115 | return len(self.train_loader) 116 | 117 | def getValidSize(self): 118 | return len(self.valid_loader) 119 | 120 | def getTestSize(self): 121 | return len(self.test_loader) 122 | 123 | 124 | class SubMapSampler(Sampler): 125 | def __init__(self, nr_submaps, sampling_method='random', nr_samples=-1): 126 | """[summary] 127 | 128 | Arguments: 129 | nr_submaps {int} -- [description] 130 | sampling_method {string} -- in [random, ordered], random: sample from specified distribution (init with uniform) 131 | 132 | Keyword Arguments: 133 | nr_samples {int} -- [description] (default: {-1}) -1: all submaps 134 | """ 135 | self.probs = None 136 | self.nr_submaps = nr_submaps 137 | if nr_samples < 0: 138 | self.nr_samples = nr_submaps 139 | else: 140 | self.nr_samples = nr_samples 141 | 142 | self.sample_fkt = getattr(self, sampling_method) 143 | self.p_func = torch.ones(self.nr_submaps, dtype=torch.float) 144 | self.dist = torch.distributions.Categorical(self.p_func) 145 | 146 | def setSampleProbs(self, probs): 147 | self.p_func = probs 148 | self.dist = torch.distributions.Categorical(self.p_func) 149 | 150 | def random(self): 151 | return (self.dist.sample() for _ in range(self.nr_samples)) 152 | 153 | def ordered(self): 154 | return (i for i in torch.arange(self.nr_samples)) 155 | 156 | def __iter__(self): 157 | return self.sample_fkt() 158 | 159 | def __len__(self): 160 | return self.nr_samples 161 | 162 | 163 | class SubMapDataSet(Dataset): 164 | def __init__(self, data_dirs, 165 | nr_submaps=0, 166 | nr_points=10000, 167 | cols=3, 168 | on_the_fly=True, 169 | init_ones=True, 170 | feature_cols=[], 171 | grid_size = 40): 172 | self.data_dirs = data_dirs 173 | self.nr_submaps = nr_submaps 174 | self.nr_points = nr_points 175 | self.cols = cols 176 | self.init_ones = init_ones 177 | self.fc = feature_cols 178 | self.submaps = createSubmaps( 179 | data_dirs, nr_submaps=self.nr_submaps, cols=cols, on_the_fly=on_the_fly,grid_size=grid_size) # list of submaps 180 | 181 | def __getitem__(self, index): 182 | out_dict = {'idx': index} 183 | self.submaps[index].initialize() 184 | if self.cols <= 3: 185 | out_dict['points'] = self.submaps[index].getRandPoints( 186 | self.nr_points, seed=index) 187 | out_dict['map'] = self.submaps[index].getPoints() 188 | if self.init_ones: 189 | out_dict['features'] = np.ones( 190 | (out_dict['points'].shape[0], 1), dtype='float32') 191 | else: 192 | points = self.submaps[index].getRandPoints( 193 | self.nr_points, seed=index) 194 | out_dict['points'] = points[:, :3] 195 | out_dict['points_attributes'] = points[:, 3:] 196 | map_ = self.submaps[index].getPoints() 197 | out_dict['map'] = map_[:, :3] 198 | out_dict['map_attributes'] = map_[:, 3:] 199 | if self.init_ones: 200 | out_dict['features'] = np.hstack( 201 | (np.ones((points.shape[0], 1), dtype='float32'), out_dict['points_attributes'][:, self.fc])) 202 | else: 203 | out_dict['features'] = out_dict['points_attributes'][:, self.fc] 204 | out_dict['features_original'] = out_dict['features'] 205 | out_dict['scale'] = self.submaps[index].getScale() 206 | return out_dict 207 | 208 | def __len__(self): 209 | return len(self.submaps) 210 | 211 | 212 | class SubMap(): 213 | def __init__(self, file, grid_size=None, on_the_fly=False, file_cols=3): 214 | self.file = file 215 | # self.embedding = torch.zeros((embedding_dim)) 216 | self.seq = file.split('/')[-2] 217 | self.id = file.split('/')[-1] 218 | self.cols = file_cols 219 | self.points = pcu.loadCloudFromBinary( 220 | self.file, cols=file_cols) if not on_the_fly else None 221 | self.normalizer = None 222 | self.grid_size = grid_size 223 | 224 | self.initialized = False 225 | if not on_the_fly: 226 | self.initialize() 227 | self.points = np.hstack((self.normalizer.normalize( 228 | self.points[:, :3]), self.points[:, 3:])) 229 | 230 | def initialize(self): 231 | if not self.initialized: 232 | self.normalizer = Normalizer( 233 | data=self.getPoints(normalize=False)[:, :3], dif=self.grid_size) 234 | self.initialized = True 235 | 236 | def normRange(self): 237 | return self.normalizer.normRange() 238 | 239 | def getScale(self): 240 | return self.normalizer.getScale() 241 | 242 | def __len__(self): 243 | # points = pcu.loadCloudFromBinary(self.file) 244 | return self.getPoints().shape[0] 245 | 246 | def getSample(self, idx): 247 | # points = pcu.loadCloudFromBinary(self.file) 248 | return self.getPoints()[idx, :] 249 | 250 | def getPoints(self, normalize=True): 251 | # points = pcu.loadCloudFromBinary(self.file) 252 | if self.points is None: 253 | points = pcu.loadCloudFromBinary(self.file, cols=self.cols) 254 | if normalize: 255 | points = np.hstack( 256 | (self.normalizer.normalize(points[:, :3]), points[:, 3:])) 257 | return points 258 | else: 259 | return self.points 260 | 261 | def getRandPoints(self, nr_points, seed=0): 262 | points = self.getPoints() 263 | act_nr_pts = points.shape[0] 264 | subm_idx = np.arange(act_nr_pts) 265 | np.random.seed(seed) 266 | np.random.shuffle(subm_idx) 267 | # print('shuffled idx',subm_idx) 268 | subm_idx = subm_idx[0:min(act_nr_pts, nr_points)] 269 | return points[subm_idx, :] 270 | 271 | 272 | def createSubmaps(folders, nr_submaps=0, cols=3, on_the_fly=False,grid_size=40): 273 | submap_files = [] 274 | for folder in sorted(folders): 275 | submap_files += sorted(glob.glob(folder+'*bin')) 276 | if int(nr_submaps) != 0: 277 | # print('taking n maps:',nr_submaps) 278 | n = min((len(submap_files), nr_submaps)) 279 | submap_files = submap_files[:n] 280 | submaps = [SubMap(f, file_cols=cols, 281 | on_the_fly=on_the_fly,grid_size=grid_size) for f in submap_files] 282 | # print(submaps[0].seq, submaps[0].id,len(submaps[0])) 283 | return submaps 284 | 285 | 286 | class Normalizer(): 287 | def __init__(self, data, dif=None): 288 | self.min = np.amin(data, axis=0, keepdims=True) 289 | self.max = np.amax(data, axis=0, keepdims=True) 290 | if dif is None: 291 | self.dif = self.max-self.min 292 | else: 293 | self.dif = dif 294 | 295 | def getScale(self): 296 | return self.dif 297 | 298 | def normalize(self, points): 299 | return (points - self.min)/self.dif 300 | 301 | def recover(self, norm_points): 302 | return (norm_points * self.dif)+self.min 303 | 304 | 305 | if __name__ == "__main__": 306 | parser = argparse.ArgumentParser("./submap_handler.py") 307 | parser.add_argument( 308 | '--cfg', '-c', 309 | type=str, 310 | required=False, 311 | default='config/arch/sample_net.yaml', 312 | help='Architecture yaml cfg file. See /config/arch for sample. No default!', 313 | ) 314 | 315 | FLAGS, unparsed = parser.parse_known_args() 316 | config = yaml.safe_load(open(FLAGS.cfg, 'r')) 317 | s = time.time() 318 | 319 | print(20*'#', 'Torch data loader', 20*'#') 320 | s = time.time() 321 | submap_parser = SubMapParser(config) 322 | print('submap init time', time.time()-s) 323 | for epoch in range(2): 324 | for it, out_dict in enumerate(submap_parser.getTrainSet()): 325 | print(it, 'sm:', out_dict) 326 | print('points shape', out_dict['points'].shape) 327 | print('map shape', out_dict['map'].shape) 328 | submap_parser.setTrainProbabilities( 329 | torch.tensor([1.0, 0, 0, 0, 0])) 330 | -------------------------------------------------------------------------------- /depoco/evaluate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from depoco.trainer import DepocoNetTrainer 4 | from ruamel import yaml 5 | import argparse 6 | import time 7 | import depoco.utils.point_cloud_utils as pcu 8 | import os 9 | 10 | if __name__ == "__main__": 11 | print('Hello') 12 | parser = argparse.ArgumentParser("./evaluate.py") 13 | parser.add_argument( 14 | '--config_cfg', '-cfg', 15 | type=str, 16 | required=False, 17 | default='config/depoco.yaml', 18 | help='configitecture yaml cfg file. See /config/config for sample. No default!', 19 | ) 20 | parser.add_argument( 21 | '--file_ext', '-fe', 22 | type=str, 23 | required=False, 24 | default='', 25 | help='Extends the output file name by the given string', 26 | ) 27 | FLAGS, unparsed = parser.parse_known_args() 28 | 29 | print('passed flags') 30 | config = yaml.safe_load(open(FLAGS.config_cfg, 'r')) 31 | print('loaded yaml flags') 32 | print('config:', FLAGS.config_cfg) 33 | trainer = DepocoNetTrainer(config) 34 | print('initialized trainer') 35 | # trainer.train() 36 | ts = time.time() 37 | test_dict = trainer.test( 38 | best=True) 39 | print('evaluation time:', time.time()-ts) 40 | # trainer.validate(load_model=True) 41 | print('rec_err', test_dict['mapwise_reconstruction_error'][0:10]) 42 | print('memory', test_dict['memory'][0:10]) 43 | 44 | if not os.path.exists(config['evaluation']['out_dir']): 45 | os.makedirs(config['evaluation']['out_dir']) 46 | 47 | file = config['evaluation']['out_dir'] + \ 48 | trainer.experiment_id+FLAGS.file_ext+".pkl" 49 | pcu.save_obj(test_dict, file) 50 | -------------------------------------------------------------------------------- /depoco/evaluation/evaluator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import chamfer3D.dist_chamfer_3D 3 | 4 | import depoco.evaluation.occupancy_grid as occupancy_grid 5 | import numpy as np 6 | from collections import defaultdict 7 | import torch.nn as nn 8 | 9 | 10 | class Evaluator(): 11 | def __init__(self, config): 12 | self.config = config 13 | self.cham_loss = chamfer3D.dist_chamfer_3D.chamfer_3DDist() 14 | self.running_loss = 0.0 15 | self.n = 0 16 | self.eval_results = defaultdict(list) 17 | self.l1_loss = nn.L1Loss() 18 | self.l2_loss = nn.MSELoss() 19 | 20 | def chamferDist(self, gt_points: torch.tensor, source_points: torch.tensor): 21 | """computes the chamfer distance between 2 point clouds 22 | 23 | Arguments: 24 | gt_points {torch.tensor} -- [description] 25 | source_points {torch.tensor} -- [description] 26 | 27 | Returns: 28 | [type] -- [description] 29 | """ 30 | gt_points = gt_points.cuda().detach() 31 | source_points = source_points.cuda().detach() 32 | d_gt2source, d_source2gt, idx3, idx4 = self.cham_loss( 33 | gt_points.unsqueeze(0), source_points.unsqueeze(0)) 34 | # mean(squared_d(gt->source)) + mean(squared_d(source->gt)) 35 | loss = (d_gt2source.mean() + d_source2gt.mean()) # /2 FIXME: 36 | self.running_loss += loss.cpu().item() 37 | self.n += 1 38 | return loss 39 | 40 | def evaluate(self, gt_points: torch.tensor, source_points: torch.tensor, gt_normals=None): 41 | """computes the chamfer distance between 2 point clouds 42 | 43 | Arguments: 44 | gt_points {torch.tensor} -- [description] 45 | source_points {torch.tensor} -- [description] 46 | 47 | Returns: 48 | [dict] -- [description] 49 | """ 50 | ##### Computing Chamfer Distances ###### 51 | gt_points = gt_points.cuda().detach() 52 | source_points = source_points.cuda().detach() 53 | d_gt2source, d_source2gt, idx3, idx4 = self.cham_loss( 54 | gt_points.unsqueeze(0), source_points.unsqueeze(0)) 55 | idx3 = idx3.long().squeeze() 56 | idx4 = idx4.long().squeeze() 57 | # mean(squared_d(gt->source)) + mean(squared_d(source->gt)) 58 | chamfer_dist = (d_gt2source.mean() + d_source2gt.mean())/2 59 | chamfer_dist_abs = (d_gt2source.sqrt().mean() + 60 | d_source2gt.sqrt().mean())/2 61 | out_dict = {} 62 | out_dict['chamfer_dist'] = chamfer_dist.cpu().item() 63 | self.eval_results['chamfer_dist'].append(out_dict['chamfer_dist']) 64 | out_dict['chamfer_dist_abs'] = chamfer_dist_abs.cpu().item() 65 | self.eval_results['chamfer_dist_abs'].append( 66 | out_dict['chamfer_dist_abs']) 67 | 68 | 69 | ############ PSNR ############## 70 | if gt_normals is not None: # Computing PSNR if we have normals 71 | gt_normals = gt_normals.cuda().detach() 72 | d_plane_gt2source = torch.sum( 73 | (gt_points - source_points[idx3, :])*gt_normals, dim=1) 74 | d_plane_source2gt = torch.sum( 75 | (source_points - gt_points[idx4, :])*gt_normals[idx4, :], dim=1) 76 | chamfer_plane = (d_plane_gt2source.abs().mean() + 77 | d_plane_source2gt.abs().mean())/2 78 | out_dict['chamfer_dist_plane'] = chamfer_plane.cpu().item() 79 | self.eval_results['chamfer_dist_plane'].append( 80 | out_dict['chamfer_dist_plane']) 81 | 82 | ###### IOU ####### 83 | gt_points_np = gt_points.cpu().numpy() 84 | source_points_np = source_points.cpu().numpy() 85 | 86 | # print('gt_points shape',g) 87 | center = (np.max(gt_points_np, axis=0, keepdims=True) + 88 | np.min(gt_points_np, axis=0, keepdims=True))/2 89 | resolution = np.array( 90 | [self.config['evaluation']['iou_grid']['resolution']]) 91 | size_meter = np.array([self.config['grid']['size']]) 92 | gt_grid = occupancy_grid.OccupancyGrid( 93 | center=center, resolution=resolution, size_meter=size_meter) 94 | gt_grid.addPoints(gt_points_np) 95 | source_grid = occupancy_grid.OccupancyGrid( 96 | center=center, resolution=resolution, size_meter=size_meter) 97 | source_grid.addPoints(source_points_np) 98 | 99 | out_dict['iou'] = occupancy_grid.gridIOU( 100 | gt_grid.grid, source_grid.grid) 101 | self.eval_results['iou'].append(out_dict['iou']) 102 | 103 | return out_dict 104 | 105 | def getRunningLoss(self): 106 | """returns the running loss: loss/n 107 | sets the loss back to 0 108 | 109 | Returns: 110 | [int] -- average chamfer distance 111 | """ 112 | if self.n == 0: 113 | return None 114 | loss = self.running_loss / self.n 115 | self.running_loss = 0.0 116 | self.n = 0 117 | return loss 118 | -------------------------------------------------------------------------------- /depoco/evaluation/occupancy_grid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class OccupancyGrid(): 5 | def __init__(self, center: np.array, resolution: np.array, size_meter: np.array): 6 | self.center = center # 1x3 7 | self.resolution = resolution 8 | self.size_meter = size_meter 9 | self.size = np.ceil(size_meter / resolution) # rows x cols x layer 10 | self.min_corner = self.center - self.size*self.resolution/2 11 | 12 | self.grid = np.zeros(np.squeeze(self.size.astype('int')), dtype='bool') 13 | 14 | def addPoints(self, points: np.array): 15 | points_l = np.floor((points - self.min_corner)/self.resolution).astype('int') 16 | valids = np.all((points_l >= 0) & (points_l < self.size),axis=1) 17 | points_l = points_l[valids,:] 18 | self.grid[points_l[:,0],points_l[:,1],points_l[:,2]]= True 19 | 20 | def gridIOU(gt_grid:np.array, source_grid:np.array): 21 | return np.sum((gt_grid & source_grid))/np.sum((gt_grid | source_grid)) 22 | 23 | if __name__ == "__main__": 24 | occ_grid = OccupancyGrid(center=np.zeros((1,3)),resolution=np.full((1,3),2),size_meter=np.full((1,3),6)) 25 | print('occ grid \n',occ_grid.grid) 26 | points = np.array([ 27 | # [0,0,0], 28 | [0.5,0.5,0.5], 29 | [0.7,0.7,0.7], 30 | [0.7,0.7,0.7], 31 | [2.5,2.5,2.5], 32 | [-2.5,2.5,2.5], 33 | [100,100,100] 34 | ]) 35 | occ_grid.addPoints(points) 36 | print('occ grid \n',occ_grid.grid) 37 | 38 | occ_grid2 = OccupancyGrid(center=np.zeros((1,3)),resolution=np.full((1,3),2),size_meter=np.full((1,3),6)) 39 | points = np.array([ 40 | # [0,0,0], 41 | [0.5,0.5,0.5], 42 | [0.7,0.7,0.7], 43 | [0.7,0.7,0.7], 44 | [2.5,-2.5,2.5], 45 | [-2.5,2.5,2.5], 46 | [100,100,100] 47 | ]) 48 | occ_grid2.addPoints(points) 49 | iou = gridIOU(occ_grid.grid,occ_grid2.grid) 50 | print('occ grid2 \n',occ_grid2.grid) 51 | print('iou',iou) 52 | ### Big grid: 53 | big_grid= OccupancyGrid(center=np.zeros((1,3)),resolution=np.array((0.2,0.2,0.1)),size_meter=np.array((40,40,15.0))) 54 | print(big_grid.grid.shape) -------------------------------------------------------------------------------- /depoco/experiments/results/kitti/e0.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/experiments/results/kitti/e0.pkl -------------------------------------------------------------------------------- /depoco/experiments/results/kitti/e1.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/experiments/results/kitti/e1.pkl -------------------------------------------------------------------------------- /depoco/experiments/results/kitti/e2.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/experiments/results/kitti/e2.pkl -------------------------------------------------------------------------------- /depoco/experiments/results/kitti/e3.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/experiments/results/kitti/e3.pkl -------------------------------------------------------------------------------- /depoco/network_files/e0/dec.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e0/dec.pth -------------------------------------------------------------------------------- /depoco/network_files/e0/dec_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e0/dec_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e0/e0.yaml: -------------------------------------------------------------------------------- 1 | train: 2 | experiment_id: e0 3 | max_epochs: 200 4 | use_adam: true 5 | batch_size: 3 6 | max_nr_pts: 30000 7 | workers: 0 8 | optimizer: 9 | enable: true 10 | start_lr: 1e-06 11 | max_lr: 0.0001 12 | end_lr: 1e-05 13 | pct_incr_cycle: 0.1 14 | anneal_strategy: cos 15 | momentum: 0.9 16 | nr_submaps: 0 17 | load_pretrained: false 18 | sampling_method: random 19 | map_prob_rate: -1 20 | loss_weights: 21 | transf2map: 1.0 22 | map2transf: 1.0 23 | upsampling_reg: 0.2 24 | validation: 25 | nr_samples: 10000 26 | report_rate: 1 27 | save_result_rate: 500 28 | grid: 29 | pose_distance: 15 30 | size: 31 | - 40.0 32 | - 40.0 33 | - 15.0 34 | dz: 4.0 35 | voxel_size: 0.1 36 | max_range: 20.0 37 | min_range: 2.0 38 | features: 39 | - intensity 40 | - label 41 | - eigenvalues 42 | - normals 43 | feature_dim: 44 | - 1 45 | - 1 46 | - 3 47 | - 3 48 | normal_eigenvalue_radius: 0.5 49 | evaluation: 50 | target_map: map 51 | iou_grid: 52 | resolution: 53 | - 0.2 54 | - 0.2 55 | - 0.1 56 | f_score_dist: 0.1 57 | out_dir: experiments/results/kitti/ 58 | float16: True 59 | network: 60 | encoder_blocks: 61 | - type: GridSampleConv 62 | number_blocks: 3 63 | parameters: 64 | in_fdim: 65 | - 1 66 | - 16 67 | - 32 68 | out_fdim: 69 | - 16 70 | - 32 71 | - 32 72 | num_kernel_points: 27 73 | max_nr_neighbors: 74 | - 70 75 | - 50 76 | - 25 77 | relu: true 78 | batchnorm: true 79 | deformable: false 80 | subsampling_dist: 3 81 | map_size: 40 82 | subsampling_factor: 83 | - 0.1 84 | - 0.5 85 | - 1.0 86 | kernel_radius: 1.0 87 | min_kernel_radius: 1.5 88 | use_dif_sampling: false 89 | - type: LinearLayer 90 | number_blocks: 1 91 | parameters: 92 | in_fdim: 32 93 | out_fdim: 3 94 | relu: false 95 | batchnorm: false 96 | decoder_blocks: 97 | - type: AdaptiveDeconv 98 | number_blocks: 4 99 | parameters: 100 | number_blocks: 4 101 | block_id: 102 | - 0 103 | - 1 104 | - 2 105 | - 3 106 | subsampling_dist: 3 107 | in_fdim: 108 | - 3 109 | - 32 110 | - 32 111 | - 32 112 | out_fdim: 32 113 | kernel_radius: 0.05 114 | relu: true 115 | use_batch_norm: false 116 | inter_fdim: 128 117 | estimate_radius: false 118 | subsampling_fct_p1: 0.006 119 | subsampling_fct_p2: 1.764 120 | - type: LinearLayer 121 | number_blocks: 1 122 | parameters: 123 | in_fdim: 32 124 | out_fdim: 3 125 | relu: false 126 | batchnorm: false 127 | out_dir: network_files/ 128 | dataset: 129 | data_folders: 130 | grid_output: /data/ 131 | prefix: /path/to/kitti/ 132 | train: 133 | - '00' 134 | - '01' 135 | - '02' 136 | - '03' 137 | - '04' 138 | - '05' 139 | - '06' 140 | - '07' 141 | - '09' 142 | - '10' 143 | valid: 144 | - validation 145 | test: 146 | - '08' 147 | git_commit_version: b'42d0bec' 148 | -------------------------------------------------------------------------------- /depoco/network_files/e0/enc.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e0/enc.pth -------------------------------------------------------------------------------- /depoco/network_files/e0/enc_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e0/enc_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e1/dec.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e1/dec.pth -------------------------------------------------------------------------------- /depoco/network_files/e1/dec_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e1/dec_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e1/e1.yaml: -------------------------------------------------------------------------------- 1 | train: 2 | experiment_id: e1 3 | max_epochs: 200 4 | use_adam: true 5 | batch_size: 3 6 | max_nr_pts: 30000 7 | workers: 0 8 | optimizer: 9 | enable: true 10 | start_lr: 1e-06 11 | max_lr: 0.0001 12 | end_lr: 1e-05 13 | pct_incr_cycle: 0.1 14 | anneal_strategy: cos 15 | momentum: 0.9 16 | nr_submaps: 0 17 | load_pretrained: false 18 | sampling_method: random 19 | map_prob_rate: -1 20 | loss_weights: 21 | transf2map: 1.0 22 | map2transf: 1.0 23 | upsampling_reg: 0.2 24 | validation: 25 | nr_samples: 10000 26 | report_rate: 1 27 | save_result_rate: 500 28 | grid: 29 | pose_distance: 15 30 | size: 31 | - 40.0 32 | - 40.0 33 | - 15.0 34 | dz: 4.0 35 | voxel_size: 0.1 36 | max_range: 20.0 37 | min_range: 2.0 38 | features: 39 | - intensity 40 | - label 41 | - eigenvalues 42 | - normals 43 | feature_dim: 44 | - 1 45 | - 1 46 | - 3 47 | - 3 48 | normal_eigenvalue_radius: 0.5 49 | evaluation: 50 | float16: True 51 | target_map: map 52 | iou_grid: 53 | resolution: 54 | - 0.2 55 | - 0.2 56 | - 0.1 57 | f_score_dist: 0.1 58 | out_dir: experiments/results/kitti/ 59 | network: 60 | encoder_blocks: 61 | - type: GridSampleConv 62 | number_blocks: 3 63 | parameters: 64 | in_fdim: 65 | - 1 66 | - 16 67 | - 32 68 | out_fdim: 69 | - 16 70 | - 32 71 | - 32 72 | num_kernel_points: 27 73 | max_nr_neighbors: 74 | - 70 75 | - 50 76 | - 25 77 | relu: true 78 | batchnorm: true 79 | deformable: false 80 | subsampling_dist: 2 81 | map_size: 40 82 | subsampling_factor: 83 | - 0.1 84 | - 0.5 85 | - 1.0 86 | kernel_radius: 1.0 87 | min_kernel_radius: 1.5 88 | use_dif_sampling: false 89 | - type: LinearLayer 90 | number_blocks: 1 91 | parameters: 92 | in_fdim: 32 93 | out_fdim: 3 94 | relu: false 95 | batchnorm: false 96 | decoder_blocks: 97 | - type: AdaptiveDeconv 98 | number_blocks: 4 99 | parameters: 100 | number_blocks: 4 101 | block_id: 102 | - 0 103 | - 1 104 | - 2 105 | - 3 106 | subsampling_dist: 2 107 | in_fdim: 108 | - 3 109 | - 32 110 | - 32 111 | - 32 112 | out_fdim: 32 113 | kernel_radius: 0.05 114 | relu: true 115 | use_batch_norm: false 116 | inter_fdim: 128 117 | estimate_radius: false 118 | subsampling_fct_p1: 0.006 119 | subsampling_fct_p2: 1.764 120 | - type: LinearLayer 121 | number_blocks: 1 122 | parameters: 123 | in_fdim: 32 124 | out_fdim: 3 125 | relu: false 126 | batchnorm: false 127 | out_dir: network_files/ 128 | dataset: 129 | data_folders: 130 | grid_output: /data/ 131 | prefix: /path/to/kitti/ 132 | train: 133 | - '00' 134 | - '01' 135 | - '02' 136 | - '03' 137 | - '04' 138 | - '05' 139 | - '06' 140 | - '07' 141 | - '09' 142 | - '10' 143 | valid: 144 | - validation 145 | test: 146 | - '08' 147 | git_commit_version: b'42d0bec' 148 | -------------------------------------------------------------------------------- /depoco/network_files/e1/enc.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e1/enc.pth -------------------------------------------------------------------------------- /depoco/network_files/e1/enc_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e1/enc_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e2/dec.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e2/dec.pth -------------------------------------------------------------------------------- /depoco/network_files/e2/dec_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e2/dec_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e2/e2.yaml: -------------------------------------------------------------------------------- 1 | train: 2 | experiment_id: e2 3 | max_epochs: 200 4 | use_adam: true 5 | batch_size: 3 6 | max_nr_pts: 30000 7 | workers: 0 8 | optimizer: 9 | enable: true 10 | start_lr: 1e-06 11 | max_lr: 0.0001 12 | end_lr: 1e-05 13 | pct_incr_cycle: 0.1 14 | anneal_strategy: cos 15 | momentum: 0.9 16 | nr_submaps: 0 17 | load_pretrained: false 18 | sampling_method: random 19 | map_prob_rate: -1 20 | loss_weights: 21 | transf2map: 1.0 22 | map2transf: 1.0 23 | upsampling_reg: 0.2 24 | validation: 25 | nr_samples: 10000 26 | report_rate: 1 27 | save_result_rate: 2 28 | grid: 29 | pose_distance: 15 30 | size: 31 | - 40.0 32 | - 40.0 33 | - 15.0 34 | dz: 4.0 35 | voxel_size: 0.1 36 | max_range: 20.0 37 | min_range: 2.0 38 | features: 39 | - intensity 40 | - label 41 | - eigenvalues 42 | - normals 43 | feature_dim: 44 | - 1 45 | - 1 46 | - 3 47 | - 3 48 | normal_eigenvalue_radius: 0.5 49 | evaluation: 50 | target_map: map 51 | float16: True 52 | iou_grid: 53 | resolution: 54 | - 0.2 55 | - 0.2 56 | - 0.1 57 | f_score_dist: 0.1 58 | out_dir: experiments/results/kitti/ 59 | network: 60 | encoder_blocks: 61 | - type: GridSampleConv 62 | number_blocks: 3 63 | parameters: 64 | in_fdim: 65 | - 1 66 | - 16 67 | - 32 68 | out_fdim: 69 | - 16 70 | - 32 71 | - 32 72 | num_kernel_points: 27 73 | max_nr_neighbors: 74 | - 70 75 | - 50 76 | - 25 77 | relu: true 78 | batchnorm: true 79 | deformable: false 80 | subsampling_dist: 1.2 81 | map_size: 40 82 | subsampling_factor: 83 | - 0.1 84 | - 0.5 85 | - 1.0 86 | kernel_radius: 1.0 87 | min_kernel_radius: 1.2 88 | use_dif_sampling: false 89 | - type: LinearLayer 90 | number_blocks: 1 91 | parameters: 92 | in_fdim: 32 93 | out_fdim: 3 94 | relu: false 95 | batchnorm: false 96 | decoder_blocks: 97 | - type: AdaptiveDeconv 98 | number_blocks: 4 99 | parameters: 100 | number_blocks: 4 101 | block_id: 102 | - 0 103 | - 1 104 | - 2 105 | - 3 106 | subsampling_dist: 1.2 107 | in_fdim: 108 | - 3 109 | - 32 110 | - 32 111 | - 32 112 | out_fdim: 32 113 | kernel_radius: 0.03 114 | relu: true 115 | use_batch_norm: false 116 | inter_fdim: 128 117 | estimate_radius: false 118 | subsampling_fct_p1: 0.006 119 | subsampling_fct_p2: 1.764 120 | - type: LinearLayer 121 | number_blocks: 1 122 | parameters: 123 | in_fdim: 32 124 | out_fdim: 3 125 | relu: false 126 | batchnorm: false 127 | out_dir: network_files/ 128 | dataset: 129 | data_folders: 130 | grid_output: /data/ 131 | prefix: /path/to/kitti/ 132 | train: 133 | - '00' 134 | - '01' 135 | - '02' 136 | - '03' 137 | - '04' 138 | - '05' 139 | - '06' 140 | - '07' 141 | - '09' 142 | - '10' 143 | valid: 144 | - validation 145 | test: 146 | - '08' 147 | git_commit_version: b'42d0bec' 148 | -------------------------------------------------------------------------------- /depoco/network_files/e2/enc.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e2/enc.pth -------------------------------------------------------------------------------- /depoco/network_files/e2/enc_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e2/enc_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e3/dec.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e3/dec.pth -------------------------------------------------------------------------------- /depoco/network_files/e3/dec_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e3/dec_best.pth -------------------------------------------------------------------------------- /depoco/network_files/e3/e3.yaml: -------------------------------------------------------------------------------- 1 | train: 2 | experiment_id: e3 3 | max_epochs: 200 4 | use_adam: true 5 | batch_size: 3 6 | max_nr_pts: 30000 7 | workers: 0 8 | optimizer: 9 | enable: true 10 | start_lr: 1e-06 11 | max_lr: 0.0001 12 | end_lr: 1e-05 13 | pct_incr_cycle: 0.1 14 | anneal_strategy: cos 15 | momentum: 0.9 16 | nr_submaps: 0 17 | load_pretrained: false 18 | sampling_method: random 19 | map_prob_rate: -1 20 | loss_weights: 21 | transf2map: 1.0 22 | map2transf: 1.0 23 | upsampling_reg: 0.2 24 | validation: 25 | nr_samples: 10000 26 | report_rate: 1 27 | save_result_rate: 500 28 | grid: 29 | pose_distance: 15 30 | size: 31 | - 40.0 32 | - 40.0 33 | - 15.0 34 | dz: 4.0 35 | voxel_size: 0.1 36 | max_range: 20.0 37 | min_range: 2.0 38 | features: 39 | - intensity 40 | - label 41 | - eigenvalues 42 | - normals 43 | feature_dim: 44 | - 1 45 | - 1 46 | - 3 47 | - 3 48 | normal_eigenvalue_radius: 0.5 49 | evaluation: 50 | float16: True 51 | target_map: map 52 | iou_grid: 53 | resolution: 54 | - 0.2 55 | - 0.2 56 | - 0.1 57 | f_score_dist: 0.1 58 | out_dir: experiments/results/kitti/ 59 | network: 60 | encoder_blocks: 61 | - type: GridSampleConv 62 | number_blocks: 3 63 | parameters: 64 | in_fdim: 65 | - 1 66 | - 16 67 | - 32 68 | out_fdim: 69 | - 16 70 | - 32 71 | - 32 72 | num_kernel_points: 27 73 | max_nr_neighbors: 74 | - 70 75 | - 50 76 | - 25 77 | relu: true 78 | batchnorm: true 79 | deformable: false 80 | subsampling_dist: 1.0 81 | map_size: 40 82 | subsampling_factor: 83 | - 0.1 84 | - 0.5 85 | - 1.0 86 | kernel_radius: 1.0 87 | min_kernel_radius: 1.2 88 | use_dif_sampling: false 89 | - type: LinearLayer 90 | number_blocks: 1 91 | parameters: 92 | in_fdim: 32 93 | out_fdim: 3 94 | relu: false 95 | batchnorm: false 96 | decoder_blocks: 97 | - type: AdaptiveDeconv 98 | number_blocks: 4 99 | parameters: 100 | number_blocks: 4 101 | block_id: 102 | - 0 103 | - 1 104 | - 2 105 | - 3 106 | subsampling_dist: 1.0 107 | in_fdim: 108 | - 3 109 | - 32 110 | - 32 111 | - 32 112 | out_fdim: 32 113 | kernel_radius: 0.05 114 | relu: true 115 | use_batch_norm: false 116 | inter_fdim: 128 117 | estimate_radius: false 118 | subsampling_fct_p1: 0.006 119 | subsampling_fct_p2: 1.764 120 | - type: LinearLayer 121 | number_blocks: 1 122 | parameters: 123 | in_fdim: 32 124 | out_fdim: 3 125 | relu: false 126 | batchnorm: false 127 | out_dir: network_files/ 128 | dataset: 129 | data_folders: 130 | grid_output: /data/ 131 | prefix: /path/to/kitti/ 132 | train: 133 | - '00' 134 | - '01' 135 | - '02' 136 | - '03' 137 | - '04' 138 | - '05' 139 | - '06' 140 | - '07' 141 | - '09' 142 | - '10' 143 | valid: 144 | - validation 145 | test: 146 | - '08' 147 | git_commit_version: b'42d0bec' 148 | -------------------------------------------------------------------------------- /depoco/network_files/e3/enc.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e3/enc.pth -------------------------------------------------------------------------------- /depoco/network_files/e3/enc_best.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/network_files/e3/enc_best.pth -------------------------------------------------------------------------------- /depoco/notebooks/visualize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "unlimited-union", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from depoco.plot_results import *\n", 11 | "path = '../experiments/results/kitti/'\n", 12 | "files = sorted(glob.glob(path+'*.pkl'))\n", 13 | "\n", 14 | "f, ax = plt.subplots(2, 2)\n", 15 | "ax = ax.flatten()\n", 16 | "f.suptitle('Kitti Results')\n", 17 | "genPlots(files, f, ax, draw_line=True, label='our', x_key='bpp')\n", 18 | "for a in ax:\n", 19 | " a.grid()\n", 20 | " # a.set_ylim([0,None])\n", 21 | " a.legend()\n", 22 | "plt.tight_layout()\n", 23 | "plt.show()\n" 24 | ] 25 | } 26 | ], 27 | "metadata": { 28 | "kernelspec": { 29 | "display_name": "Python 3", 30 | "language": "python", 31 | "name": "python3" 32 | }, 33 | "language_info": { 34 | "codemirror_mode": { 35 | "name": "ipython", 36 | "version": 3 37 | }, 38 | "file_extension": ".py", 39 | "mimetype": "text/x-python", 40 | "name": "python", 41 | "nbconvert_exporter": "python", 42 | "pygments_lexer": "ipython3", 43 | "version": "3.8.5" 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 5 48 | } 49 | -------------------------------------------------------------------------------- /depoco/plot_results.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import glob 4 | import argparse 5 | from ruamel import yaml 6 | import depoco.utils.point_cloud_utils as pcu 7 | 8 | def plotResults(files, x_key, y_key, ax, draw_line=False, label=None, set_lim=True): 9 | x = [] 10 | y = [] 11 | for f in files: 12 | eval_dict = pcu.load_obj(f) 13 | if((x_key in eval_dict.keys()) & (y_key in eval_dict.keys())): 14 | for v in eval_dict.values(): 15 | v = np.array(v) 16 | if not draw_line: 17 | ax.plot(np.mean(eval_dict[x_key]), 18 | np.mean(eval_dict[y_key]), '.') 19 | ax.text(np.mean(eval_dict[x_key]), np.mean( 20 | eval_dict[y_key]), f.split('/')[-1][:-4]) 21 | 22 | x.append(np.mean(eval_dict[x_key])) 23 | y.append(np.mean(eval_dict[y_key])) 24 | 25 | if draw_line: 26 | line, = ax.plot(x, y, '-x', label=label) 27 | # line.set_label(label) 28 | 29 | ax.set_xlabel(x_key) 30 | ax.set_ylabel(y_key) 31 | 32 | if set_lim: 33 | ax.set_xlim(0,None) 34 | ax.set_ylim(0,None) 35 | # ax.grid() 36 | 37 | 38 | def genPlots(files, f, ax, draw_line=False, label=None, x_key='memory'): 39 | # print('shape',ax[0,0]) 40 | plotResults(files, x_key=x_key, y_key='chamfer_dist_abs', 41 | ax=ax[0], draw_line=draw_line, label=label) 42 | plotResults(files, x_key=x_key, y_key='chamfer_dist_plane', 43 | ax=ax[1], draw_line=draw_line, label=label) 44 | plotResults(files, x_key=x_key, y_key='iou', 45 | ax=ax[2], draw_line=draw_line, label=label) 46 | 47 | 48 | if __name__ == "__main__": 49 | ####### radius fct ############## 50 | path = 'experiments/results/kitti/' 51 | files = sorted(glob.glob(path+'*.pkl')) 52 | 53 | f, ax = plt.subplots(1, 3) 54 | f.suptitle('Radius FPS') 55 | genPlots(files, f, ax, draw_line=True, label='our', x_key='bpp') 56 | for a in ax: 57 | a.grid() 58 | # a.set_ylim([0,None]) 59 | a.legend() 60 | plt.show() 61 | -------------------------------------------------------------------------------- /depoco/trainer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import depoco.datasets.submap_handler as submap_handler 4 | import depoco.evaluation.evaluator as evaluator 5 | import torch 6 | import time 7 | import numpy as np 8 | import torch.nn as nn 9 | import torch.optim as optim 10 | 11 | import os 12 | from torch.utils.tensorboard import SummaryWriter 13 | from ruamel import yaml 14 | import argparse 15 | import depoco.architectures.network_blocks as network 16 | import chamfer3D.dist_chamfer_3D 17 | 18 | import depoco.utils.point_cloud_utils as pcu 19 | import subprocess 20 | 21 | # import depoco.utils.checkpoint as chkpt 22 | import depoco.architectures.loss_handler as loss_handler 23 | from tqdm.auto import trange, tqdm 24 | 25 | 26 | class DepocoNetTrainer(): 27 | def __init__(self, config): 28 | t_start = time.time() 29 | # parameters 30 | config['git_commit_version'] = str(subprocess.check_output( 31 | ['git', 'rev-parse', '--short', 'HEAD']).strip()) 32 | self.config = config 33 | self.experiment_id = self.config["train"]["experiment_id"] 34 | # self.submaps = submap_handler.SubmapHandler(self.config) 35 | t_sm = time.time() 36 | self.submaps = submap_handler.SubMapParser(config) 37 | print(f'Loaded Submaps ({time.time()-t_sm}s') 38 | 39 | self.max_nr_pts = self.config["train"]["max_nr_pts"] 40 | self.device = torch.device( 41 | "cuda:0" if torch.cuda.is_available() else "cpu") 42 | 43 | ### Load Encoder and Decoder #### 44 | t_model = time.time() 45 | self.encoder_model = None 46 | self.decoder_model = None 47 | self.getModel(config) 48 | print(f'Loaded Model ({time.time()-t_model}s)') 49 | 50 | ################################## 51 | ########## Loss Attributes ####### 52 | ################################## 53 | self.cham_loss = chamfer3D.dist_chamfer_3D.chamfer_3DDist() 54 | self.pairwise_dist = nn.PairwiseDistance() 55 | self.l2_loss = nn.MSELoss(reduction='mean') 56 | self.w_transf2map = self.config['train']['loss_weights']['transf2map'] 57 | self.w_map2transf = self.config['train']['loss_weights']['map2transf'] 58 | 59 | print(f'Init Trainer ({time.time()-t_start}s)') 60 | 61 | def getModel(self, config: dict): 62 | """Loads the model specified in self.config 63 | """ 64 | arch = self.config["network"] 65 | print(f"network architecture: {arch}") 66 | self.encoder_model = network.Network( 67 | config['network']['encoder_blocks']) 68 | self.decoder_model = network.Network( 69 | config['network']['decoder_blocks']) 70 | print(self.encoder_model) 71 | print(self.decoder_model) 72 | 73 | def loadModel(self, best: bool = True, device='cuda', out_dir=None): 74 | if out_dir is None: 75 | out_dir = self.config["network"]['out_dir'] 76 | enc_path = out_dir+self.experiment_id+'/enc' 77 | dec_path = out_dir+self.experiment_id+'/dec' 78 | enc_path += '_best.pth' if best else '.pth' 79 | dec_path += '_best.pth' if best else '.pth' 80 | print("load", enc_path, ",", dec_path) 81 | if(os.path.isfile(enc_path) and os.path.isfile(dec_path)): 82 | # if(os.path.isfile(model_file)): 83 | self.encoder_model.load_state_dict(torch.load( 84 | enc_path, map_location=lambda storage, loc: storage)) 85 | 86 | self.decoder_model.load_state_dict(torch.load( 87 | dec_path, map_location=lambda storage, loc: storage)) 88 | self.encoder_model.to(device) 89 | self.decoder_model.to(device) 90 | else: 91 | print(10*'!', 'Cannot load model', 10*'!') 92 | 93 | def saveModel(self, best: bool = False): 94 | out_dir = self.config["network"]['out_dir'] 95 | enc_path = out_dir+self.experiment_id+'/enc' 96 | dec_path = out_dir+self.experiment_id+'/dec' 97 | enc_path += '_best.pth' if best else '.pth' 98 | dec_path += '_best.pth' if best else '.pth' 99 | torch.save(self.encoder_model.state_dict(), enc_path) 100 | torch.save(self.decoder_model.state_dict(), dec_path) 101 | 102 | def saveYaml(self, out_dir="network_files/"): 103 | config_path = out_dir+self.experiment_id+'/'+self.experiment_id+'.yaml' 104 | if not os.path.exists(out_dir+self.experiment_id): 105 | os.makedirs(out_dir+self.experiment_id, exist_ok=True) 106 | with open(config_path, 'w') as f: 107 | saver = yaml.YAML() 108 | saver.dump(self.config, f) 109 | 110 | def test(self, best=True): 111 | # TEST 112 | return self.evaluate(self.submaps.getTestSet(), 113 | load_model=True, 114 | best_model=best, 115 | reference_points='map', 116 | compute_memory=True, evaluate=True) 117 | 118 | def getNetworkParams(self): 119 | return list(self.encoder_model.parameters()) + list(self.decoder_model.parameters()) 120 | 121 | def getScheduler(self, optimizer, len_data_loader, batch_size): 122 | number_epochs = self.config['train']['max_epochs'] 123 | steps_per_epoch = int(len_data_loader / batch_size) 124 | max_lr = self.config["train"]["optimizer"]["max_lr"] 125 | div_factor = max_lr / \ 126 | self.config["train"]["optimizer"]["start_lr"] 127 | final_div_factor = self.config["train"]["optimizer"]["start_lr"] / \ 128 | self.config["train"]["optimizer"]["end_lr"] 129 | pct_start = self.config["train"]['optimizer']["pct_incr_cycle"] 130 | anneal_strategy = self.config["train"]['optimizer']["anneal_strategy"] 131 | return optim.lr_scheduler.OneCycleLR(optimizer, max_lr=max_lr, steps_per_epoch=steps_per_epoch, epochs=number_epochs, pct_start=pct_start, anneal_strategy=anneal_strategy, div_factor=div_factor, final_div_factor=final_div_factor) 132 | 133 | def getLogWriter(self, logdir): 134 | if(os.path.isdir(logdir)): 135 | filelist = [f for f in os.listdir(logdir)] 136 | for f in filelist: 137 | os.remove(os.path.join(logdir, f)) 138 | return SummaryWriter(logdir) 139 | 140 | def train(self, verbose=True): 141 | ###### Setup ###### 142 | if self.config['train']['load_pretrained']: 143 | self.loadModel(best=True) 144 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 145 | self.encoder_model.to(self.device) 146 | self.decoder_model.to(self.device) 147 | number_epochs = self.config['train']['max_epochs'] 148 | output_path = self.config['network']['out_dir'] 149 | writer = self.getLogWriter( 150 | output_path+'log/'+self.experiment_id+"/") 151 | batch_size = min( 152 | (self.config["train"]["batch_size"], self.submaps.getTrainSize())) 153 | 154 | ###### Init optimizer ######## 155 | lr = self.config["train"]["optimizer"]["max_lr"] 156 | optimizer = optim.Adam(self.getNetworkParams(), lr=lr, amsgrad=False) 157 | scheduler = self.getScheduler( 158 | optimizer, self.submaps.getTrainSize(), batch_size) 159 | self.saveYaml(out_dir=output_path) 160 | 161 | time_start = time.time() 162 | optimizer.zero_grad() 163 | r_batch = 0 164 | best_loss = 1e10 165 | n_pct_time = time.time() 166 | 167 | validation_it = 0 168 | nr_batches = int(self.submaps.getTrainSize()/batch_size) 169 | #################################### 170 | ####### Start training ############# 171 | #################################### 172 | print('Start Training: #self.submaps: %d, #batches: %f' % 173 | (self.submaps.getTrainSize(), nr_batches)) 174 | for epoch in range(number_epochs): 175 | running_loss = 0 176 | batch = 0 177 | for i, input_dict in enumerate(self.submaps.getTrainSet()): 178 | if i >= (nr_batches * batch_size): # drop last batch 179 | continue 180 | ######## Preprocess ####### 181 | input_dict['points'] = input_dict['points'].to(self.device) 182 | input_dict['features'] = input_dict['features'].to(self.device) 183 | input_points = input_dict['points'] 184 | 185 | ####### Encoding and decoding ######### 186 | t1 = time.time() 187 | out_dict = self.encoder_model(input_dict.copy()) 188 | out_dict = self.decoder_model(out_dict) 189 | translation = out_dict['features'][:, :3] 190 | 191 | samples = out_dict['points'] 192 | samples_transf = samples+translation 193 | ##################### 194 | ###### Loss ######### 195 | ##################### 196 | loss = self.getTrainLoss(input_points, samples, translation) 197 | loss += loss_handler.linDeconvRegularizer( 198 | self.decoder_model, 199 | weight=self.config['train']['loss_weights']['upsampling_reg'], 200 | gt_points=input_points) 201 | 202 | # print(loss) 203 | loss.backward() 204 | running_loss += loss.item() 205 | ######################################## 206 | ######### Gradient Accumulation ######## 207 | ######################################## 208 | if ((i % batch_size) == (batch_size-1)): 209 | r_batch += 1 210 | batch += 1 211 | optimizer.step() 212 | 213 | optimizer.zero_grad() 214 | running_loss /= batch_size 215 | scheduler.step() 216 | 217 | curr_lr = scheduler.get_lr()[0] 218 | 219 | # Write Log 220 | writer.add_scalar('learning loss', 221 | running_loss, 222 | r_batch) 223 | writer.add_scalar('learning rate', 224 | curr_lr, 225 | r_batch) 226 | if verbose: 227 | print('[%d, %5d] loss: %.5f, time: %.1f lr: %.5f' % 228 | (epoch + 1, batch, running_loss, time.time()-time_start, curr_lr)) 229 | 230 | time_start = time.time() 231 | self.saveModel(best=False) 232 | 233 | running_loss = 0 234 | 235 | ############################# 236 | ##### validation ############ 237 | ############################# 238 | if (epoch % self.config['train']['validation']['report_rate']) is (self.config['train']['validation']['report_rate']-1): 239 | ts_val = time.time() 240 | valid_dict = self.evaluate( 241 | dataloader=self.submaps.getValidSet()) 242 | chamf_dist = valid_dict['reconstruction_error'] 243 | writer.add_scalar('v: chamfer distance', 244 | chamf_dist, 245 | r_batch) 246 | if chamf_dist < best_loss: 247 | self.saveModel(best=True) 248 | best_loss = chamf_dist 249 | if verbose: 250 | print('[%d, valid] rec_err: %.5f, time: %.1f' % 251 | (epoch + 1, chamf_dist, time.time()-ts_val)) 252 | 253 | validation_it += 1 254 | ########################################### 255 | ##### verbose every 10 percent ############ 256 | ########################################### 257 | if(pcu.isEveryNPercent((epoch), max_it=number_epochs, percent=10)): 258 | n_pct = (epoch+1)/number_epochs*100 259 | time_est = (time.time() - n_pct_time)/n_pct*(100-n_pct) 260 | print("%4d%s in %ds, estim. time left: %ds (%dmin), best loss: %.5f" % ( 261 | n_pct, "%", time.time() - n_pct_time, time_est, time_est/60, best_loss)) 262 | 263 | def getTrainLoss(self, gt_points: torch.tensor, samples, translations,): 264 | loss = torch.tensor( 265 | 0.0, dtype=torch.float32, device=self.device) # init loss 266 | samples_transf = samples + translations 267 | 268 | # Chamfer Loss between input and samples+T 269 | d_map2transf, d_transf2map, idx3, idx4 = self.cham_loss( 270 | gt_points.unsqueeze(0), samples_transf.unsqueeze(0)) 271 | loss += (self.w_map2transf * d_map2transf.mean() + 272 | self.w_transf2map * d_transf2map.mean()) 273 | return loss 274 | 275 | def evaluate(self, dataloader, 276 | load_model=False, 277 | best_model=False, 278 | reference_points='points', 279 | compute_memory=False, 280 | evaluate=False): 281 | loss_evaluator = evaluator.Evaluator(self.config) 282 | self.encoder_model.eval() 283 | self.decoder_model.eval() 284 | with torch.no_grad(): 285 | if load_model: 286 | self.loadModel(best=best_model) 287 | self.encoder_model.to(self.device) 288 | self.decoder_model.to(self.device) 289 | print('loaded best:', best_model) 290 | for i, input_dict in enumerate(tqdm(dataloader)): 291 | map_idx = input_dict['idx'] 292 | # print('map:', map_idx) 293 | scale = input_dict['scale'] 294 | input_dict['features'] = input_dict['features'].to(self.device) 295 | input_dict['points'] = input_dict['points'].to(self.device) 296 | 297 | ####### Cast to float16 if necessary ####### 298 | out_dict = self.encoder_model(input_dict.copy()) 299 | if self.config['evaluation']['float16']: 300 | out_dict['points'] = out_dict['points'].half().float() 301 | out_dict['features'] = out_dict['features'].half().float() 302 | ####### Compute Memory ####### 303 | if compute_memory: 304 | nbytes = 2 if self.config['evaluation']['float16'] else 4 305 | mem = (out_dict['points'].numel() + 306 | out_dict['features'].numel())*nbytes 307 | loss_evaluator.eval_results['memory'].append(mem) 308 | loss_evaluator.eval_results['bpp'].append( 309 | mem/input_dict['map'].shape[0]*8) 310 | ############# Decoder ################## 311 | out_dict = self.decoder_model(out_dict) 312 | translation = out_dict['features'][:, :3] 313 | samples = out_dict['points'] 314 | samples_transf = samples+translation 315 | 316 | ################################### 317 | gt_points = input_dict[reference_points].to(self.device) 318 | 319 | # Scale to metric space 320 | samples_transf *= scale 321 | samples *= scale 322 | translation *= scale 323 | gt_points *= scale 324 | 325 | reconstruction_error = loss_evaluator.chamferDist( 326 | gt_points=gt_points, source_points=samples_transf) 327 | loss_evaluator.eval_results['mapwise_reconstruction_error'].append( 328 | reconstruction_error.item()) 329 | 330 | if evaluate: # Full evaluation for testing 331 | feat_ind = np.cumsum( 332 | [0]+self.config['grid']['feature_dim']) 333 | normal_idx = pcu.findList( 334 | self.config['grid']['features'], value='normals') 335 | normal_idx = (feat_ind[normal_idx], feat_ind[normal_idx+1]) 336 | gt_normals = input_dict[reference_points + 337 | '_attributes'][:, normal_idx[0]:normal_idx[1]].cuda() 338 | loss_evaluator.evaluate( 339 | gt_points=gt_points, source_points=samples_transf, gt_normals=gt_normals) 340 | chamfer_dist = loss_evaluator.getRunningLoss() 341 | loss_evaluator.eval_results['reconstruction_error'] = chamfer_dist 342 | self.encoder_model.train() 343 | self.decoder_model.train() 344 | 345 | return loss_evaluator.eval_results 346 | 347 | def encodeDecode(self, input_dict, float_16=True): 348 | map_idx = input_dict['idx'] 349 | # print('map:', map_idx) 350 | scale = input_dict['scale'] 351 | input_dict['features'] = input_dict['features'].to(self.device) 352 | input_dict['points'] = input_dict['points'].to(self.device) 353 | 354 | ####### Cast to float_16 if necessary ####### 355 | out_dict = self.encoder_model(input_dict.copy()) 356 | if float_16: 357 | out_dict['points'] = out_dict['points'].half().float() 358 | out_dict['features'] = out_dict['features'].half().float() 359 | nr_emb = out_dict['points'].shape 360 | ############# Decoder ################## 361 | out_dict = self.decoder_model(out_dict) 362 | translation = out_dict['features'][:, :3] 363 | samples = out_dict['points'] 364 | samples_transf = samples+translation 365 | 366 | samples_transf *= scale 367 | return samples_transf, nr_emb 368 | 369 | 370 | if __name__ == "__main__": 371 | print('Hello') 372 | parser = argparse.ArgumentParser("./sample_net_trainer.py") 373 | parser.add_argument( 374 | '--config', '-cfg', 375 | type=str, 376 | required=False, 377 | default='config/depoco.yaml', 378 | help='configitecture yaml cfg file. See /config/config for example', 379 | ) 380 | FLAGS, unparsed = parser.parse_known_args() 381 | 382 | print('passed flags') 383 | config = yaml.safe_load(open(FLAGS.config, 'r')) 384 | print('loaded yaml flags') 385 | trainer = DepocoNetTrainer(config) 386 | print('initialized trainer') 387 | trainer.train(verbose=True) 388 | -------------------------------------------------------------------------------- /depoco/utils/point_cloud_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import open3d as o3d 3 | import torch 4 | import os 5 | import time 6 | import octree_handler 7 | import random 8 | import pickle 9 | 10 | 11 | def path(text): 12 | ''' 13 | adds "/" to the end if needed 14 | ''' 15 | if text is "": 16 | return "" 17 | if(text.endswith('/')): 18 | return text 19 | else: 20 | return text+"/" 21 | 22 | 23 | def isEveryNPercent(current_it: int, max_it: int, percent: float = 10): 24 | curr_percent = current_it/max_it*100 25 | n = int(curr_percent/percent) 26 | 27 | prev_percent = (current_it-1)/max_it*100 28 | return ((curr_percent >= n * percent) & (prev_percent < n * percent)) or (curr_percent >= 100) 29 | 30 | 31 | def visPointCloud(pcd, colors=None, normals=None, downsample=None, show_normals=False): 32 | pcd_o3 = o3d.geometry.PointCloud() 33 | pcd_o3.points = o3d.utility.Vector3dVector(pcd[:, 0:3]) 34 | if colors is not None: 35 | pcd_o3.colors = o3d.utility.Vector3dVector(colors) 36 | if normals is not None: 37 | pcd_o3.normals = o3d.utility.Vector3dVector(normals) 38 | if downsample is not None: 39 | pcd_o3 = pcd_o3.voxel_down_sample(downsample) 40 | o3d.visualization.draw_geometries([pcd_o3], point_show_normal=show_normals) 41 | 42 | 43 | def visPointClouds(pcd_list, colors_list=None): 44 | pcd_o3_list = [] 45 | for i, pcd in enumerate(pcd_list): 46 | pcd_o3 = o3d.geometry.PointCloud() 47 | pcd_o3.points = o3d.utility.Vector3dVector(pcd[:, 0:3]) 48 | if colors_list is not None: 49 | if len(colors_list) > i: 50 | colors = colors_list[i] 51 | if colors is not None: 52 | pcd_o3.colors = o3d.utility.Vector3dVector(colors) 53 | else: 54 | print('have colors') 55 | 56 | pcd_o3_list += [pcd_o3] 57 | print(f'pcd_o3_list len= {len(pcd_o3_list)}') 58 | o3d.visualization.draw_geometries(pcd_o3_list) 59 | 60 | # def visAll(input,) 61 | 62 | def randomSample(nr_samples, nr_points, seed =0): 63 | """Samples nr_samples indices. All valures in range of nr_points, no duplication 64 | 65 | Args: 66 | nr_samples ([type]): [description] 67 | nr_points ([type]): [description] 68 | seed (int, optional): [description]. Defaults to 0. 69 | """ 70 | subm_idx = np.arange(nr_points) 71 | np.random.seed(seed) 72 | np.random.shuffle(subm_idx) 73 | # print('shuffled idx',subm_idx) 74 | return subm_idx[0:min(nr_points, nr_samples)] 75 | 76 | def visVectorField(start, end, ref=None, colors=None): 77 | nr_p = start.shape[0] 78 | pcd_o3 = o3d.geometry.PointCloud() 79 | pcd_o3.points = o3d.utility.Vector3dVector(end[:, 0:3]) 80 | # o3d.visualization.draw_geometries([pcd_o3]) 81 | lines = np.concatenate((np.reshape(np.arange(nr_p), (-1, 1)), 82 | np.reshape(np.arange(nr_p)+nr_p, (-1, 1))), axis=1) 83 | points = np.concatenate((start, end), axis=0) 84 | line_set = o3d.geometry.LineSet(points=o3d.utility.Vector3dVector(points), 85 | lines=o3d.utility.Vector2iVector(lines)) 86 | if ref is None: 87 | o3d.visualization.draw_geometries([line_set, pcd_o3]) 88 | else: 89 | ref_o3 = o3d.geometry.PointCloud() 90 | ref_o3.points = o3d.utility.Vector3dVector(ref[:, 0:3]) 91 | if colors is not None: 92 | ref_o3.colors = o3d.utility.Vector3dVector(colors) 93 | o3d.visualization.draw_geometries([line_set, pcd_o3, ref_o3]) 94 | 95 | 96 | def renderVectorField(start, end, ref=None, colors=None, file_path='test.png'): 97 | nr_p = start.shape[0] 98 | pcd_o3 = o3d.geometry.PointCloud() 99 | pcd_o3.points = o3d.utility.Vector3dVector(end[:, 0:3]) 100 | # o3d.visualization.draw_geometries([pcd_o3]) 101 | lines = np.concatenate((np.reshape(np.arange(nr_p), (-1, 1)), 102 | np.reshape(np.arange(nr_p)+nr_p, (-1, 1))), axis=1) 103 | points = np.concatenate((start, end), axis=0) 104 | line_set = o3d.geometry.LineSet(points=o3d.utility.Vector3dVector(points), 105 | lines=o3d.utility.Vector2iVector(lines)) 106 | if ref is None: 107 | renderO3d([line_set, pcd_o3], file_path=file_path) 108 | else: 109 | ref_o3 = o3d.geometry.PointCloud() 110 | ref_o3.points = o3d.utility.Vector3dVector(ref[:, 0:3]) 111 | if colors is not None: 112 | ref_o3.colors = o3d.utility.Vector3dVector(colors) 113 | renderO3d([line_set, pcd_o3, ref_o3], file_path=file_path) 114 | 115 | 116 | def renderO3d(o3d_list, file_path='test.png'): 117 | vis = o3d.visualization.Visualizer() 118 | vis.create_window() 119 | for o3d_g in o3d_list: 120 | vis.add_geometry(o3d_g) 121 | # vis.add_geometry(o3d_list) 122 | # vis.update_geometry() 123 | 124 | vis.poll_events() 125 | vis.update_renderer() 126 | vis.capture_screen_image(file_path) 127 | # vis.run() 128 | vis.destroy_window() 129 | 130 | 131 | def renderCloud(pcd, colors, file_path='test.png'): 132 | pcd_o3 = o3d.geometry.PointCloud() 133 | pcd_o3.points = o3d.utility.Vector3dVector(pcd[:, 0:3]) 134 | if colors is not None: 135 | pcd_o3.colors = o3d.utility.Vector3dVector(colors) 136 | vis = o3d.visualization.Visualizer() 137 | vis.create_window() 138 | vis.add_geometry(pcd_o3) 139 | # vis.update_geometry() 140 | vis.poll_events() 141 | vis.update_renderer() 142 | vis.capture_screen_image(file_path) 143 | # vis.run() 144 | vis.destroy_window() 145 | 146 | 147 | def saveCloud2Binary(cld, file, out_path=None): 148 | if out_path is None: 149 | out_path = '' 150 | else: 151 | if not os.path.exists(out_path): 152 | os.makedirs(out_path) 153 | f = open(out_path+file, "wb") 154 | f.write(cld.astype('float32').T.tobytes()) 155 | f.close() 156 | 157 | 158 | def loadCloudFromBinary(file, cols=3): 159 | f = open(file, "rb") 160 | binary_data = f.read() 161 | f.close() 162 | temp = np.frombuffer( 163 | binary_data, dtype='float32', count=-1) 164 | data = np.reshape(temp, (cols, int(temp.size/cols))) 165 | return data.T 166 | 167 | 168 | def colorizeConv(in_pcl:np.ndarray, out_pcl:np.ndarray, kernel_radius, max_nr_neighbors,kernel_pos=None, kernel_points=None): 169 | octree = octree_handler.Octree() 170 | octree.setInput(in_pcl) 171 | out_index = random.randrange(out_pcl.shape[0]) 172 | in_index = octree.radiusSearchPoints( 173 | out_pcl[out_index:out_index+1, :], max_nr_neighbors, kernel_radius) 174 | in_index = in_index[in_index < in_pcl.shape[0]] 175 | in_clr = np.zeros_like(in_pcl) 176 | in_clr[in_index, :] = np.ones_like( 177 | in_pcl[in_index, :])*np.array([1, 0, 0]) 178 | 179 | out_clr = np.zeros_like(out_pcl) 180 | out_clr[out_index, :] = np.array([1, 0, 0]) 181 | 182 | if kernel_pos is not None: 183 | out_pt = out_pcl[out_index:out_index+1,:] 184 | k_in = kernel_pos[out_index,:,:]+out_pt 185 | k_clr = np.ones_like(k_in)*np.array([0, 1, 0]) 186 | print('kernel_i coords ',kernel_pos[out_index,:,:]) 187 | print('kernel_i coords ',kernel_points) 188 | print(f'in {in_pcl.shape},out {out_pcl.shape},kernel_def {kernel_pos.shape},kernel_def_i {k_in.shape},kernel {kernel_points.shape}',) 189 | in_pcl = np.vstack((in_pcl,k_in)) 190 | in_clr = np.vstack((in_clr,k_clr)) 191 | 192 | if kernel_points is not None: 193 | out_pt = out_pcl[out_index:out_index+1,:] 194 | k_in = kernel_points+out_pt 195 | k_clr = np.ones_like(k_in)*np.array([0, 0, 1]) 196 | in_pcl = np.vstack((in_pcl,k_in)) 197 | in_clr = np.vstack((in_clr,k_clr)) 198 | # visPointClouds([in_pcl,out_pcl+1],[in_clr,out_clr]) 199 | # if kernel_pos is not None: 200 | 201 | 202 | return (in_pcl, out_pcl), (in_clr, out_clr) 203 | 204 | # def colorizeDeformedKP(in_pcl, out_pcl, kernel_radius, max_nr_neighbors,kern_pcl=None): 205 | # octree = octree_handler.Octree() 206 | # octree.setInput(in_pcl) 207 | 208 | def visualizeConv(in_out_pts, in_out_clr): 209 | """[summary] 210 | 211 | Arguments: 212 | in_out_pts [list] -- [(in_pcl1,out_pcl1),(in_pcl2,out_pcl2),...] pointclouds 213 | in_out_clr [list] -- [(in_clr1,out_clr1),(in_clr2,out_clr2),...] colors 214 | """ 215 | extend = 1.2 216 | pts = [] 217 | clrs = [] 218 | n = len(in_out_pts) 219 | for i in range(n): 220 | in_pcl, out_pcl = in_out_pts[i] 221 | row = np.array([1, 0, 0])*extend * i 222 | col = np.array([0, -1, 0])*extend 223 | pts.append(in_pcl+row) 224 | pts.append(out_pcl+row+col) 225 | clrs.append(in_out_clr[i][0]) 226 | clrs.append(in_out_clr[i][1]) 227 | visPointClouds(pts, clrs) 228 | 229 | # https://stackoverflow.com/questions/19201290/how-to-save-a-dictionary-to-a-file/32216025 230 | 231 | 232 | def save_obj(obj, name): 233 | with open(name, 'wb') as f: 234 | pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 235 | 236 | 237 | def load_obj(name): 238 | with open(name, 'rb') as f: 239 | return pickle.load(f) 240 | 241 | 242 | def findList(inp_list, value): 243 | for i, item in enumerate(inp_list): 244 | # print( item,value) 245 | if item == value: 246 | return i 247 | 248 | 249 | if __name__ == "__main__": 250 | a = np.random.rand(12, 3) 251 | # saveCloud2Binary(a,'test.bin') 252 | # b = loadCloudFromBinary('test.bin') 253 | # print(a-b) 254 | # cld = loadCloudFromBinary('/media/lwiesmann/WiesmannIPB/data/data_kitti/dataset/submaps/04/000004.bin') 255 | # visPointCloud(cld) 256 | # start = np.array([[0, 0, 0], 257 | # [0, 1, 0], 258 | # [0, 0, 1]]) 259 | 260 | # end = np.array([[0, 0, 0], 261 | # [1, 0, 0], 262 | # [0, 0, 1]])+2 263 | # visVectorField(start, end) 264 | # renderCloud(a) 265 | -------------------------------------------------------------------------------- /depoco/utils/upsampling_rating.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import argparse 3 | 4 | def getScalingFactors(nr_in, nr_out, nr_layer): 5 | sf = (nr_out/nr_in)**(1/nr_layer) 6 | factors = nr_layer*[int(sf)] 7 | factors[-1]=round(nr_out/(np.prod(factors[:-1])*nr_in)) 8 | 9 | sampling_factor = np.prod(factors) 10 | print(f'factors {factors}, upsampling rate {sampling_factor}, nr output points {sampling_factor*nr_in}') 11 | 12 | 13 | 14 | if __name__ == "__main__": 15 | print('Hello') 16 | parser = argparse.ArgumentParser("./upsample_rating.py") 17 | # parser.add_argument( 18 | # '--dataset', '-d', 19 | # type=str, 20 | # required=False, 21 | # default="/mnt/91d100fa-d283-4eeb-b68c-e2b4b199d2de/wiesmann/data/data_kitti/dataset/" , 22 | # help='Dataset to train with. No Default', 23 | # ) 24 | parser.add_argument( 25 | '--input', '-i', 26 | type=int, 27 | required=True, 28 | default=120, 29 | help='Nr input points', 30 | ) 31 | parser.add_argument( 32 | '--output', '-o', 33 | type=int, 34 | required=True, 35 | default=100000, 36 | help='Nr output points', 37 | ) 38 | parser.add_argument( 39 | '--layer', '-l', 40 | type=int, 41 | required=False, 42 | default=4, 43 | help='Nr layer', 44 | ) 45 | FLAGS, unparsed = parser.parse_known_args() 46 | 47 | getScalingFactors(FLAGS.input,FLAGS.output,FLAGS.layer) -------------------------------------------------------------------------------- /depoco/utils/visualization/visualize_layer.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def remove_sequential(network): 4 | all_layers = [] 5 | for layer in network.children(): 6 | # if sequential layer, apply recursively to layers in sequential layer 7 | if isinstance(layer, nn.Sequential): 8 | all_layers += remove_sequential(layer) 9 | else: 10 | all_layers += [layer] 11 | 12 | return all_layers 13 | 14 | 15 | 16 | def visualizeSampling(self): 17 | t_gpu = time.time() 18 | print(20*'-') 19 | print('Visualize Deformation') 20 | self.loadModel(best=False) 21 | # torch.backends.cudnn.enabled = False 22 | self.encoder_model.to(self.device) 23 | self.decoder_model.to(self.device) 24 | 25 | print(f'Model gpu ({time.time()-t_gpu}s)') 26 | 27 | t_iter = time.time() 28 | input_dict = self.submaps.train_iter_ordered.next() 29 | print(f'Got train iter ({time.time()-t_iter}s)') 30 | input_dict['points'] = input_dict['points'].cuda() 31 | input_dict['features'] = input_dict['features'].cuda() 32 | input_dict['norm_range'] = input_dict['norm_range'].to( 33 | self.device) 34 | print(20*'#') 35 | 36 | all_layers = remove_sequential(self.encoder_model) 37 | all_layers += remove_sequential(self.decoder_model) 38 | # print(all_layers) 39 | pcl = [] 40 | clr = [] 41 | print(f'Time until start ({time.time()-t_gpu}s)') 42 | 43 | for m in all_layers: 44 | print(type(m)) 45 | in_pcl = input_dict['points'].detach().cpu().numpy() 46 | input_dict = m(input_dict) 47 | out_pcl = input_dict['points'].detach().cpu().numpy() 48 | 49 | if(type(m) in [network.SampleKPConvBlock, network.InterKPConvBlock, network.RandomSampleKPConv, network.ResnetConv, network.FPSResnetConv, network.FPSSamplingConv, network.GridSampleConv]): 50 | print(input_dict['points'].shape, input_dict['features'].shape) 51 | print('kernel_radius', m.kernel_radius) 52 | pts, clrs = pcu.colorizeConv( 53 | in_pcl, out_pcl, m.kernel_radius, m.max_nr_neighbors, kernel_points=m.kp_conv.kernel_points.cpu()) 54 | pcl.append(pts) 55 | clr.append(clrs) 56 | if(type(m) in [network.LinearDeconv, network.AdaptiveDeconv]): 57 | print(input_dict['points'].shape, input_dict['features'].shape) 58 | print('kernel_radius', m.kernel_radius) 59 | pts, clrs = pcu.colorizeConv( 60 | in_pcl, out_pcl, m.kernel_radius, max_nr_neighbors=m.upsampling_rate) 61 | pcl.append(pts) 62 | clr.append(clrs) 63 | pcu.visualizeConv(pcl, clr) 64 | 65 | def visualizeDeformation(self): 66 | print(20*'-') 67 | print('Visualize Deformation') 68 | self.loadModel(best=True) 69 | # torch.backends.cudnn.enabled = False 70 | self.encoder_model.to(self.device) 71 | self.decoder_model.to(self.device) 72 | 73 | input_dict = self.submaps.train_iter_ordered.next() 74 | input_dict['points'] = input_dict['points'].cuda() 75 | input_dict['norm_range'] = input_dict['norm_range'].to( 76 | self.device) 77 | 78 | 79 | all_layers = remove_sequential(self.encoder_model) 80 | all_layers += remove_sequential(self.decoder_model) 81 | # print(all_layers) 82 | pcl = [] 83 | clr = [] 84 | for m in all_layers: 85 | print(type(m)) 86 | in_pcl = input_dict['points'].detach().cpu().numpy() 87 | input_dict = m(input_dict) 88 | out_pcl = input_dict['points'].detach().cpu().numpy() 89 | features = input_dict['features'].cpu() 90 | 91 | if isinstance(m, network.OriginalKpConv) and m.kp_conv.deformable: 92 | print(f'features min {features.min()}, max {features.max()}') 93 | print(input_dict['points'].shape, input_dict['features'].shape) 94 | print('kernel_radius', m.kernel_radius) 95 | 96 | print('deformed kp:', m.kp_conv.deformed_KP.shape) 97 | pts, clrs = pcu.colorizeConv( 98 | in_pcl, out_pcl, m.kernel_radius, m.max_nr_neighbors, kernel_pos=m.kp_conv.deformed_KP.detach().cpu().numpy(), kernel_points=m.kp_conv.kernel_points.detach().cpu().numpy()) 99 | pcl.append(pts) 100 | clr.append(clrs) 101 | pcu.visualizeConv(pcl, clr) -------------------------------------------------------------------------------- /depoco/utils/visualization/visualize_pointcloud.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRBonn/deep-point-map-compression/b2e35bb05e70ae28b159c2c602bc187414173c06/depoco/utils/visualization/visualize_pointcloud.py -------------------------------------------------------------------------------- /depoco/visualize.py: -------------------------------------------------------------------------------- 1 | import depoco.utils.point_cloud_utils as pcu 2 | import argparse 3 | import ruamel.yaml as yaml 4 | from depoco.trainer import DepocoNetTrainer 5 | import torch 6 | 7 | if __name__ == "__main__": 8 | print('Hello') 9 | parser = argparse.ArgumentParser("./sample_net_trainer.py") 10 | parser.add_argument( 11 | '--config', '-cfg', 12 | type=str, 13 | required=False, 14 | default='config/depoco.yaml', 15 | help='configitecture yaml cfg file. See /config/config for example', 16 | ) 17 | parser.add_argument( 18 | '--number', '-n', 19 | type=int, 20 | default=5, 21 | help='Number of maps to visualize', 22 | ) 23 | FLAGS, unparsed = parser.parse_known_args() 24 | 25 | print('passed flags') 26 | config = yaml.safe_load(open(FLAGS.config, 'r')) 27 | print('loaded yaml flags') 28 | trainer = DepocoNetTrainer(config) 29 | trainer.loadModel(best=False) 30 | print('initialized trainer') 31 | for i, batch in enumerate(trainer.submaps.getOrderedTrainSet()): 32 | with torch.no_grad(): 33 | points_est,nr_emb_points = trainer.encodeDecode(batch) 34 | print( 35 | f'nr embedding points: {nr_emb_points}, points out: {points_est.shape[0]}') 36 | pcu.visPointCloud(points_est.detach().cpu().numpy()) 37 | if i+1 >= FLAGS.number: 38 | break 39 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | open3d 2 | torch 3 | ruamel.yaml -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ### To not hack your paths 2 | # install via `pip3 install -U -e .` in this dir 3 | #https://stackoverflow.com/questions/6323860/sibling-package-imports/50193944#50193944 4 | 5 | from setuptools import setup, find_packages 6 | pkg = 'depoco' 7 | setup(name=pkg, version='1.0', packages=find_packages(include=(pkg+"*",))) -------------------------------------------------------------------------------- /submodules/octree_handler/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: Google 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveMacros: false 7 | AlignConsecutiveAssignments: false 8 | AlignConsecutiveDeclarations: false 9 | AlignEscapedNewlines: Left 10 | AlignOperands: true 11 | AlignTrailingComments: true 12 | AllowAllArgumentsOnNextLine: true 13 | AllowAllConstructorInitializersOnNextLine: true 14 | AllowAllParametersOfDeclarationOnNextLine: true 15 | AllowShortBlocksOnASingleLine: Never 16 | AllowShortCaseLabelsOnASingleLine: false 17 | AllowShortFunctionsOnASingleLine: All 18 | AllowShortLambdasOnASingleLine: All 19 | AllowShortIfStatementsOnASingleLine: WithoutElse 20 | AllowShortLoopsOnASingleLine: true 21 | AlwaysBreakAfterDefinitionReturnType: None 22 | AlwaysBreakAfterReturnType: None 23 | AlwaysBreakBeforeMultilineStrings: true 24 | AlwaysBreakTemplateDeclarations: Yes 25 | BinPackArguments: false 26 | BinPackParameters: false 27 | BraceWrapping: 28 | AfterCaseLabel: false 29 | AfterClass: false 30 | AfterControlStatement: false 31 | AfterEnum: false 32 | AfterFunction: false 33 | AfterNamespace: false 34 | AfterObjCDeclaration: false 35 | AfterStruct: false 36 | AfterUnion: false 37 | AfterExternBlock: false 38 | BeforeCatch: false 39 | BeforeElse: false 40 | IndentBraces: false 41 | SplitEmptyFunction: true 42 | SplitEmptyRecord: true 43 | SplitEmptyNamespace: true 44 | BreakBeforeBinaryOperators: None 45 | BreakBeforeBraces: Attach 46 | BreakBeforeInheritanceComma: false 47 | BreakInheritanceList: BeforeColon 48 | BreakBeforeTernaryOperators: true 49 | BreakConstructorInitializersBeforeComma: false 50 | BreakConstructorInitializers: BeforeColon 51 | BreakAfterJavaFieldAnnotations: false 52 | BreakStringLiterals: true 53 | ColumnLimit: 80 54 | CommentPragmas: '^ IWYU pragma:' 55 | CompactNamespaces: false 56 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 57 | ConstructorInitializerIndentWidth: 4 58 | ContinuationIndentWidth: 4 59 | Cpp11BracedListStyle: true 60 | DeriveLineEnding: true 61 | DerivePointerAlignment: true 62 | DisableFormat: false 63 | ExperimentalAutoDetectBinPacking: false 64 | FixNamespaceComments: true 65 | ForEachMacros: 66 | - foreach 67 | - Q_FOREACH 68 | - BOOST_FOREACH 69 | IncludeBlocks: Regroup 70 | IncludeCategories: 71 | - Regex: '^<.*\.h>' 72 | Priority: 1 73 | - Regex: '^<.*\.hpp>' 74 | Priority: 3 75 | - Regex: '^<.*' 76 | Priority: 2 77 | - Regex: '.*' 78 | Priority: 4 79 | IncludeIsMainRegex: '([-_](test|unittest))?$' 80 | IncludeIsMainSourceRegex: '' 81 | IndentCaseLabels: true 82 | IndentGotoLabels: true 83 | IndentPPDirectives: None 84 | IndentWidth: 2 85 | IndentWrappedFunctionNames: false 86 | JavaScriptQuotes: Leave 87 | JavaScriptWrapImports: true 88 | KeepEmptyLinesAtTheStartOfBlocks: false 89 | MacroBlockBegin: '' 90 | MacroBlockEnd: '' 91 | MaxEmptyLinesToKeep: 1 92 | NamespaceIndentation: None 93 | ObjCBinPackProtocolList: Never 94 | ObjCBlockIndentWidth: 2 95 | ObjCSpaceAfterProperty: false 96 | ObjCSpaceBeforeProtocolList: true 97 | PenaltyBreakAssignment: 2 98 | PenaltyBreakBeforeFirstCallParameter: 1 99 | PenaltyBreakComment: 300 100 | PenaltyBreakFirstLessLess: 120 101 | PenaltyBreakString: 1000 102 | PenaltyBreakTemplateDeclaration: 10 103 | PenaltyExcessCharacter: 1000000 104 | PenaltyReturnTypeOnItsOwnLine: 200 105 | PointerAlignment: Left 106 | RawStringFormats: 107 | - Language: Cpp 108 | Delimiters: 109 | - cc 110 | - CC 111 | - cpp 112 | - Cpp 113 | - CPP 114 | - 'c++' 115 | - 'C++' 116 | CanonicalDelimiter: '' 117 | BasedOnStyle: google 118 | - Language: TextProto 119 | Delimiters: 120 | - pb 121 | - PB 122 | - proto 123 | - PROTO 124 | EnclosingFunctions: 125 | - EqualsProto 126 | - EquivToProto 127 | - PARSE_PARTIAL_TEXT_PROTO 128 | - PARSE_TEST_PROTO 129 | - PARSE_TEXT_PROTO 130 | - ParseTextOrDie 131 | - ParseTextProtoOrDie 132 | CanonicalDelimiter: '' 133 | BasedOnStyle: google 134 | ReflowComments: true 135 | SortIncludes: true 136 | SortUsingDeclarations: true 137 | SpaceAfterCStyleCast: false 138 | SpaceAfterLogicalNot: false 139 | SpaceAfterTemplateKeyword: true 140 | SpaceBeforeAssignmentOperators: true 141 | SpaceBeforeCpp11BracedList: false 142 | SpaceBeforeCtorInitializerColon: true 143 | SpaceBeforeInheritanceColon: true 144 | SpaceBeforeParens: ControlStatements 145 | SpaceBeforeRangeBasedForLoopColon: true 146 | SpaceInEmptyBlock: false 147 | SpaceInEmptyParentheses: false 148 | SpacesBeforeTrailingComments: 2 149 | SpacesInAngles: false 150 | SpacesInConditionalStatement: false 151 | SpacesInContainerLiterals: true 152 | SpacesInCStyleCastParentheses: false 153 | SpacesInParentheses: false 154 | SpacesInSquareBrackets: false 155 | SpaceBeforeSquareBrackets: false 156 | Standard: Auto 157 | StatementMacros: 158 | - Q_UNUSED 159 | - QT_REQUIRE_VERSION 160 | TabWidth: 8 161 | UseCRLF: false 162 | UseTab: Never 163 | ... 164 | 165 | -------------------------------------------------------------------------------- /submodules/octree_handler/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/python,cmake,c++ 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,cmake,c++ 4 | 5 | ### C++ ### 6 | # Prerequisites 7 | *.d 8 | 9 | # Compiled Object files 10 | *.slo 11 | *.lo 12 | *.o 13 | *.obj 14 | 15 | # Precompiled Headers 16 | *.gch 17 | *.pch 18 | 19 | # Compiled Dynamic libraries 20 | *.so 21 | *.dylib 22 | *.dll 23 | 24 | # Fortran module files 25 | *.mod 26 | *.smod 27 | 28 | # Compiled Static libraries 29 | *.lai 30 | *.la 31 | *.a 32 | *.lib 33 | 34 | # Executables 35 | *.exe 36 | *.out 37 | *.app 38 | 39 | ### CMake ### 40 | CMakeLists.txt.user 41 | CMakeCache.txt 42 | CMakeFiles 43 | CMakeScripts 44 | Testing 45 | Makefile 46 | cmake_install.cmake 47 | install_manifest.txt 48 | compile_commands.json 49 | CTestTestfile.cmake 50 | _deps 51 | 52 | ### CMake Patch ### 53 | # External projects 54 | *-prefix/ 55 | 56 | ### Python ### 57 | # Byte-compiled / optimized / DLL files 58 | __pycache__/ 59 | *.py[cod] 60 | *$py.class 61 | 62 | # C extensions 63 | 64 | # Distribution / packaging 65 | .Python 66 | build/ 67 | develop-eggs/ 68 | dist/ 69 | downloads/ 70 | eggs/ 71 | .eggs/ 72 | lib/ 73 | lib64/ 74 | parts/ 75 | sdist/ 76 | var/ 77 | wheels/ 78 | pip-wheel-metadata/ 79 | share/python-wheels/ 80 | *.egg-info/ 81 | .installed.cfg 82 | *.egg 83 | MANIFEST 84 | 85 | # PyInstaller 86 | # Usually these files are written by a python script from a template 87 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 88 | *.manifest 89 | *.spec 90 | 91 | # Installer logs 92 | pip-log.txt 93 | pip-delete-this-directory.txt 94 | 95 | # Unit test / coverage reports 96 | htmlcov/ 97 | .tox/ 98 | .nox/ 99 | .coverage 100 | .coverage.* 101 | .cache 102 | nosetests.xml 103 | coverage.xml 104 | *.cover 105 | *.py,cover 106 | .hypothesis/ 107 | .pytest_cache/ 108 | pytestdebug.log 109 | 110 | # Translations 111 | *.mo 112 | *.pot 113 | 114 | # Django stuff: 115 | *.log 116 | local_settings.py 117 | db.sqlite3 118 | db.sqlite3-journal 119 | 120 | # Flask stuff: 121 | instance/ 122 | .webassets-cache 123 | 124 | # Scrapy stuff: 125 | .scrapy 126 | 127 | # Sphinx documentation 128 | docs/_build/ 129 | doc/_build/ 130 | 131 | # PyBuilder 132 | target/ 133 | 134 | # Jupyter Notebook 135 | .ipynb_checkpoints 136 | 137 | # IPython 138 | profile_default/ 139 | ipython_config.py 140 | 141 | # pyenv 142 | .python-version 143 | 144 | # pipenv 145 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 146 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 147 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 148 | # install all needed dependencies. 149 | #Pipfile.lock 150 | 151 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 152 | __pypackages__/ 153 | 154 | # Celery stuff 155 | celerybeat-schedule 156 | celerybeat.pid 157 | 158 | # SageMath parsed files 159 | *.sage.py 160 | 161 | # Environments 162 | .env 163 | .venv 164 | env/ 165 | venv/ 166 | ENV/ 167 | env.bak/ 168 | venv.bak/ 169 | pythonenv* 170 | 171 | # Spyder project settings 172 | .spyderproject 173 | .spyproject 174 | 175 | # Rope project settings 176 | .ropeproject 177 | 178 | # mkdocs documentation 179 | /site 180 | 181 | # mypy 182 | .mypy_cache/ 183 | .dmypy.json 184 | dmypy.json 185 | 186 | # Pyre type checker 187 | .pyre/ 188 | 189 | # pytype static type analyzer 190 | .pytype/ 191 | 192 | # profiling data 193 | .prof 194 | 195 | # End of https://www.toptal.com/developers/gitignore/api/python,cmake,c++ 196 | -------------------------------------------------------------------------------- /submodules/octree_handler/.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: gitlab.ipb.uni-bonn.de:4567/ipb-team/global/docker-images/ipb_default:latest 2 | 3 | run: 4 | script: 5 | - apt update && apt install -yqq pybind11-dev 6 | - pip3 install twine 7 | - python3 setup.py sdist bdist_wheel 8 | - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.ipb.uni-bonn.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/* 9 | -------------------------------------------------------------------------------- /submodules/octree_handler/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.2.0) 2 | project(octree_handler) 3 | 4 | find_package(pybind11 REQUIRED) 5 | find_package(Eigen3 REQUIRED) 6 | 7 | # Default to release if not specified 8 | if(NOT CMAKE_BUILD_TYPE) 9 | set(CMAKE_BUILD_TYPE Release) 10 | endif() 11 | 12 | # Set aditional flags 13 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 14 | set(CMAKE_CXX_STANDARD 17) 15 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra") 16 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3 -O0") 17 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g0 -O3") 18 | 19 | include_directories(${EIGEN3_INCLUDE_DIR}) 20 | add_subdirectory(src) 21 | 22 | -------------------------------------------------------------------------------- /submodules/octree_handler/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import platform 5 | import subprocess 6 | 7 | from setuptools import setup, Extension 8 | from setuptools.command.build_ext import build_ext 9 | from distutils.version import LooseVersion 10 | 11 | 12 | class CMakeExtension(Extension): 13 | def __init__(self, name, sourcedir=''): 14 | Extension.__init__(self, name, sources=[]) 15 | self.sourcedir = os.path.abspath(sourcedir) 16 | 17 | 18 | class CMakeBuild(build_ext): 19 | def run(self): 20 | try: 21 | out = subprocess.check_output(['cmake', '--version']) 22 | except OSError: 23 | raise RuntimeError("CMake must be installed to build the following extensions: " + 24 | ", ".join(e.name for e in self.extensions)) 25 | 26 | if platform.system() == "Windows": 27 | cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1)) 28 | if cmake_version < '3.1.0': 29 | raise RuntimeError("CMake >= 3.1.0 is required on Windows") 30 | 31 | for ext in self.extensions: 32 | self.build_extension(ext) 33 | 34 | def build_extension(self, ext): 35 | extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) 36 | # required for auto-detection of auxiliary "native" libs 37 | if not extdir.endswith(os.path.sep): 38 | extdir += os.path.sep 39 | 40 | cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, 41 | '-DPYTHON_EXECUTABLE=' + sys.executable] 42 | 43 | cfg = 'Debug' if self.debug else 'Release' 44 | build_args = ['--config', cfg] 45 | 46 | if platform.system() == "Windows": 47 | cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] 48 | if sys.maxsize > 2**32: 49 | cmake_args += ['-A', 'x64'] 50 | build_args += ['--', '/m'] 51 | else: 52 | cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] 53 | build_args += ['--', '-j2'] 54 | 55 | env = os.environ.copy() 56 | env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), 57 | self.distribution.get_version()) 58 | if not os.path.exists(self.build_temp): 59 | os.makedirs(self.build_temp) 60 | subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) 61 | subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) 62 | 63 | setup( 64 | name='octree_handler', 65 | version='0.0.1', 66 | author='Louis Wiesmann', 67 | author_email='lwiesmann@uni-bonn.com', 68 | description='A python wrapper for an Octree (todo)', 69 | long_description='', 70 | ext_modules=[CMakeExtension('octree_handler')], 71 | cmdclass=dict(build_ext=CMakeBuild), 72 | zip_safe=False, 73 | ) 74 | -------------------------------------------------------------------------------- /submodules/octree_handler/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | pybind11_add_module(octree_handler OctreeHandler.cpp) 2 | -------------------------------------------------------------------------------- /submodules/octree_handler/src/OctreeHandler.cpp: -------------------------------------------------------------------------------- 1 | #include "OctreeHandler.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace py = pybind11; 12 | 13 | void Octree::setInput(Eigen::MatrixXf &cloud) { 14 | points_.clear(); 15 | points_.reserve(cloud.rows()); 16 | for (uint32_t i = 0; i < cloud.rows(); i++) { 17 | points_.emplace_back( 18 | Eigen::Vector3f(cloud(i, 0), cloud(i, 1), cloud(i, 2))); 19 | } 20 | octree_.initialize(points_); 21 | } 22 | 23 | std::vector Octree::radiusSearch(const Eigen::Vector3f &pt, 24 | const float &radius) { 25 | std::vector results; 26 | octree_.radiusNeighbors>( 27 | pt, radius, results); 28 | return results; 29 | } 30 | 31 | std::vector Octree::radiusSearch(const uint32_t &pt_idx, 32 | const float &radius) { 33 | assert(pt_idx < points_.size()); 34 | std::vector results; 35 | const Eigen::Vector3f &q = points_[pt_idx]; 36 | octree_.radiusNeighbors>( 37 | q, radius, results); 38 | return results; 39 | } 40 | 41 | Eigen::MatrixXi Octree::radiusSearchAll(const uint32_t &max_nr_neighbors, 42 | const float &radius) { 43 | uint32_t act_max_neighbors = 0; 44 | Eigen::MatrixXi indices = 45 | Eigen::MatrixXi::Ones(points_.size(), max_nr_neighbors) * 46 | (int)points_.size(); 47 | for (uint32_t i = 0; i < points_.size(); i++) { 48 | std::vector results = radiusSearch(i, radius); 49 | uint32_t nr_n = std::min(max_nr_neighbors, (uint32_t)results.size()); 50 | if (nr_n > act_max_neighbors) act_max_neighbors = nr_n; 51 | for (uint32_t j = 0; j < nr_n; j++) { 52 | indices(i, j) = results[j]; 53 | } 54 | } 55 | return indices.leftCols(act_max_neighbors); 56 | } 57 | 58 | Eigen::MatrixXi Octree::radiusSearchIndices( 59 | const std::vector &pt_indices, 60 | const uint32_t &max_nr_neighbors, 61 | const float &radius) { 62 | uint32_t act_max_neighbors = 0; 63 | Eigen::MatrixXi indices = 64 | Eigen::MatrixXi::Ones(pt_indices.size(), max_nr_neighbors) * 65 | (int)points_.size(); 66 | for (uint32_t i = 0; i < pt_indices.size(); i++) { 67 | std::vector results = radiusSearch(pt_indices[i], radius); 68 | uint32_t nr_n = std::min(max_nr_neighbors, (uint32_t)results.size()); 69 | if (nr_n > act_max_neighbors) act_max_neighbors = nr_n; 70 | for (uint32_t j = 0; j < nr_n; j++) { 71 | uint32_t step = 72 | std::max((uint32_t)(results.size() * j / max_nr_neighbors), j); 73 | assert(step < results.size()); 74 | indices(i, j) = results[step]; 75 | } 76 | } 77 | return indices.leftCols(act_max_neighbors); 78 | } 79 | 80 | Eigen::MatrixXi Octree::radiusSearchPoints(Eigen::MatrixXf &query_points, 81 | const uint32_t &max_nr_neighbors, 82 | const float &radius) { 83 | assert(query_points.cols() == 3); 84 | uint32_t act_max_neighbors = 0; 85 | Eigen::MatrixXi indices = 86 | Eigen::MatrixXi::Ones(query_points.rows(), max_nr_neighbors) * 87 | (int)points_.size(); 88 | for (uint32_t i = 0; i < query_points.rows(); i++) { 89 | std::vector results = radiusSearch(query_points.row(i), radius); 90 | uint32_t nr_n = std::min(max_nr_neighbors, (uint32_t)results.size()); 91 | if (nr_n > act_max_neighbors) act_max_neighbors = nr_n; 92 | for (uint32_t j = 0; j < nr_n; j++) { 93 | uint32_t step = 94 | std::max((uint32_t)(results.size() * j / max_nr_neighbors), j); 95 | assert(step < results.size()); 96 | assert(results[step] <= points_.size()); 97 | // std::cout< Octree::randomUniformSampling(const float &max_dist) { 105 | std::vector indices; 106 | if (max_dist > 0) { 107 | octree_.randomUniformSampling(max_dist, indices); 108 | } else { 109 | indices = std::vector(points_.size()); // return all points 110 | std::iota(std::begin(indices), std::end(indices), 0); 111 | } 112 | return indices; 113 | } 114 | Eigen::MatrixXf Octree::computeScatterMatrices(const float &radius) { 115 | uint32_t n = points_.size(); 116 | Eigen::MatrixXf scatter_matrices = Eigen::MatrixXf(n, 6); 117 | for (uint32_t i = 0; i < n; i++) { 118 | std::vector neighbors_idx = radiusSearch(i, radius); 119 | uint32_t nr_neighbors = neighbors_idx.size(); 120 | Eigen::Matrix3Xf neighbors = Eigen::Matrix3Xf(3, nr_neighbors); 121 | for (uint32_t j = 0; j < nr_neighbors; j++) { 122 | neighbors.col(j) = points_[neighbors_idx[j]]; 123 | } 124 | // https://stackoverflow.com/questions/15138634/eigen-is-there-an-inbuilt-way-to-calculate-sample-covariance 125 | Eigen::Matrix3Xf centered = 126 | (neighbors.colwise() - neighbors.rowwise().mean()) / radius; 127 | Eigen::Matrix3f cov = 128 | (centered * centered.transpose()) / float(nr_neighbors - 1); 129 | Eigen::Matrix flat_cov; 130 | flat_cov << cov(0, 0), cov(1, 1), cov(2, 2), cov(0, 1), cov(0, 2), 131 | cov(1, 2); 132 | scatter_matrices.row(i) = flat_cov; 133 | } 134 | return scatter_matrices; 135 | } 136 | 137 | Eigen::MatrixXf Octree::spectralFeaturesAll(const float &radius) { 138 | uint32_t n = points_.size(); 139 | Eigen::MatrixXf eigenvalues = Eigen::MatrixXf(n, 3); 140 | for (uint32_t i = 0; i < n; i++) { 141 | std::vector neighbors_idx = radiusSearch(i, radius); 142 | uint32_t nr_neighbors = neighbors_idx.size(); 143 | Eigen::Matrix3Xf neighbors = Eigen::Matrix3Xf(3, nr_neighbors); 144 | for (uint32_t j = 0; j < nr_neighbors; j++) { 145 | neighbors.col(j) = points_[neighbors_idx[j]]; 146 | } 147 | // https://stackoverflow.com/questions/15138634/eigen-is-there-an-inbuilt-way-to-calculate-sample-covariance 148 | Eigen::Matrix3Xf centered = 149 | (neighbors.colwise() - neighbors.rowwise().mean()) / radius; 150 | Eigen::Matrix3f cov = 151 | (centered * centered.transpose()) / float(nr_neighbors - 1); 152 | Eigen::Vector3f eigenv = cov.eigenvalues().real(); 153 | std::sort( 154 | eigenv.data(), eigenv.data() + eigenv.size(), std::greater()); 155 | eigenvalues.row(i) = eigenv.transpose(); 156 | } 157 | return eigenvalues; 158 | } 159 | Eigen::MatrixXf Octree::computeEigenvaluesNormal(const float &radius) { 160 | uint32_t n = points_.size(); 161 | Eigen::MatrixXf eigenvalues = Eigen::MatrixXf(n, 6); 162 | for (uint32_t i = 0; i < n; i++) { 163 | std::vector neighbors_idx = radiusSearch(i, radius); 164 | uint32_t nr_neighbors = neighbors_idx.size(); 165 | Eigen::Matrix3Xf neighbors = Eigen::Matrix3Xf(3, nr_neighbors); 166 | for (uint32_t j = 0; j < nr_neighbors; j++) { 167 | neighbors.col(j) = points_[neighbors_idx[j]]; 168 | } 169 | // https://stackoverflow.com/questions/15138634/eigen-is-there-an-inbuilt-way-to-calculate-sample-covariance 170 | Eigen::Matrix3Xf centered = 171 | (neighbors.colwise() - neighbors.rowwise().mean()) / radius; 172 | Eigen::Matrix3f cov = 173 | (centered * centered.transpose()) / float(nr_neighbors - 1); 174 | Eigen::JacobiSVD svd(cov, Eigen::ComputeFullU); 175 | Eigen::Matrix3f eigenvectors = svd.matrixU(); 176 | // last column is the eigenvector of the smalles singular value 177 | Eigen::Vector3f normal = eigenvectors.col(2); 178 | normal.normalize(); 179 | Eigen::Vector3f singularv = svd.singularValues(); 180 | if (i < 1) { 181 | std::cout << "sing: " << singularv 182 | << "sqrt sing: " << singularv.array().sqrt().real() 183 | << "pow sing: " << singularv.array().pow(2) << "eigen " 184 | << cov.eigenvalues().real() << std::endl; 185 | } 186 | eigenvalues.row(i).leftCols(3) = singularv.transpose(); 187 | eigenvalues.row(i).rightCols(3) = normal.transpose(); 188 | } 189 | return eigenvalues; 190 | } 191 | 192 | // if fatal error: Python.h: No such file or directory 193 | // https://github.com/stevenlovegrove/Pangolin/issues/494 194 | // CPLUS_INCLUDE_PATH=/usr/include/python3.6 195 | // export CPLUS_INCLUDE_PATH 196 | PYBIND11_MODULE(octree_handler, m) { 197 | m.doc() = "pybind11 octree plugin"; // optional module docstring 198 | 199 | m.def("scale", 200 | [](pybind11::EigenDRef m, double c) { m *= c; }); 201 | py::class_(m, "Octree") 202 | .def(py::init()) 203 | .def("setInput", 204 | &Octree::setInput, 205 | "builds octree based on the input cloud", 206 | py::arg("points")) 207 | .def("radiusSearch", 208 | py::overload_cast( 209 | &Octree::radiusSearch)) 210 | .def("radiusSearchAll", &Octree::radiusSearchAll) 211 | .def("radiusSearchIndices", &Octree::radiusSearchIndices) 212 | .def("radiusSearchPoints", &Octree::radiusSearchPoints) 213 | .def("spectralFeatureAll", &Octree::spectralFeaturesAll) 214 | .def("computeEigenvaluesNormal", &Octree::computeEigenvaluesNormal) 215 | .def("computeScatterMatrices", &Octree::computeScatterMatrices) 216 | .def("randomUniformSampling", &Octree::randomUniformSampling); 217 | } 218 | -------------------------------------------------------------------------------- /submodules/octree_handler/src/OctreeHandler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "Octree.hpp" 7 | 8 | namespace unibn { 9 | namespace traits { 10 | 11 | template <> 12 | struct access { 13 | static float get(const Eigen::Vector3f &p) { return p.x(); } 14 | }; 15 | 16 | template <> 17 | struct access { 18 | static float get(const Eigen::Vector3f &p) { return p.y(); } 19 | }; 20 | 21 | template <> 22 | struct access { 23 | static float get(const Eigen::Vector3f &p) { return p.z(); } 24 | }; 25 | } // namespace traits 26 | } // namespace unibn 27 | 28 | class Octree { 29 | private: 30 | unibn::Octree octree_; 31 | std::vector points_; 32 | 33 | public: 34 | Octree() = default; 35 | ~Octree() = default; 36 | void setInput(Eigen::MatrixXf &cloud); 37 | std::vector radiusSearch(const uint32_t &pt_idx, 38 | const float &radius); 39 | std::vector radiusSearch(const Eigen::Vector3f &pt, 40 | const float &radius); 41 | Eigen::MatrixXi radiusSearchAll(const uint32_t &max_nr_neighbors, 42 | const float &radius); 43 | Eigen::MatrixXi radiusSearchIndices(const std::vector &pt_indices, 44 | const uint32_t &max_nr_neighbors, 45 | const float &radius); 46 | Eigen::MatrixXi radiusSearchPoints(Eigen::MatrixXf &query_points, 47 | const uint32_t &max_nr_neighbors, 48 | const float &radius); 49 | std::vector randomUniformSampling(const float &max_dist); 50 | Eigen::MatrixXf spectralFeaturesAll(const float &radius); 51 | Eigen::MatrixXf computeEigenvaluesNormal(const float &radius); 52 | Eigen::MatrixXf computeScatterMatrices(const float &radius); 53 | }; 54 | -------------------------------------------------------------------------------- /submodules/octree_handler/tests/octree_cpp_testing.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import octree_handler 4 | import time 5 | 6 | if __name__ == "__main__": 7 | octree = octree_handler.Octree() 8 | points = np.array([[0, -5, 0], # 0 9 | [0, -4, 0], # 1 10 | [0, -3, 0], # 2 11 | [0, -2, 0], # 3 12 | [0, -1, 0], # 4 13 | [0, 0, 0], # 5 14 | [1, 0, 0], # 6 15 | [2, 0, 0], # 7 16 | [3, 0, 0], # 8 17 | [4, 0, 0], # 9 18 | [5, 0, 0], # 10 19 | # [100, 100, 100], # 11: not valid point 20 | ], dtype='float32') 21 | print(points) 22 | octree.setInput(points) 23 | b = octree.radiusSearch(5, 3.1) 24 | print(b) 25 | print(20*"-", "search all", 20*"-") 26 | ind = octree.radiusSearchAll(20, 2.1) 27 | print(ind) 28 | print(ind.shape, ind.dtype) 29 | 30 | ############################## 31 | ###### lot of points ######### 32 | ############################## 33 | nr_p = int(1e4) 34 | print(20*"-", f"#points: {nr_p}", 20*"-") 35 | points = np.random.normal(scale=10.0, size=(nr_p, 3)) 36 | t_init = time.time() 37 | octree.setInput(points) 38 | t_init = time.time()-t_init 39 | t_search = time.time() 40 | ind = octree.radiusSearchAll(10, 0.1) 41 | t_search = time.time() - t_search 42 | t_overall = t_init+t_search 43 | print(f'time: init {t_init}s, search {t_search}s, overall {t_overall}s') 44 | print(ind.shape) 45 | --------------------------------------------------------------------------------