├── doc ├── Github_intro.png ├── pretrained_models_guide.md ├── object_segmentation_guide.md ├── object_classification_guide.md ├── scene_segmentation_guide.md ├── visualization_guide.md └── new_dataset_guide.md ├── cpp_wrappers ├── compile_wrappers.sh ├── cpp_subsampling │ ├── setup.py │ ├── grid_subsampling │ │ ├── grid_subsampling.h │ │ └── grid_subsampling.cpp │ └── wrapper.cpp └── cpp_utils │ └── cloud │ ├── cloud.cpp │ └── cloud.h ├── convert.py ├── pipenvlist.txt ├── LICENSE ├── tf_custom_ops ├── tf_neighbors │ ├── neighbors │ │ ├── neighbors.h │ │ └── neighbors.cpp │ ├── tf_neighbors.cpp │ └── tf_batch_neighbors.cpp ├── cpp_utils │ └── cloud │ │ ├── cloud.cpp │ │ └── cloud.h ├── compile_op.sh ├── notes.md └── tf_subsampling │ ├── grid_subsampling │ ├── grid_subsampling.h │ └── grid_subsampling.cpp │ ├── tf_subsampling.cpp │ └── tf_batch_subsampling.cpp ├── Instruction_Manual ├── setup.sh ├── AWS_help.md ├── Hyperparameters_help.md ├── Running test and train.md └── Launching and Setting up the instance.md ├── INSTALL.md ├── envlist.txt ├── Log_2020-09-29_02-19-56 └── F1_Score.txt ├── README.md ├── test_accuracy.py ├── aerotronic.yml ├── utils ├── mesh.py └── metrics.py ├── training_S3DIS.py ├── training_ShapeNetPart.py ├── training_Semantic3D.py ├── training_NPM3D.py ├── training_DALES.py ├── training_Scannet.py ├── training_ModelNet40.py ├── visualize_deformations.py ├── models └── KPCNN_model.py ├── visualize_ERFs.py ├── test_any_model.py └── kernels └── kernel_points.py /doc/Github_intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Arjun-NA/KPConv_for_DALES/HEAD/doc/Github_intro.png -------------------------------------------------------------------------------- /cpp_wrappers/compile_wrappers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Compile cpp subsampling 4 | cd cpp_subsampling 5 | python3 setup.py build_ext --inplace 6 | cd .. 7 | 8 | -------------------------------------------------------------------------------- /doc/pretrained_models_guide.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Test a pretrained network 4 | 5 | ### Data 6 | 7 | We provide two examples of pretrained models: 8 | - A network with rigid KPConv trained on S3DIS: link (50 MB) 9 | - A network with deformable KPConv trained on NPM3D: link (54 MB) 10 | 11 | 12 | 13 | Unzip the log folder anywhere. 14 | 15 | ### Test model 16 | 17 | In `test_any_model.py`, choose the path of the log you just unzipped with the `chosen_log` variable: 18 | 19 | chosen_log = 'path_to_pretrained_log' 20 | -------------------------------------------------------------------------------- /cpp_wrappers/cpp_subsampling/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup, Extension 2 | import numpy.distutils.misc_util 3 | 4 | # Adding OpenCV to project 5 | # ************************ 6 | 7 | # Adding sources of the project 8 | # ***************************** 9 | 10 | m_name = "grid_subsampling" 11 | 12 | SOURCES = ["../cpp_utils/cloud/cloud.cpp", 13 | "grid_subsampling/grid_subsampling.cpp", 14 | "wrapper.cpp"] 15 | 16 | module = Extension(m_name, 17 | sources=SOURCES, 18 | extra_compile_args=['-std=c++11', 19 | '-D_GLIBCXX_USE_CXX11_ABI=0']) 20 | 21 | setup(ext_modules=[module], include_dirs=numpy.distutils.misc_util.get_numpy_include_dirs()) 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /convert.py: -------------------------------------------------------------------------------- 1 | from plyfile import PlyData, PlyElement, PlyProperty, PlyListProperty 2 | from os import listdir,path 3 | 4 | to_ascii = False 5 | file_path = '.' 6 | files = [f for f in listdir(file_path) if f[-4:] == '.ply'] 7 | for each_file in files: 8 | print('\n Loading.... ', path.join(file_path, each_file) ) 9 | data = PlyData.read(path.join(file_path, each_file) ) 10 | print('\n Loaded..... ', path.join(file_path, each_file) ) 11 | data.elements[0].data.dtype.names = ['x', 'y', 'z', 'reflectance', 'class'] 12 | data.elements[0].properties = (PlyProperty('x', 'float'), PlyProperty('y', 'float'), 13 | PlyProperty('z', 'float'), PlyProperty('reflectance', 'int'), 14 | PlyProperty('class', 'int')) 15 | data1 = PlyData([data.elements[0]], text=to_ascii) 16 | data1.write(path.join(file_path,'bin_'+ each_file) ) 17 | print('\n completed.. ', each_file) 18 | data2 = PlyData.read(path.join(file_path, 'bin_'+each_file) ) 19 | print(data.elements[0]) 20 | print('\n') 21 | -------------------------------------------------------------------------------- /pipenvlist.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.9.0 2 | astor==0.8.0 3 | certifi==2020.4.5.2 4 | chardet==3.0.4 5 | cycler==0.10.0 6 | gast==0.2.2 7 | google-pasta==0.2.0 8 | grpcio==1.27.2 9 | h5py==2.10.0 10 | idna==2.9 11 | imbalanced-learn==0.7.0 12 | joblib==0.16.0 13 | Keras==2.3.1 14 | Keras-Applications==1.0.8 15 | Keras-Preprocessing==1.1.0 16 | kiwisolver==1.2.0 17 | Markdown==3.1.1 18 | matplotlib==3.2.2 19 | mkl-fft==1.1.0 20 | mkl-random==1.1.1 21 | mkl-service==2.3.0 22 | numpy==1.18.1 23 | opt-einsum==3.1.0 24 | pandas==1.0.5 25 | plyfile==0.7.2 26 | protobuf==3.12.3 27 | psutil==5.7.2 28 | pyparsing==2.4.7 29 | python-dateutil==2.8.1 30 | python-mnist==0.7 31 | pytz==2020.1 32 | PyYAML==5.3.1 33 | requests==2.24.0 34 | scikit-learn==0.23.1 35 | scipy==1.4.1 36 | six==1.15.0 37 | sklearn==0.0 38 | svgpathtools==1.3.3 39 | svgwrite==1.4 40 | tensorboard==1.14.0 41 | tensorflow==1.14.0 42 | tensorflow-estimator==1.14.0 43 | termcolor==1.1.0 44 | threadpoolctl==2.1.0 45 | tqdm==4.46.1 46 | transforms3d==0.3.1 47 | urllib3==1.25.9 48 | webencodings==0.5.1 49 | Werkzeug==0.16.1 50 | wrapt==1.12.1 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 HuguesTHOMAS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tf_custom_ops/tf_neighbors/neighbors/neighbors.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "../../cpp_utils/cloud/cloud.h" 4 | #include "../../cpp_utils/nanoflann/nanoflann.hpp" 5 | 6 | #include 7 | #include 8 | 9 | using namespace std; 10 | 11 | 12 | void ordered_neighbors(vector& queries, 13 | vector& supports, 14 | vector& neighbors_indices, 15 | float radius); 16 | 17 | void batch_ordered_neighbors(vector& queries, 18 | vector& supports, 19 | vector& q_batches, 20 | vector& s_batches, 21 | vector& neighbors_indices, 22 | float radius); 23 | 24 | void batch_nanoflann_neighbors(vector& queries, 25 | vector& supports, 26 | vector& q_batches, 27 | vector& s_batches, 28 | vector& neighbors_indices, 29 | float radius); 30 | -------------------------------------------------------------------------------- /Instruction_Manual/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## Arjun's KPConv adjustments... 3 | sudo mkdir -p -m 777 $HOME/lidar 4 | sudo chmod g+s $HOME/lidar 5 | cd $HOME/lidar 6 | git clone https://github.com/Arjun-NA/KPConv_for_DALES 7 | 8 | cd $HOME/lidar/KPConv_for_DALES 9 | if ( ! /home/ubuntu/anaconda3/bin/conda env list | grep "^aerotronic" >/dev/null 2>&1 ); then 10 | /home/ubuntu/anaconda3/bin/conda env create --file ./aerotronic.yml 11 | else 12 | echo "aerotronic conda environment is already present" 13 | echo "run conda activate aerotronic" 14 | fi 15 | 16 | eval "$(conda shell.bash hook)" 17 | conda activate aerotronic 18 | cd $HOME/lidar/KPConv_for_DALES/tf_custom_ops 19 | TF_LIB=$(python3 -c 'import tensorflow as tf;print(tf.sysconfig.get_lib())' 2>/dev/null) 20 | 21 | echo "TFLIB: " 22 | echo $TF_LIB 23 | cd $TF_LIB 24 | ls 25 | sudo cp libtensorflow_framework.so.1 libtensorflow_framework.so 26 | cd $HOME/lidar/KPConv_for_DALES/tf_custom_ops 27 | export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${TF_LIB} 28 | sh ./compile_op.sh 29 | 30 | 31 | cd $HOME/lidar/KPConv_for_DALES/cpp_wrappers 32 | sh ./compile_wrappers.sh 33 | 34 | 35 | ## Tmux Installation 36 | sudo apt-get --assume-yes install tmux 37 | -------------------------------------------------------------------------------- /cpp_wrappers/cpp_utils/cloud/cloud.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // 3 | // 0==========================0 4 | // | Local feature test | 5 | // 0==========================0 6 | // 7 | // version 1.0 : 8 | // > 9 | // 10 | //--------------------------------------------------- 11 | // 12 | // Cloud source : 13 | // Define usefull Functions/Methods 14 | // 15 | //---------------------------------------------------- 16 | // 17 | // Hugues THOMAS - 10/02/2017 18 | // 19 | 20 | 21 | #include "cloud.h" 22 | 23 | 24 | // Getters 25 | // ******* 26 | 27 | PointXYZ max_point(std::vector points) 28 | { 29 | // Initiate limits 30 | PointXYZ maxP(points[0]); 31 | 32 | // Loop over all points 33 | for (auto p : points) 34 | { 35 | if (p.x > maxP.x) 36 | maxP.x = p.x; 37 | 38 | if (p.y > maxP.y) 39 | maxP.y = p.y; 40 | 41 | if (p.z > maxP.z) 42 | maxP.z = p.z; 43 | } 44 | 45 | return maxP; 46 | } 47 | 48 | PointXYZ min_point(std::vector points) 49 | { 50 | // Initiate limits 51 | PointXYZ minP(points[0]); 52 | 53 | // Loop over all points 54 | for (auto p : points) 55 | { 56 | if (p.x < minP.x) 57 | minP.x = p.x; 58 | 59 | if (p.y < minP.y) 60 | minP.y = p.y; 61 | 62 | if (p.z < minP.z) 63 | minP.z = p.z; 64 | } 65 | 66 | return minP; 67 | } -------------------------------------------------------------------------------- /tf_custom_ops/cpp_utils/cloud/cloud.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // 3 | // 0==========================0 4 | // | Local feature test | 5 | // 0==========================0 6 | // 7 | // version 1.0 : 8 | // > 9 | // 10 | //--------------------------------------------------- 11 | // 12 | // Cloud source : 13 | // Define usefull Functions/Methods 14 | // 15 | //---------------------------------------------------- 16 | // 17 | // Hugues THOMAS - 10/02/2017 18 | // 19 | 20 | 21 | #include "cloud.h" 22 | 23 | 24 | // Getters 25 | // ******* 26 | 27 | PointXYZ max_point(std::vector points) 28 | { 29 | // Initiate limits 30 | PointXYZ maxP(points[0]); 31 | 32 | // Loop over all points 33 | for (auto p : points) 34 | { 35 | if (p.x > maxP.x) 36 | maxP.x = p.x; 37 | 38 | if (p.y > maxP.y) 39 | maxP.y = p.y; 40 | 41 | if (p.z > maxP.z) 42 | maxP.z = p.z; 43 | } 44 | 45 | return maxP; 46 | } 47 | 48 | PointXYZ min_point(std::vector points) 49 | { 50 | // Initiate limits 51 | PointXYZ minP(points[0]); 52 | 53 | // Loop over all points 54 | for (auto p : points) 55 | { 56 | if (p.x < minP.x) 57 | minP.x = p.x; 58 | 59 | if (p.y < minP.y) 60 | minP.y = p.y; 61 | 62 | if (p.z < minP.z) 63 | minP.z = p.z; 64 | } 65 | 66 | return minP; 67 | } -------------------------------------------------------------------------------- /tf_custom_ops/compile_op.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get TF variables 4 | TF_INC=$(python3 -c 'import tensorflow as tf; print(tf.sysconfig.get_include())') 5 | TF_LIB=$(python3 -c 'import tensorflow as tf; print(tf.sysconfig.get_lib())') 6 | TF_LIB2='/home/ubuntu/Arjun/KPConv/tf_custom_ops' 7 | # Neighbors op 8 | g++ -std=c++11 -shared tf_neighbors/tf_neighbors.cpp tf_neighbors/neighbors/neighbors.cpp cpp_utils/cloud/cloud.cpp -o tf_neighbors.so -fPIC -I$TF_INC -I$TF_INC/external/nsync/public -L$TF_LIB -L$TF_LIB2 -ltensorflow_framework -O2 9 | g++ -std=c++11 -shared tf_neighbors/tf_batch_neighbors.cpp tf_neighbors/neighbors/neighbors.cpp cpp_utils/cloud/cloud.cpp -o tf_batch_neighbors.so -fPIC -I$TF_INC -I$TF_INC/external/nsync/public -L$TF_LIB -L$TF_LIB2 -ltensorflow_framework -O2 10 | 11 | # Subsampling op 12 | g++ -std=c++11 -shared tf_subsampling/tf_subsampling.cpp tf_subsampling/grid_subsampling/grid_subsampling.cpp cpp_utils/cloud/cloud.cpp -o tf_subsampling.so -fPIC -I$TF_INC -I$TF_INC/external/nsync/public -L$TF_LIB -ltensorflow_framework -O2 13 | g++ -std=c++11 -shared tf_subsampling/tf_batch_subsampling.cpp tf_subsampling/grid_subsampling/grid_subsampling.cpp cpp_utils/cloud/cloud.cpp -o tf_batch_subsampling.so -fPIC -I$TF_INC -I$TF_INC/external/nsync/public -L$TF_LIB -ltensorflow_framework -O2 14 | -------------------------------------------------------------------------------- /tf_custom_ops/notes.md: -------------------------------------------------------------------------------- 1 | # **NOTES** 2 | 3 | ### 1. Error while execution compile.sh "-ltensorflow_core not found": 4 | __*Reason*__ : Library files of Tensorflow in ubuntu systems are saved in .so format. For some weird reasons versions of them are saved as so.1 or so.2 and suddenly shell forgets what they are 5 | 'So' better go to the location of the include and see which is what. 6 | 7 | To see the location use this inside python or the conda environment python (whichever you created for your project): 8 | ``` 9 | import tensorflow as tf 10 | print(tf.sysconfig.get_lib()) 11 | ``` 12 | OR you could also do the following in normal shell/cmd do the following 13 | ``` 14 | TF_INC=$(python3 -c 'import tensorflow as tf; print(tf.sysconfig.get_include())') 15 | TF_LIB=$(python3 -c 'import tensorflow as tf; print(tf.sysconfig.get_lib())') 16 | cd $TF_LIB 17 | ``` 18 | After that the variables TF_LIB will lead you to the location. 19 | Where did I magically get these from? It's over here in [compile_op.sh](./tf_custom_ops/compile_op.sh) 20 | 21 | Inside the location, you can either create a soft link like this [issue solution](https://github.com/HuguesTHOMAS/KPConv/issues/79) 22 | or You can create a duplication of libtensorflow_framework.so.1 or libtensorflow_framework.so.2 with the name libtensorflow_framework.so 23 | to do that you can use: 24 | ``` 25 | cp libtensorflow_framework.so.1 libtensorflow_framework.so 26 | ``` 27 | -------------------------------------------------------------------------------- /doc/object_segmentation_guide.md: -------------------------------------------------------------------------------- 1 | 2 | ## Object Part Segmentation on ShapeNetPart 3 | 4 | ### Data 5 | 6 | ShapeNetPart dataset can be downloaded here (635 MB). Uncompress the folder and move it to `Data/ShapeNetPart/shapenetcore_partanno_segmentation_benchmark_v0`. 7 | 8 | ### Training 9 | 10 | Simply run the following script to start the training: 11 | 12 | python3 training_ShapeNetPart.py 13 | 14 | Similarly to ModelNet40 training, the parameters can be modified in a configuration subclass called `ShapeNetPartConfig`, and the first run of this script might take some time to precompute dataset structures. 15 | 16 | ### Plot a logged training 17 | 18 | When you start a new training, it is saved in a `results` folder. A dated log folder will be created, containing many information including loss values, validation metrics, model snapshots, etc. 19 | 20 | In `plot_convergence.py`, you will find detailed comments explaining how to choose which training log you want to plot. Follow them and then run the script : 21 | 22 | python3 plot_convergence.py 23 | 24 | 25 | ### Test the trained model 26 | 27 | The test script is the same for all models (segmentation or classification). In `test_any_model.py`, you will find detailed comments explaining how to choose which logged trained model you want to test. Follow them and then run the script : 28 | 29 | python3 test_any_model.py 30 | -------------------------------------------------------------------------------- /doc/object_classification_guide.md: -------------------------------------------------------------------------------- 1 | 2 | ## Object classification on ModelNet40 3 | 4 | ### Data 5 | 6 | Regularly sampled clouds from ModelNet40 dataset can be downloaded here (1.6 GB). Uncompress the folder and move it to `Data/ModelNet40/modelnet40_normal_resampled`. 7 | 8 | N.B. If you want to place your data anywhere else, you just have to change the variable `self.path` of `ModelNet40Dataset` class (in the file `datasets/ModelNet40.py`). 9 | 10 | ### Training a model 11 | 12 | Simply run the following script to start the training: 13 | 14 | python3 training_ModelNet40.py 15 | 16 | This file contains a configuration subclass `ModelNet40Config`, inherited from the general configuration class `Config` defined in `utils/config.py`. The value of every parameter can be modified in the subclass. The first run of this script will precompute structures for the dataset which might take some time. 17 | 18 | ### Plot a logged training 19 | 20 | When you start a new training, it is saved in a `results` folder. A dated log folder will be created, containing many information including loss values, validation metrics, model snapshots, etc. 21 | 22 | In `plot_convergence.py`, you will find detailed comments explaining how to choose which training log you want to plot. Follow them and then run the script : 23 | 24 | python3 plot_convergence.py 25 | 26 | 27 | ### Test the trained model 28 | 29 | The test script is the same for all models (segmentation or classification). In `test_any_model.py`, you will find detailed comments explaining how to choose which logged trained model you want to test. Follow them and then run the script : 30 | 31 | python3 test_any_model.py 32 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | ### Installation instructions for Ubuntu 16.04 2 | 3 | * Make sure CUDA and cuDNN are installed. Three configurations have been tested: 4 | - TensorFlow 1.4.1, CUDA 8.0 and cuDNN 6.0 5 | - TensorFlow 1.12.0, CUDA 9.0 and cuDNN 7.4 6 | - ~~TensorFlow 1.13.0, CUDA 10.0 and cuDNN 7.5~~ (bug found only with this version). 7 | 8 | * Ensure all python packages are installed : 9 | 10 | sudo apt update 11 | sudo apt install python3-dev python3-pip python3-tk 12 | 13 | * Follow Tensorflow installation procedure. 14 | 15 | * Install the other dependencies with pip: 16 | - numpy 17 | - scikit-learn 18 | - psutil 19 | - matplotlib (for visualization) 20 | - mayavi (for visualization) 21 | - PyQt5 (for visualization) 22 | 23 | * Compile the customized Tensorflow operators located in `tf_custom_ops`. Open a terminal in this folder, and run: 24 | 25 | sh compile_op.sh 26 | 27 | N.B. If you installed Tensorflow in a virtual environment, it needs to be activated when running these scripts 28 | 29 | * Compile the C++ extension module for python located in `cpp_wrappers`. Open a terminal in this folder, and run: 30 | 31 | sh compile_wrappers.sh 32 | 33 | You should now be able to train Kernel-Point Convolution models 34 | 35 | ### Installation instructions for Ubuntu 18.04 (Thank to @noahtren) 36 | 37 | * Remove the `-D_GLIBCXX_USE_CXX11_ABI=0` flag for each line in `tf_custom_ops/compile_op.sh` (problem with the version of gcc). One configuration has been tested: 38 | 39 | - TensorFlow 1.12.0, CUDA 9.0 and cuDNN 7.3.1 40 | -------------------------------------------------------------------------------- /envlist.txt: -------------------------------------------------------------------------------- 1 | # This file may be used to create an environment using: 2 | # $ conda create --name --file 3 | # platform: linux-64 4 | _libgcc_mutex=0.1=main 5 | _tflow_select=2.1.0=gpu 6 | absl-py=0.9.0=py37_0 7 | astor=0.8.0=py37_0 8 | blas=1.0=mkl 9 | c-ares=1.15.0=h7b6447c_1001 10 | ca-certificates=2020.1.1=0 11 | certifi=2020.4.5.2=py37_0 12 | cudatoolkit=9.0=h13b8566_0 13 | cudnn=7.6.5=cuda9.0_0 14 | cupti=9.0.176=0 15 | gast=0.2.2=py37_0 16 | google-pasta=0.2.0=py_0 17 | grpcio=1.27.2=py37hf8bcb03_0 18 | h5py=2.10.0=py37h7918eee_0 19 | hdf5=1.10.4=hb1b8bf9_0 20 | intel-openmp=2020.1=217 21 | keras-applications=1.0.8=py_0 22 | keras-base=2.3.1=py37_0 23 | keras-gpu=2.3.1=0 24 | keras-preprocessing=1.1.0=py_1 25 | ld_impl_linux-64=2.33.1=h53a641e_7 26 | libedit=3.1.20181209=hc058e9b_0 27 | libffi=3.3=he6710b0_1 28 | libgcc-ng=9.1.0=hdf63c60_0 29 | libgfortran-ng=7.3.0=hdf63c60_0 30 | libprotobuf=3.12.3=hd408876_0 31 | libstdcxx-ng=9.1.0=hdf63c60_0 32 | markdown=3.1.1=py37_0 33 | mkl=2020.1=217 34 | mkl-service=2.3.0=py37he904b0f_0 35 | mkl_fft=1.1.0=py37h23d657b_0 36 | mkl_random=1.1.1=py37h0573a6f_0 37 | ncurses=6.2=he6710b0_1 38 | numpy=1.18.1=py37h4f9e942_0 39 | numpy-base=1.18.1=py37hde5b4d6_1 40 | openssl=1.1.1g=h7b6447c_0 41 | opt_einsum=3.1.0=py_0 42 | pip=20.1.1=py37_1 43 | protobuf=3.12.3=py37he6710b0_0 44 | python=3.7.7=hcff3b4d_5 45 | pyyaml=5.3.1=py37h7b6447c_0 46 | readline=8.0=h7b6447c_0 47 | scipy=1.4.1=py37h0b6359f_0 48 | setuptools=47.3.0=py37_0 49 | six=1.15.0=py_0 50 | sqlite=3.31.1=h62c20be_1 51 | tensorboard=1.14.0=py37hf484d3e_0 52 | tensorflow=1.14.0=gpu_py37hae64822_0 53 | tensorflow-base=1.14.0=gpu_py37h8f37b9b_0 54 | tensorflow-estimator=1.14.0=py_0 55 | tensorflow-gpu=1.14.0=h0d30ee6_0 56 | termcolor=1.1.0=py37_1 57 | tk=8.6.8=hbc83047_0 58 | webencodings=0.5.1=py37_1 59 | werkzeug=0.16.1=py_0 60 | wheel=0.34.2=py37_0 61 | wrapt=1.12.1=py37h7b6447c_1 62 | xz=5.2.5=h7b6447c_0 63 | yaml=0.1.7=had09818_2 64 | zlib=1.2.11=h7b6447c_3 65 | -------------------------------------------------------------------------------- /Log_2020-09-29_02-19-56/F1_Score.txt: -------------------------------------------------------------------------------- 1 | All files good 2 | 3 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5080_54400.ply 4 | micro F1: 0.971800472005263 5 | macro F1: 0.7125992781285472 6 | 7 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5080_54470.ply 8 | micro F1: 0.9712402459534125 9 | macro F1: 0.7603846950573112 10 | 11 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5100_54440.ply 12 | micro F1: 0.9339789109964945 13 | macro F1: 0.7407313819921052 14 | 15 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5100_54490.ply 16 | micro F1: 0.9733621763924409 17 | macro F1: 0.7564696469960107 18 | 19 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5120_54445.ply 20 | micro F1: 0.9724872673445031 21 | macro F1: 0.784785751017888 22 | 23 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5135_54430.ply 24 | micro F1: 0.9759974172148351 25 | macro F1: 0.764491141588723 26 | 27 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5135_54435.ply 28 | micro F1: 0.977462815230023 29 | macro F1: 0.7926956961914926 30 | 31 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5140_54390.ply 32 | micro F1: 0.964461245744877 33 | macro F1: 0.6541784907805973 34 | 35 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5150_54325.ply 36 | micro F1: 0.9740829226300796 37 | macro F1: 0.6994128385643351 38 | 39 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5155_54335.ply 40 | micro F1: 0.9722904007687808 41 | macro F1: 0.5379838415421019 42 | 43 | Loading.... ../Results_from_RED/Log_2020-09-29_02-19-56/predictions/bin_5175_54395.ply 44 | micro F1: 0.9705561295053557 45 | macro F1: 0.7616489924197007 46 | Final Avg micro : 0.9688836367078242 | Avg macro : 0.7241256140253466 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ![Intro figure](https://github.com/HuguesTHOMAS/KPConv/blob/master/doc/Github_intro.png) 3 | 4 | ### [Created by Hugues THOMAS](https://github.com/HuguesTHOMAS/KPConv) 5 | 6 | 7 | ### Update 27/04/2020: [PyTorch implementation](https://github.com/HuguesTHOMAS/KPConv-PyTorch) available. With SemanticKitti, and Windows supported. 8 | 9 | This repository contains the implementation of **Kernel Point Convolution** (KPConv), a point convolution operator 10 | presented in our ICCV2019 paper ([arXiv](https://arxiv.org/abs/1904.08889)). 11 | 12 | **Update 03/05/2019, bug found with TF 1.13 and CUDA 10.** 13 | We found an internal bug inside tf.matmul operation. It returns absurd values like 1e12, leading to the 14 | apparition of NaNs in our network. We advise to use the code with CUDA 9.0 and TF 1.12. 15 | More info in [issue #15](https://github.com/HuguesTHOMAS/KPConv/issues/15) 16 | 17 | 18 | ## Installation 19 | 20 | A step-by-step installation guide for Ubuntu 16.04 is provided in [INSTALL.md](./INSTALL.md). Windows is currently 21 | not supported as the code uses tensorflow custom operations. 22 | 23 | 24 | ## TO USE IN DALES DATASET 25 | 26 | Use the [convert.py](convert.py) to convert the DALES ascii ply file to bin ply file. 27 | copy the convert.py to the location of the ascii ply files and run it.
28 | Utilize [requirements](pipenvlist.txt) and [conda_env](envlist.txt) 29 | For conda env creation you can use : 30 | ``` 31 | conda create --name --file envlist.txt 32 | ``` 33 | and for pip 34 | ``` 35 | pip install -r pipenvlist.txt 36 | ``` 37 | ## MY RESULTS IN DALES DATASET 38 | The result of the testing is uploaded [here](https://indiana-my.sharepoint.com/:u:/g/personal/arjuna_iu_edu/ERlV6lBVnQtMvAyfloE354YBNUglxQbroAUnGds8x8Rjcg?e=vIk490) 39 | and the [test_accuracy.py](./test_accuracy.py) was used to calculate F1 scores which gave [this](./Log_2020-09-29_02-19-56/F1_Score.txt) result with Final Avg micro F1 : 0.97 | Avg macro F1: 0.72. This result was far better than any other point convolutions I had used. 40 | -------------------------------------------------------------------------------- /cpp_wrappers/cpp_subsampling/grid_subsampling/grid_subsampling.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "../../cpp_utils/cloud/cloud.h" 4 | 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | 10 | class SampledData 11 | { 12 | public: 13 | 14 | // Elements 15 | // ******** 16 | 17 | int count; 18 | PointXYZ point; 19 | vector features; 20 | vector> labels; 21 | 22 | 23 | // Methods 24 | // ******* 25 | 26 | // Constructor 27 | SampledData() 28 | { 29 | count = 0; 30 | point = PointXYZ(); 31 | } 32 | 33 | SampledData(const size_t fdim, const size_t ldim) 34 | { 35 | count = 0; 36 | point = PointXYZ(); 37 | features = vector(fdim); 38 | labels = vector>(ldim); 39 | } 40 | 41 | // Method Update 42 | void update_all(const PointXYZ p, vector::iterator f_begin, vector::iterator l_begin) 43 | { 44 | count += 1; 45 | point += p; 46 | transform (features.begin(), features.end(), f_begin, features.begin(), plus()); 47 | int i = 0; 48 | for(vector::iterator it = l_begin; it != l_begin + labels.size(); ++it) 49 | { 50 | labels[i][*it] += 1; 51 | i++; 52 | } 53 | return; 54 | } 55 | void update_features(const PointXYZ p, vector::iterator f_begin) 56 | { 57 | count += 1; 58 | point += p; 59 | transform (features.begin(), features.end(), f_begin, features.begin(), plus()); 60 | return; 61 | } 62 | void update_classes(const PointXYZ p, vector::iterator l_begin) 63 | { 64 | count += 1; 65 | point += p; 66 | int i = 0; 67 | for(vector::iterator it = l_begin; it != l_begin + labels.size(); ++it) 68 | { 69 | labels[i][*it] += 1; 70 | i++; 71 | } 72 | return; 73 | } 74 | void update_points(const PointXYZ p) 75 | { 76 | count += 1; 77 | point += p; 78 | return; 79 | } 80 | }; 81 | 82 | 83 | 84 | void grid_subsampling(vector& original_points, 85 | vector& subsampled_points, 86 | vector& original_features, 87 | vector& subsampled_features, 88 | vector& original_classes, 89 | vector& subsampled_classes, 90 | float sampleDl, 91 | int verbose); 92 | 93 | -------------------------------------------------------------------------------- /doc/scene_segmentation_guide.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Scene Segmentation on S3DIS 4 | 5 | ### Data 6 | 7 | S3DIS dataset can be downloaded here (4.8 GB). Download the file named `Stanford3dDataset_v1.2.zip`, uncompress the folder and move it to `Data/S3DIS/Stanford3dDataset_v1.2`. 8 | 9 | ### Training 10 | 11 | Simply run the following script to start the training: 12 | 13 | python3 training_S3DIS.py 14 | 15 | Similarly to ModelNet40 training, the parameters can be modified in a configuration subclass called `S3DISConfig`, and the first run of this script might take some time to precompute dataset structures. 16 | 17 | 18 | ## Scene Segmentation on Scannet 19 | 20 | Incoming 21 | 22 | ## Scene Segmentation on Semantic3D 23 | 24 | ### Data 25 | 26 | Semantic3D dataset can be found here. Download and unzip every point cloud as ascii files and place them in a folder called `Data/Semantic3D/original_data`. You also have to download and unzip the groundthruth labels as ascii files in the same folder 27 | 28 | 29 | ### Training 30 | 31 | Simply run the following script to start the training: 32 | 33 | python3 training_Semantic3D.py 34 | 35 | Similarly to ModelNet40 training, the parameters can be modified in a configuration subclass called `Semantic3DConfig`, and the first run of this script might take some time to precompute dataset structures. 36 | 37 | 38 | ## Scene Segmentation on NPM3D 39 | 40 | Incoming 41 | 42 | 43 | ## Plot and test trained models 44 | 45 | ### Plot a logged training 46 | 47 | When you start a new training, it is saved in a `results` folder. A dated log folder will be created, containing many information including loss values, validation metrics, model snapshots, etc. 48 | 49 | In `plot_convergence.py`, you will find detailed comments explaining how to choose which training log you want to plot. Follow them and then run the script : 50 | 51 | python3 plot_convergence.py 52 | 53 | 54 | ### Test the trained model 55 | 56 | The test script is the same for all models (segmentation or classification). In `test_any_model.py`, you will find detailed comments explaining how to choose which logged trained model you want to test. Follow them and then run the script : 57 | 58 | python3 test_any_model.py 59 | -------------------------------------------------------------------------------- /tf_custom_ops/tf_subsampling/grid_subsampling/grid_subsampling.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "../../cpp_utils/cloud/cloud.h" 4 | 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | 10 | class SampledData 11 | { 12 | public: 13 | 14 | // Elements 15 | // ******** 16 | 17 | int count; 18 | PointXYZ point; 19 | vector features; 20 | unordered_map labels; 21 | 22 | 23 | // Methods 24 | // ******* 25 | 26 | // Constructor 27 | SampledData() 28 | { 29 | count = 0; 30 | point = PointXYZ(); 31 | } 32 | 33 | SampledData(const size_t fdim) 34 | { 35 | count = 0; 36 | point = PointXYZ(); 37 | features = vector(fdim); 38 | } 39 | 40 | // Method Update 41 | void update_all(const PointXYZ p, std::vector::iterator f_begin, const int l) 42 | { 43 | count += 1; 44 | point += p; 45 | std::transform (features.begin(), features.end(), f_begin, features.begin(), std::plus()); 46 | labels[l] += 1; 47 | return; 48 | } 49 | void update_features(const PointXYZ p, std::vector::iterator f_begin) 50 | { 51 | count += 1; 52 | point += p; 53 | std::transform (features.begin(), features.end(), f_begin, features.begin(), std::plus()); 54 | return; 55 | } 56 | void update_classes(const PointXYZ p, const int l) 57 | { 58 | count += 1; 59 | point += p; 60 | labels[l] += 1; 61 | return; 62 | } 63 | void update_points(const PointXYZ p) 64 | { 65 | count += 1; 66 | point += p; 67 | return; 68 | } 69 | }; 70 | 71 | 72 | 73 | void grid_subsampling(vector& original_points, 74 | vector& subsampled_points, 75 | vector& original_features, 76 | vector& subsampled_features, 77 | vector& original_classes, 78 | vector& subsampled_classes, 79 | float sampleDl); 80 | 81 | 82 | void batch_grid_subsampling(vector& original_points, 83 | vector& subsampled_points, 84 | vector& original_features, 85 | vector& subsampled_features, 86 | vector& original_classes, 87 | vector& subsampled_classes, 88 | vector& original_batches, 89 | vector& subsampled_batches, 90 | float sampleDl); 91 | 92 | -------------------------------------------------------------------------------- /Instruction_Manual/AWS_help.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Commands for EC2 instance 3 | 4 | Basics and Advanced commands which is useful for setting up and running Amazon AMI EC2 instance. 5 | 6 | ## Managing External Storage 7 | 8 | For more detailed guide [try here](https://www.digitalocean.com/community/tutorials/how-to-partition-and-format-storage-devices-in-linux) 9 | Command used to list all usable storage 10 | ``` 11 | sudo lsblk 12 | ``` 13 | Some versions of lsblk will print all of this information if we type: 14 | ``` 15 | sudo lsblk --fs 16 | ``` 17 | Command for creating partition 18 | ``` 19 | sudo parted /dev/sda mklabel gpt 20 | sudo parted -a opt /dev/sda mkpart primary ext4 0% 100% 21 | ``` 22 | 23 | Create a Filesystem on the New Partition 24 | ``` 25 | sudo mkfs.ext4 -L datapartition /dev/sda1 26 | ``` 27 | 28 | 29 | ## Mounting an extra storage eg: HDD storage 30 | 31 | Create the mounting point using the following commands 32 | ``` 33 | cd / 34 | sudo mkdir media/NewVol/ 35 | ``` 36 | Adding storage to the running instance. (You should have already done the volume attach to the instance from the AWS Management Console) 37 | ``` 38 | cd / 39 | sudo mount -t ext4 /dev/[storage_name] /media/NewVol/ 40 | ``` 41 |
42 | [storage_name] 43 | : This should be found out on your own. This can be found out by checking command `lsblk` or the directory list using the ``` ls``` command in /dev/ folder before and after you attach the volume (Notice the change in it) to the instance via AWS Management Console. 44 |
some common names look like nvme1n1 or SDF or XDF or XVDF where F is variable 45 | 46 | 47 | 48 | ## Copying files from/to s3 49 | ``` 50 | aws s3 cp filename_from filename_to 51 | aws s3 cp --recursive folder_from folder_to 52 | ``` 53 | 54 | ## Copying folder from local storage to AWS s3 55 | As the usage of cp does not help in copying folder from the instance to the s3 cloud we utilize sync
56 | ``` 57 | aws s3 sync folder_to_copy S3_URL 58 | ``` 59 | 60 | ## Configuring AWS credentials to access S3 61 | Fix for the error **"Unable to locate credentials"** 62 | 63 | ### Quick configuration with aws configure 64 | 65 | For general use, the aws configure command is the fastest way to set up your AWS CLI installation. 66 | 67 | the AWS CLI prompts you for four pieces of information: 68 | - Access key ID (for each user have max 2. It can be found from IAM -> Users ->Select user -> Security credentials ) 69 | - Secret access key (for each user have max 2) 70 | - AWS Region (eg: us-east-2) 71 | - Output format (you can leave it at none) 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /test_accuracy.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import sys 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | from sklearn.metrics import confusion_matrix, f1_score 7 | 8 | # Custom libs 9 | 10 | # Dataset 11 | from plyfile import PlyData, PlyElement 12 | labels = ['Unclassified','Gnd', 'Trees', 'Car', 'Truck', 'Wire', 'Fence', 'Poles' , 'Bldngs'] 13 | ignored_labels = ['Unclassified'] 14 | final_labels = labels[:] 15 | for each in ignored_labels: final_labels.remove(each) 16 | # Given below is the path to test output and ground truth files. 17 | # All files in test output file should be present in ground truth folder for successful execution of this file 18 | # This Also works in windows 19 | test_predictions_path = 'results/Log_2020-09-29_02-19-56/predictions/' 20 | test_groundtruth_path = '../Data/test_bin/' 21 | files_pred = [f for f in os.listdir(test_predictions_path) if f[-4:] == '.ply'] 22 | files_ground = [f for f in os.listdir(test_groundtruth_path) if f[-4:] == '.ply'] 23 | if(all(each in files_ground for each in files_pred )): 24 | print("All files good") 25 | else: 26 | print("Error some files at ",test_predictions_path, "not matching with files at",test_groundtruth_path) 27 | exit() 28 | 29 | once = False 30 | total_list_micro = list() 31 | total_list_macro = list() 32 | Cum = None 33 | for each_file in files_pred: 34 | print('\n Loading.... ', os.path.join(test_predictions_path, each_file)) 35 | data_pred = PlyData.read(os.path.join(test_predictions_path, each_file)) 36 | data_grtr = PlyData.read(os.path.join(test_groundtruth_path, each_file)) 37 | y_true = data_grtr.elements[0]['class'] 38 | y_pred = data_pred.elements[0]['preds'] 39 | """ # Uncomment these lines for saving confusion matrix in a color scale in pdf format 40 | C = confusion_matrix(y_true, y_pred, normalize='pred') 41 | 42 | for l_ind, label_value in enumerate(labels): 43 | if label_value in ignored_labels: 44 | C = np.delete(C, l_ind, axis=0) 45 | C = np.delete(C, l_ind, axis=1) 46 | 47 | if not once: 48 | Cum = C 49 | else: 50 | Cum += C 51 | plt.imshow(Cum) 52 | ticks = range(len(final_labels)) 53 | plt.xticks(ticks=ticks,labels=final_labels) 54 | plt.yticks(ticks=ticks,labels=final_labels) 55 | if not once: plt.colorbar() 56 | once = True 57 | plt.title(" Confusion Matrix ") 58 | plt.savefig("results/"+each_file[:-4]+'.pdf') 59 | """ 60 | F1_score_micro = f1_score(y_true, y_pred, average='micro') 61 | F1_score_macro = f1_score(y_true, y_pred, average='macro') 62 | print("micro F1: \t",F1_score_micro) 63 | print("macro F1: \t",F1_score_macro) 64 | total_list_macro += [F1_score_macro] 65 | total_list_micro += [F1_score_micro] 66 | avg_micro = sum(total_list_micro)/len(total_list_micro) 67 | avg_macro = sum(total_list_macro)/len(total_list_macro) 68 | print( " Final Avg micro : ", avg_micro, "| Avg macro : ", avg_macro) 69 | -------------------------------------------------------------------------------- /Instruction_Manual/Hyperparameters_help.md: -------------------------------------------------------------------------------- 1 | # This file is a summary of hyperparameter that was adjusted to improve accuracy 2 | 3 | For KP Convolution the following parameters change the network complexity : 4 | 1. architecture: Trivially this helps in varying the complexity of the network as required. The parameter is provided as a list of words that the software recognizes. The words/ modules that exist in this particular software are the followings 5 | - 'unary' 6 | - 'simple' 7 | - 'simple_strided' 8 | - 'resnet' 9 | - 'resnetb' 10 | - 'resnetb_light' 11 | - 'resnetb_deformable' 12 | - 'inceptiong_deformable' 13 | - 'resnetb_strided' 14 | - 'resnetb_light_strided' 15 | - 'resnetb_deformable_strided' 16 | - 'inception_deformable_strided' 17 | - 'vgg' 18 | - 'max_pool' 19 | - 'global_average' 20 | - 'nearest_upsample' 21 | - 'simple_upsample' 22 | - 'resnetb_upsample' 23 | 2. num_kernel_points: This parameter Highly influences the accuracy of the model as well as the complexity of the model. Correct tuning is required and this parameter should be an odd number 24 | 3. first_subsampling_dl: This parameter reduces the input cloud into a cloud with a min distance between points with the value provided for the variable. 25 | 4. in_radius: This is the radius of the sphere which is taken in for each iteration of the convolution. 26 | 5. density_parameter: Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent. 27 | 6. KP_influence: This sets the behavior of the KPConv. Acceptable parameters are : 'linear','constant' and 'gaussian'. 28 | 7. convolution_mode: This sets another behavior of the KPConv. Acceptable parameters are: 'closest' and 'sum'. 29 | 8. batch_num: Batch number for training. 30 | 9. learning_rate: Learning rate of the network. 31 | 10. Augmentation of the input: This is another crucial way to improve the model. This includes many variables 32 | - augment_scale_anisotropic = True 33 | - augment_symmetries = [True, False, False] 34 | - augment_rotation = 'vertical' 35 | - augment_scale_min = 0.9 36 | - augment_scale_max = 1.1 37 | - augment_noise = 0.01 38 | - augment_occlusion = 'none' 39 | - augment_color = 1.0 40 | 41 | ## What I tried 42 | 43 | - The architecture, to include Squeeze and Excite which did not provide a better result 44 | - The architecture, reducing the number of layers decreased accuracy tremendously, thus 5 is the minimum number of layers for the dataset (one layer means the start of the model to a strided layer, the exception being the first layer). 45 | - Radius of KPConv, More radius gained more accuracy (20 was optimum) 46 | - density_parameter, no significant change was observed 47 | - first_sabsampling_dl, there was variations and proper tuning is needed 48 | - num_kernel_points, increasing was not possible due to Graphics card limits. 49 | - KP_influence, tried gaussian but no good change was observed. 50 | - convolution_mode, changed to closest but no improvement was observed. 51 | -------------------------------------------------------------------------------- /cpp_wrappers/cpp_utils/cloud/cloud.h: -------------------------------------------------------------------------------- 1 | // 2 | // 3 | // 0==========================0 4 | // | Local feature test | 5 | // 0==========================0 6 | // 7 | // version 1.0 : 8 | // > 9 | // 10 | //--------------------------------------------------- 11 | // 12 | // Cloud header 13 | // 14 | //---------------------------------------------------- 15 | // 16 | // Hugues THOMAS - 10/02/2017 17 | // 18 | 19 | 20 | # pragma once 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include 32 | 33 | 34 | 35 | 36 | // Point class 37 | // *********** 38 | 39 | 40 | class PointXYZ 41 | { 42 | public: 43 | 44 | // Elements 45 | // ******** 46 | 47 | float x, y, z; 48 | 49 | 50 | // Methods 51 | // ******* 52 | 53 | // Constructor 54 | PointXYZ() { x = 0; y = 0; z = 0; } 55 | PointXYZ(float x0, float y0, float z0) { x = x0; y = y0; z = z0; } 56 | 57 | // array type accessor 58 | float operator [] (int i) const 59 | { 60 | if (i == 0) return x; 61 | else if (i == 1) return y; 62 | else return z; 63 | } 64 | 65 | // opperations 66 | float dot(const PointXYZ P) const 67 | { 68 | return x * P.x + y * P.y + z * P.z; 69 | } 70 | 71 | float sq_norm() 72 | { 73 | return x*x + y*y + z*z; 74 | } 75 | 76 | PointXYZ cross(const PointXYZ P) const 77 | { 78 | return PointXYZ(y*P.z - z*P.y, z*P.x - x*P.z, x*P.y - y*P.x); 79 | } 80 | 81 | PointXYZ& operator+=(const PointXYZ& P) 82 | { 83 | x += P.x; 84 | y += P.y; 85 | z += P.z; 86 | return *this; 87 | } 88 | 89 | PointXYZ& operator-=(const PointXYZ& P) 90 | { 91 | x -= P.x; 92 | y -= P.y; 93 | z -= P.z; 94 | return *this; 95 | } 96 | 97 | PointXYZ& operator*=(const float& a) 98 | { 99 | x *= a; 100 | y *= a; 101 | z *= a; 102 | return *this; 103 | } 104 | }; 105 | 106 | 107 | // Point Opperations 108 | // ***************** 109 | 110 | inline PointXYZ operator + (const PointXYZ A, const PointXYZ B) 111 | { 112 | return PointXYZ(A.x + B.x, A.y + B.y, A.z + B.z); 113 | } 114 | 115 | inline PointXYZ operator - (const PointXYZ A, const PointXYZ B) 116 | { 117 | return PointXYZ(A.x - B.x, A.y - B.y, A.z - B.z); 118 | } 119 | 120 | inline PointXYZ operator * (const PointXYZ P, const float a) 121 | { 122 | return PointXYZ(P.x * a, P.y * a, P.z * a); 123 | } 124 | 125 | inline PointXYZ operator * (const float a, const PointXYZ P) 126 | { 127 | return PointXYZ(P.x * a, P.y * a, P.z * a); 128 | } 129 | 130 | inline std::ostream& operator << (std::ostream& os, const PointXYZ P) 131 | { 132 | return os << "[" << P.x << ", " << P.y << ", " << P.z << "]"; 133 | } 134 | 135 | inline bool operator == (const PointXYZ A, const PointXYZ B) 136 | { 137 | return A.x == B.x && A.y == B.y && A.z == B.z; 138 | } 139 | 140 | inline PointXYZ floor(const PointXYZ P) 141 | { 142 | return PointXYZ(std::floor(P.x), std::floor(P.y), std::floor(P.z)); 143 | } 144 | 145 | 146 | PointXYZ max_point(std::vector points); 147 | PointXYZ min_point(std::vector points); 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | -------------------------------------------------------------------------------- /tf_custom_ops/tf_subsampling/tf_subsampling.cpp: -------------------------------------------------------------------------------- 1 | #include "tensorflow/core/framework/op.h" 2 | #include "tensorflow/core/framework/shape_inference.h" 3 | #include "tensorflow/core/framework/op_kernel.h" 4 | #include "grid_subsampling/grid_subsampling.h" 5 | 6 | using namespace tensorflow; 7 | 8 | REGISTER_OP("GridSubsampling") 9 | .Input("points: float") 10 | .Input("dl: float") 11 | .Output("sub_points: float") 12 | .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) { 13 | ::tensorflow::shape_inference::ShapeHandle input; 14 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input)); 15 | c->set_output(0, input); 16 | return Status::OK(); 17 | }); 18 | 19 | 20 | 21 | 22 | 23 | class GridSubsamplingOp : public OpKernel { 24 | public: 25 | explicit GridSubsamplingOp(OpKernelConstruction* context) : OpKernel(context) {} 26 | 27 | void Compute(OpKernelContext* context) override 28 | { 29 | 30 | // Grab the input tensors 31 | const Tensor& points_tensor = context->input(0); 32 | const Tensor& dl_tensor = context->input(1); 33 | 34 | // check shapes of input and weights 35 | const TensorShape& points_shape = points_tensor.shape(); 36 | 37 | // check input are [N x 3] matrices 38 | DCHECK_EQ(points_shape.dims(), 2); 39 | DCHECK_EQ(points_shape.dim_size(1), 3); 40 | 41 | // Dimensions 42 | int N = (int)points_shape.dim_size(0); 43 | 44 | // get the data as std vector of points 45 | float sampleDl = dl_tensor.flat().data()[0]; 46 | vector original_points = vector((PointXYZ*)points_tensor.flat().data(), 47 | (PointXYZ*)points_tensor.flat().data() + N); 48 | 49 | // Unsupported label and features 50 | vector original_features; 51 | vector original_classes; 52 | 53 | // Create result containers 54 | vector subsampled_points; 55 | vector subsampled_features; 56 | vector subsampled_classes; 57 | 58 | // Compute results 59 | grid_subsampling(original_points, 60 | subsampled_points, 61 | original_features, 62 | subsampled_features, 63 | original_classes, 64 | subsampled_classes, 65 | sampleDl); 66 | 67 | // create output shape 68 | TensorShape output_shape; 69 | output_shape.AddDim(subsampled_points.size()); 70 | output_shape.AddDim(3); 71 | 72 | // create output tensor 73 | Tensor* output = NULL; 74 | OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); 75 | auto output_tensor = output->matrix(); 76 | 77 | // Fill output tensor 78 | for (int i = 0; i < output->shape().dim_size(0); i++) 79 | { 80 | output_tensor(i, 0) = subsampled_points[i].x; 81 | output_tensor(i, 1) = subsampled_points[i].y; 82 | output_tensor(i, 2) = subsampled_points[i].z; 83 | } 84 | } 85 | }; 86 | 87 | 88 | REGISTER_KERNEL_BUILDER(Name("GridSubsampling").Device(DEVICE_CPU), GridSubsamplingOp); -------------------------------------------------------------------------------- /tf_custom_ops/tf_neighbors/tf_neighbors.cpp: -------------------------------------------------------------------------------- 1 | #include "tensorflow/core/framework/op.h" 2 | #include "tensorflow/core/framework/shape_inference.h" 3 | #include "tensorflow/core/framework/op_kernel.h" 4 | #include "neighbors/neighbors.h" 5 | 6 | using namespace tensorflow; 7 | 8 | REGISTER_OP("OrderedNeighbors") 9 | .Input("queries: float") 10 | .Input("supports: float") 11 | .Input("radius: float") 12 | .Output("neighbors: int32") 13 | .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) { 14 | ::tensorflow::shape_inference::ShapeHandle input; 15 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input)); 16 | c->set_output(0, input); 17 | return Status::OK(); 18 | }); 19 | 20 | 21 | 22 | 23 | 24 | class OrderedNeighborsOp : public OpKernel { 25 | public: 26 | explicit OrderedNeighborsOp(OpKernelConstruction* context) : OpKernel(context) {} 27 | 28 | void Compute(OpKernelContext* context) override 29 | { 30 | 31 | // Grab the input tensors 32 | const Tensor& queries_tensor = context->input(0); 33 | const Tensor& supports_tensor = context->input(1); 34 | const Tensor& radius_tensor = context->input(2); 35 | 36 | // check shapes of input and weights 37 | const TensorShape& queries_shape = queries_tensor.shape(); 38 | const TensorShape& supports_shape = supports_tensor.shape(); 39 | 40 | // check input are [N x 3] matrices 41 | DCHECK_EQ(queries_shape.dims(), 2); 42 | DCHECK_EQ(queries_shape.dim_size(1), 3); 43 | DCHECK_EQ(supports_shape.dims(), 2); 44 | DCHECK_EQ(supports_shape.dim_size(1), 3); 45 | 46 | // Dimensions 47 | int Nq = (int)queries_shape.dim_size(0); 48 | int Ns = (int)supports_shape.dim_size(0); 49 | 50 | // get the data as std vector of points 51 | float radius = radius_tensor.flat().data()[0]; 52 | vector queries = vector((PointXYZ*)queries_tensor.flat().data(), 53 | (PointXYZ*)queries_tensor.flat().data() + Nq); 54 | 55 | vector supports = vector((PointXYZ*)supports_tensor.flat().data(), 56 | (PointXYZ*)supports_tensor.flat().data() + Ns); 57 | 58 | // Create result containers 59 | vector neighbors_indices; 60 | 61 | // Compute results 62 | ordered_neighbors(queries, supports, neighbors_indices, radius); 63 | 64 | // Maximal number of neighbors 65 | int max_neighbors = neighbors_indices.size() / Nq; 66 | 67 | // create output shape 68 | TensorShape output_shape; 69 | output_shape.AddDim(Nq); 70 | output_shape.AddDim(max_neighbors); 71 | 72 | // create output tensor 73 | Tensor* output = NULL; 74 | OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); 75 | auto output_tensor = output->matrix(); 76 | 77 | // Fill output tensor 78 | for (int i = 0; i < output->shape().dim_size(0); i++) 79 | { 80 | for (int j = 0; j < output->shape().dim_size(1); j++) 81 | { 82 | output_tensor(i, j) = neighbors_indices[max_neighbors * i + j]; 83 | } 84 | } 85 | } 86 | }; 87 | 88 | 89 | REGISTER_KERNEL_BUILDER(Name("OrderedNeighbors").Device(DEVICE_CPU), OrderedNeighborsOp); -------------------------------------------------------------------------------- /aerotronic.yml: -------------------------------------------------------------------------------- 1 | name: aerotronic 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - _libgcc_mutex=0.1=main 7 | - _tflow_select=2.1.0=gpu 8 | - absl-py=0.9.0=py37_0 9 | - astor=0.8.0=py37_0 10 | - blas=1.0=mkl 11 | - c-ares=1.15.0=h7b6447c_1001 12 | - ca-certificates=2020.11.8=ha878542_0 13 | - certifi=2020.11.8=py37h89c1867_0 14 | - cudatoolkit=9.0=h13b8566_0 15 | - cudnn=7.6.5=cuda9.0_0 16 | - cupti=9.0.176=0 17 | - cycler=0.10.0=py_2 18 | - dbus=1.13.6=he372182_0 19 | - expat=2.2.9=he1b5a44_2 20 | - fontconfig=2.13.1=he4413a7_1000 21 | - freetype=2.10.4=h7ca028e_0 22 | - gast=0.2.2=py37_0 23 | - glib=2.66.1=h92f7085_0 24 | - google-pasta=0.2.0=py_0 25 | - grpcio=1.27.2=py37hf8bcb03_0 26 | - gst-plugins-base=1.14.0=hbbd80ab_1 27 | - gstreamer=1.14.0=hb31296c_0 28 | - h5py=2.10.0=py37h7918eee_0 29 | - hdf5=1.10.4=hb1b8bf9_0 30 | - icu=58.2=hf484d3e_1000 31 | - intel-openmp=2020.1=217 32 | - joblib=0.17.0=py_0 33 | - jpeg=9d=h36c2ea0_0 34 | - keras-applications=1.0.8=py_0 35 | - keras-base=2.3.1=py37_0 36 | - keras-gpu=2.3.1=0 37 | - keras-preprocessing=1.1.0=py_1 38 | - kiwisolver=1.3.1=py37hc928c03_0 39 | - lcms2=2.11=hcbb858e_1 40 | - ld_impl_linux-64=2.33.1=h53a641e_7 41 | - libblas=3.8.0=21_mkl 42 | - libcblas=3.8.0=21_mkl 43 | - libedit=3.1.20181209=hc058e9b_0 44 | - libffi=3.3=he6710b0_1 45 | - libgcc-ng=9.1.0=hdf63c60_0 46 | - libgfortran-ng=7.3.0=hdf63c60_0 47 | - libpng=1.6.37=h21135ba_2 48 | - libprotobuf=3.12.3=hd408876_0 49 | - libstdcxx-ng=9.1.0=hdf63c60_0 50 | - libtiff=4.1.0=h4f3a223_6 51 | - libuuid=2.32.1=h14c3975_1000 52 | - libwebp-base=1.1.0=h36c2ea0_3 53 | - libxcb=1.13=h14c3975_1002 54 | - libxml2=2.9.10=hb55368b_3 55 | - lz4-c=1.9.2=he1b5a44_3 56 | - markdown=3.1.1=py37_0 57 | - matplotlib=3.3.2=0 58 | - matplotlib-base=3.3.2=py37h817c723_0 59 | - mkl=2020.1=217 60 | - mkl-service=2.3.0=py37he904b0f_0 61 | - mkl_fft=1.1.0=py37h23d657b_0 62 | - mkl_random=1.1.1=py37h0573a6f_0 63 | - ncurses=6.2=he6710b0_1 64 | - numpy=1.18.1=py37h4f9e942_0 65 | - numpy-base=1.18.1=py37hde5b4d6_1 66 | - olefile=0.46=pyh9f0ad1d_1 67 | - openssl=1.1.1h=h516909a_0 68 | - opt_einsum=3.1.0=py_0 69 | - pcre=8.44=he1b5a44_0 70 | - pillow=8.0.1=py37he98fc37_0 71 | - pip=20.1.1=py37_1 72 | - protobuf=3.12.3=py37he6710b0_0 73 | - psutil=5.7.3=py37hb5d75c8_0 74 | - pthread-stubs=0.4=h14c3975_1001 75 | - pyparsing=2.4.7=pyh9f0ad1d_0 76 | - pyqt=5.9.2=py37hcca6a23_4 77 | - python=3.7.7=hcff3b4d_5 78 | - python-dateutil=2.8.1=py_0 79 | - python_abi=3.7=1_cp37m 80 | - pyyaml=5.3.1=py37h7b6447c_0 81 | - qt=5.9.7=h5867ecd_1 82 | - readline=8.0=h7b6447c_0 83 | - scikit-learn=0.23.2=py37hddcf8d6_2 84 | - scipy=1.4.1=py37h0b6359f_0 85 | - setuptools=47.3.0=py37_0 86 | - sip=4.19.8=py37hf484d3e_0 87 | - six=1.15.0=py_0 88 | - sqlite=3.31.1=h62c20be_1 89 | - tensorboard=1.14.0=py37hf484d3e_0 90 | - tensorflow=1.14.0=gpu_py37hae64822_0 91 | - tensorflow-base=1.14.0=gpu_py37h8f37b9b_0 92 | - tensorflow-estimator=1.14.0=py_0 93 | - tensorflow-gpu=1.14.0=h0d30ee6_0 94 | - termcolor=1.1.0=py37_1 95 | - threadpoolctl=2.1.0=pyh5ca1d4c_0 96 | - tk=8.6.8=hbc83047_0 97 | - tornado=6.1=py37h4abf009_0 98 | - webencodings=0.5.1=py37_1 99 | - werkzeug=0.16.1=py_0 100 | - wheel=0.34.2=py37_0 101 | - wrapt=1.12.1=py37h7b6447c_1 102 | - xorg-libxau=1.0.9=h14c3975_0 103 | - xorg-libxdmcp=1.1.3=h516909a_0 104 | - xz=5.2.5=h7b6447c_0 105 | - yaml=0.1.7=had09818_2 106 | - zlib=1.2.11=h7b6447c_3 107 | - zstd=1.4.5=h6597ccf_2 108 | prefix: /home/ubuntu/anaconda3/envs/aerotronic 109 | 110 | -------------------------------------------------------------------------------- /Instruction_Manual/Running test and train.md: -------------------------------------------------------------------------------- 1 | # Running Test and Train in the instance 2 | (Please use this page only after completing the setup of the Instance as per the Launch and Setup guide.) 3 | 4 | This page is a guide to train and test of KPConv Model described as the following architecture: 5 | 6 | ``` 7 | 'simple', 8 | 'resnetb', 9 | 'resnetb_strided', 10 | 'resnetb', 11 | 'resnetb_strided', 12 | 'resnetb_deformable', 13 | 'resnetb_deformable_strided', 14 | 'resnetb_deformable', 15 | 'resnetb_deformable_strided', 16 | 'resnetb_deformable', 17 | 'nearest_upsample', 18 | 'unary', 19 | 'nearest_upsample', 20 | 'unary', 21 | 'nearest_upsample', 22 | 'unary', 23 | 'nearest_upsample', 24 | 'unary' 25 | ``` 26 | 27 | ## 1. Training the Model 28 | 29 | Training the model takes around 32 GB RAM and required 16 GB Graphics for the Dales dataset (This requirement is dataset dependent). 30 | Following are the steps: 31 | (If the dataset have difference in the classes compared to DALES, change it in the `dataset/DALES.py ` at line #97) 32 | 1. Connect to the ssh terminal. 33 | 2. Utilise tmux to create a new session using the following command 34 | ` tmux attach ` 35 | 3. Run the following commands: 36 | ``` 37 | cd {Location of the KPConv}/ 38 | conda activate aerotronic 39 | python -u training_DALES.py > filename.txt 40 | ``` 41 | filename.txt will contain the output log of the training. 42 | 4. Detach from the tmux session using the following 43 | * Press Ctrl-A and then press D or (if this doesn't work) 44 | * Press Ctrl-B and then press D 45 | 5. Now the session is running and the current ssh session can be safely closed. 46 | 47 | ## 2. Testing the trained model 48 | 49 | Testing the model takes around 16 GB RAM and required 16 GB Graphics for the Dales dataset (This requirement is dataset dependent). 50 | Following are the steps: 51 | 52 | 1. Edit the `test_any_model.py` file at line #235 as per the training log of your choice 53 | ` chosen_log = 'Log_YYYY-MM-DD_HH-MM-SS' ` 54 | This sets the testing script to look for the trained model in 'results/Log_YYYY-MM-DD_HH-MM-SS' 55 | 2. Connect to the ssh terminal. 56 | 3. Utilise tmux to create a new session using the following command 57 | ` tmux attach ` 58 | 4. Run the following commands: 59 | ``` 60 | cd {Location of the KPConv}/ 61 | conda activate aerotronic 62 | python -u test_any_model.py > output_filename.txt 63 | ``` 64 | output_filename.txt will contain the output log of the training. 65 | 4. Detach from the tmux session using the following 66 | * Press Ctrl-A and then press D or (if this doesn't work) 67 | * Press Ctrl-B and then press D 68 | 5. The session is running. The current ssh session can be safely closed. 69 | 70 | ## 3. Expected running times for test and train. 71 | 72 | Training is expected to take 3 Days. It can be stopped at any time by using the `Ctrl-C` break. 73 | 74 | Testing is expected to take a maximum of 1 hour. (The default number of votes is 100, this may lead to longer test time> 2Hr) To reduce the running time you can edit 'test_any_model.py' at line #197 75 | to 76 | `tester.test_cloud_segmentation(model, dataset, num_votes=1)` 77 | 78 | ## 4. Model Artifacts and Results 79 | 80 | Training model artifacts are stored in `results/Log_YYYY-MM-DD_HH-MM-SS` 81 | 82 | Testing results are stored at `test/Log_YYYY-MM-DD_HH-MM-SS/predictions` 83 | 84 | To check the accuracy of the testing `test_accuracy.py` can be utilized. Edit line #19 to the model predicted files which are by default in `test/Log_YYYY-MM-DD_HH-MM-SS/predictions`. `test_accuracy.py` 85 | compares the files from the prediction files to ground truth files which are stored in `Data/test_bin/` 86 | -------------------------------------------------------------------------------- /doc/visualization_guide.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Visualize learned features 4 | 5 | ### Intructions 6 | 7 | In order to visualize features you need a dataset and a pretrained model. You can use one of our pretrained models 8 | provided in the [pretrained models guide](./pretrained_models_guide.md), and the corresponding dataset. 9 | 10 | To start this visualization run the script: 11 | 12 | python3 visualize_features.py 13 | 14 | ### Details 15 | 16 | The visualization script has to main parts, separated in two different methods of the visualizer class in 17 | `visualizer.py`. 18 | 19 | * In the first part, implemented in the method `top_relu_activations`, the script runs the model on test examples 20 | (forward pass). At the chosen Relu layer, you have N output features that are going to be visualized. For each feature, 21 | the script keeps the top 5 examples that activated it the most, and saves them in a `visu` folder. 22 | 23 | * In the second part, implemented in the method `top_relu_activations`, the script just shows the saved examples for 24 | each feature with the level of activation as color. You can navigate through examples with keys 'g' and 'h'. 25 | 26 | N.B. This second part of the code can be started without doing the first part again if the top examples have already 27 | been computed. See details in the code. Alternatively you can visualize the saved example with a point cloud software 28 | like CloudCompare. 29 | 30 | 31 | ## Visualize kernel deformations 32 | 33 | ### Intructions 34 | 35 | In order to visualize features you need a dataset and a pretrained model that uses deformable KPConv. You can use our 36 | NPM3D pretrained model provided in the [pretrained models guide](./pretrained_models_guide.md). 37 | 38 | To start this visualization run the script: 39 | 40 | python3 visualize_deformations.py 41 | 42 | ### Details 43 | 44 | The visualization script runs the model runs the model on a batch of test examples (forward pass), and then show these 45 | examples in an interactive window. Here is a list of all keyborad shortcuts: 46 | 47 | - 'b' / 'n': smaller or larger point size. 48 | - 'g' / 'h': previous or next example in current batch. 49 | - 'k': switch between the rigid kenrel (original kernel points positions) and the deformed kernel (position of the 50 | kernel points after shift are applied) 51 | - 'z': Switch between the points displayed (input points, current layer points or both). 52 | - '0': Saves the example and deformed kernel as ply files. 53 | - mouse left click: select a point and show kernel at its location. 54 | - exit window: compute next batch. 55 | 56 | 57 | ## visualize Effective Receptive Fields 58 | 59 | ### Intructions 60 | 61 | In order to visualize features you need a dataset and a pretrained model. You can use one of our pretrained models 62 | provided in the [pretrained models guide](./pretrained_models_guide.md), and the corresponding dataset. 63 | 64 | To start this visualization run the script: 65 | 66 | python3 visualize_ERFs.py 67 | 68 | **Warning: This cript currently only works on the following datasets: NPM3D, Semantic3D, S3DIS, Scannet** 69 | 70 | ### Details 71 | 72 | The visualization script show the Effective receptive fields of a network layer at one location. If you chose another 73 | location (with left click), it has to rerun the model on the whole input point cloud to get new gradient values. Here a 74 | list of all keyborad shortcuts: 75 | 76 | - 'b' / 'n': smaller or larger point size. 77 | - 'g' / 'h': lower or higher ceiling limit. A functionality that remove points from the ceiling. Very handy for indoor 78 | point clouds. 79 | - 'z': Switch between the points displayed (input points, current layer points or both). 80 | - 'x': Go to the next input point cloud. 81 | - '0': Saves the input point cloud with ERF values and the center point used as origin of the ERF. 82 | - mouse left click: select a point and show ERF at its location. 83 | - exit window: End script. -------------------------------------------------------------------------------- /cpp_wrappers/cpp_subsampling/grid_subsampling/grid_subsampling.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "grid_subsampling.h" 3 | 4 | 5 | void grid_subsampling(vector& original_points, 6 | vector& subsampled_points, 7 | vector& original_features, 8 | vector& subsampled_features, 9 | vector& original_classes, 10 | vector& subsampled_classes, 11 | float sampleDl, 12 | int verbose) { 13 | 14 | // Initiate variables 15 | // ****************** 16 | 17 | // Number of points in the cloud 18 | size_t N = original_points.size(); 19 | 20 | // Dimension of the features 21 | size_t fdim = original_features.size() / N; 22 | size_t ldim = original_classes.size() / N; 23 | 24 | // Limits of the cloud 25 | PointXYZ minCorner = min_point(original_points); 26 | PointXYZ maxCorner = max_point(original_points); 27 | PointXYZ originCorner = floor(minCorner * (1/sampleDl)) * sampleDl; 28 | 29 | // Dimensions of the grid 30 | size_t sampleNX = (size_t)floor((maxCorner.x - originCorner.x) / sampleDl) + 1; 31 | size_t sampleNY = (size_t)floor((maxCorner.y - originCorner.y) / sampleDl) + 1; 32 | //size_t sampleNZ = (size_t)floor((maxCorner.z - originCorner.z) / sampleDl) + 1; 33 | 34 | // Check if features and classes need to be processed 35 | bool use_feature = original_features.size() > 0; 36 | bool use_classes = original_classes.size() > 0; 37 | 38 | 39 | // Create the sampled map 40 | // ********************** 41 | 42 | // Verbose parameters 43 | int i = 0; 44 | int nDisp = N / 100; 45 | 46 | // Initiate variables 47 | size_t iX, iY, iZ, mapIdx; 48 | unordered_map data; 49 | 50 | for (auto& p : original_points) 51 | { 52 | // Position of point in sample map 53 | iX = (size_t)floor((p.x - originCorner.x) / sampleDl); 54 | iY = (size_t)floor((p.y - originCorner.y) / sampleDl); 55 | iZ = (size_t)floor((p.z - originCorner.z) / sampleDl); 56 | mapIdx = iX + sampleNX*iY + sampleNX*sampleNY*iZ; 57 | 58 | // If not already created, create key 59 | if (data.count(mapIdx) < 1) 60 | data.emplace(mapIdx, SampledData(fdim, ldim)); 61 | 62 | // Fill the sample map 63 | if (use_feature && use_classes) 64 | data[mapIdx].update_all(p, original_features.begin() + i * fdim, original_classes.begin() + i * ldim); 65 | else if (use_feature) 66 | data[mapIdx].update_features(p, original_features.begin() + i * fdim); 67 | else if (use_classes) 68 | data[mapIdx].update_classes(p, original_classes.begin() + i * ldim); 69 | else 70 | data[mapIdx].update_points(p); 71 | 72 | // Display 73 | i++; 74 | if (verbose > 1 && i%nDisp == 0) 75 | std::cout << "\rSampled Map : " << std::setw(3) << i / nDisp << "%"; 76 | 77 | } 78 | 79 | // Divide for barycentre and transfer to a vector 80 | subsampled_points.reserve(data.size()); 81 | if (use_feature) 82 | subsampled_features.reserve(data.size() * fdim); 83 | if (use_classes) 84 | subsampled_classes.reserve(data.size() * ldim); 85 | for (auto& v : data) 86 | { 87 | subsampled_points.push_back(v.second.point * (1.0 / v.second.count)); 88 | if (use_feature) 89 | { 90 | float count = (float)v.second.count; 91 | transform(v.second.features.begin(), 92 | v.second.features.end(), 93 | v.second.features.begin(), 94 | [count](float f) { return f / count;}); 95 | subsampled_features.insert(subsampled_features.end(),v.second.features.begin(),v.second.features.end()); 96 | } 97 | if (use_classes) 98 | { 99 | for (int i = 0; i < ldim; i++) 100 | subsampled_classes.push_back(max_element(v.second.labels[i].begin(), v.second.labels[i].end(), 101 | [](const pair&a, const pair&b){return a.second < b.second;})->first); 102 | } 103 | } 104 | 105 | return; 106 | } 107 | -------------------------------------------------------------------------------- /tf_custom_ops/cpp_utils/cloud/cloud.h: -------------------------------------------------------------------------------- 1 | // 2 | // 3 | // 0==========================0 4 | // | Local feature test | 5 | // 0==========================0 6 | // 7 | // version 1.0 : 8 | // > 9 | // 10 | //--------------------------------------------------- 11 | // 12 | // Cloud header 13 | // 14 | //---------------------------------------------------- 15 | // 16 | // Hugues THOMAS - 10/02/2017 17 | // 18 | 19 | 20 | # pragma once 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include 32 | 33 | 34 | 35 | 36 | // Point class 37 | // *********** 38 | 39 | 40 | class PointXYZ 41 | { 42 | public: 43 | 44 | // Elements 45 | // ******** 46 | 47 | float x, y, z; 48 | 49 | 50 | // Methods 51 | // ******* 52 | 53 | // Constructor 54 | PointXYZ() { x = 0; y = 0; z = 0; } 55 | PointXYZ(float x0, float y0, float z0) { x = x0; y = y0; z = z0; } 56 | 57 | // array type accessor 58 | float operator [] (int i) const 59 | { 60 | if (i == 0) return x; 61 | else if (i == 1) return y; 62 | else return z; 63 | } 64 | 65 | // opperations 66 | float dot(const PointXYZ P) const 67 | { 68 | return x * P.x + y * P.y + z * P.z; 69 | } 70 | 71 | float sq_norm() 72 | { 73 | return x*x + y*y + z*z; 74 | } 75 | 76 | PointXYZ cross(const PointXYZ P) const 77 | { 78 | return PointXYZ(y*P.z - z*P.y, z*P.x - x*P.z, x*P.y - y*P.x); 79 | } 80 | 81 | PointXYZ& operator+=(const PointXYZ& P) 82 | { 83 | x += P.x; 84 | y += P.y; 85 | z += P.z; 86 | return *this; 87 | } 88 | 89 | PointXYZ& operator-=(const PointXYZ& P) 90 | { 91 | x -= P.x; 92 | y -= P.y; 93 | z -= P.z; 94 | return *this; 95 | } 96 | 97 | PointXYZ& operator*=(const float& a) 98 | { 99 | x *= a; 100 | y *= a; 101 | z *= a; 102 | return *this; 103 | } 104 | 105 | }; 106 | 107 | 108 | // Point Opperations 109 | // ***************** 110 | 111 | inline PointXYZ operator + (const PointXYZ A, const PointXYZ B) 112 | { 113 | return PointXYZ(A.x + B.x, A.y + B.y, A.z + B.z); 114 | } 115 | 116 | inline PointXYZ operator - (const PointXYZ A, const PointXYZ B) 117 | { 118 | return PointXYZ(A.x - B.x, A.y - B.y, A.z - B.z); 119 | } 120 | 121 | inline PointXYZ operator * (const PointXYZ P, const float a) 122 | { 123 | return PointXYZ(P.x * a, P.y * a, P.z * a); 124 | } 125 | 126 | inline PointXYZ operator * (const float a, const PointXYZ P) 127 | { 128 | return PointXYZ(P.x * a, P.y * a, P.z * a); 129 | } 130 | 131 | inline std::ostream& operator << (std::ostream& os, const PointXYZ P) 132 | { 133 | return os << "[" << P.x << ", " << P.y << ", " << P.z << "]"; 134 | } 135 | 136 | inline bool operator == (const PointXYZ A, const PointXYZ B) 137 | { 138 | return A.x == B.x && A.y == B.y && A.z == B.z; 139 | } 140 | 141 | inline PointXYZ floor(const PointXYZ P) 142 | { 143 | return PointXYZ(floor(P.x), floor(P.y), floor(P.z)); 144 | } 145 | 146 | 147 | PointXYZ max_point(std::vector points); 148 | PointXYZ min_point(std::vector points); 149 | 150 | 151 | struct PointCloud 152 | { 153 | 154 | std::vector pts; 155 | 156 | // Must return the number of data points 157 | inline size_t kdtree_get_point_count() const { return pts.size(); } 158 | 159 | // Returns the dim'th component of the idx'th point in the class: 160 | // Since this is inlined and the "dim" argument is typically an immediate value, the 161 | // "if/else's" are actually solved at compile time. 162 | inline float kdtree_get_pt(const size_t idx, const size_t dim) const 163 | { 164 | if (dim == 0) return pts[idx].x; 165 | else if (dim == 1) return pts[idx].y; 166 | else return pts[idx].z; 167 | } 168 | 169 | // Optional bounding-box computation: return false to default to a standard bbox computation loop. 170 | // Return true if the BBOX was already computed by the class and returned in "bb" so it can be avoided to redo it again. 171 | // Look at bb.size() to find out the expected dimensionality (e.g. 2 or 3 for point clouds) 172 | template 173 | bool kdtree_get_bbox(BBOX& /* bb */) const { return false; } 174 | 175 | }; 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | -------------------------------------------------------------------------------- /doc/new_dataset_guide.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Creating your own dataset 4 | 5 | ### Overview of the pipeline 6 | 7 | The training script initiates a bunch of variables and classes before starting the training on a dataset. Here are the 8 | initialization steps: 9 | 10 | * Create an instance of the `Config` class. This instance will hold all the parameters defining the network. 11 | 12 | * Create an instance of your dataset class. This instance will handle the data, and the input pipeline. **This is the 13 | class you have to implement to train our network on your own data**. 14 | 15 | * Load the input point cloud in memory. Most datasets will fit in a 32GB RAM computer. If you don't have enough memory 16 | for your dataset, you will have to redesign the input pipeline. 17 | 18 | * Initialize the tensorflow input pipeline, which is a `tf.dataset` object that will create and feed the input batches 19 | to the network. 20 | 21 | * Create an instance of the network model class. This class contains the tensorflow operations defining the network. 22 | 23 | * Create an instance of our generic `ModelTrainer` class. This class handles the training of the model 24 | 25 | Then the training can start. 26 | 27 | 28 | ### The dataset class 29 | 30 | This class has several roles. First this is where you define your dataset parameters (class names, data path, nature 31 | of the data...). Then this class will hold the point clouds loaded in memory. Eventually, it also defines the 32 | Tensorflow input pipeline. For efficiency, our implementation uses a parallel input queue, feeding batches to the 33 | network. 34 | 35 | Here we give you a description of each essential method that need to be implemented in your new dataset class. For more 36 | details, follow the implementation of the current datasets, which contains a lot of indications as comments. 37 | 38 | 39 | * The **\_\_init\_\_** method: Here you have to define the parameters of your dataset. Notice that your dataset class 40 | has to be a child of the common `Dataset` class, where generic methods are implemented. Their are a few thing that has 41 | to be defined here: 42 | - The labels: define a dictionary `self.label_to_names`, call the `self.init_labels()` method, and define which 43 | label should be ignored in `self.ignored_labels`. 44 | - The network model: the type of model that will be used on this dataset ("classification", "segmentation", 45 | "multi_segmentation" or "cloud_segmentation"). 46 | - The number of CPU threads used in the parallel input queue. 47 | - Data paths and splits: you can manage your data as you wish, these variables are only used in methods that you 48 | will implement, so you do not have to follow exactly the notations of the other dataset classes. 49 | 50 | 51 | * The **load_subsampled_clouds** method: Here you load your data in memory. Depending on your dataset (if this is a 52 | classification or segmentation task, 3D scenes or 3D models) you will not have to load the same variables. Just follow 53 | the implementation of the existing datasets. 54 | 55 | 56 | * The **get_batch_gen** method: This method should return a python generator. This will be the base generator for the 57 | `tf.dataset` object. It is called in the generic `self.init_input_pipeline` or `self.init_test_input_pipeline` methods. 58 | Along with the generator, it also has to return the generated types and shapes. You can redesign the generators or used 59 | the ones we implemented. The generator returns np.arrays, but from this point of the pipeline, they will be converted 60 | to tensorflow tensors. 61 | 62 | 63 | * The **get_tf_mapping** method: This method return a mapping function that takes the generated batches and creates all 64 | the variables for the network. Remember that from this point we are defining a tensorflow graph of operations. There is 65 | not much to implement here as most of the work is done by two generic function `self.tf_augment_input` and 66 | `self.tf_xxxxxxxxx_inputs` where xxxxxxxxx can be "classification" of "segmentation" depending on the task. The only 67 | important thing to do here is to define the features that will be fed to the network. 68 | 69 | 70 | ### The training script and configuration class 71 | 72 | In the training script you have to create a class that inherits from the `Config` class. This is where you will define 73 | all the network parameters by overwriting the attributes 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /Instruction_Manual/Launching and Setting up the instance.md: -------------------------------------------------------------------------------- 1 | # Launching and Setting up the instance 2 | 3 | This page is a guide to setup the instance with required configurations and connecting to it so as to create the environment required for the training and testing of KPConv 4 | 5 | ## 1. Recommended Instance Details 6 | 7 | ``` 8 | Instance Type : g4dn.2xlarge (for training); g4dn.xlarge (for testing) 9 | Instance OS : Deep Learning AMI (Ubuntu 18.04) Version 37.0 10 | Hard Disk : 150 GB SSD 11 | Hard Disk-2 : 500 GB HDD Magnetic 12 | ``` 13 | 14 | ## 2. Launching the instance 15 | 16 | On Amazon Console goto `EC2` from the `Services` menu in the top left corner.
17 | Create an Instance using the `Launch instance` button.
18 | * Steps to Launch Instance: 19 | 1. Choose AMI. Utilise the recommended Instance details to set this. 20 | 2. Choose Instance Type. Utilise the recommended Instance details to set this. 21 | 3. Configure Instance. In the `Advanced Details` tab, for `User data` select `As file` and use `choose file` to upload setup.sh 22 | 4. Add Storage. Utilise the recommended Instance details to set this. 23 | 5. Review and Launch. 24 | 25 | Save the key pair generated from the launch (let's call it aws_key.pem). 26 | 27 | 28 | (Error may occur due to dedicated host limit which can be increased by request).
29 | Choose a consistent region for your Amazon machines and plugs (Used US-east-2c which is the same as US Ohio). 30 | 31 | ## 3. Connecting to the instance 32 | 33 | Connecting is done via ssh. If using windows utilize 34 | 35 | * WinSCP 36 | * Putty 37 | 38 | Using putty : 39 | 40 | While creating the Instance you will get a *.pem(eg: aws_key.pem) file key. using this as the key for ssh connect using 41 | * For Linux : 42 | ` ssh -i "aws_key.pem" ubuntu@ec2-xxxxxx.compute.amazonaws.com ` 43 | 44 | * For windows : 45 | 1. If ssh works in your shell you can follow the same as the Linux method. 46 | 2. Use putty-gen to convert the key to .ppk format. Use [this link](https://www.puttygen.com/convert-pem-to-ppk) for more details. 47 | 3. After the conversion of the key to PPK format. Use "Pageant" software and add this key to it. 48 | 4. You will be able to connect with putty now, using the 'ubuntu@ec2-xxxxxx.compute.amazonaws.com' as the hostname. 49 | >**Host Name** 50 | If the hostname is unknown you can log in to the Amazon management console and choose EC2 from services which will lead you to EC2 Dashboard. 51 | In this dashboard, you can choose the running instance and click on 'connect' on the top right side. 52 | The 'SSH Client' tab will show you the methods to connect along with your actual hostname. 53 | ## 4. File transfer to and from the Instance 54 | **In Linux** 55 | You can do file transfer from your local pc to the instance via SCP commands after the ssh connection is complete. 56 | **In Windows** 57 | You can use WinSCP for the same (more convenient). For more details use [this link](https://winscp.net/eng/docs/guide_public_key) 58 | 59 | ## 5. Configuring the instance for KPConv 60 | - After successfully logging into the instance. Transfer the setup.sh file to the {Home} location and run it.- 61 | - Mount the extra 500 GB HDD to the system using the guide [AWS_help.md](AWS_help.md) 62 | - Copy the files of data into a specific folder (Preferably in the 500GB HDD):(yet to make this change in code from ../Data/) 63 | Create the folder tree looks like this 64 | ``` 65 | ├── Data 66 | │ ├── test_bin 67 | │ └── train_bin 68 | │ 69 | └── KPConv_for_DALES 70 | ├── INSTALL.md 71 | ├── LICENSE 72 | ├── README.md 73 | ├── convert.py 74 | ├── cpp_wrappers 75 | ├── datasets 76 | ├── doc 77 | ├── envlist.txt 78 | ├── kernels 79 | ├── models 80 | ├── pipenvlist.txt 81 | ├── plot_convergence.py 82 | ├── results 83 | ├── test 84 | ├── test_accuracy.py 85 | ├── test_any_model.py 86 | ├── tf_custom_ops 87 | ├── training_DALES.py 88 | ├── training_ModelNet40.py 89 | ├── training_NPM3D.py 90 | ├── training_S3DIS.py 91 | ├── training_Scannet.py 92 | ├── training_Semantic3D.py 93 | ├── training_ShapeNetPart.py 94 | ├── utils 95 | ├── visualize_ERFs.py 96 | ├── visualize_deformations.py 97 | └── visualize_features.py 98 | ``` 99 | In this tree, test_bin and train_bin contain the files for test and train respectively in binary ply format. 100 | - If necessary to convert *' ASCII ply'* file format to *'binary ply'* format utilize **"convert.py".** by copying the **convert.py** to the location of the ASCII ply files and run it. 101 | -------------------------------------------------------------------------------- /tf_custom_ops/tf_subsampling/tf_batch_subsampling.cpp: -------------------------------------------------------------------------------- 1 | #include "tensorflow/core/framework/op.h" 2 | #include "tensorflow/core/framework/shape_inference.h" 3 | #include "tensorflow/core/framework/op_kernel.h" 4 | #include "grid_subsampling/grid_subsampling.h" 5 | 6 | using namespace tensorflow; 7 | 8 | REGISTER_OP("BatchGridSubsampling") 9 | .Input("points: float") 10 | .Input("batches: int32") 11 | .Input("dl: float") 12 | .Output("sub_points: float") 13 | .Output("sub_batches: int32") 14 | .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) { 15 | ::tensorflow::shape_inference::ShapeHandle input0_shape; 16 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input0_shape)); 17 | c->set_output(0, input0_shape); 18 | c->set_output(1, c->input(1)); 19 | return Status::OK(); 20 | }); 21 | 22 | 23 | 24 | 25 | 26 | class BatchGridSubsamplingOp : public OpKernel { 27 | public: 28 | explicit BatchGridSubsamplingOp(OpKernelConstruction* context) : OpKernel(context) {} 29 | 30 | void Compute(OpKernelContext* context) override 31 | { 32 | 33 | // Grab the input tensors 34 | const Tensor& points_tensor = context->input(0); 35 | const Tensor& batches_tensor = context->input(1); 36 | const Tensor& dl_tensor = context->input(2); 37 | 38 | // check shapes of input and weights 39 | const TensorShape& points_shape = points_tensor.shape(); 40 | const TensorShape& batches_shape = batches_tensor.shape(); 41 | 42 | // check input is a [N x 3] matrix 43 | DCHECK_EQ(points_shape.dims(), 2); 44 | DCHECK_EQ(points_shape.dim_size(1), 3); 45 | 46 | // Check that Batch lengths is a vector 47 | DCHECK_EQ(batches_shape.dims(), 1); 48 | 49 | // Dimensions 50 | int N = (int)points_shape.dim_size(0); 51 | 52 | // Number of batches 53 | int Nb = (int)batches_shape.dim_size(0); 54 | 55 | // get the data as std vector of points 56 | float sampleDl = dl_tensor.flat().data()[0]; 57 | vector original_points = vector((PointXYZ*)points_tensor.flat().data(), 58 | (PointXYZ*)points_tensor.flat().data() + N); 59 | 60 | // Batches lengths 61 | vector batches = vector((int*)batches_tensor.flat().data(), 62 | (int*)batches_tensor.flat().data() + Nb); 63 | 64 | // Unsupported label and features 65 | vector original_features; 66 | vector original_classes; 67 | 68 | // Create result containers 69 | vector subsampled_points; 70 | vector subsampled_features; 71 | vector subsampled_classes; 72 | vector subsampled_batches; 73 | 74 | // Compute results 75 | batch_grid_subsampling(original_points, 76 | subsampled_points, 77 | original_features, 78 | subsampled_features, 79 | original_classes, 80 | subsampled_classes, 81 | batches, 82 | subsampled_batches, 83 | sampleDl); 84 | 85 | // Sub_points output 86 | // ***************** 87 | 88 | // create output shape 89 | TensorShape sub_points_shape; 90 | sub_points_shape.AddDim(subsampled_points.size()); 91 | sub_points_shape.AddDim(3); 92 | 93 | // create output tensor 94 | Tensor* sub_points_output = NULL; 95 | OP_REQUIRES_OK(context, context->allocate_output(0, sub_points_shape, &sub_points_output)); 96 | auto sub_points_tensor = sub_points_output->matrix(); 97 | 98 | // Fill output tensor 99 | for (int i = 0; i < subsampled_points.size(); i++) 100 | { 101 | sub_points_tensor(i, 0) = subsampled_points[i].x; 102 | sub_points_tensor(i, 1) = subsampled_points[i].y; 103 | sub_points_tensor(i, 2) = subsampled_points[i].z; 104 | } 105 | 106 | // Batch length output 107 | // ******************* 108 | 109 | // create output shape 110 | TensorShape sub_batches_shape; 111 | sub_batches_shape.AddDim(subsampled_batches.size()); 112 | 113 | // create output tensor 114 | Tensor* sub_batches_output = NULL; 115 | OP_REQUIRES_OK(context, context->allocate_output(1, sub_batches_shape, &sub_batches_output)); 116 | auto sub_batches_tensor = sub_batches_output->flat(); 117 | 118 | // Fill output tensor 119 | for (int i = 0; i < subsampled_batches.size(); i++) 120 | sub_batches_tensor(i) = subsampled_batches[i]; 121 | 122 | } 123 | }; 124 | 125 | 126 | REGISTER_KERNEL_BUILDER(Name("BatchGridSubsampling").Device(DEVICE_CPU), BatchGridSubsamplingOp); -------------------------------------------------------------------------------- /tf_custom_ops/tf_neighbors/tf_batch_neighbors.cpp: -------------------------------------------------------------------------------- 1 | #include "tensorflow/core/framework/op.h" 2 | #include "tensorflow/core/framework/shape_inference.h" 3 | #include "tensorflow/core/framework/op_kernel.h" 4 | #include "neighbors/neighbors.h" 5 | 6 | using namespace tensorflow; 7 | 8 | REGISTER_OP("BatchOrderedNeighbors") 9 | .Input("queries: float") 10 | .Input("supports: float") 11 | .Input("q_batches: int32") 12 | .Input("s_batches: int32") 13 | .Input("radius: float") 14 | .Output("neighbors: int32") 15 | .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) { 16 | 17 | // Create input shape container 18 | ::tensorflow::shape_inference::ShapeHandle input; 19 | 20 | // Check inputs rank 21 | TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input)); 22 | TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 2, &input)); 23 | TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &input)); 24 | TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &input)); 25 | 26 | // Create the output shape 27 | c->set_output(0, c->UnknownShapeOfRank(2)); 28 | 29 | return Status::OK(); 30 | }); 31 | 32 | 33 | 34 | 35 | 36 | class BatchOrderedNeighborsOp : public OpKernel { 37 | public: 38 | explicit BatchOrderedNeighborsOp(OpKernelConstruction* context) : OpKernel(context) {} 39 | 40 | void Compute(OpKernelContext* context) override 41 | { 42 | 43 | // Grab the input tensors 44 | const Tensor& queries_tensor = context->input(0); 45 | const Tensor& supports_tensor = context->input(1); 46 | const Tensor& q_batches_tensor = context->input(2); 47 | const Tensor& s_batches_tensor = context->input(3); 48 | const Tensor& radius_tensor = context->input(4); 49 | 50 | // check shapes of input and weights 51 | const TensorShape& queries_shape = queries_tensor.shape(); 52 | const TensorShape& supports_shape = supports_tensor.shape(); 53 | const TensorShape& q_batches_shape = q_batches_tensor.shape(); 54 | const TensorShape& s_batches_shape = s_batches_tensor.shape(); 55 | 56 | // check input are [N x 3] matrices 57 | DCHECK_EQ(queries_shape.dims(), 2); 58 | DCHECK_EQ(queries_shape.dim_size(1), 3); 59 | DCHECK_EQ(supports_shape.dims(), 2); 60 | DCHECK_EQ(supports_shape.dim_size(1), 3); 61 | 62 | // Check that Batch lengths are vectors and same number of batch for both query and support 63 | DCHECK_EQ(q_batches_shape.dims(), 1); 64 | DCHECK_EQ(s_batches_shape.dims(), 1); 65 | DCHECK_EQ(q_batches_shape.dim_size(0), s_batches_shape.dim_size(0)); 66 | 67 | // Points Dimensions 68 | int Nq = (int)queries_shape.dim_size(0); 69 | int Ns = (int)supports_shape.dim_size(0); 70 | 71 | // Number of batches 72 | int Nb = (int)q_batches_shape.dim_size(0); 73 | 74 | // get the data as std vector of points 75 | float radius = radius_tensor.flat().data()[0]; 76 | vector queries = vector((PointXYZ*)queries_tensor.flat().data(), 77 | (PointXYZ*)queries_tensor.flat().data() + Nq); 78 | vector supports = vector((PointXYZ*)supports_tensor.flat().data(), 79 | (PointXYZ*)supports_tensor.flat().data() + Ns); 80 | 81 | // Batches lengths 82 | vector q_batches = vector((int*)q_batches_tensor.flat().data(), 83 | (int*)q_batches_tensor.flat().data() + Nb); 84 | vector s_batches = vector((int*)s_batches_tensor.flat().data(), 85 | (int*)s_batches_tensor.flat().data() + Nb); 86 | 87 | 88 | // Create result containers 89 | vector neighbors_indices; 90 | 91 | // Compute results 92 | //batch_ordered_neighbors(queries, supports, q_batches, s_batches, neighbors_indices, radius); 93 | batch_nanoflann_neighbors(queries, supports, q_batches, s_batches, neighbors_indices, radius); 94 | 95 | // Maximal number of neighbors 96 | int max_neighbors = neighbors_indices.size() / Nq; 97 | 98 | // create output shape 99 | TensorShape output_shape; 100 | output_shape.AddDim(Nq); 101 | output_shape.AddDim(max_neighbors); 102 | 103 | // create output tensor 104 | Tensor* output = NULL; 105 | OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); 106 | auto output_tensor = output->matrix(); 107 | 108 | // Fill output tensor 109 | for (int i = 0; i < output->shape().dim_size(0); i++) 110 | { 111 | for (int j = 0; j < output->shape().dim_size(1); j++) 112 | { 113 | output_tensor(i, j) = neighbors_indices[max_neighbors * i + j]; 114 | } 115 | } 116 | } 117 | }; 118 | 119 | 120 | REGISTER_KERNEL_BUILDER(Name("BatchOrderedNeighbors").Device(DEVICE_CPU), BatchOrderedNeighborsOp); -------------------------------------------------------------------------------- /tf_custom_ops/tf_subsampling/grid_subsampling/grid_subsampling.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "grid_subsampling.h" 3 | 4 | 5 | void grid_subsampling(vector& original_points, 6 | vector& subsampled_points, 7 | vector& original_features, 8 | vector& subsampled_features, 9 | vector& original_classes, 10 | vector& subsampled_classes, 11 | float sampleDl) 12 | { 13 | 14 | // Initiate variables 15 | // ****************** 16 | 17 | // Number of points in the cloud 18 | size_t N = original_points.size(); 19 | 20 | // Dimension of the features 21 | size_t fdim = original_features.size() / N; 22 | 23 | // Limits of the cloud 24 | PointXYZ minCorner = min_point(original_points); 25 | PointXYZ maxCorner = max_point(original_points); 26 | PointXYZ originCorner = floor(minCorner * (1/sampleDl)) * sampleDl; 27 | 28 | // Dimensions of the grid 29 | size_t sampleNX = (size_t)floor((maxCorner.x - originCorner.x) / sampleDl) + 1; 30 | size_t sampleNY = (size_t)floor((maxCorner.y - originCorner.y) / sampleDl) + 1; 31 | //size_t sampleNZ = (size_t)floor((maxCorner.z - originCorner.z) / sampleDl) + 1; 32 | 33 | // Check if features and classes need to be processed 34 | bool use_feature = original_features.size() > 0; 35 | bool use_classes = original_classes.size() > 0; 36 | 37 | 38 | // Create the sampled map 39 | // ********************** 40 | 41 | // Verbose parameters 42 | int i = 0; 43 | int nDisp = N / 100; 44 | 45 | // Initiate variables 46 | size_t iX, iY, iZ, mapIdx; 47 | unordered_map data; 48 | 49 | for (auto& p : original_points) 50 | { 51 | // Position of point in sample map 52 | iX = (size_t)floor((p.x - originCorner.x) / sampleDl); 53 | iY = (size_t)floor((p.y - originCorner.y) / sampleDl); 54 | iZ = (size_t)floor((p.z - originCorner.z) / sampleDl); 55 | mapIdx = iX + sampleNX*iY + sampleNX*sampleNY*iZ; 56 | 57 | // If not already created, create key 58 | if (data.count(mapIdx) < 1) 59 | data.emplace(mapIdx, SampledData(fdim)); 60 | 61 | // Fill the sample map 62 | if (use_feature && use_classes) 63 | data[mapIdx].update_all(p, original_features.begin() + i * fdim, original_classes[i]); 64 | else if (use_feature) 65 | data[mapIdx].update_features(p, original_features.begin() + i * fdim); 66 | else if (use_classes) 67 | data[mapIdx].update_classes(p, original_classes[i]); 68 | else 69 | data[mapIdx].update_points(p); 70 | 71 | // Display 72 | i++; 73 | } 74 | 75 | // Divide for barycentre and transfer to a vector 76 | subsampled_points.reserve(data.size()); 77 | if (use_feature) 78 | subsampled_features.reserve(data.size() * fdim); 79 | if (use_classes) 80 | subsampled_classes.reserve(data.size()); 81 | for (auto& v : data) 82 | { 83 | subsampled_points.push_back(v.second.point * (1.0 / v.second.count)); 84 | if (use_feature) 85 | { 86 | float count = (float)v.second.count; 87 | transform(v.second.features.begin(), 88 | v.second.features.end(), 89 | v.second.features.begin(), 90 | [count](float f) { return f / count;}); 91 | subsampled_features.insert(subsampled_features.end(),v.second.features.begin(),v.second.features.end()); 92 | } 93 | if (use_classes) 94 | subsampled_classes.push_back(max_element(v.second.labels.begin(), v.second.labels.end(), 95 | [](const pair&a, const pair&b){return a.second < b.second;})->first); 96 | } 97 | return; 98 | } 99 | 100 | 101 | 102 | void batch_grid_subsampling(vector& original_points, 103 | vector& subsampled_points, 104 | vector& original_features, 105 | vector& subsampled_features, 106 | vector& original_classes, 107 | vector& subsampled_classes, 108 | vector& original_batches, 109 | vector& subsampled_batches, 110 | float sampleDl) 111 | { 112 | // Initiate variables 113 | // ****************** 114 | 115 | int b = 0; 116 | int sum_b = 0; 117 | 118 | // Loop over batches 119 | // ***************** 120 | 121 | for (b = 0; b < original_batches.size(); b++) 122 | { 123 | // Extract batch points 124 | vector b_original_points = vector(original_points.begin () + sum_b, 125 | original_points.begin () + sum_b + original_batches[b]); 126 | 127 | // Create result containers 128 | vector b_subsampled_points; 129 | vector b_subsampled_features; 130 | vector b_subsampled_classes; 131 | 132 | // Compute subsampling on current batch 133 | grid_subsampling(b_original_points, 134 | b_subsampled_points, 135 | original_features, 136 | b_subsampled_features, 137 | original_classes, 138 | b_subsampled_classes, 139 | sampleDl); 140 | 141 | // Stack batches points 142 | subsampled_points.insert(subsampled_points.end(), b_subsampled_points.begin(), b_subsampled_points.end()); 143 | 144 | // Stack new batch lengths 145 | subsampled_batches.push_back(b_subsampled_points.size()); 146 | sum_b += original_batches[b]; 147 | } 148 | 149 | return; 150 | } -------------------------------------------------------------------------------- /utils/mesh.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0======================0 4 | # | Mesh utilities | 5 | # 0======================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # functions related to meshes 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 10/02/2017 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Basic libs 26 | import numpy as np 27 | import time 28 | 29 | 30 | # ---------------------------------------------------------------------------------------------------------------------- 31 | # 32 | # Functions 33 | # \***************/ 34 | # 35 | 36 | 37 | def rasterize_mesh(vertices, faces, dl, verbose=False): 38 | """ 39 | Creation of point cloud from mesh via rasterization. All models are rescaled to fit in a 1 meter radius sphere 40 | :param vertices: array of vertices 41 | :param faces: array of faces 42 | :param dl: parameter controlling density. Distance between each point 43 | :param verbose: display parameter 44 | :return: point cloud 45 | """ 46 | 47 | ###################################### 48 | # Eliminate useless faces and vertices 49 | ###################################### 50 | 51 | # 3D coordinates of faces 52 | faces3D = vertices[faces, :] 53 | sides = np.stack([faces3D[:, i, :] - faces3D[:, i - 1, :] for i in [2, 0, 1]], axis=1) 54 | 55 | # Indices of big enough faces 56 | keep_bool = np.min(np.linalg.norm(sides, axis=-1), axis=-1) > 1e-9 57 | faces = faces[keep_bool] 58 | 59 | ################################## 60 | # Place random points on each face 61 | ################################## 62 | 63 | # 3D coordinates of faces 64 | faces3D = vertices[faces, :] 65 | 66 | # Area of each face 67 | opposite_sides = np.stack([faces3D[:, i, :] - faces3D[:, i - 1, :] for i in [2, 0, 1]], axis=1) 68 | lengths = np.linalg.norm(opposite_sides, axis=-1) 69 | 70 | # Points for each face 71 | all_points = [] 72 | all_vert_inds = [] 73 | for face_verts, face, l, sides in zip(faces, faces3D, lengths, opposite_sides): 74 | 75 | # All points generated for this face 76 | face_points = [] 77 | 78 | # Safe check for null faces 79 | if np.min(l) < 1e-9: 80 | continue 81 | 82 | # Smallest faces, only place one point in the center 83 | if np.max(l) < dl: 84 | face_points.append(np.mean(face, axis=0)) 85 | continue 86 | 87 | # Chose indices so that A is the largest angle 88 | A_idx = np.argmax(l) 89 | B_idx = (A_idx + 1) % 3 90 | C_idx = (A_idx + 2) % 3 91 | i = -sides[B_idx] / l[B_idx] 92 | j = sides[C_idx] / l[C_idx] 93 | 94 | # Create a mesh grid of points along the two smallest sides 95 | s1 = (l[B_idx] % dl) / 2 96 | s2 = (l[C_idx] % dl) / 2 97 | x, y = np.meshgrid(np.arange(s1, l[B_idx], dl), np.arange(s2, l[C_idx], dl)) 98 | points = face[A_idx, :] + (np.expand_dims(x.ravel(), 1) * i + np.expand_dims(y.ravel(), 1) * j) 99 | points = points[x.ravel() / l[B_idx] + y.ravel() / l[C_idx] <= 1, :] 100 | face_points.append(points) 101 | 102 | # Add points on the three edges 103 | for edge_idx in range(3): 104 | i = sides[edge_idx] / l[edge_idx] 105 | A_idx = (edge_idx + 1) % 3 106 | s1 = (l[edge_idx] % dl) / 2 107 | x = np.arange(s1, l[edge_idx], dl) 108 | points = face[A_idx, :] + np.expand_dims(x.ravel(), 1) * i 109 | face_points.append(points) 110 | 111 | # Add vertices 112 | face_points.append(face) 113 | 114 | # Compute vertex indices 115 | dists = np.sum(np.square(np.expand_dims(np.vstack(face_points), 1) - face), axis=2) 116 | all_vert_inds.append(face_verts[np.argmin(dists, axis=1)]) 117 | 118 | # Save points and inds 119 | all_points += face_points 120 | 121 | return np.vstack(all_points).astype(np.float32), np.hstack(all_vert_inds) 122 | 123 | 124 | def cylinder_mesh(cylinder, precision=24): 125 | 126 | # Get parameters 127 | center = cylinder[:3] 128 | h = cylinder[3] 129 | r = cylinder[4] 130 | 131 | # Create vertices 132 | theta = 2.0 * np.pi / precision 133 | thetas = np.arange(precision) * theta 134 | circleX = r * np.cos(thetas) 135 | circleY = r * np.sin(thetas) 136 | top_vertices = np.vstack((circleX, circleY, circleY * 0 + h / 2)).T 137 | bottom_vertices = np.vstack((circleX, circleY, circleY * 0 - h / 2)).T 138 | vertices = np.array([[0, 0, h / 2], 139 | [0, 0, -h / 2]]) 140 | vertices = np.vstack((vertices, top_vertices, bottom_vertices)) 141 | vertices += center 142 | 143 | # Create faces 144 | top_faces = [[0, 2 + i, 2 + ((i + 1) % precision)] for i in range(precision)] 145 | bottom_faces = [[1, 2 + precision + i, 2 + precision + ((i + 1) % precision)] for i in range(precision)] 146 | side_faces1 = [[2 + i, 2 + precision + i, 2 + precision + ((i + 1) % precision)] for i in range(precision)] 147 | side_faces2 = [[2 + precision + ((i + 1) % precision), 2 + i, 2 + ((i + 1) % precision)] for i in range(precision)] 148 | faces = np.array(top_faces + bottom_faces + side_faces1 + side_faces2, dtype=np.int32) 149 | 150 | return vertices.astype(np.float32), faces 151 | -------------------------------------------------------------------------------- /utils/metrics.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Metric utility functions 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Basic libs 26 | import numpy as np 27 | 28 | 29 | # ---------------------------------------------------------------------------------------------------------------------- 30 | # 31 | # Utilities 32 | # \***************/ 33 | # 34 | 35 | 36 | def metrics(confusions, ignore_unclassified=False): 37 | """ 38 | Computes different metrics from confusion matrices. 39 | :param confusions: ([..., n_c, n_c] np.int32). Can be any dimension, the confusion matrices should be described by 40 | the last axes. n_c = number of classes 41 | :param ignore_unclassified: (bool). True if the the first class should be ignored in the results 42 | :return: ([..., n_c] np.float32) precision, recall, F1 score, IoU score 43 | """ 44 | 45 | # If the first class (often "unclassified") should be ignored, erase it from the confusion. 46 | if (ignore_unclassified): 47 | confusions[..., 0, :] = 0 48 | confusions[..., :, 0] = 0 49 | 50 | # Compute TP, FP, FN. This assume that the second to last axis counts the truths (like the first axis of a 51 | # confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix) 52 | TP = np.diagonal(confusions, axis1=-2, axis2=-1) 53 | TP_plus_FP = np.sum(confusions, axis=-1) 54 | TP_plus_FN = np.sum(confusions, axis=-2) 55 | 56 | # Compute precision and recall. This assume that the second to last axis counts the truths (like the first axis of 57 | # a confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix) 58 | PRE = TP / (TP_plus_FN + 1e-6) 59 | REC = TP / (TP_plus_FP + 1e-6) 60 | 61 | # Compute Accuracy 62 | ACC = np.sum(TP, axis=-1) / (np.sum(confusions, axis=(-2, -1)) + 1e-6) 63 | 64 | # Compute F1 score 65 | F1 = 2 * TP / (TP_plus_FP + TP_plus_FN + 1e-6) 66 | 67 | # Compute IoU 68 | IoU = F1 / (2 - F1) 69 | 70 | return PRE, REC, F1, IoU, ACC 71 | 72 | 73 | def smooth_metrics(confusions, smooth_n=0, ignore_unclassified=False): 74 | """ 75 | Computes different metrics from confusion matrices. Smoothed over a number of epochs. 76 | :param confusions: ([..., n_c, n_c] np.int32). Can be any dimension, the confusion matrices should be described by 77 | the last axes. n_c = number of classes 78 | :param smooth_n: (int). smooth extent 79 | :param ignore_unclassified: (bool). True if the the first class should be ignored in the results 80 | :return: ([..., n_c] np.float32) precision, recall, F1 score, IoU score 81 | """ 82 | 83 | # If the first class (often "unclassified") should be ignored, erase it from the confusion. 84 | if ignore_unclassified: 85 | confusions[..., 0, :] = 0 86 | confusions[..., :, 0] = 0 87 | 88 | # Sum successive confusions for smoothing 89 | smoothed_confusions = confusions.copy() 90 | if confusions.ndim > 2 and smooth_n > 0: 91 | for epoch in range(confusions.shape[-3]): 92 | i0 = max(epoch - smooth_n, 0) 93 | i1 = min(epoch + smooth_n + 1, confusions.shape[-3]) 94 | smoothed_confusions[..., epoch, :, :] = np.sum(confusions[..., i0:i1, :, :], axis=-3) 95 | 96 | # Compute TP, FP, FN. This assume that the second to last axis counts the truths (like the first axis of a 97 | # confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix) 98 | TP = np.diagonal(smoothed_confusions, axis1=-2, axis2=-1) 99 | TP_plus_FP = np.sum(smoothed_confusions, axis=-2) 100 | TP_plus_FN = np.sum(smoothed_confusions, axis=-1) 101 | 102 | # Compute precision and recall. This assume that the second to last axis counts the truths (like the first axis of 103 | # a confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix) 104 | PRE = TP / (TP_plus_FN + 1e-6) 105 | REC = TP / (TP_plus_FP + 1e-6) 106 | 107 | # Compute Accuracy 108 | ACC = np.sum(TP, axis=-1) / (np.sum(smoothed_confusions, axis=(-2, -1)) + 1e-6) 109 | 110 | # Compute F1 score 111 | F1 = 2 * TP / (TP_plus_FP + TP_plus_FN + 1e-6) 112 | 113 | # Compute IoU 114 | IoU = F1 / (2 - F1) 115 | 116 | return PRE, REC, F1, IoU, ACC 117 | 118 | 119 | def IoU_from_confusions(confusions): 120 | """ 121 | Computes IoU from confusion matrices. 122 | :param confusions: ([..., n_c, n_c] np.int32). Can be any dimension, the confusion matrices should be described by 123 | the last axes. n_c = number of classes 124 | :param ignore_unclassified: (bool). True if the the first class should be ignored in the results 125 | :return: ([..., n_c] np.float32) IoU score 126 | """ 127 | 128 | # Compute TP, FP, FN. This assume that the second to last axis counts the truths (like the first axis of a 129 | # confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix) 130 | TP = np.diagonal(confusions, axis1=-2, axis2=-1) 131 | TP_plus_FN = np.sum(confusions, axis=-1) 132 | TP_plus_FP = np.sum(confusions, axis=-2) 133 | 134 | # Compute IoU 135 | IoU = TP / (TP_plus_FP + TP_plus_FN - TP + 1e-6) 136 | 137 | # Compute mIoU with only the actual classes 138 | mask = TP_plus_FN < 1e-3 139 | counts = np.sum(1 - mask, axis=-1, keepdims=True) 140 | mIoU = np.sum(IoU, axis=-1, keepdims=True) / (counts + 1e-6) 141 | 142 | # If class is absent, place mIoU in place of 0 IoU to get the actual mean later 143 | IoU += mask * mIoU 144 | 145 | return IoU 146 | 147 | -------------------------------------------------------------------------------- /training_S3DIS.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on S3DIS dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import sys 29 | 30 | # Custom libs 31 | from utils.config import Config 32 | from utils.trainer import ModelTrainer 33 | from models.KPFCNN_model import KernelPointFCNN 34 | 35 | # Dataset 36 | from datasets.S3DIS import S3DISDataset 37 | 38 | 39 | # ---------------------------------------------------------------------------------------------------------------------- 40 | # 41 | # Config Class 42 | # \******************/ 43 | # 44 | 45 | 46 | class S3DISConfig(Config): 47 | """ 48 | Override the parameters you want to modify for this dataset 49 | """ 50 | 51 | #################### 52 | # Dataset parameters 53 | #################### 54 | 55 | # Dataset name 56 | dataset = 'S3DIS' 57 | 58 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 59 | num_classes = None 60 | 61 | # Type of task performed on this dataset (also overwritten) 62 | network_model = None 63 | 64 | # Number of CPU threads for the input pipeline 65 | input_threads = 8 66 | 67 | ######################### 68 | # Architecture definition 69 | ######################### 70 | 71 | # Define layers 72 | architecture = ['simple', 73 | 'resnetb', 74 | 'resnetb_strided', 75 | 'resnetb', 76 | 'resnetb_strided', 77 | 'resnetb', 78 | 'resnetb_strided', 79 | 'resnetb', 80 | 'resnetb_strided', 81 | 'resnetb', 82 | 'nearest_upsample', 83 | 'unary', 84 | 'nearest_upsample', 85 | 'unary', 86 | 'nearest_upsample', 87 | 'unary', 88 | 'nearest_upsample', 89 | 'unary'] 90 | 91 | # KPConv specific parameters 92 | num_kernel_points = 15 93 | first_subsampling_dl = 0.04 94 | in_radius = 2.0 95 | 96 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 97 | density_parameter = 5.0 98 | 99 | # Influence function of KPConv in ('constant', 'linear', gaussian) 100 | KP_influence = 'linear' 101 | KP_extent = 1.0 102 | 103 | # Aggregation function of KPConv in ('closest', 'sum') 104 | convolution_mode = 'sum' 105 | 106 | # Can the network learn modulations in addition to deformations 107 | modulated = False 108 | 109 | # Offset loss 110 | # 'permissive' only constrains offsets to be inside the big radius 111 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 112 | offsets_loss = 'fitting' 113 | offsets_decay = 0.1 114 | 115 | # Choice of input features 116 | in_features_dim = 5 117 | 118 | # Batch normalization parameters 119 | use_batch_norm = True 120 | batch_norm_momentum = 0.98 121 | 122 | ##################### 123 | # Training parameters 124 | ##################### 125 | 126 | # Maximal number of epochs 127 | max_epoch = 500 128 | 129 | # Learning rate management 130 | learning_rate = 1e-2 131 | momentum = 0.98 132 | lr_decays = {i: 0.1**(1/100) for i in range(1, max_epoch)} 133 | grad_clip_norm = 100.0 134 | 135 | # Number of batch 136 | batch_num = 10 137 | 138 | # Number of steps per epochs (cannot be None for this dataset) 139 | epoch_steps = 500 140 | 141 | # Number of validation examples per epoch 142 | validation_size = 50 143 | 144 | # Number of epoch between each snapshot 145 | snapshot_gap = 50 146 | 147 | # Augmentations 148 | augment_scale_anisotropic = True 149 | augment_symmetries = [True, False, False] 150 | augment_rotation = 'vertical' 151 | augment_scale_min = 0.8 152 | augment_scale_max = 1.2 153 | augment_noise = 0.001 154 | augment_occlusion = 'none' 155 | augment_color = 0.8 156 | 157 | # Whether to use loss averaged on all points, or averaged per batch. 158 | batch_averaged_loss = False 159 | 160 | # Do we nee to save convergence 161 | saving = True 162 | saving_path = None 163 | 164 | 165 | # ---------------------------------------------------------------------------------------------------------------------- 166 | # 167 | # Main Call 168 | # \***************/ 169 | # 170 | 171 | 172 | if __name__ == '__main__': 173 | 174 | ########################## 175 | # Initiate the environment 176 | ########################## 177 | 178 | # Choose which gpu to use 179 | GPU_ID = '0' 180 | 181 | # Set GPU visible device 182 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 183 | 184 | # Enable/Disable warnings (set level to '0'/'3') 185 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 186 | 187 | ########################### 188 | # Load the model parameters 189 | ########################### 190 | 191 | config = S3DISConfig() 192 | 193 | ############## 194 | # Prepare Data 195 | ############## 196 | 197 | print() 198 | print('Dataset Preparation') 199 | print('*******************') 200 | 201 | # Initiate dataset configuration 202 | dataset = S3DISDataset(config.input_threads) 203 | 204 | # Create subsampled input clouds 205 | dl0 = config.first_subsampling_dl 206 | dataset.load_subsampled_clouds(dl0) 207 | 208 | # Initialize input pipelines 209 | dataset.init_input_pipeline(config) 210 | 211 | # Test the input pipeline alone with this debug function 212 | # dataset.check_input_pipeline_timing(config) 213 | 214 | ############## 215 | # Define Model 216 | ############## 217 | 218 | print('Creating Model') 219 | print('**************\n') 220 | t1 = time.time() 221 | 222 | # Model class 223 | model = KernelPointFCNN(dataset.flat_inputs, config) 224 | 225 | # Trainer class 226 | trainer = ModelTrainer(model) 227 | t2 = time.time() 228 | 229 | print('\n----------------') 230 | print('Done in {:.1f} s'.format(t2 - t1)) 231 | print('----------------\n') 232 | 233 | ################ 234 | # Start training 235 | ################ 236 | 237 | print('Start Training') 238 | print('**************\n') 239 | 240 | trainer.train(model, dataset) 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | -------------------------------------------------------------------------------- /training_ShapeNetPart.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on ShapeNetPart dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import sys 29 | 30 | # Custom libs 31 | from utils.config import Config 32 | from utils.trainer import ModelTrainer 33 | from models.KPFCNN_model import KernelPointFCNN 34 | 35 | # Dataset 36 | from datasets.ShapeNetPart import ShapeNetPartDataset 37 | 38 | 39 | # ---------------------------------------------------------------------------------------------------------------------- 40 | # 41 | # Config Class 42 | # \******************/ 43 | # 44 | 45 | 46 | class ShapeNetPartConfig(Config): 47 | """ 48 | Override the parameters you want to modify for this dataset 49 | """ 50 | 51 | #################### 52 | # Dataset parameters 53 | #################### 54 | 55 | # Dataset name in the format 'ShapeNetPart_Object' to segment an object class independently or 'ShapeNetPart_multi' 56 | # to segment all objects with a single model. 57 | dataset = 'ShapeNetPart_multi' 58 | 59 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 60 | num_classes = None 61 | 62 | # Type of task performed on this dataset (also overwritten) 63 | network_model = None 64 | 65 | # Number of CPU threads for the input pipeline 66 | input_threads = 8 67 | 68 | ######################### 69 | # Architecture definition 70 | ######################### 71 | 72 | # Define layers 73 | architecture = ['simple', 74 | 'resnetb', 75 | 'resnetb_strided', 76 | 'resnetb', 77 | 'resnetb_strided', 78 | 'resnetb_deformable', 79 | 'resnetb_deformable_strided', 80 | 'resnetb_deformable', 81 | 'resnetb_deformable_strided', 82 | 'resnetb_deformable', 83 | 'nearest_upsample', 84 | 'unary', 85 | 'nearest_upsample', 86 | 'unary', 87 | 'nearest_upsample', 88 | 'unary', 89 | 'nearest_upsample', 90 | 'unary'] 91 | 92 | # KPConv specific parameters 93 | num_kernel_points = 15 94 | first_subsampling_dl = 0.02 95 | 96 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 97 | density_parameter = 5.0 98 | 99 | # Influence function of KPConv in ('constant', 'linear', gaussian) 100 | KP_influence = 'linear' 101 | KP_extent = 1.0 102 | 103 | # Aggregation function of KPConv in ('closest', 'sum') 104 | convolution_mode = 'sum' 105 | 106 | # Can the network learn modulations in addition to deformations 107 | modulated = False 108 | 109 | # Offset loss 110 | # 'permissive' only constrains offsets inside the big radius 111 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 112 | offsets_loss = 'fitting' 113 | offsets_decay = 0.1 114 | 115 | # Choice of input features 116 | in_features_dim = 4 117 | 118 | # Batch normalization parameters 119 | use_batch_norm = True 120 | batch_norm_momentum = 0.98 121 | 122 | ##################### 123 | # Training parameters 124 | ##################### 125 | 126 | # Maximal number of epochs 127 | max_epoch = 500 128 | 129 | # Learning rate management 130 | learning_rate = 1e-2 131 | momentum = 0.98 132 | lr_decays = {i: 0.1**(1/80) for i in range(1, max_epoch)} 133 | grad_clip_norm = 100.0 134 | 135 | # Number of batch 136 | batch_num = 16 137 | 138 | # Number of steps per epochs (cannot be None for this dataset) 139 | epoch_steps = None 140 | 141 | # Number of validation examples per epoch 142 | validation_size = 50 143 | 144 | # Number of epoch between each snapshot 145 | snapshot_gap = 50 146 | 147 | # Augmentations 148 | augment_scale_anisotropic = True 149 | augment_symmetries = [False, False, False] 150 | augment_rotation = 'none' 151 | augment_scale_min = 0.9 152 | augment_scale_max = 1.1 153 | augment_noise = 0.001 154 | augment_occlusion = 'none' 155 | 156 | # Whether to use loss averaged on all points, or averaged per batch. 157 | batch_averaged_loss = False 158 | 159 | # Do we nee to save convergence 160 | saving = True 161 | saving_path = None 162 | 163 | 164 | # ---------------------------------------------------------------------------------------------------------------------- 165 | # 166 | # Main Call 167 | # \***************/ 168 | # 169 | 170 | 171 | if __name__ == '__main__': 172 | 173 | ########################## 174 | # Initiate the environment 175 | ########################## 176 | 177 | # Choose which gpu to use 178 | GPU_ID = '0' 179 | 180 | # Set GPU visible device 181 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 182 | 183 | # Enable/Disable warnings (set level to '0'/'3') 184 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 185 | 186 | ########################### 187 | # Load the model parameters 188 | ########################### 189 | 190 | config = ShapeNetPartConfig() 191 | 192 | ############## 193 | # Prepare Data 194 | ############## 195 | 196 | print() 197 | print('Dataset Preparation') 198 | print('*******************') 199 | 200 | # Initiate dataset configuration 201 | dataset = ShapeNetPartDataset(config.dataset.split('_')[1], config.input_threads) 202 | 203 | # Create subsampled input clouds 204 | dl0 = config.first_subsampling_dl 205 | dataset.load_subsampled_clouds(dl0) 206 | 207 | # Initialize input pipelines 208 | dataset.init_input_pipeline(config) 209 | 210 | # Test the input pipeline alone with this debug function 211 | # dataset.check_input_pipeline_timing(config) 212 | 213 | ############## 214 | # Define Model 215 | ############## 216 | 217 | print('Creating Model') 218 | print('**************\n') 219 | t1 = time.time() 220 | 221 | # Model class 222 | model = KernelPointFCNN(dataset.flat_inputs, config) 223 | 224 | # Trainer class 225 | trainer = ModelTrainer(model) 226 | t2 = time.time() 227 | 228 | print('\n----------------') 229 | print('Done in {:.1f} s'.format(t2 - t1)) 230 | print('----------------\n') 231 | 232 | ################ 233 | # Start training 234 | ################ 235 | 236 | print('Start Training') 237 | print('**************\n') 238 | 239 | trainer.train(model, dataset) 240 | 241 | 242 | 243 | 244 | -------------------------------------------------------------------------------- /training_Semantic3D.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on Semantic3D dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import sys 29 | 30 | import numpy as np 31 | from sklearn.metrics import confusion_matrix 32 | 33 | # Custom libs 34 | from utils.config import Config 35 | from utils.trainer import ModelTrainer 36 | from models.KPFCNN_model import KernelPointFCNN 37 | 38 | # Dataset 39 | from datasets.Semantic3D import Semantic3DDataset 40 | 41 | 42 | # ---------------------------------------------------------------------------------------------------------------------- 43 | # 44 | # Config Class 45 | # \******************/ 46 | # 47 | 48 | 49 | class Semantic3DConfig(Config): 50 | """ 51 | Override the parameters you want to modify for this dataset 52 | """ 53 | 54 | #################### 55 | # Dataset parameters 56 | #################### 57 | 58 | # Dataset name in the format 'ShapeNetPart_Object' to segment an object class independently or 'ShapeNetPart_multi' 59 | # to segment all objects with a single model. 60 | dataset = 'Semantic3D' 61 | 62 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 63 | num_classes = None 64 | 65 | # Type of task performed on this dataset (also overwritten) 66 | network_model = None 67 | 68 | # Number of CPU threads for the input pipeline 69 | input_threads = 8 70 | 71 | ######################### 72 | # Architecture definition 73 | ######################### 74 | 75 | # Define layers 76 | architecture = ['simple', 77 | 'resnetb', 78 | 'resnetb_strided', 79 | 'resnetb', 80 | 'resnetb_strided', 81 | 'resnetb', 82 | 'resnetb_strided', 83 | 'resnetb', 84 | 'resnetb_strided', 85 | 'resnetb', 86 | 'nearest_upsample', 87 | 'unary', 88 | 'nearest_upsample', 89 | 'unary', 90 | 'nearest_upsample', 91 | 'unary', 92 | 'nearest_upsample', 93 | 'unary'] 94 | 95 | # KPConv specific parameters 96 | num_kernel_points = 15 97 | first_subsampling_dl = 0.06 98 | in_radius = 3.0 99 | 100 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 101 | density_parameter = 5.0 102 | 103 | # Behavior of convolutions in ('constant', 'linear', gaussian) 104 | KP_influence = 'linear' 105 | KP_extent = 1.0 106 | 107 | # Behavior of convolutions in ('closest', 'sum') 108 | convolution_mode = 'sum' 109 | 110 | # Can the network learn modulations 111 | modulated = False 112 | 113 | # Offset loss 114 | # 'permissive' only constrains offsets inside the big radius 115 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 116 | offsets_loss = 'fitting' 117 | offsets_decay = 0.1 118 | 119 | # Choice of input features 120 | in_features_dim = 4 121 | 122 | # Batch normalization parameters 123 | use_batch_norm = True 124 | batch_norm_momentum = 0.98 125 | 126 | ##################### 127 | # Training parameters 128 | ##################### 129 | 130 | # Maximal number of epochs 131 | max_epoch = 500 132 | 133 | # Learning rate management 134 | learning_rate = 1e-2 135 | momentum = 0.98 136 | lr_decays = {i: 0.1**(1/100) for i in range(1, max_epoch)} 137 | grad_clip_norm = 100.0 138 | 139 | # Number of batch 140 | batch_num = 10 141 | 142 | # Number of steps per epochs (cannot be None for this dataset) 143 | epoch_steps = 500 144 | 145 | # Number of validation examples per epoch 146 | validation_size = 50 147 | 148 | # Number of epoch between each snapshot 149 | snapshot_gap = 50 150 | 151 | # Augmentations 152 | augment_scale_anisotropic = True 153 | augment_symmetries = [True, False, False] 154 | augment_rotation = 'vertical' 155 | augment_scale_min = 0.9 156 | augment_scale_max = 1.1 157 | augment_noise = 0.001 158 | augment_occlusion = 'none' 159 | augment_color = 1.0 160 | 161 | # Whether to use loss averaged on all points, or averaged per batch. 162 | batch_averaged_loss = False 163 | 164 | # Do we nee to save convergence 165 | saving = True 166 | saving_path = None 167 | 168 | 169 | # ---------------------------------------------------------------------------------------------------------------------- 170 | # 171 | # Main Call 172 | # \***************/ 173 | # 174 | 175 | 176 | if __name__ == '__main__': 177 | 178 | ########################## 179 | # Initiate the environment 180 | ########################## 181 | 182 | # Choose which gpu to use 183 | GPU_ID = '0' 184 | 185 | # Set GPU visible device 186 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 187 | 188 | # Enable/Disable warnings (set level to '0'/'3') 189 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 190 | 191 | ########################### 192 | # Load the model parameters 193 | ########################### 194 | 195 | config = Semantic3DConfig() 196 | 197 | ############## 198 | # Prepare Data 199 | ############## 200 | 201 | print() 202 | print('Dataset Preparation') 203 | print('*******************') 204 | 205 | # Initiate dataset configuration 206 | dataset = Semantic3DDataset(config.input_threads) 207 | 208 | # Create subsampled input clouds 209 | dl0 = config.first_subsampling_dl 210 | dataset.load_subsampled_clouds(dl0) 211 | 212 | # Initialize input pipelines 213 | dataset.init_input_pipeline(config) 214 | 215 | # Test the input pipeline alone with this debug function 216 | # dataset.check_input_pipeline_timing(config) 217 | 218 | ############## 219 | # Define Model 220 | ############## 221 | 222 | print('Creating Model') 223 | print('**************\n') 224 | t1 = time.time() 225 | 226 | # Model class 227 | model = KernelPointFCNN(dataset.flat_inputs, config) 228 | 229 | # Trainer class 230 | trainer = ModelTrainer(model) 231 | t2 = time.time() 232 | 233 | print('\n----------------') 234 | print('Done in {:.1f} s'.format(t2 - t1)) 235 | print('----------------\n') 236 | 237 | ################ 238 | # Start training 239 | ################ 240 | 241 | print('Start Training') 242 | print('**************\n') 243 | 244 | trainer.train(model, dataset) 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | -------------------------------------------------------------------------------- /training_NPM3D.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on NPM3D dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import sys 29 | 30 | import numpy as np 31 | from sklearn.metrics import confusion_matrix 32 | 33 | # Custom libs 34 | from utils.config import Config 35 | from utils.trainer import ModelTrainer 36 | from models.KPFCNN_model import KernelPointFCNN 37 | 38 | # Dataset 39 | from datasets.NPM3D import NPM3DDataset 40 | 41 | 42 | # ---------------------------------------------------------------------------------------------------------------------- 43 | # 44 | # Config Class 45 | # \******************/ 46 | # 47 | 48 | 49 | class NPM3DConfig(Config): 50 | """ 51 | Override the parameters you want to modify for this dataset 52 | """ 53 | 54 | #################### 55 | # Dataset parameters 56 | #################### 57 | 58 | # Dataset name in the format 'ShapeNetPart_Object' to segment an object class independently or 'ShapeNetPart_multi' 59 | # to segment all objects with a single model. 60 | dataset = 'NPM3D' 61 | 62 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 63 | num_classes = None 64 | 65 | # Type of task performed on this dataset (also overwritten) 66 | network_model = None 67 | 68 | # Number of CPU threads for the input pipeline 69 | input_threads = 8 70 | 71 | ######################### 72 | # Architecture definition 73 | ######################### 74 | 75 | # Define layers 76 | architecture = ['simple', 77 | 'resnetb', 78 | 'resnetb_strided', 79 | 'resnetb', 80 | 'resnetb_strided', 81 | 'resnetb_deformable', 82 | 'resnetb_deformable_strided', 83 | 'resnetb_deformable', 84 | 'resnetb_deformable_strided', 85 | 'resnetb_deformable', 86 | 'nearest_upsample', 87 | 'unary', 88 | 'nearest_upsample', 89 | 'unary', 90 | 'nearest_upsample', 91 | 'unary', 92 | 'nearest_upsample', 93 | 'unary'] 94 | 95 | # KPConv specific parameters 96 | num_kernel_points = 15 97 | first_subsampling_dl = 0.08 98 | in_radius = 4.0 99 | 100 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 101 | density_parameter = 5.0 102 | 103 | # Behavior of convolutions in ('constant', 'linear', gaussian) 104 | KP_influence = 'linear' 105 | KP_extent = 1.0 106 | 107 | # Behavior of convolutions in ('closest', 'sum') 108 | convolution_mode = 'sum' 109 | 110 | # Can the network learn modulations 111 | modulated = False 112 | 113 | # Offset loss 114 | # 'permissive' only constrains offsets inside the big radius 115 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 116 | offsets_loss = 'fitting' 117 | offsets_decay = 0.1 118 | 119 | # Choice of input features 120 | in_features_dim = 1 121 | 122 | # Batch normalization parameters 123 | use_batch_norm = True 124 | batch_norm_momentum = 0.98 125 | 126 | ##################### 127 | # Training parameters 128 | ##################### 129 | 130 | # Maximal number of epochs 131 | max_epoch = 600 132 | 133 | # Learning rate management 134 | learning_rate = 1e-2 135 | momentum = 0.98 136 | lr_decays = {i: 0.1**(1/100) for i in range(1, max_epoch)} 137 | grad_clip_norm = 100.0 138 | 139 | # Number of batch 140 | batch_num = 8 141 | 142 | # Number of steps per epochs (cannot be None for this dataset) 143 | epoch_steps = 500 144 | 145 | # Number of validation examples per epoch 146 | validation_size = 50 147 | 148 | # Number of epoch between each snapshot 149 | snapshot_gap = 50 150 | 151 | # Augmentations 152 | augment_scale_anisotropic = True 153 | augment_symmetries = [True, False, False] 154 | augment_rotation = 'vertical' 155 | augment_scale_min = 0.9 156 | augment_scale_max = 1.1 157 | augment_noise = 0.01 158 | augment_occlusion = 'none' 159 | augment_color = 1.0 160 | 161 | # Whether to use loss averaged on all points, or averaged per batch. 162 | batch_averaged_loss = False 163 | 164 | # Do we nee to save convergence 165 | saving = True 166 | saving_path = None 167 | 168 | 169 | # ---------------------------------------------------------------------------------------------------------------------- 170 | # 171 | # Main Call 172 | # \***************/ 173 | # 174 | 175 | 176 | if __name__ == '__main__': 177 | 178 | ########################## 179 | # Initiate the environment 180 | ########################## 181 | 182 | # Choose which gpu to use 183 | GPU_ID = '0' 184 | 185 | # Set GPU visible device 186 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 187 | 188 | # Enable/Disable warnings (set level to '0'/'3') 189 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 190 | 191 | ########################### 192 | # Load the model parameters 193 | ########################### 194 | 195 | config = NPM3DConfig() 196 | 197 | ############## 198 | # Prepare Data 199 | ############## 200 | 201 | print() 202 | print('Dataset Preparation') 203 | print('*******************') 204 | 205 | # Initiate dataset configuration 206 | dataset = NPM3DDataset(config.input_threads, load_test=False) 207 | 208 | # Create subsampled input clouds 209 | dl0 = config.first_subsampling_dl 210 | dataset.load_subsampled_clouds(dl0) 211 | 212 | # Initialize input pipelines 213 | dataset.init_input_pipeline(config) 214 | 215 | # Test the input pipeline alone with this debug function 216 | # dataset.check_input_pipeline_timing(config) 217 | 218 | ############## 219 | # Define Model 220 | ############## 221 | 222 | print('Creating Model') 223 | print('**************\n') 224 | t1 = time.time() 225 | 226 | # Model class 227 | model = KernelPointFCNN(dataset.flat_inputs, config) 228 | 229 | # Trainer class 230 | trainer = ModelTrainer(model) 231 | t2 = time.time() 232 | 233 | print('\n----------------') 234 | print('Done in {:.1f} s'.format(t2 - t1)) 235 | print('----------------\n') 236 | 237 | ################ 238 | # Start training 239 | ################ 240 | 241 | print('Start Training') 242 | print('**************\n') 243 | 244 | trainer.train(model, dataset) 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | -------------------------------------------------------------------------------- /training_DALES.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on NPM3D dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import sys 29 | 30 | import numpy as np 31 | from sklearn.metrics import confusion_matrix 32 | 33 | # Custom libs 34 | from utils.config import Config 35 | from utils.trainer import ModelTrainer 36 | from models.KPFCNN_model import KernelPointFCNN 37 | 38 | # Dataset 39 | from datasets.DALES import DalesDataset 40 | 41 | 42 | # ---------------------------------------------------------------------------------------------------------------------- 43 | # 44 | # Config Class 45 | # \******************/ 46 | # 47 | 48 | 49 | class DalesConfig(Config): 50 | """ 51 | Override the parameters you want to modify for this dataset 52 | """ 53 | 54 | #################### 55 | # Dataset parameters 56 | #################### 57 | 58 | # Dataset name in the format 'ShapeNetPart_Object' to segment an object class independently or 'ShapeNetPart_multi' 59 | # to segment all objects with a single model. 60 | dataset = 'DALES' 61 | 62 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 63 | num_classes = None 64 | 65 | # Type of task performed on this dataset (also overwritten) 66 | network_model = None 67 | 68 | # Number of CPU threads for the input pipeline 69 | input_threads = 8 70 | 71 | ######################### 72 | # Architecture definition 73 | ######################### 74 | 75 | # Define layers 76 | architecture = ['simple', 77 | 'resnetb', 78 | 'resnetb_strided', 79 | 'resnetb', 80 | 'resnetb_strided', 81 | 'resnetb_deformable', 82 | 'resnetb_deformable_strided', 83 | 'resnetb_deformable', 84 | 'resnetb_deformable_strided', 85 | 'resnetb_deformable', 86 | 'nearest_upsample', 87 | 'unary', 88 | 'nearest_upsample', 89 | 'unary', 90 | 'nearest_upsample', 91 | 'unary', 92 | 'nearest_upsample', 93 | 'unary'] 94 | 95 | # KPConv specific parameters 96 | num_kernel_points = 15 97 | first_subsampling_dl = 0.250 98 | in_radius = 20.0 99 | 100 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 101 | density_parameter = 5.0 102 | 103 | # Behavior of convolutions in ('constant', 'linear', gaussian) 104 | KP_influence = 'linear' 105 | KP_extent = 1.0 106 | 107 | # Behavior of convolutions in ('closest', 'sum') 108 | convolution_mode = 'sum' 109 | 110 | # Can the network learn modulations 111 | modulated = False 112 | 113 | # Offset loss 114 | # 'permissive' only constrains offsets inside the big radius 115 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 116 | offsets_loss = 'fitting' 117 | offsets_decay = 0.1 118 | 119 | # Choice of input features 120 | in_features_dim = 1 121 | 122 | # Batch normalization parameters 123 | use_batch_norm = True 124 | batch_norm_momentum = 0.98 125 | 126 | ##################### 127 | # Training parameters 128 | ##################### 129 | 130 | # Maximal number of epochs 131 | max_epoch = 850 132 | 133 | # Learning rate management 134 | learning_rate = 1e-2 135 | momentum = 0.98 136 | lr_decays = {i: 0.1**(1/100) for i in range(1, max_epoch)} 137 | grad_clip_norm = 100.0 138 | 139 | # Number of batch 140 | batch_num = 4 141 | 142 | # Number of steps per epochs (cannot be None for this dataset) 143 | epoch_steps = 741 144 | 145 | # Number of validation examples per epoch 146 | validation_size = 50 147 | 148 | # Number of epoch between each snapshot 149 | snapshot_gap = 50 150 | 151 | # Augmentations 152 | augment_scale_anisotropic = True 153 | augment_symmetries = [True, False, False] 154 | augment_rotation = 'vertical' 155 | augment_scale_min = 0.9 156 | augment_scale_max = 1.1 157 | augment_noise = 0.01 158 | augment_occlusion = 'none' 159 | augment_color = 1.0 160 | 161 | # Whether to use loss averaged on all points, or averaged per batch. 162 | batch_averaged_loss = False 163 | 164 | # Do we nee to save convergence 165 | saving = True 166 | saving_path = None 167 | 168 | 169 | # ---------------------------------------------------------------------------------------------------------------------- 170 | # 171 | # Main Call 172 | # \***************/ 173 | # 174 | 175 | 176 | if __name__ == '__main__': 177 | 178 | ########################## 179 | # Initiate the environment 180 | ########################## 181 | 182 | # Choose which gpu to use 183 | # GPU_ID = '0' 184 | 185 | # Set GPU visible device 186 | # os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 187 | 188 | # Enable/Disable warnings (set level to '0'/'3') 189 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 190 | 191 | ########################### 192 | # Load the model parameters 193 | ########################### 194 | 195 | config = DalesConfig() 196 | 197 | ############## 198 | # Prepare Data 199 | ############## 200 | 201 | print() 202 | print('Dataset Preparation') 203 | print('*******************') 204 | 205 | # Initiate dataset configuration 206 | dataset = DalesDataset(config.input_threads, load_test=False) 207 | 208 | # Create subsampled input clouds 209 | dl0 = config.first_subsampling_dl 210 | dataset.load_subsampled_clouds(dl0) 211 | 212 | # Initialize input pipelines 213 | dataset.init_input_pipeline(config) 214 | 215 | # Test the input pipeline alone with this debug function 216 | # dataset.check_input_pipeline_timing(config) 217 | 218 | ############## 219 | # Define Model 220 | ############## 221 | 222 | print('Creating Model') 223 | print('**************\n') 224 | t1 = time.time() 225 | 226 | # Model class 227 | model = KernelPointFCNN(dataset.flat_inputs, config) 228 | 229 | # Trainer class 230 | trainer = ModelTrainer(model) 231 | t2 = time.time() 232 | 233 | print('\n----------------') 234 | print('Done in {:.1f} s'.format(t2 - t1)) 235 | print('----------------\n') 236 | 237 | ################ 238 | # Start training 239 | ################ 240 | 241 | print('Start Training') 242 | print('**************\n') 243 | 244 | trainer.train(model, dataset) 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | -------------------------------------------------------------------------------- /training_Scannet.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on S3DIS dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import sys 29 | 30 | import numpy as np 31 | from sklearn.metrics import confusion_matrix 32 | 33 | # Custom libs 34 | from utils.config import Config 35 | from utils.trainer import ModelTrainer 36 | from models.KPFCNN_model import KernelPointFCNN 37 | 38 | # Dataset 39 | from datasets.Scannet import ScannetDataset 40 | 41 | 42 | # ---------------------------------------------------------------------------------------------------------------------- 43 | # 44 | # Config Class 45 | # \******************/ 46 | # 47 | 48 | 49 | class ScannetConfig(Config): 50 | """ 51 | Override the parameters you want to modify for this dataset 52 | """ 53 | 54 | #################### 55 | # Dataset parameters 56 | #################### 57 | 58 | # Dataset name in the format 'ShapeNetPart_Object' to segment an object class independently or 'ShapeNetPart_multi' 59 | # to segment all objects with a single model. 60 | dataset = 'Scannet' 61 | 62 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 63 | num_classes = None 64 | 65 | # Type of task performed on this dataset (also overwritten) 66 | network_model = None 67 | 68 | # Number of CPU threads for the input pipeline 69 | input_threads = 8 70 | 71 | ######################### 72 | # Architecture definition 73 | ######################### 74 | 75 | # Define layers 76 | architecture = ['simple', 77 | 'resnetb', 78 | 'resnetb_strided', 79 | 'resnetb', 80 | 'resnetb_strided', 81 | 'resnetb_deformable', 82 | 'resnetb_deformable_strided', 83 | 'resnetb_deformable', 84 | 'resnetb_deformable_strided', 85 | 'resnetb_deformable', 86 | 'nearest_upsample', 87 | 'unary', 88 | 'nearest_upsample', 89 | 'unary', 90 | 'nearest_upsample', 91 | 'unary', 92 | 'nearest_upsample', 93 | 'unary'] 94 | 95 | # KPConv specific parameters 96 | num_kernel_points = 15 97 | first_subsampling_dl = 0.04 98 | in_radius = 2.0 99 | 100 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 101 | density_parameter = 5.0 102 | 103 | # Behavior of convolutions in ('constant', 'linear', gaussian) 104 | KP_influence = 'linear' 105 | KP_extent = 1.0 106 | 107 | # Behavior of convolutions in ('closest', 'sum') 108 | convolution_mode = 'sum' 109 | 110 | # Can the network learn modulations 111 | modulated = False 112 | 113 | # Offset loss 114 | # 'permissive' only constrains offsets inside the big radius 115 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 116 | offsets_loss = 'fitting' 117 | offsets_decay = 0.1 118 | 119 | # Choice of input features 120 | in_features_dim = 4 121 | 122 | # Batch normalization parameters 123 | use_batch_norm = True 124 | batch_norm_momentum = 0.98 125 | 126 | ##################### 127 | # Training parameters 128 | ##################### 129 | 130 | # Maximal number of epochs 131 | max_epoch = 500 132 | 133 | # Learning rate management 134 | learning_rate = 1e-2 135 | momentum = 0.98 136 | lr_decays = {i: 0.1**(1/100) for i in range(1, max_epoch)} 137 | grad_clip_norm = 100.0 138 | 139 | # Number of batch 140 | batch_num = 10 141 | 142 | # Number of steps per epochs (cannot be None for this dataset) 143 | epoch_steps = 600 144 | 145 | # Number of validation examples per epoch 146 | validation_size = 50 147 | 148 | # Number of epoch between each snapshot 149 | snapshot_gap = 50 150 | 151 | # Augmentations 152 | augment_scale_anisotropic = True 153 | augment_symmetries = [True, False, False] 154 | augment_rotation = 'vertical' 155 | augment_scale_min = 0.9 156 | augment_scale_max = 1.1 157 | augment_noise = 0.001 158 | augment_occlusion = 'none' 159 | augment_color = 1.0 160 | 161 | # Whether to use loss averaged on all points, or averaged per batch. 162 | batch_averaged_loss = False 163 | 164 | # Do we nee to save convergence 165 | saving = True 166 | saving_path = None 167 | 168 | 169 | # ---------------------------------------------------------------------------------------------------------------------- 170 | # 171 | # Main Call 172 | # \***************/ 173 | # 174 | 175 | 176 | if __name__ == '__main__': 177 | 178 | ########################## 179 | # Initiate the environment 180 | ########################## 181 | 182 | # Choose which gpu to use 183 | GPU_ID = '0' 184 | 185 | # Set GPU visible device 186 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 187 | 188 | # Enable/Disable warnings (set level to '0'/'3') 189 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 190 | 191 | ########################### 192 | # Load the model parameters 193 | ########################### 194 | 195 | # Load config 196 | config = ScannetConfig() 197 | 198 | ############## 199 | # Prepare Data 200 | ############## 201 | 202 | print() 203 | print('Dataset Preparation') 204 | print('*******************') 205 | 206 | # Initiate dataset configuration 207 | dataset = ScannetDataset(config.input_threads, load_test=False) 208 | 209 | # Create subsampled input clouds 210 | dl0 = config.first_subsampling_dl 211 | dataset.load_subsampled_clouds(dl0) 212 | 213 | # Initialize input pipelines 214 | dataset.init_input_pipeline(config) 215 | 216 | # Test the input pipeline alone with this debug function 217 | # dataset.check_input_pipeline_timing(config) 218 | 219 | ############## 220 | # Define Model 221 | ############## 222 | 223 | print('Creating Model') 224 | print('**************\n') 225 | t1 = time.time() 226 | 227 | # Model class 228 | model = KernelPointFCNN(dataset.flat_inputs, config) 229 | 230 | # Trainer class 231 | trainer = ModelTrainer(model) 232 | t2 = time.time() 233 | 234 | print('\n----------------') 235 | print('Done in {:.1f} s'.format(t2 - t1)) 236 | print('----------------\n') 237 | 238 | ################ 239 | # Start training 240 | ################ 241 | 242 | print('Start Training') 243 | print('**************\n') 244 | 245 | trainer.train(model, dataset) 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | -------------------------------------------------------------------------------- /training_ModelNet40.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to start a training on ModelNet40 dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Common libs 26 | import time 27 | import os 28 | import numpy as np 29 | 30 | # Custom libs 31 | from utils.config import Config 32 | from utils.trainer import ModelTrainer 33 | from models.KPCNN_model import KernelPointCNN 34 | 35 | # Dataset 36 | from datasets.ModelNet40 import ModelNet40Dataset 37 | 38 | 39 | # ---------------------------------------------------------------------------------------------------------------------- 40 | # 41 | # Config Class 42 | # \******************/ 43 | # 44 | 45 | class Modelnet40Config(Config): 46 | """ 47 | Override the parameters you want to modify for this dataset 48 | """ 49 | 50 | #################### 51 | # Dataset parameters 52 | #################### 53 | 54 | # Dataset name 55 | dataset = 'ModelNet40' 56 | 57 | # Number of classes in the dataset (This value is overwritten by dataset class when initiating input pipeline). 58 | num_classes = None 59 | 60 | # Type of task performed on this dataset (also overwritten) 61 | network_model = None 62 | 63 | # Number of CPU threads for the input pipeline 64 | input_threads = 8 65 | 66 | ######################### 67 | # Architecture definition 68 | ######################### 69 | 70 | # Define layers 71 | architecture = ['simple', 72 | 'resnetb', 73 | 'resnetb_strided', 74 | 'resnetb', 75 | 'resnetb_strided', 76 | 'resnetb', 77 | 'resnetb_strided', 78 | 'resnetb_deformable', 79 | 'resnetb_deformable_strided', 80 | 'resnetb_deformable', 81 | 'global_average'] 82 | 83 | # KPConv specific parameters 84 | num_kernel_points = 15 85 | first_subsampling_dl = 0.02 86 | 87 | # Density of neighborhoods for deformable convs (which need bigger radiuses). For normal conv we use KP_extent 88 | density_parameter = 5.0 89 | 90 | # Behavior of convolutions in ('constant', 'linear', gaussian) 91 | KP_influence = 'linear' 92 | KP_extent = 1.0 93 | 94 | # Aggregation function of KPConv in ('closest', 'sum') 95 | convolution_mode = 'sum' 96 | 97 | # Choice of input features 98 | in_features_dim = 1 99 | 100 | # Can the network learn modulations 101 | modulated = False 102 | 103 | # Batch normalization parameters 104 | use_batch_norm = True 105 | batch_norm_momentum = 0.98 106 | 107 | # Offset loss 108 | # 'permissive' only constrains offsets inside the big radius 109 | # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points 110 | offsets_loss = 'fitting' 111 | offsets_decay = 0.1 112 | 113 | ##################### 114 | # Training parameters 115 | ##################### 116 | 117 | # Maximal number of epochs 118 | max_epoch = 402 119 | 120 | # Learning rate management 121 | learning_rate = 1e-3 122 | momentum = 0.98 123 | lr_decays = {i: 0.1**(1/80) for i in range(1, max_epoch)} 124 | grad_clip_norm = 100.0 125 | 126 | # Number of batch 127 | batch_num = 16 128 | 129 | # Number of steps per epochs (If None, 1 epoch = computing the whole dataset.) 130 | epoch_steps = None 131 | 132 | # Number of validation examples per epoch 133 | validation_size = 50 134 | 135 | # Number of epoch between each snapshot 136 | snapshot_gap = 50 137 | 138 | # Augmentations 139 | augment_scale_anisotropic = True 140 | augment_symmetries = [False, False, False] 141 | augment_rotation = 'none' 142 | augment_scale_min = 0.9 143 | augment_scale_max = 1.1 144 | augment_noise = 0.001 145 | augment_occlusion = 'none' 146 | augment_color = 1.0 147 | 148 | # Whether to use loss balanced according to object number of points 149 | batch_averaged_loss = False 150 | 151 | # Do we nee to save convergence 152 | saving = True 153 | saving_path = None 154 | 155 | 156 | # ---------------------------------------------------------------------------------------------------------------------- 157 | # 158 | # Main Call 159 | # \***************/ 160 | # 161 | 162 | if __name__ == '__main__': 163 | 164 | ########################## 165 | # Initiate the environment 166 | ########################## 167 | 168 | # Choose which gpu to use 169 | GPU_ID = '0' 170 | 171 | # Set GPU visible device 172 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 173 | 174 | # Enable/Disable warnings (set level to '0'/'3') 175 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 176 | 177 | ########################### 178 | # Load the model parameters 179 | ########################### 180 | 181 | config = Modelnet40Config() 182 | 183 | ############## 184 | # Prepare Data 185 | ############## 186 | 187 | print() 188 | print('Dataset Preparation') 189 | print('*******************') 190 | 191 | # Initiate dataset configuration 192 | dataset = ModelNet40Dataset(config.input_threads) 193 | 194 | # Create subsample clouds of the models 195 | dl0 = config.first_subsampling_dl 196 | dataset.load_subsampled_clouds(dl0) 197 | 198 | # Initialize input pipelines 199 | dataset.init_input_pipeline(config) 200 | 201 | # Test the input pipeline alone with this debug function 202 | #dataset.check_input_pipeline_timing(config) 203 | 204 | ############## 205 | # Define Model 206 | ############## 207 | 208 | print('Creating Model') 209 | print('**************\n') 210 | t1 = time.time() 211 | 212 | # Model class 213 | model = KernelPointCNN(dataset.flat_inputs, config) 214 | 215 | # Choose here if you want to start training from a previous snapshot 216 | previous_training_path = None 217 | step_ind = -1 218 | 219 | if previous_training_path: 220 | 221 | # Find all snapshot in the chosen training folder 222 | snap_path = os.path.join(previous_training_path, 'snapshots') 223 | snap_steps = [int(f[:-5].split('-')[-1]) for f in os.listdir(snap_path) if f[-5:] == '.meta'] 224 | 225 | # Find which snapshot to restore 226 | chosen_step = np.sort(snap_steps)[step_ind] 227 | chosen_snap = os.path.join(previous_training_path, 'snapshots', 'snap-{:d}'.format(chosen_step)) 228 | 229 | else: 230 | chosen_snap = None 231 | 232 | # Create a trainer class 233 | trainer = ModelTrainer(model, restore_snap=chosen_snap) 234 | t2 = time.time() 235 | 236 | print('\n----------------') 237 | print('Done in {:.1f} s'.format(t2 - t1)) 238 | print('----------------\n') 239 | 240 | ################ 241 | # Start training 242 | ################ 243 | 244 | print('Start Training') 245 | print('**************\n') 246 | 247 | trainer.train(model, dataset) 248 | 249 | 250 | 251 | 252 | 253 | -------------------------------------------------------------------------------- /visualize_deformations.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to test any model on any dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | # Common libs 25 | import time 26 | import os 27 | import numpy as np 28 | 29 | # My libs 30 | from utils.config import Config 31 | from utils.visualizer import ModelVisualizer 32 | from models.KPCNN_model import KernelPointCNN 33 | from models.KPFCNN_model import KernelPointFCNN 34 | 35 | # Datasets 36 | from datasets.ModelNet40 import ModelNet40Dataset 37 | from datasets.ShapeNetPart import ShapeNetPartDataset 38 | from datasets.S3DIS import S3DISDataset 39 | from datasets.Semantic3D import Semantic3DDataset 40 | from datasets.NPM3D import NPM3DDataset 41 | from datasets.Scannet import ScannetDataset 42 | 43 | 44 | # ---------------------------------------------------------------------------------------------------------------------- 45 | # 46 | # Utility functions 47 | # \***********************/ 48 | # 49 | 50 | 51 | def visu_caller(path, step_ind, deform_idx): 52 | 53 | ########################## 54 | # Initiate the environment 55 | ########################## 56 | 57 | # Choose which gpu to use 58 | GPU_ID = '0' 59 | 60 | # Set GPU visible device 61 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 62 | 63 | # Disable warnings 64 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 65 | 66 | ########################### 67 | # Load the model parameters 68 | ########################### 69 | 70 | # Load model parameters 71 | config = Config() 72 | config.load(path) 73 | 74 | ################################## 75 | # Change model parameters for test 76 | ################################## 77 | 78 | # Change parameters for the test here. For example, you can stop augmenting the input data. 79 | 80 | #config.augment_noise = 0.0001 81 | #config.augment_symmetries = False 82 | 83 | config.batch_num = 3 84 | config.in_radius = 4 85 | 86 | ############## 87 | # Prepare Data 88 | ############## 89 | 90 | print() 91 | print('Dataset Preparation') 92 | print('*******************') 93 | 94 | # Initiate dataset configuration 95 | if config.dataset.startswith('ModelNet40'): 96 | dataset = ModelNet40Dataset(config.input_threads) 97 | elif config.dataset == 'S3DIS': 98 | dataset = S3DISDataset(config.input_threads) 99 | on_val = True 100 | elif config.dataset == 'Scannet': 101 | dataset = ScannetDataset(config.input_threads, load_test=True) 102 | elif config.dataset.startswith('ShapeNetPart'): 103 | dataset = ShapeNetPartDataset(config.dataset.split('_')[1], config.input_threads) 104 | elif config.dataset == 'NPM3D': 105 | dataset = NPM3DDataset(config.input_threads, load_test=True) 106 | elif config.dataset == 'Semantic3D': 107 | dataset = Semantic3DDataset(config.input_threads) 108 | else: 109 | raise ValueError('Unsupported dataset : ' + config.dataset) 110 | 111 | # Create subsample clouds of the models 112 | dl0 = config.first_subsampling_dl 113 | dataset.load_subsampled_clouds(dl0) 114 | 115 | # Initialize input pipelines 116 | if config.dataset == 'S3DIS': 117 | dataset.init_input_pipeline(config) 118 | else: 119 | dataset.init_test_input_pipeline(config) 120 | 121 | ############## 122 | # Define Model 123 | ############## 124 | 125 | print('Creating Model') 126 | print('**************\n') 127 | t1 = time.time() 128 | 129 | if config.dataset.startswith('ShapeNetPart'): 130 | model = KernelPointFCNN(dataset.flat_inputs, config) 131 | elif config.dataset.startswith('S3DIS'): 132 | model = KernelPointFCNN(dataset.flat_inputs, config) 133 | elif config.dataset.startswith('Scannet'): 134 | model = KernelPointFCNN(dataset.flat_inputs, config) 135 | elif config.dataset.startswith('NPM3D'): 136 | model = KernelPointFCNN(dataset.flat_inputs, config) 137 | elif config.dataset.startswith('ModelNet40'): 138 | model = KernelPointCNN(dataset.flat_inputs, config) 139 | elif config.dataset.startswith('Semantic3D'): 140 | model = KernelPointFCNN(dataset.flat_inputs, config) 141 | else: 142 | raise ValueError('Unsupported dataset : ' + config.dataset) 143 | 144 | # Find all snapshot in the chosen training folder 145 | snap_path = os.path.join(path, 'snapshots') 146 | snap_steps = [int(f[:-5].split('-')[-1]) for f in os.listdir(snap_path) if f[-5:] == '.meta'] 147 | 148 | # Find which snapshot to restore 149 | chosen_step = np.sort(snap_steps)[step_ind] 150 | chosen_snap = os.path.join(path, 'snapshots', 'snap-{:d}'.format(chosen_step)) 151 | 152 | # Create a tester class 153 | visualizer = ModelVisualizer(model, restore_snap=chosen_snap) 154 | t2 = time.time() 155 | 156 | print('\n----------------') 157 | print('Done in {:.1f} s'.format(t2 - t1)) 158 | print('----------------\n') 159 | 160 | ##################### 161 | # Start visualization 162 | ##################### 163 | 164 | print('Start visualization') 165 | print('*******************\n') 166 | 167 | visualizer.show_deformable_kernels(model, dataset, deform_idx) 168 | 169 | 170 | # ---------------------------------------------------------------------------------------------------------------------- 171 | # 172 | # Main Call 173 | # \***************/ 174 | # 175 | 176 | 177 | if __name__ == '__main__': 178 | 179 | ############################### 180 | # Choose the model to visualize 181 | ############################### 182 | 183 | # 184 | # Here you can choose which model you want to test with the variable test_model. Here are the possible values : 185 | # 186 | # > 'last_ModelNet40': Automatically retrieve the last trained model on ModelNet40 187 | # 188 | # > 'last_ShapeNetPart': Automatically retrieve the last trained model on ShapeNetPart 189 | # 190 | # > 'last_S3DIS': Automatically retrieve the last trained model on S3DIS 191 | # 192 | # > 'results/Log_YYYY-MM-DD_HH-MM-SS': Directly provide the path of a trained model 193 | # 194 | 195 | chosen_log = 'results/Log_2019-03-27_20-31-49' # => NPM3D 196 | 197 | # 198 | # You can also choose the index of the snapshot to load (last by default) 199 | # 200 | 201 | chosen_snapshot = -1 202 | 203 | # 204 | # Eventually you can choose which feature is visualized (index of the deform operation in the network) 205 | # 206 | 207 | chosen_deformation = 0 208 | 209 | # 210 | # If you want to modify certain parameters in the Config class, for example, to stop augmenting the input data, 211 | # there is a section for it in the function "test_caller" defined above. 212 | # 213 | 214 | ########################### 215 | # Call the test initializer 216 | ########################### 217 | 218 | # Automatically retrieve the last trained model 219 | if chosen_log in ['last_ModelNet40', 'last_ShapeNetPart', 'last_S3DIS']: 220 | 221 | # Dataset name 222 | test_dataset = '_'.join(chosen_log.split('_')[1:]) 223 | 224 | # List all training logs 225 | logs = np.sort([os.path.join('results', f) for f in os.listdir('results') if f.startswith('Log')]) 226 | 227 | # Find the last log of asked dataset 228 | for log in logs[::-1]: 229 | log_config = Config() 230 | log_config.load(log) 231 | if log_config.dataset.startswith(test_dataset): 232 | chosen_log = log 233 | break 234 | 235 | if chosen_log in ['last_ModelNet40', 'last_ShapeNetPart', 'last_S3DIS']: 236 | raise ValueError('No log of the dataset "' + test_dataset + '" found') 237 | 238 | # Check if log exists 239 | if not os.path.exists(chosen_log): 240 | raise ValueError('The given log does not exists: ' + chosen_log) 241 | 242 | # Let's go 243 | visu_caller(chosen_log, chosen_snapshot, chosen_deformation) 244 | 245 | 246 | 247 | -------------------------------------------------------------------------------- /models/KPCNN_model.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Classification model 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Basic libs 26 | from os import makedirs 27 | from os.path import exists 28 | import time 29 | import tensorflow as tf 30 | import numpy as np 31 | 32 | # Convolution functions 33 | from models.network_blocks import assemble_CNN_blocks, classification_head, classification_loss 34 | 35 | 36 | # ---------------------------------------------------------------------------------------------------------------------- 37 | # 38 | # Model Class 39 | # \*****************/ 40 | # 41 | 42 | 43 | class KernelPointCNN: 44 | 45 | def __init__(self, flat_inputs, config): 46 | """ 47 | Initiate the model 48 | :param flat_inputs: List of input tensors (flatten) 49 | :param config: configuration class 50 | """ 51 | 52 | # Model parameters 53 | self.config = config 54 | 55 | # Path of the result folder 56 | if self.config.saving: 57 | if self.config.saving_path == None: 58 | self.saving_path = time.strftime('results/Log_%Y-%m-%d_%H-%M-%S', time.gmtime()) 59 | else: 60 | self.saving_path = self.config.saving_path 61 | if not exists(self.saving_path): 62 | makedirs(self.saving_path) 63 | 64 | ######## 65 | # Inputs 66 | ######## 67 | 68 | # Sort flatten inputs in a dictionary 69 | with tf.variable_scope('inputs'): 70 | self.inputs = dict() 71 | self.inputs['points'] = flat_inputs[:config.num_layers] 72 | self.inputs['neighbors'] = flat_inputs[config.num_layers:2 * config.num_layers] 73 | self.inputs['pools'] = flat_inputs[2 * config.num_layers:3 * config.num_layers] 74 | ind = 3 * config.num_layers 75 | self.inputs['features'] = flat_inputs[ind] 76 | ind += 1 77 | self.inputs['batch_weights'] = flat_inputs[ind] 78 | ind += 1 79 | self.inputs['in_batches'] = flat_inputs[ind] 80 | ind += 1 81 | self.inputs['out_batches'] = flat_inputs[ind] 82 | ind += 1 83 | self.inputs['labels'] = flat_inputs[ind] 84 | ind += 1 85 | self.labels = self.inputs['labels'] 86 | self.inputs['augment_scales'] = flat_inputs[ind] 87 | ind += 1 88 | self.inputs['augment_rotations'] = flat_inputs[ind] 89 | ind += 1 90 | self.inputs['object_inds'] = flat_inputs[ind] 91 | 92 | # Dropout placeholder 93 | self.dropout_prob = tf.placeholder(tf.float32, name='dropout_prob') 94 | 95 | ######## 96 | # Layers 97 | ######## 98 | 99 | # Create layers 100 | with tf.variable_scope('KernelPointNetwork'): 101 | F = assemble_CNN_blocks(self.inputs, 102 | self.config, 103 | self.dropout_prob) 104 | 105 | self.logits = classification_head(F[-1], 106 | self.config, 107 | self.dropout_prob) 108 | 109 | ######## 110 | # Losses 111 | ######## 112 | 113 | with tf.variable_scope('loss'): 114 | 115 | # Classification loss 116 | self.output_loss = classification_loss(self.logits, 117 | self.inputs) 118 | 119 | # Add regularization 120 | self.loss = self.regularization_losses() + self.output_loss 121 | 122 | return 123 | 124 | def regularization_losses(self): 125 | 126 | ##################### 127 | # Regularization loss 128 | ##################### 129 | 130 | # Get L2 norm of all weights 131 | regularization_losses = [tf.nn.l2_loss(v) for v in tf.global_variables() if 'weights' in v.name] 132 | self.regularization_loss = self.config.weights_decay * tf.add_n(regularization_losses) 133 | 134 | ############################## 135 | # Gaussian regularization loss 136 | ############################## 137 | 138 | gaussian_losses = [] 139 | for v in tf.global_variables(): 140 | if 'kernel_extents' in v.name: 141 | 142 | # Layer index 143 | layer = int(v.name.split('/')[1].split('_')[-1]) 144 | 145 | # Radius of convolution for this layer 146 | conv_radius = self.config.first_subsampling_dl * self.config.density_parameter * (2 ** (layer - 1)) 147 | 148 | # Target extent 149 | target_extent = np.float32(1.0 * conv_radius / np.power(self.config.num_kernel_points, 1 / 3)) 150 | gaussian_losses += [tf.nn.l2_loss(v - target_extent)] 151 | 152 | if len(gaussian_losses) > 0: 153 | self.gaussian_loss = self.config.gaussian_decay * tf.add_n(gaussian_losses) 154 | else: 155 | self.gaussian_loss = tf.constant(0, dtype=tf.float32) 156 | 157 | ############################# 158 | # Offsets regularization loss 159 | ############################# 160 | 161 | offset_losses = [] 162 | 163 | if self.config.offsets_loss == 'permissive': 164 | 165 | for op in tf.get_default_graph().get_operations(): 166 | if op.name.endswith('deformed_KP'): 167 | 168 | # Get deformed positions 169 | deformed_positions = op.outputs[0] 170 | 171 | # Layer index 172 | layer = int(op.name.split('/')[1].split('_')[-1]) 173 | 174 | # Radius of deformed convolution for this layer 175 | conv_radius = self.config.first_subsampling_dl * self.config.density_parameter * (2 ** layer) 176 | 177 | # Normalized KP locations 178 | KP_locs = deformed_positions/conv_radius 179 | 180 | # Loss will be zeros inside radius and linear outside radius 181 | # Mean => loss independent from the number of input points 182 | radius_outside = tf.maximum(0.0, tf.norm(KP_locs, axis=2) - 1.0) 183 | offset_losses += [tf.reduce_mean(radius_outside)] 184 | 185 | 186 | elif self.config.offsets_loss == 'fitting': 187 | 188 | for op in tf.get_default_graph().get_operations(): 189 | if op.name.endswith('deformed_d2'): 190 | 191 | # Get deformed distances 192 | deformed_d2 = op.outputs[0] 193 | 194 | # Layer index 195 | layer = int(op.name.split('/')[1].split('_')[-1]) 196 | 197 | # Radius of deformed convolution for this layer 198 | conv_radius = self.config.first_subsampling_dl * self.config.density_parameter * (2 ** layer) 199 | 200 | # Get the distance to closest input point 201 | KP_min_d2 = tf.reduce_min(deformed_d2, axis=1) 202 | 203 | # Normalize KP locations to be independant from layers 204 | KP_min_d2 = KP_min_d2 / (conv_radius**2) 205 | 206 | # Loss will be the square distance to closest input point. 207 | # Mean => loss independent from the number of input points 208 | offset_losses += [tf.reduce_mean(KP_min_d2)] 209 | 210 | elif self.config.offsets_loss != 'none': 211 | raise ValueError('Unknown offset loss') 212 | 213 | if len(offset_losses) > 0: 214 | self.offsets_loss = self.config.offsets_decay * tf.add_n(offset_losses) 215 | else: 216 | self.offsets_loss = tf.constant(0, dtype=tf.float32) 217 | 218 | return self.offsets_loss + self.gaussian_loss + self.regularization_loss 219 | 220 | def parameters_log(self): 221 | 222 | self.config.save(self.saving_path) 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | -------------------------------------------------------------------------------- /visualize_ERFs.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to test any model on any dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | # Common libs 25 | import time 26 | import os 27 | import numpy as np 28 | 29 | # My libs 30 | from utils.config import Config 31 | from utils.visualizer import ModelVisualizer 32 | from models.KPCNN_model import KernelPointCNN 33 | from models.KPFCNN_model import KernelPointFCNN 34 | 35 | # Datasets 36 | from datasets.ModelNet40 import ModelNet40Dataset 37 | from datasets.ShapeNetPart import ShapeNetPartDataset 38 | from datasets.S3DIS import S3DISDataset 39 | from datasets.Semantic3D import Semantic3DDataset 40 | from datasets.NPM3D import NPM3DDataset 41 | from datasets.Scannet import ScannetDataset 42 | 43 | 44 | # ---------------------------------------------------------------------------------------------------------------------- 45 | # 46 | # Utility functions 47 | # \***********************/ 48 | # 49 | 50 | 51 | def visu_caller(path, step_ind, relu_idx): 52 | 53 | ########################## 54 | # Initiate the environment 55 | ########################## 56 | 57 | # Choose which gpu to use 58 | GPU_ID = '0' 59 | 60 | # Set GPU visible device 61 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 62 | 63 | # Disable warnings 64 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 65 | 66 | ########################### 67 | # Load the model parameters 68 | ########################### 69 | 70 | # Load model parameters 71 | config = Config() 72 | config.load(path) 73 | 74 | ################################## 75 | # Change model parameters for test 76 | ################################## 77 | 78 | # Change parameters for the test here. For example, you can stop augmenting the input data. 79 | 80 | # No augmentation to avoid random inputs 81 | config.augment_scale_anisotropic = False 82 | config.augment_symmetries = [False, False, False] 83 | config.augment_rotation = 'none' 84 | config.augment_scale_min = 1.0 85 | config.augment_scale_max = 1.0 86 | config.augment_noise = 0.0 87 | config.augment_occlusion = 'none' 88 | config.augment_color = 1.0 89 | 90 | config.batch_num = 2 91 | config.in_radius = 5 92 | 93 | ############## 94 | # Prepare Data 95 | ############## 96 | 97 | print() 98 | print('Dataset Preparation') 99 | print('*******************') 100 | 101 | # Initiate dataset configuration 102 | if config.dataset.startswith('ModelNet40'): 103 | dataset = ModelNet40Dataset(config.input_threads) 104 | elif config.dataset == 'S3DIS': 105 | dataset = S3DISDataset(config.input_threads) 106 | on_val = True 107 | elif config.dataset == 'Scannet': 108 | dataset = ScannetDataset(config.input_threads, load_test=True) 109 | elif config.dataset.startswith('ShapeNetPart'): 110 | dataset = ShapeNetPartDataset(config.dataset.split('_')[1], config.input_threads) 111 | elif config.dataset == 'NPM3D': 112 | dataset = NPM3DDataset(config.input_threads, load_test=True) 113 | elif config.dataset == 'Semantic3D': 114 | dataset = Semantic3DDataset(config.input_threads) 115 | else: 116 | raise ValueError('Unsupported dataset : ' + config.dataset) 117 | 118 | # Create subsample clouds of the models 119 | dl0 = config.first_subsampling_dl 120 | dataset.load_subsampled_clouds(dl0) 121 | 122 | # Initiate ERF input pipeleine (only diff is that it is not random) 123 | dataset.init_ERF_input_pipeline(config) 124 | 125 | ############## 126 | # Define Model 127 | ############## 128 | 129 | print('Creating Model') 130 | print('**************\n') 131 | t1 = time.time() 132 | 133 | if config.dataset.startswith('ShapeNetPart'): 134 | model = KernelPointFCNN(dataset.flat_inputs, config) 135 | elif config.dataset.startswith('S3DIS'): 136 | model = KernelPointFCNN(dataset.flat_inputs, config) 137 | elif config.dataset.startswith('Scannet'): 138 | model = KernelPointFCNN(dataset.flat_inputs, config) 139 | elif config.dataset.startswith('NPM3D'): 140 | model = KernelPointFCNN(dataset.flat_inputs, config) 141 | elif config.dataset.startswith('ModelNet40'): 142 | model = KernelPointCNN(dataset.flat_inputs, config) 143 | elif config.dataset.startswith('Semantic3D'): 144 | model = KernelPointFCNN(dataset.flat_inputs, config) 145 | else: 146 | raise ValueError('Unsupported dataset : ' + config.dataset) 147 | 148 | # Find all snapshot in the chosen training folder 149 | snap_path = os.path.join(path, 'snapshots') 150 | snap_steps = [int(f[:-5].split('-')[-1]) for f in os.listdir(snap_path) if f[-5:] == '.meta'] 151 | 152 | # Find which snapshot to restore 153 | chosen_step = np.sort(snap_steps)[step_ind] 154 | chosen_snap = os.path.join(path, 'snapshots', 'snap-{:d}'.format(chosen_step)) 155 | 156 | # Create a tester class 157 | visualizer = ModelVisualizer(model, restore_snap=chosen_snap) 158 | t2 = time.time() 159 | 160 | print('\n----------------') 161 | print('Done in {:.1f} s'.format(t2 - t1)) 162 | print('----------------\n') 163 | 164 | ##################### 165 | # Start visualization 166 | ##################### 167 | 168 | print('Start visualization') 169 | print('*******************\n') 170 | 171 | visualizer.show_effective_recep_field(model, dataset, relu_idx) 172 | 173 | 174 | # ---------------------------------------------------------------------------------------------------------------------- 175 | # 176 | # Main Call 177 | # \***************/ 178 | # 179 | 180 | 181 | if __name__ == '__main__': 182 | 183 | ############################### 184 | # Choose the model to visualize 185 | ############################### 186 | 187 | # 188 | # Here you can choose which model you want to test with the variable test_model. Here are the possible values : 189 | # 190 | # > 'last_ModelNet40': Automatically retrieve the last trained model on ModelNet40 191 | # 192 | # > 'last_ShapeNetPart': Automatically retrieve the last trained model on ShapeNetPart 193 | # 194 | # > 'last_S3DIS': Automatically retrieve the last trained model on S3DIS 195 | # 196 | # > 'results/Log_YYYY-MM-DD_HH-MM-SS': Directly provide the path of a trained model 197 | # 198 | 199 | chosen_log = 'results/Log_2019-03-19_19-14-24' # => S3DIS, rigid KPConv 200 | 201 | # 202 | # You can also choose the index of the snapshot to load (last by default) 203 | # 204 | 205 | chosen_snapshot = -1 206 | 207 | # 208 | # Eventually you can choose which feature is visualized (index of the deform operation in the network) 209 | # 210 | 211 | chosen_relu = 20 212 | 213 | # 214 | # If you want to modify certain parameters in the Config class, for example, to stop augmenting the input data, 215 | # there is a section for it in the function "test_caller" defined above. 216 | # 217 | 218 | ########################### 219 | # Call the test initializer 220 | ########################### 221 | 222 | # Automatically retrieve the last trained model 223 | if chosen_log in ['last_ModelNet40', 'last_ShapeNetPart', 'last_S3DIS']: 224 | 225 | # Dataset name 226 | test_dataset = '_'.join(chosen_log.split('_')[1:]) 227 | 228 | # List all training logs 229 | logs = np.sort([os.path.join('results', f) for f in os.listdir('results') if f.startswith('Log')]) 230 | 231 | # Find the last log of asked dataset 232 | for log in logs[::-1]: 233 | log_config = Config() 234 | log_config.load(log) 235 | if log_config.dataset.startswith(test_dataset): 236 | chosen_log = log 237 | break 238 | 239 | if chosen_log in ['last_ModelNet40', 'last_ShapeNetPart', 'last_S3DIS']: 240 | raise ValueError('No log of the dataset "' + test_dataset + '" found') 241 | 242 | # Check if log exists 243 | if not os.path.exists(chosen_log): 244 | raise ValueError('The given log does not exists: ' + chosen_log) 245 | 246 | # Let's go 247 | visu_caller(chosen_log, chosen_snapshot, chosen_relu) 248 | 249 | 250 | 251 | -------------------------------------------------------------------------------- /tf_custom_ops/tf_neighbors/neighbors/neighbors.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "neighbors.h" 3 | 4 | 5 | void brute_neighbors(vector& queries, vector& supports, vector& neighbors_indices, float radius, int verbose) 6 | { 7 | 8 | // Initiate variables 9 | // ****************** 10 | 11 | // square radius 12 | float r2 = radius * radius; 13 | 14 | // indices 15 | int i0 = 0; 16 | 17 | // Counting vector 18 | int max_count = 0; 19 | vector> tmp(queries.size()); 20 | 21 | // Search neigbors indices 22 | // *********************** 23 | 24 | for (auto& p0 : queries) 25 | { 26 | int i = 0; 27 | for (auto& p : supports) 28 | { 29 | if ((p0 - p).sq_norm() < r2) 30 | { 31 | tmp[i0].push_back(i); 32 | if (tmp[i0].size() > max_count) 33 | max_count = tmp[i0].size(); 34 | } 35 | i++; 36 | } 37 | i0++; 38 | } 39 | 40 | // Reserve the memory 41 | neighbors_indices.resize(queries.size() * max_count); 42 | i0 = 0; 43 | for (auto& inds : tmp) 44 | { 45 | for (int j = 0; j < max_count; j++) 46 | { 47 | if (j < inds.size()) 48 | neighbors_indices[i0 * max_count + j] = inds[j]; 49 | else 50 | neighbors_indices[i0 * max_count + j] = -1; 51 | } 52 | i0++; 53 | } 54 | 55 | return; 56 | } 57 | 58 | void ordered_neighbors(vector& queries, 59 | vector& supports, 60 | vector& neighbors_indices, 61 | float radius) 62 | { 63 | 64 | // Initiate variables 65 | // ****************** 66 | 67 | // square radius 68 | float r2 = radius * radius; 69 | 70 | // indices 71 | int i0 = 0; 72 | 73 | // Counting vector 74 | int max_count = 0; 75 | float d2; 76 | vector> tmp(queries.size()); 77 | vector> dists(queries.size()); 78 | 79 | // Search neigbors indices 80 | // *********************** 81 | 82 | for (auto& p0 : queries) 83 | { 84 | int i = 0; 85 | for (auto& p : supports) 86 | { 87 | d2 = (p0 - p).sq_norm(); 88 | if (d2 < r2) 89 | { 90 | // Find order of the new point 91 | auto it = std::upper_bound(dists[i0].begin(), dists[i0].end(), d2); 92 | int index = std::distance(dists[i0].begin(), it); 93 | 94 | // Insert element 95 | dists[i0].insert(it, d2); 96 | tmp[i0].insert(tmp[i0].begin() + index, i); 97 | 98 | // Update max count 99 | if (tmp[i0].size() > max_count) 100 | max_count = tmp[i0].size(); 101 | } 102 | i++; 103 | } 104 | i0++; 105 | } 106 | 107 | // Reserve the memory 108 | neighbors_indices.resize(queries.size() * max_count); 109 | i0 = 0; 110 | for (auto& inds : tmp) 111 | { 112 | for (int j = 0; j < max_count; j++) 113 | { 114 | if (j < inds.size()) 115 | neighbors_indices[i0 * max_count + j] = inds[j]; 116 | else 117 | neighbors_indices[i0 * max_count + j] = -1; 118 | } 119 | i0++; 120 | } 121 | 122 | return; 123 | } 124 | 125 | void batch_ordered_neighbors(vector& queries, 126 | vector& supports, 127 | vector& q_batches, 128 | vector& s_batches, 129 | vector& neighbors_indices, 130 | float radius) 131 | { 132 | 133 | // Initiate variables 134 | // ****************** 135 | 136 | // square radius 137 | float r2 = radius * radius; 138 | 139 | // indices 140 | int i0 = 0; 141 | 142 | // Counting vector 143 | int max_count = 0; 144 | float d2; 145 | vector> tmp(queries.size()); 146 | vector> dists(queries.size()); 147 | 148 | // batch index 149 | int b = 0; 150 | int sum_qb = 0; 151 | int sum_sb = 0; 152 | 153 | 154 | // Search neigbors indices 155 | // *********************** 156 | 157 | for (auto& p0 : queries) 158 | { 159 | // Check if we changed batch 160 | if (i0 == sum_qb + q_batches[b]) 161 | { 162 | sum_qb += q_batches[b]; 163 | sum_sb += s_batches[b]; 164 | b++; 165 | } 166 | 167 | // Loop only over the supports of current batch 168 | vector::iterator p_it; 169 | int i = 0; 170 | for(p_it = supports.begin() + sum_sb; p_it < supports.begin() + sum_sb + s_batches[b]; p_it++ ) 171 | { 172 | d2 = (p0 - *p_it).sq_norm(); 173 | if (d2 < r2) 174 | { 175 | // Find order of the new point 176 | auto it = std::upper_bound(dists[i0].begin(), dists[i0].end(), d2); 177 | int index = std::distance(dists[i0].begin(), it); 178 | 179 | // Insert element 180 | dists[i0].insert(it, d2); 181 | tmp[i0].insert(tmp[i0].begin() + index, sum_sb + i); 182 | 183 | // Update max count 184 | if (tmp[i0].size() > max_count) 185 | max_count = tmp[i0].size(); 186 | } 187 | i++; 188 | } 189 | i0++; 190 | } 191 | 192 | // Reserve the memory 193 | neighbors_indices.resize(queries.size() * max_count); 194 | i0 = 0; 195 | for (auto& inds : tmp) 196 | { 197 | for (int j = 0; j < max_count; j++) 198 | { 199 | if (j < inds.size()) 200 | neighbors_indices[i0 * max_count + j] = inds[j]; 201 | else 202 | neighbors_indices[i0 * max_count + j] = supports.size(); 203 | } 204 | i0++; 205 | } 206 | 207 | return; 208 | } 209 | 210 | 211 | void batch_nanoflann_neighbors(vector& queries, 212 | vector& supports, 213 | vector& q_batches, 214 | vector& s_batches, 215 | vector& neighbors_indices, 216 | float radius) 217 | { 218 | 219 | // Initiate variables 220 | // ****************** 221 | 222 | // indices 223 | int i0 = 0; 224 | 225 | // Square radius 226 | float r2 = radius * radius; 227 | 228 | // Counting vector 229 | int max_count = 0; 230 | float d2; 231 | vector>> all_inds_dists(queries.size()); 232 | 233 | // batch index 234 | int b = 0; 235 | int sum_qb = 0; 236 | int sum_sb = 0; 237 | 238 | // Nanoflann related variables 239 | // *************************** 240 | 241 | // CLoud variable 242 | PointCloud current_cloud; 243 | 244 | // Tree parameters 245 | nanoflann::KDTreeSingleIndexAdaptorParams tree_params(10 /* max leaf */); 246 | 247 | // KDTree type definition 248 | typedef nanoflann::KDTreeSingleIndexAdaptor< nanoflann::L2_Simple_Adaptor , 249 | PointCloud, 250 | 3 > my_kd_tree_t; 251 | 252 | // Pointer to trees 253 | my_kd_tree_t* index; 254 | 255 | // Build KDTree for the first batch element 256 | current_cloud.pts = vector(supports.begin() + sum_sb, supports.begin() + sum_sb + s_batches[b]); 257 | index = new my_kd_tree_t(3, current_cloud, tree_params); 258 | index->buildIndex(); 259 | 260 | 261 | // Search neigbors indices 262 | // *********************** 263 | 264 | // Search params 265 | nanoflann::SearchParams search_params; 266 | search_params.sorted = true; 267 | 268 | for (auto& p0 : queries) 269 | { 270 | 271 | // Check if we changed batch 272 | if (i0 == sum_qb + q_batches[b]) 273 | { 274 | sum_qb += q_batches[b]; 275 | sum_sb += s_batches[b]; 276 | b++; 277 | 278 | // Change the points 279 | current_cloud.pts.clear(); 280 | current_cloud.pts = vector(supports.begin() + sum_sb, supports.begin() + sum_sb + s_batches[b]); 281 | 282 | // Build KDTree of the current element of the batch 283 | delete index; 284 | index = new my_kd_tree_t(3, current_cloud, tree_params); 285 | index->buildIndex(); 286 | } 287 | 288 | // Initial guess of neighbors size 289 | all_inds_dists[i0].reserve(max_count); 290 | 291 | // Find neighbors 292 | float query_pt[3] = { p0.x, p0.y, p0.z}; 293 | size_t nMatches = index->radiusSearch(query_pt, r2, all_inds_dists[i0], search_params); 294 | 295 | // Update max count 296 | if (nMatches > max_count) 297 | max_count = nMatches; 298 | 299 | // Increment query idx 300 | i0++; 301 | } 302 | 303 | // Reserve the memory 304 | neighbors_indices.resize(queries.size() * max_count); 305 | i0 = 0; 306 | sum_sb = 0; 307 | sum_qb = 0; 308 | b = 0; 309 | for (auto& inds_dists : all_inds_dists) 310 | { 311 | // Check if we changed batch 312 | if (i0 == sum_qb + q_batches[b]) 313 | { 314 | sum_qb += q_batches[b]; 315 | sum_sb += s_batches[b]; 316 | b++; 317 | } 318 | 319 | for (int j = 0; j < max_count; j++) 320 | { 321 | if (j < inds_dists.size()) 322 | neighbors_indices[i0 * max_count + j] = inds_dists[j].first + sum_sb; 323 | else 324 | neighbors_indices[i0 * max_count + j] = supports.size(); 325 | } 326 | i0++; 327 | } 328 | 329 | delete index; 330 | 331 | return; 332 | } 333 | 334 | -------------------------------------------------------------------------------- /cpp_wrappers/cpp_subsampling/wrapper.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "grid_subsampling/grid_subsampling.h" 4 | #include 5 | 6 | 7 | 8 | // docstrings for our module 9 | // ************************* 10 | 11 | static char module_docstring[] = "This module provides an interface for the subsampling of a pointcloud"; 12 | 13 | static char compute_docstring[] = "function subsampling a pointcloud"; 14 | 15 | 16 | // Declare the functions 17 | // ********************* 18 | 19 | static PyObject *grid_subsampling_compute(PyObject *self, PyObject *args, PyObject *keywds); 20 | 21 | 22 | // Specify the members of the module 23 | // ********************************* 24 | 25 | static PyMethodDef module_methods[] = 26 | { 27 | { "compute", (PyCFunction)grid_subsampling_compute, METH_VARARGS | METH_KEYWORDS, compute_docstring }, 28 | {NULL, NULL, 0, NULL} 29 | }; 30 | 31 | 32 | // Initialize the module 33 | // ********************* 34 | 35 | static struct PyModuleDef moduledef = 36 | { 37 | PyModuleDef_HEAD_INIT, 38 | "grid_subsampling", // m_name 39 | module_docstring, // m_doc 40 | -1, // m_size 41 | module_methods, // m_methods 42 | NULL, // m_reload 43 | NULL, // m_traverse 44 | NULL, // m_clear 45 | NULL, // m_free 46 | }; 47 | 48 | PyMODINIT_FUNC PyInit_grid_subsampling(void) 49 | { 50 | import_array(); 51 | return PyModule_Create(&moduledef); 52 | } 53 | 54 | 55 | // Actual wrapper 56 | // ************** 57 | 58 | static PyObject *grid_subsampling_compute(PyObject *self, PyObject *args, PyObject *keywds) 59 | { 60 | 61 | // Manage inputs 62 | // ************* 63 | 64 | // Args containers 65 | PyObject *points_obj = NULL; 66 | PyObject *features_obj = NULL; 67 | PyObject *classes_obj = NULL; 68 | 69 | // Keywords containers 70 | static char *kwlist[] = {"points", "features", "classes", "sampleDl", "method", "verbose", NULL }; 71 | float sampleDl = 0.1; 72 | const char *method_buffer = "barycenters"; 73 | int verbose = 0; 74 | 75 | // Parse the input 76 | if (!PyArg_ParseTupleAndKeywords(args, keywds, "O|$OOfsi", kwlist, &points_obj, &features_obj, &classes_obj, &sampleDl, &method_buffer, &verbose)) 77 | { 78 | PyErr_SetString(PyExc_RuntimeError, "Error parsing arguments"); 79 | return NULL; 80 | } 81 | 82 | // Get the method argument 83 | string method(method_buffer); 84 | 85 | // Interpret method 86 | if (method.compare("barycenters") && method.compare("voxelcenters")) 87 | { 88 | PyErr_SetString(PyExc_RuntimeError, "Error parsing method. Valid method names are \"barycenters\" and \"voxelcenters\" "); 89 | return NULL; 90 | } 91 | 92 | // Check if using features or classes 93 | bool use_feature = true, use_classes = true; 94 | if (features_obj == NULL) 95 | use_feature = false; 96 | if (classes_obj == NULL) 97 | use_classes = false; 98 | 99 | // Interpret the input objects as numpy arrays. 100 | PyObject *points_array = PyArray_FROM_OTF(points_obj, NPY_FLOAT, NPY_IN_ARRAY); 101 | PyObject *features_array = NULL; 102 | PyObject *classes_array = NULL; 103 | if (use_feature) 104 | features_array = PyArray_FROM_OTF(features_obj, NPY_FLOAT, NPY_IN_ARRAY); 105 | if (use_classes) 106 | classes_array = PyArray_FROM_OTF(classes_obj, NPY_INT, NPY_IN_ARRAY); 107 | 108 | // Verify data was load correctly. 109 | if (points_array == NULL) 110 | { 111 | Py_XDECREF(points_array); 112 | Py_XDECREF(classes_array); 113 | Py_XDECREF(features_array); 114 | PyErr_SetString(PyExc_RuntimeError, "Error converting input points to numpy arrays of type float32"); 115 | return NULL; 116 | } 117 | if (use_feature && features_array == NULL) 118 | { 119 | Py_XDECREF(points_array); 120 | Py_XDECREF(classes_array); 121 | Py_XDECREF(features_array); 122 | PyErr_SetString(PyExc_RuntimeError, "Error converting input features to numpy arrays of type float32"); 123 | return NULL; 124 | } 125 | if (use_classes && classes_array == NULL) 126 | { 127 | Py_XDECREF(points_array); 128 | Py_XDECREF(classes_array); 129 | Py_XDECREF(features_array); 130 | PyErr_SetString(PyExc_RuntimeError, "Error converting input classes to numpy arrays of type int32"); 131 | return NULL; 132 | } 133 | 134 | // Check that the input array respect the dims 135 | if ((int)PyArray_NDIM(points_array) != 2 || (int)PyArray_DIM(points_array, 1) != 3) 136 | { 137 | Py_XDECREF(points_array); 138 | Py_XDECREF(classes_array); 139 | Py_XDECREF(features_array); 140 | PyErr_SetString(PyExc_RuntimeError, "Wrong dimensions : points.shape is not (N, 3)"); 141 | return NULL; 142 | } 143 | if (use_feature && ((int)PyArray_NDIM(features_array) != 2)) 144 | { 145 | Py_XDECREF(points_array); 146 | Py_XDECREF(classes_array); 147 | Py_XDECREF(features_array); 148 | PyErr_SetString(PyExc_RuntimeError, "Wrong dimensions : features.shape is not (N, d)"); 149 | return NULL; 150 | } 151 | 152 | if (use_classes && (int)PyArray_NDIM(classes_array) > 2) 153 | { 154 | Py_XDECREF(points_array); 155 | Py_XDECREF(classes_array); 156 | Py_XDECREF(features_array); 157 | PyErr_SetString(PyExc_RuntimeError, "Wrong dimensions : classes.shape is not (N,) or (N, d)"); 158 | return NULL; 159 | } 160 | 161 | // Number of points 162 | int N = (int)PyArray_DIM(points_array, 0); 163 | 164 | // Dimension of the features 165 | int fdim = 0; 166 | if (use_feature) 167 | fdim = (int)PyArray_DIM(features_array, 1); 168 | 169 | //Dimension of labels 170 | int ldim = 1; 171 | if (use_classes && (int)PyArray_NDIM(classes_array) == 2) 172 | ldim = (int)PyArray_DIM(classes_array, 1); 173 | 174 | // Check that the input array respect the number of points 175 | if (use_feature && (int)PyArray_DIM(features_array, 0) != N) 176 | { 177 | Py_XDECREF(points_array); 178 | Py_XDECREF(classes_array); 179 | Py_XDECREF(features_array); 180 | PyErr_SetString(PyExc_RuntimeError, "Wrong dimensions : features.shape is not (N, d)"); 181 | return NULL; 182 | } 183 | if (use_classes && (int)PyArray_DIM(classes_array, 0) != N) 184 | { 185 | Py_XDECREF(points_array); 186 | Py_XDECREF(classes_array); 187 | Py_XDECREF(features_array); 188 | PyErr_SetString(PyExc_RuntimeError, "Wrong dimensions : classes.shape is not (N,) or (N, d)"); 189 | return NULL; 190 | } 191 | 192 | 193 | // Call the C++ function 194 | // ********************* 195 | 196 | // Create pyramid 197 | if (verbose > 0) 198 | cout << "Computing cloud pyramid with support points: " << endl; 199 | 200 | 201 | // Convert PyArray to Cloud C++ class 202 | vector original_points; 203 | vector original_features; 204 | vector original_classes; 205 | original_points = vector((PointXYZ*)PyArray_DATA(points_array), (PointXYZ*)PyArray_DATA(points_array) + N); 206 | if (use_feature) 207 | original_features = vector((float*)PyArray_DATA(features_array), (float*)PyArray_DATA(features_array) + N*fdim); 208 | if (use_classes) 209 | original_classes = vector((int*)PyArray_DATA(classes_array), (int*)PyArray_DATA(classes_array) + N*ldim); 210 | 211 | // Subsample 212 | vector subsampled_points; 213 | vector subsampled_features; 214 | vector subsampled_classes; 215 | grid_subsampling(original_points, 216 | subsampled_points, 217 | original_features, 218 | subsampled_features, 219 | original_classes, 220 | subsampled_classes, 221 | sampleDl, 222 | verbose); 223 | 224 | // Check result 225 | if (subsampled_points.size() < 1) 226 | { 227 | PyErr_SetString(PyExc_RuntimeError, "Error"); 228 | return NULL; 229 | } 230 | 231 | // Manage outputs 232 | // ************** 233 | 234 | // Dimension of input containers 235 | npy_intp* point_dims = new npy_intp[2]; 236 | point_dims[0] = subsampled_points.size(); 237 | point_dims[1] = 3; 238 | npy_intp* feature_dims = new npy_intp[2]; 239 | feature_dims[0] = subsampled_points.size(); 240 | feature_dims[1] = fdim; 241 | npy_intp* classes_dims = new npy_intp[2]; 242 | classes_dims[0] = subsampled_points.size(); 243 | classes_dims[1] = ldim; 244 | 245 | // Create output array 246 | PyObject *res_points_obj = PyArray_SimpleNew(2, point_dims, NPY_FLOAT); 247 | PyObject *res_features_obj = NULL; 248 | PyObject *res_classes_obj = NULL; 249 | PyObject *ret = NULL; 250 | 251 | // Fill output array with values 252 | size_t size_in_bytes = subsampled_points.size() * 3 * sizeof(float); 253 | memcpy(PyArray_DATA(res_points_obj), subsampled_points.data(), size_in_bytes); 254 | if (use_feature) 255 | { 256 | size_in_bytes = subsampled_points.size() * fdim * sizeof(float); 257 | res_features_obj = PyArray_SimpleNew(2, feature_dims, NPY_FLOAT); 258 | memcpy(PyArray_DATA(res_features_obj), subsampled_features.data(), size_in_bytes); 259 | } 260 | if (use_classes) 261 | { 262 | size_in_bytes = subsampled_points.size() * ldim * sizeof(int); 263 | res_classes_obj = PyArray_SimpleNew(2, classes_dims, NPY_INT); 264 | memcpy(PyArray_DATA(res_classes_obj), subsampled_classes.data(), size_in_bytes); 265 | } 266 | 267 | 268 | // Merge results 269 | if (use_feature && use_classes) 270 | ret = Py_BuildValue("NNN", res_points_obj, res_features_obj, res_classes_obj); 271 | else if (use_feature) 272 | ret = Py_BuildValue("NN", res_points_obj, res_features_obj); 273 | else if (use_classes) 274 | ret = Py_BuildValue("NN", res_points_obj, res_classes_obj); 275 | else 276 | ret = Py_BuildValue("N", res_points_obj); 277 | 278 | // Clean up 279 | // ******** 280 | 281 | Py_DECREF(points_array); 282 | Py_XDECREF(features_array); 283 | Py_XDECREF(classes_array); 284 | 285 | return ret; 286 | } -------------------------------------------------------------------------------- /test_any_model.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Callable script to test any model on any dataset 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ---------------------------------------------------------------------------------------------------------------------- 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | # Common libs 25 | import time 26 | import os 27 | import numpy as np 28 | 29 | # My libs 30 | from utils.config import Config 31 | from utils.tester import ModelTester 32 | from models.KPCNN_model import KernelPointCNN 33 | from models.KPFCNN_model import KernelPointFCNN 34 | 35 | # Datasets 36 | from datasets.ModelNet40 import ModelNet40Dataset 37 | from datasets.ShapeNetPart import ShapeNetPartDataset 38 | from datasets.S3DIS import S3DISDataset 39 | from datasets.Scannet import ScannetDataset 40 | from datasets.NPM3D import NPM3DDataset 41 | from datasets.Semantic3D import Semantic3DDataset 42 | from datasets.DALES import DalesDataset 43 | 44 | # ---------------------------------------------------------------------------------------------------------------------- 45 | # 46 | # Utility functions 47 | # \***********************/ 48 | # 49 | 50 | 51 | def test_caller(path, step_ind, on_val): 52 | 53 | ########################## 54 | # Initiate the environment 55 | ########################## 56 | 57 | # Choose which gpu to use 58 | GPU_ID = '0' 59 | 60 | # Set GPU visible device 61 | os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID 62 | 63 | # Disable warnings 64 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' 65 | 66 | ########################### 67 | # Load the model parameters 68 | ########################### 69 | 70 | # Load model parameters 71 | config = Config() 72 | config.load(path) 73 | 74 | ################################## 75 | # Change model parameters for test 76 | ################################## 77 | 78 | # Change parameters for the test here. For example, you can stop augmenting the input data. 79 | 80 | #config.augment_noise = 0.0001 81 | #config.augment_color = 1.0 82 | config.validation_size = 500 83 | #config.batch_num = 10 84 | 85 | ############## 86 | # Prepare Data 87 | ############## 88 | 89 | print() 90 | print('Dataset Preparation') 91 | print('*******************') 92 | 93 | # Initiate dataset configuration 94 | if config.dataset.startswith('ModelNet40'): 95 | dataset = ModelNet40Dataset(config.input_threads) 96 | elif config.dataset == 'S3DIS': 97 | dataset = S3DISDataset(config.input_threads) 98 | on_val = True 99 | elif config.dataset == 'Scannet': 100 | dataset = ScannetDataset(config.input_threads, load_test=(not on_val)) 101 | elif config.dataset.startswith('ShapeNetPart'): 102 | dataset = ShapeNetPartDataset(config.dataset.split('_')[1], config.input_threads) 103 | elif config.dataset == 'NPM3D': 104 | dataset = NPM3DDataset(config.input_threads, load_test=(not on_val)) 105 | elif config.dataset == 'DALES': 106 | dataset = DalesDataset(config.input_threads, load_test=(not on_val)) 107 | elif config.dataset == 'Semantic3D': 108 | dataset = Semantic3DDataset(config.input_threads) 109 | else: 110 | raise ValueError('Unsupported dataset : ' + config.dataset) 111 | 112 | # Create subsample clouds of the models 113 | dl0 = config.first_subsampling_dl 114 | dataset.load_subsampled_clouds(dl0) 115 | 116 | # Initialize input pipelines 117 | if on_val: 118 | dataset.init_input_pipeline(config) 119 | else: 120 | dataset.init_test_input_pipeline(config) 121 | 122 | 123 | ############## 124 | # Define Model 125 | ############## 126 | 127 | print('Creating Model') 128 | print('**************\n') 129 | t1 = time.time() 130 | 131 | if config.dataset.startswith('ShapeNetPart'): 132 | model = KernelPointFCNN(dataset.flat_inputs, config) 133 | elif config.dataset.startswith('S3DIS'): 134 | model = KernelPointFCNN(dataset.flat_inputs, config) 135 | elif config.dataset.startswith('Scannet'): 136 | model = KernelPointFCNN(dataset.flat_inputs, config) 137 | elif config.dataset.startswith('NPM3D'): 138 | model = KernelPointFCNN(dataset.flat_inputs, config) 139 | elif config.dataset.startswith('DALES'): 140 | model = KernelPointFCNN(dataset.flat_inputs, config) 141 | elif config.dataset.startswith('ModelNet40'): 142 | model = KernelPointCNN(dataset.flat_inputs, config) 143 | elif config.dataset.startswith('Semantic3D'): 144 | model = KernelPointFCNN(dataset.flat_inputs, config) 145 | else: 146 | raise ValueError('Unsupported dataset : ' + config.dataset) 147 | 148 | # Find all snapshot in the chosen training folder 149 | snap_path = os.path.join(path, 'snapshots') 150 | snap_steps = [int(f[:-5].split('-')[-1]) for f in os.listdir(snap_path) if f[-5:] == '.meta'] 151 | 152 | # Find which snapshot to restore 153 | chosen_step = np.sort(snap_steps)[step_ind] 154 | chosen_snap = os.path.join(path, 'snapshots', 'snap-{:d}'.format(chosen_step)) 155 | 156 | # Create a tester class 157 | tester = ModelTester(model, restore_snap=chosen_snap) 158 | t2 = time.time() 159 | 160 | print('\n----------------') 161 | print('Done in {:.1f} s'.format(t2 - t1)) 162 | print('----------------\n') 163 | 164 | ############ 165 | # Start test 166 | ############ 167 | 168 | print('Start Test') 169 | print('**********\n') 170 | 171 | if config.dataset.startswith('ShapeNetPart'): 172 | if config.dataset.split('_')[1] == 'multi': 173 | tester.test_multi_segmentation(model, dataset) 174 | else: 175 | tester.test_segmentation(model, dataset) 176 | elif config.dataset.startswith('S3DIS'): 177 | tester.test_cloud_segmentation_on_val(model, dataset) 178 | elif config.dataset.startswith('Scannet'): 179 | if on_val: 180 | tester.test_cloud_segmentation_on_val(model, dataset) 181 | else: 182 | tester.test_cloud_segmentation(model, dataset) 183 | elif config.dataset.startswith('Semantic3D'): 184 | if on_val: 185 | tester.test_cloud_segmentation_on_val(model, dataset) 186 | else: 187 | tester.test_cloud_segmentation(model, dataset) 188 | elif config.dataset.startswith('NPM3D'): 189 | if on_val: 190 | tester.test_cloud_segmentation_on_val(model, dataset) 191 | else: 192 | tester.test_cloud_segmentation(model, dataset) 193 | elif config.dataset.startswith('DALES'): 194 | if on_val: 195 | tester.test_cloud_segmentation_on_val(model, dataset) 196 | else: 197 | tester.test_cloud_segmentation(model, dataset) 198 | elif config.dataset.startswith('ModelNet40'): 199 | tester.test_classification(model, dataset) 200 | else: 201 | raise ValueError('Unsupported dataset') 202 | 203 | 204 | # ---------------------------------------------------------------------------------------------------------------------- 205 | # 206 | # Main Call 207 | # \***************/ 208 | # 209 | 210 | 211 | if __name__ == '__main__': 212 | 213 | ########################## 214 | # Choose the model to test 215 | ########################## 216 | 217 | # 218 | # Here you can choose which model you want to test with the variable test_model. Here are the possible values : 219 | # 220 | # > 'last_ModelNet40': Automatically retrieve the last trained model on ModelNet40 221 | # 222 | # > 'last_ShapeNetPart': Automatically retrieve the last trained model on ShapeNetPart 223 | # 224 | # > 'last_S3DIS': Automatically retrieve the last trained model on S3DIS 225 | # 226 | # > 'last_Scannet': Automatically retrieve the last trained model on Scannet 227 | # 228 | # > 'last_NPM3D': Automatically retrieve the last trained model on NPM3D 229 | # 230 | # > 'last_Semantic3D': Automatically retrieve the last trained model on Semantic3D 231 | # 232 | # > 'results/Log_YYYY-MM-DD_HH-MM-SS': Directly provide the path of a trained model 233 | # 234 | 235 | chosen_log = 'Log_2020-09-15_02-29-55' 236 | 237 | # 238 | # You can also choose the index of the snapshot to load (last by default) 239 | # 240 | 241 | chosen_snapshot = -1 242 | 243 | # 244 | # Eventually, you can choose to test your model on the validation set 245 | # 246 | 247 | on_val = False 248 | 249 | # 250 | # If you want to modify certain parameters in the Config class, for example, to stop augmenting the input data, 251 | # there is a section for it in the function "test_caller" defined above. 252 | # 253 | 254 | ########################### 255 | # Call the test initializer 256 | ########################### 257 | 258 | handled_logs = ['last_ModelNet40', 259 | 'last_ShapeNetPart', 260 | 'last_S3DIS', 261 | 'last_Scannet', 262 | 'last_NPM3D', 263 | 'last_Semantic3D'] 264 | 265 | # Automatically retrieve the last trained model 266 | if chosen_log in handled_logs: 267 | 268 | # Dataset name 269 | test_dataset = '_'.join(chosen_log.split('_')[1:]) 270 | 271 | # List all training logs 272 | logs = np.sort([os.path.join('results', f) for f in os.listdir('results') if f.startswith('Log')]) 273 | 274 | # Find the last log of asked dataset 275 | for log in logs[::-1]: 276 | log_config = Config() 277 | log_config.load(log) 278 | if log_config.dataset.startswith(test_dataset): 279 | chosen_log = log 280 | break 281 | 282 | if chosen_log in handled_logs: 283 | raise ValueError('No log of the dataset "' + test_dataset + '" found') 284 | 285 | # Check if log exists 286 | if not os.path.exists(chosen_log): 287 | raise ValueError('The given log does not exists: ' + chosen_log) 288 | 289 | # Let's go 290 | test_caller(chosen_log, chosen_snapshot, on_val) 291 | 292 | 293 | 294 | -------------------------------------------------------------------------------- /kernels/kernel_points.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 0=================================0 4 | # | Kernel Point Convolutions | 5 | # 0=================================0 6 | # 7 | # 8 | # ---------------------------------------------------------------------------------------------------------------------- 9 | # 10 | # Functions handling the disposition of kernel points. 11 | # 12 | # ---------------------------------------------------------------------------------------------------------------------- 13 | # 14 | # Hugues THOMAS - 11/06/2018 15 | # 16 | 17 | 18 | # ------------------------------------------------------------------------------------------ 19 | # 20 | # Imports and global variables 21 | # \**********************************/ 22 | # 23 | 24 | 25 | # Import numpy package and name it "np" 26 | import numpy as np 27 | import matplotlib.pyplot as plt 28 | from os import makedirs 29 | from os.path import join, exists 30 | 31 | from utils.ply import read_ply, write_ply 32 | 33 | 34 | # ------------------------------------------------------------------------------------------ 35 | # 36 | # Functions 37 | # \***************/ 38 | # 39 | # 40 | 41 | def kernel_point_optimization_debug(radius, num_points, num_kernels=1, dimension=3, fixed='center', ratio=1.0, verbose=0): 42 | """ 43 | Creation of kernel point via optimization of potentials. 44 | :param radius: Radius of the kernels 45 | :param num_points: points composing kernels 46 | :param num_kernels: number of wanted kernels 47 | :param dimension: dimension of the space 48 | :param fixed: fix position of certain kernel points ('none', 'center' or 'verticals') 49 | :param ratio: ratio of the radius where you want the kernels points to be placed 50 | :param verbose: display option 51 | :return: points [num_kernels, num_points, dimension] 52 | """ 53 | 54 | ####################### 55 | # Parameters definition 56 | ####################### 57 | 58 | # Radius used for optimization (points are rescaled afterwards) 59 | radius0 = 1 60 | diameter0 = 2 61 | 62 | # Factor multiplicating gradients for moving points (~learning rate) 63 | moving_factor = 1e-2 64 | continuous_moving_decay = 0.9995 65 | 66 | # Gradient threshold to stop optimization 67 | thresh = 1e-5 68 | 69 | # Gradient clipping value 70 | clip = 0.05 * radius0 71 | 72 | ####################### 73 | # Kernel initialization 74 | ####################### 75 | 76 | # Random kernel points 77 | kernel_points = np.random.rand(num_kernels * num_points - 1, dimension) * diameter0 - radius0 78 | while (kernel_points.shape[0] < num_kernels * num_points): 79 | new_points = np.random.rand(num_kernels * num_points - 1, dimension) * diameter0 - radius0 80 | kernel_points = np.vstack((kernel_points, new_points)) 81 | d2 = np.sum(np.power(kernel_points, 2), axis=1) 82 | kernel_points = kernel_points[d2 < 0.5 * radius0 * radius0, :] 83 | kernel_points = kernel_points[:num_kernels * num_points, :].reshape((num_kernels, num_points, -1)) 84 | 85 | # Optionnal fixing 86 | if fixed == 'center': 87 | kernel_points[:, 0, :] *= 0 88 | if fixed == 'verticals': 89 | kernel_points[:, :3, :] *= 0 90 | kernel_points[:, 1, -1] += 2 * radius0 / 3 91 | kernel_points[:, 2, -1] -= 2 * radius0 / 3 92 | 93 | ##################### 94 | # Kernel optimization 95 | ##################### 96 | 97 | # Initiate figure 98 | if verbose>1: 99 | fig = plt.figure() 100 | 101 | saved_gradient_norms = np.zeros((10000, num_kernels)) 102 | old_gradient_norms = np.zeros((num_kernels, num_points)) 103 | for iter in range(10000): 104 | 105 | # Compute gradients 106 | # ***************** 107 | 108 | # Derivative of the sum of potentials of all points 109 | A = np.expand_dims(kernel_points, axis=2) 110 | B = np.expand_dims(kernel_points, axis=1) 111 | interd2 = np.sum(np.power(A - B, 2), axis=-1) 112 | inter_grads = (A - B) / (np.power(np.expand_dims(interd2, -1), 3/2) + 1e-6) 113 | inter_grads = np.sum(inter_grads, axis=1) 114 | 115 | # Derivative of the radius potential 116 | circle_grads = 10*kernel_points 117 | 118 | # All gradients 119 | gradients = inter_grads + circle_grads 120 | 121 | if fixed == 'verticals': 122 | gradients[:, 1:3, :-1] = 0 123 | 124 | # Stop condition 125 | # ************** 126 | 127 | # Compute norm of gradients 128 | gradients_norms = np.sqrt(np.sum(np.power(gradients, 2), axis=-1)) 129 | saved_gradient_norms[iter, :] = np.max(gradients_norms, axis=1) 130 | 131 | # Stop if all moving points are gradients fixed (low gradients diff) 132 | 133 | if fixed == 'center' and np.max(np.abs(old_gradient_norms[:, 1:] - gradients_norms[:, 1:])) < thresh: 134 | break 135 | elif fixed == 'verticals' and np.max(np.abs(old_gradient_norms[:, 3:] - gradients_norms[:, 3:])) < thresh: 136 | break 137 | elif np.max(np.abs(old_gradient_norms - gradients_norms)) < thresh: 138 | break 139 | old_gradient_norms = gradients_norms 140 | 141 | # Move points 142 | # *********** 143 | 144 | # Clip gradient to get moving dists 145 | moving_dists = np.minimum(moving_factor * gradients_norms, clip) 146 | 147 | # Fix central point 148 | if fixed == 'center': 149 | moving_dists[:, 0] = 0 150 | if fixed == 'verticals': 151 | moving_dists[:, 0] = 0 152 | 153 | # Move points 154 | kernel_points -= np.expand_dims(moving_dists, -1) * gradients / np.expand_dims(gradients_norms + 1e-6, -1) 155 | 156 | if verbose: 157 | print('iter {:5d} / max grad = {:f}'.format(iter, np.max(gradients_norms[:, 3:]))) 158 | if verbose > 1: 159 | plt.clf() 160 | plt.plot(kernel_points[0, :, 0], kernel_points[0, :, 1], '.') 161 | circle = plt.Circle((0, 0), radius, color='r', fill=False) 162 | fig.axes[0].add_artist(circle) 163 | fig.axes[0].set_xlim((-radius*1.1, radius*1.1)) 164 | fig.axes[0].set_ylim((-radius*1.1, radius*1.1)) 165 | fig.axes[0].set_aspect('equal') 166 | plt.draw() 167 | plt.pause(0.001) 168 | plt.show(block=False) 169 | print(moving_factor) 170 | 171 | # moving factor decay 172 | moving_factor *= continuous_moving_decay 173 | 174 | # Rescale radius to fit the wanted ratio of radius 175 | r = np.sqrt(np.sum(np.power(kernel_points, 2), axis=-1)) 176 | kernel_points *= ratio / np.mean(r[:, 1:]) 177 | 178 | # Rescale kernels with real radius 179 | return kernel_points * radius, saved_gradient_norms 180 | 181 | 182 | def load_kernels(radius, num_kpoints, num_kernels, dimension, fixed): 183 | 184 | # Number of tries in the optimization process, to ensure we get the most stable disposition 185 | num_tries = 100 186 | 187 | # Kernel directory 188 | kernel_dir = 'kernels/dispositions' 189 | if not exists(kernel_dir): 190 | makedirs(kernel_dir) 191 | 192 | # Kernel_file 193 | if dimension == 3: 194 | kernel_file = join(kernel_dir, 'k_{:03d}_{:s}.ply'.format(num_kpoints, fixed)) 195 | elif dimension == 2: 196 | kernel_file = join(kernel_dir, 'k_{:03d}_{:s}_2D.ply'.format(num_kpoints, fixed)) 197 | else: 198 | raise ValueError('Unsupported dimpension of kernel : ' + str(dimension)) 199 | 200 | # Check if already done 201 | if not exists(kernel_file): 202 | 203 | # Create kernels 204 | kernel_points, grad_norms = kernel_point_optimization_debug(1.0, 205 | num_kpoints, 206 | num_kernels=num_tries, 207 | dimension=dimension, 208 | fixed=fixed, 209 | verbose=0) 210 | 211 | # Find best candidate 212 | best_k = np.argmin(grad_norms[-1, :]) 213 | 214 | # Save points 215 | original_kernel = kernel_points[best_k, :, :] 216 | write_ply(kernel_file, original_kernel, ['x', 'y', 'z']) 217 | 218 | else: 219 | data = read_ply(kernel_file) 220 | original_kernel = np.vstack((data['x'], data['y'], data['z'])).T 221 | 222 | # N.B. 2D kernels are not supported yet 223 | if dimension == 2: 224 | return original_kernel 225 | 226 | # Random rotations depending of the fixed points 227 | if fixed == 'verticals': 228 | 229 | # Create random rotations 230 | thetas = np.random.rand(num_kernels) * 2 * np.pi 231 | c, s = np.cos(thetas), np.sin(thetas) 232 | R = np.zeros((num_kernels, 3, 3), dtype=np.float32) 233 | R[:, 0, 0] = c 234 | R[:, 1, 1] = c 235 | R[:, 2, 2] = 1 236 | R[:, 0, 1] = s 237 | R[:, 1, 0] = -s 238 | 239 | # Scale kernels 240 | original_kernel = radius * np.expand_dims(original_kernel, 0) 241 | 242 | # Rotate kernels 243 | kernels = np.matmul(original_kernel, R) 244 | 245 | else: 246 | 247 | # Create random rotations 248 | u = np.ones((num_kernels, 3)) 249 | v = np.ones((num_kernels, 3)) 250 | wrongs = np.abs(np.sum(u * v, axis=1)) > 0.99 251 | while np.any(wrongs): 252 | new_u = np.random.rand(num_kernels, 3) * 2 - 1 253 | new_u = new_u / np.expand_dims(np.linalg.norm(new_u, axis=1) + 1e-9, -1) 254 | u[wrongs, :] = new_u[wrongs, :] 255 | new_v = np.random.rand(num_kernels, 3) * 2 - 1 256 | new_v = new_v / np.expand_dims(np.linalg.norm(new_v, axis=1) + 1e-9, -1) 257 | v[wrongs, :] = new_v[wrongs, :] 258 | wrongs = np.abs(np.sum(u * v, axis=1)) > 0.99 259 | 260 | # Make v perpendicular to u 261 | v -= np.expand_dims(np.sum(u * v, axis=1), -1) * u 262 | v = v / np.expand_dims(np.linalg.norm(v, axis=1) + 1e-9, -1) 263 | 264 | # Last rotation vector 265 | w = np.cross(u, v) 266 | R = np.stack((u, v, w), axis=-1) 267 | 268 | # Scale kernels 269 | original_kernel = radius * np.expand_dims(original_kernel, 0) 270 | 271 | # Rotate kernels 272 | kernels = np.matmul(original_kernel, R) 273 | 274 | # Add a small noise 275 | kernels = kernels 276 | kernels = kernels + np.random.normal(scale=radius*0.01, size=kernels.shape) 277 | 278 | return kernels 279 | 280 | 281 | 282 | 283 | --------------------------------------------------------------------------------