├── chainer_pointnet ├── models │ ├── __init__.py │ ├── kdnet │ │ ├── __init__.py │ │ ├── kdconv.py │ │ ├── kdnet_cls.py │ │ ├── kdnet_seg.py │ │ └── kddeconv.py │ ├── kdcontextnet │ │ ├── __init__.py │ │ ├── kdcontextnet_cls.py │ │ ├── kdcontextdeconv_block.py │ │ ├── kdcontextconv_block.py │ │ └── kdcontextnet_seg.py │ ├── pointnet │ │ ├── __init__.py │ │ ├── transform_net.py │ │ ├── pointnet_cls.py │ │ └── pointnet_seg.py │ ├── pointnet2 │ │ ├── __init__.py │ │ ├── pointnet2_cls_ssg.py │ │ ├── pointnet2_seg_ssg.py │ │ ├── pointnet2_cls_msg.py │ │ ├── set_abstraction_all_block.py │ │ ├── feature_propagation_block.py │ │ └── set_abstraction_block.py │ ├── linear_block.py │ └── conv_block.py ├── utils │ ├── __init__.py │ ├── sampling.py │ ├── grouping.py │ └── kdtree.py ├── _version.py └── __init__.py ├── experiments ├── s3dis │ ├── third_party │ │ ├── meta │ │ │ ├── class_names.txt │ │ │ ├── area6_data_label.txt │ │ │ ├── all_data_label.txt │ │ │ └── anno_paths.txt │ │ ├── download_data.sh │ │ ├── collect_indoor3d_data.py │ │ ├── collect_indoor3d_data_manual.py │ │ ├── gen_indoor3d_h5.py │ │ └── data_prep_util.py │ ├── s3dis_dataset.py │ └── train.py └── modelnet40 │ ├── data_check.py │ ├── ply_dataset.py │ ├── provider.py │ └── train.py ├── setup.py ├── LICENSE ├── .gitignore └── README.md /chainer_pointnet/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chainer_pointnet/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdnet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdcontextnet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chainer_pointnet/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.0' 2 | -------------------------------------------------------------------------------- /chainer_pointnet/__init__.py: -------------------------------------------------------------------------------- 1 | from chainer_pointnet import _version # NOQA 2 | 3 | __version__ = _version.__version__ 4 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/meta/class_names.txt: -------------------------------------------------------------------------------- 1 | ceiling 2 | floor 3 | wall 4 | beam 5 | column 6 | window 7 | door 8 | table 9 | chair 10 | sofa 11 | bookcase 12 | board 13 | clutter 14 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/download_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Download HDF5 for indoor 3d semantic segmentation (around 1.6GB) 4 | wget https://shapenet.cs.stanford.edu/media/indoor3d_sem_seg_hdf5_data.zip 5 | unzip indoor3d_sem_seg_hdf5_data.zip 6 | rm indoor3d_sem_seg_hdf5_data.zip 7 | 8 | -------------------------------------------------------------------------------- /experiments/modelnet40/data_check.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | 5 | # data is downloaded when importing provider 6 | import provider 7 | 8 | 9 | if __name__ == '__main__': 10 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 11 | if not os.path.exists(BASE_DIR): 12 | os.mkdir(BASE_DIR) 13 | 14 | # ModelNet40 official train/test split 15 | TRAIN_FILES = provider.getDataFiles( 16 | os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/train_files.txt')) 17 | TEST_FILES = provider.getDataFiles( 18 | os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/test_files.txt')) 19 | 20 | # these are list of file names for train h5/test h5 files resp. 21 | print('train_files', TRAIN_FILES) 22 | print('test_files', TEST_FILES) 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | import imp 3 | import os 4 | 5 | from setuptools import find_packages 6 | 7 | setup_requires = [] 8 | install_requires = [ 9 | 'chainer>=2.0', 10 | # 'chainer-chemistry>=0.3.0', 11 | ] 12 | 13 | 14 | here = os.path.abspath(os.path.dirname(__file__)) 15 | __version__ = imp.load_source( 16 | '_version', os.path.join(here, 17 | 'chainer_pointnet', '_version.py')).__version__ 18 | 19 | setup(name='chainer-pointnet', 20 | version=__version__, 21 | description='Chainer PointNet', 22 | author='corochann', 23 | author_email='', 24 | packages=find_packages(), 25 | license='MIT', 26 | # url='', 27 | setup_requires=setup_requires, 28 | install_requires=install_requires 29 | ) 30 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/collect_indoor3d_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | ROOT_DIR = os.path.dirname(BASE_DIR) 5 | sys.path.append(BASE_DIR) 6 | import indoor3d_util 7 | 8 | anno_paths = [line.rstrip() for line in open(os.path.join(BASE_DIR, 'meta/anno_paths.txt'))] 9 | anno_paths = [os.path.join(indoor3d_util.DATA_PATH, p) for p in anno_paths] 10 | 11 | output_folder = os.path.join(ROOT_DIR, 'data/stanford_indoor3d') 12 | if not os.path.exists(output_folder): 13 | os.mkdir(output_folder) 14 | 15 | # Note: there is an extra character in the v1.2 data in Area_5/hallway_6. It's fixed manually. 16 | for anno_path in anno_paths: 17 | print(anno_path) 18 | try: 19 | elements = anno_path.split('/') 20 | out_filename = elements[-3]+'_'+elements[-2]+'.npy' # Area_1_hallway_1.npy 21 | indoor3d_util.collect_point_label(anno_path, os.path.join(output_folder, out_filename), 'numpy') 22 | except: 23 | print(anno_path, 'ERROR!!') 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 corochann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/collect_indoor3d_data_manual.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | ROOT_DIR = os.path.dirname(BASE_DIR) 5 | sys.path.append(BASE_DIR) 6 | import indoor3d_util 7 | 8 | # anno_paths = [line.rstrip() for line in open(os.path.join(BASE_DIR, 'meta/anno_paths.txt'))] 9 | # anno_paths = [os.path.join(indoor3d_util.DATA_PATH, p) for p in anno_paths] 10 | 11 | output_folder = os.path.join(ROOT_DIR, 'data/stanford_indoor3d') 12 | if not os.path.exists(output_folder): 13 | os.mkdir(output_folder) 14 | 15 | # Note: there is an extra character in the v1.2 data in Area_5/hallway_6. It's fixed manually. 16 | anno_paths = [ 17 | '/home/nakago/workspace/chainer-pointnet/experiments/s3dis/data/Stanford3dDataset_v1.2_Aligned_Version/Area_5/hallway_6/Annotations',] 18 | for anno_path in anno_paths: 19 | print(anno_path) 20 | try: 21 | elements = anno_path.split('/') 22 | out_filename = elements[-3]+'_'+elements[-2]+'.npy' # Area_1_hallway_1.npy 23 | indoor3d_util.collect_point_label(anno_path, os.path.join(output_folder, out_filename), 'numpy') 24 | except Exception as e: 25 | print(e) 26 | import IPython; IPython.embed() 27 | print(anno_path, 'ERROR!!') 28 | -------------------------------------------------------------------------------- /chainer_pointnet/models/linear_block.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions 3 | from chainer import links 4 | 5 | 6 | class LinearBlock(chainer.Chain): 7 | 8 | def __init__(self, in_size, out_size=None, nobias=False, 9 | initialW=None, initial_bias=None, use_bn=True, 10 | activation=functions.relu, dropout_ratio=-1, residual=False): 11 | super(LinearBlock, self).__init__() 12 | with self.init_scope(): 13 | self.linear = links.Linear( 14 | in_size, out_size=out_size, nobias=nobias, 15 | initialW=initialW, initial_bias=initial_bias) 16 | if use_bn: 17 | self.bn = links.BatchNormalization(out_size) 18 | self.activation = activation 19 | self.use_bn = use_bn 20 | self.dropout_ratio = dropout_ratio 21 | self.residual = residual 22 | 23 | def __call__(self, x): 24 | if self.use_bn: 25 | h = self.bn(self.linear(x)) 26 | else: 27 | h = self.linear(x) 28 | if self.activation is not None: 29 | h = self.activation(h) 30 | if self.residual: 31 | raise NotImplementedError('not implemented yet') 32 | if self.dropout_ratio >= 0: 33 | h = functions.dropout(h, ratio=self.dropout_ratio) 34 | return h 35 | -------------------------------------------------------------------------------- /chainer_pointnet/models/conv_block.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions 3 | from chainer import links 4 | 5 | 6 | class ConvBlock(chainer.Chain): 7 | 8 | def __init__(self, in_channels, out_channels, ksize=None, stride=1, pad=0, 9 | nobias=False, initialW=None, initial_bias=None, use_bn=True, 10 | activation=functions.relu, dropout_ratio=-1, residual=False): 11 | super(ConvBlock, self).__init__() 12 | with self.init_scope(): 13 | self.conv = links.Convolution2D( 14 | in_channels, out_channels, ksize=ksize, stride=stride, pad=pad, 15 | nobias=nobias, initialW=initialW, initial_bias=initial_bias) 16 | if use_bn: 17 | self.bn = links.BatchNormalization(out_channels) 18 | self.activation = activation 19 | self.use_bn = use_bn 20 | self.dropout_ratio = dropout_ratio 21 | self.residual = residual 22 | 23 | def __call__(self, x): 24 | if self.use_bn: 25 | h = self.bn(self.conv(x)) 26 | else: 27 | h = self.conv(x) 28 | if self.activation is not None: 29 | h = self.activation(h) 30 | if self.residual: 31 | from chainerex.functions import residual_add 32 | h = residual_add(h, x) 33 | if self.dropout_ratio >= 0: 34 | h = functions.dropout(h, ratio=self.dropout_ratio) 35 | return h 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # PyCharm 107 | .idea/ 108 | 109 | # Datasets 110 | examples/modelnet40/data/modelnet40_ply_hdf5_2048/ 111 | experiments/modelnet40/data/modelnet40_ply_hdf5_2048/ 112 | experiments/s3dis/data/ 113 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/meta/area6_data_label.txt: -------------------------------------------------------------------------------- 1 | data/stanford_indoor3d/Area_6_conferenceRoom_1.npy 2 | data/stanford_indoor3d/Area_6_copyRoom_1.npy 3 | data/stanford_indoor3d/Area_6_hallway_1.npy 4 | data/stanford_indoor3d/Area_6_hallway_2.npy 5 | data/stanford_indoor3d/Area_6_hallway_3.npy 6 | data/stanford_indoor3d/Area_6_hallway_4.npy 7 | data/stanford_indoor3d/Area_6_hallway_5.npy 8 | data/stanford_indoor3d/Area_6_hallway_6.npy 9 | data/stanford_indoor3d/Area_6_lounge_1.npy 10 | data/stanford_indoor3d/Area_6_office_10.npy 11 | data/stanford_indoor3d/Area_6_office_11.npy 12 | data/stanford_indoor3d/Area_6_office_12.npy 13 | data/stanford_indoor3d/Area_6_office_13.npy 14 | data/stanford_indoor3d/Area_6_office_14.npy 15 | data/stanford_indoor3d/Area_6_office_15.npy 16 | data/stanford_indoor3d/Area_6_office_16.npy 17 | data/stanford_indoor3d/Area_6_office_17.npy 18 | data/stanford_indoor3d/Area_6_office_18.npy 19 | data/stanford_indoor3d/Area_6_office_19.npy 20 | data/stanford_indoor3d/Area_6_office_1.npy 21 | data/stanford_indoor3d/Area_6_office_20.npy 22 | data/stanford_indoor3d/Area_6_office_21.npy 23 | data/stanford_indoor3d/Area_6_office_22.npy 24 | data/stanford_indoor3d/Area_6_office_23.npy 25 | data/stanford_indoor3d/Area_6_office_24.npy 26 | data/stanford_indoor3d/Area_6_office_25.npy 27 | data/stanford_indoor3d/Area_6_office_26.npy 28 | data/stanford_indoor3d/Area_6_office_27.npy 29 | data/stanford_indoor3d/Area_6_office_28.npy 30 | data/stanford_indoor3d/Area_6_office_29.npy 31 | data/stanford_indoor3d/Area_6_office_2.npy 32 | data/stanford_indoor3d/Area_6_office_30.npy 33 | data/stanford_indoor3d/Area_6_office_31.npy 34 | data/stanford_indoor3d/Area_6_office_32.npy 35 | data/stanford_indoor3d/Area_6_office_33.npy 36 | data/stanford_indoor3d/Area_6_office_34.npy 37 | data/stanford_indoor3d/Area_6_office_35.npy 38 | data/stanford_indoor3d/Area_6_office_36.npy 39 | data/stanford_indoor3d/Area_6_office_37.npy 40 | data/stanford_indoor3d/Area_6_office_3.npy 41 | data/stanford_indoor3d/Area_6_office_4.npy 42 | data/stanford_indoor3d/Area_6_office_5.npy 43 | data/stanford_indoor3d/Area_6_office_6.npy 44 | data/stanford_indoor3d/Area_6_office_7.npy 45 | data/stanford_indoor3d/Area_6_office_8.npy 46 | data/stanford_indoor3d/Area_6_office_9.npy 47 | data/stanford_indoor3d/Area_6_openspace_1.npy 48 | data/stanford_indoor3d/Area_6_pantry_1.npy 49 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/pointnet2_cls_ssg.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions 3 | from chainer import links 4 | from chainer import reporter 5 | 6 | from chainer_pointnet.models.linear_block import LinearBlock 7 | from chainer_pointnet.models.pointnet2.set_abstraction_all_block import \ 8 | SetAbstractionGroupAllModule 9 | from chainer_pointnet.models.pointnet2.set_abstraction_block import \ 10 | SetAbstractionModule 11 | 12 | 13 | class PointNet2ClsSSG(chainer.Chain): 14 | 15 | """Classification PointNet++ SSG 16 | 17 | Input is (minibatch, K, N, 1), output is (minibatch, out_dim) 18 | 19 | Args: 20 | out_dim (int): output dimension, number of class for classification 21 | in_dim: input dimension for each point. default is 3, (x, y, z). 22 | dropout_ratio (float): dropout ratio 23 | use_bn (bool): use batch normalization or not. 24 | compute_accuracy (bool): compute & report accuracy or not 25 | residual (bool): use residual connection or not 26 | """ 27 | 28 | def __init__(self, out_dim, in_dim=3, dropout_ratio=0.5, 29 | use_bn=True, compute_accuracy=True, residual=False): 30 | super(PointNet2ClsSSG, self).__init__() 31 | with self.init_scope(): 32 | self.sam1 = SetAbstractionModule( 33 | k=512, num_sample_in_region=32, radius=0.2, 34 | mlp=[64, 64, 128], mlp2=None, residual=residual) 35 | self.sam2 = SetAbstractionModule( 36 | k=128, num_sample_in_region=64, radius=0.4, 37 | mlp=[128, 128, 256], mlp2=None, residual=residual) 38 | # k, num_sample_in_region, radius are ignored when group_all=True 39 | self.sam3 = SetAbstractionGroupAllModule( 40 | mlp=[256, 512, 1024], mlp2=None, residual=residual) 41 | 42 | self.fc_block4 = LinearBlock( 43 | 1024, 512, use_bn=use_bn, dropout_ratio=dropout_ratio,) 44 | self.fc_block5 = LinearBlock( 45 | 512, 256, use_bn=use_bn, dropout_ratio=dropout_ratio,) 46 | self.fc6 = links.Linear(256, out_dim) 47 | 48 | self.compute_accuracy = compute_accuracy 49 | 50 | def calc(self, x): 51 | # x: (minibatch, K, N, 1) 52 | # N - num_point 53 | # K - feature degree (this is 3 for xyz input, 64 for middle layer) 54 | assert x.ndim == 4 55 | assert x.shape[-1] == 1 56 | 57 | coord_points = functions.transpose(x[:, :, :, 0], (0, 2, 1)) 58 | # h: feature_points 59 | h = None 60 | coord_points, h, _ = self.sam1(coord_points, h) 61 | coord_points, h, _ = self.sam2(coord_points, h) 62 | coord_points, h = self.sam3(coord_points, h) 63 | # coord (bs, k, coord), h: feature (bs, k, ch') 64 | h = self.fc_block4(h) 65 | h = self.fc_block5(h) 66 | h = self.fc6(h) 67 | return h 68 | 69 | def __call__(self, x, t): 70 | h = self.calc(x) 71 | cls_loss = functions.softmax_cross_entropy(h, t) 72 | # reporter.report({'cls_loss': cls_loss}, self) 73 | loss = cls_loss 74 | reporter.report({'loss': loss}, self) 75 | if self.compute_accuracy: 76 | acc = functions.accuracy(h, t) 77 | reporter.report({'accuracy': acc}, self) 78 | return loss 79 | 80 | -------------------------------------------------------------------------------- /experiments/modelnet40/ply_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import chainer 4 | from chainer.datasets.concatenated_dataset import ConcatenatedDataset 5 | import numpy as np 6 | 7 | # data is downloaded when importing provider 8 | import provider 9 | 10 | 11 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 12 | 13 | 14 | class PlyDataset(chainer.dataset.DatasetMixin): 15 | """This if partial dataset""" 16 | 17 | def __init__(self, h5_filepath, num_point=1024, augment=False): 18 | print('loading ', h5_filepath) 19 | data, label = provider.loadDataFile(h5_filepath) 20 | assert len(data) == len(label) 21 | # data: (2048, 2048, 3) - (batchsize, point, xyz) 22 | # Reduce num point here. 23 | self.data = data[:, :num_point, :].astype(np.float32) 24 | # (2048,) - (batchsize,) 25 | self.label = np.squeeze(label).astype(np.int32) 26 | self.augment = augment 27 | self.num_point = num_point 28 | self.length = len(data) 29 | print('length ', self.length) 30 | 31 | def __len__(self): 32 | """return length of this dataset""" 33 | return self.length 34 | 35 | def get_example(self, i): 36 | """Return i-th data""" 37 | if self.augment: 38 | rotated_data = provider.rotate_point_cloud( 39 | self.data[i:i + 1, :, :]) 40 | jittered_data = provider.jitter_point_cloud(rotated_data) 41 | point_data = jittered_data[0] 42 | else: 43 | point_data = self.data[i] 44 | # pint_data (2048, 3): (num_point, k) --> convert to (k, num_point, 1) 45 | point_data = np.transpose( 46 | point_data.astype(np.float32), (1, 0))[:, :, None] 47 | assert point_data.dtype == np.float32 48 | assert self.label[i].dtype == np.int32 49 | return point_data, self.label[i] 50 | 51 | 52 | def get_train_dataset(num_point=1024): 53 | print('get train num_point ', num_point) 54 | train_files = provider.getDataFiles( 55 | os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/train_files.txt')) 56 | return ConcatenatedDataset( 57 | *(PlyDataset(filepath, num_point=num_point, augment=True) for filepath in train_files)) 58 | 59 | 60 | def get_test_dataset(num_point=1024): 61 | print('get test num_point ', num_point) 62 | test_files = provider.getDataFiles( 63 | os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/test_files.txt')) 64 | return ConcatenatedDataset( 65 | *(PlyDataset(filepath, num_point=num_point, augment=False) for filepath in test_files)) 66 | 67 | 68 | 69 | 70 | 71 | if __name__ == '__main__': 72 | # --- PlyDataset check --- 73 | train_files = provider.getDataFiles( 74 | os.path.join(BASE_DIR, 'data/modelnet40_ply_hdf5_2048/train_files.txt')) 75 | d0 = PlyDataset(train_files[0], augment=True) 76 | data_point, label = d0[3] 77 | print('data_point', data_point.shape, 'label', label) 78 | 79 | # --- Total dataset check --- 80 | train = get_train_dataset() 81 | test = get_test_dataset() 82 | # import IPython; IPython.embed() 83 | print('train', len(train), 'test', len(test)) 84 | 85 | convert_to_kdtree = True 86 | if convert_to_kdtree: 87 | from chainer.datasets import TransformDataset 88 | from chainer_pointnet.utils.kdtree import TransformKDTreeCls 89 | train = TransformDataset(train, TransformKDTreeCls()) 90 | points, split_dims, t = train[1] 91 | print(points.shape, split_dims.shape, t) 92 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet/transform_net.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions 5 | from chainer import initializers 6 | from chainer import links 7 | 8 | from chainer_pointnet.models.conv_block import ConvBlock 9 | 10 | 11 | class TransformModule(chainer.Chain): 12 | 13 | """Transform module 14 | 15 | This class produces transform matrix 16 | Input is (minibatch, K, N, 1), output is (minibatch, K, K) 17 | 18 | Args: 19 | k (int): hidden layer's coordinate dimension 20 | use_bn (bool): use batch normalization or not 21 | residual (bool): use residual connection or not 22 | """ 23 | 24 | def __init__(self, k=3, use_bn=True, residual=False): 25 | super(TransformModule, self).__init__() 26 | initial_bias = numpy.identity(k, dtype=numpy.float32).ravel() 27 | with self.init_scope(): 28 | self.conv_block1 = ConvBlock(k, 64, ksize=1, use_bn=use_bn, 29 | residual=residual) 30 | self.conv_block2 = ConvBlock(64, 128, ksize=1, use_bn=use_bn, 31 | residual=residual) 32 | self.conv_block3 = ConvBlock(128, 1024, ksize=1, use_bn=use_bn, 33 | residual=residual) 34 | # [Note] 35 | # Original paper uses BN for fc layer as well. 36 | # https://github.com/charlesq34/pointnet/blob/master/models/transform_nets.py#L34 37 | # This chanier impl. skip BN for fc layer 38 | self.fc4 = links.Linear(1024, 512) 39 | # self.bn4 = links.BatchNormalization(512) 40 | self.fc5 = links.Linear(512, 256) 41 | # self.bn5 = links.BatchNormalization(256) 42 | 43 | # initial output of transform net should be identity 44 | self.fc6 = links.Linear( 45 | 256, k * k, initialW=initializers.Zero(dtype=numpy.float32), 46 | initial_bias=initial_bias) 47 | self.k = k 48 | 49 | def __call__(self, x): 50 | # reference --> x: (minibatch, N, 1, K) <- original tf impl. 51 | # x: (minibatch, K, N, 1) <- chainer impl. 52 | # N - num_point 53 | # K - feature degree (this is 3 for xyz input, 64 for middle layer) 54 | h = self.conv_block1(x) 55 | h = self.conv_block2(h) 56 | h = self.conv_block3(h) 57 | h = functions.max_pooling_2d(h, ksize=h.shape[2:]) 58 | # h: (minibatch, K, 1, 1) 59 | h = functions.relu(self.fc4(h)) 60 | h = functions.relu(self.fc5(h)) 61 | h = self.fc6(h) 62 | bs, k2 = h.shape 63 | assert k2 == self.k ** 2 64 | h = functions.reshape(h, (bs, self.k, self.k)) 65 | return h 66 | 67 | 68 | class TransformNet(chainer.Chain): 69 | """Transform Network 70 | 71 | This class can be used for Both InputTransformNet & FeatureTransformNet 72 | Input is (minibatch, K, N, 1), 73 | output is (minibatch, K, N, 1), which is transformed 74 | 75 | Args: 76 | k (int): hidden layer's coordinate dimension 77 | use_bn (bool): use batch normalization or not 78 | residual (bool): use residual connection or not 79 | """ 80 | 81 | def __init__(self, k=3, use_bn=True, residual=False): 82 | super(TransformNet, self).__init__() 83 | with self.init_scope(): 84 | self.trans_module = TransformModule( 85 | k=k, use_bn=use_bn, residual=residual) 86 | 87 | def __call__(self, x): 88 | t = self.trans_module(x) 89 | # t: (minibatch, K, K) 90 | # x: (minibatch, K, N, 1) 91 | # h: (minibatch, K, N) 92 | # K = in_dim 93 | h = functions.matmul(t, x[:, :, :, 0]) 94 | bs, k, n = h.shape 95 | h = functions.reshape(h, (bs, k, n, 1)) 96 | return h, t 97 | -------------------------------------------------------------------------------- /experiments/s3dis/s3dis_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import h5py 4 | 5 | import numpy as np 6 | from chainer_chemistry.datasets.numpy_tuple_dataset import NumpyTupleDataset 7 | 8 | 9 | MAX_NUM_POINT = 4096 10 | 11 | 12 | def load_h5(h5_filename): 13 | f = h5py.File(h5_filename) 14 | data = f['data'][:] 15 | label = f['label'][:] 16 | return data, label 17 | 18 | 19 | def get_dataset(test_area_int=6, num_point=4096): 20 | assert isinstance(test_area_int, int) 21 | assert num_point <= MAX_NUM_POINT 22 | 23 | data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') 24 | room_filelist = [line.rstrip() for line in open( 25 | os.path.join(data_dir, 'indoor3d_sem_seg_hdf5_data/room_filelist.txt'))] 26 | 27 | # Load ALL data 28 | data_batch_list = [] 29 | label_batch_list = [] 30 | i = 0 31 | 32 | # for h5_filename in all_files: 33 | while True: 34 | h5_filename = os.path.join( 35 | data_dir, 'indoor3d_sem_seg_hdf5_data/ply_data_all_{}.h5'.format(i)) 36 | if not os.path.exists(h5_filename): 37 | print('exit at i={}'.format(i)) 38 | break 39 | print('open {}'.format(h5_filename)) 40 | data_batch, label_batch = load_h5(h5_filename) 41 | data_batch_list.append(data_batch) 42 | label_batch_list.append(label_batch) 43 | i += 1 44 | 45 | data_batches = np.concatenate(data_batch_list, 0) 46 | label_batches = np.concatenate(label_batch_list, 0) 47 | print(data_batches.shape) # (23585, 4096, 9) batchsize, num_point, ch 48 | print(label_batches.shape) # (23585, 4096) batchsize, num_point 49 | # reduce point number num_point 50 | assert data_batches.ndim == 3 51 | assert label_batches.ndim == 2 52 | assert data_batches.shape[0] == label_batches.shape[0] 53 | assert data_batches.shape[1] == MAX_NUM_POINT 54 | assert label_batches.shape[1] == MAX_NUM_POINT 55 | data_batches = data_batches[:, :num_point, :].astype(np.float32) 56 | label_batches = label_batches[:, :num_point].astype(np.int32) 57 | # data_batches (batch_size, num_point, k) -> (batch_size, k, num_point, 1) 58 | data_batches = np.transpose(data_batches, (0, 2, 1))[:, :, :, None] 59 | 60 | # test_area = 'Area_'+str(FLAGS.test_area) 61 | test_area = 'Area_' + str(test_area_int) 62 | train_idxs = [] 63 | test_idxs = [] 64 | for i, room_name in enumerate(room_filelist): 65 | if test_area in room_name: 66 | test_idxs.append(i) 67 | else: 68 | train_idxs.append(i) 69 | 70 | train_data = data_batches[train_idxs, ...] 71 | train_label = label_batches[train_idxs] 72 | test_data = data_batches[test_idxs, ...] 73 | test_label = label_batches[test_idxs] 74 | print('train shape', train_data.shape, train_label.shape) 75 | print('test shape', test_data.shape, test_label.shape) 76 | train = NumpyTupleDataset(train_data, train_label) 77 | test = NumpyTupleDataset(test_data, test_label) 78 | return train, test 79 | 80 | 81 | if __name__ == '__main__': 82 | train, test = get_dataset() 83 | print('train', len(train), 'test', len(test)) 84 | 85 | train_x, train_y = train[3] 86 | print('train_x', train_x.shape, 'train_y', train_y.shape) 87 | test_x, test_y = test[3] 88 | print('test_x', test_x.shape, 'test_y', test_y.shape) 89 | 90 | convert_to_kdtree = True 91 | if convert_to_kdtree: 92 | from chainer.datasets import TransformDataset 93 | from chainer_pointnet.utils.kdtree import TransformKDTreeSeg, \ 94 | calc_max_level 95 | num_point = train_x.shape[1] 96 | max_level = calc_max_level(num_point) 97 | train = TransformDataset(train, TransformKDTreeSeg(max_level=max_level)) 98 | points, split_dims, t = train[1] 99 | print('transformed', points.shape, split_dims.shape, t.shape) 100 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/gen_indoor3d_h5.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import sys 4 | # sys.path.append(BASE_DIR) 5 | # sys.path.append(os.path.join(ROOT_DIR, 'utils')) 6 | import data_prep_util 7 | import indoor3d_util 8 | 9 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 10 | ROOT_DIR = os.path.dirname(BASE_DIR) 11 | # Constants 12 | data_dir = os.path.join(ROOT_DIR, 'data') 13 | indoor3d_data_dir = os.path.join(data_dir, 'stanford_indoor3d') 14 | NUM_POINT = 4096 15 | H5_BATCH_SIZE = 1000 16 | data_dim = [NUM_POINT, 9] 17 | label_dim = [NUM_POINT] 18 | data_dtype = 'float32' 19 | label_dtype = 'uint8' 20 | 21 | # Set paths 22 | filelist = os.path.join(BASE_DIR, 'meta/all_data_label.txt') 23 | data_label_files = [os.path.join(indoor3d_data_dir, line.rstrip()) for line in open(filelist)] 24 | output_dir = os.path.join(data_dir, 'indoor3d_sem_seg_hdf5_data') 25 | if not os.path.exists(output_dir): 26 | os.mkdir(output_dir) 27 | output_filename_prefix = os.path.join(output_dir, 'ply_data_all') 28 | output_room_filelist = os.path.join(output_dir, 'room_filelist.txt') 29 | fout_room = open(output_room_filelist, 'w') 30 | 31 | # -------------------------------------- 32 | # ----- BATCH WRITE TO HDF5 ----- 33 | # -------------------------------------- 34 | batch_data_dim = [H5_BATCH_SIZE] + data_dim 35 | batch_label_dim = [H5_BATCH_SIZE] + label_dim 36 | h5_batch_data = np.zeros(batch_data_dim, dtype=np.float32) 37 | h5_batch_label = np.zeros(batch_label_dim, dtype=np.uint8) 38 | buffer_size = 0 # state: record how many samples are currently in buffer 39 | h5_index = 0 # state: the next h5 file to save 40 | 41 | 42 | def insert_batch(data, label, last_batch=False): 43 | global h5_batch_data, h5_batch_label 44 | global buffer_size, h5_index 45 | data_size = data.shape[0] 46 | # If there is enough space, just insert 47 | if buffer_size + data_size <= h5_batch_data.shape[0]: 48 | h5_batch_data[buffer_size:buffer_size+data_size, ...] = data 49 | h5_batch_label[buffer_size:buffer_size+data_size] = label 50 | buffer_size += data_size 51 | else: # not enough space 52 | capacity = h5_batch_data.shape[0] - buffer_size 53 | assert(capacity>=0) 54 | if capacity > 0: 55 | h5_batch_data[buffer_size:buffer_size+capacity, ...] = data[0:capacity, ...] 56 | h5_batch_label[buffer_size:buffer_size+capacity, ...] = label[0:capacity, ...] 57 | # Save batch data and label to h5 file, reset buffer_size 58 | h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5' 59 | data_prep_util.save_h5(h5_filename, h5_batch_data, h5_batch_label, data_dtype, label_dtype) 60 | print('Stored {0} with size {1}'.format(h5_filename, h5_batch_data.shape[0])) 61 | h5_index += 1 62 | buffer_size = 0 63 | # recursive call 64 | insert_batch(data[capacity:, ...], label[capacity:, ...], last_batch) 65 | if last_batch and buffer_size > 0: 66 | h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5' 67 | data_prep_util.save_h5(h5_filename, h5_batch_data[0:buffer_size, ...], h5_batch_label[0:buffer_size, ...], data_dtype, label_dtype) 68 | print('Stored {0} with size {1}'.format(h5_filename, buffer_size)) 69 | h5_index += 1 70 | buffer_size = 0 71 | return 72 | 73 | 74 | sample_cnt = 0 75 | for i, data_label_filename in enumerate(data_label_files): 76 | print(data_label_filename) 77 | data, label = indoor3d_util.room2blocks_wrapper_normalized( 78 | data_label_filename, NUM_POINT, block_size=1.0, stride=0.5, 79 | random_sample=False, sample_num=None) 80 | print('{0}, {1}'.format(data.shape, label.shape)) 81 | for _ in range(data.shape[0]): 82 | fout_room.write(os.path.basename(data_label_filename)[0:-4]+'\n') 83 | 84 | sample_cnt += data.shape[0] 85 | insert_batch(data, label, i == len(data_label_files)-1) 86 | 87 | fout_room.close() 88 | print("Total samples: {0}".format(sample_cnt)) 89 | -------------------------------------------------------------------------------- /experiments/modelnet40/provider.py: -------------------------------------------------------------------------------- 1 | """ 2 | Original author: @charlesq34 3 | This code is copied from 4 | https://github.com/charlesq34/pointnet/blob/master/provider.py 5 | 6 | [Note] 7 | It uses `wget` and it does not work on Windows. 8 | """ 9 | import os 10 | import sys 11 | import numpy as np 12 | import h5py 13 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 14 | sys.path.append(BASE_DIR) 15 | 16 | # Download dataset for point cloud classification 17 | DATA_DIR = os.path.join(BASE_DIR, 'data') 18 | if not os.path.exists(DATA_DIR): 19 | os.mkdir(DATA_DIR) 20 | if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')): 21 | www = 'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip' 22 | zipfile = os.path.basename(www) 23 | os.system('wget %s; unzip %s' % (www, zipfile)) 24 | os.system('mv %s %s' % (zipfile[:-4], DATA_DIR)) 25 | os.system('rm %s' % (zipfile)) 26 | 27 | 28 | def shuffle_data(data, labels): 29 | """ Shuffle data and labels. 30 | Input: 31 | data: B,N,... numpy array 32 | label: B,... numpy array 33 | Return: 34 | shuffled data, label and shuffle indices 35 | """ 36 | idx = np.arange(len(labels)) 37 | np.random.shuffle(idx) 38 | return data[idx, ...], labels[idx], idx 39 | 40 | 41 | def rotate_point_cloud(batch_data): 42 | """ Randomly rotate the point clouds to augument the dataset 43 | rotation is per shape based along up direction 44 | Input: 45 | BxNx3 array, original batch of point clouds 46 | Return: 47 | BxNx3 array, rotated batch of point clouds 48 | """ 49 | rotated_data = np.zeros(batch_data.shape, dtype=np.float32) 50 | for k in range(batch_data.shape[0]): 51 | rotation_angle = np.random.uniform() * 2 * np.pi 52 | cosval = np.cos(rotation_angle) 53 | sinval = np.sin(rotation_angle) 54 | rotation_matrix = np.array([[cosval, 0, sinval], 55 | [0, 1, 0], 56 | [-sinval, 0, cosval]]) 57 | shape_pc = batch_data[k, ...] 58 | rotated_data[k, ...] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix) 59 | return rotated_data 60 | 61 | 62 | def rotate_point_cloud_by_angle(batch_data, rotation_angle): 63 | """ Rotate the point cloud along up direction with certain angle. 64 | Input: 65 | BxNx3 array, original batch of point clouds 66 | Return: 67 | BxNx3 array, rotated batch of point clouds 68 | """ 69 | rotated_data = np.zeros(batch_data.shape, dtype=np.float32) 70 | for k in range(batch_data.shape[0]): 71 | #rotation_angle = np.random.uniform() * 2 * np.pi 72 | cosval = np.cos(rotation_angle) 73 | sinval = np.sin(rotation_angle) 74 | rotation_matrix = np.array([[cosval, 0, sinval], 75 | [0, 1, 0], 76 | [-sinval, 0, cosval]]) 77 | shape_pc = batch_data[k, ...] 78 | rotated_data[k, ...] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix) 79 | return rotated_data 80 | 81 | 82 | def jitter_point_cloud(batch_data, sigma=0.01, clip=0.05): 83 | """ Randomly jitter points. jittering is per point. 84 | Input: 85 | BxNx3 array, original batch of point clouds 86 | Return: 87 | BxNx3 array, jittered batch of point clouds 88 | """ 89 | B, N, C = batch_data.shape 90 | assert(clip > 0) 91 | jittered_data = np.clip(sigma * np.random.randn(B, N, C), -1*clip, clip) 92 | jittered_data += batch_data 93 | return jittered_data 94 | 95 | def getDataFiles(list_filename): 96 | return [line.rstrip() for line in open(list_filename)] 97 | 98 | def load_h5(h5_filename): 99 | f = h5py.File(h5_filename) 100 | data = f['data'][:] 101 | label = f['label'][:] 102 | return (data, label) 103 | 104 | def loadDataFile(filename): 105 | return load_h5(filename) 106 | 107 | def load_h5_data_label_seg(h5_filename): 108 | f = h5py.File(h5_filename) 109 | data = f['data'][:] 110 | label = f['label'][:] 111 | seg = f['pid'][:] 112 | return (data, label, seg) 113 | 114 | 115 | def loadDataFile_with_seg(filename): 116 | return load_h5_data_label_seg(filename) 117 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdnet/kdconv.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions, links 3 | 4 | 5 | class KDConv(chainer.Chain): 6 | 7 | """KD-Convolution. apply eq (1) or (2) of the paper 8 | 9 | Escape from Cells: Deep Kd-Networks for the Recognition of 10 | 3D Point Cloud Models. 11 | """ 12 | 13 | def __init__(self, in_channels, out_channels, ksize=1, stride=1, pad=0, 14 | nobias=False, initialW=None, initial_bias=None, use_bn=True, 15 | activation=functions.relu, cdim=3): 16 | # cdim: coordinate dimension, usually 3 (x, y, z). 17 | super(KDConv, self).__init__() 18 | in_ch = None if in_channels is None else in_channels * 2 19 | with self.init_scope(): 20 | self.conv = links.Convolution2D( 21 | in_ch, out_channels * cdim, ksize=ksize, 22 | stride=stride, pad=pad, nobias=nobias, initialW=initialW, 23 | initial_bias=initial_bias) 24 | if use_bn: 25 | self.bn = links.BatchNormalization(out_channels) 26 | self.out_channels = out_channels 27 | self.cdim = cdim 28 | self.activation = activation 29 | self.use_bn = use_bn 30 | 31 | def __call__(self, x, split_dim): 32 | """KDConv makes feature of 1-level parent node of KDTree. 33 | 34 | `num_point` of input `x` will be half in the `output`. 35 | 36 | Args: 37 | x (numpy.ndarray): 38 | 4-dim array (batchsize, in_channels, num_point, 1) 39 | split_dim (numpy.ndarray): 1d array with dtype=object with 40 | length=max_level. `split_dim[i]` is numpy array with dtype=int, 41 | represents i-th level split dimension axis. 42 | 43 | Returns (numpy.ndarray): 44 | 4-dim array (batchsize, out_channels, num_point//2, 1) 45 | 46 | """ 47 | assert x.ndim == 4 48 | assert x.shape[2] // 2 == split_dim.shape[1],\ 49 | 'x.shape {}, split_dim.shape{}'.format(x.shape, split_dim.shape) 50 | # x: (batch_size, ch, num_point, 1) 51 | bs, ch, num_point, w = x.shape 52 | assert w == 1 53 | x = functions.reshape(x, (bs, ch, num_point//2, 2)) 54 | x = functions.transpose(x, (0, 1, 3, 2)) 55 | x = functions.reshape(x, (bs, ch * 2, num_point//2, 1)) 56 | x = self.conv(x) 57 | # split_dim: (batch_size, num_point//2) dtype=np.int32 58 | # x: (batch_size, out_channels, cdim, num_point//2, 1) 59 | x = functions.reshape(x, (bs, self.out_channels, self.cdim, 60 | num_point//2)) 61 | # select `split_dim`'s output (extract KDTree's split axis conv result) 62 | x = x[self.xp.arange(bs)[:, None], :, split_dim, 63 | self.xp.arange(num_point//2)[None, :]] 64 | # x: (batch_size, num_point//2, out_channels) 65 | x = functions.transpose(x, (0, 2, 1)) 66 | x = functions.reshape(x, (bs, self.out_channels, num_point//2, 1)) 67 | # x: (batch_size, out_channels, num_point//2, 1) 68 | if self.use_bn: 69 | x = self.bn(x) 70 | if self.activation is not None: 71 | x = self.activation(x) 72 | return x 73 | 74 | 75 | if __name__ == '__main__': 76 | import numpy 77 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 78 | 79 | batchsize = 1 80 | num_point = 135 # try 100, 128, 135 81 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 82 | dim = 3 83 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 84 | point_set2 = numpy.random.rand(num_point, dim).astype(numpy.float32) 85 | print('point_set', point_set.shape) 86 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 87 | point_set, max_level=7, calc_split_positions=True) 88 | points2, split_dims2, inds2, kdtree2, split_positions2 = construct_kdtree_data( 89 | point_set2, max_level=7, calc_split_positions=True) 90 | print('points', points.shape) # 128 point here! 91 | kdconv = KDConv(3, 8) 92 | split_dim = numpy.array([split_dims[-1], split_dims2[-1]]) 93 | print('split_dim', split_dim.shape) 94 | pts = numpy.array([points, points2]) 95 | pts = numpy.transpose(pts, (0, 2, 1))[:, :, :, None] 96 | print('pts', pts.shape) 97 | out = kdconv(pts, split_dim) 98 | print('out', out.shape) 99 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/pointnet2_seg_ssg.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions 3 | from chainer import links 4 | from chainer import reporter 5 | 6 | from chainer_pointnet.models.conv_block import ConvBlock 7 | from chainer_pointnet.models.pointnet2.feature_propagation_block import \ 8 | FeaturePropagationModule 9 | from chainer_pointnet.models.pointnet2.set_abstraction_block import \ 10 | SetAbstractionModule 11 | 12 | 13 | class PointNet2SegSSG(chainer.Chain): 14 | 15 | """Segmentation PointNet++ SSG 16 | 17 | Input is (minibatch, K, N, 1), output is (minibatch, out_dim, N) 18 | 19 | Args: 20 | out_dim (int): output dimension, number of class for classification 21 | in_dim (int): input dimension for each point. default is 3, (x, y, z). 22 | dropout_ratio (float): dropout ratio 23 | use_bn (bool): use batch normalization or not. 24 | compute_accuracy (bool): compute & report accuracy or not 25 | residual (bool): use residual connection or not 26 | """ 27 | 28 | def __init__(self, out_dim, in_dim=3, dropout_ratio=0.5, 29 | use_bn=True, compute_accuracy=True, residual=False): 30 | super(PointNet2SegSSG, self).__init__() 31 | with self.init_scope(): 32 | self.sam1 = SetAbstractionModule( 33 | k=1024, num_sample_in_region=32, radius=0.1, mlp=[32, 32, 64], 34 | mlp2=None, use_bn=use_bn, return_distance=True, 35 | residual=residual) 36 | self.sam2 = SetAbstractionModule( 37 | k=256, num_sample_in_region=32, radius=0.2, mlp=[64, 64, 128], 38 | mlp2=None, use_bn=use_bn, return_distance=True, 39 | residual=residual) 40 | self.sam3 = SetAbstractionModule( 41 | k=64, num_sample_in_region=32, radius=0.4, mlp=[128, 128, 256], 42 | mlp2=None, use_bn=use_bn, return_distance=True, 43 | residual=residual) 44 | self.sam4 = SetAbstractionModule( 45 | k=16, num_sample_in_region=32, radius=0.8, mlp=[256, 256, 512], 46 | mlp2=None, use_bn=use_bn, return_distance=True, 47 | residual=residual) 48 | 49 | self.fpm5 = FeaturePropagationModule( 50 | mlp=[256, 256], use_bn=use_bn, residual=residual) 51 | self.fpm6 = FeaturePropagationModule( 52 | mlp=[256, 256], use_bn=use_bn, residual=residual) 53 | self.fpm7 = FeaturePropagationModule( 54 | mlp=[256, 128], use_bn=use_bn, residual=residual) 55 | self.fpm8 = FeaturePropagationModule( 56 | mlp=[128, 128, 128], use_bn=use_bn, residual=residual) 57 | self.conv_block9 = ConvBlock( 58 | 128, 128, ksize=1, use_bn=use_bn, residual=residual) 59 | self.conv10 = links.Convolution2D(128, out_dim, ksize=1) 60 | 61 | self.compute_accuracy = compute_accuracy 62 | 63 | def calc(self, x): 64 | # x: (minibatch, K, N, 1) 65 | # N - num_point 66 | # K - feature degree (this is 3 for xyz input, 64 for middle layer) 67 | assert x.ndim == 4 68 | assert x.shape[-1] == 1 69 | 70 | # TODO: consider support using only XYZ information like 71 | # coord_points = functions.transpose(x[:, :3, :, 0], (0, 2, 1)) 72 | coord_points = functions.transpose(x[:, :, :, 0], (0, 2, 1)) 73 | # h: feature_points (bs, num_point, ch) 74 | h0 = None 75 | coord_points, h1, d1 = self.sam1(coord_points, h0) 76 | coord_points, h2, d2 = self.sam2(coord_points, h1) 77 | coord_points, h3, d3 = self.sam3(coord_points, h2) 78 | coord_points, h4, d4 = self.sam4(coord_points, h3) 79 | 80 | del coord_points 81 | h3 = self.fpm5(d4, h3, h4) 82 | del h4, d4 83 | h2 = self.fpm6(d3, h2, h3) 84 | del h3, d3 85 | h1 = self.fpm7(d2, h1, h2) 86 | del h2, d2 87 | h0 = self.fpm8(d1, h0, h1) 88 | del h1, d1 89 | h = functions.transpose(h0, (0, 2, 1))[:, :, :, None] 90 | h = self.conv_block9(h) 91 | h = self.conv10(h) 92 | return h[:, :, :, 0] 93 | 94 | def __call__(self, x, t): 95 | h = self.calc(x) 96 | 97 | bs, ch, n = h.shape 98 | h = functions.reshape(functions.transpose(h, (0, 2, 1)), (bs * n, ch)) 99 | t = functions.reshape(t, (bs * n,)) 100 | cls_loss = functions.softmax_cross_entropy(h, t) 101 | # reporter.report({'cls_loss': cls_loss}, self) 102 | loss = cls_loss 103 | reporter.report({'loss': loss}, self) 104 | if self.compute_accuracy: 105 | acc = functions.accuracy(h, t) 106 | reporter.report({'accuracy': acc}, self) 107 | return loss 108 | 109 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/pointnet2_cls_msg.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions, cuda 5 | from chainer import links 6 | from chainer import reporter 7 | 8 | from chainer_pointnet.models.linear_block import LinearBlock 9 | from chainer_pointnet.models.pointnet2.set_abstraction_all_block import \ 10 | SetAbstractionGroupAllModule 11 | from chainer_pointnet.models.pointnet2.set_abstraction_block import \ 12 | SetAbstractionModule 13 | 14 | 15 | class PointNet2ClsMSG(chainer.Chain): 16 | 17 | """Classification PointNet++ MSG (Multi Scale Grouping) 18 | 19 | Input is (minibatch, K, N, 1), output is (minibatch, out_dim) 20 | 21 | Args: 22 | out_dim (int): output dimension, number of class for classification 23 | in_dim: input dimension for each point. default is 3, (x, y, z). 24 | dropout_ratio (float): dropout ratio 25 | use_bn (bool): use batch normalization or not. 26 | compute_accuracy (bool): compute & report accuracy or not 27 | residual (bool): use residual connection or not 28 | """ 29 | 30 | def __init__(self, out_dim, in_dim=3, dropout_ratio=0.5, 31 | use_bn=True, compute_accuracy=True, residual=False): 32 | super(PointNet2ClsMSG, self).__init__() 33 | with self.init_scope(): 34 | # initial_idx is set to ensure deterministic behavior of 35 | # fathest_point_sampling 36 | self.sam11 = SetAbstractionModule( 37 | k=512, num_sample_in_region=16, radius=0.1, 38 | mlp=[32, 32, 64], mlp2=None, initial_idx=0, 39 | residual=residual) 40 | self.sam12 = SetAbstractionModule( 41 | k=512, num_sample_in_region=32, radius=0.2, 42 | mlp=[64, 64, 128], mlp2=None, initial_idx=0, 43 | residual=residual) 44 | self.sam13 = SetAbstractionModule( 45 | k=512, num_sample_in_region=128, radius=0.4, 46 | mlp=[64, 96, 128], mlp2=None, initial_idx=0, 47 | residual=residual) 48 | self.sam21 = SetAbstractionModule( 49 | k=128, num_sample_in_region=32, radius=0.2, 50 | mlp=[64, 64, 128], mlp2=None, initial_idx=0, 51 | residual=residual) 52 | self.sam22 = SetAbstractionModule( 53 | k=128, num_sample_in_region=64, radius=0.4, 54 | mlp=[128, 128, 256], mlp2=None, initial_idx=0, 55 | residual=residual) 56 | self.sam23 = SetAbstractionModule( 57 | k=128, num_sample_in_region=128, radius=0.8, 58 | mlp=[128, 128, 256], mlp2=None, initial_idx=0, 59 | residual=residual) 60 | self.sam3 = SetAbstractionGroupAllModule( 61 | mlp=[256, 512, 1024], mlp2=None, 62 | residual=residual) 63 | 64 | self.fc_block4 = LinearBlock( 65 | 1024, 512, use_bn=use_bn, dropout_ratio=dropout_ratio,) 66 | self.fc_block5 = LinearBlock( 67 | 512, 256, use_bn=use_bn, dropout_ratio=dropout_ratio,) 68 | self.fc6 = links.Linear(256, out_dim) 69 | 70 | self.compute_accuracy = compute_accuracy 71 | 72 | def calc(self, x): 73 | # x: (minibatch, K, N, 1) 74 | # N - num_point 75 | # K - feature degree (this is 3 for xyz input, 64 for middle layer) 76 | assert x.ndim == 4 77 | assert x.shape[-1] == 1 78 | 79 | coord_points = functions.transpose(x[:, :, :, 0], (0, 2, 1)) 80 | feature_points = None 81 | cp11, fp11, _ = self.sam11(coord_points, feature_points) 82 | cp12, fp12, _ = self.sam12(coord_points, feature_points) 83 | cp13, fp13, _ = self.sam13(coord_points, feature_points) 84 | # assert numpy.allclose(cuda.to_cpu(cp11.data), cuda.to_cpu(cp12.data)) 85 | # assert numpy.allclose(cuda.to_cpu(cp11.data), cuda.to_cpu(cp13.data)) 86 | del cp12, cp13 87 | 88 | feature_points = functions.concat([fp11, fp12, fp13], axis=2) 89 | cp21, fp21, _ = self.sam21(cp11, feature_points) 90 | cp22, fp22, _ = self.sam21(cp11, feature_points) 91 | cp23, fp23, _ = self.sam21(cp11, feature_points) 92 | # assert numpy.allclose(cuda.to_cpu(cp21.data), cuda.to_cpu(cp22.data)) 93 | # assert numpy.allclose(cuda.to_cpu(cp21.data), cuda.to_cpu(cp23.data)) 94 | del cp22, cp23 95 | feature_points = functions.concat([fp21, fp22, fp23], axis=2) 96 | 97 | coord_points, feature_points = self.sam3(cp21, feature_points) 98 | # coord (bs, k, coord), feature (bs, k, ch') 99 | h = self.fc_block4(feature_points) 100 | h = self.fc_block5(h) 101 | h = self.fc6(h) 102 | return h 103 | 104 | def __call__(self, x, t): 105 | h = self.calc(x) 106 | cls_loss = functions.softmax_cross_entropy(h, t) 107 | # reporter.report({'cls_loss': cls_loss}, self) 108 | loss = cls_loss 109 | reporter.report({'loss': loss}, self) 110 | if self.compute_accuracy: 111 | acc = functions.accuracy(h, t) 112 | reporter.report({'accuracy': acc}, self) 113 | return loss 114 | 115 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdnet/kdnet_cls.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import links, reporter, functions 3 | 4 | from chainer_pointnet.models.kdnet.kdconv import KDConv 5 | 6 | 7 | class KDNetCls(chainer.Chain): 8 | 9 | """Classification KD-Network 10 | 11 | Input is points (minibatch, K, N, 1) and split_dims (minibatch, max_level) 12 | output is (minibatch, out_dim). 13 | Here, max_level=log2(N) denotes the level of KDTree. 14 | `split_dims[i]` is numpy array, represents i-th level split dimension axis. 15 | 16 | Args: 17 | out_dim (int): output dimension, number of class for classification 18 | in_dim (int or None): input dimension for each point. 19 | default is 3, (x, y, z). 20 | dropout_ratio (float): dropout ratio 21 | use_bn (bool): use batch normalization or not. 22 | compute_accuracy (bool): compute & report accuracy or not 23 | cdim (int): coordinate dimension in KDTree, usually 3 (x, y, z). 24 | """ 25 | 26 | def __init__(self, out_dim, in_dim=3, max_level=10, dropout_ratio=-1, 27 | use_bn=True, compute_accuracy=True, cdim=3): 28 | super(KDNetCls, self).__init__() 29 | if max_level <= 10: 30 | # depth 10 31 | ch_list = [in_dim] + [32, 64, 64, 128, 128, 256, 256, 32 | 512, 512, 128] 33 | ch_list = ch_list[:max_level + 1] 34 | elif max_level <= 15: 35 | # depth 15 36 | ch_list = [in_dim] + [16, 16, 32, 32, 64, 64, 128, 128, 256, 256, 37 | 512, 512, 1024, 1024, 128] 38 | ch_list = ch_list[:max_level + 1] 39 | else: 40 | raise NotImplementedError('depth {} is not implemented yet' 41 | .format(max_level)) 42 | with self.init_scope(): 43 | self.kdconvs = chainer.ChainList( 44 | *[KDConv(ch_list[i], ch_list[i+1], use_bn=use_bn, cdim=cdim) 45 | for i in range(len(ch_list)-1)]) 46 | self.linear = links.Linear(ch_list[-1], out_dim) 47 | self.compute_accuracy = compute_accuracy 48 | self.max_level = max_level 49 | self.dropout_ratio = dropout_ratio 50 | 51 | def calc(self, x, split_dims): 52 | """Main forward computation 53 | 54 | Args: 55 | x (array or Variable): 4-dim array (minibatch, K, N, 1). 56 | `K` is channel, `N` is number of points. 57 | split_dims (numpy.ndarray): 2-d array (minibatch, max_level) with 58 | dtype=object. `split_dims[i, level]` is another array, 59 | represents `i`-th example's `level` level split dimension axis 60 | 61 | Returns: 2-dim array (minibatch, out_ch), corresponds to the 62 | probability score of each label. 63 | 64 | """ 65 | bs = len(split_dims) 66 | h = x 67 | for d, kdconv in enumerate(self.kdconvs): 68 | level = self.max_level - d - 1 69 | split_dim = self.xp.array( 70 | [split_dims[i, level] for i in range(bs)]) 71 | h = kdconv(h, split_dim) 72 | if self.dropout_ratio >= 0.: 73 | h = functions.dropout(h, self.dropout_ratio) 74 | return self.linear(h) 75 | 76 | def __call__(self, x, split_dims, t): 77 | """calculate loss, accuracy 78 | 79 | Args: 80 | x (array or Variable): 4-dim array (minibatch, K, N, 1). 81 | `K` is channel, `N` is number of points. 82 | split_dims (numpy.ndarray): 2-d array (minibatch, max_level) with 83 | dtype=object. `split_dims[i, level]` is another array, 84 | represents `i`-th example's `level` level split dimension axis 85 | t (array or Variable): label 86 | 87 | Returns (Variable): loss 88 | 89 | """ 90 | h = self.calc(x, split_dims) 91 | cls_loss = functions.softmax_cross_entropy(h, t) 92 | loss = cls_loss 93 | reporter.report({'loss': loss}, self) 94 | if self.compute_accuracy: 95 | acc = functions.accuracy(h, t) 96 | reporter.report({'accuracy': acc}, self) 97 | return loss 98 | 99 | 100 | if __name__ == '__main__': 101 | import numpy 102 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 103 | 104 | batchsize = 1 105 | num_point = 135 # try 100, 128, 135 106 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 107 | dim = 3 108 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 109 | print('point_set', point_set.shape) 110 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 111 | point_set, max_level=7, calc_split_positions=True) 112 | print('points', points.shape) # 128 point here! 113 | # kdconv = KDConv(3, 8) 114 | kdnet = KDNetCls(3, max_level=7) 115 | split_dims = numpy.array(split_dims) 116 | print('split_dims', split_dims.shape, split_dims.dtype) 117 | pts = numpy.transpose(points, (1, 0))[None, :, :, None] 118 | print('pts', pts.shape, split_dims.shape) 119 | out = kdnet.calc(pts, split_dims[None, ...]) 120 | print('out', out.shape) 121 | 122 | 123 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/set_abstraction_all_block.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions 5 | from chainer import links 6 | 7 | from chainer_pointnet.models.conv_block import ConvBlock 8 | from chainer_pointnet.utils.grouping import query_ball_by_diff 9 | from chainer_pointnet.utils.sampling import farthest_point_sampling 10 | 11 | 12 | class SetAbstractionGroupAllModule(chainer.Chain): 13 | 14 | def __init__(self, mlp, mlp2, in_channels=None, use_bn=True, 15 | activation=functions.relu, residual=False): 16 | # k is number of sampled point (num_region) 17 | super(SetAbstractionGroupAllModule, self).__init__() 18 | # Feature Extractor channel list 19 | assert isinstance(mlp, list) 20 | fe_ch_list = [in_channels] + mlp 21 | # Head channel list 22 | if mlp2 is None: 23 | mlp2 = [] 24 | assert isinstance(mlp2, list) 25 | head_ch_list = [mlp[-1]] + mlp2 26 | with self.init_scope(): 27 | self.sampling_grouping = SamplingGroupingAllModule() 28 | self.feature_extractor_list = chainer.ChainList( 29 | *[ConvBlock(fe_ch_list[i], fe_ch_list[i+1], ksize=1, 30 | use_bn=use_bn, activation=activation, 31 | residual=residual 32 | ) for i in range(len(mlp))]) 33 | self.head_list = chainer.ChainList( 34 | *[ConvBlock(head_ch_list[i], head_ch_list[i + 1], ksize=1, 35 | use_bn=use_bn, activation=activation, 36 | residual=residual 37 | ) for i in range(len(mlp2))]) 38 | self.use_bn = use_bn 39 | 40 | def __call__(self, coord_points, feature_points=None): 41 | # coord_points (batch_size, num_point, coord_dim) 42 | # feature_points (batch_size, num_point, ch) 43 | # num_point, ch: coord_dim 44 | 45 | # grouped_points (batch_size, k, num_sample, channel) 46 | # center_points (batch_size, k, coord_dim) 47 | grouped_points, center_points = self.sampling_grouping( 48 | coord_points, feature_points=feature_points) 49 | # set alias `h` -> (bs, channel, num_sample, k) 50 | # Note: transpose may be removed by optimizing shape sequence for sampling_groupoing 51 | h = functions.transpose(grouped_points, (0, 3, 2, 1)) 52 | # h (bs, ch, num_sample_in_region, k=num_group) 53 | for conv_block in self.feature_extractor_list: 54 | h = conv_block(h) 55 | # TODO: try other option of pooling function 56 | h = functions.max(h, axis=2, keepdims=True) 57 | # h (bs, ch, 1, k=num_group) 58 | for conv_block in self.head_list: 59 | h = conv_block(h) 60 | h = functions.transpose(h[:, :, 0, :], (0, 2, 1)) 61 | return center_points, h # (bs, k, coord), h (bs, k, ch') 62 | 63 | 64 | def _to_array(var): 65 | """"Input: numpy, cupy array or Variable. Output: numpy or cupy array""" 66 | if isinstance(var, chainer.Variable): 67 | var = var.data 68 | return var 69 | 70 | 71 | class SamplingGroupingAllModule(chainer.Chain): 72 | 73 | def __init__(self, use_coord=True): 74 | super(SamplingGroupingAllModule, self).__init__() 75 | # number of points grouped in each region with radius 76 | self.use_coord = use_coord 77 | 78 | def __call__(self, coord_points, feature_points=None): 79 | # input: coord_points (batch_size, num_point, coord_dim) 80 | # input: feature_points (batch_size, num_point, channel) 81 | batch_size, num_point, coord_dim = coord_points.shape 82 | 83 | # grouped_points (batch_size, k=1, num_sample, coord_dim) 84 | grouped_points = coord_points[:, None, :, :] 85 | # center_points (batch_size, k=1, coord_dim) -> new_coord_points 86 | center_points = self.xp.zeros((batch_size, 1, coord_dim), 87 | self.xp.float32) 88 | 89 | if feature_points is None: 90 | new_feature_points = grouped_points 91 | else: 92 | # grouped_indices (batch_size, k, num_sample) 93 | # grouped_feature_points (batch_size, k, num_sample, channel) 94 | grouped_feature_points = feature_points[:, None, :, :] 95 | if self.use_coord: 96 | new_feature_points = functions.concat([grouped_points, grouped_feature_points], axis=3) 97 | else: 98 | new_feature_points = grouped_feature_points 99 | # new_feature_points (batch_size, k, num_sample, channel') 100 | # center_points (batch_size, k, coord_dim) -> new_coord_points 101 | return new_feature_points, center_points 102 | 103 | 104 | if __name__ == '__main__': 105 | batch_size = 3 106 | num_point = 100 107 | coord_dim = 2 108 | 109 | k = 5 110 | num_sample_in_region = 8 111 | radius = 0.4 112 | mlp = [16, 16] 113 | # mlp2 = [32, 32] 114 | mlp2 = None 115 | 116 | device = -1 117 | print('num_point', num_point, 'device', device) 118 | if device == -1: 119 | pts = numpy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 120 | else: 121 | import cupy 122 | pts = cupy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 123 | 124 | pts = pts.astype(numpy.float32) 125 | sam = SetAbstractionGroupAllModule(mlp=mlp, mlp2=mlp2) 126 | 127 | coord, h = sam(pts) 128 | print('coord', type(coord), coord.shape) # (3, 5, 2) - (bs, k, coord) 129 | print('h', type(h), h.shape) # (3, 5, 32) - (bs, k, ch') 130 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/data_prep_util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 4 | sys.path.append(BASE_DIR) 5 | from plyfile import (PlyData, PlyElement) 6 | import numpy as np 7 | import h5py 8 | 9 | SAMPLING_BIN = os.path.join(BASE_DIR, 'third_party/mesh_sampling/build/pcsample') 10 | 11 | SAMPLING_POINT_NUM = 2048 12 | SAMPLING_LEAF_SIZE = 0.005 13 | 14 | MODELNET40_PATH = '../datasets/modelnet40' 15 | def export_ply(pc, filename): 16 | vertex = np.zeros(pc.shape[0], dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) 17 | for i in range(pc.shape[0]): 18 | vertex[i] = (pc[i][0], pc[i][1], pc[i][2]) 19 | ply_out = PlyData([PlyElement.describe(vertex, 'vertex', comments=['vertices'])]) 20 | ply_out.write(filename) 21 | 22 | # Sample points on the obj shape 23 | def get_sampling_command(obj_filename, ply_filename): 24 | cmd = SAMPLING_BIN + ' ' + obj_filename 25 | cmd += ' ' + ply_filename 26 | cmd += ' -n_samples %d ' % SAMPLING_POINT_NUM 27 | cmd += ' -leaf_size %f ' % SAMPLING_LEAF_SIZE 28 | return cmd 29 | 30 | # -------------------------------------------------------------- 31 | # Following are the helper functions to load MODELNET40 shapes 32 | # -------------------------------------------------------------- 33 | 34 | # Read in the list of categories in MODELNET40 35 | def get_category_names(): 36 | shape_names_file = os.path.join(MODELNET40_PATH, 'shape_names.txt') 37 | shape_names = [line.rstrip() for line in open(shape_names_file)] 38 | return shape_names 39 | 40 | # Return all the filepaths for the shapes in MODELNET40 41 | def get_obj_filenames(): 42 | obj_filelist_file = os.path.join(MODELNET40_PATH, 'filelist.txt') 43 | obj_filenames = [os.path.join(MODELNET40_PATH, line.rstrip()) for line in open(obj_filelist_file)] 44 | print('Got %d obj files in modelnet40.' % len(obj_filenames)) 45 | return obj_filenames 46 | 47 | # Helper function to create the father folder and all subdir folders if not exist 48 | def batch_mkdir(output_folder, subdir_list): 49 | if not os.path.exists(output_folder): 50 | os.mkdir(output_folder) 51 | for subdir in subdir_list: 52 | if not os.path.exists(os.path.join(output_folder, subdir)): 53 | os.mkdir(os.path.join(output_folder, subdir)) 54 | 55 | # ---------------------------------------------------------------- 56 | # Following are the helper functions to load save/load HDF5 files 57 | # ---------------------------------------------------------------- 58 | 59 | # Write numpy array data and label to h5_filename 60 | def save_h5_data_label_normal(h5_filename, data, label, normal, 61 | data_dtype='float32', label_dtype='uint8', noral_dtype='float32'): 62 | h5_fout = h5py.File(h5_filename) 63 | h5_fout.create_dataset( 64 | 'data', data=data, 65 | compression='gzip', compression_opts=4, 66 | dtype=data_dtype) 67 | h5_fout.create_dataset( 68 | 'normal', data=normal, 69 | compression='gzip', compression_opts=4, 70 | dtype=normal_dtype) 71 | h5_fout.create_dataset( 72 | 'label', data=label, 73 | compression='gzip', compression_opts=1, 74 | dtype=label_dtype) 75 | h5_fout.close() 76 | 77 | 78 | # Write numpy array data and label to h5_filename 79 | def save_h5(h5_filename, data, label, data_dtype='uint8', label_dtype='uint8'): 80 | h5_fout = h5py.File(h5_filename) 81 | h5_fout.create_dataset( 82 | 'data', data=data, 83 | compression='gzip', compression_opts=4, 84 | dtype=data_dtype) 85 | h5_fout.create_dataset( 86 | 'label', data=label, 87 | compression='gzip', compression_opts=1, 88 | dtype=label_dtype) 89 | h5_fout.close() 90 | 91 | # Read numpy array data and label from h5_filename 92 | def load_h5_data_label_normal(h5_filename): 93 | f = h5py.File(h5_filename) 94 | data = f['data'][:] 95 | label = f['label'][:] 96 | normal = f['normal'][:] 97 | return (data, label, normal) 98 | 99 | # Read numpy array data and label from h5_filename 100 | def load_h5_data_label_seg(h5_filename): 101 | f = h5py.File(h5_filename) 102 | data = f['data'][:] 103 | label = f['label'][:] 104 | seg = f['pid'][:] 105 | return (data, label, seg) 106 | 107 | # Read numpy array data and label from h5_filename 108 | def load_h5(h5_filename): 109 | f = h5py.File(h5_filename) 110 | data = f['data'][:] 111 | label = f['label'][:] 112 | return (data, label) 113 | 114 | # ---------------------------------------------------------------- 115 | # Following are the helper functions to load save/load PLY files 116 | # ---------------------------------------------------------------- 117 | 118 | # Load PLY file 119 | def load_ply_data(filename, point_num): 120 | plydata = PlyData.read(filename) 121 | pc = plydata['vertex'].data[:point_num] 122 | pc_array = np.array([[x, y, z] for x,y,z in pc]) 123 | return pc_array 124 | 125 | # Load PLY file 126 | def load_ply_normal(filename, point_num): 127 | plydata = PlyData.read(filename) 128 | pc = plydata['normal'].data[:point_num] 129 | pc_array = np.array([[x, y, z] for x,y,z in pc]) 130 | return pc_array 131 | 132 | # Make up rows for Nxk array 133 | # Input Pad is 'edge' or 'constant' 134 | def pad_arr_rows(arr, row, pad='edge'): 135 | assert(len(arr.shape) == 2) 136 | assert(arr.shape[0] <= row) 137 | assert(pad == 'edge' or pad == 'constant') 138 | if arr.shape[0] == row: 139 | return arr 140 | if pad == 'edge': 141 | return np.lib.pad(arr, ((0, row-arr.shape[0]), (0, 0)), 'edge') 142 | if pad == 'constant': 143 | return np.lib.pad(arr, ((0, row-arr.shape[0]), (0, 0)), 'constant', (0, 0)) 144 | 145 | 146 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdnet/kdnet_seg.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import links, reporter, functions 3 | 4 | from chainer_pointnet.models.conv_block import ConvBlock 5 | from chainer_pointnet.models.kdnet.kdconv import KDConv 6 | from chainer_pointnet.models.kdnet.kddeconv import KDDeconv 7 | 8 | 9 | class KDNetSeg(chainer.Chain): 10 | 11 | """Segmentation KD-Network 12 | 13 | Input is points (minibatch, K, N, 1) and split_dims (minibatch, max_level) 14 | output is (minibatch, out_dim). 15 | Here, max_level=log2(N) denotes the level of KDTree. 16 | `split_dims[i]` is numpy array, represents i-th level split dimension axis. 17 | 18 | Args: 19 | out_dim (int): output dimension, number of class for classification 20 | in_dim: input dimension for each point. default is 3, (x, y, z). 21 | dropout_ratio (float): dropout ratio 22 | use_bn (bool): use batch normalization or not. 23 | compute_accuracy (bool): compute & report accuracy or not 24 | cdim (int): coordinate dimension in KDTree, usually 3 (x, y, z). 25 | residual (bool): use residual connection or not 26 | """ 27 | 28 | def __init__(self, out_dim, in_dim=3, max_level=10, dropout_ratio=0.0, 29 | use_bn=True, compute_accuracy=True, cdim=3, residual=False): 30 | super(KDNetSeg, self).__init__() 31 | if max_level <= 10: 32 | # depth 10 33 | ch_list = [in_dim] + [32, 64, 64, 128, 128, 256, 256, 34 | 512, 512, 128] 35 | elif max_level <= 15: 36 | # depth 15 37 | ch_list = [in_dim] + [16, 16, 32, 32, 64, 64, 128, 128, 256, 256, 38 | 512, 512, 1024, 1024, 128] 39 | else: 40 | raise NotImplementedError('depth {} is not implemented yet' 41 | .format(max_level)) 42 | ch_list = ch_list[:max_level + 1] 43 | out_ch_list = ch_list.copy() 44 | # out_ch_list[0] = 16 45 | for i in range(5): 46 | # TODO: What kind of channels are best for decoding part? 47 | # HACKING 48 | # overwrites bottom layer to have wider channels 49 | out_ch_list[i] = 64 50 | with self.init_scope(): 51 | self.kdconvs = chainer.ChainList( 52 | *[KDConv(ch_list[i], ch_list[i+1], use_bn=use_bn, cdim=cdim) 53 | for i in range(len(ch_list)-1)]) 54 | self.kddeconvs = chainer.ChainList( 55 | *[KDDeconv(out_ch_list[-i], in_channels_skip=ch_list[-i-1], 56 | out_channels=out_ch_list[-i-1], use_bn=use_bn, 57 | cdim=cdim) 58 | for i in range(1, len(out_ch_list))]) 59 | self.conv_block = ConvBlock( 60 | out_ch_list[0], out_ch_list[0], ksize=1, use_bn=use_bn, 61 | residual=residual) 62 | self.conv = links.Convolution2D(out_ch_list[0], out_dim, ksize=1) 63 | self.compute_accuracy = compute_accuracy 64 | self.max_level = max_level 65 | self.dropout_ratio = dropout_ratio 66 | 67 | def calc(self, x, split_dims): 68 | bs = len(split_dims) 69 | # construct split_dim_list 70 | split_dim_list = [] 71 | for level in range(self.max_level): 72 | split_dim_list.append(self.xp.array( 73 | [split_dims[i, level] for i in range(bs)])) 74 | 75 | h = x 76 | h_skip_list = [h] 77 | for d, kdconv in enumerate(self.kdconvs): 78 | level = self.max_level - d - 1 79 | h = kdconv(h, split_dim_list[level]) 80 | h_skip_list.append(h) 81 | 82 | h_skip_list.pop(-1) # don't use last h as skip connection. 83 | for d, kddeconv in enumerate(self.kddeconvs): 84 | level = d 85 | h_skip = h_skip_list.pop(-1) 86 | # print('h h_skip', h.shape, h_skip.shape, level, len(h_list)) 87 | h = kddeconv(h, split_dim_list[level], h_skip) 88 | assert len(h_skip_list) == 0 89 | 90 | if self.dropout_ratio > 0.: 91 | h = functions.dropout(h, self.dropout_ratio) 92 | h = self.conv_block(h) 93 | h = self.conv(h) 94 | return h[:, :, :, 0] 95 | 96 | def __call__(self, x, split_dims, t): 97 | h = self.calc(x, split_dims) 98 | bs, ch, n = h.shape 99 | h = functions.reshape(functions.transpose(h, (0, 2, 1)), (bs * n, ch)) 100 | t = functions.reshape(t, (bs * n,)) 101 | cls_loss = functions.softmax_cross_entropy(h, t) 102 | loss = cls_loss 103 | reporter.report({'loss': loss}, self) 104 | if self.compute_accuracy: 105 | acc = functions.accuracy(h, t) 106 | reporter.report({'accuracy': acc}, self) 107 | return loss 108 | 109 | 110 | if __name__ == '__main__': 111 | import numpy 112 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 113 | 114 | batchsize = 1 115 | num_point = 135 # try 100, 128, 135 116 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 117 | dim = 3 118 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 119 | print('point_set', point_set.shape) 120 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 121 | point_set, max_level=max_level, calc_split_positions=True) 122 | print('points', points.shape) # 128 point here! 123 | kdnet = KDNetSeg(3, max_level=max_level, use_bn=False) 124 | split_dims = numpy.array(split_dims) 125 | print('split_dims', split_dims.shape, split_dims.dtype) 126 | pts = numpy.transpose(points, (1, 0))[None, :, :, None] 127 | print('pts', pts.shape, split_dims.shape) 128 | out = kdnet.calc(pts, split_dims[None, ...]) 129 | print('out', out.shape) 130 | 131 | 132 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdcontextnet/kdcontextnet_cls.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions 5 | from chainer import links 6 | from chainer import reporter 7 | 8 | from chainer_pointnet.models.kdcontextnet.kdcontextconv_block import \ 9 | KDContextConvBlock 10 | from chainer_pointnet.models.linear_block import LinearBlock 11 | 12 | 13 | class KDContextNetCls(chainer.Chain): 14 | 15 | """Classification 3DContextNet 16 | 17 | Args: 18 | out_dim (int): output dimension, number of class for classification 19 | in_dim (int or None): input dimension for each point. 20 | default is 3, (x, y, z). 21 | dropout_ratio (float): dropout ratio 22 | use_bn (bool): use batch normalization or not. 23 | compute_accuracy (bool): compute & report accuracy or not 24 | levels (list): list of int. It determines block depth and each 25 | block's grouping (receptive field) size. 26 | For example if `list=[5, 6, 7]`, it will take 27 | level1 block as 5-th depth (32) of KDTree, 28 | level2 as 6-th depth (64), level3 as 7-th depth (128) resp. 29 | Receptive field (number of points in each grouping) is 30 | `2**level` for each level. 31 | feature_learning_mlp_list (list): list of list of int. 32 | Indicates each block's feature learning MLP size. 33 | feature_aggregation_mlp_list (list): list of list of int. 34 | Indicates each block's feature aggregation MLP size. 35 | fc_mlp_list (list): list of int. 36 | Indicates final fully connection MLP size. 37 | normalize (bool): apply normalization to calculate global context cues 38 | in `KDContextConvBlock`. 39 | residual (bool): use residual connection or not 40 | """ 41 | 42 | def __init__(self, out_dim, in_dim=3, dropout_ratio=0.5, 43 | use_bn=True, compute_accuracy=True, 44 | levels=None, 45 | feature_learning_mlp_list=None, 46 | feature_aggregation_mlp_list=None, 47 | fc_mlp_list=None, normalize=False, residual=False 48 | ): 49 | super(KDContextNetCls, self).__init__() 50 | levels = levels or [5, 6, 7] 51 | feature_learning_mlp_list = feature_learning_mlp_list or [ 52 | [64, 64, 128, 128], [64, 64, 256, 256], [64, 64, 512, 512]] 53 | feature_aggregation_mlp_list = feature_aggregation_mlp_list or [ 54 | [256], [512], [1024]] 55 | in_channels_list = [in_dim] + [elem[-1] for elem in 56 | feature_aggregation_mlp_list] 57 | fc_mlp_list = fc_mlp_list or [256, 256] 58 | 59 | levels_diff = numpy.diff(numpy.array([0] + levels)) 60 | print('levels {}, levels_diff {}'.format(levels, levels_diff)) 61 | fcmlps = [in_channels_list[-1]] + fc_mlp_list 62 | assert len(levels) == len(feature_learning_mlp_list) 63 | assert len(levels) == len(feature_aggregation_mlp_list) 64 | with self.init_scope(): 65 | # don't use dropout in conv blocks 66 | self.kdcontextconv_blocks = chainer.ChainList( 67 | *[KDContextConvBlock( 68 | in_channels_list[i], m=2 ** levels_diff[i], 69 | feature_learning_mlp=feature_learning_mlp_list[i], 70 | feature_aggregation_mlp=feature_aggregation_mlp_list[i], 71 | use_bn=use_bn, normalize=normalize, residual=residual 72 | ) for i in range(len(levels_diff))]) 73 | self.fc_blocks = chainer.ChainList( 74 | *[LinearBlock( 75 | fcmlps[i], fcmlps[i+1], use_bn=use_bn, 76 | dropout_ratio=dropout_ratio 77 | ) for i in range(len(fcmlps)-1)]) 78 | self.linear = links.Linear(fcmlps[-1], out_dim) 79 | self.compute_accuracy = compute_accuracy 80 | 81 | def calc(self, x): 82 | # x (bs, ch, N, 1) 83 | assert x.ndim == 4 84 | assert x.shape[3] == 1 85 | h = x 86 | for kdconv_block in self.kdcontextconv_blocks: 87 | h = kdconv_block(h) 88 | # h (bs, ch, N//2**levels[-1], 1) 89 | # TODO: support other symmetric function 90 | h = functions.max_pooling_2d(h, ksize=(h.shape[2], 1)) 91 | for fc_block in self.fc_blocks: 92 | h = fc_block(h) 93 | h = self.linear(h) 94 | return h 95 | 96 | def __call__(self, x, t): 97 | h = self.calc(x) 98 | cls_loss = functions.softmax_cross_entropy(h, t) 99 | # reporter.report({'cls_loss': cls_loss}, self) 100 | loss = cls_loss 101 | reporter.report({'loss': loss}, self) 102 | if self.compute_accuracy: 103 | acc = functions.accuracy(h, t) 104 | reporter.report({'accuracy': acc}, self) 105 | return loss 106 | 107 | 108 | if __name__ == '__main__': 109 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 110 | 111 | batchsize = 1 112 | num_point = 135 # try 100, 128, 135 113 | max_level = 8 # 2^7 -> 128. Final num_point will be 128 114 | dim = 3 115 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 116 | print('point_set', point_set.shape) 117 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 118 | point_set, max_level=max_level, calc_split_positions=True) 119 | print('points', points.shape) # 128 point here! 120 | kdnet = KDContextNetCls(out_dim=5, use_bn=False) 121 | # split_dims = numpy.array(split_dims) 122 | # print('split_dims', split_dims.shape, split_dims.dtype) 123 | pts = numpy.transpose(points, (1, 0))[None, :, :, None] 124 | print('pts', pts.shape, split_dims.shape) 125 | out = kdnet.calc(pts) 126 | print('out', out.shape) 127 | 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet/pointnet_cls.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions, cuda 3 | from chainer import links 4 | from chainer import reporter 5 | 6 | from chainer_pointnet.models.conv_block import ConvBlock 7 | from chainer_pointnet.models.linear_block import LinearBlock 8 | from chainer_pointnet.models.pointnet.transform_net import TransformNet 9 | 10 | 11 | def calc_trans_loss(t): 12 | # Loss to enforce the transformation as orthogonal matrix 13 | # t (batchsize, K, K) - transform matrix 14 | xp = cuda.get_array_module(t) 15 | bs, k1, k2 = t.shape 16 | assert k1 == k2 17 | mat_diff = functions.matmul(t, functions.transpose(t, (0, 2, 1))) 18 | mat_diff = mat_diff - xp.identity(k1, dtype=xp.float32) 19 | # divide by 2. is to make the behavior same with tf. 20 | # https://www.tensorflow.org/versions/r1.1/api_docs/python/tf/nn/l2_loss 21 | return functions.sum(functions.batch_l2_norm_squared(mat_diff)) / 2. 22 | 23 | 24 | class PointNetCls(chainer.Chain): 25 | 26 | """Classification PointNet 27 | 28 | Input is (minibatch, K, N, 1), output is (minibatch, out_dim) 29 | 30 | Args: 31 | out_dim (int): output dimension, number of class for classification 32 | in_dim: input dimension for each point. default is 3, (x, y, z). 33 | middle_dim (int): hidden layer 34 | dropout_ratio (float): dropout ratio, negative value indicates 35 | not to use dropout. 36 | use_bn (bool): use batch normalization or not. 37 | trans (bool): use TransformNet or not. 38 | False means not to use TransformNet, corresponds to 39 | PointNetVanilla. True corresponds to PointNet in the paper. 40 | trans_lam1 (float): regularization term for input transform. 41 | used in training. it is simply ignored when `trans` is False. 42 | trans_lam2 (float): regularization term for feature transform 43 | used in training. it is simply ignored when `trans` is False. 44 | compute_accuracy (bool): compute & report accuracy or not 45 | residual (bool): use residual connection or not 46 | """ 47 | 48 | def __init__(self, out_dim, in_dim=3, middle_dim=64, dropout_ratio=0.3, 49 | use_bn=True, trans=True, trans_lam1=0.001, trans_lam2=0.001, 50 | compute_accuracy=True, residual=False): 51 | super(PointNetCls, self).__init__() 52 | with self.init_scope(): 53 | if trans: 54 | self.input_transform_net = TransformNet( 55 | k=in_dim, use_bn=use_bn, residual=residual) 56 | 57 | self.conv_block1 = ConvBlock( 58 | in_dim, 64, ksize=1, use_bn=use_bn, residual=residual) 59 | self.conv_block2 = ConvBlock( 60 | 64, middle_dim, ksize=1, use_bn=use_bn, residual=residual) 61 | if trans: 62 | self.feature_transform_net = TransformNet( 63 | k=middle_dim, use_bn=use_bn, residual=residual) 64 | 65 | self.conv_block3 = ConvBlock( 66 | middle_dim, 64, ksize=1, use_bn=use_bn, residual=residual) 67 | self.conv_block4 = ConvBlock( 68 | 64, 128, ksize=1, use_bn=use_bn, residual=residual) 69 | self.conv_block5 = ConvBlock( 70 | 128, 1024, ksize=1, use_bn=use_bn, residual=residual) 71 | 72 | # original impl. uses `keep_prob=0.7`. 73 | self.fc_block6 = LinearBlock( 74 | 1024, 512, use_bn=use_bn, dropout_ratio=dropout_ratio,) 75 | self.fc_block7 = LinearBlock( 76 | 512, 256, use_bn=use_bn, dropout_ratio=dropout_ratio,) 77 | self.fc8 = links.Linear(256, out_dim) 78 | 79 | self.in_dim = in_dim 80 | self.trans = trans 81 | self.trans_lam1 = trans_lam1 82 | self.trans_lam2 = trans_lam2 83 | self.compute_accuracy = compute_accuracy 84 | 85 | def __call__(self, x, t): 86 | h, t1, t2 = self.calc(x) 87 | cls_loss = functions.softmax_cross_entropy(h, t) 88 | reporter.report({'cls_loss': cls_loss}, self) 89 | 90 | loss = cls_loss 91 | # Enforce the transformation as orthogonal matrix 92 | if self.trans and self.trans_lam1 >= 0: 93 | trans_loss1 = self.trans_lam1 * calc_trans_loss(t1) 94 | reporter.report({'trans_loss1': trans_loss1}, self) 95 | loss = loss + trans_loss1 96 | if self.trans and self.trans_lam2 >= 0: 97 | trans_loss2 = self.trans_lam2 * calc_trans_loss(t2) 98 | reporter.report({'trans_loss2': trans_loss2}, self) 99 | loss = loss + trans_loss2 100 | reporter.report({'loss': loss}, self) 101 | 102 | if self.compute_accuracy: 103 | acc = functions.accuracy(h, t) 104 | reporter.report({'accuracy': acc}, self) 105 | return loss 106 | 107 | def calc(self, x): 108 | # x: (minibatch, K, N, 1) 109 | # N - num_point 110 | # K - feature degree (this is 3 for xyz input, 64 for middle layer) 111 | assert x.ndim == 4 112 | assert x.shape[-1] == 1 113 | 114 | # --- input transform --- 115 | if self.trans: 116 | h, t1 = self.input_transform_net(x) 117 | else: 118 | h = x 119 | t1 = 0 # dummy 120 | 121 | h = self.conv_block1(h) 122 | h = self.conv_block2(h) 123 | 124 | # --- feature transform --- 125 | if self.trans: 126 | h, t2 = self.feature_transform_net(h) 127 | else: 128 | t2 = 0 # dummy 129 | 130 | h = self.conv_block3(h) 131 | h = self.conv_block4(h) 132 | h = self.conv_block5(h) 133 | 134 | # Symmetric function: max pooling 135 | h = functions.max_pooling_2d(h, ksize=h.shape[2:]) 136 | # h: (minibatch, K, 1, 1) 137 | h = self.fc_block6(h) 138 | h = self.fc_block7(h) 139 | h = self.fc8(h) 140 | return h, t1, t2 141 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdnet/kddeconv.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions, links 3 | 4 | 5 | class KDDeconv(chainer.Chain): 6 | 7 | """KD-Deconvolution. apply eq (4) of the paper 8 | 9 | Escape from Cells: Deep Kd-Networks for the Recognition of 10 | 3D Point Cloud Models. 11 | 12 | Args: 13 | in_channels (int or None): channel size for input `x` 14 | out_channels (int or None): output channels size 15 | in_channels_skip (int or None): channel size for input `x_skip` 16 | ksize (int or tuple): kernel size 17 | stride (int or tuple): stride size 18 | pad (int or tuple): padding size 19 | nobias (bool): use bias `b` or not. 20 | initialW: initiallizer of `W` 21 | initial_bias: initializer of `b` 22 | use_bn (bool): use batch normalization or not 23 | activation (callable): activation function 24 | dropout_ratio (float): dropout ratio, set negative value to skip 25 | dropout 26 | cdim (int): coordinate dimension, usually 3 (x, y, z). 27 | """ 28 | 29 | def __init__(self, in_channels, out_channels, in_channels_skip=None, 30 | ksize=1, stride=1, pad=0, 31 | nobias=False, initialW=None, initial_bias=None, use_bn=True, 32 | activation=functions.relu, dropout_ratio=-1, cdim=3): 33 | super(KDDeconv, self).__init__() 34 | out_ch = out_channels // 2 35 | out_channels_skip = out_channels - out_ch 36 | with self.init_scope(): 37 | self.conv = links.Convolution2D( 38 | in_channels, out_ch * cdim * 2, ksize=ksize, 39 | stride=stride, pad=pad, nobias=nobias, initialW=initialW, 40 | initial_bias=initial_bias) 41 | self.conv_skip = links.Convolution2D( 42 | in_channels_skip, out_channels_skip, ksize=ksize, 43 | stride=stride, pad=pad, nobias=nobias, initialW=initialW, 44 | initial_bias=initial_bias) 45 | if use_bn: 46 | self.bn = links.BatchNormalization(out_channels) 47 | self.out_ch = out_ch 48 | self.out_channels = out_channels 49 | self.cdim = cdim 50 | self.activation = activation 51 | self.use_bn = use_bn 52 | self.dropout_ratio = dropout_ratio 53 | 54 | def __call__(self, x, split_dim, x_skip): 55 | """KDDeconv makes feature of 1-level children node of KDTree. 56 | 57 | `num_point` of input `x` will be twice in the `output`. 58 | 59 | Args: 60 | x (numpy.ndarray): 61 | 4-dim array (batchsize, in_channels, num_point, 1) 62 | split_dim (numpy.ndarray): 1d array with dtype=object with 63 | length=max_level. `split_dim[i]` is numpy array with dtype=int, 64 | represents i-th level split dimension axis. 65 | x_skip (numpy.ndarray): 66 | 4-dim array (batchsize, in_channels_skip, num_point*2, 1) 67 | 68 | Returns (numpy.ndarray): 69 | 4-dim array (batchsize, out_channels, num_point*2, 1) 70 | 71 | """ 72 | assert x.ndim == 4 73 | assert x_skip.ndim == 4 74 | assert x.shape[2] == split_dim.shape[1],\ 75 | 'x.shape {}, split_dim.shape{}'.format(x.shape, split_dim.shape) 76 | # x: (batch_size, ch, num_point, 1) 77 | bs, ch, num_point, w = x.shape 78 | bs, ch_skip, num_point2, w_skip = x_skip.shape 79 | assert w == 1 80 | assert w_skip == 1 81 | assert num_point * 2 == num_point2 82 | 83 | # --- deconvolution from parent node to children part --- 84 | # (bs, ch, N, 1) -> (bs, out_ch, N*2, 1) 85 | 86 | # conv: (bs, ch, N, 1) -> (bs, out_ch*cdim*2, N, 1) 87 | h = self.conv(x) 88 | # select split_dim: -> (bs, out_ch*2, N, 1) 89 | h = functions.reshape(h, (bs, self.out_ch, self.cdim, 2, num_point)) 90 | h = h[self.xp.arange(bs)[:, None], :, split_dim, :, 91 | self.xp.arange(num_point)[None, :]] 92 | # h (bs, num_point, out_ch, 2) 93 | h = functions.transpose(h, (0, 2, 1, 3)) 94 | h = functions.reshape(h, (bs, self.out_ch, num_point2, 1)) 95 | # split to child node: -> (bs, out_ch, N*2, 1) 96 | 97 | # --- skip connection part --- 98 | # (bs, ch_skip, N*2, 1) -> (bs, out_channels_skip, N*2, 1) 99 | h_skip = self.conv_skip(x_skip) 100 | # concat deconv feature `h` and skip connection feature `h_skip` 101 | h = functions.concat([h, h_skip], axis=1) 102 | if self.use_bn: 103 | h = self.bn(h) 104 | if self.activation is not None: 105 | h = self.activation(h) 106 | if self.dropout_ratio >= 0: 107 | h = functions.dropout(h, ratio=self.dropout_ratio) 108 | return h 109 | 110 | 111 | if __name__ == '__main__': 112 | import numpy 113 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 114 | from chainer_pointnet.models.kdnet.kdconv import KDConv 115 | 116 | batchsize = 1 117 | num_point = 135 # try 100, 128, 135 118 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 119 | dim = 3 120 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 121 | point_set2 = numpy.random.rand(num_point, dim).astype(numpy.float32) 122 | print('point_set', point_set.shape) 123 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 124 | point_set, max_level=7, calc_split_positions=True) 125 | points2, split_dims2, inds2, kdtree2, split_positions2 = construct_kdtree_data( 126 | point_set2, max_level=7, calc_split_positions=True) 127 | print('points', points.shape) # 128 point here! 128 | # --- net definition --- 129 | kdconv = KDConv(3, 8) 130 | kddeconv = KDDeconv(8, out_channels=11, in_channels_skip=3) 131 | # --- net definition end --- 132 | 133 | split_dim = numpy.array([split_dims[-1], split_dims2[-1]]) 134 | print('split_dim', split_dim.shape) 135 | pts = numpy.array([points, points2]) 136 | pts = numpy.transpose(pts, (0, 2, 1))[:, :, :, None] 137 | print('pts', pts.shape) 138 | pts2 = kdconv(pts, split_dim) 139 | print('pts2', pts2.shape) 140 | out = kddeconv(pts2, split_dim, pts) 141 | print('out', out.shape) 142 | 143 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/feature_propagation_block.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions 5 | 6 | from chainer_pointnet.models.conv_block import ConvBlock 7 | from chainer_pointnet.models.pointnet2.set_abstraction_block import \ 8 | SetAbstractionModule 9 | from chainer_pointnet.utils.grouping import _l2_norm 10 | 11 | 12 | class FeaturePropagationModule(chainer.Chain): 13 | 14 | def __init__(self, mlp, in_channels=None, use_bn=True, 15 | activation=functions.relu, residual=False): 16 | super(FeaturePropagationModule, self).__init__() 17 | # Feature Extractor channel list 18 | assert isinstance(mlp, list) 19 | fe_ch_list = [in_channels] + mlp 20 | with self.init_scope(): 21 | self.interpolation = InterpolationModule() 22 | self.feature_extractor_list = chainer.ChainList( 23 | *[ConvBlock(fe_ch_list[i], fe_ch_list[i+1], ksize=1, 24 | use_bn=use_bn, activation=activation, 25 | residual=residual 26 | ) for i in range(len(mlp))]) 27 | self.use_bn = use_bn 28 | 29 | def __call__(self, distances, points1, points2): 30 | """ 31 | 32 | Args: 33 | distances (numpy.ndarray or cupy.ndarray): 34 | 3-dim array (bs, num_point2, num_point1) 35 | points1 (Variable): 3-dim (batch_size, num_point1, ch1) 36 | points2 (Variable): 3-dim (batch_size, num_point2, ch2) 37 | points2 is deeper, rich feature. num_point1 > num_point2 38 | 39 | Returns (Variable): 3-dim (batch_size, num_point1, ch1+ch2) 40 | """ 41 | # h: interpolated_points (batch_size, num_point1, ch1+ch2) 42 | h = self.interpolation(distances, points1, points2) 43 | # h: interpolated_points (batch_size, ch1+ch2, num_point1, 1) 44 | h = functions.transpose(h, (0, 2, 1))[:, :, :, None] 45 | for conv_block in self.feature_extractor_list: 46 | h = conv_block(h) 47 | h = functions.transpose(h[:, :, :, 0], (0, 2, 1)) 48 | return h # h (bs, num_point, ch') 49 | 50 | 51 | def _to_array(var): 52 | """"Input: numpy, cupy array or Variable. Output: numpy or cupy array""" 53 | if isinstance(var, chainer.Variable): 54 | var = var.data 55 | return var 56 | 57 | 58 | class InterpolationModule(chainer.Chain): 59 | 60 | def __init__(self, num_fp_point=3, eps=1e-10, metric=_l2_norm): 61 | super(InterpolationModule, self).__init__() 62 | # number of feature propagation point to interpolate 63 | self.num_fp_point = num_fp_point 64 | self.eps = eps 65 | self.metric = metric 66 | 67 | def __call__(self, distances, points1, points2): 68 | """ 69 | 70 | Args: 71 | distances (numpy.ndarray or cupy.ndarray): 72 | 3-dim array (bs, num_point2, num_point1) 73 | points1 (Variable): 3-dim (batch_size, num_point1, ch1) 74 | points2 (Variable): 3-dim (batch_size, num_point2, ch2) 75 | points2 is deeper, rich feature. num_point1 > num_point2 76 | 77 | Returns (Variable): 3-dim (batch_size, num_point1, ch1+ch2) 78 | 79 | """ 80 | # batch_size, num_point1, ch1 = points1.shape 81 | # batch_size2, num_point2, ch2 = points2.shape 82 | batch_size, num_point2, num_point1 = distances.shape 83 | # assert batch_size == batch_size2 84 | if distances is None: 85 | print('[WARNING] distances is None') 86 | # calculate distances by feature vector (not coord vector) 87 | distances = self.xp(self.metric(points1, points2)) 88 | # Better in this form 89 | # distances = self.xp(self.metric(coord1, coord2)) 90 | 91 | # --- weight calculation --- 92 | # k-nearest neighbor with k=self.num_fp_point 93 | # sorted_indices (bs, num_fp_point, num_point1) 94 | sorted_indices = self.xp.argsort( 95 | distances, axis=1)[:, :self.num_fp_point, :] 96 | # sorted_dists (bs, num_fp_point, num_point1) 97 | sorted_dists = distances[ 98 | self.xp.arange(batch_size)[:, None, None], 99 | sorted_indices, 100 | self.xp.arange(num_point1)[None, None, :]] 101 | 102 | eps_array = self.xp.ones( 103 | sorted_dists.shape, dtype=sorted_dists.dtype) * self.eps 104 | sorted_dists = functions.maximum(sorted_dists, eps_array) 105 | inv_dist = 1.0 / sorted_dists 106 | norm = functions.sum(inv_dist, axis=1, keepdims=True) 107 | norm = functions.broadcast_to(norm, sorted_dists.shape) 108 | # weight (bs, num_fp_point, num_point1) 109 | weight = inv_dist / norm 110 | # --- weight calculation end --- 111 | # point2_selected (bs, num_fp_point, num_point1, ch2) 112 | points2_selected = points2[ 113 | self.xp.arange(batch_size)[:, None, None], 114 | sorted_indices, :] 115 | # print('debug', weight.shape, points2_selected.shape) 116 | weight = functions.broadcast_to( 117 | weight[:, :, :, None], points2_selected.shape) 118 | # interpolated_points (bs, num_point1, ch2) 119 | interpolated_points = functions.sum( 120 | weight * points2_selected, axis=1) 121 | if points1 is None: 122 | return interpolated_points 123 | else: 124 | return functions.concat([interpolated_points, points1], axis=2) 125 | 126 | 127 | if __name__ == '__main__': 128 | batch_size = 4 129 | num_point1 = 100 130 | ch1 = 16 131 | 132 | k = 5 133 | num_sample_in_region = 8 134 | radius = 0.4 135 | 136 | num_point2 = k 137 | ch2 = 32 138 | mlp = [16, ch2] 139 | 140 | device = -1 141 | print('num_point', num_point1, num_point2, 'device', device) 142 | if device == -1: 143 | xp = numpy 144 | else: 145 | import cupy 146 | xp = cupy 147 | pts = xp.random.uniform(0, 1, (batch_size, num_point1, ch1)) 148 | 149 | pts = pts.astype(numpy.float32) 150 | sam = SetAbstractionModule(k, num_sample_in_region, radius, mlp, mlp2=None, 151 | return_distance=True) 152 | fpm = FeaturePropagationModule([10, 10]) 153 | 154 | coord, h, dists = sam(pts) 155 | h2 = fpm(dists, pts, h) 156 | print('coord', type(coord), coord.shape) # (4, 5, 16) - (bs, k, coord) 157 | print('h', type(h), h.shape) # (4, 5, 32) - (bs, k, ch') 158 | print('h2', type(h2), h2.shape) # (4, 100, 10) - (bs, num_point1, ch') 159 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdcontextnet/kdcontextdeconv_block.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions 3 | from chainer import links 4 | 5 | from chainer_pointnet.models.conv_block import ConvBlock 6 | from chainer_pointnet.models.kdcontextnet.kdcontextconv_block import \ 7 | KDContextConvBlock 8 | 9 | 10 | class KDContextDeconvBlock(chainer.Chain): 11 | 12 | """KD Context Deconvolution Block 13 | 14 | It performs each level feature learning & aggregation 15 | 16 | Args: 17 | in_channels (int or None): channel size for input `x` 18 | m (int): Number to group to be aggregated. 19 | out_deconv_channels (int or None): output channels size for deconv part 20 | in_channels_skip (int or None): channel size for input `x_skip` 21 | feature_learning_mlp (list): list of int, specifies MLP size for 22 | feature learning stage. 23 | feature_aggregation_mlp (list): list of int, specifies MLP size for 24 | feature aggregation stage. 25 | ksize (int or tuple): kernel size 26 | stride (int or tuple): stride size 27 | pad (int or tuple): padding size 28 | nobias (bool): use bias `b` or not. 29 | initialW: initiallizer of `W` 30 | initial_bias: initializer of `b` 31 | use_bn (bool): use batch normalization or not 32 | activation (callable): activation function 33 | dropout_ratio (float): dropout ratio, set negative value to skip 34 | dropout 35 | normalize (bool): apply normalization to calculate global context cues 36 | residual (bool): use residual connection or not 37 | """ 38 | 39 | def __init__(self, in_channels, m, in_channels_skip=None, 40 | out_deconv_channels=None, 41 | feature_learning_mlp=None, 42 | feature_aggregation_mlp=None, 43 | ksize=1, stride=1, pad=0, 44 | nobias=False, initialW=None, initial_bias=None, use_bn=True, 45 | activation=functions.relu, dropout_ratio=-1, normalize=False, 46 | residual=False): 47 | super(KDContextDeconvBlock, self).__init__() 48 | if in_channels_skip is None: 49 | in_channels_context = None 50 | else: 51 | in_channels_context = in_channels_skip + out_deconv_channels 52 | if out_deconv_channels is None: 53 | raise ValueError('currently out_deconv_channels must be set ' 54 | 'specifically') 55 | with self.init_scope(): 56 | # deconvolution part 57 | self.conv = links.Convolution2D( 58 | in_channels, out_deconv_channels * m, ksize=ksize, 59 | stride=stride, pad=pad, nobias=nobias, initialW=initialW, 60 | initial_bias=initial_bias) 61 | # set `aggregation=False` to keep num_point of output. 62 | self.kdcontextconv = KDContextConvBlock( 63 | in_channels_context, m, 64 | feature_learning_mlp=feature_learning_mlp, 65 | feature_aggregation_mlp=feature_aggregation_mlp, 66 | ksize=ksize, stride=stride, pad=pad, 67 | nobias=nobias, initialW=initialW, initial_bias=initial_bias, 68 | use_bn=use_bn, activation=activation, 69 | dropout_ratio=dropout_ratio, aggregation=False, 70 | normalize=normalize, residual=residual) 71 | self.m = m 72 | self.out_deconv_channels = out_deconv_channels 73 | 74 | def __call__(self, x, x_skip): 75 | """Main forward computation 76 | 77 | Args: 78 | x (array or Variable): Input from parent node. 79 | 4-dim array (batchsize, in_channels, num_point, 1) 80 | x_skip (array or Variable): Input from skip connection. 81 | 4-dim array (batchsize, in_channels_skip, num_point*m, 1) 82 | 83 | Returns (array or Variable): 84 | 4-dim array (batchsize, out_channels, num_point*m, 1) 85 | Number of output points are increased to `num_point*m` after 86 | deconvolution. 87 | """ 88 | assert x.ndim == 4 # (bs, ch, N, 1) 89 | assert x_skip.ndim == 4 # (bs, ch, N, 1) 90 | bs, ch, num_point, w = x.shape 91 | bs, ch_skip, num_pointm, w_skip = x_skip.shape 92 | assert w == 1 93 | assert w_skip == 1 94 | assert num_point * self.m == num_pointm 95 | # --- deconvolution from parent node to children part --- 96 | # (bs, ch, n, 1) -> (bs, out_ch*m, n, 1) -> (bs, out_ch, n*m, 1) 97 | h = self.conv(x) 98 | h = functions.reshape(h, (bs, self.out_deconv_channels, 99 | self.m * num_point, 1)) 100 | # --- skip connection part --- 101 | # Do nothing, just concat. 102 | # Other way is to put one convolution operation, 103 | # and we can reduce channels for `h_skip` here. 104 | # h_skip = self.conv_skip(x_skip) 105 | h_skip = x_skip # (bs, ch_skip, n*m, 1) 106 | 107 | h = functions.concat([h, h_skip], axis=1) 108 | h = self.kdcontextconv(h) 109 | return h 110 | 111 | 112 | if __name__ == '__main__': 113 | import numpy 114 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 115 | 116 | batchsize = 1 117 | num_point = 135 # try 100, 128, 135 118 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 119 | dim = 3 120 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 121 | point_set2 = numpy.random.rand(num_point, dim).astype(numpy.float32) 122 | print('point_set', point_set.shape) 123 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 124 | point_set, max_level=7, calc_split_positions=True) 125 | points2, split_dims2, inds2, kdtree2, split_positions2 = construct_kdtree_data( 126 | point_set2, max_level=7, calc_split_positions=True) 127 | print('points', points.shape) # 128 point here! 128 | kdconvblock = KDContextConvBlock( 129 | 3, m=2**3, feature_learning_mlp=[16], feature_aggregation_mlp=[24]) 130 | kddeconvblock = KDContextDeconvBlock( 131 | 24, m=2**3, out_deconv_channels=12, 132 | feature_learning_mlp=[28], feature_aggregation_mlp=[30]) 133 | # split_dim = numpy.array([split_dims[-1], split_dims2[-1]]) 134 | # print('split_dim', split_dim.shape) 135 | pts = numpy.array([points, points2]) 136 | pts = numpy.transpose(pts, (0, 2, 1))[:, :, :, None] 137 | print('pts', pts.shape) # (2, 3, 128, 1) 138 | pts2 = kdconvblock(pts) 139 | print('pts2', pts2.shape) # (2, 24, 16, 1) 140 | out = kddeconvblock(pts2, pts) 141 | print('out', out.shape) # (2, 30, 128, 1) 142 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet/pointnet_seg.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions, cuda 3 | from chainer import links 4 | from chainer import reporter 5 | 6 | from chainer_pointnet.models.conv_block import ConvBlock 7 | from chainer_pointnet.models.pointnet.transform_net import TransformNet 8 | 9 | 10 | def calc_trans_loss(t): 11 | # Loss to enforce the transformation as orthogonal matrix 12 | # t (batchsize, K, K) - transform matrix 13 | xp = cuda.get_array_module(t) 14 | bs, k1, k2 = t.shape 15 | assert k1 == k2 16 | mat_diff = functions.matmul(t, functions.transpose(t, (0, 2, 1))) 17 | mat_diff = mat_diff - xp.identity(k1, dtype=xp.float32) 18 | # divide by 2. is to make the behavior same with tf. 19 | # https://www.tensorflow.org/versions/r1.1/api_docs/python/tf/nn/l2_loss 20 | return functions.sum(functions.batch_l2_norm_squared(mat_diff)) / 2. 21 | 22 | 23 | class PointNetSeg(chainer.Chain): 24 | 25 | """Segmentation PointNet 26 | 27 | Input is (minibatch, K, N, 1), output is (minibatch, out_dim, N) 28 | 29 | Args: 30 | out_dim (int): output dimension, number of class for classification 31 | in_dim: input dimension for each point. default is 3, (x, y, z). 32 | middle_dim (int): hidden layer 33 | dropout_ratio (float): dropout ratio, not used now. 34 | use_bn (bool): use batch normalization or not. 35 | trans (bool): use TransformNet or not. 36 | False means not to use TransformNet, corresponds to 37 | PointNetVanilla. True corresponds to PointNet in the paper. 38 | trans_lam1 (float): regularization term for input transform. 39 | used in training. it is simply ignored when `trans` is False. 40 | trans_lam2 (float): regularization term for feature transform 41 | used in training. it is simply ignored when `trans` is False. 42 | compute_accuracy (bool): compute & report accuracy or not 43 | residual (bool): use residual connection or not 44 | """ 45 | 46 | def __init__(self, out_dim, in_dim=3, middle_dim=64, dropout_ratio=0.3, 47 | use_bn=True, trans=True, trans_lam1=0.001, trans_lam2=0.001, 48 | compute_accuracy=True, residual=False): 49 | super(PointNetSeg, self).__init__() 50 | with self.init_scope(): 51 | if trans: 52 | self.input_transform_net = TransformNet( 53 | k=in_dim, use_bn=use_bn, residual=residual) 54 | 55 | self.conv_block1 = ConvBlock( 56 | in_dim, 64, ksize=1, use_bn=use_bn, residual=residual) 57 | self.conv_block2 = ConvBlock( 58 | 64, middle_dim, ksize=1, use_bn=use_bn, residual=residual) 59 | if trans: 60 | self.feature_transform_net = TransformNet( 61 | k=middle_dim, use_bn=use_bn, residual=residual) 62 | 63 | self.conv_block3 = ConvBlock( 64 | middle_dim, 64, ksize=1, use_bn=use_bn, residual=residual) 65 | self.conv_block4 = ConvBlock( 66 | 64, 128, ksize=1, use_bn=use_bn, residual=residual) 67 | self.conv_block5 = ConvBlock( 68 | 128, 1024, ksize=1, use_bn=use_bn, residual=residual) 69 | 70 | # --- concat point_feat & global_feat --- 71 | self.conv_block6 = ConvBlock( 72 | middle_dim + 1024, 512, ksize=1, use_bn=use_bn, residual=residual) 73 | self.conv_block7 = ConvBlock( 74 | 512, 256, ksize=1, use_bn=use_bn, residual=residual) 75 | self.conv_block8 = ConvBlock( 76 | 256, 128, ksize=1, use_bn=use_bn, residual=residual) 77 | self.conv_block9 = ConvBlock( 78 | 128, 128, ksize=1, use_bn=use_bn, residual=residual) 79 | self.conv10 = links.Convolution2D( 80 | 128, out_dim, ksize=1) 81 | 82 | self.in_dim = in_dim 83 | self.trans = trans 84 | self.trans_lam1 = trans_lam1 85 | self.trans_lam2 = trans_lam2 86 | self.compute_accuracy = compute_accuracy 87 | 88 | def __call__(self, x, t): 89 | h, t1, t2 = self.calc(x) 90 | # h: (bs, ch, N), t: (bs, N) 91 | # print('h', h.shape, 't', t.shape) 92 | bs, ch, n = h.shape 93 | h = functions.reshape(functions.transpose(h, (0, 2, 1)), (bs * n, ch)) 94 | t = functions.reshape(t, (bs * n,)) 95 | cls_loss = functions.softmax_cross_entropy(h, t) 96 | reporter.report({'cls_loss': cls_loss}, self) 97 | 98 | loss = cls_loss 99 | # Enforce the transformation as orthogonal matrix 100 | if self.trans and self.trans_lam1 >= 0: 101 | trans_loss1 = self.trans_lam1 * calc_trans_loss(t1) 102 | reporter.report({'trans_loss1': trans_loss1}, self) 103 | loss = loss + trans_loss1 104 | if self.trans and self.trans_lam2 >= 0: 105 | trans_loss2 = self.trans_lam2 * calc_trans_loss(t2) 106 | reporter.report({'trans_loss2': trans_loss2}, self) 107 | loss = loss + trans_loss2 108 | reporter.report({'loss': loss}, self) 109 | 110 | if self.compute_accuracy: 111 | acc = functions.accuracy(h, t) 112 | reporter.report({'accuracy': acc}, self) 113 | return loss 114 | 115 | def calc(self, x): 116 | # x: (minibatch, K, N, 1) 117 | # N - num_point 118 | # K - feature degree (this is 3 for xyz input, 64 for middle layer) 119 | assert x.ndim == 4 120 | assert x.shape[-1] == 1 121 | 122 | # --- input transform --- 123 | if self.trans: 124 | h, t1 = self.input_transform_net(x) 125 | else: 126 | h = x 127 | t1 = 0 # dummy 128 | 129 | h = self.conv_block1(h) 130 | h = self.conv_block2(h) 131 | 132 | # --- feature transform --- 133 | if self.trans: 134 | point_feat, t2 = self.feature_transform_net(h) 135 | else: 136 | point_feat = h 137 | t2 = 0 # dummy 138 | 139 | h = self.conv_block3(point_feat) 140 | h = self.conv_block4(h) 141 | h = self.conv_block5(h) 142 | 143 | # Symmetric function: max pooling 144 | bs, k, n, tmp = h.shape 145 | assert tmp == 1 146 | h = functions.max_pooling_2d(h, ksize=h.shape[2:]) 147 | # h: (minibatch, K, 1, 1) 148 | global_feat = functions.broadcast_to(h, (bs, k, n, 1)) 149 | h = functions.concat([point_feat, global_feat], axis=1) 150 | 151 | h = self.conv_block6(h) 152 | h = self.conv_block7(h) 153 | h = self.conv_block8(h) 154 | h = self.conv_block9(h) 155 | h = self.conv10(h) 156 | return h[:, :, :, 0], t1, t2 157 | -------------------------------------------------------------------------------- /chainer_pointnet/utils/sampling.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from chainer import cuda 3 | 4 | 5 | def l2_norm(x, y): 6 | """Calculate l2 norm (distance) of `x` and `y`. 7 | 8 | Args: 9 | x (numpy.ndarray or cupy): (batch_size, num_point, coord_dim) 10 | y (numpy.ndarray): (batch_size, num_point, coord_dim) 11 | 12 | Returns (numpy.ndarray): (batch_size, num_point,) 13 | 14 | """ 15 | return ((x - y) ** 2).sum(axis=2) 16 | 17 | 18 | def farthest_point_sampling(pts, k, initial_idx=None, metrics=l2_norm, 19 | skip_initial=False, indices_dtype=numpy.int32, 20 | distances_dtype=numpy.float32): 21 | """Batch operation of farthest point sampling 22 | 23 | Code referenced from below link by @Graipher 24 | https://codereview.stackexchange.com/questions/179561/farthest-point-algorithm-in-python 25 | 26 | Args: 27 | pts (numpy.ndarray or cupy.ndarray): 2-dim array (num_point, coord_dim) 28 | or 3-dim array (batch_size, num_point, coord_dim) 29 | When input is 2-dim array, it is treated as 3-dim array with 30 | `batch_size=1`. 31 | k (int): number of points to sample 32 | initial_idx (int): initial index to start farthest point sampling. 33 | `None` indicates to sample from random index, 34 | in this case the returned value is not deterministic. 35 | metrics (callable): metrics function, indicates how to calc distance. 36 | skip_initial (bool): If True, initial point is skipped to store as 37 | farthest point. It stabilizes the function output. 38 | xp (numpy or cupy): 39 | indices_dtype (): dtype of output `indices` 40 | distances_dtype (): dtype of output `distances` 41 | 42 | Returns (tuple): `indices` and `distances`. 43 | indices (numpy.ndarray or cupy.ndarray): 2-dim array (batch_size, k, ) 44 | indices of sampled farthest points. 45 | `pts[indices[i, j]]` represents `i-th` batch element of `j-th` 46 | farthest point. 47 | distances (numpy.ndarray or cupy.ndarray): 3-dim array 48 | (batch_size, k, num_point) 49 | 50 | """ 51 | if pts.ndim == 2: 52 | # insert batch_size axis 53 | pts = pts[None, ...] 54 | assert pts.ndim == 3 55 | xp = cuda.get_array_module(pts) 56 | batch_size, num_point, coord_dim = pts.shape 57 | indices = xp.zeros((batch_size, k, ), dtype=indices_dtype) 58 | 59 | # distances[bs, i, j] is distance between i-th farthest point `pts[bs, i]` 60 | # and j-th input point `pts[bs, j]`. 61 | distances = xp.zeros((batch_size, k, num_point), dtype=distances_dtype) 62 | if initial_idx is None: 63 | indices[:, 0] = xp.random.randint(len(pts)) 64 | else: 65 | indices[:, 0] = initial_idx 66 | 67 | batch_indices = xp.arange(batch_size) 68 | farthest_point = pts[batch_indices, indices[:, 0]] 69 | # minimum distances to the sampled farthest point 70 | try: 71 | min_distances = metrics(farthest_point[:, None, :], pts) 72 | except Exception as e: 73 | import IPython; IPython.embed() 74 | 75 | if skip_initial: 76 | # Override 0-th `indices` by the farthest point of `initial_idx` 77 | indices[:, 0] = xp.argmax(min_distances, axis=1) 78 | farthest_point = pts[batch_indices, indices[:, 0]] 79 | min_distances = metrics(farthest_point[:, None, :], pts) 80 | 81 | distances[:, 0, :] = min_distances 82 | for i in range(1, k): 83 | indices[:, i] = xp.argmax(min_distances, axis=1) 84 | farthest_point = pts[batch_indices, indices[:, i]] 85 | dist = metrics(farthest_point[:, None, :], pts) 86 | distances[:, i, :] = dist 87 | min_distances = xp.minimum(min_distances, dist) 88 | return indices, distances 89 | 90 | 91 | if __name__ == '__main__': 92 | # when num_point = 10000 & k = 1000 & batch_size = 32, 93 | # CPU takes 6 sec, GPU takes 0.5 sec. 94 | 95 | from contextlib import contextmanager 96 | from time import time 97 | 98 | @contextmanager 99 | def timer(name): 100 | t0 = time() 101 | yield 102 | t1 = time() 103 | print('[{}] done in {:.3f} s'.format(name, t1-t0)) 104 | 105 | # batch_size = 32 106 | # num_point = 10000 107 | # coord_dim = 2 108 | # k = 1000 109 | # do_plot = False 110 | batch_size = 3 111 | num_point = 100 112 | coord_dim = 2 113 | k = 5 114 | do_plot = True 115 | 116 | device = -1 117 | print('num_point', num_point, 'device', device) 118 | if device == -1: 119 | pts = numpy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 120 | else: 121 | import cupy 122 | pts = cupy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 123 | 124 | with timer('1st'): 125 | farthest_indices, distances = farthest_point_sampling(pts, k) 126 | 127 | with timer('2nd'): # time measuring twice. 128 | farthest_indices, distances = farthest_point_sampling(pts, k) 129 | 130 | with timer('3rd'): # time measuring twice. 131 | farthest_indices, distances = farthest_point_sampling( 132 | pts, k, skip_initial=True) 133 | 134 | # with timer('gpu'): 135 | # farthest_indices = farthest_point_sampling_gpu(pts, k) 136 | print('farthest_indices', farthest_indices.shape, type(farthest_indices)) 137 | 138 | if do_plot: 139 | import matplotlib 140 | matplotlib.use('Agg') 141 | import matplotlib.pyplot as plt 142 | import os 143 | pts = cuda.to_cpu(pts) 144 | farthest_indices = cuda.to_cpu(farthest_indices) 145 | if not os.path.exists('results'): 146 | os.mkdir('results') 147 | for index in range(batch_size): 148 | fig, ax = plt.subplots() 149 | plt.grid(False) 150 | plt.scatter(pts[index, :, 0], pts[index, :, 1], c='k', s=4) 151 | plt.scatter(pts[index, farthest_indices[index], 0], pts[index, farthest_indices[index], 1], c='r', s=4) 152 | # plt.show() 153 | plt.savefig('results/farthest_point_sampling_{}.png'.format(index)) 154 | 155 | # --- To extract farthest_points, you can use this kind of advanced indexing --- 156 | farthest_points = pts[numpy.arange(batch_size)[:, None], 157 | farthest_indices, :] 158 | print('farthest_points', farthest_points.shape) 159 | for index in range(batch_size): 160 | farthest_pts_index = pts[index, farthest_indices[index], :] 161 | print('farthest', farthest_points[index].shape, 162 | farthest_pts_index.shape, 163 | numpy.sum(farthest_points[index] - farthest_pts_index)) 164 | assert numpy.allclose(farthest_points[index], farthest_pts_index) 165 | 166 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/meta/all_data_label.txt: -------------------------------------------------------------------------------- 1 | Area_1_conferenceRoom_1.npy 2 | Area_1_conferenceRoom_2.npy 3 | Area_1_copyRoom_1.npy 4 | Area_1_hallway_1.npy 5 | Area_1_hallway_2.npy 6 | Area_1_hallway_3.npy 7 | Area_1_hallway_4.npy 8 | Area_1_hallway_5.npy 9 | Area_1_hallway_6.npy 10 | Area_1_hallway_7.npy 11 | Area_1_hallway_8.npy 12 | Area_1_office_10.npy 13 | Area_1_office_11.npy 14 | Area_1_office_12.npy 15 | Area_1_office_13.npy 16 | Area_1_office_14.npy 17 | Area_1_office_15.npy 18 | Area_1_office_16.npy 19 | Area_1_office_17.npy 20 | Area_1_office_18.npy 21 | Area_1_office_19.npy 22 | Area_1_office_1.npy 23 | Area_1_office_20.npy 24 | Area_1_office_21.npy 25 | Area_1_office_22.npy 26 | Area_1_office_23.npy 27 | Area_1_office_24.npy 28 | Area_1_office_25.npy 29 | Area_1_office_26.npy 30 | Area_1_office_27.npy 31 | Area_1_office_28.npy 32 | Area_1_office_29.npy 33 | Area_1_office_2.npy 34 | Area_1_office_30.npy 35 | Area_1_office_31.npy 36 | Area_1_office_3.npy 37 | Area_1_office_4.npy 38 | Area_1_office_5.npy 39 | Area_1_office_6.npy 40 | Area_1_office_7.npy 41 | Area_1_office_8.npy 42 | Area_1_office_9.npy 43 | Area_1_pantry_1.npy 44 | Area_1_WC_1.npy 45 | Area_2_auditorium_1.npy 46 | Area_2_auditorium_2.npy 47 | Area_2_conferenceRoom_1.npy 48 | Area_2_hallway_10.npy 49 | Area_2_hallway_11.npy 50 | Area_2_hallway_12.npy 51 | Area_2_hallway_1.npy 52 | Area_2_hallway_2.npy 53 | Area_2_hallway_3.npy 54 | Area_2_hallway_4.npy 55 | Area_2_hallway_5.npy 56 | Area_2_hallway_6.npy 57 | Area_2_hallway_7.npy 58 | Area_2_hallway_8.npy 59 | Area_2_hallway_9.npy 60 | Area_2_office_10.npy 61 | Area_2_office_11.npy 62 | Area_2_office_12.npy 63 | Area_2_office_13.npy 64 | Area_2_office_14.npy 65 | Area_2_office_1.npy 66 | Area_2_office_2.npy 67 | Area_2_office_3.npy 68 | Area_2_office_4.npy 69 | Area_2_office_5.npy 70 | Area_2_office_6.npy 71 | Area_2_office_7.npy 72 | Area_2_office_8.npy 73 | Area_2_office_9.npy 74 | Area_2_storage_1.npy 75 | Area_2_storage_2.npy 76 | Area_2_storage_3.npy 77 | Area_2_storage_4.npy 78 | Area_2_storage_5.npy 79 | Area_2_storage_6.npy 80 | Area_2_storage_7.npy 81 | Area_2_storage_8.npy 82 | Area_2_storage_9.npy 83 | Area_2_WC_1.npy 84 | Area_2_WC_2.npy 85 | Area_3_conferenceRoom_1.npy 86 | Area_3_hallway_1.npy 87 | Area_3_hallway_2.npy 88 | Area_3_hallway_3.npy 89 | Area_3_hallway_4.npy 90 | Area_3_hallway_5.npy 91 | Area_3_hallway_6.npy 92 | Area_3_lounge_1.npy 93 | Area_3_lounge_2.npy 94 | Area_3_office_10.npy 95 | Area_3_office_1.npy 96 | Area_3_office_2.npy 97 | Area_3_office_3.npy 98 | Area_3_office_4.npy 99 | Area_3_office_5.npy 100 | Area_3_office_6.npy 101 | Area_3_office_7.npy 102 | Area_3_office_8.npy 103 | Area_3_office_9.npy 104 | Area_3_storage_1.npy 105 | Area_3_storage_2.npy 106 | Area_3_WC_1.npy 107 | Area_3_WC_2.npy 108 | Area_4_conferenceRoom_1.npy 109 | Area_4_conferenceRoom_2.npy 110 | Area_4_conferenceRoom_3.npy 111 | Area_4_hallway_10.npy 112 | Area_4_hallway_11.npy 113 | Area_4_hallway_12.npy 114 | Area_4_hallway_13.npy 115 | Area_4_hallway_14.npy 116 | Area_4_hallway_1.npy 117 | Area_4_hallway_2.npy 118 | Area_4_hallway_3.npy 119 | Area_4_hallway_4.npy 120 | Area_4_hallway_5.npy 121 | Area_4_hallway_6.npy 122 | Area_4_hallway_7.npy 123 | Area_4_hallway_8.npy 124 | Area_4_hallway_9.npy 125 | Area_4_lobby_1.npy 126 | Area_4_lobby_2.npy 127 | Area_4_office_10.npy 128 | Area_4_office_11.npy 129 | Area_4_office_12.npy 130 | Area_4_office_13.npy 131 | Area_4_office_14.npy 132 | Area_4_office_15.npy 133 | Area_4_office_16.npy 134 | Area_4_office_17.npy 135 | Area_4_office_18.npy 136 | Area_4_office_19.npy 137 | Area_4_office_1.npy 138 | Area_4_office_20.npy 139 | Area_4_office_21.npy 140 | Area_4_office_22.npy 141 | Area_4_office_2.npy 142 | Area_4_office_3.npy 143 | Area_4_office_4.npy 144 | Area_4_office_5.npy 145 | Area_4_office_6.npy 146 | Area_4_office_7.npy 147 | Area_4_office_8.npy 148 | Area_4_office_9.npy 149 | Area_4_storage_1.npy 150 | Area_4_storage_2.npy 151 | Area_4_storage_3.npy 152 | Area_4_storage_4.npy 153 | Area_4_WC_1.npy 154 | Area_4_WC_2.npy 155 | Area_4_WC_3.npy 156 | Area_4_WC_4.npy 157 | Area_5_conferenceRoom_1.npy 158 | Area_5_conferenceRoom_2.npy 159 | Area_5_conferenceRoom_3.npy 160 | Area_5_hallway_10.npy 161 | Area_5_hallway_11.npy 162 | Area_5_hallway_12.npy 163 | Area_5_hallway_13.npy 164 | Area_5_hallway_14.npy 165 | Area_5_hallway_15.npy 166 | Area_5_hallway_1.npy 167 | Area_5_hallway_2.npy 168 | Area_5_hallway_3.npy 169 | Area_5_hallway_4.npy 170 | Area_5_hallway_5.npy 171 | Area_5_hallway_6.npy 172 | Area_5_hallway_7.npy 173 | Area_5_hallway_8.npy 174 | Area_5_hallway_9.npy 175 | Area_5_lobby_1.npy 176 | Area_5_office_10.npy 177 | Area_5_office_11.npy 178 | Area_5_office_12.npy 179 | Area_5_office_13.npy 180 | Area_5_office_14.npy 181 | Area_5_office_15.npy 182 | Area_5_office_16.npy 183 | Area_5_office_17.npy 184 | Area_5_office_18.npy 185 | Area_5_office_19.npy 186 | Area_5_office_1.npy 187 | Area_5_office_20.npy 188 | Area_5_office_21.npy 189 | Area_5_office_22.npy 190 | Area_5_office_23.npy 191 | Area_5_office_24.npy 192 | Area_5_office_25.npy 193 | Area_5_office_26.npy 194 | Area_5_office_27.npy 195 | Area_5_office_28.npy 196 | Area_5_office_29.npy 197 | Area_5_office_2.npy 198 | Area_5_office_30.npy 199 | Area_5_office_31.npy 200 | Area_5_office_32.npy 201 | Area_5_office_33.npy 202 | Area_5_office_34.npy 203 | Area_5_office_35.npy 204 | Area_5_office_36.npy 205 | Area_5_office_37.npy 206 | Area_5_office_38.npy 207 | Area_5_office_39.npy 208 | Area_5_office_3.npy 209 | Area_5_office_40.npy 210 | Area_5_office_41.npy 211 | Area_5_office_42.npy 212 | Area_5_office_4.npy 213 | Area_5_office_5.npy 214 | Area_5_office_6.npy 215 | Area_5_office_7.npy 216 | Area_5_office_8.npy 217 | Area_5_office_9.npy 218 | Area_5_pantry_1.npy 219 | Area_5_storage_1.npy 220 | Area_5_storage_2.npy 221 | Area_5_storage_3.npy 222 | Area_5_storage_4.npy 223 | Area_5_WC_1.npy 224 | Area_5_WC_2.npy 225 | Area_6_conferenceRoom_1.npy 226 | Area_6_copyRoom_1.npy 227 | Area_6_hallway_1.npy 228 | Area_6_hallway_2.npy 229 | Area_6_hallway_3.npy 230 | Area_6_hallway_4.npy 231 | Area_6_hallway_5.npy 232 | Area_6_hallway_6.npy 233 | Area_6_lounge_1.npy 234 | Area_6_office_10.npy 235 | Area_6_office_11.npy 236 | Area_6_office_12.npy 237 | Area_6_office_13.npy 238 | Area_6_office_14.npy 239 | Area_6_office_15.npy 240 | Area_6_office_16.npy 241 | Area_6_office_17.npy 242 | Area_6_office_18.npy 243 | Area_6_office_19.npy 244 | Area_6_office_1.npy 245 | Area_6_office_20.npy 246 | Area_6_office_21.npy 247 | Area_6_office_22.npy 248 | Area_6_office_23.npy 249 | Area_6_office_24.npy 250 | Area_6_office_25.npy 251 | Area_6_office_26.npy 252 | Area_6_office_27.npy 253 | Area_6_office_28.npy 254 | Area_6_office_29.npy 255 | Area_6_office_2.npy 256 | Area_6_office_30.npy 257 | Area_6_office_31.npy 258 | Area_6_office_32.npy 259 | Area_6_office_33.npy 260 | Area_6_office_34.npy 261 | Area_6_office_35.npy 262 | Area_6_office_36.npy 263 | Area_6_office_37.npy 264 | Area_6_office_3.npy 265 | Area_6_office_4.npy 266 | Area_6_office_5.npy 267 | Area_6_office_6.npy 268 | Area_6_office_7.npy 269 | Area_6_office_8.npy 270 | Area_6_office_9.npy 271 | Area_6_openspace_1.npy 272 | Area_6_pantry_1.npy 273 | -------------------------------------------------------------------------------- /chainer_pointnet/utils/grouping.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from chainer import cuda 3 | 4 | from chainer_pointnet.utils.sampling import farthest_point_sampling 5 | 6 | 7 | def _l2_norm(x, y): 8 | """Calculate l2 norm (distance) of `x` and `y`. 9 | 10 | Args: 11 | x (numpy.ndarray or cupy.ndarray): 12 | input points, 3-dim array (batch_size, num_point, coord_dim) 13 | y (numpy.ndarray or cupy.ndarray): 14 | query points, 3-dim array (batch_size, k, coord_dim) 15 | 16 | Returns (numpy.ndarray): (batch_size, k, num_point,) 17 | 18 | """ 19 | return ((x[:, None, :, :] - y[:, :, None, :]) ** 2).sum(axis=3) 20 | 21 | 22 | def query_ball_point(pts, indices, num_sample, radius=None, metrics=_l2_norm): 23 | """query ball point 24 | 25 | `num_sample` nearest neighbor search for each `indices` among `pts`. 26 | When `radius` is set, query point must be less than `radius`, 27 | = 28 | Args: 29 | pts (numpy.ndarray or cupy.ndarray): input points 30 | 3-dim array (batch_size, num_point, coord_dim) 31 | indices: 2-dim array (batch_size, k) indices of `pts` to be center. 32 | num_sample (int): number of points selected in each region 33 | radius (float or None): radius of each region to search. 34 | When `None`, it is equivalent to the case radius is infinity, 35 | and the behavior is same k-nearest neighbor with `k=num_sample`. 36 | metrics (callable): metrics function to calculate distance 37 | 38 | Returns (numpy.ndarray or cupy.ndarray): grouped indices 39 | 3-dim array (batch_size, k, num_sample) 40 | 41 | """ 42 | # --- calc diff --- 43 | diff = calc_diff(pts, indices, metrics=metrics) 44 | # --- sort & select nearest indices --- 45 | return query_ball_by_diff(diff, num_sample=num_sample, radius=radius) 46 | 47 | 48 | def calc_diff(pts, indices, metrics=_l2_norm): 49 | batch_size = pts.shape[0] 50 | # query_pts (batch_size, k, coor_dim) -> k points to be center 51 | query_pts = pts[numpy.arange(batch_size)[:, None], indices, :] 52 | 53 | # diff (batch_size, k, num_point) 54 | diff = metrics(pts, query_pts) 55 | return diff 56 | 57 | 58 | def query_ball_by_diff(diff, num_sample, radius=None): 59 | """ 60 | 61 | Args: 62 | diff (numpy.ndarray or cupy.ndarray): 3-dim array 63 | (batch_size, k, num_point) 64 | num_sample (int): number of sample in each region 65 | radius (float): radius of each inregion 66 | 67 | Returns (numpy.ndarray or cupy.ndarray): grouped indices 68 | 3-dim array (batch_size, k, num_sample) 69 | 70 | """ 71 | xp = cuda.get_array_module(diff) 72 | batch_size, k, num_point = diff.shape 73 | diff_sorted_indices = xp.argsort(diff, axis=2) 74 | if radius is None: 75 | return diff_sorted_indices[:, :, :num_sample] 76 | else: 77 | diff_sorted_indices = diff_sorted_indices[:, :, :num_sample] 78 | diff_sorted = diff[xp.arange(batch_size)[:, None, None], 79 | xp.arange(k)[None, :, None], 80 | diff_sorted_indices] 81 | # take original value when it is smaller than radius. 82 | return xp.where(diff_sorted < radius, diff_sorted_indices, 83 | diff_sorted_indices[:, :, 0:1]) 84 | 85 | 86 | if __name__ == '__main__': 87 | # when num_point = 10000 & k = 1000 & batch_size = 32, 88 | # CPU takes 6 sec, GPU takes 0.5 sec. 89 | 90 | from contextlib import contextmanager 91 | from time import time 92 | 93 | @contextmanager 94 | def timer(name): 95 | t0 = time() 96 | yield 97 | t1 = time() 98 | print('[{}] done in {:.3f} s'.format(name, t1-t0)) 99 | 100 | # batch_size = 8 101 | # num_point = 10000 102 | # coord_dim = 2 103 | # k = 1000 104 | # do_plot = False 105 | batch_size = 3 106 | num_point = 200 107 | coord_dim = 2 108 | k = 8 109 | do_plot = False 110 | 111 | # for grouping 112 | num_sample = 10 113 | # radius = None 114 | # radius = 10.0 115 | radius = 0.0000001 116 | 117 | device = -1 118 | print('num_point', num_point, 'device', device) 119 | if device == -1: 120 | pts = numpy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 121 | else: 122 | import cupy 123 | pts = cupy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 124 | 125 | with timer('farthest_point_sampling'): 126 | farthest_indices, distances = farthest_point_sampling( 127 | pts, k, skip_initial=True) 128 | print('farthest_indices', farthest_indices.shape, type(farthest_indices)) 129 | 130 | # efficient calculation using `distances` 131 | with timer('query_ball_by_diff'): 132 | grouped_indices = query_ball_by_diff(distances, num_sample, radius=radius) 133 | print('grouped_indices', grouped_indices.shape, type(grouped_indices)) 134 | # (batch_size, k, num_sample) 135 | 136 | # query_ball_point, it will calculate `diff` <- same with `distances` above, 137 | # takes time for calculation 138 | with timer('query_ball_point'): 139 | grouped_indices2 = query_ball_point(pts, farthest_indices, num_sample, radius=radius) 140 | print('grouped_indices2', grouped_indices.shape, type(grouped_indices)) 141 | assert numpy.allclose(cuda.to_cpu(grouped_indices), 142 | cuda.to_cpu(grouped_indices2)) 143 | 144 | # take grouped points 145 | grouped_points = pts[numpy.arange(batch_size)[:, None, None], grouped_indices, :] 146 | print('grouped_points', grouped_points.shape) 147 | for i in range(grouped_points.shape[0]): 148 | for j in range(grouped_points.shape[1]): 149 | for k in range(grouped_points.shape[2]): 150 | index = grouped_indices[i, j, k] 151 | numpy.allclose(grouped_points[i, j, k, :], pts[i, index, :]) 152 | # print('test grouped_points', 153 | # numpy.sum(grouped_points[i, j, k, :] - pts[i, index, :])) 154 | 155 | if do_plot: 156 | import matplotlib 157 | matplotlib.use('Agg') 158 | import matplotlib.pyplot as plt 159 | import os 160 | pts = cuda.to_cpu(pts) 161 | farthest_indices = cuda.to_cpu(farthest_indices) 162 | grouped_indices_flatten = cuda.to_cpu( 163 | grouped_indices).reshape(batch_size, k * num_sample) 164 | if not os.path.exists('results'): 165 | os.mkdir('results') 166 | for index in range(batch_size): 167 | fig, ax = plt.subplots() 168 | plt.grid(False) 169 | plt.scatter(pts[index, :, 0], pts[index, :, 1], c='k', s=4) 170 | # farthest point itself is also inside grouped point, so write grouped points before drawing farthest points. 171 | plt.scatter(pts[index, grouped_indices_flatten[index], 0], pts[index, grouped_indices_flatten[index], 1], c='b', s=4) 172 | plt.scatter(pts[index, farthest_indices[index], 0], pts[index, farthest_indices[index], 1], c='r', s=4) 173 | # plt.show() 174 | plt.savefig('results/farthest_point_sampling_grouping_{}.png'.format(index)) 175 | -------------------------------------------------------------------------------- /chainer_pointnet/models/pointnet2/set_abstraction_block.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions 5 | 6 | from chainer_pointnet.models.conv_block import ConvBlock 7 | from chainer_pointnet.utils.grouping import query_ball_by_diff 8 | from chainer_pointnet.utils.sampling import farthest_point_sampling 9 | 10 | 11 | class SetAbstractionModule(chainer.Chain): 12 | 13 | def __init__(self, k, num_sample_in_region, radius, 14 | mlp, mlp2=None, in_channels=None, use_bn=True, 15 | activation=functions.relu, initial_idx=None, 16 | skip_initial=True, return_distance=False, 17 | residual=False): 18 | # k is number of sampled point (num_region) 19 | super(SetAbstractionModule, self).__init__() 20 | # Feature Extractor channel list 21 | assert isinstance(mlp, list) 22 | fe_ch_list = [in_channels] + mlp 23 | # Head channel list 24 | if mlp2 is None: 25 | mlp2 = [] 26 | assert isinstance(mlp2, list) 27 | head_ch_list = [mlp[-1]] + mlp2 28 | with self.init_scope(): 29 | self.sampling_grouping = SamplingGroupingModule( 30 | k=k, num_sample_in_region=num_sample_in_region, 31 | radius=radius, initial_idx=initial_idx, 32 | skip_initial=skip_initial, return_distance=return_distance) 33 | self.feature_extractor_list = chainer.ChainList( 34 | *[ConvBlock(fe_ch_list[i], fe_ch_list[i+1], ksize=1, 35 | use_bn=use_bn, activation=activation, 36 | residual=residual 37 | ) for i in range(len(mlp))]) 38 | self.head_list = chainer.ChainList( 39 | *[ConvBlock(head_ch_list[i], head_ch_list[i + 1], ksize=1, 40 | use_bn=use_bn, activation=activation, 41 | residual=residual 42 | ) for i in range(len(mlp2))]) 43 | self.use_bn = use_bn 44 | 45 | def __call__(self, coord_points, feature_points=None): 46 | # coord_points (batch_size, num_point, coord_dim) 47 | # feature_points (batch_size, num_point, ch) 48 | # num_point, ch: coord_dim 49 | 50 | # grouped_points (batch_size, k, num_sample, channel) 51 | # center_points (batch_size, k, coord_dim) 52 | grouped_points, center_points, dist = self.sampling_grouping( 53 | coord_points, feature_points=feature_points) 54 | # set alias `h` -> (bs, channel, num_sample, k) 55 | # Note: transpose may be removed by optimizing shape sequence for sampling_groupoing 56 | h = functions.transpose(grouped_points, (0, 3, 2, 1)) 57 | # h (bs, ch, num_sample_in_region, k=num_group) 58 | for conv_block in self.feature_extractor_list: 59 | h = conv_block(h) 60 | # TODO: try other option of pooling function 61 | h = functions.max(h, axis=2, keepdims=True) 62 | # h (bs, ch, 1, k=num_group) 63 | for conv_block in self.head_list: 64 | h = conv_block(h) 65 | h = functions.transpose(h[:, :, 0, :], (0, 2, 1)) 66 | # points (bs, k, coord), h (bs, k, ch'), dist (bs, k, num_point) 67 | return center_points, h, dist 68 | 69 | 70 | def _to_array(var): 71 | """"Input: numpy, cupy array or Variable. Output: numpy or cupy array""" 72 | if isinstance(var, chainer.Variable): 73 | var = var.data 74 | return var 75 | 76 | 77 | #TODO: this does not have parameter, change it to function instead of chain. 78 | class SamplingGroupingModule(chainer.Chain): 79 | 80 | def __init__(self, k, num_sample_in_region, radius=None, use_coord=True, 81 | initial_idx=None, skip_initial=True, return_distance=False): 82 | super(SamplingGroupingModule, self).__init__() 83 | # number of center point (sampled point) 84 | self.k = k 85 | 86 | # number of points grouped in each region with radius 87 | self.radius = radius 88 | self.num_sample_in_region = num_sample_in_region 89 | self.use_coord = use_coord 90 | self.initial_idx = initial_idx 91 | self.skip_initial = skip_initial 92 | self.return_distance = return_distance 93 | 94 | def __call__(self, coord_points, feature_points=None): 95 | # input: coord_points (batch_size, num_point, coord_dim) 96 | # input: feature_points (batch_size, num_point, channel) 97 | batch_size, num_point, coord_dim = coord_points.shape 98 | 99 | # sampling -> this only decides indices, calculation is done by array 100 | farthest_indices, distances = farthest_point_sampling( 101 | _to_array(coord_points), self.k, initial_idx=self.initial_idx, 102 | skip_initial=self.skip_initial) 103 | # grouping 104 | grouped_indices = query_ball_by_diff( 105 | distances, self.num_sample_in_region, radius=self.radius) 106 | if not self.return_distance: 107 | distances = None # release memory 108 | 109 | # grouped_indices (batch_size, k, num_sample) 110 | # grouped_points (batch_size, k, num_sample, coord_dim) 111 | grouped_points = coord_points[self.xp.arange(batch_size)[:, None, None], grouped_indices, :] 112 | # center_points (batch_size, k, coord_dim) -> new_coord_points 113 | center_points = coord_points[self.xp.arange(batch_size)[:, None], farthest_indices, :] 114 | 115 | # calculate relative coordinate 116 | grouped_points = grouped_points - functions.broadcast_to( 117 | center_points[:, :, None, :], grouped_points.shape) 118 | 119 | if feature_points is None: 120 | new_feature_points = grouped_points 121 | else: 122 | # grouped_indices (batch_size, k, num_sample) 123 | # grouped_feature_points (batch_size, k, num_sample, channel) 124 | grouped_feature_points = feature_points[ 125 | self.xp.arange(batch_size)[:, None, None], 126 | grouped_indices, :] 127 | if self.use_coord: 128 | new_feature_points = functions.concat([grouped_points, grouped_feature_points], axis=3) 129 | else: 130 | new_feature_points = grouped_feature_points 131 | # new_feature_points (batch_size, k, num_sample, channel') 132 | # center_points (batch_size, k, coord_dim) -> new_coord_points 133 | # distances (batch_size, k, num_point) it is for feature_propagation 134 | return new_feature_points, center_points, distances 135 | 136 | 137 | if __name__ == '__main__': 138 | batch_size = 3 139 | num_point = 100 140 | coord_dim = 2 141 | 142 | k = 5 143 | num_sample_in_region = 8 144 | radius = 0.4 145 | mlp = [16, 16] 146 | # mlp2 = [32, 32] 147 | mlp2 = None 148 | 149 | device = -1 150 | print('num_point', num_point, 'device', device) 151 | if device == -1: 152 | pts = numpy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 153 | else: 154 | import cupy 155 | pts = cupy.random.uniform(0, 1, (batch_size, num_point, coord_dim)) 156 | 157 | pts = pts.astype(numpy.float32) 158 | sam = SetAbstractionModule(k, num_sample_in_region, radius, mlp, mlp2=mlp2) 159 | 160 | coord, h = sam(pts) 161 | print('coord', type(coord), coord.shape) # (3, 5, 2) - (bs, k, coord) 162 | print('h', type(h), h.shape) # (3, 5, 32) - (bs, k, ch') 163 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdcontextnet/kdcontextconv_block.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | from chainer import functions 3 | 4 | from chainer_pointnet.models.conv_block import ConvBlock 5 | 6 | 7 | class KDContextConvBlock(chainer.Chain): 8 | 9 | """KD Context Convolution Block 10 | 11 | It performs each level feature learning & aggregation 12 | 13 | Args: 14 | in_channels (int or None): input channels 15 | m (int): Number to group to be aggregated. 16 | feature_learning_mlp (list): list of int, specifies MLP size for 17 | feature learning stage. 18 | feature_aggregation_mlp (list): list of int, specifies MLP size for 19 | feature aggregation stage. 20 | ksize (int or tuple): kernel size 21 | stride (int or tuple): stride size 22 | pad (int or tuple): padding size 23 | nobias (bool): use bias `b` or not. 24 | initialW: initiallizer of `W` 25 | initial_bias: initializer of `b` 26 | use_bn (bool): use batch normalization or not 27 | activation (callable): activation function 28 | dropout_ratio (float): dropout ratio, set negative value to skip 29 | dropout 30 | aggregation (bool): apply aggregation or not. 31 | Default is `True`, `num_point` will be `num_point//m` in output. 32 | `False` is set for deconvolution part, not reduce number of points. 33 | normalize (bool): apply normalization to calculate global context cues 34 | residual (bool): use residual connection or not 35 | """ 36 | 37 | def __init__(self, in_channels, m, 38 | feature_learning_mlp=None, 39 | feature_aggregation_mlp=None, 40 | ksize=1, stride=1, pad=0, 41 | nobias=False, initialW=None, initial_bias=None, use_bn=True, 42 | activation=functions.relu, dropout_ratio=-1, 43 | aggregation=True, normalize=False, residual=False): 44 | super(KDContextConvBlock, self).__init__() 45 | if feature_learning_mlp is None: 46 | print('feature_learning_mlp is None, value set automatically') 47 | feature_learning_mlp = [16] 48 | if feature_aggregation_mlp is None: 49 | print('feature_aggregation_mlp is None, value set automatically') 50 | feature_aggregation_mlp = [256] 51 | 52 | # must be multiple of 4. 53 | # Used for (y_i, \sigma(g(Y)), G(x_i)-\theta&\phi, H(x_i)) respectively 54 | # First 2 are for Local Contextual Cues and 55 | # Latter 2 are for Global Contextual Cues. 56 | assert feature_learning_mlp[-1] % 4 == 0 57 | # TODO: support dense-net calculation for flmlp 58 | flmlp = [in_channels] + feature_learning_mlp 59 | famlp = [flmlp[-1]//2] + feature_aggregation_mlp 60 | with self.init_scope(): 61 | self.flconvs = chainer.ChainList( 62 | *[ConvBlock(flmlp[i], flmlp[i+1], ksize=ksize, stride=stride, 63 | pad=pad, nobias=nobias, initialW=initialW, 64 | initial_bias=initial_bias, use_bn=use_bn, 65 | activation=activation, dropout_ratio=dropout_ratio, 66 | residual=residual 67 | ) for i in range(len(flmlp)-1)]) 68 | self.faconvs = chainer.ChainList( 69 | *[ConvBlock(famlp[i], famlp[i + 1], ksize=ksize, stride=stride, 70 | pad=pad, nobias=nobias, initialW=initialW, 71 | initial_bias=initial_bias, use_bn=use_bn, 72 | activation=activation, dropout_ratio=dropout_ratio, 73 | residual=residual 74 | ) for i in range(len(famlp)-1)]) 75 | self.m = m 76 | self.aggregation = aggregation 77 | self.normalize = normalize 78 | 79 | def __call__(self, x): 80 | """Main forward computation 81 | 82 | Args: 83 | x (array or Variable): 4-dim array (minibatch, K, N, 1). 84 | `K=in_channels` is channel, `N` is number of points. 85 | 86 | Returns (array or Variable): 4-dim array (minibatch, K', N//m, 1). 87 | `K'=feature_aggregation_mlp[-1]` is channel, 88 | `N` is number of input points. 89 | Number of output points are reduced to `N//m` after feature 90 | aggregation. 91 | If `aggregation=False`, then output shape is (minibatch, K', N, 1) 92 | 93 | """ 94 | assert x.ndim == 4 # (bs, ch, N, 1) 95 | assert x.shape[3] == 1 96 | h = x 97 | # Feature Learning Stage 98 | for conv in self.flconvs: 99 | h = conv(h) 100 | h0, h1, h2, h3 = functions.split_axis(h, 4, axis=1) 101 | # 1. Local Contextual Cues 102 | # TODO: support other symmetric function 103 | gy = functions.max_pooling_2d(h0, ksize=(self.m, 1)) 104 | bs, ch, n, _ = h1.shape 105 | assert n % self.m == 0 106 | h1 = functions.reshape(h1, (bs, ch, n//self.m, self.m)) 107 | h_local = functions.broadcast_to(functions.sigmoid(gy), h1.shape) * h1 108 | del gy, h0, h1 109 | h_local = functions.reshape(h_local, (bs, ch, n, 1)) 110 | # 2. Global Contextual Cues 111 | # See also https://arxiv.org/pdf/1711.07971.pdf 112 | # h2, h3 (batchsize, ch, N, 1) -> (bs, ch, N, m) 113 | # h2 is used for both \theta(x_i) and \phi(x_j). 114 | # h3 is used for H(x_j) of Eq(3) of 3DContextNet paper. 115 | bs, ch, n, _ = h2.shape 116 | assert n % self.m == 0 117 | 118 | # TODO: support other symmetric function 119 | # h2, h3 (batchsize, ch, N, 1) -> (bs, ch, N//m) 120 | h2 = functions.max_pooling_2d(h2, ksize=(self.m, 1))[:, :, :, 0] 121 | h3 = functions.max_pooling_2d(h3, ksize=(self.m, 1))[:, :, :, 0] 122 | if self.normalize: 123 | eps = 1e-3 124 | h2 = h2 * functions.broadcast_to(functions.rsqrt( 125 | functions.sum(h2 * h2, axis=1, keepdims=True) + eps), h2.shape) 126 | 127 | # g (bs, hw, hw), where hw=`n//self.m` 128 | g = functions.matmul(h2, h2, transa=True) 129 | # g(x_i, x_j) / C(x) can be replaced by softmax calculation. 130 | h_global = functions.matmul(h3, functions.softmax(g, axis=1)) 131 | del g, h2, h3 132 | # (bs, ch, hw=N//m) -> (bs, ch, N, 1) 133 | h_global = functions.reshape(h_global, (bs, ch, n//self.m, 1)) 134 | h_global = functions.broadcast_to(h_global, (bs, ch, n//self.m, self.m)) 135 | h_global = functions.reshape(h_global, (bs, ch, n, 1)) 136 | 137 | # Feature Aggregation Stage 138 | h = functions.concat([h_local, h_global], axis=1) 139 | del h_local, h_global 140 | for conv in self.faconvs: 141 | h = conv(h) 142 | if self.aggregation: 143 | # TODO: support other symmetric function 144 | h = functions.max_pooling_2d(h, ksize=(self.m, 1)) 145 | return h 146 | 147 | 148 | if __name__ == '__main__': 149 | import numpy 150 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 151 | 152 | batchsize = 1 153 | num_point = 135 # try 100, 128, 135 154 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 155 | dim = 3 156 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 157 | point_set2 = numpy.random.rand(num_point, dim).astype(numpy.float32) 158 | print('point_set', point_set.shape) 159 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 160 | point_set, max_level=7, calc_split_positions=True) 161 | points2, split_dims2, inds2, kdtree2, split_positions2 = construct_kdtree_data( 162 | point_set2, max_level=7, calc_split_positions=True) 163 | print('points', points.shape) # 128 point here! 164 | kdconvblock = KDContextConvBlock( 165 | 3, m=2**3, feature_learning_mlp=[16], feature_aggregation_mlp=[24], 166 | normalize=True) 167 | # split_dim = numpy.array([split_dims[-1], split_dims2[-1]]) 168 | # print('split_dim', split_dim.shape) 169 | pts = numpy.array([points, points2]) 170 | pts = numpy.transpose(pts, (0, 2, 1))[:, :, :, None] 171 | print('pts', pts.shape) 172 | out = kdconvblock(pts) 173 | print('out', out.shape) 174 | -------------------------------------------------------------------------------- /chainer_pointnet/models/kdcontextnet/kdcontextnet_seg.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | import chainer 4 | from chainer import functions 5 | from chainer import links 6 | from chainer import reporter 7 | 8 | from chainer_pointnet.models.conv_block import ConvBlock 9 | from chainer_pointnet.models.kdcontextnet.kdcontextconv_block import \ 10 | KDContextConvBlock 11 | from chainer_pointnet.models.kdcontextnet.kdcontextdeconv_block import \ 12 | KDContextDeconvBlock 13 | 14 | 15 | class KDContextNetSeg(chainer.Chain): 16 | 17 | """Segmentation 3DContextNet 18 | 19 | Args: 20 | out_dim (int): output dimension, number of class for classification 21 | in_dim (int or None): input dimension for each point. 22 | default is 3, (x, y, z). 23 | dropout_ratio (float): dropout ratio 24 | use_bn (bool): use batch normalization or not. 25 | compute_accuracy (bool): compute & report accuracy or not 26 | levels (list): list of int. It determines block depth and each 27 | block's grouping (receptive field) size. 28 | For example if `list=[5, 6, 7]`, it will take 29 | level1 block as 5-th depth (32) of KDTree, 30 | level2 as 6-th depth (64), level3 as 7-th depth (128) resp. 31 | Receptive field (number of points in each grouping) is 32 | `2**level` for each level. 33 | feature_learning_mlp_enc_list (list): list of list of int. 34 | Indicates each block's feature learning MLP size for encoder. 35 | feature_aggregation_mlp_enc_list (list): list of list of int. 36 | Indicates each block's feature aggregation MLP size for encoder. 37 | feature_learning_mlp_dec_list (list): list of list of int. 38 | Indicates each block's feature learning MLP size for decoder. 39 | feature_aggregation_mlp_dec_list (list): list of list of int. 40 | Indicates each block's feature aggregation MLP size for decoder. 41 | normalize (bool): apply normalization to calculate global context cues 42 | in `KDContextConvBlock` & KDContextDeconvBlock. 43 | residual (bool): use residual connection or not 44 | """ 45 | 46 | def __init__(self, out_dim, in_dim=3, dropout_ratio=-1, 47 | use_bn=True, compute_accuracy=True, 48 | levels=None, 49 | feature_learning_mlp_enc_list=None, 50 | feature_aggregation_mlp_enc_list=None, 51 | feature_learning_mlp_dec_list=None, 52 | feature_aggregation_mlp_dec_list=None, 53 | fc_mlp_list=None, normalize=False, residual=False 54 | ): 55 | super(KDContextNetSeg, self).__init__() 56 | levels = levels or [5, 7, 9] # (32, 128, 512) receptive field. 57 | if feature_learning_mlp_enc_list is None: 58 | feature_learning_mlp_enc_list = [ 59 | [64, 64, 128, 128], [64, 64, 256, 256], [64, 64, 512, 512]] 60 | if feature_aggregation_mlp_enc_list is None: 61 | feature_aggregation_mlp_enc_list = [[256], [512], [1024]] 62 | if feature_learning_mlp_dec_list is None: 63 | feature_learning_mlp_dec_list = list(reversed( 64 | feature_learning_mlp_enc_list)) 65 | if feature_aggregation_mlp_dec_list is None: 66 | feature_aggregation_mlp_dec_list = list(reversed( 67 | feature_aggregation_mlp_enc_list)) 68 | fc_mlp_list = fc_mlp_list or [128] 69 | 70 | in_channels_enc_list = [in_dim] + [ 71 | elem[-1] for elem in feature_aggregation_mlp_enc_list] 72 | in_channels_dec_list = [in_channels_enc_list[-1]] + [ 73 | elem[-1] for elem in feature_aggregation_mlp_dec_list] 74 | levels_diff = numpy.diff(numpy.array([0] + levels)) 75 | print('levels {}, levels_diff {}'.format(levels, levels_diff)) 76 | fcmlps = [feature_aggregation_mlp_dec_list[-1][-1]] + fc_mlp_list 77 | assert len(levels) == len(feature_learning_mlp_enc_list) 78 | assert len(levels) == len(feature_aggregation_mlp_enc_list) 79 | assert len(levels) == len(feature_learning_mlp_dec_list) 80 | assert len(levels) == len(feature_aggregation_mlp_dec_list) 81 | with self.init_scope(): 82 | # don't use dropout in conv blocks 83 | self.kdcontextconv_blocks = chainer.ChainList( 84 | *[KDContextConvBlock( 85 | in_channels_enc_list[i], m=2 ** levels_diff[i], 86 | feature_learning_mlp=feature_learning_mlp_enc_list[i], 87 | feature_aggregation_mlp=feature_aggregation_mlp_enc_list[i], 88 | use_bn=use_bn, normalize=normalize, residual=residual 89 | ) for i in range(len(levels_diff))]) 90 | self.kdcontextdeconv_blocks = chainer.ChainList( 91 | *[KDContextDeconvBlock( 92 | in_channels_dec_list[i], m=2 ** levels_diff[-i-1], 93 | out_deconv_channels=in_channels_dec_list[i]//2, 94 | feature_learning_mlp=feature_learning_mlp_dec_list[i], 95 | feature_aggregation_mlp=feature_aggregation_mlp_dec_list[i], 96 | use_bn=use_bn, normalize=normalize, residual=residual 97 | ) for i in range(len(levels_diff))]) 98 | self.conv_blocks = chainer.ChainList( 99 | *[ConvBlock( 100 | fcmlps[i], fcmlps[i+1], ksize=1, use_bn=use_bn, 101 | residual=residual 102 | ) for i in range(len(fcmlps)-1)]) 103 | self.conv = links.Convolution2D(fcmlps[-1], out_dim, ksize=1) 104 | self.compute_accuracy = compute_accuracy 105 | self.dropout_ratio = dropout_ratio 106 | 107 | def calc(self, x): 108 | # x (bs, ch, N, 1) 109 | assert x.ndim == 4 110 | assert x.shape[3] == 1 111 | h = x 112 | h_skip_list = [h] 113 | for kdconv_block in self.kdcontextconv_blocks: 114 | h = kdconv_block(h) 115 | h_skip_list.append(h) 116 | # h (bs, ch, N//2**levels[-1], 1) 117 | # TODO: support other symmetric function 118 | # TODO: Support concatenating whole global feature. 119 | # h = functions.max_pooling_2d(h, ksize=(h.shape[2], 1)) 120 | 121 | h_skip_list.pop(-1) # don't use last h as skip connection. 122 | for kddeconv_block in self.kdcontextdeconv_blocks: 123 | h_skip = h_skip_list.pop(-1) 124 | h = kddeconv_block(h, h_skip) 125 | assert len(h_skip_list) == 0 126 | for conv_block in self.conv_blocks: 127 | h = conv_block(h) 128 | if self.dropout_ratio > 0.: 129 | h = functions.dropout(h, self.dropout_ratio) 130 | h = self.conv(h) 131 | return h[:, :, :, 0] 132 | 133 | def __call__(self, x, t): 134 | h = self.calc(x) 135 | cls_loss = functions.softmax_cross_entropy(h, t) 136 | # reporter.report({'cls_loss': cls_loss}, self) 137 | loss = cls_loss 138 | reporter.report({'loss': loss}, self) 139 | if self.compute_accuracy: 140 | acc = functions.accuracy(h, t) 141 | reporter.report({'accuracy': acc}, self) 142 | return loss 143 | 144 | 145 | if __name__ == '__main__': 146 | from chainer_pointnet.utils.kdtree import construct_kdtree_data 147 | 148 | device = -1 149 | batchsize = 1 150 | num_point = 135 # try 100, 128, 135 151 | max_level = 9 # 2^7 -> 128. Final num_point will be 128 152 | big_data_test = True 153 | if big_data_test: 154 | device = 0 155 | # max_level=18, num_point 262144 run on GPU with batchsize=1 156 | max_level = 18 157 | num_point = 2 ** max_level 158 | print('max_level', max_level, 'num_point', num_point) 159 | dim = 3 160 | point_set = numpy.random.rand(num_point, dim).astype(numpy.float32) 161 | print('point_set', point_set.shape) 162 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 163 | point_set, max_level=max_level, calc_split_positions=True) 164 | print('points', points.shape) # 128 point here! 165 | kdnet = KDContextNetSeg(out_dim=5, use_bn=False) 166 | # split_dims = numpy.array(split_dims) 167 | # print('split_dims', split_dims.shape, split_dims.dtype) 168 | pts = numpy.transpose(points, (1, 0))[None, :, :, None] 169 | print('pts', pts.shape, split_dims.shape) 170 | if device >= 0: 171 | chainer.cuda.get_device_from_id(device).use() # Make a specified GPU current 172 | kdnet.to_gpu() # Copy the model to the GPU 173 | pts = chainer.cuda.to_gpu(pts) 174 | out = kdnet.calc(pts) 175 | print('out', out.shape) 176 | 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /experiments/s3dis/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | import argparse 5 | from distutils.util import strtobool 6 | import os 7 | 8 | import chainer 9 | from chainer import serializers 10 | from chainer import iterators 11 | from chainer import optimizers 12 | from chainer import training 13 | from chainer.dataset import to_device, concat_examples 14 | from chainer.datasets import TransformDataset 15 | from chainer.training import extensions as E 16 | 17 | from chainer_pointnet.models.kdcontextnet.kdcontextnet_seg import \ 18 | KDContextNetSeg 19 | from chainer_pointnet.models.kdnet.kdnet_seg import KDNetSeg 20 | from chainer_pointnet.models.pointnet.pointnet_seg import PointNetSeg 21 | from chainer_pointnet.models.pointnet2.pointnet2_seg_ssg import PointNet2SegSSG 22 | 23 | from s3dis_dataset import get_dataset 24 | 25 | from chainer_pointnet.utils.kdtree import calc_max_level, TransformKDTreeSeg 26 | 27 | 28 | def main(): 29 | parser = argparse.ArgumentParser( 30 | description='S3DIS segmentation') 31 | parser.add_argument('--method', '-m', type=str, default='point_seg') 32 | parser.add_argument('--batchsize', '-b', type=int, default=32) 33 | parser.add_argument('--dropout_ratio', type=float, default=0.0) 34 | parser.add_argument('--num_point', type=int, default=4096) 35 | parser.add_argument('--gpu', '-g', type=int, default=-1) 36 | parser.add_argument('--out', '-o', type=str, default='result') 37 | parser.add_argument('--epoch', '-e', type=int, default=250) 38 | parser.add_argument('--seed', '-s', type=int, default=777) 39 | parser.add_argument('--protocol', type=int, default=2) 40 | parser.add_argument('--model_filename', type=str, default='model.npz') 41 | parser.add_argument('--resume', type=str, default='') 42 | parser.add_argument('--trans', type=strtobool, default='false') 43 | parser.add_argument('--use_bn', type=strtobool, default='true') 44 | parser.add_argument('--normalize', type=strtobool, default='false') 45 | parser.add_argument('--residual', type=strtobool, default='false') 46 | args = parser.parse_args() 47 | 48 | seed = args.seed 49 | out_dir = args.out 50 | method = args.method 51 | num_point = args.num_point 52 | 53 | try: 54 | os.makedirs(out_dir, exist_ok=True) 55 | import chainerex.utils as cl 56 | fp = os.path.join(out_dir, 'args.json') 57 | cl.save_json(fp, vars(args)) 58 | print('save args to', fp) 59 | except ImportError: 60 | pass 61 | 62 | # S3DIS dataset has 13 labels 63 | num_class = 13 64 | in_dim = 9 65 | 66 | # Dataset preparation 67 | train, val = get_dataset(num_point=num_point) 68 | if method == 'kdnet_seg' or method == 'kdcontextnet_seg': 69 | from chainer_pointnet.utils.kdtree import TransformKDTreeSeg, \ 70 | calc_max_level 71 | max_level = calc_max_level(num_point) 72 | print('kdnet max_level {}'.format(max_level)) 73 | return_split_dims = (method == 'kdnet_seg') 74 | train = TransformDataset(train, TransformKDTreeSeg( 75 | max_level=max_level, return_split_dims=return_split_dims)) 76 | val = TransformDataset(val, TransformKDTreeSeg( 77 | max_level=max_level, return_split_dims=return_split_dims)) 78 | if method == 'kdnet_seg': 79 | # Debug print 80 | points, split_dims, t = train[0] 81 | print('converted to kdnet dataset train', points.shape, split_dims.shape, t.shape) 82 | points, split_dims, t = val[0] 83 | print('converted to kdnet dataset val', points.shape, split_dims.shape, t.shape) 84 | if method == 'kdcontextnet_seg': 85 | # Debug print 86 | points, t = train[0] 87 | print('converted to kdcontextnet dataset train', points.shape, t.shape) 88 | points, t = val[0] 89 | print('converted to kdcontextnet dataset val', points.shape, t.shape) 90 | 91 | # Network 92 | trans = args.trans 93 | use_bn = args.use_bn 94 | dropout_ratio = args.dropout_ratio 95 | normalize = args.normalize 96 | residual = args.residual 97 | converter = concat_examples 98 | if method == 'point_seg': 99 | print('Train PointNetSeg model... trans={} use_bn={} dropout={}' 100 | .format(trans, use_bn, dropout_ratio)) 101 | model = PointNetSeg( 102 | out_dim=num_class, in_dim=in_dim, middle_dim=64, 103 | dropout_ratio=dropout_ratio, 104 | trans=trans, trans_lam1=0.001, trans_lam2=0.001, use_bn=use_bn, 105 | residual=residual) 106 | elif method == 'point2_seg_ssg': 107 | print('Train PointNet2SegSSG model... use_bn={} dropout={}' 108 | .format(use_bn, dropout_ratio)) 109 | model = PointNet2SegSSG( 110 | out_dim=num_class, in_dim=in_dim, 111 | dropout_ratio=dropout_ratio, use_bn=use_bn, residual=residual) 112 | elif method == 'kdnet_seg': 113 | print('Train KDNetSeg model... use_bn={} dropout={}' 114 | .format(use_bn, dropout_ratio)) 115 | model = KDNetSeg( 116 | out_dim=num_class, in_dim=in_dim, 117 | dropout_ratio=dropout_ratio, use_bn=use_bn, max_level=max_level, 118 | residual=residual) 119 | 120 | def kdnet_converter(batch, device=None, padding=None): 121 | # concat_examples to CPU at first. 122 | result = concat_examples(batch, device=None, padding=padding) 123 | out_list = [] 124 | for elem in result: 125 | if elem.dtype != object: 126 | # Send to GPU for int/float dtype array. 127 | out_list.append(to_device(device, elem)) 128 | else: 129 | # Do NOT send to GPU for dtype=object array. 130 | out_list.append(elem) 131 | return tuple(out_list) 132 | 133 | converter = kdnet_converter 134 | elif method == 'kdcontextnet_seg': 135 | print('Train KDContextNetSeg model... use_bn={} dropout={} ' 136 | 'normalize={} residual={}' 137 | .format(use_bn, dropout_ratio, normalize, residual)) 138 | model = KDContextNetSeg( 139 | out_dim=num_class, in_dim=in_dim, 140 | dropout_ratio=dropout_ratio, use_bn=use_bn, normalize=True, 141 | residual=residual) 142 | else: 143 | raise ValueError('[ERROR] Invalid method {}'.format(method)) 144 | 145 | train_iter = iterators.SerialIterator(train, args.batchsize) 146 | val_iter = iterators.SerialIterator( 147 | val, args.batchsize, repeat=False, shuffle=False) 148 | 149 | device = args.gpu 150 | # classifier = Classifier(model, device=device) 151 | classifier = model 152 | load_model = False 153 | if load_model: 154 | serializers.load_npz( 155 | os.path.join(args.out, args.model_filename), classifier) 156 | if device >= 0: 157 | chainer.cuda.get_device_from_id(device).use() 158 | classifier.to_gpu() # Copy the model to the GPU 159 | 160 | optimizer = optimizers.Adam() 161 | optimizer.setup(classifier) 162 | 163 | updater = training.StandardUpdater( 164 | train_iter, optimizer, device=args.gpu, converter=converter) 165 | 166 | trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) 167 | 168 | from chainerex.training.extensions import schedule_optimizer_value 169 | from chainer.training.extensions import observe_value 170 | # trainer.extend(observe_lr) 171 | observation_key = 'lr' 172 | trainer.extend(observe_value( 173 | observation_key, 174 | lambda trainer: trainer.updater.get_optimizer('main').alpha)) 175 | trainer.extend(schedule_optimizer_value( 176 | [10, 20, 100, 150, 200, 230], 177 | [0.003, 0.001, 0.0003, 0.0001, 0.00003, 0.00001])) 178 | 179 | trainer.extend(E.Evaluator( 180 | val_iter, classifier, device=args.gpu, converter=converter)) 181 | trainer.extend(E.snapshot(), trigger=(args.epoch, 'epoch')) 182 | trainer.extend(E.LogReport()) 183 | trainer.extend(E.PrintReport( 184 | ['epoch', 'main/loss', 'main/cls_loss', 'main/trans_loss1', 185 | 'main/trans_loss2', 'main/accuracy', 'validation/main/loss', 186 | # 'validation/main/cls_loss', 187 | # 'validation/main/trans_loss1', 'validation/main/trans_loss2', 188 | 'validation/main/accuracy', 'lr', 'elapsed_time'])) 189 | trainer.extend(E.ProgressBar(update_interval=10)) 190 | 191 | if args.resume: 192 | serializers.load_npz(args.resume, trainer) 193 | trainer.run() 194 | 195 | # --- save classifier --- 196 | # protocol = args.protocol 197 | # classifier.save_pickle( 198 | # os.path.join(args.out, args.model_filename), protocol=protocol) 199 | serializers.save_npz( 200 | os.path.join(args.out, args.model_filename), classifier) 201 | 202 | 203 | if __name__ == '__main__': 204 | main() 205 | -------------------------------------------------------------------------------- /experiments/s3dis/third_party/meta/anno_paths.txt: -------------------------------------------------------------------------------- 1 | Area_1/conferenceRoom_1/Annotations 2 | Area_1/conferenceRoom_2/Annotations 3 | Area_1/copyRoom_1/Annotations 4 | Area_1/hallway_1/Annotations 5 | Area_1/hallway_2/Annotations 6 | Area_1/hallway_3/Annotations 7 | Area_1/hallway_4/Annotations 8 | Area_1/hallway_5/Annotations 9 | Area_1/hallway_6/Annotations 10 | Area_1/hallway_7/Annotations 11 | Area_1/hallway_8/Annotations 12 | Area_1/office_10/Annotations 13 | Area_1/office_11/Annotations 14 | Area_1/office_12/Annotations 15 | Area_1/office_13/Annotations 16 | Area_1/office_14/Annotations 17 | Area_1/office_15/Annotations 18 | Area_1/office_16/Annotations 19 | Area_1/office_17/Annotations 20 | Area_1/office_18/Annotations 21 | Area_1/office_19/Annotations 22 | Area_1/office_1/Annotations 23 | Area_1/office_20/Annotations 24 | Area_1/office_21/Annotations 25 | Area_1/office_22/Annotations 26 | Area_1/office_23/Annotations 27 | Area_1/office_24/Annotations 28 | Area_1/office_25/Annotations 29 | Area_1/office_26/Annotations 30 | Area_1/office_27/Annotations 31 | Area_1/office_28/Annotations 32 | Area_1/office_29/Annotations 33 | Area_1/office_2/Annotations 34 | Area_1/office_30/Annotations 35 | Area_1/office_31/Annotations 36 | Area_1/office_3/Annotations 37 | Area_1/office_4/Annotations 38 | Area_1/office_5/Annotations 39 | Area_1/office_6/Annotations 40 | Area_1/office_7/Annotations 41 | Area_1/office_8/Annotations 42 | Area_1/office_9/Annotations 43 | Area_1/pantry_1/Annotations 44 | Area_1/WC_1/Annotations 45 | Area_2/auditorium_1/Annotations 46 | Area_2/auditorium_2/Annotations 47 | Area_2/conferenceRoom_1/Annotations 48 | Area_2/hallway_10/Annotations 49 | Area_2/hallway_11/Annotations 50 | Area_2/hallway_12/Annotations 51 | Area_2/hallway_1/Annotations 52 | Area_2/hallway_2/Annotations 53 | Area_2/hallway_3/Annotations 54 | Area_2/hallway_4/Annotations 55 | Area_2/hallway_5/Annotations 56 | Area_2/hallway_6/Annotations 57 | Area_2/hallway_7/Annotations 58 | Area_2/hallway_8/Annotations 59 | Area_2/hallway_9/Annotations 60 | Area_2/office_10/Annotations 61 | Area_2/office_11/Annotations 62 | Area_2/office_12/Annotations 63 | Area_2/office_13/Annotations 64 | Area_2/office_14/Annotations 65 | Area_2/office_1/Annotations 66 | Area_2/office_2/Annotations 67 | Area_2/office_3/Annotations 68 | Area_2/office_4/Annotations 69 | Area_2/office_5/Annotations 70 | Area_2/office_6/Annotations 71 | Area_2/office_7/Annotations 72 | Area_2/office_8/Annotations 73 | Area_2/office_9/Annotations 74 | Area_2/storage_1/Annotations 75 | Area_2/storage_2/Annotations 76 | Area_2/storage_3/Annotations 77 | Area_2/storage_4/Annotations 78 | Area_2/storage_5/Annotations 79 | Area_2/storage_6/Annotations 80 | Area_2/storage_7/Annotations 81 | Area_2/storage_8/Annotations 82 | Area_2/storage_9/Annotations 83 | Area_2/WC_1/Annotations 84 | Area_2/WC_2/Annotations 85 | Area_3/conferenceRoom_1/Annotations 86 | Area_3/hallway_1/Annotations 87 | Area_3/hallway_2/Annotations 88 | Area_3/hallway_3/Annotations 89 | Area_3/hallway_4/Annotations 90 | Area_3/hallway_5/Annotations 91 | Area_3/hallway_6/Annotations 92 | Area_3/lounge_1/Annotations 93 | Area_3/lounge_2/Annotations 94 | Area_3/office_10/Annotations 95 | Area_3/office_1/Annotations 96 | Area_3/office_2/Annotations 97 | Area_3/office_3/Annotations 98 | Area_3/office_4/Annotations 99 | Area_3/office_5/Annotations 100 | Area_3/office_6/Annotations 101 | Area_3/office_7/Annotations 102 | Area_3/office_8/Annotations 103 | Area_3/office_9/Annotations 104 | Area_3/storage_1/Annotations 105 | Area_3/storage_2/Annotations 106 | Area_3/WC_1/Annotations 107 | Area_3/WC_2/Annotations 108 | Area_4/conferenceRoom_1/Annotations 109 | Area_4/conferenceRoom_2/Annotations 110 | Area_4/conferenceRoom_3/Annotations 111 | Area_4/hallway_10/Annotations 112 | Area_4/hallway_11/Annotations 113 | Area_4/hallway_12/Annotations 114 | Area_4/hallway_13/Annotations 115 | Area_4/hallway_14/Annotations 116 | Area_4/hallway_1/Annotations 117 | Area_4/hallway_2/Annotations 118 | Area_4/hallway_3/Annotations 119 | Area_4/hallway_4/Annotations 120 | Area_4/hallway_5/Annotations 121 | Area_4/hallway_6/Annotations 122 | Area_4/hallway_7/Annotations 123 | Area_4/hallway_8/Annotations 124 | Area_4/hallway_9/Annotations 125 | Area_4/lobby_1/Annotations 126 | Area_4/lobby_2/Annotations 127 | Area_4/office_10/Annotations 128 | Area_4/office_11/Annotations 129 | Area_4/office_12/Annotations 130 | Area_4/office_13/Annotations 131 | Area_4/office_14/Annotations 132 | Area_4/office_15/Annotations 133 | Area_4/office_16/Annotations 134 | Area_4/office_17/Annotations 135 | Area_4/office_18/Annotations 136 | Area_4/office_19/Annotations 137 | Area_4/office_1/Annotations 138 | Area_4/office_20/Annotations 139 | Area_4/office_21/Annotations 140 | Area_4/office_22/Annotations 141 | Area_4/office_2/Annotations 142 | Area_4/office_3/Annotations 143 | Area_4/office_4/Annotations 144 | Area_4/office_5/Annotations 145 | Area_4/office_6/Annotations 146 | Area_4/office_7/Annotations 147 | Area_4/office_8/Annotations 148 | Area_4/office_9/Annotations 149 | Area_4/storage_1/Annotations 150 | Area_4/storage_2/Annotations 151 | Area_4/storage_3/Annotations 152 | Area_4/storage_4/Annotations 153 | Area_4/WC_1/Annotations 154 | Area_4/WC_2/Annotations 155 | Area_4/WC_3/Annotations 156 | Area_4/WC_4/Annotations 157 | Area_5/conferenceRoom_1/Annotations 158 | Area_5/conferenceRoom_2/Annotations 159 | Area_5/conferenceRoom_3/Annotations 160 | Area_5/hallway_10/Annotations 161 | Area_5/hallway_11/Annotations 162 | Area_5/hallway_12/Annotations 163 | Area_5/hallway_13/Annotations 164 | Area_5/hallway_14/Annotations 165 | Area_5/hallway_15/Annotations 166 | Area_5/hallway_1/Annotations 167 | Area_5/hallway_2/Annotations 168 | Area_5/hallway_3/Annotations 169 | Area_5/hallway_4/Annotations 170 | Area_5/hallway_5/Annotations 171 | Area_5/hallway_6/Annotations 172 | Area_5/hallway_7/Annotations 173 | Area_5/hallway_8/Annotations 174 | Area_5/hallway_9/Annotations 175 | Area_5/lobby_1/Annotations 176 | Area_5/office_10/Annotations 177 | Area_5/office_11/Annotations 178 | Area_5/office_12/Annotations 179 | Area_5/office_13/Annotations 180 | Area_5/office_14/Annotations 181 | Area_5/office_15/Annotations 182 | Area_5/office_16/Annotations 183 | Area_5/office_17/Annotations 184 | Area_5/office_18/Annotations 185 | Area_5/office_19/Annotations 186 | Area_5/office_1/Annotations 187 | Area_5/office_20/Annotations 188 | Area_5/office_21/Annotations 189 | Area_5/office_22/Annotations 190 | Area_5/office_23/Annotations 191 | Area_5/office_24/Annotations 192 | Area_5/office_25/Annotations 193 | Area_5/office_26/Annotations 194 | Area_5/office_27/Annotations 195 | Area_5/office_28/Annotations 196 | Area_5/office_29/Annotations 197 | Area_5/office_2/Annotations 198 | Area_5/office_30/Annotations 199 | Area_5/office_31/Annotations 200 | Area_5/office_32/Annotations 201 | Area_5/office_33/Annotations 202 | Area_5/office_34/Annotations 203 | Area_5/office_35/Annotations 204 | Area_5/office_36/Annotations 205 | Area_5/office_37/Annotations 206 | Area_5/office_38/Annotations 207 | Area_5/office_39/Annotations 208 | Area_5/office_3/Annotations 209 | Area_5/office_40/Annotations 210 | Area_5/office_41/Annotations 211 | Area_5/office_42/Annotations 212 | Area_5/office_4/Annotations 213 | Area_5/office_5/Annotations 214 | Area_5/office_6/Annotations 215 | Area_5/office_7/Annotations 216 | Area_5/office_8/Annotations 217 | Area_5/office_9/Annotations 218 | Area_5/pantry_1/Annotations 219 | Area_5/storage_1/Annotations 220 | Area_5/storage_2/Annotations 221 | Area_5/storage_3/Annotations 222 | Area_5/storage_4/Annotations 223 | Area_5/WC_1/Annotations 224 | Area_5/WC_2/Annotations 225 | Area_6/conferenceRoom_1/Annotations 226 | Area_6/copyRoom_1/Annotations 227 | Area_6/hallway_1/Annotations 228 | Area_6/hallway_2/Annotations 229 | Area_6/hallway_3/Annotations 230 | Area_6/hallway_4/Annotations 231 | Area_6/hallway_5/Annotations 232 | Area_6/hallway_6/Annotations 233 | Area_6/lounge_1/Annotations 234 | Area_6/office_10/Annotations 235 | Area_6/office_11/Annotations 236 | Area_6/office_12/Annotations 237 | Area_6/office_13/Annotations 238 | Area_6/office_14/Annotations 239 | Area_6/office_15/Annotations 240 | Area_6/office_16/Annotations 241 | Area_6/office_17/Annotations 242 | Area_6/office_18/Annotations 243 | Area_6/office_19/Annotations 244 | Area_6/office_1/Annotations 245 | Area_6/office_20/Annotations 246 | Area_6/office_21/Annotations 247 | Area_6/office_22/Annotations 248 | Area_6/office_23/Annotations 249 | Area_6/office_24/Annotations 250 | Area_6/office_25/Annotations 251 | Area_6/office_26/Annotations 252 | Area_6/office_27/Annotations 253 | Area_6/office_28/Annotations 254 | Area_6/office_29/Annotations 255 | Area_6/office_2/Annotations 256 | Area_6/office_30/Annotations 257 | Area_6/office_31/Annotations 258 | Area_6/office_32/Annotations 259 | Area_6/office_33/Annotations 260 | Area_6/office_34/Annotations 261 | Area_6/office_35/Annotations 262 | Area_6/office_36/Annotations 263 | Area_6/office_37/Annotations 264 | Area_6/office_3/Annotations 265 | Area_6/office_4/Annotations 266 | Area_6/office_5/Annotations 267 | Area_6/office_6/Annotations 268 | Area_6/office_7/Annotations 269 | Area_6/office_8/Annotations 270 | Area_6/office_9/Annotations 271 | Area_6/openspace_1/Annotations 272 | Area_6/pantry_1/Annotations 273 | -------------------------------------------------------------------------------- /experiments/modelnet40/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | import argparse 5 | from distutils.util import strtobool 6 | import os 7 | 8 | import chainer 9 | from chainer import serializers 10 | from chainer import iterators 11 | from chainer import optimizers 12 | from chainer import training 13 | from chainer.dataset import to_device 14 | from chainer.datasets import TransformDataset, SubDataset 15 | from chainer.training import extensions as E 16 | 17 | from chainer_pointnet.models.kdcontextnet.kdcontextnet_cls import \ 18 | KDContextNetCls 19 | from chainer_pointnet.models.kdnet.kdnet_cls import KDNetCls 20 | from chainer_pointnet.models.pointnet.pointnet_cls import PointNetCls 21 | from chainer_pointnet.models.pointnet2.pointnet2_cls_msg import PointNet2ClsMSG 22 | from chainer_pointnet.models.pointnet2.pointnet2_cls_ssg import PointNet2ClsSSG 23 | 24 | from ply_dataset import get_train_dataset, get_test_dataset 25 | 26 | from chainer_pointnet.utils.kdtree import calc_max_level 27 | 28 | 29 | def main(): 30 | parser = argparse.ArgumentParser( 31 | description='ModelNet40 classification') 32 | # parser.add_argument('--conv-layers', '-c', type=int, default=4) 33 | parser.add_argument('--method', '-m', type=str, default='point_cls') 34 | parser.add_argument('--batchsize', '-b', type=int, default=32) 35 | parser.add_argument('--dropout_ratio', type=float, default=0.3) 36 | parser.add_argument('--num_point', type=int, default=1024) 37 | parser.add_argument('--gpu', '-g', type=int, default=-1) 38 | parser.add_argument('--out', '-o', type=str, default='result') 39 | parser.add_argument('--epoch', '-e', type=int, default=250) 40 | # parser.add_argument('--unit-num', '-u', type=int, default=16) 41 | parser.add_argument('--seed', '-s', type=int, default=777) 42 | parser.add_argument('--protocol', type=int, default=2) 43 | parser.add_argument('--model_filename', type=str, default='model.npz') 44 | parser.add_argument('--resume', type=str, default='') 45 | parser.add_argument('--trans', type=strtobool, default='true') 46 | parser.add_argument('--use_bn', type=strtobool, default='true') 47 | parser.add_argument('--normalize', type=strtobool, default='false') 48 | parser.add_argument('--residual', type=strtobool, default='false') 49 | args = parser.parse_args() 50 | 51 | seed = args.seed 52 | method = args.method 53 | num_point = args.num_point 54 | out_dir = args.out 55 | num_class = 40 56 | debug = False 57 | try: 58 | os.makedirs(out_dir, exist_ok=True) 59 | import chainerex.utils as cl 60 | fp = os.path.join(out_dir, 'args.json') 61 | cl.save_json(fp, vars(args)) 62 | print('save args to', fp) 63 | except ImportError: 64 | pass 65 | 66 | # Dataset preparation 67 | train = get_train_dataset(num_point=num_point) 68 | val = get_test_dataset(num_point=num_point) 69 | if method == 'kdnet_cls' or method == 'kdcontextnet_cls': 70 | from chainer_pointnet.utils.kdtree import TransformKDTreeCls 71 | max_level = calc_max_level(num_point) 72 | print('kdnet_cls max_level {}'.format(max_level)) 73 | return_split_dims = (method == 'kdnet_cls') 74 | train = TransformDataset(train, TransformKDTreeCls( 75 | max_level=max_level, return_split_dims=return_split_dims)) 76 | val = TransformDataset(val, TransformKDTreeCls( 77 | max_level=max_level, return_split_dims=return_split_dims)) 78 | if method == 'kdnet_cls': 79 | # Debug print 80 | points, split_dims, t = train[0] 81 | print('converted to kdnet dataset train', points.shape, split_dims.shape, t) 82 | points, split_dims, t = val[0] 83 | print('converted to kdnet dataset val', points.shape, split_dims.shape, t) 84 | if method == 'kdcontextnet_cls': 85 | # Debug print 86 | points, t = train[0] 87 | print('converted to kdcontextnet dataset train', points.shape, t) 88 | points, t = val[0] 89 | print('converted to kdcontextnet dataset val', points.shape, t) 90 | 91 | if debug: 92 | # use few train dataset 93 | train = SubDataset(train, 0, 50) 94 | 95 | # Network 96 | # n_unit = args.unit_num 97 | # conv_layers = args.conv_layers 98 | trans = args.trans 99 | use_bn = args.use_bn 100 | normalize = args.normalize 101 | residual = args.residual 102 | dropout_ratio = args.dropout_ratio 103 | from chainer.dataset.convert import concat_examples 104 | converter = concat_examples 105 | 106 | if method == 'point_cls': 107 | print('Train PointNetCls model... trans={} use_bn={} dropout={}' 108 | .format(trans, use_bn, dropout_ratio)) 109 | model = PointNetCls( 110 | out_dim=num_class, in_dim=3, middle_dim=64, dropout_ratio=dropout_ratio, 111 | trans=trans, trans_lam1=0.001, trans_lam2=0.001, use_bn=use_bn, 112 | residual=residual) 113 | elif method == 'point2_cls_ssg': 114 | print('Train PointNet2ClsSSG model... use_bn={} dropout={}' 115 | .format(use_bn, dropout_ratio)) 116 | model = PointNet2ClsSSG( 117 | out_dim=num_class, in_dim=3, 118 | dropout_ratio=dropout_ratio, use_bn=use_bn, residual=residual) 119 | elif method == 'point2_cls_msg': 120 | print('Train PointNet2ClsMSG model... use_bn={} dropout={}' 121 | .format(use_bn, dropout_ratio)) 122 | model = PointNet2ClsMSG( 123 | out_dim=num_class, in_dim=3, 124 | dropout_ratio=dropout_ratio, use_bn=use_bn, residual=residual) 125 | elif method == 'kdnet_cls': 126 | print('Train KDNetCls model... use_bn={} dropout={}' 127 | .format(use_bn, dropout_ratio)) 128 | model = KDNetCls( 129 | out_dim=num_class, in_dim=3, 130 | dropout_ratio=dropout_ratio, use_bn=use_bn, max_level=max_level,) 131 | 132 | def kdnet_converter(batch, device=None, padding=None): 133 | # concat_examples to CPU at first. 134 | result = concat_examples(batch, device=None, padding=padding) 135 | out_list = [] 136 | for elem in result: 137 | if elem.dtype != object: 138 | # Send to GPU for int/float dtype array. 139 | out_list.append(to_device(device, elem)) 140 | else: 141 | # Do NOT send to GPU for dtype=object array. 142 | out_list.append(elem) 143 | return tuple(out_list) 144 | 145 | converter = kdnet_converter 146 | elif method == 'kdcontextnet_cls': 147 | print('Train KDContextNetCls model... use_bn={} dropout={}' 148 | 'normalize={} residual={}' 149 | .format(use_bn, dropout_ratio, normalize, residual)) 150 | model = KDContextNetCls( 151 | out_dim=num_class, in_dim=3, 152 | dropout_ratio=dropout_ratio, use_bn=use_bn, 153 | # Below is for non-default customization 154 | levels=[3, 6, 9], 155 | feature_learning_mlp_list=[ 156 | [32, 32, 128], [64, 64, 256], [128, 128, 512]], 157 | feature_aggregation_mlp_list=[[128], [256], [512]], 158 | normalize=normalize, residual=residual 159 | ) 160 | else: 161 | raise ValueError('[ERROR] Invalid method {}'.format(method)) 162 | 163 | train_iter = iterators.SerialIterator(train, args.batchsize) 164 | val_iter = iterators.SerialIterator( 165 | val, args.batchsize, repeat=False, shuffle=False) 166 | 167 | device = args.gpu 168 | # classifier = Classifier(model, device=device) 169 | classifier = model 170 | load_model = False 171 | if load_model: 172 | serializers.load_npz( 173 | os.path.join(out_dir, args.model_filename), classifier) 174 | if device >= 0: 175 | print('using gpu {}'.format(device)) 176 | chainer.cuda.get_device_from_id(device).use() 177 | classifier.to_gpu() # Copy the model to the GPU 178 | 179 | optimizer = optimizers.Adam() 180 | optimizer.setup(classifier) 181 | 182 | updater = training.StandardUpdater( 183 | train_iter, optimizer, converter=converter, device=args.gpu) 184 | 185 | trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=out_dir) 186 | 187 | from chainerex.training.extensions import schedule_optimizer_value 188 | from chainer.training.extensions import observe_value 189 | # trainer.extend(observe_lr) 190 | observation_key = 'lr' 191 | trainer.extend(observe_value( 192 | observation_key, 193 | lambda trainer: trainer.updater.get_optimizer('main').alpha)) 194 | trainer.extend(schedule_optimizer_value( 195 | [10, 20, 100, 150, 200, 230], 196 | [0.003, 0.001, 0.0003, 0.0001, 0.00003, 0.00001])) 197 | 198 | trainer.extend(E.Evaluator( 199 | val_iter, classifier, converter=converter, device=args.gpu)) 200 | trainer.extend(E.snapshot(), trigger=(args.epoch, 'epoch')) 201 | trainer.extend(E.LogReport()) 202 | trainer.extend(E.PrintReport( 203 | ['epoch', 'main/loss', 'main/cls_loss', 'main/trans_loss1', 204 | 'main/trans_loss2', 'main/accuracy', 'validation/main/loss', 205 | # 'validation/main/cls_loss', 206 | # 'validation/main/trans_loss1', 'validation/main/trans_loss2', 207 | 'validation/main/accuracy', 'lr', 'elapsed_time'])) 208 | trainer.extend(E.ProgressBar(update_interval=10)) 209 | 210 | if args.resume: 211 | serializers.load_npz(args.resume, trainer) 212 | trainer.run() 213 | 214 | # --- save classifier --- 215 | # protocol = args.protocol 216 | # classifier.save_pickle( 217 | # os.path.join(out_dir, args.model_filename), protocol=protocol) 218 | serializers.save_npz( 219 | os.path.join(out_dir, args.model_filename), classifier) 220 | 221 | 222 | if __name__ == '__main__': 223 | main() 224 | -------------------------------------------------------------------------------- /chainer_pointnet/utils/kdtree.py: -------------------------------------------------------------------------------- 1 | """Used in Kd-Network 2 | 3 | Implementation referenced from 4 | https://github.com/fxia22/kdnet.pytorch 5 | kdtree code by @wassname 6 | https://github.com/fxia22/kdnet.pytorch/blob/master/kdtree.py 7 | """ 8 | import scipy.spatial 9 | import numpy 10 | 11 | 12 | def _parse_split_dims(tree, split_dims, level=0, parent=None, max_level=7, 13 | split_positions=None): 14 | """Traverse KDTree in DFS order, to extract `split_dims`&`split_positions` 15 | 16 | Args: 17 | tree (scipy.spatial.ckdtree.cKDTreeNode): kdtree node 18 | split_dims (list): list of list. `split_dims[i]` will store 19 | `i`-th level `split_dim`. 20 | level (int): current parsing level 21 | parent (scipy.spatial.ckdtree.cKDTreeNode): 22 | max_level (int): max level of KDTree 23 | split_positions (list): list of list. `split_positions[i]` will store 24 | `i`-th level `split`. If None, updating this value is skipped. 25 | 26 | """ 27 | if level == max_level: 28 | # this is leaf tree, and split_dim=-1. 29 | return 30 | 31 | if tree.lesser is not None: 32 | _parse_split_dims(tree.lesser, split_dims, level=level+1, parent=tree, 33 | max_level=max_level, split_positions=split_positions) 34 | else: 35 | # This will happen when the point is overlapped and `split_dim==-1`, 36 | # in this case just use parent `tree`. 37 | # print('tree.lesser is None, level {}'.format(level)) 38 | _parse_split_dims(tree, split_dims, level=level+1, parent=tree, 39 | max_level=max_level, split_positions=split_positions) 40 | if tree.greater is not None: 41 | _parse_split_dims(tree.greater, split_dims, level=level+1, parent=tree, 42 | max_level=max_level, split_positions=split_positions) 43 | else: 44 | # This will happen when the point is overlapped and `split_dim==-1`, 45 | # in this case just use parent `tree`. 46 | # print('[WARNING] tree.greater is None, level {}'.format(level)) 47 | _parse_split_dims(tree, split_dims, level=level+1, parent=tree, 48 | max_level=max_level, split_positions=split_positions) 49 | 50 | if level < max_level: 51 | split_dim = tree.split_dim 52 | if split_dim == -1: 53 | # since we repeated premature leafs, we get invalid splits 54 | # in this case just use the parents. 55 | # This case happen when the points are overlapped. 56 | # print('split_dim is -1 at level', level) 57 | split_dim = parent.split_dim if (parent.split_dim > -1) else 0 58 | split_dims[level].append(split_dim) 59 | if split_positions is not None: 60 | split = tree.split 61 | if split_dim == -1: 62 | split = parent.split if (parent.split_dim > -1) else 0 63 | split_positions[level].append(split) 64 | 65 | 66 | def calc_max_level(num_point): 67 | """Calculate estimated max_level based on `num_point` 68 | 69 | Args: 70 | num_point (int): number of points 71 | 72 | Returns (int): max_level 73 | 74 | """ 75 | return int(numpy.ceil(numpy.log2(num_point))) 76 | 77 | 78 | def construct_kdtree_data(points, max_level=-1, calc_split_positions=False): 79 | """Construct preprocessing data for KD-network 80 | 81 | Args: 82 | points (numpy.ndarray): 83 | 2-dim array (num_point, coord_dim) 84 | max_level (int): depth of the KDTree. The target size of output points 85 | is 2**max_level. When -1 is set, minimum target size is 86 | automatically inferred. 87 | calc_split_positions (bool): calculate `split_positions` or not. 88 | 89 | Returns: 90 | new_points (numpy.ndarray): 91 | 2-dim array (num_point', coord_dim), where num_point'=2**max_level 92 | Its order is updated according to `KDTree.indices`. 93 | split_dims (numpy.ndarray): list of int array. `split_dims[i]` will 94 | store `i`-th level `split_dim`. 95 | inds (numpy.ndarray or slice): 1d array or slice to represent how the 96 | `new_points` are constructed from input `points`. 97 | kdtree (scipy.spatial.ckdtree.cKDTree): KDTree instance 98 | split_positions (numpy.ndarray): list of float array. 99 | `split_positions[i]` will store `i`-th level `split`. 100 | If `calc_split_positions=False`, `None` is returned. 101 | This is mainly for debug purpose. 102 | """ 103 | assert points.ndim == 2, 'points.ndim must be 2, got points with shape {}'\ 104 | .format(points.shape) 105 | num_point = points.shape[0] 106 | if max_level <= -1: 107 | max_level = calc_max_level(num_point) 108 | # print('max_level', max_level, 'num_point', num_point) 109 | target_size = 2 ** max_level 110 | if target_size > num_point: 111 | # augment point to make power of 2 112 | remainder = target_size - num_point 113 | print('[DEBUG] add points, target_size={}, remainder={}' 114 | .format(target_size, remainder)) 115 | # select remainder randomly 116 | inds = numpy.random.choice(range(num_point), remainder) 117 | inds = numpy.concatenate([numpy.arange(len(points)), inds], axis=0) 118 | points = points[inds] 119 | elif target_size < num_point: 120 | # Reduce number of points 121 | inds = numpy.random.permutation(num_point)[:target_size] 122 | points = points[inds] 123 | else: 124 | inds = numpy.arange(num_point) 125 | assert points.shape[0] == target_size 126 | kdtree = scipy.spatial.cKDTree(points, leafsize=1, balanced_tree=True) 127 | tree = kdtree.tree 128 | # split_dims[i] will store `split_dim` for the level `i`. 129 | split_dims = [[] for _ in range(max_level)] 130 | if calc_split_positions: 131 | split_positions = [[] for _ in range(max_level)] 132 | else: 133 | split_positions = None 134 | _parse_split_dims(tree, split_dims, max_level=max_level, 135 | split_positions=split_positions) 136 | split_dims = numpy.array([numpy.array(elem) for elem in split_dims]) 137 | 138 | if split_positions is not None: 139 | # convert list to numpy array with object type. 140 | split_positions = numpy.array( 141 | [numpy.array(elem) for elem in split_positions]) 142 | return points[tree.indices], split_dims, inds[tree.indices], kdtree, split_positions 143 | 144 | 145 | class TransformKDTreeCls(object): 146 | 147 | def __init__(self, max_level=10, return_split_dims=True): 148 | super(TransformKDTreeCls, self).__init__() 149 | self.max_level = max_level 150 | self.return_split_dims = return_split_dims 151 | 152 | def __call__(self, in_data): 153 | original_points, label = in_data 154 | # print('original_points', original_points.shape, 'label', label) 155 | pts = numpy.transpose(original_points[:, :, 0], (1, 0)) 156 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 157 | pts, max_level=self.max_level, 158 | calc_split_positions=False) 159 | points = numpy.transpose(points, (1, 0))[:, :, None] 160 | if self.return_split_dims: 161 | # Used in `kdnet_cls`, which needs `split_dims` information. 162 | return points, split_dims, label 163 | else: 164 | # Used in `kdcontextnet_cls`, which only needs permutated points, 165 | # but not `split_dims`. 166 | return points, label 167 | 168 | 169 | class TransformKDTreeSeg(object): 170 | 171 | def __init__(self, max_level=10, return_split_dims=True): 172 | super(TransformKDTreeSeg, self).__init__() 173 | self.max_level = max_level 174 | self.return_split_dims = return_split_dims 175 | 176 | def __call__(self, in_data): 177 | # shape points (cdim, num_point, 1), label (num_point,) 178 | original_points, label_points = in_data 179 | # print('original_points', original_points.shape, 'label', label) 180 | pts = numpy.transpose(original_points[:, :, 0], (1, 0)) 181 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 182 | pts, max_level=self.max_level, 183 | calc_split_positions=False) 184 | points = numpy.transpose(points, (1, 0))[:, :, None] 185 | label_points = label_points[inds] 186 | if self.return_split_dims: 187 | # Used in `kdnet_cls`, which needs `split_dims` information. 188 | return points, split_dims, label_points 189 | else: 190 | # Used in `kdcontextnet_cls`, which only needs permutated points, 191 | # but not `split_dims`. 192 | return points, label_points 193 | 194 | 195 | if __name__ == '__main__': 196 | batchsize = 1 197 | num_point = 135 # try 100, 128, 135 198 | max_level = 7 # 2^7 -> 128. Final num_point will be 128 199 | dim = 3 200 | point_set = numpy.random.rand(num_point, dim) 201 | print('point_set', point_set.shape) 202 | points, split_dims, inds, kdtree, split_positions = construct_kdtree_data( 203 | point_set, max_level=max_level, calc_split_positions=True) 204 | print('kdtree', kdtree.indices) 205 | print('inds', inds) 206 | # print('kdtree.tree', kdtree.tree.indices) # same with kdtree.indices 207 | print('points', points.shape) # 128 point here! 208 | print(points[0:2]) 209 | print(points[-2:]) 210 | print('split_dims', len(split_dims), 'type', split_dims.dtype) 211 | print('split_positions', len(split_positions)) 212 | for i in range(len(split_dims)): 213 | print('i {}, {} type {}'.format(i, split_dims[i], split_dims[i].dtype)) 214 | print('i {}, {} type {}'.format(i, split_positions[i], 215 | split_positions.dtype)) 216 | 217 | # test TransformKDTreeSeg 218 | num_point = 50 219 | perm = numpy.random.permutation(num_point) 220 | pts = numpy.arange(num_point).astype(numpy.float32)[perm] 221 | label = numpy.arange(num_point).astype(numpy.int32)[perm] 222 | print('pts', pts.shape, pts[:5]) 223 | print('label', label.shape, label[:5]) 224 | pts = numpy.broadcast_to(pts[None, :], (3, num_point))[:, :, None] 225 | t = TransformKDTreeSeg(max_level=calc_max_level(num_point)) 226 | out_pts, split_dims, out_labels = t((pts, label)) 227 | print('out_pts', out_pts.shape, out_pts[0, :10, 0]) 228 | print('out_labels', out_labels.shape, out_labels[:10]) 229 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # chainer-pointnet 2 | 3 | Various point cloud based deep neural network implementation by 4 | [Chainer](https://github.com/chainer/chainer) [1]. 5 | 6 | It includes PointNet, PointNet++, Kd-Network and Kd context net (3DContextNet). 7 | 8 | ## Installation 9 | 10 | Please install [Chainer](https://github.com/chainer/chainer) 11 | (and [cupy](https://github.com/cupy/cupy) if you want to use GPU) beforehand, 12 | ```bash 13 | # Install chainer 14 | pip install chainer 15 | 16 | # Install cupy if you use GPU (please change based on your CUDA version) 17 | # For example, if you use CUDA 8.0, 18 | pip install cupy-cuda80 19 | # similarly cupy-cuda90 or cupy-cuda91 is avaialble. 20 | ``` 21 | 22 | and then type following to install master branch. 23 | 24 | ```bash 25 | git clone https://github.com/corochann/chainer-pointnet.git 26 | pip install -e chainer-pointnet 27 | ``` 28 | 29 | Also, some extension library is used in some of the code, 30 | ```bash 31 | # Chainer Chemistry 32 | git clone https://github.com/pfnet-research/chainer-chemistry.git 33 | pip install -e chainer-chemistry 34 | # ChainerEX 35 | git clone https://github.com/corochann/chainerex.git 36 | pip install -e chainerex 37 | ``` 38 | 39 | 40 | ## Model structure 41 | 42 | ### PointNet [2] 43 | 44 | Implementations are in `models/pointnet`. 45 | 46 | Both classification and segmentation network are implemented. 47 | 48 | `models/pointnet_cls` can be used for classification task. 49 | `trans` option represents to use `TransformNet` or not. 50 | `trans=False` corresponds `PointNetVanilla` (basic), 51 | and `trans=True` corresponds `PointNet` in the paper, respectively. 52 | 53 | In my experiment `PointNetVanilla` performs already very well, 54 | the gain in `PoinetNet` is few (maybe only 1-2% gain) while computation becomes 55 | much huge (around 3 times slower). 56 | 57 | Original implementation (in tensorflow) can be found on github under MIT license. 58 | 59 | - [charlesq34/pointnet](https://github.com/charlesq34/pointnet) 60 | 61 | #### tips 62 | 63 | I found the batch normalizations in the last linear layers are quite important. 64 | The accuracy dramatically changes (10% or more) with BatchNormalization at FC 65 | layers. 66 | 67 | ### PointNet++ [3] 68 | 69 | Implementations are in `models/pointnet2`. 70 | 71 | Both classification and segmentation network are implemented. 72 | 73 | Just note that thanks to [cupy](https://github.com/cupy/cupy), 74 | there is no C language implementation for farthest point sampling, grouping or 75 | feature propagation on GPU. 76 | You don't need to build any C binary to use this function. 77 | Please refer `utils/sampling.py` and `utils/grouping.py` for sampling & grouping. 78 | 79 | Original implementation (in tensorflow) can be found on github under MIT license. 80 | 81 | - [charlesq34/pointnet2](https://github.com/charlesq34/pointnet2) 82 | 83 | 84 | ### Kd-Network [4] 85 | 86 | Implementations are in `models/kdnet` 87 | 88 | Both classification and segmentation network are implemented. 89 | 90 | [Original implementation](https://github.com/Regenerator/kdnets) 91 | constructs KDTree by their own implementation, 92 | which supports random splitting. 93 | 94 | However in my implementation, `scipy.spatial.cKDTree` is used for 95 | easy and faster implementation, by following [fxia22/kdnet.pytorch](https://github.com/fxia22/kdnet.pytorch). 96 | So implementation in this repo does not support random tree splitting. 97 | 98 | Original implementation (in tensorflow) can be found on github under MIT license. 99 | 100 | - [Regenerator/kdnets](https://github.com/Regenerator/kdnets) 101 | 102 | Also, pytorch implementations can be found on github 103 | 104 | - [fxia22/kdnet.pytorch](https://github.com/fxia22/kdnet.pytorch) 105 | 106 | ### Kd Context Network (3DContextNet) [7] 107 | 108 | Originally called 3DContextNet, but I named `KDContextNet` in my python program. 109 | 110 | Implementations are in `models/kdcontextnet` 111 | 112 | Both classification and segmentation network are implemented. 113 | 114 | I could not find the implementation so far, and this is "inferred" 115 | implementation. 116 | Especially for segmentation part, how to "upsample" is not written in detail. 117 | So this implementation might be different from author's implementation. 118 | 119 | ## Experiments 120 | 121 | Experiments in each dataset is located under `expriments` folder. 122 | Each folder is independent, so you can refer independently. 123 | 124 | ### ModelNet40 [5] 125 | 126 | This is point cloud classification task of 40 category. 127 | Download script & code is from [charlesq34/pointnet](https://github.com/charlesq34/pointnet) 128 | 129 | - http://modelnet.cs.princeton.edu/ 130 | 131 | The dataset is automatically downloaded and preprocessed. 132 | Dataset shape is `train: (9840, 3, num_point=1024, 1), test (2468, 3, num_point=1024, 1)`. 133 | `ch=3`, meaning it only contains `(x, y, z)` coordinate information. 134 | 135 | You can simply execute train code to train `PointNet` or `PointNetVanilla`. 136 | 137 | ```angular2html 138 | # use gpu with id 0, train PointNetVanilla 139 | $ python train.py -g 0 --trans=false --method=point_cls --out=results/point_vanilla 140 | 141 | # use gpu with id 0, train PointNet 142 | $ python train.py -g 0 --method=point_cls --out=results/point 143 | 144 | # use gpu with id 0, train PointNet++ 145 | $ python train.py -g 0 --method=point2_cls_ssg --out=results/point2_ssg 146 | $ python train.py -g 0 --method=point2_cls_msg --out=results/point2_msg 147 | 148 | # use gpu with id 0, train KDNet 149 | $ python train.py -g 0 --method=kdnet_cls --dropout_ratio=0.3 --use_bn=1 --out=results/kdnet 150 | 151 | # use gpu with id 0, train KDContextNet 152 | $ python train.py -g 0 --method=kdcontextnet_cls --dropout_ratio=0.3 --use_bn=1 --out=results/kdcontextnet 153 | python train.py -g 0 --method=kdcontextnet_cls --dropout_ratio=0.3 --use_bn=1 --out=results/kdcontextnet_level369 154 | ``` 155 | 156 | ```text 157 | python train.py --use_bn=1 --dropout_ratio=0.3 158 | # PointNetVanilla 159 | epoch main/loss main/cls_loss main/trans_loss1 main/trans_loss2 main/accuracy validation/main/loss validation/main/accuracy lr elapsed_time 160 | 250 0.111644 0.111644 0.958367 0.606223 0.872596 1e-05 3560.81 161 | # PointNet 162 | 250 0.119699 0.117399 0.00227531 2.40329e-05 0.95684 0.587751 0.871795 1e-05 10358 163 | # PointNet2 SSG 164 | 250 0.0226786 0.989821 0.631495 0.898638 1e-05 54240.2 165 | # PointNet2 MSG 166 | 250 0.0217696 0.991653 0.610621 0.892628 1e-05 160451 167 | 168 | # KDNet with bn & dropout_ratio=0.3 169 | 250 0.10106 0.962235 1.01367 0.820913 1e-05 20324 170 | # KDContextNet with bn & dropout_ratio=0.3 171 | 250 0.126861 0.952769 0.835642 0.825321 1e-05 31900.6 172 | # KDContextNet with bn & dropout_ratio=0.3 & normalize=1 & residual=1 173 | 250 0.0804102 0.9716 0.824233 0.838542 1e-05 28028.5 174 | ``` 175 | 176 | KDNet seems "overfit" to the train data, meaning that its representation power is strong but it fails to generalize to test data. 177 | Maybe this is due to its property of non-rotational invariance. 178 | 179 | ### S3DIS [6] 180 | 181 | Stanford Large-Scale 3D Indoor Spaces Dataset (S3DIS) for point cloud segmentation task. 182 | Download the dataset from, 183 | 184 | - [S3DIS Dataset](http://buildingparser.stanford.edu/dataset.html) 185 | 186 | Prerocessing code adopted from [charlesq34/pointnet](https://github.com/charlesq34/pointnet) 187 | under `third_party` directory. 188 | 189 | Dataset shape: 190 | ```text 191 | train shape (20291, 9, 4096, 1) (20291, 4096) 192 | test shape (3294, 9, 4096, 1) (3294, 4096) 193 | ``` 194 | `ch=9`, it contains `0-2 ch: XYZ, 3-5ch: RGB, 6-8 normalized XYZ` coordinate 195 | information respectively. 196 | 197 | 198 | Steps: 199 | 200 | 1. Go to download link: download S3DIS dataset from 201 | [S3DIS Dataset](http://buildingparser.stanford.edu/dataset.html). 202 | You need to send application form. 203 | 204 | 2. Download `Stanford3dDataset_v1.2_Aligned_Version.zip` file (4GB), 205 | place it under `s3dis/data` directory. 206 | 207 | 2'. Fix mis label manually. 208 | 209 | `Stanford3dDataset_v1.2_Aligned_Version/Area_5/hallway_6/Annotations/ceiling_1.txt` 210 | has wrong charcter at line 180389. Please fix it manually. 211 | 212 | 3. Preprocessing 213 | 214 | `collect_indoor3d_data.py` is for data re-organization and 215 | `gen_indoor3d_h5.py` is to generate HDF5 files. (cite from [charlesq34/pointnet](https://github.com/charlesq34/pointnet/tree/master/sem_seg#dataset)) 216 | 217 | ```angular2html 218 | $ cd third_party 219 | $ python collect_indoor3d_data.py 220 | $ python gen_indoor3d_h5.py 221 | ``` 222 | 223 | 4. Training 224 | 225 | ```bash 226 | # use gpu with id 0, train PointNetVanilla 227 | $ python train.py -g 0 --method=point_seg --trans=false --out=results/pointnet_vanilla 228 | 229 | # use gpu with id 0, train PointNet 230 | $ python train.py -g 0 --method=point_seg --out=results/pointnet 231 | 232 | # use gpu with id 0, train PointNet++ 233 | $ python train.py -g 0 --method=point2_seg_ssg --out=results/pointnet2 234 | 235 | # use gpu with id 0, train KDNet 236 | $ python train.py -g 0 --method=kdnet_seg --dropout_ratio=0 --use_bn=1 237 | $ python train.py -g 0 --method=kdnet_seg --dropout_ratio=0.3 --use_bn=1 --out=results/kdnet 238 | 239 | # use gpu with id 0, train KDContextNet 240 | $ python train.py -g 0 --method=kdcontextnet_seg --dropout_ratio=0.3 --use_bn=1 --out=results/kdcontextnet 241 | ``` 242 | 243 | ```text 244 | python train.py --use_bn=1 --dropout_ratio=0.3 245 | # PointNetVanilla 246 | epoch main/loss main/cls_loss main/trans_loss1 main/trans_loss2 main/accuracy validation/main/loss validation/main/accuracy lr elapsed_time 247 | 250 0.0305516 0.0305516 0.988418 0.690604 0.891049 1e-05 101405 248 | ``` 249 | 250 | ### ScanNet 251 | 252 | Point cloud semantic segmentation task of indoor scenes. 253 | 254 | - [ScanNet: Richly-annotated 3D Reconstructions of Indoor Scenes (CVPR 2017 Spotlight)](https://www.youtube.com/watch?v=Olx4OnoZWQQ) 255 | 256 | ## LICENSE 257 | MIT License. 258 | 259 | No warranty or support for this implementation. 260 | Each model performance is not guaranteed, and may not achieve the score reported in each paper. Use it at your own risk. 261 | 262 | Please see the [LICENSE](https://github.com/corochann/chainer-pointnet/blob/master/LICENSE) file for details. 263 | 264 | I appreciate the authors who open sourced their code for the reference under permissive license. 265 | 266 | ## Reference 267 | 268 | [1] Seiya Tokui, Kenta Oono, Shohei Hido, and Justin Clayton. 269 | Chainer: a next-generation open source framework for deep learning. 270 | In *Proceedings of Workshop on Machine Learning Systems (LearningSys) in Advances in Neural Information Processing System (NIPS) 28*, 2015. 271 | 272 | - [paper](http://learningsys.org/papers/LearningSys_2015_paper_33.pdf) 273 | - [official page](https://chainer.org/) 274 | - [code: chainer/chainer](https://github.com/chainer/chainer) 275 | 276 | [2] Qi, Charles R and Su, Hao and Mo, Kaichun and Guibas, Leonidas J. 277 | PointNet: Deep Learning on Point Sets for 3D Classification and Segmentation. 278 | *arXiv preprint arXiv:1612.00593*, 2016. 279 | 280 | - [paper on arXiv](https://arxiv.org/abs/1612.00593) 281 | - [project page](http://stanford.edu/~rqi/pointnet/) 282 | - [code: charlesq34/pointnet](https://github.com/charlesq34/pointnet) 283 | - CVPR oral 284 | 285 | [3] Qi, Charles R and Yi, Li and Su, Hao and Guibas, Leonidas J. 286 | PointNet++: Deep Hierarchical Feature Learning on Point Sets in a Metric Space. 287 | *arXiv preprint arXiv:1706.02413* 2017. 288 | 289 | - [paper on arXiv](https://arxiv.org/abs/1706.02413) 290 | - [project page](http://stanford.edu/~rqi/pointnet2/) 291 | - [code: charlesq34/pointnet2](https://github.com/charlesq34/pointnet2) 292 | - NIPS 2017 293 | 294 | [4] Roman, Klokov and Victor, Lempitsky. 295 | Escape from Cells: Deep Kd-Networks for the Recognition of 3D Point Cloud Models. 296 | *arXiv preprint arXiv:1704.01222* 2017. 297 | 298 | - [paper on arXiv](https://arxiv.org/abs/1704.01222) 299 | - [project page](http://sites.skoltech.ru/compvision/kdnets) 300 | - [code: Regenerator/kdnets](https://github.com/Regenerator/kdnets) 301 | - ICCV 2017 302 | 303 | [5] Z, Wu and S, Song and A, Khosla and F, Yu and L, Zhang and X, Tang and J, Xiao. 304 | 3D ShapeNets: A Deep Representation for Volumetric Shapes. 305 | Proceedings of 28th IEEE Conference on Computer Vision and Pattern Recognition (CVPR2015), 2015. 306 | 307 | - [project page](http://modelnet.cs.princeton.edu/) 308 | 309 | [6] Iro, Armeni and Alexander, Sax and Amir, R., Zamir and Silvio, Savarese. 310 | Joint 2D-3D-Semantic Data for Indoor Scene Understanding. 311 | *arXiv preprint arXiv:1702.01105* 2017. 312 | 313 | - [paper on arXiv](https://arxiv.org/abs/1702.01105) 314 | - [project page](http://buildingparser.stanford.edu/dataset.html) 315 | - [code: alexsax/2D-3D-Semantics](https://github.com/alexsax/2D-3D-Semantics) 316 | 317 | [7] Wei, Zeng and Theo, Gevers. 318 | 3DContextNet: K-d Tree Guided Hierarchical Learning of Point Clouds Using Local and Global Contextual Cues. 319 | *arXiv preprint arXiv:1711.11379* 2017. 320 | 321 | - [paper on arXiv](https://arxiv.org/abs/1711.11379) 322 | 323 | --------------------------------------------------------------------------------