├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── assets └── teaser.jpg ├── downstream ├── insseg │ ├── .gitignore │ ├── .style.yapf │ ├── config │ │ └── default.yaml │ ├── datasets │ │ ├── __init__.py │ │ ├── dataloader.py │ │ ├── dataset.py │ │ ├── evaluation │ │ │ ├── evaluate_semantic_instance.py │ │ │ ├── evaluate_semantic_label.py │ │ │ └── scannet_benchmark_utils │ │ │ │ ├── scripts │ │ │ │ ├── ._evaluate_semantic_instance.py │ │ │ │ ├── ._evaluate_semantic_label.py │ │ │ │ ├── ._util.py │ │ │ │ ├── ._util_3d.py │ │ │ │ ├── evaluate_semantic_instance.py │ │ │ │ ├── evaluate_semantic_label.py │ │ │ │ ├── util.py │ │ │ │ └── util_3d.py │ │ │ │ ├── util.py │ │ │ │ └── util_3d.py │ │ ├── scannet.py │ │ ├── stanford.py │ │ ├── synthia.py │ │ ├── transforms.py │ │ └── voxelizer.py │ ├── ddp_main.py │ ├── lib │ │ ├── __init__.py │ │ ├── bfs │ │ │ ├── bfs.py │ │ │ └── ops │ │ │ │ ├── PG_OP.egg-info │ │ │ │ ├── PKG-INFO │ │ │ │ ├── SOURCES.txt │ │ │ │ ├── dependency_links.txt │ │ │ │ └── top_level.txt │ │ │ │ ├── build │ │ │ │ └── temp.linux-x86_64-3.7 │ │ │ │ │ ├── .ninja_deps │ │ │ │ │ ├── .ninja_log │ │ │ │ │ └── build.ninja │ │ │ │ ├── dist │ │ │ │ └── PG_OP-0.0.0-py3.7-linux-x86_64.egg │ │ │ │ ├── ops.py │ │ │ │ ├── setup.py │ │ │ │ └── src │ │ │ │ ├── bfs_cluster.cpp │ │ │ │ └── bfs_cluster_kernel.cu │ │ ├── ddp_trainer.py │ │ ├── distributed.py │ │ ├── io3d.py │ │ ├── layers.py │ │ ├── math_functions.py │ │ ├── pc_utils.py │ │ ├── solvers.py │ │ ├── test.py │ │ └── utils.py │ ├── models │ │ ├── __init__.py │ │ ├── common.py │ │ ├── model.py │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── common.py │ │ │ ├── resnet_block.py │ │ │ └── senet_block.py │ │ ├── res16unet.py │ │ ├── residual_block.py │ │ ├── resnet.py │ │ ├── resunet.py │ │ └── wrapper.py │ └── scripts │ │ ├── data_efficient │ │ ├── by_points.sh │ │ └── by_scenes.sh │ │ ├── test_scannet.sh │ │ ├── test_scannet_benchmark.sh │ │ ├── test_stanford3d.sh │ │ ├── train_scannet.sh │ │ ├── train_scannet_benchmark.sh │ │ └── train_stanford3d.sh ├── semseg │ ├── .gitignore │ ├── .style.yapf │ ├── config │ │ └── default.yaml │ ├── datasets │ │ ├── __init__.py │ │ ├── dataloader.py │ │ ├── dataset.py │ │ ├── evaluation │ │ │ ├── evaluate_semantic_instance.py │ │ │ ├── evaluate_semantic_label.py │ │ │ └── scannet_benchmark_utils │ │ │ │ ├── scripts │ │ │ │ ├── ._evaluate_semantic_instance.py │ │ │ │ ├── ._evaluate_semantic_label.py │ │ │ │ ├── ._util.py │ │ │ │ ├── ._util_3d.py │ │ │ │ ├── evaluate_semantic_instance.py │ │ │ │ ├── evaluate_semantic_label.py │ │ │ │ ├── util.py │ │ │ │ └── util_3d.py │ │ │ │ ├── util.py │ │ │ │ └── util_3d.py │ │ ├── preprocessing │ │ │ ├── scannet │ │ │ │ ├── collect_indoor3d_data.py │ │ │ │ ├── scannetv2-labels.combined.tsv │ │ │ │ └── split │ │ │ │ │ ├── scannetv2_test.txt │ │ │ │ │ ├── scannetv2_train.txt │ │ │ │ │ └── scannetv2_val.txt │ │ │ └── stanford │ │ │ │ └── stanford.py │ │ ├── scannet.py │ │ ├── stanford.py │ │ ├── synthia.py │ │ ├── transforms.py │ │ └── voxelizer.py │ ├── ddp_main.py │ ├── lib │ │ ├── __init__.py │ │ ├── ddp_trainer.py │ │ ├── distributed.py │ │ ├── io3d.py │ │ ├── layers.py │ │ ├── math_functions.py │ │ ├── pc_utils.py │ │ ├── plot_graph.py │ │ ├── sampling_points.py │ │ ├── solvers.py │ │ ├── test.py │ │ └── utils.py │ ├── models │ │ ├── __init__.py │ │ ├── common.py │ │ ├── model.py │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── common.py │ │ │ ├── resnet_block.py │ │ │ └── senet_block.py │ │ ├── res16unet.py │ │ ├── residual_block.py │ │ ├── resnet.py │ │ ├── resunet.py │ │ └── wrapper.py │ └── scripts │ │ ├── data_efficient │ │ ├── by_points.sh │ │ └── by_scenes.sh │ │ ├── inference_features.sh │ │ ├── test_scannet.sh │ │ ├── test_stanford3d.sh │ │ ├── train_scannet.sh │ │ └── train_stanford3d.sh └── votenet │ ├── config │ └── default.yaml │ ├── datasets │ ├── evaluation │ │ ├── evaluate_object_detection.py │ │ ├── evaluate_object_detection_helper.py │ │ ├── metric_util.py │ │ ├── util.py │ │ └── util_3d.py │ ├── scannet │ │ ├── README.md │ │ ├── batch_load_scannet_data.py │ │ ├── data_viz.py │ │ ├── load_scannet_data.py │ │ ├── meta_data │ │ │ ├── scannet_means.npz │ │ │ ├── scannetv2-labels.combined.tsv │ │ │ ├── scannetv2_test.txt │ │ │ ├── scannetv2_train.txt │ │ │ └── scannetv2_val.txt │ │ ├── model_util_scannet.py │ │ ├── scannet_detection_dataset.py │ │ ├── scannet_train_detection_data │ │ └── scannet_utils.py │ └── sunrgbd │ │ ├── README.md │ │ ├── __pycache__ │ │ └── sunrgbd_utils.cpython-37.pyc │ │ ├── model_util_sunrgbd.py │ │ ├── sunrgbd_data.py │ │ ├── sunrgbd_detection_dataset.py │ │ ├── sunrgbd_pc_bbox_votes_50k_v1_train │ │ ├── sunrgbd_pc_bbox_votes_50k_v1_val │ │ └── sunrgbd_utils.py │ ├── ddp_main.py │ ├── lib │ ├── ddp_trainer.py │ ├── distributed.py │ └── utils │ │ ├── box_util.py │ │ ├── eval_det.py │ │ ├── io3d.py │ │ ├── metric_util.py │ │ ├── nms.py │ │ ├── nn_distance.py │ │ ├── pc_util.py │ │ ├── tf_logger.py │ │ └── tf_visualizer.py │ ├── models │ ├── ap_helper.py │ ├── backbone │ │ ├── pointnet2 │ │ │ ├── __pycache__ │ │ │ │ ├── pointnet2_modules.cpython-37.pyc │ │ │ │ ├── pointnet2_utils.cpython-37.pyc │ │ │ │ └── pytorch_utils.cpython-37.pyc │ │ │ ├── _ext_src │ │ │ │ ├── include │ │ │ │ │ ├── ball_query.h │ │ │ │ │ ├── cuda_utils.h │ │ │ │ │ ├── group_points.h │ │ │ │ │ ├── interpolate.h │ │ │ │ │ ├── sampling.h │ │ │ │ │ └── utils.h │ │ │ │ └── src │ │ │ │ │ ├── ball_query.cpp │ │ │ │ │ ├── ball_query_gpu.cu │ │ │ │ │ ├── bindings.cpp │ │ │ │ │ ├── group_points.cpp │ │ │ │ │ ├── group_points_gpu.cu │ │ │ │ │ ├── interpolate.cpp │ │ │ │ │ ├── interpolate_gpu.cu │ │ │ │ │ ├── sampling.cpp │ │ │ │ │ └── sampling_gpu.cu │ │ │ ├── pointnet2_modules.py │ │ │ ├── pointnet2_test.py │ │ │ ├── pointnet2_utils.py │ │ │ ├── pytorch_utils.py │ │ │ └── setup.py │ │ └── sparseconv │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── config.cpython-37.pyc │ │ │ └── voxelized_dataset.cpython-37.pyc │ │ │ ├── config.py │ │ │ ├── lib │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-37.pyc │ │ │ │ └── math_functions.cpython-37.pyc │ │ │ └── math_functions.py │ │ │ ├── models_sparseconv │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-37.pyc │ │ │ │ ├── conditional_random_fields.cpython-37.pyc │ │ │ │ ├── model.cpython-37.pyc │ │ │ │ ├── res16unet.cpython-37.pyc │ │ │ │ ├── resnet.cpython-37.pyc │ │ │ │ ├── resunet.cpython-37.pyc │ │ │ │ └── wrapper.cpython-37.pyc │ │ │ ├── conditional_random_fields.py │ │ │ ├── model.py │ │ │ ├── modules │ │ │ │ ├── __init__.py │ │ │ │ ├── __pycache__ │ │ │ │ │ ├── __init__.cpython-36.pyc │ │ │ │ │ ├── __init__.cpython-37.pyc │ │ │ │ │ ├── common.cpython-36.pyc │ │ │ │ │ ├── common.cpython-37.pyc │ │ │ │ │ ├── resnet_block.cpython-36.pyc │ │ │ │ │ └── resnet_block.cpython-37.pyc │ │ │ │ ├── common.py │ │ │ │ ├── resnet_block.py │ │ │ │ └── senet_block.py │ │ │ ├── res16unet.py │ │ │ ├── resnet.py │ │ │ ├── resunet.py │ │ │ └── wrapper.py │ │ │ ├── voxelized_dataset.py │ │ │ └── voxelizer.py │ ├── backbone_module.py │ ├── boxnet.py │ ├── dump_helper.py │ ├── loss_helper.py │ ├── loss_helper_boxnet.py │ ├── proposal_module.py │ ├── votenet.py │ └── voting_module.py │ └── scripts │ ├── data_efficient │ ├── by_bboxes.sh │ └── by_scenes.sh │ ├── test_scannet.sh │ ├── test_scannet_pointnet.sh │ ├── test_sunrgbd.sh │ ├── train_scannet.sh │ ├── train_scannet_pointnet.sh │ └── train_sunrgbd.sh ├── pretrain ├── contrastive_scene_contexts │ ├── .gitignore │ ├── config │ │ └── defaults.yaml │ ├── ddp_train.py │ ├── lib │ │ ├── __init__.py │ │ ├── criterion.py │ │ ├── data_sampler.py │ │ ├── ddp_data_loaders.py │ │ ├── ddp_trainer.py │ │ ├── distributed.py │ │ ├── error_handler.py │ │ ├── evaluation │ │ │ ├── evaluate_semantic_instance.py │ │ │ ├── evaluate_semantic_label.py │ │ │ └── scannet_benchmark_utils │ │ │ │ ├── util.py │ │ │ │ └── util_3d.py │ │ ├── io3d.py │ │ ├── multiprocessing_utils.py │ │ ├── shape_context.py │ │ ├── timer.py │ │ └── transforms.py │ ├── model │ │ ├── __init__.py │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── common.py │ │ │ └── resnet_block.py │ │ ├── pointnet2 │ │ │ ├── _ext_src │ │ │ │ ├── include │ │ │ │ │ ├── ball_query.h │ │ │ │ │ ├── cuda_utils.h │ │ │ │ │ ├── group_points.h │ │ │ │ │ ├── interpolate.h │ │ │ │ │ ├── sampling.h │ │ │ │ │ └── utils.h │ │ │ │ └── src │ │ │ │ │ ├── ball_query.cpp │ │ │ │ │ ├── ball_query_gpu.cu │ │ │ │ │ ├── bindings.cpp │ │ │ │ │ ├── group_points.cpp │ │ │ │ │ ├── group_points_gpu.cu │ │ │ │ │ ├── interpolate.cpp │ │ │ │ │ ├── interpolate_gpu.cu │ │ │ │ │ ├── sampling.cpp │ │ │ │ │ └── sampling_gpu.cu │ │ │ ├── pointnet2_modules.py │ │ │ ├── pointnet2_test.py │ │ │ ├── pointnet2_utils.py │ │ │ ├── pytorch_utils.py │ │ │ └── setup.py │ │ ├── pointnet2backbone.py │ │ ├── res16unet.py │ │ └── resnet.py │ └── scripts │ │ ├── pretrain_pointnet2.sh │ │ └── pretrain_sparseconv.sh └── scannet_pair │ ├── SensorData.py │ ├── __pycache__ │ ├── SensorData.cpython-36.pyc │ ├── SensorData.cpython-38.pyc │ └── plyfile.cpython-36.pyc │ ├── compute_full_overlapping.py │ ├── generage_list.py │ ├── plyfile.py │ ├── point_cloud_extractor.py │ ├── preprocess.sh │ └── reader.py └── requirements.txt /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Contrastive Scene Contexts 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `master`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need 17 | to do this once to work on any of Facebook's open source projects. 18 | 19 | Complete your CLA here: 20 | 21 | ## Issues 22 | We use GitHub issues to track public bugs. Please ensure your description is 23 | clear and has sufficient instructions to be able to reproduce the issue. 24 | 25 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 26 | disclosure of security bugs. In those cases, please go through the process 27 | outlined on that page and do not file a public issue. 28 | 29 | ## License 30 | By contributing to Contrastive Scene Contexts, you agree that your contributions will be licensed 31 | under the LICENSE file in the root directory of this source tree. 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Facebook, Inc. and its affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /assets/teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/assets/teaser.jpg -------------------------------------------------------------------------------- /downstream/insseg/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized 2 | __pycache__/ 3 | *.py[cod] 4 | *.o 5 | *.so 6 | 7 | # Text edits 8 | *.swp 9 | *.swo 10 | *.orig 11 | 12 | # Training 13 | outputs/ 14 | 15 | # Profiling 16 | callgrind.out* 17 | *.dSYM 18 | 19 | # Misc 20 | .DS_Store 21 | -------------------------------------------------------------------------------- /downstream/insseg/.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = chromium 3 | column_limit = 100 4 | indent_width = 2 5 | -------------------------------------------------------------------------------- /downstream/insseg/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #from lib.datasets import synthia 8 | from datasets import stanford 9 | from datasets import scannet 10 | #from lib.datasets import shapenet 11 | 12 | DATASETS = [] 13 | 14 | 15 | def add_datasets(module): 16 | DATASETS.extend([getattr(module, a) for a in dir(module) if 'Dataset' in a]) 17 | 18 | 19 | add_datasets(stanford) 20 | #add_datasets(synthia) 21 | add_datasets(scannet) 22 | #add_datasets(shapenet) 23 | 24 | def load_dataset(name): 25 | '''Creates and returns an instance of the datasets given its name. 26 | ''' 27 | # Find the model class from its name 28 | mdict = {dataset.__name__: dataset for dataset in DATASETS} 29 | if name not in mdict: 30 | print('Invalid dataset index. Options are:') 31 | # Display a list of valid dataset names 32 | for dataset in DATASETS: 33 | print('\t* {}'.format(dataset.__name__)) 34 | raise ValueError(f'Dataset {name} not defined') 35 | DatasetClass = mdict[name] 36 | 37 | return DatasetClass 38 | -------------------------------------------------------------------------------- /downstream/insseg/datasets/dataloader.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import math 7 | import torch 8 | import torch.distributed as dist 9 | from torch.utils.data.sampler import Sampler 10 | 11 | 12 | class InfSampler(Sampler): 13 | """Samples elements randomly, without replacement. 14 | 15 | Arguments: 16 | data_source (Dataset): dataset to sample from 17 | """ 18 | 19 | def __init__(self, data_source, shuffle=False): 20 | self.data_source = data_source 21 | self.shuffle = shuffle 22 | self.reset_permutation() 23 | 24 | def reset_permutation(self): 25 | perm = len(self.data_source) 26 | if self.shuffle: 27 | perm = torch.randperm(perm) 28 | self._perm = perm.tolist() 29 | 30 | def __iter__(self): 31 | return self 32 | 33 | def __next__(self): 34 | if len(self._perm) == 0: 35 | self.reset_permutation() 36 | 37 | return self._perm.pop() 38 | 39 | def __len__(self): 40 | return len(self.data_source) 41 | 42 | next = __next__ # Python 2 compatibility 43 | 44 | 45 | class DistributedInfSampler(InfSampler): 46 | def __init__(self, data_source, num_replicas=None, rank=None, shuffle=True): 47 | if num_replicas is None: 48 | if not dist.is_available(): 49 | raise RuntimeError("Requires distributed package to be available") 50 | num_replicas = dist.get_world_size() 51 | if rank is None: 52 | if not dist.is_available(): 53 | raise RuntimeError("Requires distributed package to be available") 54 | rank = dist.get_rank() 55 | 56 | self.data_source = data_source 57 | self.num_replicas = num_replicas 58 | self.rank = rank 59 | self.epoch = 0 60 | self.it = 0 61 | self.num_samples = int(math.ceil(len(self.data_source) * 1.0 / self.num_replicas)) 62 | self.total_size = self.num_samples * self.num_replicas 63 | self.shuffle = shuffle 64 | self.reset_permutation() 65 | 66 | def __next__(self): 67 | it = self.it * self.num_replicas + self.rank 68 | value = self._perm[it % len(self._perm)] 69 | self.it = self.it + 1 70 | 71 | if (self.it * self.num_replicas) >= len(self._perm): 72 | self.reset_permutation() 73 | self.it = 0 74 | return value 75 | 76 | def __len__(self): 77 | return self.num_samples -------------------------------------------------------------------------------- /downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_instance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_instance.py -------------------------------------------------------------------------------- /downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_label.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_label.py -------------------------------------------------------------------------------- /downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util.py -------------------------------------------------------------------------------- /downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util_3d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util_3d.py -------------------------------------------------------------------------------- /downstream/insseg/datasets/evaluation/scannet_benchmark_utils/scripts/util.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import csv 3 | try: 4 | import numpy as np 5 | except: 6 | print("Failed to import numpy package.") 7 | sys.exit(-1) 8 | try: 9 | import imageio 10 | except: 11 | print("Please install the module 'imageio' for image processing, e.g.") 12 | print("pip install imageio") 13 | sys.exit(-1) 14 | 15 | # print an error message and quit 16 | def print_error(message, user_fault=False): 17 | sys.stderr.write('ERROR: ' + str(message) + '\n') 18 | if user_fault: 19 | sys.exit(2) 20 | sys.exit(-1) 21 | 22 | 23 | # if string s represents an int 24 | def represents_int(s): 25 | try: 26 | int(s) 27 | return True 28 | except ValueError: 29 | return False 30 | 31 | 32 | def read_label_mapping(filename, label_from='raw_category', label_to='nyu40id'): 33 | assert os.path.isfile(filename) 34 | mapping = dict() 35 | with open(filename) as csvfile: 36 | reader = csv.DictReader(csvfile, delimiter='\t') 37 | for row in reader: 38 | mapping[row[label_from]] = int(row[label_to]) 39 | # if ints convert 40 | if represents_int([key for key in mapping.keys()][0]): 41 | mapping = {int(k):v for k,v in mapping.items()} 42 | return mapping 43 | 44 | 45 | # input: scene_types.txt or scene_types_all.txt 46 | def read_scene_types_mapping(filename, remove_spaces=True): 47 | assert os.path.isfile(filename) 48 | mapping = dict() 49 | lines = open(filename).read().splitlines() 50 | lines = [line.split('\t') for line in lines] 51 | if remove_spaces: 52 | mapping = { x[1].strip():int(x[0]) for x in lines } 53 | else: 54 | mapping = { x[1]:int(x[0]) for x in lines } 55 | return mapping 56 | 57 | 58 | # color by label 59 | def visualize_label_image(filename, image): 60 | height = image.shape[0] 61 | width = image.shape[1] 62 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 63 | color_palette = create_color_palette() 64 | for idx, color in enumerate(color_palette): 65 | vis_image[image==idx] = color 66 | imageio.imwrite(filename, vis_image) 67 | 68 | 69 | # color by different instances (mod length of color palette) 70 | def visualize_instance_image(filename, image): 71 | height = image.shape[0] 72 | width = image.shape[1] 73 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 74 | color_palette = create_color_palette() 75 | instances = np.unique(image) 76 | for idx, inst in enumerate(instances): 77 | vis_image[image==inst] = color_palette[inst%len(color_palette)] 78 | imageio.imwrite(filename, vis_image) 79 | 80 | -------------------------------------------------------------------------------- /downstream/insseg/datasets/evaluation/scannet_benchmark_utils/util.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import csv 3 | try: 4 | import numpy as np 5 | except: 6 | print("Failed to import numpy package.") 7 | sys.exit(-1) 8 | try: 9 | import imageio 10 | except: 11 | print("Please install the module 'imageio' for image processing, e.g.") 12 | print("pip install imageio") 13 | sys.exit(-1) 14 | 15 | # print an error message and quit 16 | def print_error(message, user_fault=False): 17 | sys.stderr.write('ERROR: ' + str(message) + '\n') 18 | if user_fault: 19 | sys.exit(2) 20 | sys.exit(-1) 21 | 22 | 23 | # if string s represents an int 24 | def represents_int(s): 25 | try: 26 | int(s) 27 | return True 28 | except ValueError: 29 | return False 30 | 31 | 32 | def read_label_mapping(filename, label_from='raw_category', label_to='nyu40id'): 33 | assert os.path.isfile(filename) 34 | mapping = dict() 35 | with open(filename) as csvfile: 36 | reader = csv.DictReader(csvfile, delimiter='\t') 37 | for row in reader: 38 | mapping[row[label_from]] = int(row[label_to]) 39 | # if ints convert 40 | if represents_int([key for key in mapping.keys()][0]): 41 | mapping = {int(k):v for k,v in mapping.items()} 42 | return mapping 43 | 44 | 45 | # input: scene_types.txt or scene_types_all.txt 46 | def read_scene_types_mapping(filename, remove_spaces=True): 47 | assert os.path.isfile(filename) 48 | mapping = dict() 49 | lines = open(filename).read().splitlines() 50 | lines = [line.split('\t') for line in lines] 51 | if remove_spaces: 52 | mapping = { x[1].strip():int(x[0]) for x in lines } 53 | else: 54 | mapping = { x[1]:int(x[0]) for x in lines } 55 | return mapping 56 | 57 | 58 | # color by label 59 | def visualize_label_image(filename, image): 60 | height = image.shape[0] 61 | width = image.shape[1] 62 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 63 | color_palette = create_color_palette() 64 | for idx, color in enumerate(color_palette): 65 | vis_image[image==idx] = color 66 | imageio.imwrite(filename, vis_image) 67 | 68 | 69 | # color by different instances (mod length of color palette) 70 | def visualize_instance_image(filename, image): 71 | height = image.shape[0] 72 | width = image.shape[1] 73 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 74 | color_palette = create_color_palette() 75 | instances = np.unique(image) 76 | for idx, inst in enumerate(instances): 77 | vis_image[image==inst] = color_palette[inst%len(color_palette)] 78 | imageio.imwrite(filename, vis_image) 79 | 80 | -------------------------------------------------------------------------------- /downstream/insseg/ddp_main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import os 7 | import sys 8 | import hydra 9 | import torch 10 | import numpy as np 11 | 12 | from lib.ddp_trainer import SegmentationTrainer 13 | from lib.distributed import multi_proc_run 14 | 15 | def single_proc_run(config): 16 | if not torch.cuda.is_available(): 17 | raise Exception('No GPUs FOUND.') 18 | trainer = SegmentationTrainer(config) 19 | if config.train.is_train: 20 | trainer.train() 21 | else: 22 | trainer.test() 23 | 24 | @hydra.main(config_path='config', config_name='default.yaml') 25 | def main(config): 26 | 27 | # fix seed 28 | np.random.seed(config.misc.seed) 29 | torch.manual_seed(config.misc.seed) 30 | torch.cuda.manual_seed(config.misc.seed) 31 | 32 | # Convert to dict 33 | if config.misc.num_gpus > 1: 34 | multi_proc_run(config.misc.num_gpus, fun=single_proc_run, fun_args=(config,)) 35 | else: 36 | single_proc_run(config) 37 | 38 | if __name__ == '__main__': 39 | __spec__ = None 40 | os.environ['MKL_THREADING_LAYER'] = 'GNU' 41 | main() 42 | -------------------------------------------------------------------------------- /downstream/insseg/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/PG_OP.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.0 2 | Name: PG-OP 3 | Version: 0.0.0 4 | Summary: UNKNOWN 5 | Home-page: UNKNOWN 6 | Author: UNKNOWN 7 | Author-email: UNKNOWN 8 | License: UNKNOWN 9 | Description: UNKNOWN 10 | Platform: UNKNOWN 11 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/PG_OP.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | setup.py 2 | PG_OP.egg-info/PKG-INFO 3 | PG_OP.egg-info/SOURCES.txt 4 | PG_OP.egg-info/dependency_links.txt 5 | PG_OP.egg-info/top_level.txt 6 | src/bfs_cluster.cpp 7 | src/bfs_cluster_kernel.cu -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/PG_OP.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/PG_OP.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | PG_OP 2 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/.ninja_deps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/.ninja_deps -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/.ninja_log: -------------------------------------------------------------------------------- 1 | # ninja log v5 2 | 1 660 1619447017255716929 /rhome/jhou/csc/downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/src/bfs_cluster_kernel.o d5477a5f479e9fe9 3 | 1 20399 1619447036979599521 /rhome/jhou/csc/downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/src/bfs_cluster.o 21be16bcfecb8bdb 4 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/build.ninja: -------------------------------------------------------------------------------- 1 | ninja_required_version = 1.3 2 | cxx = c++ 3 | nvcc = /usr/local/cuda-10.2/bin/nvcc 4 | 5 | cflags = -pthread -B /rhome/jhou/.local/anaconda3/envs/sparseconv043/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include/torch/csrc/api/include -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include/TH -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include/THC -I/usr/local/cuda-10.2/include -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/include/python3.7m -c 6 | post_cflags = -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=PG_OP -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14 7 | cuda_cflags = -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include/torch/csrc/api/include -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include/TH -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/lib/python3.7/site-packages/torch/include/THC -I/usr/local/cuda-10.2/include -I/rhome/jhou/.local/anaconda3/envs/sparseconv043/include/python3.7m -c 8 | cuda_post_cflags = -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=PG_OP -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_75,code=sm_75 -std=c++14 9 | ldflags = 10 | 11 | rule compile 12 | command = $cxx -MMD -MF $out.d $cflags -c $in -o $out $post_cflags 13 | depfile = $out.d 14 | deps = gcc 15 | 16 | rule cuda_compile 17 | command = $nvcc $cuda_cflags -c $in -o $out $cuda_post_cflags 18 | 19 | 20 | 21 | build /rhome/jhou/csc/downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/src/bfs_cluster.o: compile /rhome/jhou/csc/downstream/insseg/lib/bfs/ops/src/bfs_cluster.cpp 22 | build /rhome/jhou/csc/downstream/insseg/lib/bfs/ops/build/temp.linux-x86_64-3.7/src/bfs_cluster_kernel.o: cuda_compile /rhome/jhou/csc/downstream/insseg/lib/bfs/ops/src/bfs_cluster_kernel.cu 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/dist/PG_OP-0.0.0-py3.7-linux-x86_64.egg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/lib/bfs/ops/dist/PG_OP-0.0.0-py3.7-linux-x86_64.egg -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/ops.py: -------------------------------------------------------------------------------- 1 | ''' 2 | PointGroup operations 3 | Written by Li Jiang 4 | ''' 5 | 6 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 3 | 4 | setup( 5 | name='PG_OP', 6 | ext_modules=[ 7 | CUDAExtension('PG_OP', [ 8 | 'src/bfs_cluster.cpp', 9 | 'src/bfs_cluster_kernel.cu', 10 | ]) 11 | ], 12 | cmdclass={'build_ext': BuildExtension} 13 | ) 14 | -------------------------------------------------------------------------------- /downstream/insseg/lib/bfs/ops/src/bfs_cluster_kernel.cu: -------------------------------------------------------------------------------- 1 | /* 2 | Ball Query with BatchIdx 3 | Written by Li Jiang 4 | All Rights Reserved 2020. 5 | */ 6 | #include 7 | #include 8 | #include 9 | 10 | #define TOTAL_THREADS 1024 11 | #define THREADS_PER_BLOCK 512 12 | #define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0)) 13 | 14 | 15 | /* ================================== ballquery_batch_p ================================== */ 16 | __global__ void ballquery_batch_p_cuda_(int n, int meanActive, float radius, const float *xyz, const int *batch_idxs, const int *batch_offsets, int *idx, int *start_len, int *cumsum) { 17 | int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; 18 | if (pt_idx >= n) return; 19 | 20 | start_len += (pt_idx * 2); 21 | int idx_temp[1000]; 22 | 23 | float radius2 = radius * radius; 24 | float o_x = xyz[pt_idx * 3 + 0]; 25 | float o_y = xyz[pt_idx * 3 + 1]; 26 | float o_z = xyz[pt_idx * 3 + 2]; 27 | 28 | int batch_idx = batch_idxs[pt_idx]; 29 | int start = batch_offsets[batch_idx]; 30 | int end = batch_offsets[batch_idx + 1]; 31 | 32 | int cnt = 0; 33 | for(int k = start; k < end; k++){ 34 | float x = xyz[k * 3 + 0]; 35 | float y = xyz[k * 3 + 1]; 36 | float z = xyz[k * 3 + 2]; 37 | float d2 = (o_x - x) * (o_x - x) + (o_y - y) * (o_y - y) + (o_z - z) * (o_z - z); 38 | if(d2 < radius2){ 39 | if(cnt < 1000){ 40 | idx_temp[cnt] = k; 41 | } 42 | else{ 43 | break; 44 | } 45 | ++cnt; 46 | } 47 | } 48 | 49 | start_len[0] = atomicAdd(cumsum, cnt); 50 | start_len[1] = cnt; 51 | 52 | int thre = n * meanActive; 53 | if(start_len[0] >= thre) return; 54 | 55 | idx += start_len[0]; 56 | if(start_len[0] + cnt >= thre) cnt = thre - start_len[0]; 57 | 58 | for(int k = 0; k < cnt; k++){ 59 | idx[k] = idx_temp[k]; 60 | } 61 | } 62 | 63 | 64 | int ballquery_batch_p_cuda(int n, int meanActive, float radius, const float *xyz, const int *batch_idxs, const int *batch_offsets, int *idx, int *start_len, cudaStream_t stream) { 65 | // param xyz: (n, 3) 66 | // param batch_idxs: (n) 67 | // param batch_offsets: (B + 1) 68 | // output idx: (n * meanActive) dim 0 for number of points in the ball, idx in n 69 | // output start_len: (n, 2), int 70 | 71 | cudaError_t err; 72 | 73 | dim3 blocks(DIVUP(n, THREADS_PER_BLOCK)); 74 | dim3 threads(THREADS_PER_BLOCK); 75 | 76 | int cumsum = 0; 77 | int* p_cumsum; 78 | cudaMalloc((void**)&p_cumsum, sizeof(int)); 79 | cudaMemcpy(p_cumsum, &cumsum, sizeof(int), cudaMemcpyHostToDevice); 80 | 81 | ballquery_batch_p_cuda_<<>>(n, meanActive, radius, xyz, batch_idxs, batch_offsets, idx, start_len, p_cumsum); 82 | 83 | err = cudaGetLastError(); 84 | if (cudaSuccess != err) { 85 | fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err)); 86 | exit(-1); 87 | } 88 | 89 | cudaMemcpy(&cumsum, p_cumsum, sizeof(int), cudaMemcpyDeviceToHost); 90 | return cumsum; 91 | } 92 | -------------------------------------------------------------------------------- /downstream/insseg/lib/io3d.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import numpy as np 8 | import trimesh 9 | 10 | # color palette for nyu40 labels 11 | def create_color_palette(): 12 | return [ 13 | (0, 0, 0), 14 | (174, 199, 232), # wall 15 | (152, 223, 138), # floor 16 | (31, 119, 180), # cabinet 17 | (255, 187, 120), # bed 18 | (188, 189, 34), # chair 19 | (140, 86, 75), # sofa 20 | (255, 152, 150), # table 21 | (214, 39, 40), # door 22 | (197, 176, 213), # window 23 | (148, 103, 189), # bookshelf 24 | (196, 156, 148), # picture 25 | (23, 190, 207), # counter 26 | (178, 76, 76), 27 | (247, 182, 210), # desk 28 | (66, 188, 102), 29 | (219, 219, 141), # curtain 30 | (140, 57, 197), 31 | (202, 185, 52), 32 | (51, 176, 203), 33 | (200, 54, 131), 34 | (92, 193, 61), 35 | (78, 71, 183), 36 | (172, 114, 82), 37 | (255, 127, 14), # refrigerator 38 | (91, 163, 138), 39 | (153, 98, 156), 40 | (140, 153, 101), 41 | (158, 218, 229), # shower curtain 42 | (100, 125, 154), 43 | (178, 127, 135), 44 | (120, 185, 128), 45 | (146, 111, 194), 46 | (44, 160, 44), # toilet 47 | (112, 128, 144), # sink 48 | (96, 207, 209), 49 | (227, 119, 194), # bathtub 50 | (213, 92, 176), 51 | (94, 106, 211), 52 | (82, 84, 163), # otherfurn 53 | (100, 85, 144), 54 | ] 55 | 56 | def write_triangle_mesh(vertices, colors, faces, outputFile): 57 | mesh = trimesh.Trimesh(vertices=vertices, vertex_colors=colors, faces=faces, process=False) 58 | mesh.export(outputFile) 59 | 60 | def read_triangle_mesh(filename): 61 | mesh = trimesh.load_mesh(filename, process=False) 62 | if isinstance(mesh, trimesh.PointCloud): 63 | vertices = mesh.vertices 64 | colors = mesh.colors 65 | faces = None 66 | elif isinstance(mesh, trimesh.Trimesh): 67 | vertices = mesh.vertices 68 | colors = mesh.visual.vertex_colors 69 | faces = mesh.faces 70 | 71 | return vertices, colors, faces -------------------------------------------------------------------------------- /downstream/insseg/lib/layers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import torch 8 | import torch.nn as nn 9 | 10 | from MinkowskiEngine import MinkowskiGlobalPooling, MinkowskiBroadcastAddition, MinkowskiBroadcastMultiplication 11 | 12 | 13 | class MinkowskiLayerNorm(nn.Module): 14 | 15 | def __init__(self, num_features, eps=1e-5, D=-1): 16 | super(MinkowskiLayerNorm, self).__init__() 17 | self.num_features = num_features 18 | self.eps = eps 19 | self.weight = nn.Parameter(torch.ones(1, num_features)) 20 | self.bias = nn.Parameter(torch.zeros(1, num_features)) 21 | 22 | self.mean_in = MinkowskiGlobalPooling(dimension=D) 23 | self.glob_sum = MinkowskiBroadcastAddition(dimension=D) 24 | self.glob_sum2 = MinkowskiBroadcastAddition(dimension=D) 25 | self.glob_mean = MinkowskiGlobalPooling(dimension=D) 26 | self.glob_times = MinkowskiBroadcastMultiplication(dimension=D) 27 | self.D = D 28 | self.reset_parameters() 29 | 30 | def __repr__(self): 31 | s = f'(D={self.D})' 32 | return self.__class__.__name__ + s 33 | 34 | def reset_parameters(self): 35 | self.weight.data.fill_(1) 36 | self.bias.data.zero_() 37 | 38 | def _check_input_dim(self, input): 39 | if input.F.dim() != 2: 40 | raise ValueError('expected 2D input (got {}D input)'.format(input.dim())) 41 | 42 | def forward(self, x): 43 | self._check_input_dim(x) 44 | mean = self.mean_in(x).F.mean(-1, keepdim=True) 45 | mean = mean + torch.zeros(mean.size(0), self.num_features).type_as(mean) 46 | temp = self.glob_sum(x.F, -mean)**2 47 | var = self.glob_mean(temp.data).mean(-1, keepdim=True) 48 | var = var + torch.zeros(var.size(0), self.num_features).type_as(var) 49 | instd = 1 / (var + self.eps).sqrt() 50 | 51 | x = self.glob_times(self.glob_sum2(x, -mean), instd) 52 | return x * self.weight + self.bias 53 | 54 | 55 | class MinkowskiInstanceNorm(nn.Module): 56 | 57 | def __init__(self, num_features, eps=1e-5, D=-1): 58 | super(MinkowskiInstanceNorm, self).__init__() 59 | self.eps = eps 60 | self.weight = nn.Parameter(torch.ones(1, num_features)) 61 | self.bias = nn.Parameter(torch.zeros(1, num_features)) 62 | 63 | self.mean_in = MinkowskiGlobalPooling(dimension=D) 64 | self.glob_sum = MinkowskiBroadcastAddition(dimension=D) 65 | self.glob_sum2 = MinkowskiBroadcastAddition(dimension=D) 66 | self.glob_mean = MinkowskiGlobalPooling(dimension=D) 67 | self.glob_times = MinkowskiBroadcastMultiplication(dimension=D) 68 | self.D = D 69 | self.reset_parameters() 70 | 71 | def __repr__(self): 72 | s = f'(pixel_dist={self.pixel_dist}, D={self.D})' 73 | return self.__class__.__name__ + s 74 | 75 | def reset_parameters(self): 76 | self.weight.data.fill_(1) 77 | self.bias.data.zero_() 78 | 79 | def _check_input_dim(self, input): 80 | if input.dim() != 2: 81 | raise ValueError('expected 2D input (got {}D input)'.format(input.dim())) 82 | 83 | def forward(self, x): 84 | self._check_input_dim(x) 85 | mean_in = self.mean_in(x) 86 | temp = self.glob_sum(x, -mean_in)**2 87 | var_in = self.glob_mean(temp.data) 88 | instd_in = 1 / (var_in + self.eps).sqrt() 89 | 90 | x = self.glob_times(self.glob_sum2(x, -mean_in), instd_in) 91 | return x * self.weight + self.bias 92 | -------------------------------------------------------------------------------- /downstream/insseg/lib/math_functions.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from scipy.sparse import csr_matrix 8 | import torch 9 | 10 | 11 | class SparseMM(torch.autograd.Function): 12 | """ 13 | Sparse x dense matrix multiplication with autograd support. 14 | Implementation by Soumith Chintala: 15 | https://discuss.pytorch.org/t/ 16 | does-pytorch-support-autograd-on-sparse-matrix/6156/7 17 | """ 18 | 19 | def forward(self, matrix1, matrix2): 20 | self.save_for_backward(matrix1, matrix2) 21 | return torch.mm(matrix1, matrix2) 22 | 23 | def backward(self, grad_output): 24 | matrix1, matrix2 = self.saved_tensors 25 | grad_matrix1 = grad_matrix2 = None 26 | 27 | if self.needs_input_grad[0]: 28 | grad_matrix1 = torch.mm(grad_output, matrix2.t()) 29 | 30 | if self.needs_input_grad[1]: 31 | grad_matrix2 = torch.mm(matrix1.t(), grad_output) 32 | 33 | return grad_matrix1, grad_matrix2 34 | 35 | 36 | def sparse_float_tensor(values, indices, size=None): 37 | """ 38 | Return a torch sparse matrix give values and indices (row_ind, col_ind). 39 | If the size is an integer, return a square matrix with side size. 40 | If the size is a torch.Size, use it to initialize the out tensor. 41 | If none, the size is inferred. 42 | """ 43 | indices = torch.stack(indices).int() 44 | sargs = [indices, values.float()] 45 | if size is not None: 46 | # Use the provided size 47 | if isinstance(size, int): 48 | size = torch.Size((size, size)) 49 | sargs.append(size) 50 | if values.is_cuda: 51 | return torch.cuda.sparse.FloatTensor(*sargs) 52 | else: 53 | return torch.sparse.FloatTensor(*sargs) 54 | 55 | 56 | def diags(values, size=None): 57 | values = values.view(-1) 58 | n = values.nelement() 59 | size = torch.Size((n, n)) 60 | indices = (torch.arange(0, n), torch.arange(0, n)) 61 | return sparse_float_tensor(values, indices, size) 62 | 63 | 64 | def sparse_to_csr_matrix(tensor): 65 | tensor = tensor.cpu() 66 | inds = tensor._indices().numpy() 67 | vals = tensor._values().numpy() 68 | return csr_matrix((vals, (inds[0], inds[1])), shape=[s for s in tensor.shape]) 69 | 70 | 71 | def csr_matrix_to_sparse(mat): 72 | row_ind, col_ind = mat.nonzero() 73 | return sparse_float_tensor( 74 | torch.from_numpy(mat.data), 75 | (torch.from_numpy(row_ind), torch.from_numpy(col_ind)), 76 | size=torch.Size(mat.shape)) 77 | -------------------------------------------------------------------------------- /downstream/insseg/lib/solvers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import logging 8 | 9 | from torch.optim import SGD, Adam 10 | from torch.optim.lr_scheduler import LambdaLR, StepLR 11 | 12 | 13 | class LambdaStepLR(LambdaLR): 14 | 15 | def __init__(self, optimizer, lr_lambda, last_step=-1): 16 | super(LambdaStepLR, self).__init__(optimizer, lr_lambda, last_step) 17 | 18 | @property 19 | def last_step(self): 20 | """Use last_epoch for the step counter""" 21 | return self.last_epoch 22 | 23 | @last_step.setter 24 | def last_step(self, v): 25 | self.last_epoch = v 26 | 27 | 28 | class PolyLR(LambdaStepLR): 29 | """DeepLab learning rate policy""" 30 | 31 | def __init__(self, optimizer, max_iter, power=0.9, last_step=-1): 32 | super(PolyLR, self).__init__(optimizer, lambda s: (1 - s / (max_iter + 1))**power, last_step) 33 | 34 | 35 | class SquaredLR(LambdaStepLR): 36 | """ Used for SGD Lars""" 37 | 38 | def __init__(self, optimizer, max_iter, last_step=-1): 39 | super(SquaredLR, self).__init__(optimizer, lambda s: (1 - s / (max_iter + 1))**2, last_step) 40 | 41 | 42 | class ExpLR(LambdaStepLR): 43 | 44 | def __init__(self, optimizer, step_size, gamma=0.9, last_step=-1): 45 | # (0.9 ** 21.854) = 0.1, (0.95 ** 44.8906) = 0.1 46 | # To get 0.1 every N using gamma 0.9, N * log(0.9)/log(0.1) = 0.04575749 N 47 | # To get 0.1 every N using gamma g, g ** N = 0.1 -> N * log(g) = log(0.1) -> g = np.exp(log(0.1) / N) 48 | super(ExpLR, self).__init__(optimizer, lambda s: gamma**(s / step_size), last_step) 49 | 50 | 51 | def initialize_optimizer(params, config): 52 | assert config.optimizer in ['SGD', 'Adagrad', 'Adam', 'RMSProp', 'Rprop', 'SGDLars'] 53 | 54 | if config.optimizer == 'SGD': 55 | return SGD( 56 | params, 57 | lr=config.lr, 58 | momentum=config.sgd_momentum, 59 | dampening=config.sgd_dampening, 60 | weight_decay=config.weight_decay) 61 | elif config.optimizer == 'Adam': 62 | return Adam( 63 | params, 64 | lr=config.lr, 65 | betas=(config.adam_beta1, config.adam_beta2), 66 | weight_decay=config.weight_decay) 67 | else: 68 | logging.error('Optimizer type not supported') 69 | raise ValueError('Optimizer type not supported') 70 | 71 | 72 | def initialize_scheduler(optimizer, config, last_step=-1): 73 | if config.scheduler == 'StepLR': 74 | return StepLR( 75 | optimizer, step_size=config.step_size, gamma=config.step_gamma, last_epoch=last_step) 76 | elif config.scheduler == 'PolyLR': 77 | return PolyLR(optimizer, max_iter=config.max_iter, power=config.poly_power, last_step=last_step) 78 | elif config.scheduler == 'SquaredLR': 79 | return SquaredLR(optimizer, max_iter=config.max_iter, last_step=last_step) 80 | elif config.scheduler == 'ExpLR': 81 | return ExpLR( 82 | optimizer, step_size=config.exp_step_size, gamma=config.exp_gamma, last_step=last_step) 83 | else: 84 | logging.error('Scheduler not supported') 85 | -------------------------------------------------------------------------------- /downstream/insseg/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import models.resunet as resunet 8 | import models.res16unet as res16unet 9 | 10 | MODELS = [] 11 | 12 | 13 | def add_models(module): 14 | MODELS.extend([getattr(module, a) for a in dir(module) if 'Net' in a]) 15 | 16 | 17 | add_models(resunet) 18 | add_models(res16unet) 19 | 20 | def get_models(): 21 | '''Returns a tuple of sample models.''' 22 | return MODELS 23 | 24 | def load_model(name): 25 | '''Creates and returns an instance of the model given its class name. 26 | ''' 27 | # Find the model class from its name 28 | all_models = get_models() 29 | mdict = {model.__name__: model for model in all_models} 30 | if name not in mdict: 31 | print('Invalid model index. Options are:') 32 | # Display a list of valid model names 33 | for model in all_models: 34 | print('\t* {}'.format(model.__name__)) 35 | return None 36 | NetClass = mdict[name] 37 | 38 | return NetClass 39 | -------------------------------------------------------------------------------- /downstream/insseg/models/common.py: -------------------------------------------------------------------------------- 1 | import MinkowskiEngine as ME 2 | 3 | 4 | def get_norm(norm_type, num_feats, bn_momentum=0.05, D=-1): 5 | if norm_type == 'BN': 6 | return ME.MinkowskiBatchNorm(num_feats, momentum=bn_momentum) 7 | elif norm_type == 'IN': 8 | return ME.MinkowskiInstanceNorm(num_feats, dimension=D) 9 | else: 10 | raise ValueError(f'Type {norm_type}, not defined') 11 | -------------------------------------------------------------------------------- /downstream/insseg/models/model.py: -------------------------------------------------------------------------------- 1 | from MinkowskiEngine import MinkowskiNetwork 2 | 3 | 4 | class Model(MinkowskiNetwork): 5 | """ 6 | Base network for all sparse convnet 7 | 8 | By default, all networks are segmentation networks. 9 | """ 10 | OUT_PIXEL_DIST = -1 11 | 12 | def __init__(self, in_channels, out_channels, config, D, **kwargs): 13 | super(Model, self).__init__(D) 14 | self.in_channels = in_channels 15 | self.out_channels = out_channels 16 | self.config = config 17 | 18 | 19 | class HighDimensionalModel(Model): 20 | """ 21 | Base network for all spatio (temporal) chromatic sparse convnet 22 | """ 23 | 24 | def __init__(self, in_channels, out_channels, config, D, **kwargs): 25 | assert D > 4, "Num dimension smaller than 5" 26 | super(HighDimensionalModel, self).__init__(in_channels, out_channels, config, D, **kwargs) 27 | -------------------------------------------------------------------------------- /downstream/insseg/models/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/insseg/models/modules/__init__.py -------------------------------------------------------------------------------- /downstream/insseg/models/residual_block.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | from models.common import get_norm 4 | 5 | import MinkowskiEngine as ME 6 | import MinkowskiEngine.MinkowskiFunctional as MEF 7 | 8 | 9 | class BasicBlockBase(nn.Module): 10 | expansion = 1 11 | NORM_TYPE = 'BN' 12 | 13 | def __init__(self, 14 | inplanes, 15 | planes, 16 | stride=1, 17 | dilation=1, 18 | downsample=None, 19 | bn_momentum=0.1, 20 | D=3): 21 | super(BasicBlockBase, self).__init__() 22 | 23 | self.conv1 = ME.MinkowskiConvolution( 24 | inplanes, planes, kernel_size=3, stride=stride, dimension=D) 25 | self.norm1 = get_norm(self.NORM_TYPE, planes, bn_momentum=bn_momentum, D=D) 26 | self.conv2 = ME.MinkowskiConvolution( 27 | planes, 28 | planes, 29 | kernel_size=3, 30 | stride=1, 31 | dilation=dilation, 32 | has_bias=False, 33 | dimension=D) 34 | self.norm2 = get_norm(self.NORM_TYPE, planes, bn_momentum=bn_momentum, D=D) 35 | self.downsample = downsample 36 | 37 | def forward(self, x): 38 | residual = x 39 | 40 | out = self.conv1(x) 41 | out = self.norm1(out) 42 | out = MEF.relu(out) 43 | 44 | out = self.conv2(out) 45 | out = self.norm2(out) 46 | 47 | if self.downsample is not None: 48 | residual = self.downsample(x) 49 | 50 | out += residual 51 | out = MEF.relu(out) 52 | 53 | return out 54 | 55 | 56 | class BasicBlockBN(BasicBlockBase): 57 | NORM_TYPE = 'BN' 58 | 59 | 60 | class BasicBlockIN(BasicBlockBase): 61 | NORM_TYPE = 'IN' 62 | 63 | 64 | def get_block(norm_type, 65 | inplanes, 66 | planes, 67 | stride=1, 68 | dilation=1, 69 | downsample=None, 70 | bn_momentum=0.1, 71 | D=3): 72 | if norm_type == 'BN': 73 | return BasicBlockBN(inplanes, planes, stride, dilation, downsample, bn_momentum, D) 74 | elif norm_type == 'IN': 75 | return BasicBlockIN(inplanes, planes, stride, dilation, downsample, bn_momentum, D) 76 | else: 77 | raise ValueError(f'Type {norm_type}, not defined') 78 | -------------------------------------------------------------------------------- /downstream/insseg/models/wrapper.py: -------------------------------------------------------------------------------- 1 | import random 2 | from torch.nn import Module 3 | 4 | from MinkowskiEngine import SparseTensor 5 | 6 | 7 | class Wrapper(Module): 8 | """ 9 | Wrapper for the segmentation networks. 10 | """ 11 | OUT_PIXEL_DIST = -1 12 | 13 | def __init__(self, NetClass, in_nchannel, out_nchannel, config): 14 | super(Wrapper, self).__init__() 15 | self.initialize_filter(NetClass, in_nchannel, out_nchannel, config) 16 | 17 | def initialize_filter(self, NetClass, in_nchannel, out_nchannel, config): 18 | raise NotImplementedError('Must initialize a model and a filter') 19 | 20 | def forward(self, x, coords, colors=None): 21 | soutput = self.model(x) 22 | 23 | # During training, make the network invariant to the filter 24 | if not self.training or random.random() < 0.5: 25 | # Filter requires the model to finish the forward pass 26 | wrapper_coords = self.filter.initialize_coords(self.model, coords, colors) 27 | finput = SparseTensor(soutput.F, wrapper_coords) 28 | soutput = self.filter(finput) 29 | 30 | return soutput 31 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/data_efficient/by_points.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py -m \ 11 | train.train_phase=train \ 12 | train.is_train=True \ 13 | train.lenient_weight_loading=True \ 14 | train.stat_freq=2 \ 15 | train.val_freq=250 \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | augmentation.normalize_color=True \ 19 | data.dataset=ScannetVoxelization2cmDataset \ 20 | data.batch_size=48 \ 21 | data.sampled_inds=$SAMPLED_INDS \ 22 | data.num_workers=2 \ 23 | data.scannet_path=${DATAPATH} \ 24 | optimizer.lr=0.1 \ 25 | optimizer.scheduler=PolyLR \ 26 | optimizer.max_iter=10000 \ 27 | misc.log_dir=${LOG_DIR} \ 28 | misc.num_gpus=8 \ 29 | hydra.launcher.partition=priority \ 30 | hydra.launcher.comment=CVPR_Deadline \ 31 | net.weights=$PRETRAIN \ 32 | 33 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/data_efficient/by_scenes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.train_phase=train \ 12 | train.is_train=True \ 13 | train.lenient_weight_loading=True \ 14 | train.stat_freq=2 \ 15 | train.val_freq=250 \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | augmentation.normalize_color=True \ 19 | data.dataset=ScannetVoxelization2cmDataset \ 20 | data.train_file=$TRAIN_FILE \ 21 | data.batch_size=48 \ 22 | data.num_workers=2 \ 23 | data.scannet_path=${DATAPATH} \ 24 | optimizer.lr=0.1 \ 25 | optimizer.scheduler=PolyLR \ 26 | optimizer.max_iter=5000 \ 27 | misc.log_dir=${LOG_DIR} \ 28 | misc.num_gpus=8 \ 29 | hydra.launcher.partition=learnfair \ 30 | hydra.launcher.comment=CVPR_Deadline \ 31 | net.weights=$PRETRAIN \ 32 | 33 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/test_scannet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | MODEL=Res16UNet34C 8 | 9 | python ddp_main.py \ 10 | train.is_train=False \ 11 | train.lenient_weight_loading=True \ 12 | net.model=${MODEL} \ 13 | net.conv1_kernel_size=3 \ 14 | augmentation.normalize_color=True \ 15 | data.dataset=ScannetVoxelization2cmDataset \ 16 | data.num_workers=1 \ 17 | data.scannet_path=${DATAPATH} \ 18 | data.return_transformation=False \ 19 | test.test_phase=val \ 20 | test.evaluate_benchmark=False \ 21 | misc.log_dir=${LOG_DIR} \ 22 | misc.num_gpus=1 \ 23 | net.weights=$PRETRAIN \ 24 | 25 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/test_scannet_benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | MODEL=Res16UNet34C 8 | 9 | python ddp_main.py \ 10 | train.is_train=False \ 11 | train.val_freq=5 \ 12 | train.lenient_weight_loading=True \ 13 | net.model=${MODEL} \ 14 | net.conv1_kernel_size=3 \ 15 | augmentation.normalize_color=True \ 16 | data.dataset=ScannetVoxelization2cmDataset \ 17 | data.num_workers=1 \ 18 | data.scannet_path=${DATAPATH} \ 19 | data.return_transformation=True \ 20 | test.test_phase=test \ 21 | test.evaluate_benchmark=True \ 22 | test.dual_set_cluster=True \ 23 | misc.log_dir=${LOG_DIR} \ 24 | misc.num_gpus=1 \ 25 | net.weights=$PRETRAIN \ 26 | 27 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/test_stanford3d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.is_train=False \ 12 | train.lenient_weight_loading=True \ 13 | net.model=${MODEL} \ 14 | net.conv1_kernel_size=3 \ 15 | net.weights=$PRETRAIN \ 16 | data.dataset=StanfordArea5Dataset \ 17 | data.voxel_size=0.05 \ 18 | data.num_workers=1 \ 19 | data.stanford3d_path=${DATAPATH} \ 20 | augmentation.data_aug_color_trans_ratio=0.05 \ 21 | augmentation.data_aug_color_jitter_std=0.005 \ 22 | misc.log_dir=${LOG_DIR} \ 23 | misc.train_stuff=True \ 24 | hydra.launcher.partition=learnfair \ 25 | hydra.launcher.comment=CVPR_Deadline \ 26 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/train_scannet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | MODEL=Res16UNet34C 8 | 9 | python ddp_main.py \ 10 | train.train_phase=train \ 11 | train.val_phase=val \ 12 | train.is_train=True \ 13 | train.lenient_weight_loading=True \ 14 | train.stat_freq=5 \ 15 | train.val_freq=250 \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | augmentation.normalize_color=True \ 19 | data.dataset=ScannetVoxelization2cmDataset \ 20 | data.batch_size=48 \ 21 | data.num_workers=2 \ 22 | data.scannet_path=${DATAPATH} \ 23 | data.return_transformation=True \ 24 | optimizer.lr=0.1 \ 25 | optimizer.scheduler=PolyLR \ 26 | optimizer.max_iter=20000 \ 27 | misc.log_dir=${LOG_DIR} \ 28 | misc.num_gpus=8 \ 29 | hydra.launcher.partition=dev \ 30 | hydra.launcher.comment=CVPR_rebuttal \ 31 | net.weights=$PRETRAIN \ 32 | 33 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/train_scannet_benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | MODEL=Res16UNet34C 8 | 9 | python ddp_main.py \ 10 | train.train_phase=trainval \ 11 | train.val_phase=val \ 12 | train.is_train=True \ 13 | train.lenient_weight_loading=True \ 14 | train.stat_freq=5 \ 15 | train.val_freq=500 \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | augmentation.normalize_color=True \ 19 | data.dataset=ScannetVoxelization2cmDataset \ 20 | data.batch_size=48 \ 21 | data.num_workers=2 \ 22 | data.scannet_path=${DATAPATH} \ 23 | data.return_transformation=True \ 24 | optimizer.lr=0.1 \ 25 | optimizer.scheduler=PolyLR \ 26 | optimizer.max_iter=20000 \ 27 | misc.log_dir=${LOG_DIR} \ 28 | misc.num_gpus=8 \ 29 | hydra.launcher.partition=dev \ 30 | hydra.launcher.comment=CVPR_rebuttal \ 31 | net.weights=$PRETRAIN0 \ 32 | 33 | -------------------------------------------------------------------------------- /downstream/insseg/scripts/train_stanford3d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | MODEL=Res16UNet34C 8 | 9 | python ddp_main.py \ 10 | train.train_phase=train \ 11 | train.is_train=True \ 12 | train.lenient_weight_loading=True \ 13 | train.stat_freq=5 \ 14 | train.val_freq=200 \ 15 | train.overwrite_weights=False \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | data.dataset=StanfordArea5Dataset \ 19 | data.batch_size=48 \ 20 | data.voxel_size=0.05 \ 21 | data.num_workers=2 \ 22 | data.stanford3d_path=${DATAPATH} \ 23 | augmentation.data_aug_color_trans_ratio=0.05 \ 24 | augmentation.data_aug_color_jitter_std=0.005 \ 25 | optimizer.lr=0.1 \ 26 | optimizer.scheduler=PolyLR \ 27 | optimizer.max_iter=15000 \ 28 | misc.log_dir=${LOG_DIR} \ 29 | misc.num_gpus=8 \ 30 | misc.train_stuff=True \ 31 | hydra.launcher.partition=priority \ 32 | hydra.launcher.comment=CriticalEXP \ 33 | net.weights=$PRETRAIN \ 34 | -------------------------------------------------------------------------------- /downstream/semseg/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized 2 | __pycache__/ 3 | *.py[cod] 4 | *.o 5 | *.so 6 | 7 | # Text edits 8 | *.swp 9 | *.swo 10 | *.orig 11 | 12 | # Training 13 | outputs/ 14 | 15 | # Profiling 16 | callgrind.out* 17 | *.dSYM 18 | 19 | # Misc 20 | .DS_Store 21 | -------------------------------------------------------------------------------- /downstream/semseg/.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = chromium 3 | column_limit = 100 4 | indent_width = 2 5 | -------------------------------------------------------------------------------- /downstream/semseg/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | #from lib.datasets import synthia 2 | #from lib.datasets import shapenet 3 | from datasets import stanford 4 | from datasets import scannet 5 | 6 | DATASETS = [] 7 | 8 | 9 | def add_datasets(module): 10 | DATASETS.extend([getattr(module, a) for a in dir(module) if 'Dataset' in a]) 11 | 12 | 13 | add_datasets(stanford) 14 | #add_datasets(synthia) 15 | add_datasets(scannet) 16 | #add_datasets(shapenet) 17 | 18 | def load_dataset(name): 19 | '''Creates and returns an instance of the datasets given its name. 20 | ''' 21 | # Find the model class from its name 22 | mdict = {dataset.__name__: dataset for dataset in DATASETS} 23 | if name not in mdict: 24 | print('Invalid dataset index. Options are:') 25 | # Display a list of valid dataset names 26 | for dataset in DATASETS: 27 | print('\t* {}'.format(dataset.__name__)) 28 | raise ValueError(f'Dataset {name} not defined') 29 | DatasetClass = mdict[name] 30 | 31 | return DatasetClass 32 | -------------------------------------------------------------------------------- /downstream/semseg/datasets/dataloader.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.distributed as dist 5 | from torch.utils.data.sampler import Sampler 6 | 7 | 8 | class InfSampler(Sampler): 9 | """Samples elements randomly, without replacement. 10 | 11 | Arguments: 12 | data_source (Dataset): dataset to sample from 13 | """ 14 | 15 | def __init__(self, data_source, shuffle=False): 16 | self.data_source = data_source 17 | self.shuffle = shuffle 18 | self.reset_permutation() 19 | 20 | def reset_permutation(self): 21 | perm = len(self.data_source) 22 | if self.shuffle: 23 | perm = torch.randperm(perm) 24 | self._perm = perm.tolist() 25 | 26 | def __iter__(self): 27 | return self 28 | 29 | def __next__(self): 30 | if len(self._perm) == 0: 31 | self.reset_permutation() 32 | 33 | return self._perm.pop() 34 | 35 | def __len__(self): 36 | return len(self.data_source) 37 | 38 | next = __next__ # Python 2 compatibility 39 | 40 | 41 | class DistributedInfSampler(InfSampler): 42 | def __init__(self, data_source, num_replicas=None, rank=None, shuffle=True): 43 | if num_replicas is None: 44 | if not dist.is_available(): 45 | raise RuntimeError("Requires distributed package to be available") 46 | num_replicas = dist.get_world_size() 47 | if rank is None: 48 | if not dist.is_available(): 49 | raise RuntimeError("Requires distributed package to be available") 50 | rank = dist.get_rank() 51 | 52 | self.data_source = data_source 53 | self.num_replicas = num_replicas 54 | self.rank = rank 55 | self.epoch = 0 56 | self.it = 0 57 | self.num_samples = int(math.ceil(len(self.data_source) * 1.0 / self.num_replicas)) 58 | self.total_size = self.num_samples * self.num_replicas 59 | self.shuffle = shuffle 60 | self.reset_permutation() 61 | 62 | def __next__(self): 63 | it = self.it * self.num_replicas + self.rank 64 | value = self._perm[it % len(self._perm)] 65 | self.it = self.it + 1 66 | 67 | if (self.it * self.num_replicas) >= len(self._perm): 68 | self.reset_permutation() 69 | self.it = 0 70 | return value 71 | 72 | def __len__(self): 73 | return self.num_samples -------------------------------------------------------------------------------- /downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_instance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_instance.py -------------------------------------------------------------------------------- /downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_label.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._evaluate_semantic_label.py -------------------------------------------------------------------------------- /downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util.py -------------------------------------------------------------------------------- /downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util_3d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/._util_3d.py -------------------------------------------------------------------------------- /downstream/semseg/datasets/evaluation/scannet_benchmark_utils/scripts/util.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import csv 3 | try: 4 | import numpy as np 5 | except: 6 | print("Failed to import numpy package.") 7 | sys.exit(-1) 8 | try: 9 | import imageio 10 | except: 11 | print("Please install the module 'imageio' for image processing, e.g.") 12 | print("pip install imageio") 13 | sys.exit(-1) 14 | 15 | # print an error message and quit 16 | def print_error(message, user_fault=False): 17 | sys.stderr.write('ERROR: ' + str(message) + '\n') 18 | if user_fault: 19 | sys.exit(2) 20 | sys.exit(-1) 21 | 22 | 23 | # if string s represents an int 24 | def represents_int(s): 25 | try: 26 | int(s) 27 | return True 28 | except ValueError: 29 | return False 30 | 31 | 32 | def read_label_mapping(filename, label_from='raw_category', label_to='nyu40id'): 33 | assert os.path.isfile(filename) 34 | mapping = dict() 35 | with open(filename) as csvfile: 36 | reader = csv.DictReader(csvfile, delimiter='\t') 37 | for row in reader: 38 | mapping[row[label_from]] = int(row[label_to]) 39 | # if ints convert 40 | if represents_int([key for key in mapping.keys()][0]): 41 | mapping = {int(k):v for k,v in mapping.items()} 42 | return mapping 43 | 44 | 45 | # input: scene_types.txt or scene_types_all.txt 46 | def read_scene_types_mapping(filename, remove_spaces=True): 47 | assert os.path.isfile(filename) 48 | mapping = dict() 49 | lines = open(filename).read().splitlines() 50 | lines = [line.split('\t') for line in lines] 51 | if remove_spaces: 52 | mapping = { x[1].strip():int(x[0]) for x in lines } 53 | else: 54 | mapping = { x[1]:int(x[0]) for x in lines } 55 | return mapping 56 | 57 | 58 | # color by label 59 | def visualize_label_image(filename, image): 60 | height = image.shape[0] 61 | width = image.shape[1] 62 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 63 | color_palette = create_color_palette() 64 | for idx, color in enumerate(color_palette): 65 | vis_image[image==idx] = color 66 | imageio.imwrite(filename, vis_image) 67 | 68 | 69 | # color by different instances (mod length of color palette) 70 | def visualize_instance_image(filename, image): 71 | height = image.shape[0] 72 | width = image.shape[1] 73 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 74 | color_palette = create_color_palette() 75 | instances = np.unique(image) 76 | for idx, inst in enumerate(instances): 77 | vis_image[image==inst] = color_palette[inst%len(color_palette)] 78 | imageio.imwrite(filename, vis_image) 79 | 80 | -------------------------------------------------------------------------------- /downstream/semseg/datasets/evaluation/scannet_benchmark_utils/util.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import csv 3 | try: 4 | import numpy as np 5 | except: 6 | print("Failed to import numpy package.") 7 | sys.exit(-1) 8 | try: 9 | import imageio 10 | except: 11 | print("Please install the module 'imageio' for image processing, e.g.") 12 | print("pip install imageio") 13 | sys.exit(-1) 14 | 15 | # print an error message and quit 16 | def print_error(message, user_fault=False): 17 | sys.stderr.write('ERROR: ' + str(message) + '\n') 18 | if user_fault: 19 | sys.exit(2) 20 | sys.exit(-1) 21 | 22 | 23 | # if string s represents an int 24 | def represents_int(s): 25 | try: 26 | int(s) 27 | return True 28 | except ValueError: 29 | return False 30 | 31 | 32 | def read_label_mapping(filename, label_from='raw_category', label_to='nyu40id'): 33 | assert os.path.isfile(filename) 34 | mapping = dict() 35 | with open(filename) as csvfile: 36 | reader = csv.DictReader(csvfile, delimiter='\t') 37 | for row in reader: 38 | mapping[row[label_from]] = int(row[label_to]) 39 | # if ints convert 40 | if represents_int([key for key in mapping.keys()][0]): 41 | mapping = {int(k):v for k,v in mapping.items()} 42 | return mapping 43 | 44 | 45 | # input: scene_types.txt or scene_types_all.txt 46 | def read_scene_types_mapping(filename, remove_spaces=True): 47 | assert os.path.isfile(filename) 48 | mapping = dict() 49 | lines = open(filename).read().splitlines() 50 | lines = [line.split('\t') for line in lines] 51 | if remove_spaces: 52 | mapping = { x[1].strip():int(x[0]) for x in lines } 53 | else: 54 | mapping = { x[1]:int(x[0]) for x in lines } 55 | return mapping 56 | 57 | 58 | # color by label 59 | def visualize_label_image(filename, image): 60 | height = image.shape[0] 61 | width = image.shape[1] 62 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 63 | color_palette = create_color_palette() 64 | for idx, color in enumerate(color_palette): 65 | vis_image[image==idx] = color 66 | imageio.imwrite(filename, vis_image) 67 | 68 | 69 | # color by different instances (mod length of color palette) 70 | def visualize_instance_image(filename, image): 71 | height = image.shape[0] 72 | width = image.shape[1] 73 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 74 | color_palette = create_color_palette() 75 | instances = np.unique(image) 76 | for idx, inst in enumerate(instances): 77 | vis_image[image==inst] = color_palette[inst%len(color_palette)] 78 | imageio.imwrite(filename, vis_image) 79 | 80 | -------------------------------------------------------------------------------- /downstream/semseg/datasets/preprocessing/scannet/collect_indoor3d_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import plyfile 4 | import json 5 | import time 6 | import torch 7 | import argparse 8 | import numpy as np 9 | 10 | def get_raw2scannet_label_map(): 11 | lines = [line.rstrip() for line in open('scannetv2-labels.combined.tsv')] 12 | lines = lines[1:] 13 | raw2scannet = {} 14 | for i in range(len(lines)): 15 | elements = lines[i].split('\t') 16 | # raw_name = elements[0] 17 | # nyu40_name = elements[6] 18 | raw_name = elements[1] 19 | nyu40_id = elements[4] 20 | nyu40_name = elements[7] 21 | raw2scannet[raw_name] = nyu40_id 22 | return raw2scannet 23 | g_raw2scannet = get_raw2scannet_label_map() 24 | RAW2SCANNET = g_raw2scannet 25 | 26 | def parse_args(): 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument('--input', default='/canis/Datasets/ScanNet/public/v2/scans/') 29 | parser.add_argument('--output', default='./output') 30 | opt = parser.parse_args() 31 | return opt 32 | 33 | def main(config): 34 | for scene_name in os.listdir(config.input): 35 | print(scene_name) 36 | # Over-segmented segments: maps from segment to vertex/point IDs 37 | segid_to_pointid = {} 38 | segfile = os.path.join(config.input, scene_name, '%s_vh_clean_2.0.010000.segs.json'%(scene_name)) 39 | with open(segfile) as jsondata: 40 | d = json.load(jsondata) 41 | seg = d['segIndices'] 42 | for i in range(len(seg)): 43 | if seg[i] not in segid_to_pointid: 44 | segid_to_pointid[seg[i]] = [] 45 | segid_to_pointid[seg[i]].append(i) 46 | 47 | # Raw points in XYZRGBA 48 | ply_filename = os.path.join(config.input, scene_name, '%s_vh_clean_2.ply' % (scene_name)) 49 | f = plyfile.PlyData().read(ply_filename) 50 | points = np.array([list(x) for x in f.elements[0]]) 51 | 52 | # Instances over-segmented segment IDs: annotation on segments 53 | instance_segids = [] 54 | labels = [] 55 | annotation_filename = os.path.join(config.input, scene_name, '%s.aggregation.json'%(scene_name)) 56 | with open(annotation_filename) as jsondata: 57 | d = json.load(jsondata) 58 | for x in d['segGroups']: 59 | instance_segids.append(x['segments']) 60 | labels.append(x['label']) 61 | 62 | 63 | # Each instance's points 64 | instance_labels = np.zeros(points.shape[0]) 65 | semantic_labels = np.zeros(points.shape[0]) 66 | for i in range(len(instance_segids)): 67 | segids = instance_segids[i] 68 | pointids = [] 69 | for segid in segids: 70 | pointids += segid_to_pointid[segid] 71 | pointids = np.array(pointids) 72 | instance_labels[pointids] = i+1 73 | semantic_labels[pointids] = RAW2SCANNET[labels[i]] 74 | 75 | colors = points[:,3:6] 76 | points = points[:,0:3] # XYZ+RGB+NORMAL 77 | torch.save((points, colors, semantic_labels, instance_labels), os.path.join(config.output, scene_name+'.pth')) 78 | 79 | if __name__=='__main__': 80 | config = parse_args() 81 | os.makedirs(config.output, exist_ok=True) 82 | main(config) 83 | -------------------------------------------------------------------------------- /downstream/semseg/datasets/preprocessing/scannet/split/scannetv2_test.txt: -------------------------------------------------------------------------------- 1 | scene0707_00 2 | scene0708_00 3 | scene0709_00 4 | scene0710_00 5 | scene0711_00 6 | scene0712_00 7 | scene0713_00 8 | scene0714_00 9 | scene0715_00 10 | scene0716_00 11 | scene0717_00 12 | scene0718_00 13 | scene0719_00 14 | scene0720_00 15 | scene0721_00 16 | scene0722_00 17 | scene0723_00 18 | scene0724_00 19 | scene0725_00 20 | scene0726_00 21 | scene0727_00 22 | scene0728_00 23 | scene0729_00 24 | scene0730_00 25 | scene0731_00 26 | scene0732_00 27 | scene0733_00 28 | scene0734_00 29 | scene0735_00 30 | scene0736_00 31 | scene0737_00 32 | scene0738_00 33 | scene0739_00 34 | scene0740_00 35 | scene0741_00 36 | scene0742_00 37 | scene0743_00 38 | scene0744_00 39 | scene0745_00 40 | scene0746_00 41 | scene0747_00 42 | scene0748_00 43 | scene0749_00 44 | scene0750_00 45 | scene0751_00 46 | scene0752_00 47 | scene0753_00 48 | scene0754_00 49 | scene0755_00 50 | scene0756_00 51 | scene0757_00 52 | scene0758_00 53 | scene0759_00 54 | scene0760_00 55 | scene0761_00 56 | scene0762_00 57 | scene0763_00 58 | scene0764_00 59 | scene0765_00 60 | scene0766_00 61 | scene0767_00 62 | scene0768_00 63 | scene0769_00 64 | scene0770_00 65 | scene0771_00 66 | scene0772_00 67 | scene0773_00 68 | scene0774_00 69 | scene0775_00 70 | scene0776_00 71 | scene0777_00 72 | scene0778_00 73 | scene0779_00 74 | scene0780_00 75 | scene0781_00 76 | scene0782_00 77 | scene0783_00 78 | scene0784_00 79 | scene0785_00 80 | scene0786_00 81 | scene0787_00 82 | scene0788_00 83 | scene0789_00 84 | scene0790_00 85 | scene0791_00 86 | scene0792_00 87 | scene0793_00 88 | scene0794_00 89 | scene0795_00 90 | scene0796_00 91 | scene0797_00 92 | scene0798_00 93 | scene0799_00 94 | scene0800_00 95 | scene0801_00 96 | scene0802_00 97 | scene0803_00 98 | scene0804_00 99 | scene0805_00 100 | scene0806_00 101 | -------------------------------------------------------------------------------- /downstream/semseg/ddp_main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import os 7 | import sys 8 | import hydra 9 | import torch 10 | 11 | from lib.ddp_trainer import SegmentationTrainer 12 | from lib.distributed import multi_proc_run 13 | 14 | def single_proc_run(config): 15 | if not torch.cuda.is_available(): 16 | raise Exception('No GPUs FOUND.') 17 | torch.manual_seed(config.misc.seed) 18 | torch.cuda.manual_seed(config.misc.seed) 19 | trainer = SegmentationTrainer(config) 20 | if config.train.is_train: 21 | trainer.train() 22 | else: 23 | trainer.test() 24 | 25 | @hydra.main(config_path='config', config_name='default.yaml') 26 | def main(config): 27 | # Convert to dict 28 | if config.misc.num_gpus > 1: 29 | multi_proc_run(config.misc.num_gpus, fun=single_proc_run, fun_args=(config,)) 30 | else: 31 | single_proc_run(config) 32 | 33 | if __name__ == '__main__': 34 | __spec__ = None 35 | os.environ['MKL_THREADING_LAYER'] = 'GNU' 36 | main() 37 | -------------------------------------------------------------------------------- /downstream/semseg/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /downstream/semseg/lib/io3d.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import numpy as np 8 | import trimesh 9 | 10 | # color palette for nyu40 labels 11 | def create_color_palette(): 12 | return [ 13 | (0, 0, 0), 14 | (174, 199, 232), # wall 15 | (152, 223, 138), # floor 16 | (31, 119, 180), # cabinet 17 | (255, 187, 120), # bed 18 | (188, 189, 34), # chair 19 | (140, 86, 75), # sofa 20 | (255, 152, 150), # table 21 | (214, 39, 40), # door 22 | (197, 176, 213), # window 23 | (148, 103, 189), # bookshelf 24 | (196, 156, 148), # picture 25 | (23, 190, 207), # counter 26 | (178, 76, 76), 27 | (247, 182, 210), # desk 28 | (66, 188, 102), 29 | (219, 219, 141), # curtain 30 | (140, 57, 197), 31 | (202, 185, 52), 32 | (51, 176, 203), 33 | (200, 54, 131), 34 | (92, 193, 61), 35 | (78, 71, 183), 36 | (172, 114, 82), 37 | (255, 127, 14), # refrigerator 38 | (91, 163, 138), 39 | (153, 98, 156), 40 | (140, 153, 101), 41 | (158, 218, 229), # shower curtain 42 | (100, 125, 154), 43 | (178, 127, 135), 44 | (120, 185, 128), 45 | (146, 111, 194), 46 | (44, 160, 44), # toilet 47 | (112, 128, 144), # sink 48 | (96, 207, 209), 49 | (227, 119, 194), # bathtub 50 | (213, 92, 176), 51 | (94, 106, 211), 52 | (82, 84, 163), # otherfurn 53 | (100, 85, 144), 54 | ] 55 | 56 | def write_triangle_mesh(vertices, colors, faces, outputFile): 57 | mesh = trimesh.Trimesh(vertices=vertices, vertex_colors=colors, faces=faces, process=False) 58 | mesh.export(outputFile) 59 | 60 | def read_triangle_mesh(filename): 61 | mesh = trimesh.load_mesh(filename, process=False) 62 | if isinstance(mesh, trimesh.PointCloud): 63 | vertices = mesh.vertices 64 | colors = mesh.colors 65 | faces = None 66 | elif isinstance(mesh, trimesh.Trimesh): 67 | vertices = mesh.vertices 68 | colors = mesh.visual.vertex_colors 69 | faces = mesh.faces 70 | 71 | return vertices, colors, faces 72 | -------------------------------------------------------------------------------- /downstream/semseg/lib/layers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | 8 | import torch 9 | import torch.nn as nn 10 | 11 | from MinkowskiEngine import MinkowskiGlobalPooling, MinkowskiBroadcastAddition, MinkowskiBroadcastMultiplication 12 | 13 | 14 | class MinkowskiLayerNorm(nn.Module): 15 | 16 | def __init__(self, num_features, eps=1e-5, D=-1): 17 | super(MinkowskiLayerNorm, self).__init__() 18 | self.num_features = num_features 19 | self.eps = eps 20 | self.weight = nn.Parameter(torch.ones(1, num_features)) 21 | self.bias = nn.Parameter(torch.zeros(1, num_features)) 22 | 23 | self.mean_in = MinkowskiGlobalPooling(dimension=D) 24 | self.glob_sum = MinkowskiBroadcastAddition(dimension=D) 25 | self.glob_sum2 = MinkowskiBroadcastAddition(dimension=D) 26 | self.glob_mean = MinkowskiGlobalPooling(dimension=D) 27 | self.glob_times = MinkowskiBroadcastMultiplication(dimension=D) 28 | self.D = D 29 | self.reset_parameters() 30 | 31 | def __repr__(self): 32 | s = f'(D={self.D})' 33 | return self.__class__.__name__ + s 34 | 35 | def reset_parameters(self): 36 | self.weight.data.fill_(1) 37 | self.bias.data.zero_() 38 | 39 | def _check_input_dim(self, input): 40 | if input.F.dim() != 2: 41 | raise ValueError('expected 2D input (got {}D input)'.format(input.dim())) 42 | 43 | def forward(self, x): 44 | self._check_input_dim(x) 45 | mean = self.mean_in(x).F.mean(-1, keepdim=True) 46 | mean = mean + torch.zeros(mean.size(0), self.num_features).type_as(mean) 47 | temp = self.glob_sum(x.F, -mean)**2 48 | var = self.glob_mean(temp.data).mean(-1, keepdim=True) 49 | var = var + torch.zeros(var.size(0), self.num_features).type_as(var) 50 | instd = 1 / (var + self.eps).sqrt() 51 | 52 | x = self.glob_times(self.glob_sum2(x, -mean), instd) 53 | return x * self.weight + self.bias 54 | 55 | 56 | class MinkowskiInstanceNorm(nn.Module): 57 | 58 | def __init__(self, num_features, eps=1e-5, D=-1): 59 | super(MinkowskiInstanceNorm, self).__init__() 60 | self.eps = eps 61 | self.weight = nn.Parameter(torch.ones(1, num_features)) 62 | self.bias = nn.Parameter(torch.zeros(1, num_features)) 63 | 64 | self.mean_in = MinkowskiGlobalPooling(dimension=D) 65 | self.glob_sum = MinkowskiBroadcastAddition(dimension=D) 66 | self.glob_sum2 = MinkowskiBroadcastAddition(dimension=D) 67 | self.glob_mean = MinkowskiGlobalPooling(dimension=D) 68 | self.glob_times = MinkowskiBroadcastMultiplication(dimension=D) 69 | self.D = D 70 | self.reset_parameters() 71 | 72 | def __repr__(self): 73 | s = f'(pixel_dist={self.pixel_dist}, D={self.D})' 74 | return self.__class__.__name__ + s 75 | 76 | def reset_parameters(self): 77 | self.weight.data.fill_(1) 78 | self.bias.data.zero_() 79 | 80 | def _check_input_dim(self, input): 81 | if input.dim() != 2: 82 | raise ValueError('expected 2D input (got {}D input)'.format(input.dim())) 83 | 84 | def forward(self, x): 85 | self._check_input_dim(x) 86 | mean_in = self.mean_in(x) 87 | temp = self.glob_sum(x, -mean_in)**2 88 | var_in = self.glob_mean(temp.data) 89 | instd_in = 1 / (var_in + self.eps).sqrt() 90 | 91 | x = self.glob_times(self.glob_sum2(x, -mean_in), instd_in) 92 | return x * self.weight + self.bias 93 | -------------------------------------------------------------------------------- /downstream/semseg/lib/math_functions.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from scipy.sparse import csr_matrix 8 | import torch 9 | 10 | 11 | class SparseMM(torch.autograd.Function): 12 | """ 13 | Sparse x dense matrix multiplication with autograd support. 14 | Implementation by Soumith Chintala: 15 | https://discuss.pytorch.org/t/ 16 | does-pytorch-support-autograd-on-sparse-matrix/6156/7 17 | """ 18 | 19 | def forward(self, matrix1, matrix2): 20 | self.save_for_backward(matrix1, matrix2) 21 | return torch.mm(matrix1, matrix2) 22 | 23 | def backward(self, grad_output): 24 | matrix1, matrix2 = self.saved_tensors 25 | grad_matrix1 = grad_matrix2 = None 26 | 27 | if self.needs_input_grad[0]: 28 | grad_matrix1 = torch.mm(grad_output, matrix2.t()) 29 | 30 | if self.needs_input_grad[1]: 31 | grad_matrix2 = torch.mm(matrix1.t(), grad_output) 32 | 33 | return grad_matrix1, grad_matrix2 34 | 35 | 36 | def sparse_float_tensor(values, indices, size=None): 37 | """ 38 | Return a torch sparse matrix give values and indices (row_ind, col_ind). 39 | If the size is an integer, return a square matrix with side size. 40 | If the size is a torch.Size, use it to initialize the out tensor. 41 | If none, the size is inferred. 42 | """ 43 | indices = torch.stack(indices).int() 44 | sargs = [indices, values.float()] 45 | if size is not None: 46 | # Use the provided size 47 | if isinstance(size, int): 48 | size = torch.Size((size, size)) 49 | sargs.append(size) 50 | if values.is_cuda: 51 | return torch.cuda.sparse.FloatTensor(*sargs) 52 | else: 53 | return torch.sparse.FloatTensor(*sargs) 54 | 55 | 56 | def diags(values, size=None): 57 | values = values.view(-1) 58 | n = values.nelement() 59 | size = torch.Size((n, n)) 60 | indices = (torch.arange(0, n), torch.arange(0, n)) 61 | return sparse_float_tensor(values, indices, size) 62 | 63 | 64 | def sparse_to_csr_matrix(tensor): 65 | tensor = tensor.cpu() 66 | inds = tensor._indices().numpy() 67 | vals = tensor._values().numpy() 68 | return csr_matrix((vals, (inds[0], inds[1])), shape=[s for s in tensor.shape]) 69 | 70 | 71 | def csr_matrix_to_sparse(mat): 72 | row_ind, col_ind = mat.nonzero() 73 | return sparse_float_tensor( 74 | torch.from_numpy(mat.data), 75 | (torch.from_numpy(row_ind), torch.from_numpy(col_ind)), 76 | size=torch.Size(mat.shape)) 77 | -------------------------------------------------------------------------------- /downstream/semseg/lib/solvers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import logging 8 | from torch.optim import SGD, Adam 9 | from torch.optim.lr_scheduler import LambdaLR, StepLR 10 | 11 | 12 | class LambdaStepLR(LambdaLR): 13 | 14 | def __init__(self, optimizer, lr_lambda, last_step=-1): 15 | super(LambdaStepLR, self).__init__(optimizer, lr_lambda, last_step) 16 | 17 | @property 18 | def last_step(self): 19 | """Use last_epoch for the step counter""" 20 | return self.last_epoch 21 | 22 | @last_step.setter 23 | def last_step(self, v): 24 | self.last_epoch = v 25 | 26 | 27 | class PolyLR(LambdaStepLR): 28 | """DeepLab learning rate policy""" 29 | 30 | def __init__(self, optimizer, max_iter, power=0.9, last_step=-1): 31 | super(PolyLR, self).__init__(optimizer, lambda s: (1 - s / (max_iter + 1))**power, last_step) 32 | 33 | 34 | class SquaredLR(LambdaStepLR): 35 | """ Used for SGD Lars""" 36 | 37 | def __init__(self, optimizer, max_iter, last_step=-1): 38 | super(SquaredLR, self).__init__(optimizer, lambda s: (1 - s / (max_iter + 1))**2, last_step) 39 | 40 | 41 | class ExpLR(LambdaStepLR): 42 | 43 | def __init__(self, optimizer, step_size, gamma=0.9, last_step=-1): 44 | # (0.9 ** 21.854) = 0.1, (0.95 ** 44.8906) = 0.1 45 | # To get 0.1 every N using gamma 0.9, N * log(0.9)/log(0.1) = 0.04575749 N 46 | # To get 0.1 every N using gamma g, g ** N = 0.1 -> N * log(g) = log(0.1) -> g = np.exp(log(0.1) / N) 47 | super(ExpLR, self).__init__(optimizer, lambda s: gamma**(s / step_size), last_step) 48 | 49 | 50 | def initialize_optimizer(params, config): 51 | assert config.optimizer in ['SGD', 'Adagrad', 'Adam', 'RMSProp', 'Rprop', 'SGDLars'] 52 | 53 | if config.optimizer == 'SGD': 54 | return SGD( 55 | params, 56 | lr=config.lr, 57 | momentum=config.sgd_momentum, 58 | dampening=config.sgd_dampening, 59 | weight_decay=config.weight_decay) 60 | elif config.optimizer == 'Adam': 61 | return Adam( 62 | params, 63 | lr=config.lr, 64 | betas=(config.adam_beta1, config.adam_beta2), 65 | weight_decay=config.weight_decay) 66 | else: 67 | logging.error('Optimizer type not supported') 68 | raise ValueError('Optimizer type not supported') 69 | 70 | 71 | def initialize_scheduler(optimizer, config, last_step=-1): 72 | if config.scheduler == 'StepLR': 73 | return StepLR( 74 | optimizer, step_size=config.step_size, gamma=config.step_gamma, last_epoch=last_step) 75 | elif config.scheduler == 'PolyLR': 76 | return PolyLR(optimizer, max_iter=config.max_iter, power=config.poly_power, last_step=last_step) 77 | elif config.scheduler == 'SquaredLR': 78 | return SquaredLR(optimizer, max_iter=config.max_iter, last_step=last_step) 79 | elif config.scheduler == 'ExpLR': 80 | return ExpLR( 81 | optimizer, step_size=config.exp_step_size, gamma=config.exp_gamma, last_step=last_step) 82 | else: 83 | logging.error('Scheduler not supported') 84 | -------------------------------------------------------------------------------- /downstream/semseg/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import models.resunet as resunet 7 | import models.res16unet as res16unet 8 | 9 | MODELS = [] 10 | 11 | def add_models(module): 12 | MODELS.extend([getattr(module, a) for a in dir(module) if 'Net' in a]) 13 | add_models(resunet) 14 | add_models(res16unet) 15 | 16 | def get_models(): 17 | '''Returns a tuple of sample models.''' 18 | return MODELS 19 | 20 | def load_model(name): 21 | '''Creates and returns an instance of the model given its class name. 22 | ''' 23 | # Find the model class from its name 24 | all_models = get_models() 25 | mdict = {model.__name__: model for model in all_models} 26 | if name not in mdict: 27 | print('Invalid model index. Options are:') 28 | # Display a list of valid model names 29 | for model in all_models: 30 | print('\t* {}'.format(model.__name__)) 31 | return None 32 | NetClass = mdict[name] 33 | 34 | return NetClass 35 | 36 | -------------------------------------------------------------------------------- /downstream/semseg/models/common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import MinkowskiEngine as ME 8 | 9 | 10 | def get_norm(norm_type, num_feats, bn_momentum=0.05, D=-1): 11 | if norm_type == 'BN': 12 | return ME.MinkowskiBatchNorm(num_feats, momentum=bn_momentum) 13 | elif norm_type == 'IN': 14 | return ME.MinkowskiInstanceNorm(num_feats, dimension=D) 15 | else: 16 | raise ValueError(f'Type {norm_type}, not defined') 17 | -------------------------------------------------------------------------------- /downstream/semseg/models/model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from MinkowskiEngine import MinkowskiNetwork 7 | 8 | 9 | class Model(MinkowskiNetwork): 10 | """ 11 | Base network for all sparse convnet 12 | 13 | By default, all networks are segmentation networks. 14 | """ 15 | OUT_PIXEL_DIST = -1 16 | 17 | def __init__(self, in_channels, out_channels, config, D, **kwargs): 18 | super(Model, self).__init__(D) 19 | self.in_channels = in_channels 20 | self.out_channels = out_channels 21 | self.config = config 22 | 23 | 24 | class HighDimensionalModel(Model): 25 | """ 26 | Base network for all spatio (temporal) chromatic sparse convnet 27 | """ 28 | 29 | def __init__(self, in_channels, out_channels, config, D, **kwargs): 30 | assert D > 4, "Num dimension smaller than 5" 31 | super(HighDimensionalModel, self).__init__(in_channels, out_channels, config, D, **kwargs) 32 | -------------------------------------------------------------------------------- /downstream/semseg/models/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /downstream/semseg/models/residual_block.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch.nn as nn 8 | 9 | from models.common import get_norm 10 | 11 | import MinkowskiEngine as ME 12 | import MinkowskiEngine.MinkowskiFunctional as MEF 13 | 14 | 15 | class BasicBlockBase(nn.Module): 16 | expansion = 1 17 | NORM_TYPE = 'BN' 18 | 19 | def __init__(self, 20 | inplanes, 21 | planes, 22 | stride=1, 23 | dilation=1, 24 | downsample=None, 25 | bn_momentum=0.1, 26 | D=3): 27 | super(BasicBlockBase, self).__init__() 28 | 29 | self.conv1 = ME.MinkowskiConvolution( 30 | inplanes, planes, kernel_size=3, stride=stride, dimension=D) 31 | self.norm1 = get_norm(self.NORM_TYPE, planes, bn_momentum=bn_momentum, D=D) 32 | self.conv2 = ME.MinkowskiConvolution( 33 | planes, 34 | planes, 35 | kernel_size=3, 36 | stride=1, 37 | dilation=dilation, 38 | has_bias=False, 39 | dimension=D) 40 | self.norm2 = get_norm(self.NORM_TYPE, planes, bn_momentum=bn_momentum, D=D) 41 | self.downsample = downsample 42 | 43 | def forward(self, x): 44 | residual = x 45 | 46 | out = self.conv1(x) 47 | out = self.norm1(out) 48 | out = MEF.relu(out) 49 | 50 | out = self.conv2(out) 51 | out = self.norm2(out) 52 | 53 | if self.downsample is not None: 54 | residual = self.downsample(x) 55 | 56 | out += residual 57 | out = MEF.relu(out) 58 | 59 | return out 60 | 61 | 62 | class BasicBlockBN(BasicBlockBase): 63 | NORM_TYPE = 'BN' 64 | 65 | 66 | class BasicBlockIN(BasicBlockBase): 67 | NORM_TYPE = 'IN' 68 | 69 | 70 | def get_block(norm_type, 71 | inplanes, 72 | planes, 73 | stride=1, 74 | dilation=1, 75 | downsample=None, 76 | bn_momentum=0.1, 77 | D=3): 78 | if norm_type == 'BN': 79 | return BasicBlockBN(inplanes, planes, stride, dilation, downsample, bn_momentum, D) 80 | elif norm_type == 'IN': 81 | return BasicBlockIN(inplanes, planes, stride, dilation, downsample, bn_momentum, D) 82 | else: 83 | raise ValueError(f'Type {norm_type}, not defined') 84 | -------------------------------------------------------------------------------- /downstream/semseg/models/wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import random 8 | from torch.nn import Module 9 | 10 | from MinkowskiEngine import SparseTensor 11 | 12 | 13 | class Wrapper(Module): 14 | """ 15 | Wrapper for the segmentation networks. 16 | """ 17 | OUT_PIXEL_DIST = -1 18 | 19 | def __init__(self, NetClass, in_nchannel, out_nchannel, config): 20 | super(Wrapper, self).__init__() 21 | self.initialize_filter(NetClass, in_nchannel, out_nchannel, config) 22 | 23 | def initialize_filter(self, NetClass, in_nchannel, out_nchannel, config): 24 | raise NotImplementedError('Must initialize a model and a filter') 25 | 26 | def forward(self, x, coords, colors=None): 27 | soutput = self.model(x) 28 | 29 | # During training, make the network invariant to the filter 30 | if not self.training or random.random() < 0.5: 31 | # Filter requires the model to finish the forward pass 32 | wrapper_coords = self.filter.initialize_coords(self.model, coords, colors) 33 | finput = SparseTensor(soutput.F, wrapper_coords) 34 | soutput = self.filter(finput) 35 | 36 | return soutput 37 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/data_efficient/by_points.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | #!/bin/bash 7 | 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.train_phase=train \ 12 | train.is_train=True \ 13 | train.lenient_weight_loading=True \ 14 | train.stat_freq=2 \ 15 | train.val_freq=200 \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | augmentation.normalize_color=True \ 19 | data.dataset=ScannetVoxelization2cmDataset \ 20 | data.batch_size=48 \ 21 | data.sampled_inds=$SAMPLED_INDS \ 22 | data.num_workers=1 \ 23 | data.num_val_workers=1 \ 24 | data.scannet_path=${DATAPATH} \ 25 | data.return_transformation=False \ 26 | optimizer.lr=0.1 \ 27 | optimizer.scheduler=PolyLR \ 28 | optimizer.max_iter=10000 \ 29 | misc.log_dir=${LOG_DIR} \ 30 | misc.num_gpus=8 \ 31 | hydra.launcher.partition=priority \ 32 | hydra.launcher.comment=CVPR_Deadline \ 33 | net.weights=$PRETRAIN \ 34 | 35 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/data_efficient/by_scenes.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | #!/bin/bash 7 | 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.train_phase=train \ 12 | train.val_phase=val \ 13 | train.is_train=True \ 14 | train.lenient_weight_loading=True \ 15 | train.stat_freq=2 \ 16 | train.val_freq=200 \ 17 | net.model=${MODEL} \ 18 | net.conv1_kernel_size=3 \ 19 | augmentation.normalize_color=True \ 20 | data.dataset=ScannetVoxelization2cmDataset \ 21 | data.train_file=$TRAIN_FILE \ 22 | data.batch_size=6 \ 23 | data.num_workers=2 \ 24 | data.scannet_path=${DATAPATH} \ 25 | data.return_transformation=False \ 26 | optimizer.lr=0.8 \ 27 | optimizer.scheduler=PolyLR \ 28 | optimizer.max_iter=5000 \ 29 | misc.log_dir=${LOG_DIR} \ 30 | misc.num_gpus=1 \ 31 | hydra.launcher.partition=learnfair \ 32 | hydra.launcher.comment=CVPR_Deadline \ 33 | net.weights=$PRETRAIN \ 34 | 35 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/inference_features.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/bin/bash 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.is_train=False \ 12 | train.lenient_weight_loading=True \ 13 | net.model=${MODEL} \ 14 | net.weights=$PRETRAIN \ 15 | net.conv1_kernel_size=3 \ 16 | augmentation.normalize_color=True \ 17 | data.dataset=ScannetVoxelization2cmDataset \ 18 | data.num_workers=1 \ 19 | data.scannet_path=${DATAPATH} \ 20 | data.return_transformation=True \ 21 | test.save_features=True \ 22 | test.test_phase=train \ 23 | misc.log_dir=${LOG_DIR} \ 24 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/test_scannet.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/bin/bash 8 | 9 | MODEL=Res16UNet34C 10 | 11 | python ddp_main.py \ 12 | train.is_train=False \ 13 | train.val_freq=5 \ 14 | train.lenient_weight_loading=True \ 15 | net.model=${MODEL} \ 16 | net.conv1_kernel_size=3 \ 17 | augmentation.normalize_color=True \ 18 | data.dataset=ScannetVoxelization2cmDataset \ 19 | data.batch_size=6 \ 20 | data.num_workers=1 \ 21 | data.scannet_path=${DATAPATH} \ 22 | test.test_phase=val \ 23 | misc.log_dir=${LOG_DIR} \ 24 | misc.num_gpus=1 \ 25 | net.weights=$PRETRAIN \ 26 | 27 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/test_stanford3d.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/bin/bash 8 | 9 | MODEL=Res16UNet34C 10 | 11 | python ddp_main.py \ 12 | train.is_train=False \ 13 | train.lenient_weight_loading=True \ 14 | net.model=${MODEL} \ 15 | net.conv1_kernel_size=3 \ 16 | net.weights=$PRETRAIN \ 17 | data.dataset=StanfordArea5Dataset \ 18 | data.voxel_size=0.05 \ 19 | data.num_workers=2 \ 20 | data.stanford3d_path=${DATAPATH} \ 21 | augmentation.data_aug_color_trans_ratio=0.05 \ 22 | augmentation.data_aug_color_jitter_std=0.005 \ 23 | misc.log_dir=${LOG_DIR} \ 24 | hydra.launcher.partition=learnfair \ 25 | hydra.launcher.comment=CVPR_Deadline \ 26 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/train_scannet.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/bin/bash 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.train_phase=train \ 12 | train.val_phase=val \ 13 | train.is_train=True \ 14 | train.lenient_weight_loading=True \ 15 | train.stat_freq=2 \ 16 | train.val_freq=500 \ 17 | net.model=${MODEL} \ 18 | net.conv1_kernel_size=3 \ 19 | augmentation.normalize_color=True \ 20 | data.dataset=ScannetVoxelization2cmDataset \ 21 | data.batch_size=48 \ 22 | data.num_workers=2 \ 23 | data.scannet_path=${DATAPATH} \ 24 | data.return_transformation=False \ 25 | optimizer.lr=0.8 \ 26 | optimizer.scheduler=PolyLR \ 27 | optimizer.max_iter=20000 \ 28 | misc.log_dir=${LOG_DIR} \ 29 | misc.num_gpus=8 \ 30 | hydra.launcher.partition=dev \ 31 | hydra.launcher.comment=CVPR_rebuttal \ 32 | net.weights=$PRETRAIN \ 33 | 34 | -------------------------------------------------------------------------------- /downstream/semseg/scripts/train_stanford3d.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/bin/bash 8 | MODEL=Res16UNet34C 9 | 10 | python ddp_main.py \ 11 | train.train_phase=train \ 12 | train.is_train=True \ 13 | train.lenient_weight_loading=True \ 14 | train.stat_freq=5 \ 15 | train.val_freq=200 \ 16 | net.model=${MODEL} \ 17 | net.conv1_kernel_size=3 \ 18 | data.dataset=StanfordArea5Dataset \ 19 | data.batch_size=48 \ 20 | data.voxel_size=0.05 \ 21 | data.num_workers=2 \ 22 | data.stanford3d_path=${DATAPATH} \ 23 | augmentation.data_aug_color_trans_ratio=0.05 \ 24 | augmentation.data_aug_color_jitter_std=0.005 \ 25 | optimizer.lr=0.1 \ 26 | optimizer.scheduler=PolyLR \ 27 | optimizer.max_iter=60000 \ 28 | misc.log_dir=${LOG_DIR} \ 29 | misc.num_gpus=8 \ 30 | misc.train_stuff=True \ 31 | hydra.launcher.partition=priority \ 32 | hydra.launcher.comment=CriticalEXP \ 33 | net.weights=$PRETRAIN \ 34 | -------------------------------------------------------------------------------- /downstream/votenet/config/default.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | defaults: 7 | - hydra/launcher: submitit_slurm 8 | - hydra/hydra_logging: colorlog 9 | 10 | net: 11 | model: votenet 12 | is_train: True 13 | backbone: pointnet2 # sparseconv 14 | weights: 15 | num_target: 256 16 | vote_factor: 1 17 | cluster_sampling: vote_fps 18 | 19 | data: 20 | dataset: sunrgbd 21 | by_scenes: 22 | by_points: 23 | num_points: 20000 24 | num_workers: 4 25 | batch_size: 8 26 | no_height: False 27 | use_color: False 28 | use_sunrgbd_v2: False 29 | overwrite: False 30 | dump_results: False 31 | voxelization: False 32 | voxel_size: 0.05 33 | shuffle_datast: False 34 | 35 | optimizer: 36 | max_epoch: 180 37 | learning_rate: 0.001 38 | lr_decay_steps: [80,120,160] 39 | lr_decay_rates: [0.1,0.1,0.1] 40 | weight_decay: 0 41 | bn_decay_step: 80 42 | bn_decay_rate: 0.5 43 | 44 | test: 45 | use_3d_nms: True 46 | use_cls_nms: True 47 | use_old_type_nms: False 48 | per_class_proposal: True 49 | nms_iou: 0.25 50 | conf_thresh: 0.05 51 | faster_eval: False, #Faster evaluation by skippling empty bounding box removal. 52 | ap_iou_thresholds: [0.25, 0.5] 53 | save_vis: False 54 | ap_iou: 0.5 55 | write_to_benchmark: False 56 | phase: val 57 | 58 | misc: 59 | log_dir: log 60 | num_gpus: 1 61 | seed: 0 62 | 63 | hydra: 64 | run: 65 | dir: ${misc.log_dir} 66 | sweep: 67 | dir: ${misc.log_dir} 68 | launcher: 69 | partition: debug 70 | submitit_folder: ${hydra.sweep.dir}/.submitit/%j 71 | name: ${hydra.job.name} 72 | timeout_min: 5200 73 | cpus_per_task: 10 74 | gpus_per_node: ${misc.num_gpus} 75 | tasks_per_node: 1 76 | mem_gb: 64 77 | nodes: 1 78 | max_num_timeout: 3 79 | exclude: balrog,gimli,pegasus,char 80 | comment: criticalEXP 81 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/evaluation/metric_util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ Utility functions for metric evaluation. 7 | 8 | Author: Or Litany and Charles R. Qi 9 | """ 10 | 11 | import os 12 | import sys 13 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 14 | sys.path.append(BASE_DIR) 15 | 16 | import numpy as np 17 | 18 | def calc_iou(box_a, box_b): 19 | """Computes IoU of two axis aligned bboxes. 20 | Args: 21 | box_a, box_b: 6D of center and lengths 22 | Returns: 23 | iou 24 | """ 25 | 26 | max_a = box_a[3:6] 27 | max_b = box_b[3:6] 28 | min_max = np.array([max_a, max_b]).min(0) 29 | 30 | min_a = box_a[0:3] 31 | min_b = box_b[0:3] 32 | max_min = np.array([min_a, min_b]).max(0) 33 | if not ((min_max > max_min).all()): 34 | return 0.0 35 | 36 | intersection = (min_max - max_min).prod() 37 | vol_a = (box_a[3:6] - box_a[0:3]).prod() 38 | vol_b = (box_b[3:6] - box_b[0:3]).prod() 39 | union = vol_a + vol_b - intersection 40 | return 1.0*intersection / union 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/scannet/README.md: -------------------------------------------------------------------------------- 1 | ### Prepare ScanNet Data 2 | 3 | 1. Download ScanNet v2 data [HERE](https://github.com/ScanNet/ScanNet). Move/link the `scans` folder such that under `scans` there should be folders with names such as `scene0001_01`. 4 | 5 | 2. Extract point clouds and annotations (semantic seg, instance seg etc.) by running `python batch_load_scannet_data.py`, which will create a folder named `scannet_train_detection_data` here. 6 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/scannet/data_viz.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import sys 7 | import os 8 | 9 | BASE_DIR = os.path.dirname(__file__) 10 | sys.path.append(BASE_DIR) 11 | 12 | import numpy as np 13 | import pc_util 14 | 15 | scene_name = 'scannet_train_detection_data/scene0002_00' 16 | output_folder = 'data_viz_dump' 17 | 18 | data = np.load(scene_name+'_vert.npy') 19 | scene_points = data[:,0:3] 20 | colors = data[:,3:] 21 | instance_labels = np.load(scene_name+'_ins_label.npy') 22 | semantic_labels = np.load(scene_name+'_sem_label.npy') 23 | instance_bboxes = np.load(scene_name+'_bbox.npy') 24 | 25 | print(np.unique(instance_labels)) 26 | print(np.unique(semantic_labels)) 27 | input() 28 | if not os.path.exists(output_folder): 29 | os.mkdir(output_folder) 30 | 31 | # Write scene as OBJ file for visualization 32 | pc_util.write_ply_rgb(scene_points, colors, os.path.join(output_folder, 'scene.obj')) 33 | pc_util.write_ply_color(scene_points, instance_labels, os.path.join(output_folder, 'scene_instance.obj')) 34 | pc_util.write_ply_color(scene_points, semantic_labels, os.path.join(output_folder, 'scene_semantic.obj')) 35 | 36 | from model_util_scannet import ScannetDatasetConfig 37 | DC = ScannetDatasetConfig() 38 | print(instance_bboxes.shape) 39 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/scannet/meta_data/scannet_means.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/datasets/scannet/meta_data/scannet_means.npz -------------------------------------------------------------------------------- /downstream/votenet/datasets/scannet/meta_data/scannetv2_test.txt: -------------------------------------------------------------------------------- 1 | scene0707_00 2 | scene0708_00 3 | scene0709_00 4 | scene0710_00 5 | scene0711_00 6 | scene0712_00 7 | scene0713_00 8 | scene0714_00 9 | scene0715_00 10 | scene0716_00 11 | scene0717_00 12 | scene0718_00 13 | scene0719_00 14 | scene0720_00 15 | scene0721_00 16 | scene0722_00 17 | scene0723_00 18 | scene0724_00 19 | scene0725_00 20 | scene0726_00 21 | scene0727_00 22 | scene0728_00 23 | scene0729_00 24 | scene0730_00 25 | scene0731_00 26 | scene0732_00 27 | scene0733_00 28 | scene0734_00 29 | scene0735_00 30 | scene0736_00 31 | scene0737_00 32 | scene0738_00 33 | scene0739_00 34 | scene0740_00 35 | scene0741_00 36 | scene0742_00 37 | scene0743_00 38 | scene0744_00 39 | scene0745_00 40 | scene0746_00 41 | scene0747_00 42 | scene0748_00 43 | scene0749_00 44 | scene0750_00 45 | scene0751_00 46 | scene0752_00 47 | scene0753_00 48 | scene0754_00 49 | scene0755_00 50 | scene0756_00 51 | scene0757_00 52 | scene0758_00 53 | scene0759_00 54 | scene0760_00 55 | scene0761_00 56 | scene0762_00 57 | scene0763_00 58 | scene0764_00 59 | scene0765_00 60 | scene0766_00 61 | scene0767_00 62 | scene0768_00 63 | scene0769_00 64 | scene0770_00 65 | scene0771_00 66 | scene0772_00 67 | scene0773_00 68 | scene0774_00 69 | scene0775_00 70 | scene0776_00 71 | scene0777_00 72 | scene0778_00 73 | scene0779_00 74 | scene0780_00 75 | scene0781_00 76 | scene0782_00 77 | scene0783_00 78 | scene0784_00 79 | scene0785_00 80 | scene0786_00 81 | scene0787_00 82 | scene0788_00 83 | scene0789_00 84 | scene0790_00 85 | scene0791_00 86 | scene0792_00 87 | scene0793_00 88 | scene0794_00 89 | scene0795_00 90 | scene0796_00 91 | scene0797_00 92 | scene0798_00 93 | scene0799_00 94 | scene0800_00 95 | scene0801_00 96 | scene0802_00 97 | scene0803_00 98 | scene0804_00 99 | scene0805_00 100 | scene0806_00 101 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/scannet/scannet_train_detection_data: -------------------------------------------------------------------------------- 1 | /rhome/jhou/data/dataset/scannet/detection_data -------------------------------------------------------------------------------- /downstream/votenet/datasets/scannet/scannet_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | ''' Ref: https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts ''' 7 | import os 8 | import sys 9 | import json 10 | import csv 11 | 12 | try: 13 | import numpy as np 14 | except: 15 | print("Failed to import numpy package.") 16 | sys.exit(-1) 17 | 18 | try: 19 | from plyfile import PlyData, PlyElement 20 | except: 21 | print("Please install the module 'plyfile' for PLY i/o, e.g.") 22 | print("pip install plyfile") 23 | sys.exit(-1) 24 | 25 | def represents_int(s): 26 | ''' if string s represents an int. ''' 27 | try: 28 | int(s) 29 | return True 30 | except ValueError: 31 | return False 32 | 33 | 34 | def read_label_mapping(filename, label_from='raw_category', label_to='nyu40id'): 35 | assert os.path.isfile(filename) 36 | mapping = dict() 37 | with open(filename) as csvfile: 38 | reader = csv.DictReader(csvfile, delimiter='\t') 39 | for row in reader: 40 | mapping[row[label_from]] = int(row[label_to]) 41 | if represents_int(list(mapping.keys())[0]): 42 | mapping = {int(k):v for k,v in mapping.items()} 43 | return mapping 44 | 45 | def read_mesh_vertices(filename): 46 | """ read XYZ for each vertex. 47 | """ 48 | assert os.path.isfile(filename) 49 | with open(filename, 'rb') as f: 50 | plydata = PlyData.read(f) 51 | num_verts = plydata['vertex'].count 52 | vertices = np.zeros(shape=[num_verts, 3], dtype=np.float32) 53 | vertices[:,0] = plydata['vertex'].data['x'] 54 | vertices[:,1] = plydata['vertex'].data['y'] 55 | vertices[:,2] = plydata['vertex'].data['z'] 56 | return vertices 57 | 58 | def read_mesh_vertices_rgb(filename): 59 | """ read XYZ RGB for each vertex. 60 | Note: RGB values are in 0-255 61 | """ 62 | assert os.path.isfile(filename) 63 | with open(filename, 'rb') as f: 64 | plydata = PlyData.read(f) 65 | num_verts = plydata['vertex'].count 66 | vertices = np.zeros(shape=[num_verts, 6], dtype=np.float32) 67 | vertices[:,0] = plydata['vertex'].data['x'] 68 | vertices[:,1] = plydata['vertex'].data['y'] 69 | vertices[:,2] = plydata['vertex'].data['z'] 70 | vertices[:,3] = plydata['vertex'].data['red'] 71 | vertices[:,4] = plydata['vertex'].data['green'] 72 | vertices[:,5] = plydata['vertex'].data['blue'] 73 | return vertices 74 | 75 | 76 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/sunrgbd/README.md: -------------------------------------------------------------------------------- 1 | ### Prepare SUN RGB-D Data 2 | 3 | 1. Download SUNRGBD v2 data [HERE](http://rgbd.cs.princeton.edu/data/) (SUNRGBD.zip, SUNRGBDMeta2DBB_v2.mat, SUNRGBDMeta3DBB_v2.mat) and the toolkits (SUNRGBDtoolbox.zip). Move all the downloaded files under OFFICIAL_SUNRGBD. Unzip the zip files. 4 | 5 | 2. Extract point clouds and annotations (class, v2 2D -- xmin,ymin,xmax,ymax, and 3D bounding boxes -- centroids, size, 2D heading) by running `extract_split.m`, `extract_rgbd_data_v2.m` and `extract_rgbd_data_v1.m` under the `matlab` folder. 6 | 7 | 3. Prepare data by running `python sunrgbd_data.py --gen_v1_data` 8 | 9 | You can also examine and visualize the data with `python sunrgbd_data.py --viz` and use MeshLab to view the generated PLY files at `data_viz_dump`. 10 | 11 | NOTE: SUNRGBDtoolbox.zip should have MD5 hash `18d22e1761d36352f37232cba102f91f` (you can check the hash with `md5 SUNRGBDtoolbox.zip` on Mac OS or `md5sum SUNRGBDtoolbox.zip` on Linux) 12 | -------------------------------------------------------------------------------- /downstream/votenet/datasets/sunrgbd/__pycache__/sunrgbd_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/datasets/sunrgbd/__pycache__/sunrgbd_utils.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/datasets/sunrgbd/sunrgbd_pc_bbox_votes_50k_v1_train: -------------------------------------------------------------------------------- 1 | /rhome/jhou/data/dataset/sunrgbd/sunrgbd_pc_bbox_votes_50k_v1_train -------------------------------------------------------------------------------- /downstream/votenet/datasets/sunrgbd/sunrgbd_pc_bbox_votes_50k_v1_val: -------------------------------------------------------------------------------- 1 | /rhome/jhou/data/dataset/sunrgbd/sunrgbd_pc_bbox_votes_50k_v1_val -------------------------------------------------------------------------------- /downstream/votenet/ddp_main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import os 7 | import random 8 | import torch 9 | import hydra 10 | import numpy as np 11 | from lib.ddp_trainer import DetectionTrainer 12 | from lib.distributed import multi_proc_run 13 | 14 | def single_proc_run(config): 15 | if not torch.cuda.is_available(): 16 | raise Exception('No GPUs FOUND.') 17 | trainer = DetectionTrainer(config) 18 | if config.net.is_train: 19 | trainer.train() 20 | else: 21 | trainer.test() 22 | 23 | @hydra.main(config_path='config', config_name='default.yaml') 24 | def main(config): 25 | # fix seed 26 | np.random.seed(config.misc.seed) 27 | torch.manual_seed(config.misc.seed) 28 | torch.cuda.manual_seed(config.misc.seed) 29 | port = random.randint(10001, 20001) 30 | 31 | if config.misc.num_gpus > 1: 32 | multi_proc_run(config.misc.num_gpus, port, fun=single_proc_run, fun_args=(config,)) 33 | else: 34 | single_proc_run(config) 35 | 36 | if __name__ == '__main__': 37 | __spec__ = None 38 | os.environ['MKL_THREADING_LAYER'] = 'GNU' 39 | main() 40 | -------------------------------------------------------------------------------- /downstream/votenet/lib/utils/nn_distance.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ Chamfer distance in Pytorch. 7 | Author: Charles R. Qi 8 | """ 9 | 10 | import torch 11 | import torch.nn as nn 12 | import numpy as np 13 | 14 | 15 | def huber_loss(error, delta=1.0): 16 | """ 17 | Args: 18 | error: Torch tensor (d1,d2,...,dk) 19 | Returns: 20 | loss: Torch tensor (d1,d2,...,dk) 21 | 22 | x = error = pred - gt or dist(pred,gt) 23 | 0.5 * |x|^2 if |x|<=d 24 | 0.5 * d^2 + d * (|x|-d) if |x|>d 25 | Ref: https://github.com/charlesq34/frustum-pointnets/blob/master/models/model_util.py 26 | """ 27 | abs_error = torch.abs(error) 28 | #quadratic = torch.min(abs_error, torch.FloatTensor([delta])) 29 | quadratic = torch.clamp(abs_error, max=delta) 30 | linear = (abs_error - quadratic) 31 | loss = 0.5 * quadratic**2 + delta * linear 32 | return loss 33 | 34 | def nn_distance(pc1, pc2, l1smooth=False, delta=1.0, l1=False): 35 | """ 36 | Input: 37 | pc1: (B,N,C) torch tensor 38 | pc2: (B,M,C) torch tensor 39 | l1smooth: bool, whether to use l1smooth loss 40 | delta: scalar, the delta used in l1smooth loss 41 | Output: 42 | dist1: (B,N) torch float32 tensor 43 | idx1: (B,N) torch int64 tensor 44 | dist2: (B,M) torch float32 tensor 45 | idx2: (B,M) torch int64 tensor 46 | """ 47 | N = pc1.shape[1] 48 | M = pc2.shape[1] 49 | pc1_expand_tile = pc1.unsqueeze(2).repeat(1,1,M,1) 50 | pc2_expand_tile = pc2.unsqueeze(1).repeat(1,N,1,1) 51 | pc_diff = pc1_expand_tile - pc2_expand_tile 52 | 53 | if l1smooth: 54 | pc_dist = torch.sum(huber_loss(pc_diff, delta), dim=-1) # (B,N,M) 55 | elif l1: 56 | pc_dist = torch.sum(torch.abs(pc_diff), dim=-1) # (B,N,M) 57 | else: 58 | pc_dist = torch.sum(pc_diff**2, dim=-1) # (B,N,M) 59 | dist1, idx1 = torch.min(pc_dist, dim=2) # (B,N) 60 | dist2, idx2 = torch.min(pc_dist, dim=1) # (B,M) 61 | return dist1, idx1, dist2, idx2 62 | 63 | def demo_nn_distance(): 64 | np.random.seed(0) 65 | pc1arr = np.random.random((1,5,3)) 66 | pc2arr = np.random.random((1,6,3)) 67 | pc1 = torch.from_numpy(pc1arr.astype(np.float32)) 68 | pc2 = torch.from_numpy(pc2arr.astype(np.float32)) 69 | dist1, idx1, dist2, idx2 = nn_distance(pc1, pc2) 70 | print(dist1) 71 | print(idx1) 72 | dist = np.zeros((5,6)) 73 | for i in range(5): 74 | for j in range(6): 75 | dist[i,j] = np.sum((pc1arr[0,i,:] - pc2arr[0,j,:]) ** 2) 76 | print(dist) 77 | print('-'*30) 78 | print('L1smooth dists:') 79 | dist1, idx1, dist2, idx2 = nn_distance(pc1, pc2, True) 80 | print(dist1) 81 | print(idx1) 82 | dist = np.zeros((5,6)) 83 | for i in range(5): 84 | for j in range(6): 85 | error = np.abs(pc1arr[0,i,:] - pc2arr[0,j,:]) 86 | quad = np.minimum(error, 1.0) 87 | linear = error - quad 88 | loss = 0.5*quad**2 + 1.0*linear 89 | dist[i,j] = np.sum(loss) 90 | print(dist) 91 | 92 | 93 | if __name__ == '__main__': 94 | demo_nn_distance() 95 | -------------------------------------------------------------------------------- /downstream/votenet/lib/utils/tf_logger.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import tensorflow as tf 7 | import numpy as np 8 | import scipy.misc 9 | try: 10 | from StringIO import StringIO # Python 2.7 11 | except ImportError: 12 | from io import BytesIO # Python 3.x 13 | 14 | 15 | class Logger(object): 16 | 17 | def __init__(self, log_dir): 18 | """Create a summary writer logging to log_dir.""" 19 | self.writer = tf.summary.FileWriter(log_dir) 20 | 21 | def scalar_summary(self, tag, value, step): 22 | """Log a scalar variable.""" 23 | summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)]) 24 | self.writer.add_summary(summary, step) 25 | 26 | def image_summary(self, tag, images, step): 27 | """Log a list of images.""" 28 | 29 | img_summaries = [] 30 | for i, img in enumerate(images): 31 | # Write the image to a string 32 | try: 33 | s = StringIO() 34 | except: 35 | s = BytesIO() 36 | scipy.misc.toimage(img).save(s, format="png") 37 | 38 | # Create an Image object 39 | img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), 40 | height=img.shape[0], 41 | width=img.shape[1]) 42 | # Create a Summary value 43 | img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum)) 44 | 45 | # Create and write Summary 46 | summary = tf.Summary(value=img_summaries) 47 | self.writer.add_summary(summary, step) 48 | 49 | def histo_summary(self, tag, values, step, bins=1000): 50 | """Log a histogram of the tensor of values.""" 51 | 52 | # Create a histogram using numpy 53 | counts, bin_edges = np.histogram(values, bins=bins) 54 | 55 | # Fill the fields of the histogram proto 56 | hist = tf.HistogramProto() 57 | hist.min = float(np.min(values)) 58 | hist.max = float(np.max(values)) 59 | hist.num = int(np.prod(values.shape)) 60 | hist.sum = float(np.sum(values)) 61 | hist.sum_squares = float(np.sum(values**2)) 62 | 63 | # Drop the start of the first bin 64 | bin_edges = bin_edges[1:] 65 | 66 | # Add bin edges and counts 67 | for edge in bin_edges: 68 | hist.bucket_limit.append(edge) 69 | for c in counts: 70 | hist.bucket.append(c) 71 | 72 | # Create and write Summary 73 | summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) 74 | self.writer.add_summary(summary, step) 75 | self.writer.flush() 76 | -------------------------------------------------------------------------------- /downstream/votenet/lib/utils/tf_visualizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | '''Code adapted from https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix''' 7 | import os 8 | import time 9 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 10 | import sys 11 | sys.path.append(BASE_DIR) 12 | import tf_logger 13 | 14 | 15 | class Visualizer(): 16 | def __init__(self, opt, name='train'): 17 | # self.opt = opt 18 | #self.logger = tf_logger.Logger(os.path.join(opt.logging_dir, opt.name)) 19 | #self.log_name = os.path.join(opt.checkpoint_dir, opt.name, 'loss_log.txt') 20 | self.logger = tf_logger.Logger(os.path.join(opt.log_dir, name)) 21 | self.log_name = os.path.join(opt.log_dir, 'tf_visualizer_log.txt') 22 | with open(self.log_name, "a") as log_file: 23 | now = time.strftime("%c") 24 | log_file.write('================ Training Loss (%s) ================\n' % now) 25 | 26 | # |visuals|: dictionary of images to save 27 | def log_images(self, visuals, step): 28 | for label, image_numpy in visuals.items(): 29 | self.logger.image_summary( 30 | label, [image_numpy], step) 31 | 32 | # scalars: dictionary of scalar labels and values 33 | def log_scalars(self, scalars, step): 34 | for label, val in scalars.items(): 35 | self.logger.scalar_summary(label, val, step) 36 | 37 | # scatter plots 38 | def plot_current_points(self, points, disp_offset=10): 39 | pass 40 | 41 | # scalars: same format as |scalars| of plot_current_scalars 42 | def print_current_scalars(self, epoch, i, scalars): 43 | message = '(epoch: %d, iters: %d) ' % (epoch, i) 44 | for k, v in scalars.items(): 45 | message += '%s: %.3f ' % (k, v) 46 | 47 | print(message) 48 | with open(self.log_name, "a") as log_file: 49 | log_file.write('%s\n' % message) 50 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/__pycache__/pointnet2_modules.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/pointnet2/__pycache__/pointnet2_modules.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/__pycache__/pointnet2_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/pointnet2/__pycache__/pointnet2_utils.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/__pycache__/pytorch_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/pointnet2/__pycache__/pytorch_utils.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/include/ball_query.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor ball_query(at::Tensor new_xyz, at::Tensor xyz, const float radius, 5 | const int nsample); 6 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/include/cuda_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef _CUDA_UTILS_H 2 | #define _CUDA_UTILS_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #define TOTAL_THREADS 512 14 | 15 | inline int opt_n_threads(int work_size) { 16 | const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0); 17 | 18 | return max(min(1 << pow_2, TOTAL_THREADS), 1); 19 | } 20 | 21 | inline dim3 opt_block_config(int x, int y) { 22 | const int x_threads = opt_n_threads(x); 23 | const int y_threads = 24 | max(min(opt_n_threads(y), TOTAL_THREADS / x_threads), 1); 25 | dim3 block_config(x_threads, y_threads, 1); 26 | 27 | return block_config; 28 | } 29 | 30 | #define CUDA_CHECK_ERRORS() \ 31 | do { \ 32 | cudaError_t err = cudaGetLastError(); \ 33 | if (cudaSuccess != err) { \ 34 | fprintf(stderr, "CUDA kernel failed : %s\n%s at L:%d in %s\n", \ 35 | cudaGetErrorString(err), __PRETTY_FUNCTION__, __LINE__, \ 36 | __FILE__); \ 37 | exit(-1); \ 38 | } \ 39 | } while (0) 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/include/group_points.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor group_points(at::Tensor points, at::Tensor idx); 5 | at::Tensor group_points_grad(at::Tensor grad_out, at::Tensor idx, const int n); 6 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/include/interpolate.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | std::vector three_nn(at::Tensor unknowns, at::Tensor knows); 7 | at::Tensor three_interpolate(at::Tensor points, at::Tensor idx, 8 | at::Tensor weight); 9 | at::Tensor three_interpolate_grad(at::Tensor grad_out, at::Tensor idx, 10 | at::Tensor weight, const int m); 11 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/include/sampling.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor gather_points(at::Tensor points, at::Tensor idx); 5 | at::Tensor gather_points_grad(at::Tensor grad_out, at::Tensor idx, const int n); 6 | at::Tensor furthest_point_sampling(at::Tensor points, const int nsamples); 7 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/include/utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #define CHECK_CUDA(x) \ 6 | do { \ 7 | AT_ASSERT(x.is_cuda(), #x " must be a CUDA tensor"); \ 8 | } while (0) 9 | 10 | #define CHECK_CONTIGUOUS(x) \ 11 | do { \ 12 | AT_ASSERT(x.is_contiguous(), #x " must be a contiguous tensor"); \ 13 | } while (0) 14 | 15 | #define CHECK_IS_INT(x) \ 16 | do { \ 17 | AT_ASSERT(x.scalar_type() == at::ScalarType::Int, \ 18 | #x " must be an int tensor"); \ 19 | } while (0) 20 | 21 | #define CHECK_IS_FLOAT(x) \ 22 | do { \ 23 | AT_ASSERT(x.scalar_type() == at::ScalarType::Float, \ 24 | #x " must be a float tensor"); \ 25 | } while (0) 26 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/src/ball_query.cpp: -------------------------------------------------------------------------------- 1 | #include "ball_query.h" 2 | #include "utils.h" 3 | 4 | void query_ball_point_kernel_wrapper(int b, int n, int m, float radius, 5 | int nsample, const float *new_xyz, 6 | const float *xyz, int *idx); 7 | 8 | at::Tensor ball_query(at::Tensor new_xyz, at::Tensor xyz, const float radius, 9 | const int nsample) { 10 | CHECK_CONTIGUOUS(new_xyz); 11 | CHECK_CONTIGUOUS(xyz); 12 | CHECK_IS_FLOAT(new_xyz); 13 | CHECK_IS_FLOAT(xyz); 14 | 15 | if (new_xyz.is_cuda()) { 16 | CHECK_CUDA(xyz); 17 | } 18 | 19 | at::Tensor idx = 20 | torch::zeros({new_xyz.size(0), new_xyz.size(1), nsample}, 21 | at::device(new_xyz.device()).dtype(at::ScalarType::Int)); 22 | 23 | if (new_xyz.is_cuda()) { 24 | query_ball_point_kernel_wrapper(xyz.size(0), xyz.size(1), new_xyz.size(1), 25 | radius, nsample, new_xyz.data_ptr(), 26 | xyz.data_ptr(), idx.data_ptr()); 27 | } else { 28 | AT_ASSERT(false, "CPU not supported"); 29 | } 30 | 31 | return idx; 32 | } 33 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/src/ball_query_gpu.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "cuda_utils.h" 6 | 7 | // input: new_xyz(b, m, 3) xyz(b, n, 3) 8 | // output: idx(b, m, nsample) 9 | __global__ void query_ball_point_kernel(int b, int n, int m, float radius, 10 | int nsample, 11 | const float *__restrict__ new_xyz, 12 | const float *__restrict__ xyz, 13 | int *__restrict__ idx) { 14 | int batch_index = blockIdx.x; 15 | xyz += batch_index * n * 3; 16 | new_xyz += batch_index * m * 3; 17 | idx += m * nsample * batch_index; 18 | 19 | int index = threadIdx.x; 20 | int stride = blockDim.x; 21 | 22 | float radius2 = radius * radius; 23 | for (int j = index; j < m; j += stride) { 24 | float new_x = new_xyz[j * 3 + 0]; 25 | float new_y = new_xyz[j * 3 + 1]; 26 | float new_z = new_xyz[j * 3 + 2]; 27 | for (int k = 0, cnt = 0; k < n && cnt < nsample; ++k) { 28 | float x = xyz[k * 3 + 0]; 29 | float y = xyz[k * 3 + 1]; 30 | float z = xyz[k * 3 + 2]; 31 | float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + 32 | (new_z - z) * (new_z - z); 33 | if (d2 < radius2) { 34 | if (cnt == 0) { 35 | for (int l = 0; l < nsample; ++l) { 36 | idx[j * nsample + l] = k; 37 | } 38 | } 39 | idx[j * nsample + cnt] = k; 40 | ++cnt; 41 | } 42 | } 43 | } 44 | } 45 | 46 | void query_ball_point_kernel_wrapper(int b, int n, int m, float radius, 47 | int nsample, const float *new_xyz, 48 | const float *xyz, int *idx) { 49 | cudaStream_t stream = at::cuda::getCurrentCUDAStream(); 50 | query_ball_point_kernel<<>>( 51 | b, n, m, radius, nsample, new_xyz, xyz, idx); 52 | 53 | CUDA_CHECK_ERRORS(); 54 | } 55 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/src/bindings.cpp: -------------------------------------------------------------------------------- 1 | #include "ball_query.h" 2 | #include "group_points.h" 3 | #include "interpolate.h" 4 | #include "sampling.h" 5 | 6 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 7 | m.def("gather_points", &gather_points); 8 | m.def("gather_points_grad", &gather_points_grad); 9 | m.def("furthest_point_sampling", &furthest_point_sampling); 10 | 11 | m.def("three_nn", &three_nn); 12 | m.def("three_interpolate", &three_interpolate); 13 | m.def("three_interpolate_grad", &three_interpolate_grad); 14 | 15 | m.def("ball_query", &ball_query); 16 | 17 | m.def("group_points", &group_points); 18 | m.def("group_points_grad", &group_points_grad); 19 | } 20 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/src/group_points.cpp: -------------------------------------------------------------------------------- 1 | #include "group_points.h" 2 | #include "utils.h" 3 | 4 | void group_points_kernel_wrapper(int b, int c, int n, int npoints, int nsample, 5 | const float *points, const int *idx, 6 | float *out); 7 | 8 | void group_points_grad_kernel_wrapper(int b, int c, int n, int npoints, 9 | int nsample, const float *grad_out, 10 | const int *idx, float *grad_points); 11 | 12 | at::Tensor group_points(at::Tensor points, at::Tensor idx) { 13 | CHECK_CONTIGUOUS(points); 14 | CHECK_CONTIGUOUS(idx); 15 | CHECK_IS_FLOAT(points); 16 | CHECK_IS_INT(idx); 17 | 18 | if (points.is_cuda()) { 19 | CHECK_CUDA(idx); 20 | } 21 | 22 | at::Tensor output = 23 | torch::zeros({points.size(0), points.size(1), idx.size(1), idx.size(2)}, 24 | at::device(points.device()).dtype(at::ScalarType::Float)); 25 | 26 | if (points.is_cuda()) { 27 | group_points_kernel_wrapper(points.size(0), points.size(1), points.size(2), 28 | idx.size(1), idx.size(2), 29 | points.data_ptr(), idx.data_ptr(), 30 | output.data_ptr()); 31 | } else { 32 | AT_ASSERT(false, "CPU not supported"); 33 | } 34 | 35 | return output; 36 | } 37 | 38 | at::Tensor group_points_grad(at::Tensor grad_out, at::Tensor idx, const int n) { 39 | CHECK_CONTIGUOUS(grad_out); 40 | CHECK_CONTIGUOUS(idx); 41 | CHECK_IS_FLOAT(grad_out); 42 | CHECK_IS_INT(idx); 43 | 44 | if (grad_out.is_cuda()) { 45 | CHECK_CUDA(idx); 46 | } 47 | 48 | at::Tensor output = 49 | torch::zeros({grad_out.size(0), grad_out.size(1), n}, 50 | at::device(grad_out.device()).dtype(at::ScalarType::Float)); 51 | 52 | if (grad_out.is_cuda()) { 53 | group_points_grad_kernel_wrapper( 54 | grad_out.size(0), grad_out.size(1), n, idx.size(1), idx.size(2), 55 | grad_out.data_ptr(), idx.data_ptr(), 56 | output.data_ptr()); 57 | } else { 58 | AT_ASSERT(false, "CPU not supported"); 59 | } 60 | 61 | return output; 62 | } 63 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/src/group_points_gpu.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "cuda_utils.h" 5 | 6 | // input: points(b, c, n) idx(b, npoints, nsample) 7 | // output: out(b, c, npoints, nsample) 8 | __global__ void group_points_kernel(int b, int c, int n, int npoints, 9 | int nsample, 10 | const float *__restrict__ points, 11 | const int *__restrict__ idx, 12 | float *__restrict__ out) { 13 | int batch_index = blockIdx.x; 14 | points += batch_index * n * c; 15 | idx += batch_index * npoints * nsample; 16 | out += batch_index * npoints * nsample * c; 17 | 18 | const int index = threadIdx.y * blockDim.x + threadIdx.x; 19 | const int stride = blockDim.y * blockDim.x; 20 | for (int i = index; i < c * npoints; i += stride) { 21 | const int l = i / npoints; 22 | const int j = i % npoints; 23 | for (int k = 0; k < nsample; ++k) { 24 | int ii = idx[j * nsample + k]; 25 | out[(l * npoints + j) * nsample + k] = points[l * n + ii]; 26 | } 27 | } 28 | } 29 | 30 | void group_points_kernel_wrapper(int b, int c, int n, int npoints, int nsample, 31 | const float *points, const int *idx, 32 | float *out) { 33 | cudaStream_t stream = at::cuda::getCurrentCUDAStream(); 34 | 35 | group_points_kernel<<>>( 36 | b, c, n, npoints, nsample, points, idx, out); 37 | 38 | CUDA_CHECK_ERRORS(); 39 | } 40 | 41 | // input: grad_out(b, c, npoints, nsample), idx(b, npoints, nsample) 42 | // output: grad_points(b, c, n) 43 | __global__ void group_points_grad_kernel(int b, int c, int n, int npoints, 44 | int nsample, 45 | const float *__restrict__ grad_out, 46 | const int *__restrict__ idx, 47 | float *__restrict__ grad_points) { 48 | int batch_index = blockIdx.x; 49 | grad_out += batch_index * npoints * nsample * c; 50 | idx += batch_index * npoints * nsample; 51 | grad_points += batch_index * n * c; 52 | 53 | const int index = threadIdx.y * blockDim.x + threadIdx.x; 54 | const int stride = blockDim.y * blockDim.x; 55 | for (int i = index; i < c * npoints; i += stride) { 56 | const int l = i / npoints; 57 | const int j = i % npoints; 58 | for (int k = 0; k < nsample; ++k) { 59 | int ii = idx[j * nsample + k]; 60 | atomicAdd(grad_points + l * n + ii, 61 | grad_out[(l * npoints + j) * nsample + k]); 62 | } 63 | } 64 | } 65 | 66 | void group_points_grad_kernel_wrapper(int b, int c, int n, int npoints, 67 | int nsample, const float *grad_out, 68 | const int *idx, float *grad_points) { 69 | cudaStream_t stream = at::cuda::getCurrentCUDAStream(); 70 | 71 | group_points_grad_kernel<<>>( 72 | b, c, n, npoints, nsample, grad_out, idx, grad_points); 73 | 74 | CUDA_CHECK_ERRORS(); 75 | } 76 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/_ext_src/src/sampling.cpp: -------------------------------------------------------------------------------- 1 | #include "sampling.h" 2 | #include "utils.h" 3 | 4 | void gather_points_kernel_wrapper(int b, int c, int n, int npoints, 5 | const float *points, const int *idx, 6 | float *out); 7 | void gather_points_grad_kernel_wrapper(int b, int c, int n, int npoints, 8 | const float *grad_out, const int *idx, 9 | float *grad_points); 10 | 11 | void furthest_point_sampling_kernel_wrapper(int b, int n, int m, 12 | const float *dataset, float *temp, 13 | int *idxs); 14 | 15 | at::Tensor gather_points(at::Tensor points, at::Tensor idx) { 16 | CHECK_CONTIGUOUS(points); 17 | CHECK_CONTIGUOUS(idx); 18 | CHECK_IS_FLOAT(points); 19 | CHECK_IS_INT(idx); 20 | 21 | if (points.is_cuda()) { 22 | CHECK_CUDA(idx); 23 | } 24 | 25 | at::Tensor output = 26 | torch::zeros({points.size(0), points.size(1), idx.size(1)}, 27 | at::device(points.device()).dtype(at::ScalarType::Float)); 28 | 29 | if (points.is_cuda()) { 30 | gather_points_kernel_wrapper(points.size(0), points.size(1), points.size(2), 31 | idx.size(1), points.data_ptr(), 32 | idx.data_ptr(), output.data_ptr()); 33 | } else { 34 | AT_ASSERT(false, "CPU not supported"); 35 | } 36 | 37 | return output; 38 | } 39 | 40 | at::Tensor gather_points_grad(at::Tensor grad_out, at::Tensor idx, 41 | const int n) { 42 | CHECK_CONTIGUOUS(grad_out); 43 | CHECK_CONTIGUOUS(idx); 44 | CHECK_IS_FLOAT(grad_out); 45 | CHECK_IS_INT(idx); 46 | 47 | if (grad_out.is_cuda()) { 48 | CHECK_CUDA(idx); 49 | } 50 | 51 | at::Tensor output = 52 | torch::zeros({grad_out.size(0), grad_out.size(1), n}, 53 | at::device(grad_out.device()).dtype(at::ScalarType::Float)); 54 | 55 | if (grad_out.is_cuda()) { 56 | gather_points_grad_kernel_wrapper(grad_out.size(0), grad_out.size(1), n, 57 | idx.size(1), grad_out.data_ptr(), 58 | idx.data_ptr(), 59 | output.data_ptr()); 60 | } else { 61 | AT_ASSERT(false, "CPU not supported"); 62 | } 63 | 64 | return output; 65 | } 66 | at::Tensor furthest_point_sampling(at::Tensor points, const int nsamples) { 67 | CHECK_CONTIGUOUS(points); 68 | CHECK_IS_FLOAT(points); 69 | 70 | at::Tensor output = 71 | torch::zeros({points.size(0), nsamples}, 72 | at::device(points.device()).dtype(at::ScalarType::Int)); 73 | 74 | at::Tensor tmp = 75 | torch::full({points.size(0), points.size(1)}, 1e10, 76 | at::device(points.device()).dtype(at::ScalarType::Float)); 77 | 78 | if (points.is_cuda()) { 79 | furthest_point_sampling_kernel_wrapper( 80 | points.size(0), points.size(1), nsamples, points.data_ptr(), 81 | tmp.data_ptr(), output.data_ptr()); 82 | } else { 83 | AT_ASSERT(false, "CPU not supported"); 84 | } 85 | 86 | return output; 87 | } 88 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/pointnet2_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | ''' Testing customized ops. ''' 7 | 8 | import torch 9 | from torch.autograd import gradcheck 10 | import numpy as np 11 | 12 | import os 13 | import sys 14 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 15 | sys.path.append(BASE_DIR) 16 | import pointnet2_utils 17 | 18 | def test_interpolation_grad(): 19 | batch_size = 1 20 | feat_dim = 2 21 | m = 4 22 | feats = torch.randn(batch_size, feat_dim, m, requires_grad=True).float().cuda() 23 | 24 | def interpolate_func(inputs): 25 | idx = torch.from_numpy(np.array([[[0,1,2],[1,2,3]]])).int().cuda() 26 | weight = torch.from_numpy(np.array([[[1,1,1],[2,2,2]]])).float().cuda() 27 | interpolated_feats = pointnet2_utils.three_interpolate(inputs, idx, weight) 28 | return interpolated_feats 29 | 30 | assert (gradcheck(interpolate_func, feats, atol=1e-1, rtol=1e-1)) 31 | 32 | if __name__=='__main__': 33 | test_interpolation_grad() 34 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/pointnet2/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import glob 7 | import os 8 | 9 | from setuptools import setup 10 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 11 | 12 | this_dir = os.path.dirname(os.path.abspath(__file__)) 13 | _ext_src_root = "_ext_src" 14 | _ext_sources = glob.glob("{}/src/*.cpp".format(_ext_src_root)) + glob.glob( 15 | "{}/src/*.cu".format(_ext_src_root) 16 | ) 17 | 18 | setup( 19 | name='pointnet2', 20 | ext_modules=[ 21 | CUDAExtension( 22 | name='pointnet2._ext', 23 | sources=_ext_sources, 24 | extra_compile_args={ 25 | "cxx": ["-O3"], 26 | "nvcc": ["-O3", "-Xfatbin", "-compress-all"], 27 | }, 28 | include_dirs=[os.path.join(this_dir, _ext_src_root, "include")], 29 | ) 30 | ], 31 | cmdclass={ 32 | 'build_ext': BuildExtension 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/__pycache__/voxelized_dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/__pycache__/voxelized_dataset.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/lib/__init__.py -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/lib/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/lib/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/lib/__pycache__/math_functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/lib/__pycache__/math_functions.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/lib/math_functions.py: -------------------------------------------------------------------------------- 1 | from scipy.sparse import csr_matrix 2 | import torch 3 | 4 | 5 | class SparseMM(torch.autograd.Function): 6 | """ 7 | Sparse x dense matrix multiplication with autograd support. 8 | Implementation by Soumith Chintala: 9 | https://discuss.pytorch.org/t/ 10 | does-pytorch-support-autograd-on-sparse-matrix/6156/7 11 | """ 12 | 13 | def forward(self, matrix1, matrix2): 14 | self.save_for_backward(matrix1, matrix2) 15 | return torch.mm(matrix1, matrix2) 16 | 17 | def backward(self, grad_output): 18 | matrix1, matrix2 = self.saved_tensors 19 | grad_matrix1 = grad_matrix2 = None 20 | 21 | if self.needs_input_grad[0]: 22 | grad_matrix1 = torch.mm(grad_output, matrix2.t()) 23 | 24 | if self.needs_input_grad[1]: 25 | grad_matrix2 = torch.mm(matrix1.t(), grad_output) 26 | 27 | return grad_matrix1, grad_matrix2 28 | 29 | 30 | def sparse_float_tensor(values, indices, size=None): 31 | """ 32 | Return a torch sparse matrix give values and indices (row_ind, col_ind). 33 | If the size is an integer, return a square matrix with side size. 34 | If the size is a torch.Size, use it to initialize the out tensor. 35 | If none, the size is inferred. 36 | """ 37 | indices = torch.stack(indices).int() 38 | sargs = [indices, values.float()] 39 | if size is not None: 40 | # Use the provided size 41 | if isinstance(size, int): 42 | size = torch.Size((size, size)) 43 | sargs.append(size) 44 | if values.is_cuda: 45 | return torch.cuda.sparse.FloatTensor(*sargs) 46 | else: 47 | return torch.sparse.FloatTensor(*sargs) 48 | 49 | 50 | def diags(values, size=None): 51 | values = values.view(-1) 52 | n = values.nelement() 53 | size = torch.Size((n, n)) 54 | indices = (torch.arange(0, n), torch.arange(0, n)) 55 | return sparse_float_tensor(values, indices, size) 56 | 57 | 58 | def sparse_to_csr_matrix(tensor): 59 | tensor = tensor.cpu() 60 | inds = tensor._indices().numpy() 61 | vals = tensor._values().numpy() 62 | return csr_matrix((vals, (inds[0], inds[1])), shape=[s for s in tensor.shape]) 63 | 64 | 65 | def csr_matrix_to_sparse(mat): 66 | row_ind, col_ind = mat.nonzero() 67 | return sparse_float_tensor( 68 | torch.from_numpy(mat.data), 69 | (torch.from_numpy(row_ind), torch.from_numpy(col_ind)), 70 | size=torch.Size(mat.shape)) 71 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from models.backbone.sparseconv.models_sparseconv import resunet as resunet 7 | from models.backbone.sparseconv.models_sparseconv import res16unet as res16unet 8 | 9 | # from models.trilateral_crf import TrilateralCRF 10 | from models.backbone.sparseconv.models_sparseconv.conditional_random_fields import BilateralCRF, TrilateralCRF 11 | 12 | MODELS = [] 13 | 14 | 15 | def add_models(module): 16 | MODELS.extend([getattr(module, a) for a in dir(module) if 'Net' in a]) 17 | 18 | 19 | add_models(resunet) 20 | add_models(res16unet) 21 | 22 | WRAPPERS = [BilateralCRF, TrilateralCRF] 23 | 24 | 25 | def get_models(): 26 | '''Returns a tuple of sample models.''' 27 | return MODELS 28 | 29 | 30 | def get_wrappers(): 31 | return WRAPPERS 32 | 33 | 34 | def load_model(name): 35 | '''Creates and returns an instance of the model given its class name. 36 | ''' 37 | # Find the model class from its name 38 | all_models = get_models() 39 | mdict = {model.__name__: model for model in all_models} 40 | if name not in mdict: 41 | print('Invalid model index. Options are:') 42 | # Display a list of valid model names 43 | for model in all_models: 44 | print('\t* {}'.format(model.__name__)) 45 | return None 46 | NetClass = mdict[name] 47 | 48 | return NetClass 49 | 50 | 51 | def load_wrapper(name): 52 | '''Creates and returns an instance of the model given its class name. 53 | ''' 54 | # Find the model class from its name 55 | all_wrappers = get_wrappers() 56 | mdict = {wrapper.__name__: wrapper for wrapper in all_wrappers} 57 | if name not in mdict: 58 | print('Invalid wrapper index. Options are:') 59 | # Display a list of valid model names 60 | for wrapper in all_wrappers: 61 | print('\t* {}'.format(wrapper.__name__)) 62 | return None 63 | WrapperClass = mdict[name] 64 | 65 | return WrapperClass 66 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/conditional_random_fields.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/conditional_random_fields.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/res16unet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/res16unet.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/resunet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/resunet.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/wrapper.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/__pycache__/wrapper.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from MinkowskiEngine import MinkowskiNetwork 8 | 9 | 10 | class Model(MinkowskiNetwork): 11 | """ 12 | Base network for all sparse convnet 13 | 14 | By default, all networks are segmentation networks. 15 | """ 16 | OUT_PIXEL_DIST = -1 17 | 18 | def __init__(self, in_channels, out_channels, config, D, **kwargs): 19 | super(Model, self).__init__(D) 20 | self.in_channels = in_channels 21 | self.out_channels = out_channels 22 | self.config = config 23 | 24 | 25 | class HighDimensionalModel(Model): 26 | """ 27 | Base network for all spatio (temporal) chromatic sparse convnet 28 | """ 29 | 30 | def __init__(self, in_channels, out_channels, config, D, **kwargs): 31 | assert D > 4, "Num dimension smaller than 5" 32 | super(HighDimensionalModel, self).__init__(in_channels, out_channels, config, D, **kwargs) 33 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__init__.py -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/common.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/common.cpython-36.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/common.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/common.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/resnet_block.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/resnet_block.cpython-36.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/resnet_block.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/downstream/votenet/models/backbone/sparseconv/models_sparseconv/modules/__pycache__/resnet_block.cpython-37.pyc -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/models_sparseconv/wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import random 7 | from torch.nn import Module 8 | 9 | from MinkowskiEngine import SparseTensor 10 | 11 | 12 | class Wrapper(Module): 13 | """ 14 | Wrapper for the segmentation networks. 15 | """ 16 | OUT_PIXEL_DIST = -1 17 | 18 | def __init__(self, NetClass, in_nchannel, out_nchannel, config): 19 | super(Wrapper, self).__init__() 20 | self.initialize_filter(NetClass, in_nchannel, out_nchannel, config) 21 | 22 | def initialize_filter(self, NetClass, in_nchannel, out_nchannel, config): 23 | raise NotImplementedError('Must initialize a model and a filter') 24 | 25 | def forward(self, x, coords, colors=None): 26 | soutput = self.model(x) 27 | 28 | # During training, make the network invariant to the filter 29 | if not self.training or random.random() < 0.5: 30 | # Filter requires the model to finish the forward pass 31 | wrapper_coords = self.filter.initialize_coords(self.model, coords, colors) 32 | finput = SparseTensor(soutput.F, wrapper_coords) 33 | soutput = self.filter(finput) 34 | 35 | return soutput 36 | -------------------------------------------------------------------------------- /downstream/votenet/models/backbone/sparseconv/voxelized_dataset.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import sys 9 | import numpy as np 10 | import torch 11 | 12 | from torch.utils.data import Dataset 13 | from torch.utils.data._utils.collate import default_collate 14 | 15 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 16 | ROOT_DIR = os.path.dirname(BASE_DIR) 17 | sys.path.append(BASE_DIR) 18 | sys.path.append(os.path.join(ROOT_DIR, 'utils')) 19 | 20 | import MinkowskiEngine as ME 21 | 22 | class VoxelizationDataset(Dataset): 23 | """ 24 | Wrapper dataset which voxelize the original point clouds 25 | """ 26 | def __init__(self, dataset, voxel_size=0.05): 27 | self.dataset = dataset 28 | self.VOXEL_SIZE = voxel_size 29 | 30 | def __len__(self): 31 | return len(self.dataset) 32 | 33 | def __getitem__(self, idx): 34 | ret_dict = self.dataset[idx] 35 | 36 | # voxelization 37 | coords = np.floor(ret_dict['point_clouds'] / self.VOXEL_SIZE) 38 | inds = ME.utils.sparse_quantize(coords, return_index=True) 39 | coords = coords[inds].astype(np.int32) 40 | colors = ret_dict['pcl_color'][inds] 41 | 42 | ret_dict['voxel'] = (coords, np.array(inds, dtype=np.int32), colors) 43 | return ret_dict 44 | 45 | 46 | def collate_fn(samples): 47 | data, voxel = [], [] 48 | for sample in samples: 49 | data.append({w: sample[w] for w in sample if w != 'voxel'}) 50 | voxel.append(sample['voxel']) 51 | 52 | # for non-voxel data, use default collate 53 | data_batch = default_collate(data) 54 | 55 | batch_ids = np.array( 56 | [b for b, v in enumerate(voxel) for _ in range(v[0].shape[0])]) 57 | voxel_ids = np.concatenate([v[1] for v in voxel], 0) 58 | 59 | coords = np.concatenate([v[0] for v in voxel], 0) 60 | coords = np.concatenate([batch_ids[:, None], coords], 1) 61 | 62 | colors = np.concatenate([v[2] for v in voxel], 0) 63 | 64 | data_batch['voxel_coords'] = torch.from_numpy(coords) 65 | data_batch['voxel_inds'] = torch.from_numpy(voxel_ids) 66 | #data_batch['voxel_feats'] = data_batch['point_clouds'].new_ones(batch_ids.shape[0], 3) 67 | data_batch['voxel_feats'] = torch.from_numpy(colors).float() 68 | 69 | return data_batch 70 | -------------------------------------------------------------------------------- /downstream/votenet/models/voting_module.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | ''' Voting module: generate votes from XYZ and features of seed points. 7 | 8 | Date: July, 2019 9 | Author: Charles R. Qi and Or Litany 10 | ''' 11 | 12 | import torch 13 | import torch.nn as nn 14 | import torch.nn.functional as F 15 | 16 | class VotingModule(nn.Module): 17 | def __init__(self, vote_factor, seed_feature_dim): 18 | """ Votes generation from seed point features. 19 | 20 | Args: 21 | vote_facotr: int 22 | number of votes generated from each seed point 23 | seed_feature_dim: int 24 | number of channels of seed point features 25 | vote_feature_dim: int 26 | number of channels of vote features 27 | """ 28 | super().__init__() 29 | self.vote_factor = vote_factor 30 | self.in_dim = seed_feature_dim 31 | self.out_dim = self.in_dim # due to residual feature, in_dim has to be == out_dim 32 | self.conv1 = torch.nn.Conv1d(self.in_dim, self.in_dim, 1) 33 | self.conv2 = torch.nn.Conv1d(self.in_dim, self.in_dim, 1) 34 | self.conv3 = torch.nn.Conv1d(self.in_dim, (3+self.out_dim) * self.vote_factor, 1) 35 | self.bn1 = torch.nn.BatchNorm1d(self.in_dim) 36 | self.bn2 = torch.nn.BatchNorm1d(self.in_dim) 37 | 38 | def forward(self, seed_xyz, seed_features): 39 | """ Forward pass. 40 | 41 | Arguments: 42 | seed_xyz: (batch_size, num_seed, 3) Pytorch tensor 43 | seed_features: (batch_size, feature_dim, num_seed) Pytorch tensor 44 | Returns: 45 | vote_xyz: (batch_size, num_seed*vote_factor, 3) 46 | vote_features: (batch_size, vote_feature_dim, num_seed*vote_factor) 47 | """ 48 | batch_size = seed_xyz.shape[0] 49 | num_seed = seed_xyz.shape[1] 50 | num_vote = num_seed*self.vote_factor 51 | net = F.relu(self.bn1(self.conv1(seed_features))) 52 | net = F.relu(self.bn2(self.conv2(net))) 53 | net = self.conv3(net) # (batch_size, (3+out_dim)*vote_factor, num_seed) 54 | 55 | net = net.transpose(2,1).view(batch_size, num_seed, self.vote_factor, 3+self.out_dim) 56 | offset = net[:,:,:,0:3] 57 | vote_xyz = seed_xyz.unsqueeze(2) + offset 58 | vote_xyz = vote_xyz.contiguous().view(batch_size, num_vote, 3) 59 | 60 | residual_features = net[:,:,:,3:] # (batch_size, num_seed, vote_factor, out_dim) 61 | vote_features = seed_features.transpose(2,1).unsqueeze(2) + residual_features 62 | vote_features = vote_features.contiguous().view(batch_size, num_vote, self.out_dim) 63 | vote_features = vote_features.transpose(2,1).contiguous() 64 | 65 | return vote_xyz, vote_features 66 | 67 | if __name__=='__main__': 68 | net = VotingModule(2, 256).cuda() 69 | xyz, features = net(torch.rand(8,1024,3).cuda(), torch.rand(8,256,1024).cuda()) 70 | print('xyz', xyz.shape) 71 | print('features', features.shape) 72 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/data_efficient/by_bboxes.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py -m \ 13 | net.backbone=sparseconv \ 14 | data.dataset=scannet \ 15 | data.batch_size=32 \ 16 | data.num_workers=4 \ 17 | data.num_points=40000 \ 18 | data.no_height=True \ 19 | data.by_points=$SAMPLED_BBOX \ 20 | optimizer.learning_rate=0.001 \ 21 | optimizer.max_epoch=500 \ 22 | data.voxelization=True \ 23 | data.voxel_size=0.025 \ 24 | misc.log_dir=$LOG_DIR \ 25 | net.is_train=True \ 26 | net.weights=$PRETRAIN \ 27 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/data_efficient/by_scenes.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py \ 13 | net.backbone=sparseconv \ 14 | data.dataset=scannet \ 15 | data.batch_size=32 \ 16 | data.num_workers=8 \ 17 | data.num_points=40000 \ 18 | data.no_height=True \ 19 | data.by_scenes=$TRAIN_FILE \ 20 | optimizer.learning_rate=0.001 \ 21 | optimizer.max_epoch=500 \ 22 | data.voxelization=True \ 23 | data.voxel_size=0.025 \ 24 | misc.log_dir=$LOG_DIR \ 25 | net.is_train=True \ 26 | net.weights=$PRETRAIN \ 27 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/test_scannet.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | #! /bin/bash 7 | 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py \ 13 | net.backbone=sparseconv \ 14 | data.dataset=scannet \ 15 | data.num_workers=8 \ 16 | data.batch_size=1 \ 17 | data.num_points=40000 \ 18 | data.use_color=False \ 19 | data.no_height=True \ 20 | optimizer.learning_rate=0.001 \ 21 | data.voxelization=True \ 22 | data.voxel_size=0.025 \ 23 | misc.log_dir=$LOG_DIR \ 24 | misc.num_gpus=1 \ 25 | net.is_train=False \ 26 | test.ap_iou=0.5 \ 27 | net.weights=$PRETRAIN \ 28 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/test_scannet_pointnet.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | 10 | mkdir -p $LOG_DIR 11 | 12 | # main script 13 | python ddp_main.py \ 14 | net.backbone=pointnet2 \ 15 | data.dataset=scannet \ 16 | data.num_workers=4 \ 17 | data.num_points=40000 \ 18 | data.use_color=False \ 19 | data.no_height=True \ 20 | optimizer.learning_rate=0.001 \ 21 | data.voxelization=False \ 22 | misc.log_dir=$LOG_DIR \ 23 | misc.num_gpus=1 \ 24 | net.is_train=False \ 25 | net.weights=$PRETRAIN \ 26 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/test_sunrgbd.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py \ 13 | net.is_train=False \ 14 | net.backbone=sparseconv \ 15 | data.dataset=sunrgbd \ 16 | data.num_workers=4 \ 17 | data.num_points=20000 \ 18 | data.no_height=True \ 19 | data.voxelization=True \ 20 | data.voxel_size=0.025 \ 21 | test.ap_iou=0.5 \ 22 | optimizer.learning_rate=0.001 \ 23 | misc.log_dir=$LOG_DIR \ 24 | net.weights=$PRETRAIN \ 25 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/train_scannet.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py -m \ 13 | net.backbone=sparseconv \ 14 | data.dataset=scannet \ 15 | data.num_workers=8 \ 16 | data.batch_size=32 \ 17 | data.num_points=40000 \ 18 | data.use_color=False \ 19 | data.no_height=True \ 20 | optimizer.learning_rate=0.001 \ 21 | data.voxelization=True \ 22 | data.voxel_size=0.025 \ 23 | misc.log_dir=$LOG_DIR \ 24 | misc.num_gpus=1 \ 25 | net.is_train=True \ 26 | test.ap_iou=0.5 \ 27 | net.weights=$PRETRAIN \ 28 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/train_scannet_pointnet.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py -m \ 13 | net.backbone=pointnet2 \ 14 | data.dataset=scannet \ 15 | data.num_workers=4 \ 16 | data.batch_size=8 \ 17 | data.num_points=40000 \ 18 | data.use_color=False \ 19 | data.no_height=True \ 20 | optimizer.learning_rate=0.001 \ 21 | data.voxelization=False \ 22 | data.voxel_size=0.025 \ 23 | misc.log_dir=$LOG_DIR \ 24 | misc.num_gpus=1 \ 25 | test.ap_iou=0.5 \ 26 | net.is_train=True \ 27 | net.weights=$PRETRAIN \ 28 | -------------------------------------------------------------------------------- /downstream/votenet/scripts/train_sunrgbd.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #! /bin/bash 8 | 9 | mkdir -p $LOG_DIR 10 | 11 | # main script 12 | python ddp_main.py -m \ 13 | net.is_train=True \ 14 | net.backbone=sparseconv \ 15 | data.dataset=sunrgbd \ 16 | data.num_workers=8 \ 17 | data.num_points=20000 \ 18 | data.batch_size=64 \ 19 | data.no_height=True \ 20 | data.voxelization=True \ 21 | data.voxel_size=0.025 \ 22 | test.ap_iou=0.5 \ 23 | optimizer.learning_rate=0.001 \ 24 | misc.log_dir=$LOG_DIR \ 25 | misc.num_gpus=2 \ 26 | net.weights=$PRETRAIN \ 27 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/.gitignore: -------------------------------------------------------------------------------- 1 | # Temp files 2 | __pycache__ 3 | *.swp 4 | *.swo 5 | *.orig 6 | .idea 7 | .nfs* 8 | outputs/ 9 | outputs_dir/ 10 | *.pyc 11 | data/ 12 | debug/ 13 | launch_scripts 14 | *.npy 15 | *.pcd 16 | .*/ 17 | downstream/ 18 | .vscode/settings.json 19 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/config/defaults.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | defaults: 7 | - hydra/launcher: submitit_slurm 8 | - hydra/hydra_logging: colorlog 9 | 10 | trainer: 11 | trainer: ContrastiveLossTrainer 12 | overwrite_checkpoint: False 13 | batch_size: 4 14 | 15 | # Metric learning loss 16 | neg_thresh: 1.4 17 | pos_thresh: 0.1 18 | neg_weight: 1.0 19 | 20 | # Data loader configs 21 | stat_freq: 40 22 | lr_update_freq: 1000 23 | checkpoint_freq: 1000 24 | positive_pair_search_voxel_size_multiplier: 1.5 25 | 26 | # Network specific configurations 27 | net: 28 | model: Res16UNet34C 29 | weight: 30 | model_n_out: 32 #Feature dimension 31 | conv1_kernel_size: 3 32 | normalize_feature: True 33 | dist_type: L2 34 | 35 | # Optimizer arguments 36 | opt: 37 | optimizer: SGD 38 | max_iter: 300000 39 | lr: 1e-1 40 | momentum: 0.8 41 | sgd_momentum: 0.9 42 | sgd_dampening: 0.1 43 | adam_beta1: 0.9 44 | adam_beta2: 0.999 45 | weight_decay: 1e-4 46 | bn_momentum: 0.05 47 | exp_gamma: 0.99 48 | scheduler: ExpLR 49 | 50 | shape_context: 51 | r1: -1e9 52 | r2: 1e9 53 | nbins_xy: 1 54 | nbins_zy: 1 55 | weight_inner: False 56 | fast_partition: False 57 | 58 | misc: 59 | out_dir: /checkpoint/jihou/checkpoints/default 60 | use_gpu: True 61 | num_gpus: 1 62 | lenient_weight_loading: False # Weights with the same size will be loaded 63 | 64 | train_num_thread: 2 65 | nn_max_n: 500 # The maximum number of features to find nearest neighbors in batch 66 | 67 | # NCE related 68 | nceT: 0.07 69 | npos: 4096 70 | 71 | # TODO(s9xie): all args for scannet training 72 | train_limit_numpoints: 0 73 | data_aug_scale_min: 0.9 74 | data_aug_scale_max: 1.1 75 | 76 | cache_data: False 77 | 78 | ignore_label: 255 79 | return_transformation: False 80 | is_train: True 81 | 82 | # Dataset specific configurations 83 | data: 84 | dataset: ScanNetMatchPairDataset 85 | voxel_size: 0.025 86 | voxelize: False 87 | num_points: 40000 88 | scannet_match_dir: /checkpoint/jihou/data/scannet/partial_frames/ 89 | random_rotation_xyz: False 90 | world_space: False 91 | 92 | hydra: 93 | run: 94 | dir: ${misc.out_dir} 95 | sweep: 96 | dir: ${misc.out_dir} 97 | launcher: 98 | partition: debug 99 | submitit_folder: ${hydra.sweep.dir}/.submitit/%j 100 | name: ${hydra.job.name} 101 | timeout_min: 3600 102 | cpus_per_task: 70 103 | gpus_per_node: 8 104 | tasks_per_node: 1 105 | mem_gb: 320 106 | nodes: 1 107 | exclude: balrog,gimli 108 | max_num_timeout: 3 109 | signal_delay_s: 300 110 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/ddp_train.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import sys 7 | import os 8 | import json 9 | import logging 10 | import torch 11 | from omegaconf import OmegaConf 12 | 13 | from easydict import EasyDict as edict 14 | 15 | import lib.multiprocessing_utils as mpu 16 | import hydra 17 | 18 | from lib.ddp_trainer import PointNCELossTrainer, PartitionPointNCELossTrainer, PartitionPointNCELossTrainerPointNet 19 | 20 | ch = logging.StreamHandler(sys.stdout) 21 | logging.getLogger().setLevel(logging.INFO) 22 | logging.basicConfig( 23 | format='%(asctime)s %(message)s', datefmt='%m/%d %H:%M:%S', handlers=[ch]) 24 | 25 | torch.manual_seed(0) 26 | torch.cuda.manual_seed(0) 27 | 28 | logging.basicConfig(level=logging.INFO, format="") 29 | 30 | def get_trainer(trainer): 31 | if trainer == 'PointNCELossTrainer': 32 | return PointNCELossTrainer 33 | elif trainer == 'PartitionPointNCELossTrainer': 34 | return PartitionPointNCELossTrainer 35 | elif trainer == 'PartitionPointNCELossTrainerPointNet': 36 | return PartitionPointNCELossTrainerPointNet 37 | else: 38 | raise ValueError(f'Trainer {trainer} not found') 39 | 40 | @hydra.main(config_path='config', config_name='defaults.yaml') 41 | def main(config): 42 | if os.path.exists('config.yaml'): 43 | logging.info('===> Loading exsiting config file') 44 | config = OmegaConf.load('config.yaml') 45 | logging.info('===> Loaded exsiting config file') 46 | logging.info('===> Configurations') 47 | logging.info(config.pretty()) 48 | 49 | # Convert to dict 50 | if config.misc.num_gpus > 1: 51 | mpu.multi_proc_run(config.misc.num_gpus, 52 | fun=single_proc_run, fun_args=(config,)) 53 | else: 54 | single_proc_run(config) 55 | 56 | def single_proc_run(config): 57 | from lib.ddp_data_loaders import make_data_loader 58 | 59 | train_loader = make_data_loader( 60 | config, 61 | int(config.trainer.batch_size / config.misc.num_gpus), 62 | num_threads=int(config.misc.train_num_thread / config.misc.num_gpus)) 63 | 64 | Trainer = get_trainer(config.trainer.trainer) 65 | trainer = Trainer(config=config, data_loader=train_loader) 66 | 67 | if config.misc.is_train: 68 | trainer.train() 69 | else: 70 | trainer.test() 71 | 72 | 73 | if __name__ == "__main__": 74 | os.environ['MKL_THREADING_LAYER'] = 'GNU' 75 | main() 76 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/criterion.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | from torch import nn 8 | 9 | class NCESoftmaxLoss(nn.Module): 10 | def __init__(self): 11 | super(NCESoftmaxLoss, self).__init__() 12 | self.criterion = nn.CrossEntropyLoss() 13 | 14 | def forward(self, x, label): 15 | bsz = x.shape[0] 16 | x = x.squeeze() 17 | loss = self.criterion(x, label) 18 | return loss 19 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/data_sampler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | from torch.utils.data.sampler import Sampler 8 | import torch.distributed as dist 9 | 10 | import math 11 | 12 | class InfSampler(Sampler): 13 | def __init__(self, data_source, shuffle=False): 14 | self.data_source = data_source 15 | self.shuffle = shuffle 16 | self.reset_permutation() 17 | 18 | def reset_permutation(self): 19 | perm = len(self.data_source) 20 | if self.shuffle: 21 | perm = torch.randperm(perm) 22 | self._perm = perm.tolist() 23 | 24 | def __iter__(self): 25 | return self 26 | 27 | def __next__(self): 28 | if len(self._perm) == 0: 29 | self.reset_permutation() 30 | return self._perm.pop() 31 | 32 | def __len__(self): 33 | return len(self.data_source) 34 | 35 | next = __next__ # Python 2 compatibility 36 | 37 | 38 | class DistributedInfSampler(InfSampler): 39 | def __init__(self, data_source, num_replicas=None, rank=None, shuffle=True): 40 | if num_replicas is None: 41 | if not dist.is_available(): 42 | raise RuntimeError("Requires distributed package to be available") 43 | num_replicas = dist.get_world_size() 44 | if rank is None: 45 | if not dist.is_available(): 46 | raise RuntimeError("Requires distributed package to be available") 47 | rank = dist.get_rank() 48 | 49 | self.data_source = data_source 50 | self.num_replicas = num_replicas 51 | self.rank = rank 52 | self.epoch = 0 53 | self.it = 0 54 | self.num_samples = int(math.ceil(len(self.data_source) * 1.0 / self.num_replicas)) 55 | self.total_size = self.num_samples * self.num_replicas 56 | self.shuffle = shuffle 57 | self.reset_permutation() 58 | 59 | def __next__(self): 60 | it = self.it * self.num_replicas + self.rank 61 | value = self._perm[it % len(self._perm)] 62 | self.it = self.it + 1 63 | 64 | if (self.it * self.num_replicas) >= len(self._perm): 65 | self.reset_permutation() 66 | self.it = 0 67 | return value 68 | 69 | def __len__(self): 70 | return self.num_samples 71 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/error_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/usr/bin/env python3 8 | 9 | """Multiprocessing error handler.""" 10 | 11 | import os 12 | import signal 13 | import threading 14 | 15 | 16 | class ChildException(Exception): 17 | """Wraps an exception from a child process.""" 18 | 19 | def __init__(self, child_trace): 20 | super(ChildException, self).__init__(child_trace) 21 | 22 | 23 | class ErrorHandler(object): 24 | """Multiprocessing error handler (based on fairseq's). 25 | 26 | Listens for errors in child processes and 27 | propagates the tracebacks to the parent process. 28 | """ 29 | 30 | def __init__(self, error_queue): 31 | # Shared error queue 32 | self.error_queue = error_queue 33 | # Children processes sharing the error queue 34 | self.children_pids = [] 35 | # Start a thread listening to errors 36 | self.error_listener = threading.Thread(target=self.listen, daemon=True) 37 | self.error_listener.start() 38 | # Register the signal handler 39 | signal.signal(signal.SIGUSR1, self.signal_handler) 40 | 41 | def add_child(self, pid): 42 | """Registers a child process.""" 43 | self.children_pids.append(pid) 44 | 45 | def listen(self): 46 | """Listens for errors in the error queue.""" 47 | # Wait until there is an error in the queue 48 | child_trace = self.error_queue.get() 49 | # Put the error back for the signal handler 50 | self.error_queue.put(child_trace) 51 | # Invoke the signal handler 52 | os.kill(os.getpid(), signal.SIGUSR1) 53 | 54 | def signal_handler(self, sig_num, stack_frame): 55 | """Signal handler.""" 56 | # Kill children processes 57 | for pid in self.children_pids: 58 | os.kill(pid, signal.SIGINT) 59 | # Propagate the error from the child process 60 | raise ChildException(self.error_queue.get()) 61 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/evaluation/scannet_benchmark_utils/util.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import csv 3 | try: 4 | import numpy as np 5 | except: 6 | print("Failed to import numpy package.") 7 | sys.exit(-1) 8 | try: 9 | import imageio 10 | except: 11 | print("Please install the module 'imageio' for image processing, e.g.") 12 | print("pip install imageio") 13 | sys.exit(-1) 14 | 15 | # print an error message and quit 16 | def print_error(message, user_fault=False): 17 | sys.stderr.write('ERROR: ' + str(message) + '\n') 18 | if user_fault: 19 | sys.exit(2) 20 | sys.exit(-1) 21 | 22 | 23 | # if string s represents an int 24 | def represents_int(s): 25 | try: 26 | int(s) 27 | return True 28 | except ValueError: 29 | return False 30 | 31 | 32 | def read_label_mapping(filename, label_from='raw_category', label_to='nyu40id'): 33 | assert os.path.isfile(filename) 34 | mapping = dict() 35 | with open(filename) as csvfile: 36 | reader = csv.DictReader(csvfile, delimiter='\t') 37 | for row in reader: 38 | mapping[row[label_from]] = int(row[label_to]) 39 | # if ints convert 40 | if represents_int([key for key in mapping.keys()][0]): 41 | mapping = {int(k):v for k,v in mapping.items()} 42 | return mapping 43 | 44 | 45 | # input: scene_types.txt or scene_types_all.txt 46 | def read_scene_types_mapping(filename, remove_spaces=True): 47 | assert os.path.isfile(filename) 48 | mapping = dict() 49 | lines = open(filename).read().splitlines() 50 | lines = [line.split('\t') for line in lines] 51 | if remove_spaces: 52 | mapping = { x[1].strip():int(x[0]) for x in lines } 53 | else: 54 | mapping = { x[1]:int(x[0]) for x in lines } 55 | return mapping 56 | 57 | 58 | # color by label 59 | def visualize_label_image(filename, image): 60 | height = image.shape[0] 61 | width = image.shape[1] 62 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 63 | color_palette = create_color_palette() 64 | for idx, color in enumerate(color_palette): 65 | vis_image[image==idx] = color 66 | imageio.imwrite(filename, vis_image) 67 | 68 | 69 | # color by different instances (mod length of color palette) 70 | def visualize_instance_image(filename, image): 71 | height = image.shape[0] 72 | width = image.shape[1] 73 | vis_image = np.zeros([height, width, 3], dtype=np.uint8) 74 | color_palette = create_color_palette() 75 | instances = np.unique(image) 76 | for idx, inst in enumerate(instances): 77 | vis_image[image==inst] = color_palette[inst%len(color_palette)] 78 | imageio.imwrite(filename, vis_image) 79 | 80 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/io3d.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import numpy as np 8 | import trimesh 9 | 10 | # color palette for nyu40 labels 11 | def create_color_palette(): 12 | return [ 13 | (0, 0, 0), 14 | (174, 199, 232), # wall 15 | (152, 223, 138), # floor 16 | (31, 119, 180), # cabinet 17 | (255, 187, 120), # bed 18 | (188, 189, 34), # chair 19 | (140, 86, 75), # sofa 20 | (255, 152, 150), # table 21 | (214, 39, 40), # door 22 | (197, 176, 213), # window 23 | (148, 103, 189), # bookshelf 24 | (196, 156, 148), # picture 25 | (23, 190, 207), # counter 26 | (178, 76, 76), 27 | (247, 182, 210), # desk 28 | (66, 188, 102), 29 | (219, 219, 141), # curtain 30 | (140, 57, 197), 31 | (202, 185, 52), 32 | (51, 176, 203), 33 | (200, 54, 131), 34 | (92, 193, 61), 35 | (78, 71, 183), 36 | (172, 114, 82), 37 | (255, 127, 14), # refrigerator 38 | (91, 163, 138), 39 | (153, 98, 156), 40 | (140, 153, 101), 41 | (158, 218, 229), # shower curtain 42 | (100, 125, 154), 43 | (178, 127, 135), 44 | (120, 185, 128), 45 | (146, 111, 194), 46 | (44, 160, 44), # toilet 47 | (112, 128, 144), # sink 48 | (96, 207, 209), 49 | (227, 119, 194), # bathtub 50 | (213, 92, 176), 51 | (94, 106, 211), 52 | (82, 84, 163), # otherfurn 53 | (100, 85, 144), 54 | ] 55 | 56 | def write_triangle_mesh(vertices, colors, faces, outputFile): 57 | mesh = trimesh.Trimesh(vertices=vertices, vertex_colors=colors, faces=faces, process=False) 58 | mesh.export(outputFile) 59 | 60 | def read_triangle_mesh(filename): 61 | mesh = trimesh.load_mesh(filename, process=False) 62 | if isinstance(mesh, trimesh.PointCloud): 63 | vertices = mesh.vertices 64 | colors = mesh.colors 65 | faces = None 66 | elif isinstance(mesh, trimesh.Trimesh): 67 | vertices = mesh.vertices 68 | colors = mesh.visual.vertex_colors 69 | faces = mesh.faces 70 | 71 | return vertices, colors, faces -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/multiprocessing_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/usr/bin/env python3 8 | 9 | """Multiprocessing helpers.""" 10 | 11 | import multiprocessing as mp 12 | import traceback 13 | 14 | from lib.error_handler import ErrorHandler 15 | 16 | import lib.distributed as du 17 | 18 | 19 | def run(proc_rank, world_size, error_queue, fun, fun_args, fun_kwargs): 20 | # Initialize the process group 21 | 22 | """Runs a function from a child process.""" 23 | try: 24 | # Initialize the process group 25 | du.init_process_group(proc_rank, world_size) 26 | # Run the function 27 | fun(*fun_args, **fun_kwargs) 28 | except KeyboardInterrupt: 29 | # Killed by the parent process 30 | pass 31 | except Exception: 32 | # Propagate exception to the parent process 33 | error_queue.put(traceback.format_exc()) 34 | finally: 35 | # Destroy the process group 36 | du.destroy_process_group() 37 | 38 | 39 | def multi_proc_run(num_proc, fun, fun_args=(), fun_kwargs={}): 40 | """Runs a function in a multi-proc setting.""" 41 | 42 | # Handle errors from training subprocesses 43 | error_queue = mp.SimpleQueue() 44 | error_handler = ErrorHandler(error_queue) 45 | 46 | # Run each training subprocess 47 | ps = [] 48 | for i in range(num_proc): 49 | p_i = mp.Process( 50 | target=run, 51 | args=(i, num_proc, error_queue, fun, fun_args, fun_kwargs) 52 | ) 53 | ps.append(p_i) 54 | p_i.start() 55 | error_handler.add_child(p_i.pid) 56 | 57 | # Wait for each subprocess to finish 58 | for p in ps: 59 | p.join() 60 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/timer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import time 8 | 9 | 10 | class AverageMeter(object): 11 | """Computes and stores the average and current value""" 12 | 13 | def __init__(self): 14 | self.reset() 15 | 16 | def reset(self): 17 | self.val = 0 18 | self.avg = 0 19 | self.sum = 0.0 20 | self.sq_sum = 0.0 21 | self.count = 0 22 | 23 | def update(self, val, n=1): 24 | self.val = val 25 | self.sum += val * n 26 | self.count += n 27 | self.avg = self.sum / self.count 28 | self.sq_sum += val**2 * n 29 | self.var = self.sq_sum / self.count - self.avg ** 2 30 | 31 | 32 | class Timer(object): 33 | """A simple timer.""" 34 | 35 | def __init__(self): 36 | self.total_time = 0. 37 | self.calls = 0 38 | self.start_time = 0. 39 | self.diff = 0. 40 | self.avg = 0. 41 | 42 | def reset(self): 43 | self.total_time = 0 44 | self.calls = 0 45 | self.start_time = 0 46 | self.diff = 0 47 | self.avg = 0 48 | 49 | def tic(self): 50 | # using time.time instead of time.clock because time time.clock 51 | # does not normalize for multithreading 52 | self.start_time = time.time() 53 | 54 | def toc(self, average=True): 55 | self.diff = time.time() - self.start_time 56 | self.total_time += self.diff 57 | self.calls += 1 58 | self.avg = self.total_time / self.calls 59 | if average: 60 | return self.avg 61 | else: 62 | return self.diff 63 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/lib/transforms.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import numpy as np 8 | import random 9 | 10 | 11 | class Compose: 12 | 13 | def __init__(self, transforms): 14 | self.transforms = transforms 15 | 16 | def __call__(self, coords, feats): 17 | for transform in self.transforms: 18 | coords, feats = transform(coords, feats) 19 | return coords, feats 20 | 21 | 22 | class Jitter: 23 | 24 | def __init__(self, mu=0, sigma=0.01): 25 | self.mu = mu 26 | self.sigma = sigma 27 | 28 | def __call__(self, coords, feats): 29 | if random.random() < 0.95: 30 | feats += np.random.normal(self.mu, self.sigma, (feats.shape[0], feats.shape[1])) 31 | return coords, feats 32 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import model.res16unet as res16unet 8 | import model.pointnet2backbone as pointnet2 9 | 10 | MODELS = [] 11 | 12 | 13 | def add_models(module): 14 | MODELS.extend([getattr(module, a) for a in dir(module) if 'Net' in a]) 15 | 16 | add_models(res16unet) 17 | add_models(pointnet2) 18 | 19 | def get_models(): 20 | '''Returns a tuple of sample models.''' 21 | return MODELS 22 | 23 | def load_model(name): 24 | '''Creates and returns an instance of the model given its class name. 25 | ''' 26 | all_models = get_models() 27 | mdict = {model.__name__: model for model in all_models} 28 | if name not in mdict: 29 | print('Invalid model index. Options are:') 30 | for model in all_models: 31 | print('\t* {}'.format(model.__name__)) 32 | return None 33 | NetClass = mdict[name] 34 | return NetClass 35 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/include/ball_query.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor ball_query(at::Tensor new_xyz, at::Tensor xyz, const float radius, 5 | const int nsample); 6 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/include/cuda_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef _CUDA_UTILS_H 2 | #define _CUDA_UTILS_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #define TOTAL_THREADS 512 14 | 15 | inline int opt_n_threads(int work_size) { 16 | const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0); 17 | 18 | return max(min(1 << pow_2, TOTAL_THREADS), 1); 19 | } 20 | 21 | inline dim3 opt_block_config(int x, int y) { 22 | const int x_threads = opt_n_threads(x); 23 | const int y_threads = 24 | max(min(opt_n_threads(y), TOTAL_THREADS / x_threads), 1); 25 | dim3 block_config(x_threads, y_threads, 1); 26 | 27 | return block_config; 28 | } 29 | 30 | #define CUDA_CHECK_ERRORS() \ 31 | do { \ 32 | cudaError_t err = cudaGetLastError(); \ 33 | if (cudaSuccess != err) { \ 34 | fprintf(stderr, "CUDA kernel failed : %s\n%s at L:%d in %s\n", \ 35 | cudaGetErrorString(err), __PRETTY_FUNCTION__, __LINE__, \ 36 | __FILE__); \ 37 | exit(-1); \ 38 | } \ 39 | } while (0) 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/include/group_points.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor group_points(at::Tensor points, at::Tensor idx); 5 | at::Tensor group_points_grad(at::Tensor grad_out, at::Tensor idx, const int n); 6 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/include/interpolate.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | std::vector three_nn(at::Tensor unknowns, at::Tensor knows); 7 | at::Tensor three_interpolate(at::Tensor points, at::Tensor idx, 8 | at::Tensor weight); 9 | at::Tensor three_interpolate_grad(at::Tensor grad_out, at::Tensor idx, 10 | at::Tensor weight, const int m); 11 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/include/sampling.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor gather_points(at::Tensor points, at::Tensor idx); 5 | at::Tensor gather_points_grad(at::Tensor grad_out, at::Tensor idx, const int n); 6 | at::Tensor furthest_point_sampling(at::Tensor points, const int nsamples); 7 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/include/utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #define CHECK_CUDA(x) \ 6 | do { \ 7 | AT_ASSERT(x.is_cuda(), #x " must be a CUDA tensor"); \ 8 | } while (0) 9 | 10 | #define CHECK_CONTIGUOUS(x) \ 11 | do { \ 12 | AT_ASSERT(x.is_contiguous(), #x " must be a contiguous tensor"); \ 13 | } while (0) 14 | 15 | #define CHECK_IS_INT(x) \ 16 | do { \ 17 | AT_ASSERT(x.scalar_type() == at::ScalarType::Int, \ 18 | #x " must be an int tensor"); \ 19 | } while (0) 20 | 21 | #define CHECK_IS_FLOAT(x) \ 22 | do { \ 23 | AT_ASSERT(x.scalar_type() == at::ScalarType::Float, \ 24 | #x " must be a float tensor"); \ 25 | } while (0) 26 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/src/ball_query.cpp: -------------------------------------------------------------------------------- 1 | #include "ball_query.h" 2 | #include "utils.h" 3 | 4 | void query_ball_point_kernel_wrapper(int b, int n, int m, float radius, 5 | int nsample, const float *new_xyz, 6 | const float *xyz, int *idx); 7 | 8 | at::Tensor ball_query(at::Tensor new_xyz, at::Tensor xyz, const float radius, 9 | const int nsample) { 10 | CHECK_CONTIGUOUS(new_xyz); 11 | CHECK_CONTIGUOUS(xyz); 12 | CHECK_IS_FLOAT(new_xyz); 13 | CHECK_IS_FLOAT(xyz); 14 | 15 | if (new_xyz.is_cuda()) { 16 | CHECK_CUDA(xyz); 17 | } 18 | 19 | at::Tensor idx = 20 | torch::zeros({new_xyz.size(0), new_xyz.size(1), nsample}, 21 | at::device(new_xyz.device()).dtype(at::ScalarType::Int)); 22 | 23 | if (new_xyz.is_cuda()) { 24 | query_ball_point_kernel_wrapper(xyz.size(0), xyz.size(1), new_xyz.size(1), 25 | radius, nsample, new_xyz.data_ptr(), 26 | xyz.data_ptr(), idx.data_ptr()); 27 | } else { 28 | AT_ASSERT(false, "CPU not supported"); 29 | } 30 | 31 | return idx; 32 | } 33 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/src/ball_query_gpu.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "cuda_utils.h" 6 | 7 | // input: new_xyz(b, m, 3) xyz(b, n, 3) 8 | // output: idx(b, m, nsample) 9 | __global__ void query_ball_point_kernel(int b, int n, int m, float radius, 10 | int nsample, 11 | const float *__restrict__ new_xyz, 12 | const float *__restrict__ xyz, 13 | int *__restrict__ idx) { 14 | int batch_index = blockIdx.x; 15 | xyz += batch_index * n * 3; 16 | new_xyz += batch_index * m * 3; 17 | idx += m * nsample * batch_index; 18 | 19 | int index = threadIdx.x; 20 | int stride = blockDim.x; 21 | 22 | float radius2 = radius * radius; 23 | for (int j = index; j < m; j += stride) { 24 | float new_x = new_xyz[j * 3 + 0]; 25 | float new_y = new_xyz[j * 3 + 1]; 26 | float new_z = new_xyz[j * 3 + 2]; 27 | for (int k = 0, cnt = 0; k < n && cnt < nsample; ++k) { 28 | float x = xyz[k * 3 + 0]; 29 | float y = xyz[k * 3 + 1]; 30 | float z = xyz[k * 3 + 2]; 31 | float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + 32 | (new_z - z) * (new_z - z); 33 | if (d2 < radius2) { 34 | if (cnt == 0) { 35 | for (int l = 0; l < nsample; ++l) { 36 | idx[j * nsample + l] = k; 37 | } 38 | } 39 | idx[j * nsample + cnt] = k; 40 | ++cnt; 41 | } 42 | } 43 | } 44 | } 45 | 46 | void query_ball_point_kernel_wrapper(int b, int n, int m, float radius, 47 | int nsample, const float *new_xyz, 48 | const float *xyz, int *idx) { 49 | cudaStream_t stream = at::cuda::getCurrentCUDAStream(); 50 | query_ball_point_kernel<<>>( 51 | b, n, m, radius, nsample, new_xyz, xyz, idx); 52 | 53 | CUDA_CHECK_ERRORS(); 54 | } 55 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/src/bindings.cpp: -------------------------------------------------------------------------------- 1 | #include "ball_query.h" 2 | #include "group_points.h" 3 | #include "interpolate.h" 4 | #include "sampling.h" 5 | 6 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 7 | m.def("gather_points", &gather_points); 8 | m.def("gather_points_grad", &gather_points_grad); 9 | m.def("furthest_point_sampling", &furthest_point_sampling); 10 | 11 | m.def("three_nn", &three_nn); 12 | m.def("three_interpolate", &three_interpolate); 13 | m.def("three_interpolate_grad", &three_interpolate_grad); 14 | 15 | m.def("ball_query", &ball_query); 16 | 17 | m.def("group_points", &group_points); 18 | m.def("group_points_grad", &group_points_grad); 19 | } 20 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/src/group_points.cpp: -------------------------------------------------------------------------------- 1 | #include "group_points.h" 2 | #include "utils.h" 3 | 4 | void group_points_kernel_wrapper(int b, int c, int n, int npoints, int nsample, 5 | const float *points, const int *idx, 6 | float *out); 7 | 8 | void group_points_grad_kernel_wrapper(int b, int c, int n, int npoints, 9 | int nsample, const float *grad_out, 10 | const int *idx, float *grad_points); 11 | 12 | at::Tensor group_points(at::Tensor points, at::Tensor idx) { 13 | CHECK_CONTIGUOUS(points); 14 | CHECK_CONTIGUOUS(idx); 15 | CHECK_IS_FLOAT(points); 16 | CHECK_IS_INT(idx); 17 | 18 | if (points.is_cuda()) { 19 | CHECK_CUDA(idx); 20 | } 21 | 22 | at::Tensor output = 23 | torch::zeros({points.size(0), points.size(1), idx.size(1), idx.size(2)}, 24 | at::device(points.device()).dtype(at::ScalarType::Float)); 25 | 26 | if (points.is_cuda()) { 27 | group_points_kernel_wrapper(points.size(0), points.size(1), points.size(2), 28 | idx.size(1), idx.size(2), 29 | points.data_ptr(), idx.data_ptr(), 30 | output.data_ptr()); 31 | } else { 32 | AT_ASSERT(false, "CPU not supported"); 33 | } 34 | 35 | return output; 36 | } 37 | 38 | at::Tensor group_points_grad(at::Tensor grad_out, at::Tensor idx, const int n) { 39 | CHECK_CONTIGUOUS(grad_out); 40 | CHECK_CONTIGUOUS(idx); 41 | CHECK_IS_FLOAT(grad_out); 42 | CHECK_IS_INT(idx); 43 | 44 | if (grad_out.is_cuda()) { 45 | CHECK_CUDA(idx); 46 | } 47 | 48 | at::Tensor output = 49 | torch::zeros({grad_out.size(0), grad_out.size(1), n}, 50 | at::device(grad_out.device()).dtype(at::ScalarType::Float)); 51 | 52 | if (grad_out.is_cuda()) { 53 | group_points_grad_kernel_wrapper( 54 | grad_out.size(0), grad_out.size(1), n, idx.size(1), idx.size(2), 55 | grad_out.data_ptr(), idx.data_ptr(), 56 | output.data_ptr()); 57 | } else { 58 | AT_ASSERT(false, "CPU not supported"); 59 | } 60 | 61 | return output; 62 | } 63 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/src/group_points_gpu.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "cuda_utils.h" 5 | 6 | // input: points(b, c, n) idx(b, npoints, nsample) 7 | // output: out(b, c, npoints, nsample) 8 | __global__ void group_points_kernel(int b, int c, int n, int npoints, 9 | int nsample, 10 | const float *__restrict__ points, 11 | const int *__restrict__ idx, 12 | float *__restrict__ out) { 13 | int batch_index = blockIdx.x; 14 | points += batch_index * n * c; 15 | idx += batch_index * npoints * nsample; 16 | out += batch_index * npoints * nsample * c; 17 | 18 | const int index = threadIdx.y * blockDim.x + threadIdx.x; 19 | const int stride = blockDim.y * blockDim.x; 20 | for (int i = index; i < c * npoints; i += stride) { 21 | const int l = i / npoints; 22 | const int j = i % npoints; 23 | for (int k = 0; k < nsample; ++k) { 24 | int ii = idx[j * nsample + k]; 25 | out[(l * npoints + j) * nsample + k] = points[l * n + ii]; 26 | } 27 | } 28 | } 29 | 30 | void group_points_kernel_wrapper(int b, int c, int n, int npoints, int nsample, 31 | const float *points, const int *idx, 32 | float *out) { 33 | cudaStream_t stream = at::cuda::getCurrentCUDAStream(); 34 | 35 | group_points_kernel<<>>( 36 | b, c, n, npoints, nsample, points, idx, out); 37 | 38 | CUDA_CHECK_ERRORS(); 39 | } 40 | 41 | // input: grad_out(b, c, npoints, nsample), idx(b, npoints, nsample) 42 | // output: grad_points(b, c, n) 43 | __global__ void group_points_grad_kernel(int b, int c, int n, int npoints, 44 | int nsample, 45 | const float *__restrict__ grad_out, 46 | const int *__restrict__ idx, 47 | float *__restrict__ grad_points) { 48 | int batch_index = blockIdx.x; 49 | grad_out += batch_index * npoints * nsample * c; 50 | idx += batch_index * npoints * nsample; 51 | grad_points += batch_index * n * c; 52 | 53 | const int index = threadIdx.y * blockDim.x + threadIdx.x; 54 | const int stride = blockDim.y * blockDim.x; 55 | for (int i = index; i < c * npoints; i += stride) { 56 | const int l = i / npoints; 57 | const int j = i % npoints; 58 | for (int k = 0; k < nsample; ++k) { 59 | int ii = idx[j * nsample + k]; 60 | atomicAdd(grad_points + l * n + ii, 61 | grad_out[(l * npoints + j) * nsample + k]); 62 | } 63 | } 64 | } 65 | 66 | void group_points_grad_kernel_wrapper(int b, int c, int n, int npoints, 67 | int nsample, const float *grad_out, 68 | const int *idx, float *grad_points) { 69 | cudaStream_t stream = at::cuda::getCurrentCUDAStream(); 70 | 71 | group_points_grad_kernel<<>>( 72 | b, c, n, npoints, nsample, grad_out, idx, grad_points); 73 | 74 | CUDA_CHECK_ERRORS(); 75 | } 76 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/_ext_src/src/sampling.cpp: -------------------------------------------------------------------------------- 1 | #include "sampling.h" 2 | #include "utils.h" 3 | 4 | void gather_points_kernel_wrapper(int b, int c, int n, int npoints, 5 | const float *points, const int *idx, 6 | float *out); 7 | void gather_points_grad_kernel_wrapper(int b, int c, int n, int npoints, 8 | const float *grad_out, const int *idx, 9 | float *grad_points); 10 | 11 | void furthest_point_sampling_kernel_wrapper(int b, int n, int m, 12 | const float *dataset, float *temp, 13 | int *idxs); 14 | 15 | at::Tensor gather_points(at::Tensor points, at::Tensor idx) { 16 | CHECK_CONTIGUOUS(points); 17 | CHECK_CONTIGUOUS(idx); 18 | CHECK_IS_FLOAT(points); 19 | CHECK_IS_INT(idx); 20 | 21 | if (points.is_cuda()) { 22 | CHECK_CUDA(idx); 23 | } 24 | 25 | at::Tensor output = 26 | torch::zeros({points.size(0), points.size(1), idx.size(1)}, 27 | at::device(points.device()).dtype(at::ScalarType::Float)); 28 | 29 | if (points.is_cuda()) { 30 | gather_points_kernel_wrapper(points.size(0), points.size(1), points.size(2), 31 | idx.size(1), points.data_ptr(), 32 | idx.data_ptr(), output.data_ptr()); 33 | } else { 34 | AT_ASSERT(false, "CPU not supported"); 35 | } 36 | 37 | return output; 38 | } 39 | 40 | at::Tensor gather_points_grad(at::Tensor grad_out, at::Tensor idx, 41 | const int n) { 42 | CHECK_CONTIGUOUS(grad_out); 43 | CHECK_CONTIGUOUS(idx); 44 | CHECK_IS_FLOAT(grad_out); 45 | CHECK_IS_INT(idx); 46 | 47 | if (grad_out.is_cuda()) { 48 | CHECK_CUDA(idx); 49 | } 50 | 51 | at::Tensor output = 52 | torch::zeros({grad_out.size(0), grad_out.size(1), n}, 53 | at::device(grad_out.device()).dtype(at::ScalarType::Float)); 54 | 55 | if (grad_out.is_cuda()) { 56 | gather_points_grad_kernel_wrapper(grad_out.size(0), grad_out.size(1), n, 57 | idx.size(1), grad_out.data_ptr(), 58 | idx.data_ptr(), 59 | output.data_ptr()); 60 | } else { 61 | AT_ASSERT(false, "CPU not supported"); 62 | } 63 | 64 | return output; 65 | } 66 | at::Tensor furthest_point_sampling(at::Tensor points, const int nsamples) { 67 | CHECK_CONTIGUOUS(points); 68 | CHECK_IS_FLOAT(points); 69 | 70 | at::Tensor output = 71 | torch::zeros({points.size(0), nsamples}, 72 | at::device(points.device()).dtype(at::ScalarType::Int)); 73 | 74 | at::Tensor tmp = 75 | torch::full({points.size(0), points.size(1)}, 1e10, 76 | at::device(points.device()).dtype(at::ScalarType::Float)); 77 | 78 | if (points.is_cuda()) { 79 | furthest_point_sampling_kernel_wrapper( 80 | points.size(0), points.size(1), nsamples, points.data_ptr(), 81 | tmp.data_ptr(), output.data_ptr()); 82 | } else { 83 | AT_ASSERT(false, "CPU not supported"); 84 | } 85 | 86 | return output; 87 | } 88 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/pointnet2_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | ''' Testing customized ops. ''' 7 | 8 | import torch 9 | from torch.autograd import gradcheck 10 | import numpy as np 11 | 12 | import os 13 | import sys 14 | BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 15 | sys.path.append(BASE_DIR) 16 | import pointnet2_utils 17 | 18 | def test_interpolation_grad(): 19 | batch_size = 1 20 | feat_dim = 2 21 | m = 4 22 | feats = torch.randn(batch_size, feat_dim, m, requires_grad=True).float().cuda() 23 | 24 | def interpolate_func(inputs): 25 | idx = torch.from_numpy(np.array([[[0,1,2],[1,2,3]]])).int().cuda() 26 | weight = torch.from_numpy(np.array([[[1,1,1],[2,2,2]]])).float().cuda() 27 | interpolated_feats = pointnet2_utils.three_interpolate(inputs, idx, weight) 28 | return interpolated_feats 29 | 30 | assert (gradcheck(interpolate_func, feats, atol=1e-1, rtol=1e-1)) 31 | 32 | if __name__=='__main__': 33 | test_interpolation_grad() 34 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/model/pointnet2/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import glob 7 | import os 8 | 9 | from setuptools import setup 10 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 11 | 12 | this_dir = os.path.dirname(os.path.abspath(__file__)) 13 | _ext_src_root = "_ext_src" 14 | _ext_sources = glob.glob("{}/src/*.cpp".format(_ext_src_root)) + glob.glob( 15 | "{}/src/*.cu".format(_ext_src_root) 16 | ) 17 | 18 | setup( 19 | name='pointnet2', 20 | ext_modules=[ 21 | CUDAExtension( 22 | name='pointnet2._ext', 23 | sources=_ext_sources, 24 | extra_compile_args={ 25 | "cxx": ["-O3"], 26 | "nvcc": ["-O3", "-Xfatbin", "-compress-all"], 27 | }, 28 | include_dirs=[os.path.join(this_dir, _ext_src_root, "include")], 29 | ) 30 | ], 31 | cmdclass={ 32 | 'build_ext': BuildExtension 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/scripts/pretrain_pointnet2.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | #!/bin/bash 7 | 8 | MODEL=PointNet2Backbone 9 | 10 | python ddp_train.py -m \ 11 | net.model=$MODEL \ 12 | net.conv1_kernel_size=3 \ 13 | net.model_n_out=32 \ 14 | opt.lr=0.1 \ 15 | opt.max_iter=100000 \ 16 | data.dataset=ScanNetMatchPairDataset \ 17 | data.voxelize=False \ 18 | data.num_points=20000 \ 19 | data.world_space=True \ 20 | trainer.trainer=PartitionPointNCELossTrainerPointNet \ 21 | trainer.batch_size=32 \ 22 | trainer.stat_freq=5 \ 23 | trainer.checkpoint_freq=1000 \ 24 | trainer.lr_update_freq=1000 \ 25 | shape_context.r1=0.05 \ 26 | shape_context.r2=0.5 \ 27 | shape_context.nbins_xy=2 \ 28 | shape_context.nbins_zy=2 \ 29 | shape_context.weight_inner=False \ 30 | shape_context.fast_partition=True \ 31 | misc.num_gpus=8 \ 32 | misc.train_num_thread=2 \ 33 | misc.npos=4096 \ 34 | misc.nceT=0.4 \ 35 | misc.out_dir=${OUT_DIR} \ 36 | hydra.launcher.partition=priority \ 37 | hydra.launcher.timeout_min=3600 \ 38 | hydra.launcher.max_num_timeout=5 \ 39 | hydra.launcher.comment=criticalExp \ 40 | -------------------------------------------------------------------------------- /pretrain/contrastive_scene_contexts/scripts/pretrain_sparseconv.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | #!/bin/bash 8 | 9 | MODEL=Res16UNet34C 10 | 11 | python ddp_train.py -m \ 12 | net.model=$MODEL \ 13 | net.conv1_kernel_size=3 \ 14 | net.model_n_out=32 \ 15 | opt.lr=0.1 \ 16 | opt.max_iter=100000 \ 17 | data.dataset=ScanNetMatchPairDataset \ 18 | data.voxel_size=0.025\ 19 | data.voxelize=True \ 20 | data.scannet_match_dir=$DATASET \ 21 | data.world_space=True \ 22 | trainer.trainer=PartitionPointNCELossTrainer \ 23 | trainer.batch_size=32 \ 24 | trainer.stat_freq=5 \ 25 | trainer.checkpoint_freq=1000 \ 26 | trainer.lr_update_freq=1000 \ 27 | shape_context.r1=2.0 \ 28 | shape_context.r2=20.0 \ 29 | shape_context.nbins_xy=2 \ 30 | shape_context.nbins_zy=2 \ 31 | shape_context.weight_inner=False \ 32 | shape_context.fast_partition=True \ 33 | misc.num_gpus=8 \ 34 | misc.train_num_thread=2 \ 35 | misc.npos=4096 \ 36 | misc.nceT=0.4 \ 37 | misc.out_dir=${OUT_DIR} \ 38 | hydra.launcher.partition=priority \ 39 | hydra.launcher.timeout_min=3600 \ 40 | hydra.launcher.max_num_timeout=5 \ 41 | hydra.launcher.comment=criticalExp \ 42 | -------------------------------------------------------------------------------- /pretrain/scannet_pair/__pycache__/SensorData.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/pretrain/scannet_pair/__pycache__/SensorData.cpython-36.pyc -------------------------------------------------------------------------------- /pretrain/scannet_pair/__pycache__/SensorData.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/pretrain/scannet_pair/__pycache__/SensorData.cpython-38.pyc -------------------------------------------------------------------------------- /pretrain/scannet_pair/__pycache__/plyfile.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/ContrastiveSceneContexts/83515bef4754b3d90fc3b3a437fa939e0e861af8/pretrain/scannet_pair/__pycache__/plyfile.cpython-36.pyc -------------------------------------------------------------------------------- /pretrain/scannet_pair/compute_full_overlapping.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import copy 7 | import numpy as np 8 | import math 9 | import glob, os 10 | import argparse 11 | import open3d as o3d 12 | 13 | 14 | def make_open3d_point_cloud(xyz, color=None, voxel_size=None): 15 | if np.isnan(xyz).any(): 16 | return None 17 | 18 | xyz = xyz[:,:3] 19 | pcd = o3d.geometry.PointCloud() 20 | pcd.points = o3d.utility.Vector3dVector(xyz) 21 | if color is not None: 22 | pcd.colors = o3d.utility.Vector3dVector(color) 23 | if voxel_size is not None: 24 | pcd = pcd.voxel_down_sample(voxel_size) 25 | 26 | return pcd 27 | 28 | 29 | def compute_overlap_ratio(pcd0, pcd1, voxel_size): 30 | pcd0_down = pcd0.voxel_down_sample(voxel_size) 31 | pcd1_down = pcd1.voxel_down_sample(voxel_size) 32 | matching01 = get_matching_indices(pcd0_down, pcd1_down, voxel_size * 1.5, 1) 33 | matching10 = get_matching_indices(pcd1_down, pcd0_down, voxel_size * 1.5, 1) 34 | overlap0 = float(len(matching01)) / float(len(pcd0_down.points)) 35 | overlap1 = float(len(matching10)) / float(len(pcd1_down.points)) 36 | return max(overlap0, overlap1) 37 | 38 | 39 | def get_matching_indices(source, pcd_tree, search_voxel_size, K=None): 40 | match_inds = [] 41 | for i, point in enumerate(source.points): 42 | [_, idx, _] = pcd_tree.search_radius_vector_3d(point, search_voxel_size) 43 | if K is not None: 44 | idx = idx[:K] 45 | for j in idx: 46 | match_inds.append((i, j)) 47 | return match_inds 48 | 49 | # params 50 | parser = argparse.ArgumentParser() 51 | # data paths 52 | parser.add_argument('--input_path', required=True, help='path to sens file to read') 53 | parser.add_argument('--voxel_size', type=float, default=0.05) 54 | opt = parser.parse_args() 55 | print(opt) 56 | 57 | print('load point clouds and downsampling...') 58 | 59 | _points = [ 60 | (pcd_name, make_open3d_point_cloud(np.load(pcd_name)['pcd'], voxel_size=opt.voxel_size)) 61 | for pcd_name in glob.glob(os.path.join(opt.input_path, "*.npz")) 62 | ] 63 | points = [(pcd_name, pcd) for (pcd_name, pcd) in _points if pcd is not None] 64 | print('load {} point clouds ({} invalid has been filtered), computing matching/overlapping'.format( 65 | len(points), len(_points) - len(points))) 66 | 67 | matching_matrix = np.zeros((len(points), len(points))) 68 | for i, (pcd0_name, pcd0) in enumerate(points): 69 | print('matching to...{}'.format(pcd0_name)) 70 | pcd0_tree = o3d.geometry.KDTreeFlann(copy.deepcopy(pcd0)) 71 | for j, (pcd1_name, pcd1) in enumerate(points): 72 | if i == j: 73 | continue 74 | matching_matrix[i, j] = float(len(get_matching_indices(pcd1, pcd0_tree, 1.5 * opt.voxel_size, 1))) / float(len(pcd1.points)) 75 | 76 | # write to file 77 | print('writing to file') 78 | with open(os.path.join(opt.input_path, "overlap.txt"), 'w') as f: 79 | for i, (pcd0_name, pcd0) in enumerate(points): 80 | for j, (pcd1_name, pcd1) in enumerate(points): 81 | if i < j: 82 | overlap = max(matching_matrix[i, j], matching_matrix[j, i]) 83 | f.write("{} {} {}\n".format(pcd0_name, pcd1_name, overlap)) 84 | 85 | print('done.') 86 | -------------------------------------------------------------------------------- /pretrain/scannet_pair/generage_list.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import argparse 8 | import glob, os, sys 9 | 10 | from SensorData import SensorData 11 | 12 | # params 13 | parser = argparse.ArgumentParser() 14 | # data paths 15 | parser.add_argument('--target_dir', required=True, help='path to the target dir') 16 | 17 | opt = parser.parse_args() 18 | print(opt) 19 | 20 | def main(): 21 | overlaps = glob.glob(os.path.join(opt.target_dir, "*/pcd/overlap.txt")) 22 | with open(os.path.join(opt.target_dir, 'overlap30.txt'), 'w') as f: 23 | for fo in overlaps: 24 | for line in open(fo): 25 | pcd0, pcd1, op = line.strip().split() 26 | if float(op) >= 0.3: 27 | print('{} {} {}'.format(pcd0, pcd1, op), file=f) 28 | print('done') 29 | 30 | if __name__ == '__main__': 31 | main() -------------------------------------------------------------------------------- /pretrain/scannet_pair/preprocess.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | export SCANNET_DIR= 7 | export TARGET= # data destination (change here) 8 | 9 | reader() { 10 | filename=$1 11 | frame_skip=25 12 | 13 | scene=$(basename -- "$filename") 14 | scene="${scene%.*}" 15 | echo "Find sens data: $filename $scene" 16 | python -u reader.py --filename $filename --output_path $TARGET/$scene --frame_skip $frame_skip --export_depth_images --export_poses --export_intrinsics 17 | echo "Extract point-cloud data" 18 | python -u point_cloud_extractor.py --input_path $TARGET/$scene --output_path $TARGET/$scene/pcd --save_npz 19 | echo "Compute partial scan overlapping" 20 | python -u compute_full_overlapping.py --input_path $TARGET/$scene/pcd 21 | } 22 | 23 | export -f reader 24 | 25 | parallel -j 64 --linebuffer time reader ::: `find $SCANNET_DIR/scans/scene*/*.sens` 26 | -------------------------------------------------------------------------------- /pretrain/scannet_pair/reader.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os, sys 3 | 4 | from SensorData import SensorData 5 | 6 | # params 7 | parser = argparse.ArgumentParser() 8 | # data paths 9 | parser.add_argument('--filename', required=True, help='path to sens file to read') 10 | parser.add_argument('--output_path', required=True, help='path to output folder') 11 | parser.add_argument('--export_depth_images', dest='export_depth_images', action='store_true') 12 | parser.add_argument('--export_color_images', dest='export_color_images', action='store_true') 13 | parser.add_argument('--export_poses', dest='export_poses', action='store_true') 14 | parser.add_argument('--export_intrinsics', dest='export_intrinsics', action='store_true') 15 | parser.add_argument('--frame_skip', type=int, default=25) 16 | parser.set_defaults(export_depth_images=True, export_color_images=True, export_poses=True, export_intrinsics=True) 17 | 18 | opt = parser.parse_args() 19 | print(opt) 20 | 21 | 22 | def main(): 23 | if not os.path.exists(opt.output_path): 24 | os.makedirs(opt.output_path) 25 | 26 | # load the data 27 | print('loading %s...' % opt.filename) 28 | sd = SensorData(opt.filename) 29 | print('loaded!\n') 30 | if opt.export_depth_images: 31 | sd.export_depth_images(os.path.join(opt.output_path, 'depth'), frame_skip=opt.frame_skip) 32 | if opt.export_color_images: 33 | sd.export_color_images(os.path.join(opt.output_path, 'color'), frame_skip=opt.frame_skip) 34 | if opt.export_poses: 35 | sd.export_poses(os.path.join(opt.output_path, 'pose'), frame_skip=opt.frame_skip) 36 | if opt.export_intrinsics: 37 | sd.export_intrinsics(os.path.join(opt.output_path, 'intrinsic')) 38 | 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | hydra-core==1.0.0 2 | tensorboardX==2.0 3 | scipy==1.5.4 4 | scikit-learn==0.23.1 5 | plyfile==0.4 6 | pandas==1.0.5 7 | trimesh==3.7.5 8 | imageio==2.8.0 9 | hydra-colorlog==1.0.0 10 | hydra-submitit-launcher==1.1.0 11 | matplotlib==3.2.2 12 | opencv-python==4.5.1.48 13 | --------------------------------------------------------------------------------