├── second ├── __init__.py ├── data │ ├── __init__.py │ └── dataset.py ├── builder │ ├── __init__.py │ ├── preprocess_builder.py │ ├── dbsampler_builder.py │ ├── voxel_builder.py │ ├── similarity_calculator_builder.py │ ├── anchor_generator_builder.py │ ├── target_assigner_builder.py │ └── dataset_builder.py ├── protos │ ├── __init__.py │ ├── layers.proto │ ├── model.proto │ ├── voxel_generator.proto │ ├── target.proto │ ├── sampler.proto │ ├── pipeline.proto │ ├── box_coder.proto │ ├── activations.proto │ ├── similarity.proto │ ├── layers_pb2.py │ ├── train.proto │ ├── preprocess.proto │ ├── input_reader.proto │ ├── anchors.proto │ ├── second.proto │ ├── optimizer.proto │ ├── model_pb2.py │ ├── voxel_generator_pb2.py │ ├── pipeline_pb2.py │ ├── losses.proto │ └── target_pb2.py ├── pytorch │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ └── box_coders.py │ ├── builder │ │ ├── __init__.py │ │ ├── box_coder_builder.py │ │ ├── input_reader_builder.py │ │ ├── optimizer_builder.py │ │ ├── lr_scheduler_builder.py │ │ └── second_builder.py │ ├── models │ │ └── __init__.py │ ├── utils.py │ └── inference.py ├── utils │ ├── __init__.py │ ├── buildtools │ │ ├── __init__.py │ │ └── pybind11_build.py │ ├── check.py │ └── loader.py ├── kittiviewer │ ├── __init__.py │ └── frontend │ │ ├── textures │ │ └── sprites │ │ │ └── disc.png │ │ ├── js │ │ ├── shaders │ │ │ ├── CopyShader.js │ │ │ ├── LuminosityHighPassShader.js │ │ │ ├── ConvolutionShader.js │ │ │ ├── FocusShader.js │ │ │ └── FilmShader.js │ │ ├── Toast.js │ │ ├── postprocessing │ │ │ ├── RenderPass.js │ │ │ ├── ShaderPass.js │ │ │ ├── FilmPass.js │ │ │ ├── MaskPass.js │ │ │ ├── BloomPass.js │ │ │ └── EffectComposer.js │ │ └── renderers │ │ │ └── CSS2DRenderer.js │ │ └── css │ │ └── main.css ├── core │ ├── point_cloud │ │ ├── __init__.py │ │ └── bev_ops.py │ ├── __init__.py │ ├── non_max_suppression │ │ └── __init__.py │ ├── cc │ │ ├── box_ops.cc │ │ ├── point_cloud_ops.cc │ │ ├── nms │ │ │ ├── nms.h │ │ │ └── nms.cc │ │ ├── point_cloud_ops.h │ │ └── box_ops.h │ ├── voxel_generator.py │ ├── box_coders.py │ ├── anchor_generator.py │ ├── target_assigner.py │ ├── inference.py │ └── region_similarity.py └── configs │ ├── pointpillars │ ├── README.md │ ├── car │ │ ├── xyres_20.proto │ │ ├── xyres_16.proto │ │ ├── xyres_24.proto │ │ └── xyres_28.proto │ └── ped_cycle │ │ └── xyres_20.proto │ └── car.tiny.config ├── torchplus ├── ops │ ├── __init__.py │ └── array_ops.py ├── nn │ ├── modules │ │ ├── __init__.py │ │ ├── normalization.py │ │ └── common.py │ ├── __init__.py │ └── functional.py ├── __init__.py ├── train │ ├── __init__.py │ ├── common.py │ └── optim.py └── tools.py ├── images ├── kittibox.png ├── simpleguide.png ├── viewerweb.png └── pointpillars_kitti_results.png ├── LICENSE ├── .gitignore ├── Dockerfile └── README.md /second/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/builder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/protos/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /torchplus/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/kittiviewer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /torchplus/nn/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/core/point_cloud/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/builder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/utils/buildtools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/protos/layers.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second; 4 | 5 | -------------------------------------------------------------------------------- /images/kittibox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nutonomy/second.pytorch/HEAD/images/kittibox.png -------------------------------------------------------------------------------- /images/simpleguide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nutonomy/second.pytorch/HEAD/images/simpleguide.png -------------------------------------------------------------------------------- /images/viewerweb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nutonomy/second.pytorch/HEAD/images/viewerweb.png -------------------------------------------------------------------------------- /second/core/__init__.py: -------------------------------------------------------------------------------- 1 | # from . import box_np_ops, box_tf_ops, geometry, preprocess, non_max_suppression -------------------------------------------------------------------------------- /second/pytorch/models/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch import nn 4 | 5 | from torch.nn import functional -------------------------------------------------------------------------------- /images/pointpillars_kitti_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nutonomy/second.pytorch/HEAD/images/pointpillars_kitti_results.png -------------------------------------------------------------------------------- /second/kittiviewer/frontend/textures/sprites/disc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nutonomy/second.pytorch/HEAD/second/kittiviewer/frontend/textures/sprites/disc.png -------------------------------------------------------------------------------- /second/configs/pointpillars/README.md: -------------------------------------------------------------------------------- 1 | # PointPillars Configs 2 | 3 | The configuration files in these directories can be used to reproduce the results published in PointPillars. 4 | -------------------------------------------------------------------------------- /torchplus/nn/__init__.py: -------------------------------------------------------------------------------- 1 | from torchplus.nn.functional import one_hot 2 | from torchplus.nn.modules.common import Empty, Sequential 3 | from torchplus.nn.modules.normalization import GroupNorm 4 | -------------------------------------------------------------------------------- /second/protos/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/second.proto"; 5 | message DetectionModel{ 6 | oneof model { 7 | VoxelNet second = 1; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /torchplus/__init__.py: -------------------------------------------------------------------------------- 1 | from . import train 2 | from . import nn 3 | from . import metrics 4 | from . import tools 5 | 6 | from .tools import change_default_args 7 | from torchplus.ops.array_ops import scatter_nd, gather_nd 8 | -------------------------------------------------------------------------------- /second/core/non_max_suppression/__init__.py: -------------------------------------------------------------------------------- 1 | from second.core.non_max_suppression.nms_cpu import nms_jit, soft_nms_jit 2 | from second.core.non_max_suppression.nms_gpu import (nms_gpu, rotate_iou_gpu, 3 | rotate_nms_gpu) 4 | -------------------------------------------------------------------------------- /torchplus/nn/functional.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def one_hot(tensor, depth, dim=-1, on_value=1.0, dtype=torch.float32): 4 | tensor_onehot = torch.zeros( 5 | *list(tensor.shape), depth, dtype=dtype, device=tensor.device) 6 | tensor_onehot.scatter_(dim, tensor.unsqueeze(dim).long(), on_value) 7 | return tensor_onehot 8 | -------------------------------------------------------------------------------- /torchplus/nn/modules/normalization.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class GroupNorm(torch.nn.GroupNorm): 5 | def __init__(self, num_channels, num_groups, eps=1e-5, affine=True): 6 | super().__init__( 7 | num_groups=num_groups, 8 | num_channels=num_channels, 9 | eps=eps, 10 | affine=affine) 11 | -------------------------------------------------------------------------------- /torchplus/train/__init__.py: -------------------------------------------------------------------------------- 1 | from torchplus.train.checkpoint import (latest_checkpoint, restore, 2 | restore_latest_checkpoints, 3 | restore_models, save, save_models, 4 | try_restore_latest_checkpoints) 5 | from torchplus.train.common import create_folder 6 | from torchplus.train.optim import MixedPrecisionWrapper 7 | -------------------------------------------------------------------------------- /second/protos/voxel_generator.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | message VoxelGenerator{ 6 | repeated float voxel_size = 1; 7 | repeated float point_cloud_range = 2; 8 | uint32 max_number_of_points_per_voxel = 3; 9 | // uint32 max_voxels = 4; // limit GPU memory usage 10 | bool submanifold_group = 4; 11 | repeated uint32 submanifold_size = 5; 12 | uint32 submanifold_max_points = 6; 13 | } 14 | -------------------------------------------------------------------------------- /second/protos/target.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/anchors.proto"; 5 | import "second/protos/similarity.proto"; 6 | 7 | message TargetAssigner{ 8 | repeated AnchorGeneratorCollection anchor_generators = 1; 9 | float sample_positive_fraction = 2; 10 | uint32 sample_size = 3; 11 | bool use_rotate_iou = 4; 12 | string class_name = 5; 13 | RegionSimilarityCalculator region_similarity_calculator = 6; 14 | } -------------------------------------------------------------------------------- /second/protos/sampler.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/preprocess.proto"; 5 | 6 | message Group{ 7 | map name_to_max_num = 1; 8 | } 9 | 10 | message Sampler{ 11 | string database_info_path = 1; 12 | repeated Group sample_groups = 2; 13 | repeated DatabasePreprocessingStep database_prep_steps = 3; 14 | repeated float global_random_rotation_range_per_object = 4; 15 | float rate = 5; 16 | } 17 | -------------------------------------------------------------------------------- /second/utils/check.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def is_array_like(x): 4 | return isinstance(x, (list, tuple, np.ndarray)) 5 | 6 | def shape_mergeable(x, expected_shape): 7 | mergeable = True 8 | if is_array_like(x) and is_array_like(expected_shape): 9 | x = np.array(x) 10 | if len(x.shape) == len(expected_shape): 11 | for s, s_ex in zip(x.shape, expected_shape): 12 | if s_ex is not None and s != s_ex: 13 | mergeable = False 14 | break 15 | return mergeable -------------------------------------------------------------------------------- /second/protos/pipeline.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | import "second/protos/input_reader.proto"; 6 | import "second/protos/model.proto"; 7 | import "second/protos/train.proto"; 8 | // Convenience message for configuring a training and eval pipeline. Allows all 9 | // of the pipeline parameters to be configured from one file. 10 | message TrainEvalPipelineConfig { 11 | DetectionModel model = 1; 12 | InputReader train_input_reader = 2; 13 | TrainConfig train_config = 3; 14 | InputReader eval_input_reader = 4; 15 | } 16 | 17 | -------------------------------------------------------------------------------- /second/core/cc/box_ops.cc: -------------------------------------------------------------------------------- 1 | #include "box_ops.h" 2 | 3 | PYBIND11_MODULE(box_ops_cc, m) { 4 | m.doc() = "box ops written by c++"; 5 | m.def("rbbox_iou", &rbbox_iou, 6 | py::return_value_policy::reference_internal, "rbbox iou", 7 | "box_corners"_a = 1, "qbox_corners"_a = 2, "standup_iou"_a = 3, 8 | "standup_thresh"_a = 4); 9 | m.def("rbbox_iou", &rbbox_iou, 10 | py::return_value_policy::reference_internal, "rbbox iou", 11 | "box_corners"_a = 1, "qbox_corners"_a = 2, "standup_iou"_a = 3, 12 | "standup_thresh"_a = 4); 13 | } 14 | -------------------------------------------------------------------------------- /second/protos/box_coder.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Configuration proto for the box coder to be used in the object detection 6 | // pipeline. See core/box_coder.py for details. 7 | message BoxCoder { 8 | oneof box_coder { 9 | GroundBox3dCoder ground_box3d_coder = 1; 10 | BevBoxCoder bev_box_coder = 2; 11 | } 12 | } 13 | 14 | message GroundBox3dCoder { 15 | bool linear_dim = 1; 16 | bool encode_angle_vector = 2; 17 | } 18 | 19 | message BevBoxCoder { 20 | bool linear_dim = 1; 21 | bool encode_angle_vector = 2; 22 | float z_fixed = 3; 23 | float h_fixed = 4; 24 | } 25 | -------------------------------------------------------------------------------- /second/builder/preprocess_builder.py: -------------------------------------------------------------------------------- 1 | import second.core.preprocess as prep 2 | 3 | def build_db_preprocess(db_prep_config): 4 | prep_type = db_prep_config.WhichOneof('database_preprocessing_step') 5 | 6 | if prep_type == 'filter_by_difficulty': 7 | cfg = db_prep_config.filter_by_difficulty 8 | return prep.DBFilterByDifficulty(list(cfg.removed_difficulties)) 9 | elif prep_type == 'filter_by_min_num_points': 10 | cfg = db_prep_config.filter_by_min_num_points 11 | return prep.DBFilterByMinNumPoint(dict(cfg.min_num_point_pairs)) 12 | else: 13 | raise ValueError("unknown database prep type") 14 | 15 | -------------------------------------------------------------------------------- /second/protos/activations.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | message ReLU {} 6 | 7 | message LeakyReLU { float leakness = 1; } 8 | 9 | message Swish {} 10 | message ELU { float alpha = 1; } 11 | message Softplus { 12 | float beta = 1; 13 | float threshold = 2; 14 | } 15 | message Softsign {} 16 | message ReLU6 {} 17 | message SELU {} 18 | 19 | message Activation { 20 | oneof activation { 21 | ReLU relu = 1; 22 | LeakyReLU leaky_relu = 2; 23 | Swish swish = 3; 24 | ELU elu = 4; 25 | Softplus softplus = 5; 26 | Softsign softsign = 6; 27 | ReLU6 relu6 = 7; 28 | SELU selu = 8; 29 | } 30 | } -------------------------------------------------------------------------------- /second/core/cc/point_cloud_ops.cc: -------------------------------------------------------------------------------- 1 | 2 | #include "point_cloud_ops.h" 3 | 4 | PYBIND11_MODULE(point_cloud_ops_cc, m) { 5 | m.doc() = "pybind11 example plugin"; 6 | m.def("points_to_voxel", &points_to_voxel, "matrix tensor_square", 7 | "points"_a = 1, "voxels"_a = 2, "coors"_a = 3, 8 | "num_points_per_voxel"_a = 4, "voxel_size"_a = 5, "coors_range"_a = 6, 9 | "max_points"_a = 7, "max_voxels"_a = 8); 10 | m.def("points_to_voxel", &points_to_voxel, "matrix tensor_square", 11 | "points"_a = 1, "voxels"_a = 2, "coors"_a = 3, 12 | "num_points_per_voxel"_a = 4, "voxel_size"_a = 5, "coors_range"_a = 6, 13 | "max_points"_a = 7, "max_voxels"_a = 8); 14 | } -------------------------------------------------------------------------------- /torchplus/train/common.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import shutil 4 | 5 | def create_folder(prefix, add_time=True, add_str=None, delete=False): 6 | additional_str = '' 7 | if delete is True: 8 | if os.path.exists(prefix): 9 | shutil.rmtree(prefix) 10 | os.makedirs(prefix) 11 | folder = prefix 12 | if add_time is True: 13 | # additional_str has a form such as '170903_220351' 14 | additional_str += datetime.datetime.now().strftime("%y%m%d_%H%M%S") 15 | if add_str is not None: 16 | folder += '/' + additional_str + '_' + add_str 17 | else: 18 | folder += '/' + additional_str 19 | if delete is True: 20 | if os.path.exists(folder): 21 | shutil.rmtree(folder) 22 | os.makedirs(folder) 23 | return folder -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/CopyShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Full-screen textured quad shader 5 | */ 6 | 7 | THREE.CopyShader = { 8 | 9 | uniforms: { 10 | 11 | "tDiffuse": { value: null }, 12 | "opacity": { value: 1.0 } 13 | 14 | }, 15 | 16 | vertexShader: [ 17 | 18 | "varying vec2 vUv;", 19 | 20 | "void main() {", 21 | 22 | "vUv = uv;", 23 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 24 | 25 | "}" 26 | 27 | ].join( "\n" ), 28 | 29 | fragmentShader: [ 30 | 31 | "uniform float opacity;", 32 | 33 | "uniform sampler2D tDiffuse;", 34 | 35 | "varying vec2 vUv;", 36 | 37 | "void main() {", 38 | 39 | "vec4 texel = texture2D( tDiffuse, vUv );", 40 | "gl_FragColor = opacity * texel;", 41 | 42 | "}" 43 | 44 | ].join( "\n" ) 45 | 46 | }; 47 | -------------------------------------------------------------------------------- /second/protos/similarity.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Configuration proto for region similarity calculators. See 6 | // core/region_similarity_calculator.py for details. 7 | message RegionSimilarityCalculator { 8 | oneof region_similarity { 9 | RotateIouSimilarity rotate_iou_similarity = 1; 10 | NearestIouSimilarity nearest_iou_similarity = 2; 11 | DistanceSimilarity distance_similarity = 3; 12 | } 13 | } 14 | 15 | // Configuration for intersection-over-union (IOU) similarity calculator. 16 | message RotateIouSimilarity { 17 | } 18 | 19 | // Configuration for intersection-over-union (IOU) similarity calculator. 20 | message NearestIouSimilarity { 21 | } 22 | 23 | // Configuration for intersection-over-union (IOU) similarity calculator. 24 | message DistanceSimilarity { 25 | float distance_norm = 1; 26 | bool with_rotation = 2; 27 | float rotation_alpha = 3; 28 | } -------------------------------------------------------------------------------- /second/pytorch/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def get_paddings_indicator(actual_num, max_num, axis=0): 5 | """Create boolean mask by actually number of a padded tensor. 6 | 7 | Args: 8 | actual_num ([type]): [description] 9 | max_num ([type]): [description] 10 | 11 | Returns: 12 | [type]: [description] 13 | """ 14 | 15 | actual_num = torch.unsqueeze(actual_num, axis + 1) 16 | # tiled_actual_num: [N, M, 1] 17 | max_num_shape = [1] * len(actual_num.shape) 18 | max_num_shape[axis + 1] = -1 19 | max_num = torch.arange( 20 | max_num, dtype=torch.int, device=actual_num.device).view(max_num_shape) 21 | # tiled_actual_num: [[3,3,3,3,3], [4,4,4,4,4], [2,2,2,2,2]] 22 | # tiled_max_num: [[0,1,2,3,4], [0,1,2,3,4], [0,1,2,3,4]] 23 | paddings_indicator = actual_num.int() > max_num 24 | # paddings_indicator shape: [batch_size, max_num] 25 | return paddings_indicator 26 | -------------------------------------------------------------------------------- /second/protos/layers_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/layers.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | 16 | 17 | DESCRIPTOR = _descriptor.FileDescriptor( 18 | name='second/protos/layers.proto', 19 | package='second', 20 | syntax='proto3', 21 | serialized_options=None, 22 | serialized_pb=_b('\n\x1asecond/protos/layers.proto\x12\x06secondb\x06proto3') 23 | ) 24 | 25 | 26 | 27 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 28 | 29 | 30 | # @@protoc_insertion_point(module_scope) 31 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/Toast.js: -------------------------------------------------------------------------------- 1 | var Toast = function(toasts, timeout = 3000){ 2 | this.toasts = toasts; 3 | this.type = ['error', 'message', 'warning', 'success']; 4 | this.timeout = timeout; 5 | } 6 | 7 | Toast.prototype = { 8 | _addToast : function(type, text){ 9 | var toast; 10 | toast = document.createElement('li'); 11 | toast.classList.add(type); 12 | setTimeout(function(){ 13 | toast.remove(); 14 | }, this.timeout); 15 | this.toasts.appendChild(toast); 16 | return toast.innerHTML = `${type}: ${text}`; 17 | }, 18 | 19 | message : function(text){ 20 | return this._addToast("message", text); 21 | }, 22 | warning : function(text){ 23 | return this._addToast("warning", text); 24 | }, 25 | error : function(text){ 26 | return this._addToast("error", text); 27 | }, 28 | success : function(text){ 29 | return this._addToast("success", text); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /second/builder/dbsampler_builder.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import second.core.preprocess as prep 4 | from second.builder import preprocess_builder 5 | from second.core.preprocess import DataBasePreprocessor 6 | from second.core.sample_ops import DataBaseSamplerV2 7 | 8 | 9 | def build(sampler_config): 10 | cfg = sampler_config 11 | groups = list(cfg.sample_groups) 12 | prepors = [ 13 | preprocess_builder.build_db_preprocess(c) 14 | for c in cfg.database_prep_steps 15 | ] 16 | db_prepor = DataBasePreprocessor(prepors) 17 | rate = cfg.rate 18 | grot_range = cfg.global_random_rotation_range_per_object 19 | groups = [dict(g.name_to_max_num) for g in groups] 20 | info_path = cfg.database_info_path 21 | with open(info_path, 'rb') as f: 22 | db_infos = pickle.load(f) 23 | grot_range = list(grot_range) 24 | if len(grot_range) == 0: 25 | grot_range = None 26 | sampler = DataBaseSamplerV2(db_infos, groups, db_prepor, rate, grot_range) 27 | return sampler 28 | -------------------------------------------------------------------------------- /second/builder/voxel_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.core.voxel_generator import VoxelGenerator 4 | from second.protos import voxel_generator_pb2 5 | 6 | 7 | def build(voxel_config): 8 | """Builds a tensor dictionary based on the InputReader config. 9 | 10 | Args: 11 | input_reader_config: A input_reader_pb2.InputReader object. 12 | 13 | Returns: 14 | A tensor dict based on the input_reader_config. 15 | 16 | Raises: 17 | ValueError: On invalid input reader proto. 18 | ValueError: If no input paths are specified. 19 | """ 20 | if not isinstance(voxel_config, (voxel_generator_pb2.VoxelGenerator)): 21 | raise ValueError('input_reader_config not of type ' 22 | 'input_reader_pb2.InputReader.') 23 | voxel_generator = VoxelGenerator( 24 | voxel_size=list(voxel_config.voxel_size), 25 | point_cloud_range=list(voxel_config.point_cloud_range), 26 | max_num_points=voxel_config.max_number_of_points_per_voxel, 27 | max_voxels=20000) 28 | return voxel_generator 29 | -------------------------------------------------------------------------------- /second/pytorch/builder/box_coder_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.protos import box_coder_pb2 4 | from second.pytorch.core.box_coders import (BevBoxCoderTorch, 5 | GroundBox3dCoderTorch) 6 | 7 | 8 | def build(box_coder_config): 9 | """Create optimizer based on config. 10 | 11 | Args: 12 | optimizer_config: A Optimizer proto message. 13 | 14 | Returns: 15 | An optimizer and a list of variables for summary. 16 | 17 | Raises: 18 | ValueError: when using an unsupported input data type. 19 | """ 20 | box_coder_type = box_coder_config.WhichOneof('box_coder') 21 | if box_coder_type == 'ground_box3d_coder': 22 | cfg = box_coder_config.ground_box3d_coder 23 | return GroundBox3dCoderTorch(cfg.linear_dim, cfg.encode_angle_vector) 24 | elif box_coder_type == 'bev_box_coder': 25 | cfg = box_coder_config.bev_box_coder 26 | return BevBoxCoderTorch(cfg.linear_dim, cfg.encode_angle_vector, cfg.z_fixed, cfg.h_fixed) 27 | else: 28 | raise ValueError("unknown box_coder type") 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /second/core/cc/nms/nms.h: -------------------------------------------------------------------------------- 1 | #ifndef NMS_H 2 | #define NMS_H 3 | #include 4 | // must include pybind11/stl.h if using containers in STL in arguments. 5 | #include 6 | #include 7 | 8 | template 9 | int _nms_gpu(int *keep_out, const DType *boxes_host, int boxes_num, 10 | int boxes_dim, DType nms_overlap_thresh, int device_id); 11 | 12 | constexpr int const threadsPerBlock = sizeof(unsigned long long) * 8; 13 | 14 | namespace py = pybind11; 15 | using namespace pybind11::literals; 16 | template 17 | int non_max_suppression( 18 | py::array_t boxes, 19 | py::array_t keep_out, 20 | DType nms_overlap_thresh, 21 | int device_id) 22 | { 23 | py::buffer_info info = boxes.request(); 24 | auto boxes_ptr = static_cast(info.ptr); 25 | py::buffer_info info_k = keep_out.request(); 26 | auto keep_out_ptr = static_cast(info_k.ptr); 27 | 28 | return _nms_gpu(keep_out_ptr, boxes_ptr, boxes.shape(0), boxes.shape(1), nms_overlap_thresh, device_id); 29 | 30 | } 31 | #endif -------------------------------------------------------------------------------- /torchplus/ops/array_ops.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import math 3 | import time 4 | import torch 5 | 6 | 7 | def scatter_nd(indices, updates, shape): 8 | """pytorch edition of tensorflow scatter_nd. 9 | this function don't contain except handle code. so use this carefully 10 | when indice repeats, don't support repeat add which is supported 11 | in tensorflow. 12 | """ 13 | ret = torch.zeros(*shape, dtype=updates.dtype, device=updates.device) 14 | ndim = indices.shape[-1] 15 | output_shape = list(indices.shape[:-1]) + shape[indices.shape[-1]:] 16 | flatted_indices = indices.view(-1, ndim) 17 | slices = [flatted_indices[:, i] for i in range(ndim)] 18 | slices += [Ellipsis] 19 | ret[slices] = updates.view(*output_shape) 20 | return ret 21 | 22 | 23 | def gather_nd(params, indices): 24 | # this function has a limit that MAX_ADVINDEX_CALC_DIMS=5 25 | ndim = indices.shape[-1] 26 | output_shape = list(indices.shape[:-1]) + list(params.shape[indices.shape[-1]:]) 27 | flatted_indices = indices.view(-1, ndim) 28 | slices = [flatted_indices[:, i] for i in range(ndim)] 29 | slices += [Ellipsis] 30 | return params[slices].view(*output_shape) 31 | -------------------------------------------------------------------------------- /second/builder/similarity_calculator_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.core import region_similarity 4 | from second.protos import similarity_pb2 5 | 6 | 7 | def build(similarity_config): 8 | """Create optimizer based on config. 9 | 10 | Args: 11 | optimizer_config: A Optimizer proto message. 12 | 13 | Returns: 14 | An optimizer and a list of variables for summary. 15 | 16 | Raises: 17 | ValueError: when using an unsupported input data type. 18 | """ 19 | similarity_type = similarity_config.WhichOneof('region_similarity') 20 | if similarity_type == 'rotate_iou_similarity': 21 | return region_similarity.RotateIouSimilarity() 22 | elif similarity_type == 'nearest_iou_similarity': 23 | return region_similarity.NearestIouSimilarity() 24 | elif similarity_type == 'distance_similarity': 25 | cfg = similarity_config.distance_similarity 26 | return region_similarity.DistanceSimilarity( 27 | distance_norm=cfg.distance_norm, 28 | with_rotation=cfg.with_rotation, 29 | rotation_alpha=cfg.rotation_alpha) 30 | else: 31 | raise ValueError("unknown similarity type") 32 | -------------------------------------------------------------------------------- /second/protos/train.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | import "second/protos/optimizer.proto"; 6 | import "second/protos/preprocess.proto"; 7 | 8 | message Regularizer { 9 | oneof regularizer_oneof { 10 | L1Regularizer l1_regularizer = 1; 11 | L2Regularizer l2_regularizer = 2; 12 | } 13 | } 14 | 15 | // Configuration proto for L1 Regularizer. 16 | // See https://www.tensorflow.org/api_docs/python/tf/contrib/layers/l1_regularizer 17 | message L1Regularizer { 18 | float weight = 1; 19 | } 20 | 21 | // Configuration proto for L2 Regularizer. 22 | // See https://www.tensorflow.org/api_docs/python/tf/contrib/layers/l2_regularizer 23 | message L2Regularizer { 24 | float weight = 1; 25 | } 26 | 27 | 28 | message TrainConfig{ 29 | Optimizer optimizer = 1; 30 | // repeated PreprocessingStep data_augmentation_options = 2; 31 | uint32 inter_op_parallelism_threads = 3; 32 | uint32 intra_op_parallelism_threads = 4; 33 | uint32 steps = 5; 34 | uint32 steps_per_eval = 6; 35 | uint32 save_checkpoints_secs = 7; 36 | uint32 save_summary_steps = 8; 37 | bool enable_mixed_precision = 9; 38 | float loss_scale_factor = 10; 39 | bool clear_metrics_every_epoch = 11; 40 | 41 | // Regularizer regularizer = 2; 42 | } -------------------------------------------------------------------------------- /second/protos/preprocess.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | 6 | message Preprocess{ 7 | float random_global_rotation_min_rad = 1; 8 | float random_global_rotation_max_rad = 2; 9 | float random_global_scaling_min = 3; 10 | float random_global_scaling_max = 4; 11 | float random_noise_per_groundtruth_min_rad = 5; 12 | float random_noise_per_groundtruth_max_rad = 6; 13 | float random_noise_per_groundtruth_position_std = 7; 14 | } 15 | 16 | message DatabasePreprocessingStep { 17 | oneof database_preprocessing_step { 18 | DBFilterByDifficulty filter_by_difficulty = 1; 19 | DBFilterByMinNumPointInGroundTruth filter_by_min_num_points = 2; 20 | } 21 | } 22 | 23 | message DBFilterByDifficulty{ 24 | repeated int32 removed_difficulties = 1; 25 | } 26 | 27 | message DBFilterByMinNumPointInGroundTruth{ 28 | map min_num_point_pairs = 1; 29 | } 30 | 31 | message PreprocessingStep { 32 | oneof preprocessing_step { 33 | RandomGlobalScaling random_global_scaling = 1; 34 | RandomGlobalRotation random_global_rotation = 2; 35 | } 36 | } 37 | 38 | message RandomGlobalScaling{ 39 | float min_scale = 1; 40 | float max_scale = 2; 41 | } 42 | 43 | message RandomGlobalRotation{ 44 | float min_rad = 1; 45 | float max_rad = 2; 46 | } 47 | 48 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/css/main.css: -------------------------------------------------------------------------------- 1 | .btn { 2 | /*height: 100%;*/ 3 | text-align: center; 4 | position: absolute; 5 | top: 90%; 6 | left: 40%; 7 | z-index: 2; 8 | font-size: 40pt; 9 | color:#bbb; 10 | } 11 | 12 | .prev { 13 | left: 38%; 14 | /*transform: scale(3, 1);*/ 15 | } 16 | .imgidx { 17 | background: transparent; 18 | width: 120px; 19 | height: 40px; 20 | font-size: 12pt; 21 | left: 47%; 22 | 23 | border: 2px solid blue; 24 | border-radius: 5px; 25 | 26 | } 27 | 28 | .next { 29 | left: 58%; 30 | /*transform: scale(3, 1);*/ 31 | } 32 | 33 | 34 | .toasts { 35 | position: fixed; 36 | max-width: 100%; 37 | width: 250px; 38 | right: 0; 39 | bottom: 0; 40 | padding: 0 10px; 41 | box-sizing: border-box; 42 | transition: height 1s; 43 | z-index: 100; 44 | } 45 | 46 | .toasts li { 47 | width: 100%; 48 | list-style-type: none; 49 | box-sizing: border-box; 50 | font-family: sans-serif; 51 | padding: 15px 20px; 52 | background: #222; 53 | color: white; 54 | border-radius: 2px; 55 | margin: 10px 0; 56 | } 57 | 58 | .toasts li.message { 59 | background: #358; 60 | } 61 | 62 | .toasts li.error { 63 | background: brown; 64 | } 65 | 66 | .toasts li.warning { 67 | background: chocolate; 68 | } 69 | 70 | .toasts li.success { 71 | background: seagreen; 72 | } -------------------------------------------------------------------------------- /second/pytorch/core/box_coders.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from abc import abstractproperty 4 | from second.core.box_coders import GroundBox3dCoder, BevBoxCoder 5 | from second.pytorch.core import box_torch_ops 6 | import torch 7 | 8 | class GroundBox3dCoderTorch(GroundBox3dCoder): 9 | def encode_torch(self, boxes, anchors): 10 | return box_torch_ops.second_box_encode(boxes, anchors, self.vec_encode, self.linear_dim) 11 | 12 | def decode_torch(self, boxes, anchors): 13 | return box_torch_ops.second_box_decode(boxes, anchors, self.vec_encode, self.linear_dim) 14 | 15 | 16 | 17 | class BevBoxCoderTorch(BevBoxCoder): 18 | def encode_torch(self, boxes, anchors): 19 | anchors = anchors[..., [0, 1, 3, 4, 6]] 20 | boxes = boxes[..., [0, 1, 3, 4, 6]] 21 | return box_torch_ops.bev_box_encode(boxes, anchors, self.vec_encode, self.linear_dim) 22 | 23 | def decode_torch(self, encodings, anchors): 24 | anchors = anchors[..., [0, 1, 3, 4, 6]] 25 | ret = box_torch_ops.bev_box_decode(encodings, anchors, self.vec_encode, self.linear_dim) 26 | z_fixed = torch.full([*ret.shape[:-1], 1], self.z_fixed, dtype=ret.dtype, device=ret.device) 27 | h_fixed = torch.full([*ret.shape[:-1], 1], self.h_fixed, dtype=ret.dtype, device=ret.device) 28 | return torch.cat([ret[..., :2], z_fixed, ret[..., 2:4], h_fixed, ret[..., 4:]], dim=-1) 29 | 30 | 31 | -------------------------------------------------------------------------------- /second/protos/input_reader.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | import "second/protos/target.proto"; 6 | import "second/protos/preprocess.proto"; 7 | import "second/protos/sampler.proto"; 8 | message InputReader { 9 | string record_file_path = 1; 10 | repeated string class_names = 2; 11 | uint32 batch_size = 3; 12 | uint32 max_num_epochs = 4; 13 | uint32 prefetch_size = 5; 14 | uint32 max_number_of_voxels = 6; 15 | TargetAssigner target_assigner = 7; 16 | string kitti_info_path = 8; 17 | string kitti_root_path = 9; 18 | bool shuffle_points = 10; 19 | repeated float groundtruth_localization_noise_std = 11; 20 | repeated float groundtruth_rotation_uniform_noise = 12; 21 | repeated float global_rotation_uniform_noise = 13; 22 | repeated float global_scaling_uniform_noise = 14; 23 | bool remove_unknown_examples = 15; 24 | uint32 num_workers = 16; 25 | float anchor_area_threshold = 17; 26 | bool remove_points_after_sample = 18; 27 | float groundtruth_points_drop_percentage = 19; 28 | uint32 groundtruth_drop_max_keep_points = 20; 29 | bool remove_environment = 21; 30 | bool unlabeled_training = 22; 31 | repeated float global_random_rotation_range_per_object = 23; 32 | repeated DatabasePreprocessingStep database_prep_steps = 24; 33 | Sampler database_sampler = 25; 34 | bool use_group_id = 26; // this will enable group sample and noise 35 | Sampler unlabeled_database_sampler = 27; 36 | } 37 | -------------------------------------------------------------------------------- /second/core/cc/nms/nms.cc: -------------------------------------------------------------------------------- 1 | #include "nms.h" 2 | #include "nms_cpu.h" 3 | PYBIND11_MODULE(nms, m) 4 | { 5 | m.doc() = "non_max_suppression asd"; 6 | m.def("non_max_suppression", &non_max_suppression, py::return_value_policy::reference_internal, "bbox iou", 7 | "boxes"_a = 1, "keep_out"_a = 2, "nms_overlap_thresh"_a = 3, "device_id"_a = 4); 8 | m.def("non_max_suppression", &non_max_suppression, py::return_value_policy::reference_internal, "bbox iou", 9 | "boxes"_a = 1, "keep_out"_a = 2, "nms_overlap_thresh"_a = 3, "device_id"_a = 4); 10 | m.def("non_max_suppression_cpu", &non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", 11 | "boxes"_a = 1, "order"_a = 2, "nms_overlap_thresh"_a = 3, "eps"_a = 4); 12 | m.def("non_max_suppression_cpu", &non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", 13 | "boxes"_a = 1, "order"_a = 2, "nms_overlap_thresh"_a = 3, "eps"_a = 4); 14 | m.def("rotate_non_max_suppression_cpu", &rotate_non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", 15 | "box_corners"_a = 1, "order"_a = 2, "standup_iou"_a = 3, "thresh"_a = 4); 16 | m.def("rotate_non_max_suppression_cpu", &rotate_non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", 17 | "box_corners"_a = 1, "order"_a = 2, "standup_iou"_a = 3, "thresh"_a = 4); 18 | } -------------------------------------------------------------------------------- /second/core/voxel_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from second.core.point_cloud.point_cloud_ops import points_to_voxel 3 | 4 | 5 | class VoxelGenerator: 6 | def __init__(self, 7 | voxel_size, 8 | point_cloud_range, 9 | max_num_points, 10 | max_voxels=20000): 11 | point_cloud_range = np.array(point_cloud_range, dtype=np.float32) 12 | # [0, -40, -3, 70.4, 40, 1] 13 | voxel_size = np.array(voxel_size, dtype=np.float32) 14 | grid_size = ( 15 | point_cloud_range[3:] - point_cloud_range[:3]) / voxel_size 16 | grid_size = np.round(grid_size).astype(np.int64) 17 | 18 | self._voxel_size = voxel_size 19 | self._point_cloud_range = point_cloud_range 20 | self._max_num_points = max_num_points 21 | self._max_voxels = max_voxels 22 | self._grid_size = grid_size 23 | 24 | def generate(self, points, max_voxels): 25 | return points_to_voxel( 26 | points, self._voxel_size, self._point_cloud_range, 27 | self._max_num_points, True, max_voxels) 28 | 29 | @property 30 | def voxel_size(self): 31 | return self._voxel_size 32 | 33 | @property 34 | def max_num_points_per_voxel(self): 35 | return self._max_num_points 36 | 37 | 38 | @property 39 | def point_cloud_range(self): 40 | return self._point_cloud_range 41 | 42 | @property 43 | def grid_size(self): 44 | return self._grid_size -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/LuminosityHighPassShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author bhouston / http://clara.io/ 3 | * 4 | * Luminosity 5 | * http://en.wikipedia.org/wiki/Luminosity 6 | */ 7 | 8 | THREE.LuminosityHighPassShader = { 9 | 10 | shaderID: "luminosityHighPass", 11 | 12 | uniforms: { 13 | 14 | "tDiffuse": { type: "t", value: null }, 15 | "luminosityThreshold": { type: "f", value: 1.0 }, 16 | "smoothWidth": { type: "f", value: 1.0 }, 17 | "defaultColor": { type: "c", value: new THREE.Color( 0x000000 ) }, 18 | "defaultOpacity": { type: "f", value: 0.0 } 19 | 20 | }, 21 | 22 | vertexShader: [ 23 | 24 | "varying vec2 vUv;", 25 | 26 | "void main() {", 27 | 28 | "vUv = uv;", 29 | 30 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 31 | 32 | "}" 33 | 34 | ].join("\n"), 35 | 36 | fragmentShader: [ 37 | 38 | "uniform sampler2D tDiffuse;", 39 | "uniform vec3 defaultColor;", 40 | "uniform float defaultOpacity;", 41 | "uniform float luminosityThreshold;", 42 | "uniform float smoothWidth;", 43 | 44 | "varying vec2 vUv;", 45 | 46 | "void main() {", 47 | 48 | "vec4 texel = texture2D( tDiffuse, vUv );", 49 | 50 | "vec3 luma = vec3( 0.299, 0.587, 0.114 );", 51 | 52 | "float v = dot( texel.xyz, luma );", 53 | 54 | "vec4 outputColor = vec4( defaultColor.rgb, defaultOpacity );", 55 | 56 | "float alpha = smoothstep( luminosityThreshold, luminosityThreshold + smoothWidth, v );", 57 | 58 | "gl_FragColor = mix( outputColor, texel, alpha );", 59 | 60 | "}" 61 | 62 | ].join("\n") 63 | 64 | }; 65 | -------------------------------------------------------------------------------- /second/protos/anchors.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | message AnchorGeneratorOld { 6 | string class_name = 1; 7 | repeated float bev_range = 2; 8 | float anchor_center_z = 3; 9 | repeated float anchor_sizes = 4; 10 | float matched_threshold = 5; 11 | float unmatched_threshold = 6; 12 | } 13 | 14 | message AnchorGeneratorStride { 15 | string class_name = 1; 16 | repeated float sizes = 2; 17 | repeated float strides = 3; 18 | repeated float offsets = 4; 19 | repeated float rotations = 5; 20 | float matched_threshold = 6; 21 | float unmatched_threshold = 7; 22 | } 23 | 24 | message AnchorGeneratorRange { 25 | string class_name = 1; 26 | repeated float sizes = 2; 27 | repeated float anchor_ranges = 3; 28 | repeated float rotations = 4; 29 | float matched_threshold = 5; 30 | float unmatched_threshold = 6; 31 | } 32 | 33 | message AnchorGeneratorCollection { 34 | oneof anchor_generator { 35 | AnchorGeneratorStride anchor_generator_stride = 1; 36 | AnchorGeneratorRange anchor_generator_range = 2; 37 | AnchorGeneratorOld anchor_generator_old = 3; 38 | } 39 | } 40 | 41 | message AnchorGenerator_depara { 42 | string class_name = 1; 43 | repeated float sizes = 2; 44 | repeated float strides = 3; 45 | repeated float offsets = 4; 46 | repeated float rotations = 5; 47 | float matched_threshold = 6; 48 | float unmatched_threshold = 7; 49 | } 50 | message AnchorGeneratorV1 { 51 | float anchor_center_z = 1; 52 | repeated float anchor_sizes = 2; 53 | float matched_threshold = 3; 54 | float unmatched_threshold = 4; 55 | } -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/RenderPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.RenderPass = function ( scene, camera, overrideMaterial, clearColor, clearAlpha ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | this.scene = scene; 10 | this.camera = camera; 11 | 12 | this.overrideMaterial = overrideMaterial; 13 | 14 | this.clearColor = clearColor; 15 | this.clearAlpha = ( clearAlpha !== undefined ) ? clearAlpha : 0; 16 | 17 | this.clear = true; 18 | this.clearDepth = false; 19 | this.needsSwap = false; 20 | 21 | }; 22 | 23 | THREE.RenderPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 24 | 25 | constructor: THREE.RenderPass, 26 | 27 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 28 | 29 | var oldAutoClear = renderer.autoClear; 30 | renderer.autoClear = false; 31 | 32 | this.scene.overrideMaterial = this.overrideMaterial; 33 | 34 | var oldClearColor, oldClearAlpha; 35 | 36 | if ( this.clearColor ) { 37 | 38 | oldClearColor = renderer.getClearColor().getHex(); 39 | oldClearAlpha = renderer.getClearAlpha(); 40 | 41 | renderer.setClearColor( this.clearColor, this.clearAlpha ); 42 | 43 | } 44 | 45 | if ( this.clearDepth ) { 46 | 47 | renderer.clearDepth(); 48 | 49 | } 50 | 51 | renderer.render( this.scene, this.camera, this.renderToScreen ? null : readBuffer, this.clear ); 52 | 53 | if ( this.clearColor ) { 54 | 55 | renderer.setClearColor( oldClearColor, oldClearAlpha ); 56 | 57 | } 58 | 59 | this.scene.overrideMaterial = null; 60 | renderer.autoClear = oldAutoClear; 61 | } 62 | 63 | } ); 64 | -------------------------------------------------------------------------------- /second/builder/anchor_generator_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.protos import box_coder_pb2 4 | from second.core.anchor_generator import ( 5 | AnchorGeneratorStride, AnchorGeneratorRange) 6 | 7 | 8 | def build(anchor_config): 9 | """Create optimizer based on config. 10 | 11 | Args: 12 | optimizer_config: A Optimizer proto message. 13 | 14 | Returns: 15 | An optimizer and a list of variables for summary. 16 | 17 | Raises: 18 | ValueError: when using an unsupported input data type. 19 | """ 20 | ag_type = anchor_config.WhichOneof('anchor_generator') 21 | 22 | if ag_type == 'anchor_generator_stride': 23 | config = anchor_config.anchor_generator_stride 24 | ag = AnchorGeneratorStride( 25 | sizes=list(config.sizes), 26 | anchor_strides=list(config.strides), 27 | anchor_offsets=list(config.offsets), 28 | rotations=list(config.rotations), 29 | match_threshold=config.matched_threshold, 30 | unmatch_threshold=config.unmatched_threshold, 31 | class_id=config.class_name) 32 | return ag 33 | elif ag_type == 'anchor_generator_range': 34 | config = anchor_config.anchor_generator_range 35 | ag = AnchorGeneratorRange( 36 | sizes=list(config.sizes), 37 | anchor_ranges=list(config.anchor_ranges), 38 | rotations=list(config.rotations), 39 | match_threshold=config.matched_threshold, 40 | unmatch_threshold=config.unmatched_threshold, 41 | class_id=config.class_name) 42 | return ag 43 | else: 44 | raise ValueError(" unknown anchor generator type") -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/ShaderPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.ShaderPass = function ( shader, textureID ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | this.textureID = ( textureID !== undefined ) ? textureID : "tDiffuse"; 10 | 11 | if ( shader instanceof THREE.ShaderMaterial ) { 12 | 13 | this.uniforms = shader.uniforms; 14 | 15 | this.material = shader; 16 | 17 | } else if ( shader ) { 18 | 19 | this.uniforms = THREE.UniformsUtils.clone( shader.uniforms ); 20 | 21 | this.material = new THREE.ShaderMaterial( { 22 | 23 | defines: Object.assign( {}, shader.defines ), 24 | uniforms: this.uniforms, 25 | vertexShader: shader.vertexShader, 26 | fragmentShader: shader.fragmentShader 27 | 28 | } ); 29 | 30 | } 31 | 32 | this.camera = new THREE.OrthographicCamera( - 1, 1, 1, - 1, 0, 1 ); 33 | this.scene = new THREE.Scene(); 34 | 35 | this.quad = new THREE.Mesh( new THREE.PlaneBufferGeometry( 2, 2 ), null ); 36 | this.quad.frustumCulled = false; // Avoid getting clipped 37 | this.scene.add( this.quad ); 38 | 39 | }; 40 | 41 | THREE.ShaderPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 42 | 43 | constructor: THREE.ShaderPass, 44 | 45 | render: function( renderer, writeBuffer, readBuffer, delta, maskActive ) { 46 | 47 | if ( this.uniforms[ this.textureID ] ) { 48 | 49 | this.uniforms[ this.textureID ].value = readBuffer.texture; 50 | 51 | } 52 | 53 | this.quad.material = this.material; 54 | 55 | if ( this.renderToScreen ) { 56 | 57 | renderer.render( this.scene, this.camera ); 58 | 59 | } else { 60 | 61 | renderer.render( this.scene, this.camera, writeBuffer, this.clear ); 62 | 63 | } 64 | 65 | } 66 | 67 | } ); 68 | -------------------------------------------------------------------------------- /second/builder/target_assigner_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.core.target_assigner import TargetAssigner 4 | from second.protos import target_pb2, anchors_pb2 5 | from second.builder import similarity_calculator_builder 6 | from second.builder import anchor_generator_builder 7 | 8 | def build(target_assigner_config, bv_range, box_coder): 9 | """Builds a tensor dictionary based on the InputReader config. 10 | 11 | Args: 12 | input_reader_config: A input_reader_pb2.InputReader object. 13 | 14 | Returns: 15 | A tensor dict based on the input_reader_config. 16 | 17 | Raises: 18 | ValueError: On invalid input reader proto. 19 | ValueError: If no input paths are specified. 20 | """ 21 | if not isinstance(target_assigner_config, (target_pb2.TargetAssigner)): 22 | raise ValueError('input_reader_config not of type ' 23 | 'input_reader_pb2.InputReader.') 24 | anchor_cfg = target_assigner_config.anchor_generators 25 | anchor_generators = [] 26 | for a_cfg in anchor_cfg: 27 | anchor_generator = anchor_generator_builder.build(a_cfg) 28 | anchor_generators.append(anchor_generator) 29 | similarity_calc = similarity_calculator_builder.build( 30 | target_assigner_config.region_similarity_calculator) 31 | positive_fraction = target_assigner_config.sample_positive_fraction 32 | if positive_fraction < 0: 33 | positive_fraction = None 34 | target_assigner = TargetAssigner( 35 | box_coder=box_coder, 36 | anchor_generators=anchor_generators, 37 | region_similarity_calculator=similarity_calc, 38 | positive_fraction=positive_fraction, 39 | sample_size=target_assigner_config.sample_size) 40 | return target_assigner 41 | 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | *.o 9 | *.out 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | .vscode 109 | -------------------------------------------------------------------------------- /torchplus/tools.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import inspect 3 | import sys 4 | from collections import OrderedDict 5 | 6 | import numba 7 | import numpy as np 8 | import torch 9 | 10 | 11 | def get_pos_to_kw_map(func): 12 | pos_to_kw = {} 13 | fsig = inspect.signature(func) 14 | pos = 0 15 | for name, info in fsig.parameters.items(): 16 | if info.kind is info.POSITIONAL_OR_KEYWORD: 17 | pos_to_kw[pos] = name 18 | pos += 1 19 | return pos_to_kw 20 | 21 | 22 | def get_kw_to_default_map(func): 23 | kw_to_default = {} 24 | fsig = inspect.signature(func) 25 | for name, info in fsig.parameters.items(): 26 | if info.kind is info.POSITIONAL_OR_KEYWORD: 27 | if info.default is not info.empty: 28 | kw_to_default[name] = info.default 29 | return kw_to_default 30 | 31 | 32 | def change_default_args(**kwargs): 33 | def layer_wrapper(layer_class): 34 | class DefaultArgLayer(layer_class): 35 | def __init__(self, *args, **kw): 36 | pos_to_kw = get_pos_to_kw_map(layer_class.__init__) 37 | kw_to_pos = {kw: pos for pos, kw in pos_to_kw.items()} 38 | for key, val in kwargs.items(): 39 | if key not in kw and kw_to_pos[key] > len(args): 40 | kw[key] = val 41 | super().__init__(*args, **kw) 42 | 43 | return DefaultArgLayer 44 | 45 | return layer_wrapper 46 | 47 | def torch_to_np_dtype(ttype): 48 | type_map = { 49 | torch.float16: np.dtype(np.float16), 50 | torch.float32: np.dtype(np.float32), 51 | torch.float16: np.dtype(np.float64), 52 | torch.int32: np.dtype(np.int32), 53 | torch.int64: np.dtype(np.int64), 54 | torch.uint8: np.dtype(np.uint8), 55 | } 56 | return type_map[ttype] -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/FilmPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.FilmPass = function ( noiseIntensity, scanlinesIntensity, scanlinesCount, grayscale ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | if ( THREE.FilmShader === undefined ) 10 | console.error( "THREE.FilmPass relies on THREE.FilmShader" ); 11 | 12 | var shader = THREE.FilmShader; 13 | 14 | this.uniforms = THREE.UniformsUtils.clone( shader.uniforms ); 15 | 16 | this.material = new THREE.ShaderMaterial( { 17 | 18 | uniforms: this.uniforms, 19 | vertexShader: shader.vertexShader, 20 | fragmentShader: shader.fragmentShader 21 | 22 | } ); 23 | 24 | if ( grayscale !== undefined ) this.uniforms.grayscale.value = grayscale; 25 | if ( noiseIntensity !== undefined ) this.uniforms.nIntensity.value = noiseIntensity; 26 | if ( scanlinesIntensity !== undefined ) this.uniforms.sIntensity.value = scanlinesIntensity; 27 | if ( scanlinesCount !== undefined ) this.uniforms.sCount.value = scanlinesCount; 28 | 29 | this.camera = new THREE.OrthographicCamera( - 1, 1, 1, - 1, 0, 1 ); 30 | this.scene = new THREE.Scene(); 31 | 32 | this.quad = new THREE.Mesh( new THREE.PlaneBufferGeometry( 2, 2 ), null ); 33 | this.quad.frustumCulled = false; // Avoid getting clipped 34 | this.scene.add( this.quad ); 35 | 36 | }; 37 | 38 | THREE.FilmPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 39 | 40 | constructor: THREE.FilmPass, 41 | 42 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 43 | 44 | this.uniforms[ "tDiffuse" ].value = readBuffer.texture; 45 | this.uniforms[ "time" ].value += delta; 46 | 47 | this.quad.material = this.material; 48 | 49 | if ( this.renderToScreen ) { 50 | 51 | renderer.render( this.scene, this.camera ); 52 | 53 | } else { 54 | 55 | renderer.render( this.scene, this.camera, writeBuffer, this.clear ); 56 | 57 | } 58 | 59 | } 60 | 61 | } ); 62 | -------------------------------------------------------------------------------- /second/protos/second.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/losses.proto"; 5 | import "second/protos/box_coder.proto"; 6 | import "second/protos/target.proto"; 7 | import "second/protos/voxel_generator.proto"; 8 | 9 | message VoxelNet { 10 | uint32 num_class = 1; 11 | message VoxelFeatureExtractor { 12 | string module_class_name = 1; 13 | repeated uint32 num_filters = 2; 14 | bool with_distance = 3; 15 | } 16 | VoxelFeatureExtractor voxel_feature_extractor = 2; 17 | message MiddleFeatureExtractor { 18 | string module_class_name = 1; 19 | repeated uint32 num_filters_down1 = 2; 20 | repeated uint32 num_filters_down2 = 3; 21 | } 22 | MiddleFeatureExtractor middle_feature_extractor = 3; 23 | message RPN { 24 | string module_class_name = 1; 25 | repeated uint32 layer_nums = 2; 26 | repeated uint32 layer_strides = 3; 27 | repeated uint32 num_filters = 4; 28 | repeated uint32 upsample_strides = 5; 29 | repeated uint32 num_upsample_filters = 6; 30 | bool use_groupnorm = 7; 31 | uint32 num_groups = 8; 32 | } 33 | RPN rpn = 4; 34 | bool use_sigmoid_score = 5; 35 | Loss loss = 6; 36 | bool encode_rad_error_by_sin = 7; 37 | bool encode_background_as_zeros = 8; 38 | bool use_aux_classifier = 9; 39 | bool use_rotate_nms = 10; 40 | bool use_multi_class_nms = 11; 41 | uint32 nms_pre_max_size = 12; 42 | uint32 nms_post_max_size = 13; 43 | float nms_score_threshold = 14; 44 | float nms_iou_threshold = 15; 45 | repeated float post_center_limit_range = 16; 46 | bool use_direction_classifier = 17; 47 | float direction_loss_weight = 18; 48 | float pos_class_weight = 19; 49 | float neg_class_weight = 20; 50 | enum LossNormType { 51 | NormByNumExamples = 0; 52 | NormByNumPositives = 1; 53 | NormByNumPosNeg = 2; 54 | } 55 | LossNormType loss_norm_type = 21; 56 | bool use_bev = 22; 57 | bool without_reflectivity = 23; 58 | bool encode_angle_to_vector = 24; 59 | BoxCoder box_coder = 25; 60 | TargetAssigner target_assigner = 26; 61 | bool lidar_input = 27; 62 | uint32 num_point_features = 28; 63 | VoxelGenerator voxel_generator = 29; 64 | } -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/MaskPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.MaskPass = function ( scene, camera ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | this.scene = scene; 10 | this.camera = camera; 11 | 12 | this.clear = true; 13 | this.needsSwap = false; 14 | 15 | this.inverse = false; 16 | 17 | }; 18 | 19 | THREE.MaskPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 20 | 21 | constructor: THREE.MaskPass, 22 | 23 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 24 | 25 | var context = renderer.context; 26 | var state = renderer.state; 27 | 28 | // don't update color or depth 29 | 30 | state.buffers.color.setMask( false ); 31 | state.buffers.depth.setMask( false ); 32 | 33 | // lock buffers 34 | 35 | state.buffers.color.setLocked( true ); 36 | state.buffers.depth.setLocked( true ); 37 | 38 | // set up stencil 39 | 40 | var writeValue, clearValue; 41 | 42 | if ( this.inverse ) { 43 | 44 | writeValue = 0; 45 | clearValue = 1; 46 | 47 | } else { 48 | 49 | writeValue = 1; 50 | clearValue = 0; 51 | 52 | } 53 | 54 | state.buffers.stencil.setTest( true ); 55 | state.buffers.stencil.setOp( context.REPLACE, context.REPLACE, context.REPLACE ); 56 | state.buffers.stencil.setFunc( context.ALWAYS, writeValue, 0xffffffff ); 57 | state.buffers.stencil.setClear( clearValue ); 58 | 59 | // draw into the stencil buffer 60 | 61 | renderer.render( this.scene, this.camera, readBuffer, this.clear ); 62 | renderer.render( this.scene, this.camera, writeBuffer, this.clear ); 63 | 64 | // unlock color and depth buffer for subsequent rendering 65 | 66 | state.buffers.color.setLocked( false ); 67 | state.buffers.depth.setLocked( false ); 68 | 69 | // only render where stencil is set to 1 70 | 71 | state.buffers.stencil.setFunc( context.EQUAL, 1, 0xffffffff ); // draw if == 1 72 | state.buffers.stencil.setOp( context.KEEP, context.KEEP, context.KEEP ); 73 | 74 | } 75 | 76 | } ); 77 | 78 | 79 | THREE.ClearMaskPass = function () { 80 | 81 | THREE.Pass.call( this ); 82 | 83 | this.needsSwap = false; 84 | 85 | }; 86 | 87 | THREE.ClearMaskPass.prototype = Object.create( THREE.Pass.prototype ); 88 | 89 | Object.assign( THREE.ClearMaskPass.prototype, { 90 | 91 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 92 | 93 | renderer.state.buffers.stencil.setTest( false ); 94 | 95 | } 96 | 97 | } ); 98 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/ConvolutionShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Convolution shader 5 | * ported from o3d sample to WebGL / GLSL 6 | * http://o3d.googlecode.com/svn/trunk/samples/convolution.html 7 | */ 8 | 9 | THREE.ConvolutionShader = { 10 | 11 | defines: { 12 | 13 | "KERNEL_SIZE_FLOAT": "25.0", 14 | "KERNEL_SIZE_INT": "25" 15 | 16 | }, 17 | 18 | uniforms: { 19 | 20 | "tDiffuse": { value: null }, 21 | "uImageIncrement": { value: new THREE.Vector2( 0.001953125, 0.0 ) }, 22 | "cKernel": { value: [] } 23 | 24 | }, 25 | 26 | vertexShader: [ 27 | 28 | "uniform vec2 uImageIncrement;", 29 | 30 | "varying vec2 vUv;", 31 | 32 | "void main() {", 33 | 34 | "vUv = uv - ( ( KERNEL_SIZE_FLOAT - 1.0 ) / 2.0 ) * uImageIncrement;", 35 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 36 | 37 | "}" 38 | 39 | ].join( "\n" ), 40 | 41 | fragmentShader: [ 42 | 43 | "uniform float cKernel[ KERNEL_SIZE_INT ];", 44 | 45 | "uniform sampler2D tDiffuse;", 46 | "uniform vec2 uImageIncrement;", 47 | 48 | "varying vec2 vUv;", 49 | 50 | "void main() {", 51 | 52 | "vec2 imageCoord = vUv;", 53 | "vec4 sum = vec4( 0.0, 0.0, 0.0, 0.0 );", 54 | 55 | "for( int i = 0; i < KERNEL_SIZE_INT; i ++ ) {", 56 | 57 | "sum += texture2D( tDiffuse, imageCoord ) * cKernel[ i ];", 58 | "imageCoord += uImageIncrement;", 59 | 60 | "}", 61 | 62 | "gl_FragColor = sum;", 63 | 64 | "}" 65 | 66 | 67 | ].join( "\n" ), 68 | 69 | buildKernel: function ( sigma ) { 70 | 71 | // We lop off the sqrt(2 * pi) * sigma term, since we're going to normalize anyway. 72 | 73 | function gauss( x, sigma ) { 74 | 75 | return Math.exp( - ( x * x ) / ( 2.0 * sigma * sigma ) ); 76 | 77 | } 78 | 79 | var i, values, sum, halfWidth, kMaxKernelSize = 25, kernelSize = 2 * Math.ceil( sigma * 3.0 ) + 1; 80 | 81 | if ( kernelSize > kMaxKernelSize ) kernelSize = kMaxKernelSize; 82 | halfWidth = ( kernelSize - 1 ) * 0.5; 83 | 84 | values = new Array( kernelSize ); 85 | sum = 0.0; 86 | for ( i = 0; i < kernelSize; ++ i ) { 87 | 88 | values[ i ] = gauss( i - halfWidth, sigma ); 89 | sum += values[ i ]; 90 | 91 | } 92 | 93 | // normalize the kernel 94 | 95 | for ( i = 0; i < kernelSize; ++ i ) values[ i ] /= sum; 96 | 97 | return values; 98 | 99 | } 100 | 101 | }; 102 | -------------------------------------------------------------------------------- /second/data/dataset.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import pickle 3 | import time 4 | from functools import partial 5 | 6 | import numpy as np 7 | 8 | from second.core import box_np_ops 9 | from second.core import preprocess as prep 10 | from second.data import kitti_common as kitti 11 | from second.data.preprocess import _read_and_prep_v9 12 | 13 | 14 | class Dataset(object): 15 | """An abstract class representing a pytorch-like Dataset. 16 | All other datasets should subclass it. All subclasses should override 17 | ``__len__``, that provides the size of the dataset, and ``__getitem__``, 18 | supporting integer indexing in range from 0 to len(self) exclusive. 19 | """ 20 | 21 | def __getitem__(self, index): 22 | raise NotImplementedError 23 | 24 | def __len__(self): 25 | raise NotImplementedError 26 | 27 | 28 | 29 | class KittiDataset(Dataset): 30 | def __init__(self, info_path, root_path, num_point_features, 31 | target_assigner, feature_map_size, prep_func): 32 | with open(info_path, 'rb') as f: 33 | infos = pickle.load(f) 34 | #self._kitti_infos = kitti.filter_infos_by_used_classes(infos, class_names) 35 | self._root_path = root_path 36 | self._kitti_infos = infos 37 | self._num_point_features = num_point_features 38 | print("remain number of infos:", len(self._kitti_infos)) 39 | # generate anchors cache 40 | # [352, 400] 41 | ret = target_assigner.generate_anchors(feature_map_size) 42 | anchors = ret["anchors"] 43 | anchors = anchors.reshape([-1, 7]) 44 | matched_thresholds = ret["matched_thresholds"] 45 | unmatched_thresholds = ret["unmatched_thresholds"] 46 | anchors_bv = box_np_ops.rbbox2d_to_near_bbox( 47 | anchors[:, [0, 1, 3, 4, 6]]) 48 | anchor_cache = { 49 | "anchors": anchors, 50 | "anchors_bv": anchors_bv, 51 | "matched_thresholds": matched_thresholds, 52 | "unmatched_thresholds": unmatched_thresholds, 53 | } 54 | self._prep_func = partial(prep_func, anchor_cache=anchor_cache) 55 | 56 | def __len__(self): 57 | return len(self._kitti_infos) 58 | 59 | @property 60 | def kitti_infos(self): 61 | return self._kitti_infos 62 | 63 | def __getitem__(self, idx): 64 | return _read_and_prep_v9( 65 | info=self._kitti_infos[idx], 66 | root_path=self._root_path, 67 | num_point_features=self._num_point_features, 68 | prep_func=self._prep_func) 69 | -------------------------------------------------------------------------------- /second/core/cc/point_cloud_ops.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | namespace py = pybind11; 10 | using namespace pybind11::literals; 11 | 12 | template 13 | int points_to_voxel(py::array_t points, py::array_t voxels, 14 | py::array_t coors, 15 | py::array_t num_points_per_voxel, 16 | py::array_t coor_to_voxelidx, 17 | std::vector voxel_size, 18 | std::vector coors_range, int max_points, 19 | int max_voxels) { 20 | auto points_rw = points.template mutable_unchecked<2>(); 21 | auto voxels_rw = voxels.template mutable_unchecked<3>(); 22 | auto coors_rw = coors.mutable_unchecked<2>(); 23 | auto num_points_per_voxel_rw = num_points_per_voxel.mutable_unchecked<1>(); 24 | auto coor_to_voxelidx_rw = coor_to_voxelidx.mutable_unchecked(); 25 | auto N = points_rw.shape(0); 26 | auto num_features = points_rw.shape(1); 27 | // auto ndim = points_rw.shape(1) - 1; 28 | constexpr int ndim_minus_1 = NDim - 1; 29 | int voxel_num = 0; 30 | bool failed = false; 31 | int coor[NDim]; 32 | int c; 33 | int grid_size[NDim]; 34 | for (int i = 0; i < NDim; ++i) { 35 | grid_size[i] = 36 | round((coors_range[NDim + i] - coors_range[i]) / voxel_size[i]); 37 | } 38 | int voxelidx, num; 39 | for (int i = 0; i < N; ++i) { 40 | failed = false; 41 | for (int j = 0; j < NDim; ++j) { 42 | c = floor((points_rw(i, j) - coors_range[j]) / voxel_size[j]); 43 | if ((c < 0 || c >= grid_size[j])) { 44 | failed = true; 45 | break; 46 | } 47 | coor[ndim_minus_1 - j] = c; 48 | } 49 | if (failed) 50 | continue; 51 | voxelidx = coor_to_voxelidx_rw(coor[0], coor[1], coor[2]); 52 | if (voxelidx == -1) { 53 | voxelidx = voxel_num; 54 | if (voxel_num >= max_voxels) 55 | break; 56 | voxel_num += 1; 57 | coor_to_voxelidx_rw(coor[0], coor[1], coor[2]) = voxelidx; 58 | for (int k = 0; k < NDim; ++k) { 59 | coors_rw(voxelidx, k) = coor[k]; 60 | } 61 | } 62 | num = num_points_per_voxel_rw(voxelidx); 63 | if (num < max_points) { 64 | for (int k = 0; k < num_features; ++k) { 65 | voxels_rw(voxelidx, num, k) = points_rw(i, k); 66 | } 67 | num_points_per_voxel_rw(voxelidx) += 1; 68 | } 69 | } 70 | return voxel_num; 71 | } 72 | -------------------------------------------------------------------------------- /second/core/box_coders.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from abc import abstractproperty 4 | from second.core import box_np_ops 5 | import numpy as np 6 | 7 | class BoxCoder(object): 8 | """Abstract base class for box coder.""" 9 | __metaclass__ = ABCMeta 10 | 11 | @abstractproperty 12 | def code_size(self): 13 | pass 14 | 15 | def encode(self, boxes, anchors): 16 | return self._encode(boxes, anchors) 17 | 18 | def decode(self, rel_codes, anchors): 19 | return self._decode(rel_codes, anchors) 20 | 21 | @abstractmethod 22 | def _encode(self, boxes, anchors): 23 | pass 24 | 25 | @abstractmethod 26 | def _decode(self, rel_codes, anchors): 27 | pass 28 | 29 | 30 | class GroundBox3dCoder(BoxCoder): 31 | def __init__(self, linear_dim=False, vec_encode=False): 32 | super().__init__() 33 | self.linear_dim = linear_dim 34 | self.vec_encode = vec_encode 35 | 36 | @property 37 | def code_size(self): 38 | return 8 if self.vec_encode else 7 39 | 40 | def _encode(self, boxes, anchors): 41 | return box_np_ops.second_box_encode(boxes, anchors, self.vec_encode, self.linear_dim) 42 | 43 | def _decode(self, encodings, anchors): 44 | return box_np_ops.second_box_decode(encodings, anchors, self.vec_encode, self.linear_dim) 45 | 46 | 47 | class BevBoxCoder(BoxCoder): 48 | """WARNING: this coder will return encoding with size=5, but 49 | takes size=7 boxes, anchors 50 | """ 51 | def __init__(self, linear_dim=False, vec_encode=False, z_fixed=-1.0, h_fixed=2.0): 52 | super().__init__() 53 | self.linear_dim = linear_dim 54 | self.z_fixed = z_fixed 55 | self.h_fixed = h_fixed 56 | self.vec_encode = vec_encode 57 | 58 | @property 59 | def code_size(self): 60 | return 6 if self.vec_encode else 5 61 | 62 | def _encode(self, boxes, anchors): 63 | anchors = anchors[..., [0, 1, 3, 4, 6]] 64 | boxes = boxes[..., [0, 1, 3, 4, 6]] 65 | return box_np_ops.bev_box_encode(boxes, anchors, self.vec_encode, self.linear_dim) 66 | 67 | def _decode(self, encodings, anchors): 68 | anchors = anchors[..., [0, 1, 3, 4, 6]] 69 | ret = box_np_ops.bev_box_decode(encodings, anchors, self.vec_encode, self.linear_dim) 70 | z_fixed = np.full([*ret.shape[:-1], 1], self.z_fixed, dtype=ret.dtype) 71 | h_fixed = np.full([*ret.shape[:-1], 1], self.h_fixed, dtype=ret.dtype) 72 | return np.concatenate([ret[..., :2], z_fixed, ret[..., 2:4], h_fixed, ret[..., 4:]], axis=-1) 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /second/pytorch/builder/input_reader_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Input reader builder. 16 | 17 | Creates data sources for DetectionModels from an InputReader config. See 18 | input_reader.proto for options. 19 | 20 | Note: If users wishes to also use their own InputReaders with the Object 21 | Detection configuration framework, they should define their own builder function 22 | that wraps the build function. 23 | """ 24 | 25 | from torch.utils.data import Dataset 26 | 27 | from second.builder import dataset_builder 28 | from second.protos import input_reader_pb2 29 | 30 | 31 | class DatasetWrapper(Dataset): 32 | """ convert our dataset to Dataset class in pytorch. 33 | """ 34 | 35 | def __init__(self, dataset): 36 | self._dataset = dataset 37 | 38 | def __len__(self): 39 | return len(self._dataset) 40 | 41 | def __getitem__(self, idx): 42 | return self._dataset[idx] 43 | 44 | @property 45 | def dataset(self): 46 | return self._dataset 47 | 48 | 49 | def build(input_reader_config, 50 | model_config, 51 | training, 52 | voxel_generator, 53 | target_assigner=None) -> DatasetWrapper: 54 | """Builds a tensor dictionary based on the InputReader config. 55 | 56 | Args: 57 | input_reader_config: A input_reader_pb2.InputReader object. 58 | 59 | Returns: 60 | A tensor dict based on the input_reader_config. 61 | 62 | Raises: 63 | ValueError: On invalid input reader proto. 64 | ValueError: If no input paths are specified. 65 | """ 66 | if not isinstance(input_reader_config, input_reader_pb2.InputReader): 67 | raise ValueError('input_reader_config not of type ' 68 | 'input_reader_pb2.InputReader.') 69 | dataset = dataset_builder.build(input_reader_config, model_config, 70 | training, voxel_generator, target_assigner) 71 | dataset = DatasetWrapper(dataset) 72 | return dataset 73 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/FocusShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Focus shader 5 | * based on PaintEffect postprocess from ro.me 6 | * http://code.google.com/p/3-dreams-of-black/source/browse/deploy/js/effects/PaintEffect.js 7 | */ 8 | 9 | THREE.FocusShader = { 10 | 11 | uniforms : { 12 | 13 | "tDiffuse": { value: null }, 14 | "screenWidth": { value: 1024 }, 15 | "screenHeight": { value: 1024 }, 16 | "sampleDistance": { value: 0.94 }, 17 | "waveFactor": { value: 0.00125 } 18 | 19 | }, 20 | 21 | vertexShader: [ 22 | 23 | "varying vec2 vUv;", 24 | 25 | "void main() {", 26 | 27 | "vUv = uv;", 28 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 29 | 30 | "}" 31 | 32 | ].join( "\n" ), 33 | 34 | fragmentShader: [ 35 | 36 | "uniform float screenWidth;", 37 | "uniform float screenHeight;", 38 | "uniform float sampleDistance;", 39 | "uniform float waveFactor;", 40 | 41 | "uniform sampler2D tDiffuse;", 42 | 43 | "varying vec2 vUv;", 44 | 45 | "void main() {", 46 | 47 | "vec4 color, org, tmp, add;", 48 | "float sample_dist, f;", 49 | "vec2 vin;", 50 | "vec2 uv = vUv;", 51 | 52 | "add = color = org = texture2D( tDiffuse, uv );", 53 | 54 | "vin = ( uv - vec2( 0.5 ) ) * vec2( 1.4 );", 55 | "sample_dist = dot( vin, vin ) * 2.0;", 56 | 57 | "f = ( waveFactor * 100.0 + sample_dist ) * sampleDistance * 4.0;", 58 | 59 | "vec2 sampleSize = vec2( 1.0 / screenWidth, 1.0 / screenHeight ) * vec2( f );", 60 | 61 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.111964, 0.993712 ) * sampleSize );", 62 | "if( tmp.b < color.b ) color = tmp;", 63 | 64 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.846724, 0.532032 ) * sampleSize );", 65 | "if( tmp.b < color.b ) color = tmp;", 66 | 67 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.943883, -0.330279 ) * sampleSize );", 68 | "if( tmp.b < color.b ) color = tmp;", 69 | 70 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.330279, -0.943883 ) * sampleSize );", 71 | "if( tmp.b < color.b ) color = tmp;", 72 | 73 | "add += tmp = texture2D( tDiffuse, uv + vec2( -0.532032, -0.846724 ) * sampleSize );", 74 | "if( tmp.b < color.b ) color = tmp;", 75 | 76 | "add += tmp = texture2D( tDiffuse, uv + vec2( -0.993712, -0.111964 ) * sampleSize );", 77 | "if( tmp.b < color.b ) color = tmp;", 78 | 79 | "add += tmp = texture2D( tDiffuse, uv + vec2( -0.707107, 0.707107 ) * sampleSize );", 80 | "if( tmp.b < color.b ) color = tmp;", 81 | 82 | "color = color * vec4( 2.0 ) - ( add / vec4( 8.0 ) );", 83 | "color = color + ( add / vec4( 8.0 ) - color ) * ( vec4( 1.0 ) - vec4( sample_dist * 0.5 ) );", 84 | 85 | "gl_FragColor = vec4( color.rgb * color.rgb * vec3( 0.95 ) + color.rgb, 1.0 );", 86 | 87 | "}" 88 | 89 | 90 | ].join( "\n" ) 91 | }; 92 | -------------------------------------------------------------------------------- /second/protos/optimizer.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Messages for configuring the optimizing strategy for training object 6 | // detection models. 7 | 8 | // Top level optimizer message. 9 | message Optimizer { 10 | oneof optimizer { 11 | RMSPropOptimizer rms_prop_optimizer = 1; 12 | MomentumOptimizer momentum_optimizer = 2; 13 | AdamOptimizer adam_optimizer = 3; 14 | } 15 | bool use_moving_average = 4; 16 | float moving_average_decay = 5; 17 | } 18 | 19 | // Configuration message for the RMSPropOptimizer 20 | // See: https://www.tensorflow.org/api_docs/python/tf/train/RMSPropOptimizer 21 | message RMSPropOptimizer { 22 | LearningRate learning_rate = 1; 23 | float momentum_optimizer_value = 2; 24 | float decay = 3; 25 | float epsilon = 4; 26 | float weight_decay = 5; 27 | } 28 | 29 | // Configuration message for the MomentumOptimizer 30 | // See: https://www.tensorflow.org/api_docs/python/tf/train/MomentumOptimizer 31 | message MomentumOptimizer { 32 | LearningRate learning_rate = 1; 33 | float momentum_optimizer_value = 2; 34 | float weight_decay = 3; 35 | } 36 | 37 | // Configuration message for the AdamOptimizer 38 | // See: https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer 39 | message AdamOptimizer { 40 | LearningRate learning_rate = 1; 41 | float weight_decay = 2; 42 | } 43 | 44 | // Configuration message for optimizer learning rate. 45 | message LearningRate { 46 | oneof learning_rate { 47 | ConstantLearningRate constant_learning_rate = 1; 48 | ExponentialDecayLearningRate exponential_decay_learning_rate = 2; 49 | ManualStepLearningRate manual_step_learning_rate = 3; 50 | CosineDecayLearningRate cosine_decay_learning_rate = 4; 51 | } 52 | } 53 | 54 | // Configuration message for a constant learning rate. 55 | message ConstantLearningRate { 56 | float learning_rate = 1; 57 | } 58 | 59 | // Configuration message for an exponentially decaying learning rate. 60 | // See https://www.tensorflow.org/versions/master/api_docs/python/train/ \ 61 | // decaying_the_learning_rate#exponential_decay 62 | message ExponentialDecayLearningRate { 63 | float initial_learning_rate = 1; 64 | uint32 decay_steps = 2; 65 | float decay_factor = 3; 66 | bool staircase = 4; 67 | } 68 | 69 | // Configuration message for a manually defined learning rate schedule. 70 | message ManualStepLearningRate { 71 | float initial_learning_rate = 1; 72 | message LearningRateSchedule { 73 | uint32 step = 1; 74 | float learning_rate = 2; 75 | } 76 | repeated LearningRateSchedule schedule = 2; 77 | } 78 | 79 | // Configuration message for a cosine decaying learning rate as defined in 80 | // object_detection/utils/learning_schedules.py 81 | message CosineDecayLearningRate { 82 | float learning_rate_base = 1; 83 | uint32 total_steps = 2; 84 | float warmup_learning_rate = 3; 85 | uint32 warmup_steps = 4; 86 | } -------------------------------------------------------------------------------- /second/utils/loader.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from pathlib import Path 3 | import sys 4 | import os 5 | import logging 6 | logger = logging.getLogger('second.utils.loader') 7 | 8 | CUSTOM_LOADED_MODULES = {} 9 | 10 | 11 | def _get_possible_module_path(paths): 12 | ret = [] 13 | for p in paths: 14 | p = Path(p) 15 | for path in p.glob("*"): 16 | if path.suffix in ["py", ".so"] or (path.is_dir()): 17 | if path.stem.isidentifier(): 18 | ret.append(path) 19 | return ret 20 | 21 | 22 | def _get_regular_import_name(path, module_paths): 23 | path = Path(path) 24 | for mp in module_paths: 25 | mp = Path(mp) 26 | if mp == path: 27 | return path.stem 28 | try: 29 | relative_path = path.relative_to(Path(mp)) 30 | parts = list((relative_path.parent / relative_path.stem).parts) 31 | module_name = '.'.join([mp.stem] + parts) 32 | return module_name 33 | except: 34 | pass 35 | return None 36 | 37 | 38 | def import_file(path, name: str = None, add_to_sys=True, 39 | disable_warning=False): 40 | global CUSTOM_LOADED_MODULES 41 | path = Path(path) 42 | module_name = path.stem 43 | try: 44 | user_paths = os.environ['PYTHONPATH'].split(os.pathsep) 45 | except KeyError: 46 | user_paths = [] 47 | possible_paths = _get_possible_module_path(user_paths) 48 | model_import_name = _get_regular_import_name(path, possible_paths) 49 | if model_import_name is not None: 50 | return import_name(model_import_name) 51 | if name is not None: 52 | module_name = name 53 | spec = importlib.util.spec_from_file_location(module_name, path) 54 | module = importlib.util.module_from_spec(spec) 55 | spec.loader.exec_module(module) 56 | if not disable_warning: 57 | logger.warning(( 58 | f"Failed to perform regular import for file {path}. " 59 | "this means this file isn't in any folder in PYTHONPATH " 60 | "or don't have __init__.py in that project. " 61 | "directly file import may fail and some reflecting features are " 62 | "disabled even if import succeed. please add your project to PYTHONPATH " 63 | "or add __init__.py to ensure this file can be regularly imported. " 64 | )) 65 | 66 | if add_to_sys: # this will enable find objects defined in a file. 67 | # avoid replace system modules. 68 | if module_name in sys.modules and module_name not in CUSTOM_LOADED_MODULES: 69 | raise ValueError(f"{module_name} exists in system.") 70 | CUSTOM_LOADED_MODULES[module_name] = module 71 | sys.modules[module_name] = module 72 | return module 73 | 74 | 75 | def import_name(name, package=None): 76 | module = importlib.import_module(name, package) 77 | return module -------------------------------------------------------------------------------- /second/core/anchor_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from second.core import box_np_ops 3 | 4 | 5 | class AnchorGeneratorStride: 6 | def __init__(self, 7 | sizes=[1.6, 3.9, 1.56], 8 | anchor_strides=[0.4, 0.4, 1.0], 9 | anchor_offsets=[0.2, -39.8, -1.78], 10 | rotations=[0, np.pi / 2], 11 | class_id=None, 12 | match_threshold=-1, 13 | unmatch_threshold=-1, 14 | dtype=np.float32): 15 | self._sizes = sizes 16 | self._anchor_strides = anchor_strides 17 | self._anchor_offsets = anchor_offsets 18 | self._rotations = rotations 19 | self._dtype = dtype 20 | self._class_id = class_id 21 | self._match_threshold = match_threshold 22 | self._unmatch_threshold = unmatch_threshold 23 | 24 | @property 25 | def class_id(self): 26 | return self._class_id 27 | 28 | @property 29 | def match_threshold(self): 30 | return self._match_threshold 31 | 32 | @property 33 | def unmatch_threshold(self): 34 | return self._unmatch_threshold 35 | 36 | @property 37 | def num_anchors_per_localization(self): 38 | num_rot = len(self._rotations) 39 | num_size = np.array(self._sizes).reshape([-1, 3]).shape[0] 40 | return num_rot * num_size 41 | 42 | def generate(self, feature_map_size): 43 | return box_np_ops.create_anchors_3d_stride( 44 | feature_map_size, self._sizes, self._anchor_strides, 45 | self._anchor_offsets, self._rotations, self._dtype) 46 | 47 | class AnchorGeneratorRange: 48 | def __init__(self, 49 | anchor_ranges, 50 | sizes=[1.6, 3.9, 1.56], 51 | rotations=[0, np.pi / 2], 52 | class_id=None, 53 | match_threshold=-1, 54 | unmatch_threshold=-1, 55 | dtype=np.float32): 56 | self._sizes = sizes 57 | self._anchor_ranges = anchor_ranges 58 | self._rotations = rotations 59 | self._dtype = dtype 60 | self._class_id = class_id 61 | self._match_threshold = match_threshold 62 | self._unmatch_threshold = unmatch_threshold 63 | 64 | @property 65 | def class_id(self): 66 | return self._class_id 67 | 68 | @property 69 | def match_threshold(self): 70 | return self._match_threshold 71 | 72 | @property 73 | def unmatch_threshold(self): 74 | return self._unmatch_threshold 75 | 76 | @property 77 | def num_anchors_per_localization(self): 78 | num_rot = len(self._rotations) 79 | num_size = np.array(self._sizes).reshape([-1, 3]).shape[0] 80 | return num_rot * num_size 81 | 82 | def generate(self, feature_map_size): 83 | return box_np_ops.create_anchors_3d_range( 84 | feature_map_size, self._anchor_ranges, self._sizes, 85 | self._rotations, self._dtype) 86 | -------------------------------------------------------------------------------- /second/protos/model_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/model.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from second.protos import second_pb2 as second_dot_protos_dot_second__pb2 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='second/protos/model.proto', 20 | package='second.protos', 21 | syntax='proto3', 22 | serialized_options=None, 23 | serialized_pb=_b('\n\x19second/protos/model.proto\x12\rsecond.protos\x1a\x1asecond/protos/second.proto\"D\n\x0e\x44\x65tectionModel\x12)\n\x06second\x18\x01 \x01(\x0b\x32\x17.second.protos.VoxelNetH\x00\x42\x07\n\x05modelb\x06proto3') 24 | , 25 | dependencies=[second_dot_protos_dot_second__pb2.DESCRIPTOR,]) 26 | 27 | 28 | 29 | 30 | _DETECTIONMODEL = _descriptor.Descriptor( 31 | name='DetectionModel', 32 | full_name='second.protos.DetectionModel', 33 | filename=None, 34 | file=DESCRIPTOR, 35 | containing_type=None, 36 | fields=[ 37 | _descriptor.FieldDescriptor( 38 | name='second', full_name='second.protos.DetectionModel.second', index=0, 39 | number=1, type=11, cpp_type=10, label=1, 40 | has_default_value=False, default_value=None, 41 | message_type=None, enum_type=None, containing_type=None, 42 | is_extension=False, extension_scope=None, 43 | serialized_options=None, file=DESCRIPTOR), 44 | ], 45 | extensions=[ 46 | ], 47 | nested_types=[], 48 | enum_types=[ 49 | ], 50 | serialized_options=None, 51 | is_extendable=False, 52 | syntax='proto3', 53 | extension_ranges=[], 54 | oneofs=[ 55 | _descriptor.OneofDescriptor( 56 | name='model', full_name='second.protos.DetectionModel.model', 57 | index=0, containing_type=None, fields=[]), 58 | ], 59 | serialized_start=72, 60 | serialized_end=140, 61 | ) 62 | 63 | _DETECTIONMODEL.fields_by_name['second'].message_type = second_dot_protos_dot_second__pb2._VOXELNET 64 | _DETECTIONMODEL.oneofs_by_name['model'].fields.append( 65 | _DETECTIONMODEL.fields_by_name['second']) 66 | _DETECTIONMODEL.fields_by_name['second'].containing_oneof = _DETECTIONMODEL.oneofs_by_name['model'] 67 | DESCRIPTOR.message_types_by_name['DetectionModel'] = _DETECTIONMODEL 68 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 69 | 70 | DetectionModel = _reflection.GeneratedProtocolMessageType('DetectionModel', (_message.Message,), dict( 71 | DESCRIPTOR = _DETECTIONMODEL, 72 | __module__ = 'second.protos.model_pb2' 73 | # @@protoc_insertion_point(class_scope:second.protos.DetectionModel) 74 | )) 75 | _sym_db.RegisterMessage(DetectionModel) 76 | 77 | 78 | # @@protoc_insertion_point(module_scope) 79 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/FilmShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Film grain & scanlines shader 5 | * 6 | * - ported from HLSL to WebGL / GLSL 7 | * http://www.truevision3d.com/forums/showcase/staticnoise_colorblackwhite_scanline_shaders-t18698.0.html 8 | * 9 | * Screen Space Static Postprocessor 10 | * 11 | * Produces an analogue noise overlay similar to a film grain / TV static 12 | * 13 | * Original implementation and noise algorithm 14 | * Pat 'Hawthorne' Shearon 15 | * 16 | * Optimized scanlines + noise version with intensity scaling 17 | * Georg 'Leviathan' Steinrohder 18 | * 19 | * This version is provided under a Creative Commons Attribution 3.0 License 20 | * http://creativecommons.org/licenses/by/3.0/ 21 | */ 22 | 23 | THREE.FilmShader = { 24 | 25 | uniforms: { 26 | 27 | "tDiffuse": { value: null }, 28 | "time": { value: 0.0 }, 29 | "nIntensity": { value: 0.5 }, 30 | "sIntensity": { value: 0.05 }, 31 | "sCount": { value: 4096 }, 32 | "grayscale": { value: 1 } 33 | 34 | }, 35 | 36 | vertexShader: [ 37 | 38 | "varying vec2 vUv;", 39 | 40 | "void main() {", 41 | 42 | "vUv = uv;", 43 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 44 | 45 | "}" 46 | 47 | ].join( "\n" ), 48 | 49 | fragmentShader: [ 50 | 51 | "#include ", 52 | 53 | // control parameter 54 | "uniform float time;", 55 | 56 | "uniform bool grayscale;", 57 | 58 | // noise effect intensity value (0 = no effect, 1 = full effect) 59 | "uniform float nIntensity;", 60 | 61 | // scanlines effect intensity value (0 = no effect, 1 = full effect) 62 | "uniform float sIntensity;", 63 | 64 | // scanlines effect count value (0 = no effect, 4096 = full effect) 65 | "uniform float sCount;", 66 | 67 | "uniform sampler2D tDiffuse;", 68 | 69 | "varying vec2 vUv;", 70 | 71 | "void main() {", 72 | 73 | // sample the source 74 | "vec4 cTextureScreen = texture2D( tDiffuse, vUv );", 75 | 76 | // make some noise 77 | "float dx = rand( vUv + time );", 78 | 79 | // add noise 80 | "vec3 cResult = cTextureScreen.rgb + cTextureScreen.rgb * clamp( 0.1 + dx, 0.0, 1.0 );", 81 | 82 | // get us a sine and cosine 83 | "vec2 sc = vec2( sin( vUv.y * sCount ), cos( vUv.y * sCount ) );", 84 | 85 | // add scanlines 86 | "cResult += cTextureScreen.rgb * vec3( sc.x, sc.y, sc.x ) * sIntensity;", 87 | 88 | // interpolate between source and result by intensity 89 | "cResult = cTextureScreen.rgb + clamp( nIntensity, 0.0,1.0 ) * ( cResult - cTextureScreen.rgb );", 90 | 91 | // convert to grayscale if desired 92 | "if( grayscale ) {", 93 | 94 | "cResult = vec3( cResult.r * 0.3 + cResult.g * 0.59 + cResult.b * 0.11 );", 95 | 96 | "}", 97 | 98 | "gl_FragColor = vec4( cResult, cTextureScreen.a );", 99 | 100 | "}" 101 | 102 | ].join( "\n" ) 103 | 104 | }; 105 | -------------------------------------------------------------------------------- /torchplus/nn/modules/common.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from collections import OrderedDict 3 | 4 | import torch 5 | from torch.nn import functional as F 6 | 7 | class Empty(torch.nn.Module): 8 | def __init__(self, *args, **kwargs): 9 | super(Empty, self).__init__() 10 | 11 | def forward(self, *args, **kwargs): 12 | if len(args) == 1: 13 | return args[0] 14 | elif len(args) == 0: 15 | return None 16 | return args 17 | 18 | class Sequential(torch.nn.Module): 19 | r"""A sequential container. 20 | Modules will be added to it in the order they are passed in the constructor. 21 | Alternatively, an ordered dict of modules can also be passed in. 22 | 23 | To make it easier to understand, given is a small example:: 24 | 25 | # Example of using Sequential 26 | model = Sequential( 27 | nn.Conv2d(1,20,5), 28 | nn.ReLU(), 29 | nn.Conv2d(20,64,5), 30 | nn.ReLU() 31 | ) 32 | 33 | # Example of using Sequential with OrderedDict 34 | model = Sequential(OrderedDict([ 35 | ('conv1', nn.Conv2d(1,20,5)), 36 | ('relu1', nn.ReLU()), 37 | ('conv2', nn.Conv2d(20,64,5)), 38 | ('relu2', nn.ReLU()) 39 | ])) 40 | 41 | # Example of using Sequential with kwargs(python 3.6+) 42 | model = Sequential( 43 | conv1=nn.Conv2d(1,20,5), 44 | relu1=nn.ReLU(), 45 | conv2=nn.Conv2d(20,64,5), 46 | relu2=nn.ReLU() 47 | ) 48 | """ 49 | 50 | def __init__(self, *args, **kwargs): 51 | super(Sequential, self).__init__() 52 | if len(args) == 1 and isinstance(args[0], OrderedDict): 53 | for key, module in args[0].items(): 54 | self.add_module(key, module) 55 | else: 56 | for idx, module in enumerate(args): 57 | self.add_module(str(idx), module) 58 | for name, module in kwargs.items(): 59 | if sys.version_info < (3, 6): 60 | raise ValueError("kwargs only supported in py36+") 61 | if name in self._modules: 62 | raise ValueError("name exists.") 63 | self.add_module(name, module) 64 | 65 | def __getitem__(self, idx): 66 | if not (-len(self) <= idx < len(self)): 67 | raise IndexError('index {} is out of range'.format(idx)) 68 | if idx < 0: 69 | idx += len(self) 70 | it = iter(self._modules.values()) 71 | for i in range(idx): 72 | next(it) 73 | return next(it) 74 | 75 | def __len__(self): 76 | return len(self._modules) 77 | 78 | def add(self, module, name=None): 79 | if name is None: 80 | name = str(len(self._modules)) 81 | if name in self._modules: 82 | raise KeyError("name exists") 83 | self.add_module(name, module) 84 | 85 | def forward(self, input): 86 | # i = 0 87 | for module in self._modules.values(): 88 | # print(i) 89 | input = module(input) 90 | # i += 1 91 | return input -------------------------------------------------------------------------------- /second/core/cc/box_ops.h: -------------------------------------------------------------------------------- 1 | #ifndef BOX_OPS_H 2 | #define BOX_OPS_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | namespace py = pybind11; 9 | using namespace pybind11::literals; 10 | 11 | template 12 | inline py::array_t constant(ShapeContainer shape, DType value){ 13 | // create ROWMAJOR array. 14 | py::array_t array(shape); 15 | std::fill(array.mutable_data(), array.mutable_data() + array.size(), value); 16 | return array; 17 | } 18 | 19 | template 20 | inline py::array_t zeros(std::vector shape){ 21 | return constant>(shape, 0); 22 | } 23 | template 24 | py::array_t 25 | rbbox_iou(py::array_t box_corners, py::array_t qbox_corners, 26 | py::array_t standup_iou, DType standup_thresh) { 27 | namespace bg = boost::geometry; 28 | typedef bg::model::point point_t; 29 | typedef bg::model::polygon polygon_t; 30 | polygon_t poly, qpoly; 31 | std::vector poly_inter, poly_union; 32 | DType inter_area, union_area; 33 | auto box_corners_r = box_corners.template unchecked<3>(); 34 | auto qbox_corners_r = qbox_corners.template unchecked<3>(); 35 | auto standup_iou_r = standup_iou.template unchecked<2>(); 36 | auto N = box_corners_r.shape(0); 37 | auto K = qbox_corners_r.shape(0); 38 | py::array_t overlaps = zeros({N, K}); 39 | auto overlaps_rw = overlaps.template mutable_unchecked<2>(); 40 | if (N == 0 || K == 0) { 41 | return overlaps; 42 | } 43 | for (int k = 0; k < K; ++k) { 44 | for (int n = 0; n < N; ++n) { 45 | if (standup_iou_r(n, k) <= standup_thresh) 46 | continue; 47 | bg::append(poly, point_t(box_corners_r(n, 0, 0), box_corners_r(n, 0, 1))); 48 | bg::append(poly, point_t(box_corners_r(n, 1, 0), box_corners_r(n, 1, 1))); 49 | bg::append(poly, point_t(box_corners_r(n, 2, 0), box_corners_r(n, 2, 1))); 50 | bg::append(poly, point_t(box_corners_r(n, 3, 0), box_corners_r(n, 3, 1))); 51 | bg::append(poly, point_t(box_corners_r(n, 0, 0), box_corners_r(n, 0, 1))); 52 | bg::append(qpoly, 53 | point_t(qbox_corners_r(k, 0, 0), qbox_corners_r(k, 0, 1))); 54 | bg::append(qpoly, 55 | point_t(qbox_corners_r(k, 1, 0), qbox_corners_r(k, 1, 1))); 56 | bg::append(qpoly, 57 | point_t(qbox_corners_r(k, 2, 0), qbox_corners_r(k, 2, 1))); 58 | bg::append(qpoly, 59 | point_t(qbox_corners_r(k, 3, 0), qbox_corners_r(k, 3, 1))); 60 | bg::append(qpoly, 61 | point_t(qbox_corners_r(k, 0, 0), qbox_corners_r(k, 0, 1))); 62 | 63 | bg::intersection(poly, qpoly, poly_inter); 64 | 65 | if (!poly_inter.empty()) { 66 | inter_area = bg::area(poly_inter.front()); 67 | bg::union_(poly, qpoly, poly_union); 68 | if (!poly_union.empty()) { 69 | union_area = bg::area(poly_union.front()); 70 | overlaps_rw(n, k) = inter_area / union_area; 71 | } 72 | poly_union.clear(); 73 | } 74 | poly.clear(); 75 | qpoly.clear(); 76 | poly_inter.clear(); 77 | } 78 | } 79 | return overlaps; 80 | } 81 | 82 | 83 | #endif -------------------------------------------------------------------------------- /second/pytorch/inference.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | import torch 5 | 6 | import torchplus 7 | from second.core import box_np_ops 8 | from second.core.inference import InferenceContext 9 | from second.builder import target_assigner_builder, voxel_builder 10 | from second.pytorch.builder import box_coder_builder, second_builder 11 | from second.pytorch.models.voxelnet import VoxelNet 12 | from second.pytorch.train import predict_kitti_to_anno, example_convert_to_torch 13 | 14 | 15 | class TorchInferenceContext(InferenceContext): 16 | def __init__(self): 17 | super().__init__() 18 | self.net = None 19 | self.anchor_cache = None 20 | 21 | def _build(self): 22 | config = self.config 23 | input_cfg = config.eval_input_reader 24 | model_cfg = config.model.second 25 | train_cfg = config.train_config 26 | batch_size = 1 27 | voxel_generator = voxel_builder.build(model_cfg.voxel_generator) 28 | bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] 29 | grid_size = voxel_generator.grid_size 30 | self.voxel_generator = voxel_generator 31 | vfe_num_filters = list(model_cfg.voxel_feature_extractor.num_filters) 32 | 33 | box_coder = box_coder_builder.build(model_cfg.box_coder) 34 | target_assigner_cfg = model_cfg.target_assigner 35 | target_assigner = target_assigner_builder.build( 36 | target_assigner_cfg, bv_range, box_coder) 37 | self.target_assigner = target_assigner 38 | out_size_factor = model_cfg.rpn.layer_strides[0] // model_cfg.rpn.upsample_strides[0] 39 | self.net = second_builder.build(model_cfg, voxel_generator, 40 | target_assigner) 41 | self.net.cuda().eval() 42 | if train_cfg.enable_mixed_precision: 43 | self.net.half() 44 | self.net.metrics_to_float() 45 | self.net.convert_norm_to_float(self.net) 46 | feature_map_size = grid_size[:2] // out_size_factor 47 | feature_map_size = [*feature_map_size, 1][::-1] 48 | ret = target_assigner.generate_anchors(feature_map_size) 49 | anchors = ret["anchors"] 50 | anchors = anchors.reshape([-1, 7]) 51 | matched_thresholds = ret["matched_thresholds"] 52 | unmatched_thresholds = ret["unmatched_thresholds"] 53 | anchors_bv = box_np_ops.rbbox2d_to_near_bbox( 54 | anchors[:, [0, 1, 3, 4, 6]]) 55 | self.anchor_cache = { 56 | "anchors": anchors, 57 | "anchors_bv": anchors_bv, 58 | "matched_thresholds": matched_thresholds, 59 | "unmatched_thresholds": unmatched_thresholds, 60 | } 61 | 62 | def _restore(self, ckpt_path): 63 | ckpt_path = Path(ckpt_path) 64 | assert ckpt_path.suffix == ".tckpt" 65 | torchplus.train.restore(str(ckpt_path), self.net) 66 | 67 | def _inference(self, example): 68 | train_cfg = self.config.train_config 69 | input_cfg = self.config.eval_input_reader 70 | model_cfg = self.config.model.second 71 | example_torch = example_convert_to_torch(example) 72 | if train_cfg.enable_mixed_precision: 73 | float_dtype = torch.float16 74 | else: 75 | float_dtype = torch.float32 76 | 77 | result_annos = predict_kitti_to_anno( 78 | self.net, example_torch, list( 79 | input_cfg.class_names), 80 | model_cfg.post_center_limit_range, model_cfg.lidar_input) 81 | return result_annos 82 | 83 | def _ctx(self): 84 | return None 85 | -------------------------------------------------------------------------------- /second/core/target_assigner.py: -------------------------------------------------------------------------------- 1 | from second.core import box_np_ops 2 | from second.core.target_ops import create_target_np 3 | from second.core import region_similarity 4 | import numpy as np 5 | 6 | class TargetAssigner: 7 | def __init__(self, 8 | box_coder, 9 | anchor_generators, 10 | region_similarity_calculator=None, 11 | positive_fraction=None, 12 | sample_size=512): 13 | self._region_similarity_calculator = region_similarity_calculator 14 | self._box_coder = box_coder 15 | self._anchor_generators = anchor_generators 16 | self._positive_fraction = positive_fraction 17 | self._sample_size = sample_size 18 | 19 | @property 20 | def box_coder(self): 21 | return self._box_coder 22 | 23 | def assign(self, 24 | anchors, 25 | gt_boxes, 26 | anchors_mask=None, 27 | gt_classes=None, 28 | matched_thresholds=None, 29 | unmatched_thresholds=None): 30 | if anchors_mask is not None: 31 | prune_anchor_fn = lambda _: np.where(anchors_mask)[0] 32 | else: 33 | prune_anchor_fn = None 34 | 35 | def similarity_fn(anchors, gt_boxes): 36 | anchors_rbv = anchors[:, [0, 1, 3, 4, 6]] 37 | gt_boxes_rbv = gt_boxes[:, [0, 1, 3, 4, 6]] 38 | return self._region_similarity_calculator.compare( 39 | anchors_rbv, gt_boxes_rbv) 40 | 41 | def box_encoding_fn(boxes, anchors): 42 | return self._box_coder.encode(boxes, anchors) 43 | 44 | return create_target_np( 45 | anchors, 46 | gt_boxes, 47 | similarity_fn, 48 | box_encoding_fn, 49 | prune_anchor_fn=prune_anchor_fn, 50 | gt_classes=gt_classes, 51 | matched_threshold=matched_thresholds, 52 | unmatched_threshold=unmatched_thresholds, 53 | positive_fraction=self._positive_fraction, 54 | rpn_batch_size=self._sample_size, 55 | norm_by_num_examples=False, 56 | box_code_size=self.box_coder.code_size) 57 | 58 | def generate_anchors(self, feature_map_size): 59 | anchors_list = [] 60 | matched_thresholds = [ 61 | a.match_threshold for a in self._anchor_generators 62 | ] 63 | unmatched_thresholds = [ 64 | a.unmatch_threshold for a in self._anchor_generators 65 | ] 66 | match_list, unmatch_list = [], [] 67 | for anchor_generator, match_thresh, unmatch_thresh in zip( 68 | self._anchor_generators, matched_thresholds, 69 | unmatched_thresholds): 70 | anchors = anchor_generator.generate(feature_map_size) 71 | anchors = anchors.reshape([*anchors.shape[:3], -1, 7]) 72 | anchors_list.append(anchors) 73 | num_anchors = np.prod(anchors.shape[:-1]) 74 | match_list.append( 75 | np.full([num_anchors], match_thresh, anchors.dtype)) 76 | unmatch_list.append( 77 | np.full([num_anchors], unmatch_thresh, anchors.dtype)) 78 | anchors = np.concatenate(anchors_list, axis=-2) 79 | matched_thresholds = np.concatenate(match_list, axis=0) 80 | unmatched_thresholds = np.concatenate(unmatch_list, axis=0) 81 | return { 82 | "anchors": anchors, 83 | "matched_thresholds": matched_thresholds, 84 | "unmatched_thresholds": unmatched_thresholds 85 | } 86 | 87 | @property 88 | def num_anchors_per_location(self): 89 | num = 0 90 | for a_generator in self._anchor_generators: 91 | num += a_generator.num_anchors_per_localization 92 | return num -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # From https://github.com/ufoym/deepo/blob/master/docker/Dockerfile.pytorch-py36-cu90 2 | 3 | # ================================================================== 4 | # module list 5 | # ------------------------------------------------------------------ 6 | # python 3.6 (apt) 7 | # pytorch latest (pip) 8 | # ================================================================== 9 | 10 | FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 11 | RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ 12 | PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ 13 | GIT_CLONE="git clone --depth 10" && \ 14 | rm -rf /var/lib/apt/lists/* \ 15 | /etc/apt/sources.list.d/cuda.list \ 16 | /etc/apt/sources.list.d/nvidia-ml.list && \ 17 | apt-get update && \ 18 | # ================================================================== 19 | # tools 20 | # ------------------------------------------------------------------ 21 | DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ 22 | build-essential \ 23 | ca-certificates \ 24 | cmake \ 25 | wget \ 26 | git \ 27 | vim \ 28 | fish \ 29 | libsparsehash-dev \ 30 | && \ 31 | # ================================================================== 32 | # python 33 | # ------------------------------------------------------------------ 34 | DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ 35 | software-properties-common \ 36 | && \ 37 | add-apt-repository ppa:deadsnakes/ppa && \ 38 | apt-get update && \ 39 | DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ 40 | python3.6 \ 41 | python3.6-dev \ 42 | && \ 43 | wget -O ~/get-pip.py \ 44 | https://bootstrap.pypa.io/get-pip.py && \ 45 | python3.6 ~/get-pip.py && \ 46 | ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ 47 | ln -s /usr/bin/python3.6 /usr/local/bin/python && \ 48 | $PIP_INSTALL \ 49 | setuptools \ 50 | && \ 51 | $PIP_INSTALL \ 52 | numpy \ 53 | scipy \ 54 | matplotlib \ 55 | Cython \ 56 | && \ 57 | # ================================================================== 58 | # pytorch 59 | # ------------------------------------------------------------------ 60 | $PIP_INSTALL \ 61 | torch_nightly -f \ 62 | https://download.pytorch.org/whl/nightly/cu90/torch_nightly.html \ 63 | && \ 64 | $PIP_INSTALL \ 65 | torchvision_nightly \ 66 | && \ 67 | # ================================================================== 68 | # config & cleanup 69 | # ------------------------------------------------------------------ 70 | ldconfig && \ 71 | apt-get clean && \ 72 | apt-get autoremove && \ 73 | rm -rf /var/lib/apt/lists/* /tmp/* ~/* 74 | 75 | RUN PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ 76 | $PIP_INSTALL \ 77 | shapely fire pybind11 tensorboardX protobuf \ 78 | scikit-image numba pillow 79 | 80 | WORKDIR /root 81 | RUN wget https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz 82 | RUN tar xzvf boost_1_68_0.tar.gz 83 | RUN cp -r ./boost_1_68_0/boost /usr/include 84 | RUN rm -rf ./boost_1_68_0 85 | RUN rm -rf ./boost_1_68_0.tar.gz 86 | RUN git clone https://github.com/traveller59/second.pytorch.git --depth 10 87 | RUN git clone https://github.com/traveller59/SparseConvNet.git --depth 10 88 | RUN cd ./SparseConvNet && python setup.py install && cd .. && rm -rf SparseConvNet 89 | ENV NUMBAPRO_CUDA_DRIVER=/usr/lib/x86_64-linux-gnu/libcuda.so 90 | ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so 91 | ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice 92 | ENV PYTHONPATH=/root/second.pytorch 93 | 94 | VOLUME ["/root/data"] 95 | VOLUME ["/root/model"] 96 | WORKDIR /root/second.pytorch/second 97 | 98 | ENTRYPOINT ["fish"] 99 | -------------------------------------------------------------------------------- /second/pytorch/builder/optimizer_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Functions to build DetectionModel training optimizers.""" 16 | 17 | from torchplus.train import learning_schedules 18 | import torch 19 | 20 | 21 | def build(optimizer_config, params, name=None): 22 | """Create optimizer based on config. 23 | 24 | Args: 25 | optimizer_config: A Optimizer proto message. 26 | 27 | Returns: 28 | An optimizer and a list of variables for summary. 29 | 30 | Raises: 31 | ValueError: when using an unsupported input data type. 32 | """ 33 | optimizer_type = optimizer_config.WhichOneof('optimizer') 34 | optimizer = None 35 | 36 | if optimizer_type == 'rms_prop_optimizer': 37 | config = optimizer_config.rms_prop_optimizer 38 | optimizer = torch.optim.RMSprop( 39 | params, 40 | lr=_get_base_lr_by_lr_scheduler(config.learning_rate), 41 | alpha=config.decay, 42 | momentum=config.momentum_optimizer_value, 43 | eps=config.epsilon, 44 | weight_decay=config.weight_decay) 45 | 46 | if optimizer_type == 'momentum_optimizer': 47 | config = optimizer_config.momentum_optimizer 48 | optimizer = torch.optim.SGD( 49 | params, 50 | lr=_get_base_lr_by_lr_scheduler(config.learning_rate), 51 | momentum=config.momentum_optimizer_value, 52 | weight_decay=config.weight_decay) 53 | 54 | if optimizer_type == 'adam_optimizer': 55 | config = optimizer_config.adam_optimizer 56 | optimizer = torch.optim.Adam( 57 | params, 58 | lr=_get_base_lr_by_lr_scheduler(config.learning_rate), 59 | weight_decay=config.weight_decay) 60 | 61 | if optimizer is None: 62 | raise ValueError('Optimizer %s not supported.' % optimizer_type) 63 | 64 | if optimizer_config.use_moving_average: 65 | raise ValueError('torch don\'t support moving average') 66 | if name is None: 67 | # assign a name to optimizer for checkpoint system 68 | optimizer.name = optimizer_type 69 | else: 70 | optimizer.name = name 71 | return optimizer 72 | 73 | 74 | def _get_base_lr_by_lr_scheduler(learning_rate_config): 75 | base_lr = None 76 | learning_rate_type = learning_rate_config.WhichOneof('learning_rate') 77 | if learning_rate_type == 'constant_learning_rate': 78 | config = learning_rate_config.constant_learning_rate 79 | base_lr = config.learning_rate 80 | 81 | if learning_rate_type == 'exponential_decay_learning_rate': 82 | config = learning_rate_config.exponential_decay_learning_rate 83 | base_lr = config.initial_learning_rate 84 | 85 | if learning_rate_type == 'manual_step_learning_rate': 86 | config = learning_rate_config.manual_step_learning_rate 87 | base_lr = config.initial_learning_rate 88 | if not config.schedule: 89 | raise ValueError('Empty learning rate schedule.') 90 | 91 | if learning_rate_type == 'cosine_decay_learning_rate': 92 | config = learning_rate_config.cosine_decay_learning_rate 93 | base_lr = config.learning_rate_base 94 | if base_lr is None: 95 | raise ValueError( 96 | 'Learning_rate %s not supported.' % learning_rate_type) 97 | 98 | return base_lr 99 | -------------------------------------------------------------------------------- /second/core/inference.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import contextlib 3 | 4 | import numpy as np 5 | from google.protobuf import text_format 6 | 7 | from second.data.preprocess import merge_second_batch, prep_pointcloud 8 | from second.protos import pipeline_pb2 9 | 10 | 11 | class InferenceContext: 12 | def __init__(self): 13 | self.config = None 14 | self.root_path = None 15 | self.target_assigner = None 16 | self.voxel_generator = None 17 | self.anchor_cache = None 18 | self.built = False 19 | 20 | def get_inference_input_dict(self, info, points): 21 | assert self.anchor_cache is not None 22 | assert self.target_assigner is not None 23 | assert self.voxel_generator is not None 24 | assert self.config is not None 25 | assert self.built is True 26 | rect = info['calib/R0_rect'] 27 | P2 = info['calib/P2'] 28 | Trv2c = info['calib/Tr_velo_to_cam'] 29 | input_cfg = self.config.eval_input_reader 30 | model_cfg = self.config.model.second 31 | 32 | input_dict = { 33 | 'points': points, 34 | 'rect': rect, 35 | 'Trv2c': Trv2c, 36 | 'P2': P2, 37 | 'image_shape': np.array(info["img_shape"], dtype=np.int32), 38 | 'image_idx': info['image_idx'], 39 | 'image_path': info['img_path'], 40 | # 'pointcloud_num_features': num_point_features, 41 | } 42 | out_size_factor = model_cfg.rpn.layer_strides[0] // model_cfg.rpn.upsample_strides[0] 43 | example = prep_pointcloud( 44 | input_dict=input_dict, 45 | root_path=str(self.root_path), 46 | voxel_generator=self.voxel_generator, 47 | target_assigner=self.target_assigner, 48 | max_voxels=input_cfg.max_number_of_voxels, 49 | class_names=list(input_cfg.class_names), 50 | training=False, 51 | create_targets=False, 52 | shuffle_points=input_cfg.shuffle_points, 53 | generate_bev=False, 54 | without_reflectivity=model_cfg.without_reflectivity, 55 | num_point_features=model_cfg.num_point_features, 56 | anchor_area_threshold=input_cfg.anchor_area_threshold, 57 | anchor_cache=self.anchor_cache, 58 | out_size_factor=out_size_factor, 59 | out_dtype=np.float32) 60 | example["image_idx"] = info['image_idx'] 61 | example["image_shape"] = input_dict["image_shape"] 62 | example["points"] = points 63 | if "anchors_mask" in example: 64 | example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) 65 | ############# 66 | # convert example to batched example 67 | ############# 68 | example = merge_second_batch([example]) 69 | return example 70 | 71 | def get_config(self, path): 72 | config = pipeline_pb2.TrainEvalPipelineConfig() 73 | with open(path, "r") as f: 74 | proto_str = f.read() 75 | text_format.Merge(proto_str, config) 76 | return config 77 | 78 | @abc.abstractclassmethod 79 | def _build(self): 80 | raise NotImplementedError() 81 | 82 | def build(self, config_path): 83 | self.config = self.get_config(config_path) 84 | ret = self._build() 85 | self.built = True 86 | return ret 87 | 88 | @abc.abstractclassmethod 89 | def _inference(self, example): 90 | raise NotImplementedError() 91 | 92 | def inference(self, example): 93 | return self._inference(example) 94 | 95 | @abc.abstractclassmethod 96 | def _restore(self, ckpt_path): 97 | raise NotImplementedError() 98 | 99 | def restore(self, ckpt_path): 100 | return self._restore(ckpt_path) 101 | 102 | @abc.abstractclassmethod 103 | def _ctx(self): 104 | raise NotImplementedError() 105 | 106 | @contextlib.contextmanager 107 | def ctx(self): 108 | yield self._ctx() 109 | -------------------------------------------------------------------------------- /second/utils/buildtools/pybind11_build.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import subprocess 3 | import tempfile 4 | from pathlib import Path 5 | 6 | from second.utils.loader import import_file 7 | from second.utils.find import find_cuda_device_arch 8 | from .command import CUDALink, Gpp, Nvcc, compile_libraries, out 9 | 10 | 11 | class Pybind11Link(Gpp): 12 | def __init__(self, 13 | sources, 14 | target, 15 | std="c++11", 16 | includes: list = None, 17 | defines: dict = None, 18 | cflags: str = None, 19 | libraries: dict = None, 20 | lflags: str = None, 21 | extra_cflags: str = None, 22 | extra_lflags: str = None, 23 | build_directory: str = None): 24 | pb11_includes = subprocess.check_output( 25 | "python3 -m pybind11 --includes", 26 | shell=True).decode('utf8').strip("\n") 27 | cflags = cflags or '-fPIC -O3 ' 28 | cflags += pb11_includes 29 | super().__init__( 30 | sources, 31 | target, 32 | std, 33 | includes, 34 | defines, 35 | cflags, 36 | link=True, 37 | libraries=libraries, 38 | lflags=lflags, 39 | extra_cflags=extra_cflags, 40 | extra_lflags=extra_lflags, 41 | build_directory=build_directory) 42 | 43 | 44 | class Pybind11CUDALink(CUDALink): 45 | def __init__(self, 46 | sources, 47 | target, 48 | std="c++11", 49 | includes: list = None, 50 | defines: dict = None, 51 | cflags: str = None, 52 | libraries: dict = None, 53 | lflags: str = None, 54 | extra_cflags: str = None, 55 | extra_lflags: str = None, 56 | build_directory: str = None): 57 | pb11_includes = subprocess.check_output( 58 | "python3 -m pybind11 --includes", 59 | shell=True).decode('utf8').strip("\n") 60 | cflags = cflags or '-fPIC -O3 ' 61 | cflags += pb11_includes 62 | super().__init__( 63 | sources, 64 | target, 65 | std, 66 | includes, 67 | defines, 68 | cflags, 69 | libraries=libraries, 70 | lflags=lflags, 71 | extra_cflags=extra_cflags, 72 | extra_lflags=extra_lflags, 73 | build_directory=build_directory) 74 | 75 | 76 | def load_pb11(sources, 77 | target, 78 | cwd='.', 79 | cuda=False, 80 | arch=None, 81 | num_workers=4, 82 | includes: list = None, 83 | build_directory=None, 84 | compiler="g++"): 85 | cmd_groups = [] 86 | cmds = [] 87 | outs = [] 88 | main_sources = [] 89 | if arch is None: 90 | arch = find_cuda_device_arch() 91 | 92 | for s in sources: 93 | s = str(s) 94 | if ".cu" in s or ".cu.cc" in s: 95 | assert cuda is True, "cuda must be true if contain cuda file" 96 | cmds.append(Nvcc(s, out(s), arch)) 97 | outs.append(out(s)) 98 | else: 99 | main_sources.append(s) 100 | 101 | if cuda is True and arch is None: 102 | raise ValueError("you must specify arch if sources contains" 103 | " cuda files") 104 | cmd_groups.append(cmds) 105 | if cuda: 106 | cmd_groups.append( 107 | [Pybind11CUDALink(outs + main_sources, target, includes=includes)]) 108 | else: 109 | cmd_groups.append( 110 | [Pybind11Link(outs + main_sources, target, includes=includes)]) 111 | for cmds in cmd_groups: 112 | compile_libraries( 113 | cmds, cwd, num_workers=num_workers, compiler=compiler) 114 | 115 | return import_file(target, add_to_sys=False, disable_warning=True) 116 | -------------------------------------------------------------------------------- /second/pytorch/builder/lr_scheduler_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Functions to build DetectionModel training optimizers.""" 17 | 18 | from torchplus.train import learning_schedules 19 | import torch 20 | 21 | def build(optimizer_config, optimizer, last_step=-1): 22 | """Create lr scheduler based on config. note that 23 | lr_scheduler must accept a optimizer that has been restored. 24 | 25 | Args: 26 | optimizer_config: A Optimizer proto message. 27 | 28 | Returns: 29 | An optimizer and a list of variables for summary. 30 | 31 | Raises: 32 | ValueError: when using an unsupported input data type. 33 | """ 34 | optimizer_type = optimizer_config.WhichOneof('optimizer') 35 | 36 | if optimizer_type == 'rms_prop_optimizer': 37 | config = optimizer_config.rms_prop_optimizer 38 | lr_scheduler = _create_learning_rate_scheduler( 39 | config.learning_rate, optimizer, last_step=last_step) 40 | 41 | if optimizer_type == 'momentum_optimizer': 42 | config = optimizer_config.momentum_optimizer 43 | lr_scheduler = _create_learning_rate_scheduler( 44 | config.learning_rate, optimizer, last_step=last_step) 45 | 46 | if optimizer_type == 'adam_optimizer': 47 | config = optimizer_config.adam_optimizer 48 | lr_scheduler = _create_learning_rate_scheduler( 49 | config.learning_rate, optimizer, last_step=last_step) 50 | 51 | return lr_scheduler 52 | 53 | def _create_learning_rate_scheduler(learning_rate_config, optimizer, last_step=-1): 54 | """Create optimizer learning rate scheduler based on config. 55 | 56 | Args: 57 | learning_rate_config: A LearningRate proto message. 58 | 59 | Returns: 60 | A learning rate. 61 | 62 | Raises: 63 | ValueError: when using an unsupported input data type. 64 | """ 65 | lr_scheduler = None 66 | learning_rate_type = learning_rate_config.WhichOneof('learning_rate') 67 | if learning_rate_type == 'constant_learning_rate': 68 | config = learning_rate_config.constant_learning_rate 69 | lr_scheduler = learning_schedules.Constant( 70 | optimizer, last_step=last_step) 71 | 72 | if learning_rate_type == 'exponential_decay_learning_rate': 73 | config = learning_rate_config.exponential_decay_learning_rate 74 | lr_scheduler = learning_schedules.ExponentialDecay( 75 | optimizer, config.decay_steps, 76 | config.decay_factor, config.staircase, last_step=last_step) 77 | 78 | if learning_rate_type == 'manual_step_learning_rate': 79 | config = learning_rate_config.manual_step_learning_rate 80 | if not config.schedule: 81 | raise ValueError('Empty learning rate schedule.') 82 | learning_rate_step_boundaries = [x.step for x in config.schedule] 83 | learning_rate_sequence = [config.initial_learning_rate] 84 | learning_rate_sequence += [x.learning_rate for x in config.schedule] 85 | lr_scheduler = learning_schedules.ManualStepping( 86 | optimizer, learning_rate_step_boundaries, learning_rate_sequence, 87 | last_step=last_step) 88 | 89 | if learning_rate_type == 'cosine_decay_learning_rate': 90 | config = learning_rate_config.cosine_decay_learning_rate 91 | lr_scheduler = learning_schedules.CosineDecayWithWarmup( 92 | optimizer, config.total_steps, 93 | config.warmup_learning_rate, config.warmup_steps, 94 | last_step=last_step) 95 | 96 | if lr_scheduler is None: 97 | raise ValueError('Learning_rate %s not supported.' % learning_rate_type) 98 | 99 | return lr_scheduler -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/BloomPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.BloomPass = function ( strength, kernelSize, sigma, resolution ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | strength = ( strength !== undefined ) ? strength : 1; 10 | kernelSize = ( kernelSize !== undefined ) ? kernelSize : 25; 11 | sigma = ( sigma !== undefined ) ? sigma : 4.0; 12 | resolution = ( resolution !== undefined ) ? resolution : 256; 13 | 14 | // render targets 15 | 16 | var pars = { minFilter: THREE.LinearFilter, magFilter: THREE.LinearFilter, format: THREE.RGBAFormat }; 17 | 18 | this.renderTargetX = new THREE.WebGLRenderTarget( resolution, resolution, pars ); 19 | this.renderTargetX.texture.name = "BloomPass.x"; 20 | this.renderTargetY = new THREE.WebGLRenderTarget( resolution, resolution, pars ); 21 | this.renderTargetY.texture.name = "BloomPass.y"; 22 | 23 | // copy material 24 | 25 | if ( THREE.CopyShader === undefined ) 26 | console.error( "THREE.BloomPass relies on THREE.CopyShader" ); 27 | 28 | var copyShader = THREE.CopyShader; 29 | 30 | this.copyUniforms = THREE.UniformsUtils.clone( copyShader.uniforms ); 31 | 32 | this.copyUniforms[ "opacity" ].value = strength; 33 | 34 | this.materialCopy = new THREE.ShaderMaterial( { 35 | 36 | uniforms: this.copyUniforms, 37 | vertexShader: copyShader.vertexShader, 38 | fragmentShader: copyShader.fragmentShader, 39 | blending: THREE.AdditiveBlending, 40 | transparent: true 41 | 42 | } ); 43 | 44 | // convolution material 45 | 46 | if ( THREE.ConvolutionShader === undefined ) 47 | console.error( "THREE.BloomPass relies on THREE.ConvolutionShader" ); 48 | 49 | var convolutionShader = THREE.ConvolutionShader; 50 | 51 | this.convolutionUniforms = THREE.UniformsUtils.clone( convolutionShader.uniforms ); 52 | 53 | this.convolutionUniforms[ "uImageIncrement" ].value = THREE.BloomPass.blurX; 54 | this.convolutionUniforms[ "cKernel" ].value = THREE.ConvolutionShader.buildKernel( sigma ); 55 | 56 | this.materialConvolution = new THREE.ShaderMaterial( { 57 | 58 | uniforms: this.convolutionUniforms, 59 | vertexShader: convolutionShader.vertexShader, 60 | fragmentShader: convolutionShader.fragmentShader, 61 | defines: { 62 | "KERNEL_SIZE_FLOAT": kernelSize.toFixed( 1 ), 63 | "KERNEL_SIZE_INT": kernelSize.toFixed( 0 ) 64 | } 65 | 66 | } ); 67 | 68 | this.needsSwap = false; 69 | 70 | this.camera = new THREE.OrthographicCamera( - 1, 1, 1, - 1, 0, 1 ); 71 | this.scene = new THREE.Scene(); 72 | 73 | this.quad = new THREE.Mesh( new THREE.PlaneBufferGeometry( 2, 2 ), null ); 74 | this.quad.frustumCulled = false; // Avoid getting clipped 75 | this.scene.add( this.quad ); 76 | 77 | }; 78 | 79 | THREE.BloomPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 80 | 81 | constructor: THREE.BloomPass, 82 | 83 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 84 | 85 | if ( maskActive ) renderer.context.disable( renderer.context.STENCIL_TEST ); 86 | 87 | // Render quad with blured scene into texture (convolution pass 1) 88 | 89 | this.quad.material = this.materialConvolution; 90 | 91 | this.convolutionUniforms[ "tDiffuse" ].value = readBuffer.texture; 92 | this.convolutionUniforms[ "uImageIncrement" ].value = THREE.BloomPass.blurX; 93 | 94 | renderer.render( this.scene, this.camera, this.renderTargetX, true ); 95 | 96 | 97 | // Render quad with blured scene into texture (convolution pass 2) 98 | 99 | this.convolutionUniforms[ "tDiffuse" ].value = this.renderTargetX.texture; 100 | this.convolutionUniforms[ "uImageIncrement" ].value = THREE.BloomPass.blurY; 101 | 102 | renderer.render( this.scene, this.camera, this.renderTargetY, true ); 103 | 104 | // Render original scene with superimposed blur to texture 105 | 106 | this.quad.material = this.materialCopy; 107 | 108 | this.copyUniforms[ "tDiffuse" ].value = this.renderTargetY.texture; 109 | 110 | if ( maskActive ) renderer.context.enable( renderer.context.STENCIL_TEST ); 111 | 112 | renderer.render( this.scene, this.camera, readBuffer, this.clear ); 113 | 114 | } 115 | 116 | } ); 117 | 118 | THREE.BloomPass.blurX = new THREE.Vector2( 0.001953125, 0.0 ); 119 | THREE.BloomPass.blurY = new THREE.Vector2( 0.0, 0.001953125 ); 120 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/renderers/CSS2DRenderer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author mrdoob / http://mrdoob.com/ 3 | */ 4 | 5 | THREE.CSS2DObject = function ( element ) { 6 | 7 | THREE.Object3D.call( this ); 8 | 9 | this.element = element; 10 | this.element.style.position = 'absolute'; 11 | 12 | this.addEventListener( 'removed', function ( event ) { 13 | 14 | if ( this.element.parentNode !== null ) { 15 | 16 | this.element.parentNode.removeChild( this.element ); 17 | 18 | } 19 | 20 | } ); 21 | 22 | }; 23 | 24 | THREE.CSS2DObject.prototype = Object.create( THREE.Object3D.prototype ); 25 | THREE.CSS2DObject.prototype.constructor = THREE.CSS2DObject; 26 | 27 | // 28 | 29 | THREE.CSS2DRenderer = function () { 30 | 31 | console.log( 'THREE.CSS2DRenderer', THREE.REVISION ); 32 | 33 | var _width, _height; 34 | var _widthHalf, _heightHalf; 35 | 36 | var vector = new THREE.Vector3(); 37 | var viewMatrix = new THREE.Matrix4(); 38 | var viewProjectionMatrix = new THREE.Matrix4(); 39 | 40 | var cache = { 41 | objects: new WeakMap() 42 | }; 43 | 44 | var domElement = document.createElement( 'div' ); 45 | domElement.style.overflow = 'hidden'; 46 | 47 | this.domElement = domElement; 48 | 49 | this.getSize = function () { 50 | 51 | return { 52 | width: _width, 53 | height: _height 54 | }; 55 | 56 | }; 57 | 58 | this.setSize = function ( width, height ) { 59 | 60 | _width = width; 61 | _height = height; 62 | 63 | _widthHalf = _width / 2; 64 | _heightHalf = _height / 2; 65 | 66 | domElement.style.width = width + 'px'; 67 | domElement.style.height = height + 'px'; 68 | 69 | }; 70 | 71 | var renderObject = function ( object, camera ) { 72 | 73 | if ( object instanceof THREE.CSS2DObject ) { 74 | 75 | vector.setFromMatrixPosition( object.matrixWorld ); 76 | vector.applyMatrix4( viewProjectionMatrix ); 77 | 78 | var element = object.element; 79 | var style = 'translate(-50%,-50%) translate(' + ( vector.x * _widthHalf + _widthHalf ) + 'px,' + ( - vector.y * _heightHalf + _heightHalf ) + 'px)'; 80 | 81 | element.style.WebkitTransform = style; 82 | element.style.MozTransform = style; 83 | element.style.oTransform = style; 84 | element.style.transform = style; 85 | 86 | var objectData = { 87 | distanceToCameraSquared: getDistanceToSquared( camera, object ) 88 | }; 89 | 90 | cache.objects.set( object, objectData ); 91 | 92 | if ( element.parentNode !== domElement ) { 93 | 94 | domElement.appendChild( element ); 95 | 96 | } 97 | 98 | } 99 | 100 | for ( var i = 0, l = object.children.length; i < l; i ++ ) { 101 | 102 | renderObject( object.children[ i ], camera ); 103 | 104 | } 105 | 106 | }; 107 | 108 | var getDistanceToSquared = function () { 109 | 110 | var a = new THREE.Vector3(); 111 | var b = new THREE.Vector3(); 112 | 113 | return function ( object1, object2 ) { 114 | 115 | a.setFromMatrixPosition( object1.matrixWorld ); 116 | b.setFromMatrixPosition( object2.matrixWorld ); 117 | 118 | return a.distanceToSquared( b ); 119 | 120 | }; 121 | 122 | }(); 123 | 124 | var filterAndFlatten = function ( scene ) { 125 | 126 | var result = []; 127 | 128 | scene.traverse( function ( object ) { 129 | 130 | if ( object instanceof THREE.CSS2DObject ) result.push( object ); 131 | 132 | } ); 133 | 134 | return result; 135 | 136 | }; 137 | 138 | var zOrder = function ( scene ) { 139 | 140 | var sorted = filterAndFlatten( scene ).sort( function ( a, b ) { 141 | 142 | var distanceA = cache.objects.get( a ).distanceToCameraSquared; 143 | var distanceB = cache.objects.get( b ).distanceToCameraSquared; 144 | 145 | return distanceA - distanceB; 146 | 147 | } ); 148 | 149 | var zMax = sorted.length; 150 | 151 | for ( var i = 0, l = sorted.length; i < l; i ++ ) { 152 | 153 | sorted[ i ].element.style.zIndex = zMax - i; 154 | 155 | } 156 | 157 | }; 158 | 159 | this.render = function ( scene, camera ) { 160 | 161 | scene.updateMatrixWorld(); 162 | 163 | if ( camera.parent === null ) camera.updateMatrixWorld(); 164 | 165 | viewMatrix.copy( camera.matrixWorldInverse ); 166 | viewProjectionMatrix.multiplyMatrices( camera.projectionMatrix, viewMatrix ); 167 | 168 | renderObject( scene, camera ); 169 | zOrder( scene ); 170 | 171 | }; 172 | 173 | }; 174 | -------------------------------------------------------------------------------- /torchplus/train/optim.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict, Iterable 2 | 3 | import torch 4 | from copy import deepcopy 5 | from itertools import chain 6 | from torch.autograd import Variable 7 | 8 | required = object() 9 | 10 | def param_fp32_copy(params): 11 | param_copy = [ 12 | param.clone().type(torch.cuda.FloatTensor).detach() for param in params 13 | ] 14 | for param in param_copy: 15 | param.requires_grad = True 16 | return param_copy 17 | 18 | def set_grad(params, params_with_grad, scale=1.0): 19 | for param, param_w_grad in zip(params, params_with_grad): 20 | if param.grad is None: 21 | param.grad = torch.nn.Parameter( 22 | param.data.new().resize_(*param.data.size())) 23 | grad = param_w_grad.grad.data 24 | if scale is not None: 25 | grad /= scale 26 | if torch.isnan(grad).any() or torch.isinf(grad).any(): 27 | return True # invalid grad 28 | param.grad.data.copy_(grad) 29 | return False 30 | 31 | class MixedPrecisionWrapper(object): 32 | """mixed precision optimizer wrapper. 33 | Arguments: 34 | optimizer (torch.optim.Optimizer): an instance of 35 | :class:`torch.optim.Optimizer` 36 | scale: (float): a scalar for grad scale. 37 | auto_scale: (bool): whether enable auto scale. 38 | The algorihm of auto scale is discribled in 39 | http://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html 40 | """ 41 | 42 | def __init__(self, 43 | optimizer, 44 | scale=None, 45 | auto_scale=True, 46 | inc_factor=2.0, 47 | dec_factor=0.5, 48 | num_iters_be_stable=500): 49 | if not isinstance(optimizer, torch.optim.Optimizer): 50 | raise ValueError("must provide a torch.optim.Optimizer") 51 | self.optimizer = optimizer 52 | if hasattr(self.optimizer, 'name'): 53 | self.name = self.optimizer.name # for ckpt system 54 | param_groups_copy = [] 55 | for i, group in enumerate(optimizer.param_groups): 56 | group_copy = {n: v for n, v in group.items() if n != 'params'} 57 | group_copy['params'] = param_fp32_copy(group['params']) 58 | param_groups_copy.append(group_copy) 59 | 60 | # switch param_groups, may be dangerous 61 | self.param_groups = optimizer.param_groups 62 | optimizer.param_groups = param_groups_copy 63 | self.grad_scale = scale 64 | self.auto_scale = auto_scale 65 | self.inc_factor = inc_factor 66 | self.dec_factor = dec_factor 67 | self.stable_iter_count = 0 68 | self.num_iters_be_stable = num_iters_be_stable 69 | 70 | def __getstate__(self): 71 | return self.optimizer.__getstate__() 72 | 73 | def __setstate__(self, state): 74 | return self.optimizer.__setstate__(state) 75 | 76 | def __repr__(self): 77 | return self.optimizer.__repr__() 78 | 79 | def state_dict(self): 80 | return self.optimizer.state_dict() 81 | 82 | def load_state_dict(self, state_dict): 83 | return self.optimizer.load_state_dict(state_dict) 84 | 85 | def zero_grad(self): 86 | return self.optimizer.zero_grad() 87 | 88 | def step(self, closure=None): 89 | for g, g_copy in zip(self.param_groups, self.optimizer.param_groups): 90 | invalid = set_grad(g_copy['params'], g['params'], self.grad_scale) 91 | if invalid: 92 | if self.grad_scale is None or self.auto_scale is False: 93 | raise ValueError("nan/inf detected but auto_scale disabled.") 94 | self.grad_scale *= self.dec_factor 95 | print('scale decay to {}'.format(self.grad_scale)) 96 | return 97 | if self.auto_scale is True: 98 | self.stable_iter_count += 1 99 | if self.stable_iter_count > self.num_iters_be_stable: 100 | if self.grad_scale is not None: 101 | self.grad_scale *= self.inc_factor 102 | self.stable_iter_count = 0 103 | 104 | if closure is None: 105 | self.optimizer.step() 106 | else: 107 | self.optimizer.step(closure) 108 | for g, g_copy in zip(self.param_groups, self.optimizer.param_groups): 109 | for p_copy, p in zip(g_copy['params'], g['params']): 110 | p.data.copy_(p_copy.data) 111 | 112 | -------------------------------------------------------------------------------- /second/core/region_similarity.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Region Similarity Calculators for BoxLists. 17 | 18 | Region Similarity Calculators compare a pairwise measure of similarity 19 | between the boxes in two BoxLists. 20 | """ 21 | from abc import ABCMeta 22 | from abc import abstractmethod 23 | 24 | from second.core import box_np_ops 25 | 26 | class RegionSimilarityCalculator(object): 27 | """Abstract base class for 2d region similarity calculator.""" 28 | __metaclass__ = ABCMeta 29 | 30 | def compare(self, boxes1, boxes2): 31 | """Computes matrix of pairwise similarity between BoxLists. 32 | 33 | This op (to be overriden) computes a measure of pairwise similarity between 34 | the boxes in the given BoxLists. Higher values indicate more similarity. 35 | 36 | Note that this method simply measures similarity and does not explicitly 37 | perform a matching. 38 | 39 | Args: 40 | boxes1: [N, 5] [x,y,w,l,r] tensor. 41 | boxes2: [M, 5] [x,y,w,l,r] tensor. 42 | 43 | Returns: 44 | a (float32) tensor of shape [N, M] with pairwise similarity score. 45 | """ 46 | return self._compare(boxes1, boxes2) 47 | 48 | @abstractmethod 49 | def _compare(self, boxes1, boxes2): 50 | pass 51 | 52 | 53 | class RotateIouSimilarity(RegionSimilarityCalculator): 54 | """Class to compute similarity based on Intersection over Union (IOU) metric. 55 | 56 | This class computes pairwise similarity between two BoxLists based on IOU. 57 | """ 58 | 59 | def _compare(self, boxes1, boxes2): 60 | """Compute pairwise IOU similarity between the two BoxLists. 61 | 62 | Args: 63 | boxlist1: BoxList holding N boxes. 64 | boxlist2: BoxList holding M boxes. 65 | 66 | Returns: 67 | A tensor with shape [N, M] representing pairwise iou scores. 68 | """ 69 | 70 | return box_np_ops.riou_cc(boxes1, boxes2) 71 | 72 | 73 | class NearestIouSimilarity(RegionSimilarityCalculator): 74 | """Class to compute similarity based on the squared distance metric. 75 | 76 | This class computes pairwise similarity between two BoxLists based on the 77 | negative squared distance metric. 78 | """ 79 | 80 | def _compare(self, boxes1, boxes2): 81 | """Compute matrix of (negated) sq distances. 82 | 83 | Args: 84 | boxlist1: BoxList holding N boxes. 85 | boxlist2: BoxList holding M boxes. 86 | 87 | Returns: 88 | A tensor with shape [N, M] representing negated pairwise squared distance. 89 | """ 90 | boxes1_bv = box_np_ops.rbbox2d_to_near_bbox(boxes1) 91 | boxes2_bv = box_np_ops.rbbox2d_to_near_bbox(boxes2) 92 | ret = box_np_ops.iou_jit(boxes1_bv, boxes2_bv, eps=0.0) 93 | return ret 94 | 95 | 96 | class DistanceSimilarity(RegionSimilarityCalculator): 97 | """Class to compute similarity based on Intersection over Area (IOA) metric. 98 | 99 | This class computes pairwise similarity between two BoxLists based on their 100 | pairwise intersections divided by the areas of second BoxLists. 101 | """ 102 | def __init__(self, distance_norm, with_rotation=False, rotation_alpha=0.5): 103 | self._distance_norm = distance_norm 104 | self._with_rotation = with_rotation 105 | self._rotation_alpha = rotation_alpha 106 | 107 | def _compare(self, boxes1, boxes2): 108 | """Compute matrix of (negated) sq distances. 109 | 110 | Args: 111 | boxlist1: BoxList holding N boxes. 112 | boxlist2: BoxList holding M boxes. 113 | 114 | Returns: 115 | A tensor with shape [N, M] representing negated pairwise squared distance. 116 | """ 117 | return box_np_ops.distance_similarity( 118 | boxes1[..., [0, 1, -1]], 119 | boxes2[..., [0, 1, -1]], 120 | dist_norm=self._distance_norm, 121 | with_rotation=self._with_rotation, 122 | rot_alpha=self._rotation_alpha) 123 | -------------------------------------------------------------------------------- /second/pytorch/builder/second_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 yanyan. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """VoxelNet builder. 16 | """ 17 | 18 | from second.protos import second_pb2 19 | from second.pytorch.builder import losses_builder 20 | from second.pytorch.models.voxelnet import LossNormType, VoxelNet 21 | 22 | 23 | def build(model_cfg: second_pb2.VoxelNet, voxel_generator, 24 | target_assigner) -> VoxelNet: 25 | """build second pytorch instance. 26 | """ 27 | if not isinstance(model_cfg, second_pb2.VoxelNet): 28 | raise ValueError('model_cfg not of type ' 'second_pb2.VoxelNet.') 29 | vfe_num_filters = list(model_cfg.voxel_feature_extractor.num_filters) 30 | vfe_with_distance = model_cfg.voxel_feature_extractor.with_distance 31 | grid_size = voxel_generator.grid_size 32 | dense_shape = [1] + grid_size[::-1].tolist() + [vfe_num_filters[-1]] 33 | num_class = model_cfg.num_class 34 | 35 | num_input_features = model_cfg.num_point_features 36 | if model_cfg.without_reflectivity: 37 | num_input_features = 3 38 | loss_norm_type_dict = { 39 | 0: LossNormType.NormByNumExamples, 40 | 1: LossNormType.NormByNumPositives, 41 | 2: LossNormType.NormByNumPosNeg, 42 | } 43 | loss_norm_type = loss_norm_type_dict[model_cfg.loss_norm_type] 44 | 45 | losses = losses_builder.build(model_cfg.loss) 46 | encode_rad_error_by_sin = model_cfg.encode_rad_error_by_sin 47 | cls_loss_ftor, loc_loss_ftor, cls_weight, loc_weight, _ = losses 48 | pos_cls_weight = model_cfg.pos_class_weight 49 | neg_cls_weight = model_cfg.neg_class_weight 50 | direction_loss_weight = model_cfg.direction_loss_weight 51 | 52 | net = VoxelNet( 53 | dense_shape, 54 | num_class=num_class, 55 | vfe_class_name=model_cfg.voxel_feature_extractor.module_class_name, 56 | vfe_num_filters=vfe_num_filters, 57 | middle_class_name=model_cfg.middle_feature_extractor.module_class_name, 58 | middle_num_filters_d1=list( 59 | model_cfg.middle_feature_extractor.num_filters_down1), 60 | middle_num_filters_d2=list( 61 | model_cfg.middle_feature_extractor.num_filters_down2), 62 | rpn_class_name=model_cfg.rpn.module_class_name, 63 | rpn_layer_nums=list(model_cfg.rpn.layer_nums), 64 | rpn_layer_strides=list(model_cfg.rpn.layer_strides), 65 | rpn_num_filters=list(model_cfg.rpn.num_filters), 66 | rpn_upsample_strides=list(model_cfg.rpn.upsample_strides), 67 | rpn_num_upsample_filters=list(model_cfg.rpn.num_upsample_filters), 68 | use_norm=True, 69 | use_rotate_nms=model_cfg.use_rotate_nms, 70 | multiclass_nms=model_cfg.use_multi_class_nms, 71 | nms_score_threshold=model_cfg.nms_score_threshold, 72 | nms_pre_max_size=model_cfg.nms_pre_max_size, 73 | nms_post_max_size=model_cfg.nms_post_max_size, 74 | nms_iou_threshold=model_cfg.nms_iou_threshold, 75 | use_sigmoid_score=model_cfg.use_sigmoid_score, 76 | encode_background_as_zeros=model_cfg.encode_background_as_zeros, 77 | use_direction_classifier=model_cfg.use_direction_classifier, 78 | use_bev=model_cfg.use_bev, 79 | num_input_features=num_input_features, 80 | num_groups=model_cfg.rpn.num_groups, 81 | use_groupnorm=model_cfg.rpn.use_groupnorm, 82 | with_distance=vfe_with_distance, 83 | cls_loss_weight=cls_weight, 84 | loc_loss_weight=loc_weight, 85 | pos_cls_weight=pos_cls_weight, 86 | neg_cls_weight=neg_cls_weight, 87 | direction_loss_weight=direction_loss_weight, 88 | loss_norm_type=loss_norm_type, 89 | encode_rad_error_by_sin=encode_rad_error_by_sin, 90 | loc_loss_ftor=loc_loss_ftor, 91 | cls_loss_ftor=cls_loss_ftor, 92 | target_assigner=target_assigner, 93 | voxel_size=voxel_generator.voxel_size, 94 | pc_range=voxel_generator.point_cloud_range 95 | ) 96 | return net 97 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/EffectComposer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.EffectComposer = function ( renderer, renderTarget ) { 6 | 7 | this.renderer = renderer; 8 | 9 | if ( renderTarget === undefined ) { 10 | 11 | var parameters = { 12 | minFilter: THREE.LinearFilter, 13 | magFilter: THREE.LinearFilter, 14 | format: THREE.RGBAFormat, 15 | stencilBuffer: false 16 | }; 17 | 18 | var size = renderer.getDrawingBufferSize(); 19 | renderTarget = new THREE.WebGLRenderTarget( size.width, size.height, parameters ); 20 | renderTarget.texture.name = 'EffectComposer.rt1'; 21 | 22 | } 23 | 24 | this.renderTarget1 = renderTarget; 25 | this.renderTarget2 = renderTarget.clone(); 26 | this.renderTarget2.texture.name = 'EffectComposer.rt2'; 27 | 28 | this.writeBuffer = this.renderTarget1; 29 | this.readBuffer = this.renderTarget2; 30 | 31 | this.passes = []; 32 | 33 | // dependencies 34 | 35 | if ( THREE.CopyShader === undefined ) { 36 | 37 | console.error( 'THREE.EffectComposer relies on THREE.CopyShader' ); 38 | 39 | } 40 | 41 | if ( THREE.ShaderPass === undefined ) { 42 | 43 | console.error( 'THREE.EffectComposer relies on THREE.ShaderPass' ); 44 | 45 | } 46 | 47 | this.copyPass = new THREE.ShaderPass( THREE.CopyShader ); 48 | 49 | }; 50 | 51 | Object.assign( THREE.EffectComposer.prototype, { 52 | 53 | swapBuffers: function () { 54 | 55 | var tmp = this.readBuffer; 56 | this.readBuffer = this.writeBuffer; 57 | this.writeBuffer = tmp; 58 | 59 | }, 60 | 61 | addPass: function ( pass ) { 62 | 63 | this.passes.push( pass ); 64 | 65 | var size = this.renderer.getDrawingBufferSize(); 66 | pass.setSize( size.width, size.height ); 67 | 68 | }, 69 | 70 | insertPass: function ( pass, index ) { 71 | 72 | this.passes.splice( index, 0, pass ); 73 | 74 | }, 75 | 76 | render: function ( delta ) { 77 | 78 | var maskActive = false; 79 | 80 | var pass, i, il = this.passes.length; 81 | 82 | for ( i = 0; i < il; i ++ ) { 83 | 84 | pass = this.passes[ i ]; 85 | 86 | if ( pass.enabled === false ) continue; 87 | 88 | pass.render( this.renderer, this.writeBuffer, this.readBuffer, delta, maskActive ); 89 | 90 | if ( pass.needsSwap ) { 91 | 92 | if ( maskActive ) { 93 | 94 | var context = this.renderer.context; 95 | 96 | context.stencilFunc( context.NOTEQUAL, 1, 0xffffffff ); 97 | 98 | this.copyPass.render( this.renderer, this.writeBuffer, this.readBuffer, delta ); 99 | 100 | context.stencilFunc( context.EQUAL, 1, 0xffffffff ); 101 | 102 | } 103 | 104 | this.swapBuffers(); 105 | 106 | } 107 | 108 | if ( THREE.MaskPass !== undefined ) { 109 | 110 | if ( pass instanceof THREE.MaskPass ) { 111 | 112 | maskActive = true; 113 | 114 | } else if ( pass instanceof THREE.ClearMaskPass ) { 115 | 116 | maskActive = false; 117 | 118 | } 119 | 120 | } 121 | 122 | } 123 | 124 | }, 125 | 126 | reset: function ( renderTarget ) { 127 | 128 | if ( renderTarget === undefined ) { 129 | 130 | var size = this.renderer.getDrawingBufferSize(); 131 | 132 | renderTarget = this.renderTarget1.clone(); 133 | renderTarget.setSize( size.width, size.height ); 134 | 135 | } 136 | 137 | this.renderTarget1.dispose(); 138 | this.renderTarget2.dispose(); 139 | this.renderTarget1 = renderTarget; 140 | this.renderTarget2 = renderTarget.clone(); 141 | 142 | this.writeBuffer = this.renderTarget1; 143 | this.readBuffer = this.renderTarget2; 144 | 145 | }, 146 | 147 | setSize: function ( width, height ) { 148 | 149 | this.renderTarget1.setSize( width, height ); 150 | this.renderTarget2.setSize( width, height ); 151 | 152 | for ( var i = 0; i < this.passes.length; i ++ ) { 153 | 154 | this.passes[ i ].setSize( width, height ); 155 | 156 | } 157 | 158 | } 159 | 160 | } ); 161 | 162 | 163 | THREE.Pass = function () { 164 | 165 | // if set to true, the pass is processed by the composer 166 | this.enabled = true; 167 | 168 | // if set to true, the pass indicates to swap read and write buffer after rendering 169 | this.needsSwap = true; 170 | 171 | // if set to true, the pass clears its buffer before rendering 172 | this.clear = false; 173 | 174 | // if set to true, the result of the pass is rendered to screen 175 | this.renderToScreen = false; 176 | 177 | }; 178 | 179 | Object.assign( THREE.Pass.prototype, { 180 | 181 | setSize: function ( width, height ) {}, 182 | 183 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 184 | 185 | console.error( 'THREE.Pass: .render() must be implemented in derived pass.' ); 186 | 187 | } 188 | 189 | } ); 190 | -------------------------------------------------------------------------------- /second/protos/voxel_generator_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/voxel_generator.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | 16 | 17 | DESCRIPTOR = _descriptor.FileDescriptor( 18 | name='second/protos/voxel_generator.proto', 19 | package='second.protos', 20 | syntax='proto3', 21 | serialized_options=None, 22 | serialized_pb=_b('\n#second/protos/voxel_generator.proto\x12\rsecond.protos\"\xbc\x01\n\x0eVoxelGenerator\x12\x12\n\nvoxel_size\x18\x01 \x03(\x02\x12\x19\n\x11point_cloud_range\x18\x02 \x03(\x02\x12&\n\x1emax_number_of_points_per_voxel\x18\x03 \x01(\r\x12\x19\n\x11submanifold_group\x18\x04 \x01(\x08\x12\x18\n\x10submanifold_size\x18\x05 \x03(\r\x12\x1e\n\x16submanifold_max_points\x18\x06 \x01(\rb\x06proto3') 23 | ) 24 | 25 | 26 | 27 | 28 | _VOXELGENERATOR = _descriptor.Descriptor( 29 | name='VoxelGenerator', 30 | full_name='second.protos.VoxelGenerator', 31 | filename=None, 32 | file=DESCRIPTOR, 33 | containing_type=None, 34 | fields=[ 35 | _descriptor.FieldDescriptor( 36 | name='voxel_size', full_name='second.protos.VoxelGenerator.voxel_size', index=0, 37 | number=1, type=2, cpp_type=6, label=3, 38 | has_default_value=False, default_value=[], 39 | message_type=None, enum_type=None, containing_type=None, 40 | is_extension=False, extension_scope=None, 41 | serialized_options=None, file=DESCRIPTOR), 42 | _descriptor.FieldDescriptor( 43 | name='point_cloud_range', full_name='second.protos.VoxelGenerator.point_cloud_range', index=1, 44 | number=2, type=2, cpp_type=6, label=3, 45 | has_default_value=False, default_value=[], 46 | message_type=None, enum_type=None, containing_type=None, 47 | is_extension=False, extension_scope=None, 48 | serialized_options=None, file=DESCRIPTOR), 49 | _descriptor.FieldDescriptor( 50 | name='max_number_of_points_per_voxel', full_name='second.protos.VoxelGenerator.max_number_of_points_per_voxel', index=2, 51 | number=3, type=13, cpp_type=3, label=1, 52 | has_default_value=False, default_value=0, 53 | message_type=None, enum_type=None, containing_type=None, 54 | is_extension=False, extension_scope=None, 55 | serialized_options=None, file=DESCRIPTOR), 56 | _descriptor.FieldDescriptor( 57 | name='submanifold_group', full_name='second.protos.VoxelGenerator.submanifold_group', index=3, 58 | number=4, type=8, cpp_type=7, label=1, 59 | has_default_value=False, default_value=False, 60 | message_type=None, enum_type=None, containing_type=None, 61 | is_extension=False, extension_scope=None, 62 | serialized_options=None, file=DESCRIPTOR), 63 | _descriptor.FieldDescriptor( 64 | name='submanifold_size', full_name='second.protos.VoxelGenerator.submanifold_size', index=4, 65 | number=5, type=13, cpp_type=3, label=3, 66 | has_default_value=False, default_value=[], 67 | message_type=None, enum_type=None, containing_type=None, 68 | is_extension=False, extension_scope=None, 69 | serialized_options=None, file=DESCRIPTOR), 70 | _descriptor.FieldDescriptor( 71 | name='submanifold_max_points', full_name='second.protos.VoxelGenerator.submanifold_max_points', index=5, 72 | number=6, type=13, cpp_type=3, label=1, 73 | has_default_value=False, default_value=0, 74 | message_type=None, enum_type=None, containing_type=None, 75 | is_extension=False, extension_scope=None, 76 | serialized_options=None, file=DESCRIPTOR), 77 | ], 78 | extensions=[ 79 | ], 80 | nested_types=[], 81 | enum_types=[ 82 | ], 83 | serialized_options=None, 84 | is_extendable=False, 85 | syntax='proto3', 86 | extension_ranges=[], 87 | oneofs=[ 88 | ], 89 | serialized_start=55, 90 | serialized_end=243, 91 | ) 92 | 93 | DESCRIPTOR.message_types_by_name['VoxelGenerator'] = _VOXELGENERATOR 94 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 95 | 96 | VoxelGenerator = _reflection.GeneratedProtocolMessageType('VoxelGenerator', (_message.Message,), dict( 97 | DESCRIPTOR = _VOXELGENERATOR, 98 | __module__ = 'second.protos.voxel_generator_pb2' 99 | # @@protoc_insertion_point(class_scope:second.protos.VoxelGenerator) 100 | )) 101 | _sym_db.RegisterMessage(VoxelGenerator) 102 | 103 | 104 | # @@protoc_insertion_point(module_scope) 105 | -------------------------------------------------------------------------------- /second/core/point_cloud/bev_ops.py: -------------------------------------------------------------------------------- 1 | import numba 2 | import numpy as np 3 | import math 4 | 5 | 6 | @numba.jit(nopython=True) 7 | def _points_to_bevmap_reverse_kernel(points, 8 | voxel_size, 9 | coors_range, 10 | coor_to_voxelidx, 11 | # coors_2d, 12 | bev_map, 13 | height_lowers, 14 | # density_norm_num=16, 15 | with_reflectivity=False, 16 | max_voxels=40000): 17 | # put all computations to one loop. 18 | # we shouldn't create large array in main jit code, otherwise 19 | # reduce performance 20 | N = points.shape[0] 21 | ndim = points.shape[1] - 1 22 | # ndim = 3 23 | ndim_minus_1 = ndim - 1 24 | grid_size = (coors_range[3:] - coors_range[:3]) / voxel_size 25 | # np.round(grid_size) 26 | # grid_size = np.round(grid_size).astype(np.int64)(np.int32) 27 | grid_size = np.round(grid_size, 0, grid_size).astype(np.int32) 28 | height_slice_size = voxel_size[-1] 29 | coor = np.zeros(shape=(3, ), dtype=np.int32) # DHW 30 | voxel_num = 0 31 | failed = False 32 | for i in range(N): 33 | failed = False 34 | for j in range(ndim): 35 | c = np.floor((points[i, j] - coors_range[j]) / voxel_size[j]) 36 | if c < 0 or c >= grid_size[j]: 37 | failed = True 38 | break 39 | coor[ndim_minus_1 - j] = c 40 | if failed: 41 | continue 42 | voxelidx = coor_to_voxelidx[coor[0], coor[1], coor[2]] 43 | if voxelidx == -1: 44 | voxelidx = voxel_num 45 | if voxel_num >= max_voxels: 46 | break 47 | voxel_num += 1 48 | coor_to_voxelidx[coor[0], coor[1], coor[2]] = voxelidx 49 | # coors_2d[voxelidx] = coor[1:] 50 | bev_map[-1, coor[1], coor[2]] += 1 51 | height_norm = bev_map[coor[0], coor[1], coor[2]] 52 | incomimg_height_norm = ( 53 | points[i, 2] - height_lowers[coor[0]]) / height_slice_size 54 | if incomimg_height_norm > height_norm: 55 | bev_map[coor[0], coor[1], coor[2]] = incomimg_height_norm 56 | if with_reflectivity: 57 | bev_map[-2, coor[1], coor[2]] = points[i, 3] 58 | # return voxel_num 59 | 60 | 61 | def points_to_bev(points, 62 | voxel_size, 63 | coors_range, 64 | with_reflectivity=False, 65 | density_norm_num=16, 66 | max_voxels=40000): 67 | """convert kitti points(N, 4) to a bev map. return [C, H, W] map. 68 | this function based on algorithm in points_to_voxel. 69 | takes 5ms in a reduced pointcloud with voxel_size=[0.1, 0.1, 0.8] 70 | 71 | Args: 72 | points: [N, ndim] float tensor. points[:, :3] contain xyz points and 73 | points[:, 3] contain reflectivity. 74 | voxel_size: [3] list/tuple or array, float. xyz, indicate voxel size 75 | coors_range: [6] list/tuple or array, float. indicate voxel range. 76 | format: xyzxyz, minmax 77 | with_reflectivity: bool. if True, will add a intensity map to bev map. 78 | Returns: 79 | bev_map: [num_height_maps + 1(2), H, W] float tensor. 80 | `WARNING`: bev_map[-1] is num_points map, NOT density map, 81 | because calculate density map need more time in cpu rather than gpu. 82 | if with_reflectivity is True, bev_map[-2] is intensity map. 83 | """ 84 | if not isinstance(voxel_size, np.ndarray): 85 | voxel_size = np.array(voxel_size, dtype=points.dtype) 86 | if not isinstance(coors_range, np.ndarray): 87 | coors_range = np.array(coors_range, dtype=points.dtype) 88 | voxelmap_shape = (coors_range[3:] - coors_range[:3]) / voxel_size 89 | voxelmap_shape = tuple(np.round(voxelmap_shape).astype(np.int32).tolist()) 90 | voxelmap_shape = voxelmap_shape[::-1] # DHW format 91 | coor_to_voxelidx = -np.ones(shape=voxelmap_shape, dtype=np.int32) 92 | # coors_2d = np.zeros(shape=(max_voxels, 2), dtype=np.int32) 93 | bev_map_shape = list(voxelmap_shape) 94 | bev_map_shape[0] += 1 95 | height_lowers = np.linspace( 96 | coors_range[2], coors_range[5], voxelmap_shape[0], endpoint=False) 97 | if with_reflectivity: 98 | bev_map_shape[0] += 1 99 | bev_map = np.zeros(shape=bev_map_shape, dtype=points.dtype) 100 | _points_to_bevmap_reverse_kernel( 101 | points, voxel_size, coors_range, coor_to_voxelidx, bev_map, 102 | height_lowers, with_reflectivity, max_voxels) 103 | return bev_map 104 | -------------------------------------------------------------------------------- /second/protos/pipeline_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/pipeline.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from second.protos import input_reader_pb2 as second_dot_protos_dot_input__reader__pb2 16 | from second.protos import model_pb2 as second_dot_protos_dot_model__pb2 17 | from second.protos import train_pb2 as second_dot_protos_dot_train__pb2 18 | 19 | 20 | DESCRIPTOR = _descriptor.FileDescriptor( 21 | name='second/protos/pipeline.proto', 22 | package='second.protos', 23 | syntax='proto3', 24 | serialized_options=None, 25 | serialized_pb=_b('\n\x1csecond/protos/pipeline.proto\x12\rsecond.protos\x1a second/protos/input_reader.proto\x1a\x19second/protos/model.proto\x1a\x19second/protos/train.proto\"\xe8\x01\n\x17TrainEvalPipelineConfig\x12,\n\x05model\x18\x01 \x01(\x0b\x32\x1d.second.protos.DetectionModel\x12\x36\n\x12train_input_reader\x18\x02 \x01(\x0b\x32\x1a.second.protos.InputReader\x12\x30\n\x0ctrain_config\x18\x03 \x01(\x0b\x32\x1a.second.protos.TrainConfig\x12\x35\n\x11\x65val_input_reader\x18\x04 \x01(\x0b\x32\x1a.second.protos.InputReaderb\x06proto3') 26 | , 27 | dependencies=[second_dot_protos_dot_input__reader__pb2.DESCRIPTOR,second_dot_protos_dot_model__pb2.DESCRIPTOR,second_dot_protos_dot_train__pb2.DESCRIPTOR,]) 28 | 29 | 30 | 31 | 32 | _TRAINEVALPIPELINECONFIG = _descriptor.Descriptor( 33 | name='TrainEvalPipelineConfig', 34 | full_name='second.protos.TrainEvalPipelineConfig', 35 | filename=None, 36 | file=DESCRIPTOR, 37 | containing_type=None, 38 | fields=[ 39 | _descriptor.FieldDescriptor( 40 | name='model', full_name='second.protos.TrainEvalPipelineConfig.model', index=0, 41 | number=1, type=11, cpp_type=10, label=1, 42 | has_default_value=False, default_value=None, 43 | message_type=None, enum_type=None, containing_type=None, 44 | is_extension=False, extension_scope=None, 45 | serialized_options=None, file=DESCRIPTOR), 46 | _descriptor.FieldDescriptor( 47 | name='train_input_reader', full_name='second.protos.TrainEvalPipelineConfig.train_input_reader', index=1, 48 | number=2, type=11, cpp_type=10, label=1, 49 | has_default_value=False, default_value=None, 50 | message_type=None, enum_type=None, containing_type=None, 51 | is_extension=False, extension_scope=None, 52 | serialized_options=None, file=DESCRIPTOR), 53 | _descriptor.FieldDescriptor( 54 | name='train_config', full_name='second.protos.TrainEvalPipelineConfig.train_config', index=2, 55 | number=3, type=11, cpp_type=10, label=1, 56 | has_default_value=False, default_value=None, 57 | message_type=None, enum_type=None, containing_type=None, 58 | is_extension=False, extension_scope=None, 59 | serialized_options=None, file=DESCRIPTOR), 60 | _descriptor.FieldDescriptor( 61 | name='eval_input_reader', full_name='second.protos.TrainEvalPipelineConfig.eval_input_reader', index=3, 62 | number=4, type=11, cpp_type=10, label=1, 63 | has_default_value=False, default_value=None, 64 | message_type=None, enum_type=None, containing_type=None, 65 | is_extension=False, extension_scope=None, 66 | serialized_options=None, file=DESCRIPTOR), 67 | ], 68 | extensions=[ 69 | ], 70 | nested_types=[], 71 | enum_types=[ 72 | ], 73 | serialized_options=None, 74 | is_extendable=False, 75 | syntax='proto3', 76 | extension_ranges=[], 77 | oneofs=[ 78 | ], 79 | serialized_start=136, 80 | serialized_end=368, 81 | ) 82 | 83 | _TRAINEVALPIPELINECONFIG.fields_by_name['model'].message_type = second_dot_protos_dot_model__pb2._DETECTIONMODEL 84 | _TRAINEVALPIPELINECONFIG.fields_by_name['train_input_reader'].message_type = second_dot_protos_dot_input__reader__pb2._INPUTREADER 85 | _TRAINEVALPIPELINECONFIG.fields_by_name['train_config'].message_type = second_dot_protos_dot_train__pb2._TRAINCONFIG 86 | _TRAINEVALPIPELINECONFIG.fields_by_name['eval_input_reader'].message_type = second_dot_protos_dot_input__reader__pb2._INPUTREADER 87 | DESCRIPTOR.message_types_by_name['TrainEvalPipelineConfig'] = _TRAINEVALPIPELINECONFIG 88 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 89 | 90 | TrainEvalPipelineConfig = _reflection.GeneratedProtocolMessageType('TrainEvalPipelineConfig', (_message.Message,), dict( 91 | DESCRIPTOR = _TRAINEVALPIPELINECONFIG, 92 | __module__ = 'second.protos.pipeline_pb2' 93 | # @@protoc_insertion_point(class_scope:second.protos.TrainEvalPipelineConfig) 94 | )) 95 | _sym_db.RegisterMessage(TrainEvalPipelineConfig) 96 | 97 | 98 | # @@protoc_insertion_point(module_scope) 99 | -------------------------------------------------------------------------------- /second/builder/dataset_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Input reader builder. 16 | 17 | Creates data sources for DetectionModels from an InputReader config. See 18 | input_reader.proto for options. 19 | 20 | Note: If users wishes to also use their own InputReaders with the Object 21 | Detection configuration framework, they should define their own builder function 22 | that wraps the build function. 23 | """ 24 | 25 | from second.protos import input_reader_pb2 26 | from second.data.dataset import KittiDataset 27 | from second.data.preprocess import prep_pointcloud 28 | import numpy as np 29 | from second.builder import dbsampler_builder 30 | from functools import partial 31 | 32 | 33 | def build(input_reader_config, 34 | model_config, 35 | training, 36 | voxel_generator, 37 | target_assigner=None): 38 | """Builds a tensor dictionary based on the InputReader config. 39 | 40 | Args: 41 | input_reader_config: A input_reader_pb2.InputReader object. 42 | 43 | Returns: 44 | A tensor dict based on the input_reader_config. 45 | 46 | Raises: 47 | ValueError: On invalid input reader proto. 48 | ValueError: If no input paths are specified. 49 | """ 50 | if not isinstance(input_reader_config, input_reader_pb2.InputReader): 51 | raise ValueError('input_reader_config not of type ' 52 | 'input_reader_pb2.InputReader.') 53 | generate_bev = model_config.use_bev 54 | without_reflectivity = model_config.without_reflectivity 55 | num_point_features = model_config.num_point_features 56 | out_size_factor = model_config.rpn.layer_strides[0] // model_config.rpn.upsample_strides[0] 57 | 58 | cfg = input_reader_config 59 | db_sampler_cfg = input_reader_config.database_sampler 60 | db_sampler = None 61 | if len(db_sampler_cfg.sample_groups) > 0: # enable sample 62 | db_sampler = dbsampler_builder.build(db_sampler_cfg) 63 | u_db_sampler_cfg = input_reader_config.unlabeled_database_sampler 64 | u_db_sampler = None 65 | if len(u_db_sampler_cfg.sample_groups) > 0: # enable sample 66 | u_db_sampler = dbsampler_builder.build(u_db_sampler_cfg) 67 | grid_size = voxel_generator.grid_size 68 | # [352, 400] 69 | feature_map_size = grid_size[:2] // out_size_factor 70 | feature_map_size = [*feature_map_size, 1][::-1] 71 | 72 | prep_func = partial( 73 | prep_pointcloud, 74 | root_path=cfg.kitti_root_path, 75 | class_names=list(cfg.class_names), 76 | voxel_generator=voxel_generator, 77 | target_assigner=target_assigner, 78 | training=training, 79 | max_voxels=cfg.max_number_of_voxels, 80 | remove_outside_points=False, 81 | remove_unknown=cfg.remove_unknown_examples, 82 | create_targets=training, 83 | shuffle_points=cfg.shuffle_points, 84 | gt_rotation_noise=list(cfg.groundtruth_rotation_uniform_noise), 85 | gt_loc_noise_std=list(cfg.groundtruth_localization_noise_std), 86 | global_rotation_noise=list(cfg.global_rotation_uniform_noise), 87 | global_scaling_noise=list(cfg.global_scaling_uniform_noise), 88 | global_loc_noise_std=(0.2, 0.2, 0.2), 89 | global_random_rot_range=list( 90 | cfg.global_random_rotation_range_per_object), 91 | db_sampler=db_sampler, 92 | unlabeled_db_sampler=u_db_sampler, 93 | generate_bev=generate_bev, 94 | without_reflectivity=without_reflectivity, 95 | num_point_features=num_point_features, 96 | anchor_area_threshold=cfg.anchor_area_threshold, 97 | gt_points_drop=cfg.groundtruth_points_drop_percentage, 98 | gt_drop_max_keep=cfg.groundtruth_drop_max_keep_points, 99 | remove_points_after_sample=cfg.remove_points_after_sample, 100 | remove_environment=cfg.remove_environment, 101 | use_group_id=cfg.use_group_id, 102 | out_size_factor=out_size_factor) 103 | dataset = KittiDataset( 104 | info_path=cfg.kitti_info_path, 105 | root_path=cfg.kitti_root_path, 106 | num_point_features=num_point_features, 107 | target_assigner=target_assigner, 108 | feature_map_size=feature_map_size, 109 | prep_func=prep_func) 110 | 111 | return dataset 112 | -------------------------------------------------------------------------------- /second/protos/losses.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Message for configuring the localization loss, classification loss and hard 6 | // example miner used for training object detection models. See core/losses.py 7 | // for details 8 | message Loss { 9 | // Localization loss to use. 10 | LocalizationLoss localization_loss = 1; 11 | 12 | // Classification loss to use. 13 | ClassificationLoss classification_loss = 2; 14 | 15 | // If not left to default, applies hard example mining. 16 | HardExampleMiner hard_example_miner = 3; 17 | 18 | // Classification loss weight. 19 | float classification_weight = 4; 20 | 21 | // Localization loss weight. 22 | float localization_weight = 5; 23 | 24 | } 25 | 26 | // Configuration for bounding box localization loss function. 27 | message LocalizationLoss { 28 | oneof localization_loss { 29 | WeightedL2LocalizationLoss weighted_l2 = 1; 30 | WeightedSmoothL1LocalizationLoss weighted_smooth_l1 = 2; 31 | } 32 | bool encode_rad_error_by_sin = 3; 33 | } 34 | 35 | // L2 location loss: 0.5 * ||weight * (a - b)|| ^ 2 36 | message WeightedL2LocalizationLoss { 37 | // DEPRECATED, do not use. 38 | // Output loss per anchor. 39 | bool anchorwise_output = 1; 40 | repeated float code_weight = 2; 41 | } 42 | 43 | // SmoothL1 (Huber) location loss: .5 * x ^ 2 if |x| < 1 else |x| - .5 44 | message WeightedSmoothL1LocalizationLoss { 45 | // DEPRECATED, do not use. 46 | // Output loss per anchor. 47 | bool anchorwise_output = 1; 48 | float sigma = 2; 49 | repeated float code_weight = 3; 50 | } 51 | 52 | 53 | // Configuration for class prediction loss function. 54 | message ClassificationLoss { 55 | oneof classification_loss { 56 | WeightedSigmoidClassificationLoss weighted_sigmoid = 1; 57 | WeightedSoftmaxClassificationLoss weighted_softmax = 2; 58 | BootstrappedSigmoidClassificationLoss bootstrapped_sigmoid = 3; 59 | SigmoidFocalClassificationLoss weighted_sigmoid_focal = 4; 60 | SoftmaxFocalClassificationLoss weighted_softmax_focal = 5; 61 | } 62 | } 63 | 64 | // Classification loss using a sigmoid function over class predictions. 65 | message WeightedSigmoidClassificationLoss { 66 | // DEPRECATED, do not use. 67 | // Output loss per anchor. 68 | bool anchorwise_output = 1; 69 | } 70 | 71 | // Sigmoid Focal cross entropy loss as described in 72 | // https://arxiv.org/abs/1708.02002 73 | message SigmoidFocalClassificationLoss { 74 | // DEPRECATED, do not use. 75 | bool anchorwise_output = 1; 76 | // modulating factor for the loss. 77 | float gamma = 2; 78 | // alpha weighting factor for the loss. 79 | float alpha = 3; 80 | } 81 | // Sigmoid Focal cross entropy loss as described in 82 | // https://arxiv.org/abs/1708.02002 83 | message SoftmaxFocalClassificationLoss { 84 | // DEPRECATED, do not use. 85 | bool anchorwise_output = 1; 86 | // modulating factor for the loss. 87 | float gamma = 2; 88 | // alpha weighting factor for the loss. 89 | float alpha = 3; 90 | } 91 | // Classification loss using a softmax function over class predictions. 92 | message WeightedSoftmaxClassificationLoss { 93 | // DEPRECATED, do not use. 94 | // Output loss per anchor. 95 | bool anchorwise_output = 1; 96 | // Scale logit (input) value before calculating softmax classification loss. 97 | // Typically used for softmax distillation. 98 | float logit_scale = 2; 99 | } 100 | 101 | // Classification loss using a sigmoid function over the class prediction with 102 | // the highest prediction score. 103 | message BootstrappedSigmoidClassificationLoss { 104 | // Interpolation weight between 0 and 1. 105 | float alpha = 1; 106 | 107 | // Whether hard boot strapping should be used or not. If true, will only use 108 | // one class favored by model. Othewise, will use all predicted class 109 | // probabilities. 110 | bool hard_bootstrap = 2; 111 | 112 | // DEPRECATED, do not use. 113 | // Output loss per anchor. 114 | bool anchorwise_output = 3; 115 | } 116 | 117 | // Configuation for hard example miner. 118 | message HardExampleMiner { 119 | // Maximum number of hard examples to be selected per image (prior to 120 | // enforcing max negative to positive ratio constraint). If set to 0, 121 | // all examples obtained after NMS are considered. 122 | int32 num_hard_examples = 1; 123 | 124 | // Minimum intersection over union for an example to be discarded during NMS. 125 | float iou_threshold = 2; 126 | 127 | // Whether to use classification losses ('cls', default), localization losses 128 | // ('loc') or both losses ('both'). In the case of 'both', cls_loss_weight and 129 | // loc_loss_weight are used to compute weighted sum of the two losses. 130 | enum LossType { 131 | BOTH = 0; 132 | CLASSIFICATION = 1; 133 | LOCALIZATION = 2; 134 | } 135 | LossType loss_type = 3; 136 | 137 | // Maximum number of negatives to retain for each positive anchor. If 138 | // num_negatives_per_positive is 0 no prespecified negative:positive ratio is 139 | // enforced. 140 | int32 max_negatives_per_positive = 4; 141 | 142 | // Minimum number of negative anchors to sample for a given image. Setting 143 | // this to a positive number samples negatives in an image without any 144 | // positive anchors and thus not bias the model towards having at least one 145 | // detection per image. 146 | int32 min_negatives_per_image = 5; 147 | } 148 | -------------------------------------------------------------------------------- /second/protos/target_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/target.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from second.protos import anchors_pb2 as second_dot_protos_dot_anchors__pb2 16 | from second.protos import similarity_pb2 as second_dot_protos_dot_similarity__pb2 17 | 18 | 19 | DESCRIPTOR = _descriptor.FileDescriptor( 20 | name='second/protos/target.proto', 21 | package='second.protos', 22 | syntax='proto3', 23 | serialized_options=None, 24 | serialized_pb=_b('\n\x1asecond/protos/target.proto\x12\rsecond.protos\x1a\x1bsecond/protos/anchors.proto\x1a\x1esecond/protos/similarity.proto\"\x89\x02\n\x0eTargetAssigner\x12\x43\n\x11\x61nchor_generators\x18\x01 \x03(\x0b\x32(.second.protos.AnchorGeneratorCollection\x12 \n\x18sample_positive_fraction\x18\x02 \x01(\x02\x12\x13\n\x0bsample_size\x18\x03 \x01(\r\x12\x16\n\x0euse_rotate_iou\x18\x04 \x01(\x08\x12\x12\n\nclass_name\x18\x05 \x01(\t\x12O\n\x1cregion_similarity_calculator\x18\x06 \x01(\x0b\x32).second.protos.RegionSimilarityCalculatorb\x06proto3') 25 | , 26 | dependencies=[second_dot_protos_dot_anchors__pb2.DESCRIPTOR,second_dot_protos_dot_similarity__pb2.DESCRIPTOR,]) 27 | 28 | 29 | 30 | 31 | _TARGETASSIGNER = _descriptor.Descriptor( 32 | name='TargetAssigner', 33 | full_name='second.protos.TargetAssigner', 34 | filename=None, 35 | file=DESCRIPTOR, 36 | containing_type=None, 37 | fields=[ 38 | _descriptor.FieldDescriptor( 39 | name='anchor_generators', full_name='second.protos.TargetAssigner.anchor_generators', index=0, 40 | number=1, type=11, cpp_type=10, label=3, 41 | has_default_value=False, default_value=[], 42 | message_type=None, enum_type=None, containing_type=None, 43 | is_extension=False, extension_scope=None, 44 | serialized_options=None, file=DESCRIPTOR), 45 | _descriptor.FieldDescriptor( 46 | name='sample_positive_fraction', full_name='second.protos.TargetAssigner.sample_positive_fraction', index=1, 47 | number=2, type=2, cpp_type=6, label=1, 48 | has_default_value=False, default_value=float(0), 49 | message_type=None, enum_type=None, containing_type=None, 50 | is_extension=False, extension_scope=None, 51 | serialized_options=None, file=DESCRIPTOR), 52 | _descriptor.FieldDescriptor( 53 | name='sample_size', full_name='second.protos.TargetAssigner.sample_size', index=2, 54 | number=3, type=13, cpp_type=3, label=1, 55 | has_default_value=False, default_value=0, 56 | message_type=None, enum_type=None, containing_type=None, 57 | is_extension=False, extension_scope=None, 58 | serialized_options=None, file=DESCRIPTOR), 59 | _descriptor.FieldDescriptor( 60 | name='use_rotate_iou', full_name='second.protos.TargetAssigner.use_rotate_iou', index=3, 61 | number=4, type=8, cpp_type=7, label=1, 62 | has_default_value=False, default_value=False, 63 | message_type=None, enum_type=None, containing_type=None, 64 | is_extension=False, extension_scope=None, 65 | serialized_options=None, file=DESCRIPTOR), 66 | _descriptor.FieldDescriptor( 67 | name='class_name', full_name='second.protos.TargetAssigner.class_name', index=4, 68 | number=5, type=9, cpp_type=9, label=1, 69 | has_default_value=False, default_value=_b("").decode('utf-8'), 70 | message_type=None, enum_type=None, containing_type=None, 71 | is_extension=False, extension_scope=None, 72 | serialized_options=None, file=DESCRIPTOR), 73 | _descriptor.FieldDescriptor( 74 | name='region_similarity_calculator', full_name='second.protos.TargetAssigner.region_similarity_calculator', index=5, 75 | number=6, type=11, cpp_type=10, label=1, 76 | has_default_value=False, default_value=None, 77 | message_type=None, enum_type=None, containing_type=None, 78 | is_extension=False, extension_scope=None, 79 | serialized_options=None, file=DESCRIPTOR), 80 | ], 81 | extensions=[ 82 | ], 83 | nested_types=[], 84 | enum_types=[ 85 | ], 86 | serialized_options=None, 87 | is_extendable=False, 88 | syntax='proto3', 89 | extension_ranges=[], 90 | oneofs=[ 91 | ], 92 | serialized_start=107, 93 | serialized_end=372, 94 | ) 95 | 96 | _TARGETASSIGNER.fields_by_name['anchor_generators'].message_type = second_dot_protos_dot_anchors__pb2._ANCHORGENERATORCOLLECTION 97 | _TARGETASSIGNER.fields_by_name['region_similarity_calculator'].message_type = second_dot_protos_dot_similarity__pb2._REGIONSIMILARITYCALCULATOR 98 | DESCRIPTOR.message_types_by_name['TargetAssigner'] = _TARGETASSIGNER 99 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 100 | 101 | TargetAssigner = _reflection.GeneratedProtocolMessageType('TargetAssigner', (_message.Message,), dict( 102 | DESCRIPTOR = _TARGETASSIGNER, 103 | __module__ = 'second.protos.target_pb2' 104 | # @@protoc_insertion_point(class_scope:second.protos.TargetAssigner) 105 | )) 106 | _sym_db.RegisterMessage(TargetAssigner) 107 | 108 | 109 | # @@protoc_insertion_point(module_scope) 110 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_20.proto: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | voxel_generator { 4 | point_cloud_range : [0, -40, -3, 70.4, 40, 1] 5 | voxel_size : [0.2, 0.2, 4] 6 | max_number_of_points_per_voxel : 100 7 | } 8 | num_class: 1 9 | voxel_feature_extractor: { 10 | module_class_name: "PillarFeatureNet" 11 | num_filters: [64] 12 | with_distance: false 13 | } 14 | middle_feature_extractor: { 15 | module_class_name: "PointPillarsScatter" 16 | } 17 | rpn: { 18 | module_class_name: "RPN" 19 | layer_nums: [3, 5, 5] 20 | layer_strides: [2, 2, 2] 21 | num_filters: [64, 128, 256] 22 | upsample_strides: [1, 2, 4] 23 | num_upsample_filters: [128, 128, 128] 24 | use_groupnorm: false 25 | num_groups: 32 26 | } 27 | loss: { 28 | classification_loss: { 29 | weighted_sigmoid_focal: { 30 | alpha: 0.25 31 | gamma: 2.0 32 | anchorwise_output: true 33 | } 34 | } 35 | localization_loss: { 36 | weighted_smooth_l1: { 37 | sigma: 3.0 38 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 39 | } 40 | } 41 | classification_weight: 1.0 42 | localization_weight: 2.0 43 | } 44 | # Outputs 45 | use_sigmoid_score: true 46 | encode_background_as_zeros: true 47 | encode_rad_error_by_sin: true 48 | 49 | use_direction_classifier: true 50 | direction_loss_weight: 0.2 51 | use_aux_classifier: false 52 | # Loss 53 | pos_class_weight: 1.0 54 | neg_class_weight: 1.0 55 | 56 | loss_norm_type: NormByNumPositives 57 | # Postprocess 58 | post_center_limit_range: [0, -40, -5, 70.4, 40, 5] 59 | use_rotate_nms: false 60 | use_multi_class_nms: false 61 | nms_pre_max_size: 1000 62 | nms_post_max_size: 300 63 | nms_score_threshold: 0.05 64 | nms_iou_threshold: 0.5 65 | 66 | use_bev: false 67 | num_point_features: 4 68 | without_reflectivity: false 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | anchor_generators: { 77 | anchor_generator_stride: { 78 | sizes: [1.6, 3.9, 1.56] # wlh 79 | strides: [0.4, 0.4, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.2, -39.8, -1.78] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | matched_threshold : 0.6 83 | unmatched_threshold : 0.45 84 | } 85 | } 86 | 87 | sample_positive_fraction : -1 88 | sample_size : 512 89 | region_similarity_calculator: { 90 | nearest_iou_similarity: { 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | 98 | train_input_reader: { 99 | record_file_path: "/data/sets/kitti_second/kitti_train.tfrecord" 100 | class_names: ["Car"] 101 | max_num_epochs : 160 102 | batch_size: 2 103 | prefetch_size : 25 104 | max_number_of_voxels: 12000 105 | shuffle_points: true 106 | num_workers: 2 107 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 108 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 109 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 110 | global_scaling_uniform_noise: [0.95, 1.05] 111 | global_random_rotation_range_per_object: [0, 0] 112 | anchor_area_threshold: 1 113 | remove_points_after_sample: false 114 | groundtruth_points_drop_percentage: 0.0 115 | groundtruth_drop_max_keep_points: 15 116 | database_sampler { 117 | database_info_path: "/data/sets/kitti_second/kitti_dbinfos_train.pkl" 118 | sample_groups { 119 | name_to_max_num { 120 | key: "Car" 121 | value: 15 122 | } 123 | } 124 | database_prep_steps { 125 | filter_by_min_num_points { 126 | min_num_point_pairs { 127 | key: "Car" 128 | value: 5 129 | } 130 | } 131 | } 132 | database_prep_steps { 133 | filter_by_difficulty { 134 | removed_difficulties: [-1] 135 | } 136 | } 137 | global_random_rotation_range_per_object: [0, 0] 138 | rate: 1.0 139 | } 140 | 141 | remove_unknown_examples: false 142 | remove_environment: false 143 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_train.pkl" 144 | kitti_root_path: "/data/sets/kitti_second" 145 | } 146 | 147 | train_config: { 148 | optimizer: { 149 | adam_optimizer: { 150 | learning_rate: { 151 | exponential_decay_learning_rate: { 152 | initial_learning_rate: 0.0002 153 | decay_steps: 27840 # 1856 steps per epoch * 15 epochs 154 | decay_factor: 0.8 155 | staircase: true 156 | } 157 | } 158 | weight_decay: 0.0001 159 | } 160 | use_moving_average: false 161 | 162 | } 163 | inter_op_parallelism_threads: 4 164 | intra_op_parallelism_threads: 4 165 | steps: 296960 # 1856 steps per epoch * 160 epochs 166 | steps_per_eval: 9280 # 1856 steps per epoch * 5 epochs 167 | save_checkpoints_secs : 1800 # half hour 168 | save_summary_steps : 10 169 | enable_mixed_precision: false 170 | loss_scale_factor : 512.0 171 | clear_metrics_every_epoch: false 172 | } 173 | 174 | eval_input_reader: { 175 | record_file_path: "/data/sets/kitti_second/kitti_val.tfrecord" 176 | class_names: ["Car"] 177 | batch_size: 2 178 | max_num_epochs : 160 179 | prefetch_size : 25 180 | max_number_of_voxels: 12000 181 | shuffle_points: false 182 | num_workers: 3 183 | anchor_area_threshold: 1 184 | remove_environment: false 185 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_val.pkl" 186 | kitti_root_path: "/data/sets/kitti_second" 187 | } 188 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_16.proto: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | voxel_generator { 4 | point_cloud_range : [0, -39.68, -3, 69.12, 39.68, 1] 5 | voxel_size : [0.16, 0.16, 4] 6 | max_number_of_points_per_voxel : 100 7 | } 8 | num_class: 1 9 | voxel_feature_extractor: { 10 | module_class_name: "PillarFeatureNet" 11 | num_filters: [64] 12 | with_distance: false 13 | } 14 | middle_feature_extractor: { 15 | module_class_name: "PointPillarsScatter" 16 | } 17 | rpn: { 18 | module_class_name: "RPN" 19 | layer_nums: [3, 5, 5] 20 | layer_strides: [2, 2, 2] 21 | num_filters: [64, 128, 256] 22 | upsample_strides: [1, 2, 4] 23 | num_upsample_filters: [128, 128, 128] 24 | use_groupnorm: false 25 | num_groups: 32 26 | } 27 | loss: { 28 | classification_loss: { 29 | weighted_sigmoid_focal: { 30 | alpha: 0.25 31 | gamma: 2.0 32 | anchorwise_output: true 33 | } 34 | } 35 | localization_loss: { 36 | weighted_smooth_l1: { 37 | sigma: 3.0 38 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 39 | } 40 | } 41 | classification_weight: 1.0 42 | localization_weight: 2.0 43 | } 44 | # Outputs 45 | use_sigmoid_score: true 46 | encode_background_as_zeros: true 47 | encode_rad_error_by_sin: true 48 | 49 | use_direction_classifier: true 50 | direction_loss_weight: 0.2 51 | use_aux_classifier: false 52 | # Loss 53 | pos_class_weight: 1.0 54 | neg_class_weight: 1.0 55 | 56 | loss_norm_type: NormByNumPositives 57 | # Postprocess 58 | post_center_limit_range: [0, -39.68, -5, 69.12, 39.68, 5] 59 | use_rotate_nms: false 60 | use_multi_class_nms: false 61 | nms_pre_max_size: 1000 62 | nms_post_max_size: 300 63 | nms_score_threshold: 0.05 64 | nms_iou_threshold: 0.5 65 | 66 | use_bev: false 67 | num_point_features: 4 68 | without_reflectivity: false 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | anchor_generators: { 77 | anchor_generator_stride: { 78 | sizes: [1.6, 3.9, 1.56] # wlh 79 | strides: [0.32, 0.32, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.16, -39.52, -1.78] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | matched_threshold : 0.6 83 | unmatched_threshold : 0.45 84 | } 85 | } 86 | 87 | sample_positive_fraction : -1 88 | sample_size : 512 89 | region_similarity_calculator: { 90 | nearest_iou_similarity: { 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | 98 | train_input_reader: { 99 | record_file_path: "/data/sets/kitti_second/kitti_train.tfrecord" 100 | class_names: ["Car"] 101 | max_num_epochs : 160 102 | batch_size: 2 103 | prefetch_size : 25 104 | max_number_of_voxels: 12000 105 | shuffle_points: true 106 | num_workers: 2 107 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 108 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 109 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 110 | global_scaling_uniform_noise: [0.95, 1.05] 111 | global_random_rotation_range_per_object: [0, 0] 112 | anchor_area_threshold: 1 113 | remove_points_after_sample: false 114 | groundtruth_points_drop_percentage: 0.0 115 | groundtruth_drop_max_keep_points: 15 116 | database_sampler { 117 | database_info_path: "/data/sets/kitti_second/kitti_dbinfos_train.pkl" 118 | sample_groups { 119 | name_to_max_num { 120 | key: "Car" 121 | value: 15 122 | } 123 | } 124 | database_prep_steps { 125 | filter_by_min_num_points { 126 | min_num_point_pairs { 127 | key: "Car" 128 | value: 5 129 | } 130 | } 131 | } 132 | database_prep_steps { 133 | filter_by_difficulty { 134 | removed_difficulties: [-1] 135 | } 136 | } 137 | global_random_rotation_range_per_object: [0, 0] 138 | rate: 1.0 139 | } 140 | 141 | remove_unknown_examples: false 142 | remove_environment: false 143 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_train.pkl" 144 | kitti_root_path: "/data/sets/kitti_second" 145 | } 146 | 147 | train_config: { 148 | optimizer: { 149 | adam_optimizer: { 150 | learning_rate: { 151 | exponential_decay_learning_rate: { 152 | initial_learning_rate: 0.0002 153 | decay_steps: 27840 # 1856 steps per epoch * 15 epochs 154 | decay_factor: 0.8 155 | staircase: true 156 | } 157 | } 158 | weight_decay: 0.0001 159 | } 160 | use_moving_average: false 161 | 162 | } 163 | inter_op_parallelism_threads: 4 164 | intra_op_parallelism_threads: 4 165 | steps: 296960 # 1856 steps per epoch * 160 epochs 166 | steps_per_eval: 9280 # 1856 steps per epoch * 5 epochs 167 | save_checkpoints_secs : 1800 # half hour 168 | save_summary_steps : 10 169 | enable_mixed_precision: false 170 | loss_scale_factor : 512.0 171 | clear_metrics_every_epoch: false 172 | } 173 | 174 | eval_input_reader: { 175 | record_file_path: "/data/sets/kitti_second/kitti_val.tfrecord" 176 | class_names: ["Car"] 177 | batch_size: 2 178 | max_num_epochs : 160 179 | prefetch_size : 25 180 | max_number_of_voxels: 12000 181 | shuffle_points: false 182 | num_workers: 3 183 | anchor_area_threshold: 1 184 | remove_environment: false 185 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_val.pkl" 186 | kitti_root_path: "/data/sets/kitti_second" 187 | } 188 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_24.proto: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | voxel_generator { 4 | point_cloud_range : [0, -40.32, -3, 71.04, 40.32, 1] 5 | voxel_size : [0.24, 0.24, 4] 6 | max_number_of_points_per_voxel : 100 7 | } 8 | num_class: 1 9 | voxel_feature_extractor: { 10 | module_class_name: "PillarFeatureNet" 11 | num_filters: [64] 12 | with_distance: false 13 | } 14 | middle_feature_extractor: { 15 | module_class_name: "PointPillarsScatter" 16 | } 17 | rpn: { 18 | module_class_name: "RPN" 19 | layer_nums: [3, 5, 5] 20 | layer_strides: [2, 2, 2] 21 | num_filters: [64, 128, 256] 22 | upsample_strides: [1, 2, 4] 23 | num_upsample_filters: [128, 128, 128] 24 | use_groupnorm: false 25 | num_groups: 32 26 | } 27 | loss: { 28 | classification_loss: { 29 | weighted_sigmoid_focal: { 30 | alpha: 0.25 31 | gamma: 2.0 32 | anchorwise_output: true 33 | } 34 | } 35 | localization_loss: { 36 | weighted_smooth_l1: { 37 | sigma: 3.0 38 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 39 | } 40 | } 41 | classification_weight: 1.0 42 | localization_weight: 2.0 43 | } 44 | # Outputs 45 | use_sigmoid_score: true 46 | encode_background_as_zeros: true 47 | encode_rad_error_by_sin: true 48 | 49 | use_direction_classifier: true 50 | direction_loss_weight: 0.2 51 | use_aux_classifier: false 52 | # Loss 53 | pos_class_weight: 1.0 54 | neg_class_weight: 1.0 55 | 56 | loss_norm_type: NormByNumPositives 57 | # Postprocess 58 | post_center_limit_range: [0, -40.32, -5, 71.04, 40.32, 5] 59 | use_rotate_nms: false 60 | use_multi_class_nms: false 61 | nms_pre_max_size: 1000 62 | nms_post_max_size: 300 63 | nms_score_threshold: 0.05 64 | nms_iou_threshold: 0.5 65 | 66 | use_bev: false 67 | num_point_features: 4 68 | without_reflectivity: false 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | anchor_generators: { 77 | anchor_generator_stride: { 78 | sizes: [1.6, 3.9, 1.56] # wlh 79 | strides: [0.48, 0.48, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.24, -40.08, -1.78] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | matched_threshold : 0.6 83 | unmatched_threshold : 0.45 84 | } 85 | } 86 | 87 | sample_positive_fraction : -1 88 | sample_size : 512 89 | region_similarity_calculator: { 90 | nearest_iou_similarity: { 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | 98 | train_input_reader: { 99 | record_file_path: "/data/sets/kitti_second/kitti_train.tfrecord" 100 | class_names: ["Car"] 101 | max_num_epochs : 160 102 | batch_size: 2 103 | prefetch_size : 25 104 | max_number_of_voxels: 12000 105 | shuffle_points: true 106 | num_workers: 2 107 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 108 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 109 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 110 | global_scaling_uniform_noise: [0.95, 1.05] 111 | global_random_rotation_range_per_object: [0, 0] 112 | anchor_area_threshold: 1 113 | remove_points_after_sample: false 114 | groundtruth_points_drop_percentage: 0.0 115 | groundtruth_drop_max_keep_points: 15 116 | database_sampler { 117 | database_info_path: "/data/sets/kitti_second/kitti_dbinfos_train.pkl" 118 | sample_groups { 119 | name_to_max_num { 120 | key: "Car" 121 | value: 15 122 | } 123 | } 124 | database_prep_steps { 125 | filter_by_min_num_points { 126 | min_num_point_pairs { 127 | key: "Car" 128 | value: 5 129 | } 130 | } 131 | } 132 | database_prep_steps { 133 | filter_by_difficulty { 134 | removed_difficulties: [-1] 135 | } 136 | } 137 | global_random_rotation_range_per_object: [0, 0] 138 | rate: 1.0 139 | } 140 | 141 | remove_unknown_examples: false 142 | remove_environment: false 143 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_train.pkl" 144 | kitti_root_path: "/data/sets/kitti_second" 145 | } 146 | 147 | train_config: { 148 | optimizer: { 149 | adam_optimizer: { 150 | learning_rate: { 151 | exponential_decay_learning_rate: { 152 | initial_learning_rate: 0.0002 153 | decay_steps: 27840 # 1856 steps per epoch * 15 epochs 154 | decay_factor: 0.8 155 | staircase: true 156 | } 157 | } 158 | weight_decay: 0.0001 159 | } 160 | use_moving_average: false 161 | 162 | } 163 | inter_op_parallelism_threads: 4 164 | intra_op_parallelism_threads: 4 165 | steps: 296960 # 1856 steps per epoch * 160 epochs 166 | steps_per_eval: 9280 # 1856 steps per epoch * 5 epochs 167 | save_checkpoints_secs : 1800 # half hour 168 | save_summary_steps : 10 169 | enable_mixed_precision: false 170 | loss_scale_factor : 512.0 171 | clear_metrics_every_epoch: false 172 | } 173 | 174 | eval_input_reader: { 175 | record_file_path: "/data/sets/kitti_second/kitti_val.tfrecord" 176 | class_names: ["Car"] 177 | batch_size: 2 178 | max_num_epochs : 160 179 | prefetch_size : 25 180 | max_number_of_voxels: 12000 181 | shuffle_points: false 182 | num_workers: 3 183 | anchor_area_threshold: 1 184 | remove_environment: false 185 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_val.pkl" 186 | kitti_root_path: "/data/sets/kitti_second" 187 | } 188 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_28.proto: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | voxel_generator { 4 | point_cloud_range : [0, -40.32, -3, 71.68, 40.32, 1] 5 | voxel_size : [0.28, 0.28, 4] 6 | max_number_of_points_per_voxel : 100 7 | } 8 | num_class: 1 9 | voxel_feature_extractor: { 10 | module_class_name: "PillarFeatureNet" 11 | num_filters: [64] 12 | with_distance: false 13 | } 14 | middle_feature_extractor: { 15 | module_class_name: "PointPillarsScatter" 16 | } 17 | rpn: { 18 | module_class_name: "RPN" 19 | layer_nums: [3, 5, 5] 20 | layer_strides: [2, 2, 2] 21 | num_filters: [64, 128, 256] 22 | upsample_strides: [1, 2, 4] 23 | num_upsample_filters: [128, 128, 128] 24 | use_groupnorm: false 25 | num_groups: 32 26 | } 27 | loss: { 28 | classification_loss: { 29 | weighted_sigmoid_focal: { 30 | alpha: 0.25 31 | gamma: 2.0 32 | anchorwise_output: true 33 | } 34 | } 35 | localization_loss: { 36 | weighted_smooth_l1: { 37 | sigma: 3.0 38 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 39 | } 40 | } 41 | classification_weight: 1.0 42 | localization_weight: 2.0 43 | } 44 | # Outputs 45 | use_sigmoid_score: true 46 | encode_background_as_zeros: true 47 | encode_rad_error_by_sin: true 48 | 49 | use_direction_classifier: true 50 | direction_loss_weight: 0.2 51 | use_aux_classifier: false 52 | # Loss 53 | pos_class_weight: 1.0 54 | neg_class_weight: 1.0 55 | 56 | loss_norm_type: NormByNumPositives 57 | # Postprocess 58 | post_center_limit_range: [0, -40.32, -5, 71.68, 40.32, 5] 59 | use_rotate_nms: false 60 | use_multi_class_nms: false 61 | nms_pre_max_size: 1000 62 | nms_post_max_size: 300 63 | nms_score_threshold: 0.05 64 | nms_iou_threshold: 0.5 65 | 66 | use_bev: false 67 | num_point_features: 4 68 | without_reflectivity: false 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | anchor_generators: { 77 | anchor_generator_stride: { 78 | sizes: [1.6, 3.9, 1.56] # wlh 79 | strides: [0.56, 0.56, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.28, -40.04, -1.78] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | matched_threshold : 0.6 83 | unmatched_threshold : 0.45 84 | } 85 | } 86 | 87 | sample_positive_fraction : -1 88 | sample_size : 512 89 | region_similarity_calculator: { 90 | nearest_iou_similarity: { 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | 98 | train_input_reader: { 99 | record_file_path: "/data/sets/kitti_second/kitti_train.tfrecord" 100 | class_names: ["Car"] 101 | max_num_epochs : 160 102 | batch_size: 2 103 | prefetch_size : 25 104 | max_number_of_voxels: 12000 105 | shuffle_points: true 106 | num_workers: 2 107 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 108 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 109 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 110 | global_scaling_uniform_noise: [0.95, 1.05] 111 | global_random_rotation_range_per_object: [0, 0] 112 | anchor_area_threshold: 1 113 | remove_points_after_sample: false 114 | groundtruth_points_drop_percentage: 0.0 115 | groundtruth_drop_max_keep_points: 15 116 | database_sampler { 117 | database_info_path: "/data/sets/kitti_second/kitti_dbinfos_train.pkl" 118 | sample_groups { 119 | name_to_max_num { 120 | key: "Car" 121 | value: 15 122 | } 123 | } 124 | database_prep_steps { 125 | filter_by_min_num_points { 126 | min_num_point_pairs { 127 | key: "Car" 128 | value: 5 129 | } 130 | } 131 | } 132 | database_prep_steps { 133 | filter_by_difficulty { 134 | removed_difficulties: [-1] 135 | } 136 | } 137 | global_random_rotation_range_per_object: [0, 0] 138 | rate: 1.0 139 | } 140 | 141 | remove_unknown_examples: false 142 | remove_environment: false 143 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_train.pkl" 144 | kitti_root_path: "/data/sets/kitti_second" 145 | } 146 | 147 | train_config: { 148 | optimizer: { 149 | adam_optimizer: { 150 | learning_rate: { 151 | exponential_decay_learning_rate: { 152 | initial_learning_rate: 0.0002 153 | decay_steps: 27840 # 1856 steps per epoch * 15 epochs 154 | decay_factor: 0.8 155 | staircase: true 156 | } 157 | } 158 | weight_decay: 0.0001 159 | } 160 | use_moving_average: false 161 | 162 | } 163 | inter_op_parallelism_threads: 4 164 | intra_op_parallelism_threads: 4 165 | steps: 296960 # 1856 steps per epoch * 160 epochs 166 | steps_per_eval: 9280 # 1856 steps per epoch * 5 epochs 167 | save_checkpoints_secs : 1800 # half hour 168 | save_summary_steps : 10 169 | enable_mixed_precision: false 170 | loss_scale_factor : 512.0 171 | clear_metrics_every_epoch: false 172 | } 173 | 174 | eval_input_reader: { 175 | record_file_path: "/data/sets/kitti_second/kitti_val.tfrecord" 176 | class_names: ["Car"] 177 | batch_size: 2 178 | max_num_epochs : 160 179 | prefetch_size : 25 180 | max_number_of_voxels: 12000 181 | shuffle_points: false 182 | num_workers: 3 183 | anchor_area_threshold: 1 184 | remove_environment: false 185 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_val.pkl" 186 | kitti_root_path: "/data/sets/kitti_second" 187 | } 188 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PointPillars 2 | 3 | Welcome to PointPillars. 4 | 5 | This repo demonstrates how to reproduce the results from 6 | [_PointPillars: Fast Encoders for Object Detection from Point Clouds_](https://arxiv.org/abs/1812.05784) (to be published at CVPR 2019) on the 7 | [KITTI dataset](http://www.cvlibs.net/datasets/kitti/) by making the minimum required changes from the preexisting 8 | open source codebase [SECOND](https://github.com/traveller59/second.pytorch). 9 | 10 | This is not an official nuTonomy codebase, but it can be used to match the published PointPillars results. 11 | 12 | **WARNING: This code is not being actively maintained. This code can be used to reproduce the results in the first version of the paper, https://arxiv.org/abs/1812.05784v1. For an actively maintained repository that can also reproduce PointPillars results on nuScenes, we recommend using [SECOND](https://github.com/traveller59/second.pytorch). We are not the owners of the repository, but we have worked with the author and endorse his code.** 13 | 14 | ![Example Results](https://raw.githubusercontent.com/nutonomy/second.pytorch/master/images/pointpillars_kitti_results.png) 15 | 16 | 17 | ## Getting Started 18 | 19 | This is a fork of [SECOND for KITTI object detection](https://github.com/traveller59/second.pytorch) and the relevant 20 | subset of the original README is reproduced here. 21 | 22 | ### Code Support 23 | 24 | ONLY supports python 3.6+, pytorch 0.4.1+. Code has only been tested on Ubuntu 16.04/18.04. 25 | 26 | ### Install 27 | 28 | #### 1. Clone code 29 | 30 | ```bash 31 | git clone https://github.com/nutonomy/second.pytorch.git 32 | ``` 33 | 34 | #### 2. Install Python packages 35 | 36 | It is recommend to use the Anaconda package manager. 37 | 38 | First, use Anaconda to configure as many packages as possible. 39 | ```bash 40 | conda create -n pointpillars python=3.7 anaconda 41 | source activate pointpillars 42 | conda install shapely pybind11 protobuf scikit-image numba pillow 43 | conda install pytorch torchvision -c pytorch 44 | conda install google-sparsehash -c bioconda 45 | ``` 46 | 47 | Then use pip for the packages missing from Anaconda. 48 | ```bash 49 | pip install --upgrade pip 50 | pip install fire tensorboardX 51 | ``` 52 | 53 | Finally, install SparseConvNet. This is not required for PointPillars, but the general SECOND code base expects this 54 | to be correctly configured. 55 | ```bash 56 | git clone git@github.com:facebookresearch/SparseConvNet.git 57 | cd SparseConvNet/ 58 | bash build.sh 59 | # NOTE: if bash build.sh fails, try bash develop.sh instead 60 | ``` 61 | 62 | Additionally, you may need to install Boost geometry: 63 | 64 | ```bash 65 | sudo apt-get install libboost-all-dev 66 | ``` 67 | 68 | 69 | #### 3. Setup cuda for numba 70 | 71 | You need to add following environment variables for numba to ~/.bashrc: 72 | 73 | ```bash 74 | export NUMBAPRO_CUDA_DRIVER=/usr/lib/x86_64-linux-gnu/libcuda.so 75 | export NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so 76 | export NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice 77 | ``` 78 | 79 | #### 4. PYTHONPATH 80 | 81 | Add second.pytorch/ to your PYTHONPATH. 82 | 83 | ### Prepare dataset 84 | 85 | #### 1. Dataset preparation 86 | 87 | Download KITTI dataset and create some directories first: 88 | 89 | ```plain 90 | └── KITTI_DATASET_ROOT 91 | ├── training <-- 7481 train data 92 | | ├── image_2 <-- for visualization 93 | | ├── calib 94 | | ├── label_2 95 | | ├── velodyne 96 | | └── velodyne_reduced <-- empty directory 97 | └── testing <-- 7580 test data 98 | ├── image_2 <-- for visualization 99 | ├── calib 100 | ├── velodyne 101 | └── velodyne_reduced <-- empty directory 102 | ``` 103 | 104 | Note: PointPillar's protos use ```KITTI_DATASET_ROOT=/data/sets/kitti_second/```. 105 | 106 | #### 2. Create kitti infos: 107 | 108 | ```bash 109 | python create_data.py create_kitti_info_file --data_path=KITTI_DATASET_ROOT 110 | ``` 111 | 112 | #### 3. Create reduced point cloud: 113 | 114 | ```bash 115 | python create_data.py create_reduced_point_cloud --data_path=KITTI_DATASET_ROOT 116 | ``` 117 | 118 | #### 4. Create groundtruth-database infos: 119 | 120 | ```bash 121 | python create_data.py create_groundtruth_database --data_path=KITTI_DATASET_ROOT 122 | ``` 123 | 124 | #### 5. Modify config file 125 | 126 | The config file needs to be edited to point to the above datasets: 127 | 128 | ```bash 129 | train_input_reader: { 130 | ... 131 | database_sampler { 132 | database_info_path: "/path/to/kitti_dbinfos_train.pkl" 133 | ... 134 | } 135 | kitti_info_path: "/path/to/kitti_infos_train.pkl" 136 | kitti_root_path: "KITTI_DATASET_ROOT" 137 | } 138 | ... 139 | eval_input_reader: { 140 | ... 141 | kitti_info_path: "/path/to/kitti_infos_val.pkl" 142 | kitti_root_path: "KITTI_DATASET_ROOT" 143 | } 144 | ``` 145 | 146 | 147 | ### Train 148 | 149 | ```bash 150 | cd ~/second.pytorch/second 151 | python ./pytorch/train.py train --config_path=./configs/pointpillars/car/xyres_16.proto --model_dir=/path/to/model_dir 152 | ``` 153 | 154 | * If you want to train a new model, make sure "/path/to/model_dir" doesn't exist. 155 | * If "/path/to/model_dir" does exist, training will be resumed from the last checkpoint. 156 | * Training only supports a single GPU. 157 | * Training uses a batchsize=2 which should fit in memory on most standard GPUs. 158 | * On a single 1080Ti, training xyres_16 requires approximately 20 hours for 160 epochs. 159 | 160 | 161 | ### Evaluate 162 | 163 | 164 | ```bash 165 | cd ~/second.pytorch/second/ 166 | python pytorch/train.py evaluate --config_path= configs/pointpillars/car/xyres_16.proto --model_dir=/path/to/model_dir 167 | ``` 168 | 169 | * Detection result will saved in model_dir/eval_results/step_xxx. 170 | * By default, results are stored as a result.pkl file. To save as official KITTI label format use --pickle_result=False. 171 | -------------------------------------------------------------------------------- /second/configs/car.tiny.config: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | voxel_generator { 4 | point_cloud_range : [0, -32.0, -3, 52.8, 32.0, 1] 5 | voxel_size : [0.2, 0.2, 0.4] 6 | max_number_of_points_per_voxel : 35 7 | } 8 | 9 | num_class: 1 10 | voxel_feature_extractor: { 11 | module_class_name: "VoxelFeatureExtractor" 12 | num_filters: [32, 64] 13 | with_distance: false 14 | } 15 | middle_feature_extractor: { 16 | module_class_name: "SparseMiddleExtractor" 17 | # num_filters_down1: [64] 18 | # num_filters_down2: [64, 64] 19 | } 20 | rpn: { 21 | module_class_name: "RPN" 22 | layer_nums: [3, 5, 5] 23 | layer_strides: [2, 2, 2] 24 | num_filters: [64, 128, 128] 25 | upsample_strides: [1, 2, 4] 26 | num_upsample_filters: [128, 128, 128] 27 | use_groupnorm: false 28 | num_groups: 32 29 | } 30 | loss: { 31 | classification_loss: { 32 | weighted_sigmoid_focal: { 33 | alpha: 0.25 34 | gamma: 2.0 35 | anchorwise_output: true 36 | } 37 | } 38 | localization_loss: { 39 | weighted_smooth_l1: { 40 | sigma: 3.0 41 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 42 | } 43 | } 44 | classification_weight: 1.0 45 | localization_weight: 2.0 46 | } 47 | # Outputs 48 | use_sigmoid_score: true 49 | encode_background_as_zeros: true 50 | encode_rad_error_by_sin: true 51 | 52 | use_direction_classifier: true # this can help for orientation benchmark 53 | direction_loss_weight: 0.2 # enough. 54 | use_aux_classifier: false 55 | # Loss 56 | pos_class_weight: 1.0 57 | neg_class_weight: 1.0 58 | 59 | loss_norm_type: NormByNumPositives 60 | # Postprocess 61 | post_center_limit_range: [0, -40, -5.0, 70.4, 40, 5.0] 62 | use_rotate_nms: true 63 | use_multi_class_nms: false 64 | nms_pre_max_size: 1000 65 | nms_post_max_size: 100 66 | nms_score_threshold: 0.3 # 0.4 in submit, but 0.3 can get better hard performance 67 | nms_iou_threshold: 0.1 68 | 69 | use_bev: false 70 | num_point_features: 4 71 | without_reflectivity: false 72 | box_coder: { 73 | ground_box3d_coder: { 74 | linear_dim: false 75 | encode_angle_vector: false 76 | } 77 | } 78 | target_assigner: { 79 | anchor_generators: { 80 | anchor_generator_range: { 81 | sizes: [1.6, 3.9, 1.56] # wlh 82 | anchor_ranges: [0, -32.0, -1.78, 52.8, 32.0, -1.78] # carefully set z center 83 | rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code. 84 | matched_threshold : 0.6 85 | unmatched_threshold : 0.45 86 | } 87 | } 88 | sample_positive_fraction : -1 89 | sample_size : 512 90 | region_similarity_calculator: { 91 | nearest_iou_similarity: { 92 | } 93 | } 94 | } 95 | } 96 | } 97 | 98 | 99 | train_input_reader: { 100 | record_file_path: "/root/data/kitti/kitti_train.tfrecord" 101 | class_names: ["Car"] 102 | max_num_epochs : 160 103 | batch_size: 3 # sparse conv use 7633MB GPU memory when batch_size=3 104 | prefetch_size : 25 105 | max_number_of_voxels: 6500 # to support batchsize=2 in 1080Ti 106 | shuffle_points: true 107 | num_workers: 3 108 | groundtruth_localization_noise_std: [1.0, 1.0, 0.5] 109 | # groundtruth_rotation_uniform_noise: [-0.3141592654, 0.3141592654] 110 | groundtruth_rotation_uniform_noise: [-0.78539816, 0.78539816] 111 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 112 | global_scaling_uniform_noise: [0.95, 1.05] 113 | # global_random_rotation_range_per_object: [0.7854, 2.356] # pi/4 ~ 3pi/4 114 | global_random_rotation_range_per_object: [0, 0] # pi/4 ~ 3pi/4 115 | anchor_area_threshold: 1 116 | remove_points_after_sample: false 117 | groundtruth_points_drop_percentage: 0.0 118 | groundtruth_drop_max_keep_points: 15 119 | database_sampler { 120 | database_info_path: "/root/data/kitti/kitti_dbinfos_train.pkl" 121 | sample_groups { 122 | name_to_max_num { 123 | key: "Car" 124 | value: 15 125 | } 126 | } 127 | database_prep_steps { 128 | filter_by_min_num_points { 129 | min_num_point_pairs { 130 | key: "Car" 131 | value: 5 132 | } 133 | } 134 | } 135 | database_prep_steps { 136 | filter_by_difficulty { 137 | removed_difficulties: [-1] 138 | } 139 | } 140 | global_random_rotation_range_per_object: [0, 0] 141 | rate: 1.0 142 | } 143 | 144 | remove_unknown_examples: false 145 | remove_environment: false 146 | kitti_info_path: "/root/data/kitti/kitti_infos_train.pkl" 147 | kitti_root_path: "/root/data/kitti" 148 | } 149 | 150 | train_config: { 151 | optimizer: { 152 | adam_optimizer: { 153 | learning_rate: { 154 | exponential_decay_learning_rate: { 155 | initial_learning_rate: 0.0002 156 | decay_steps: 18570 157 | decay_factor: 0.8 158 | staircase: true 159 | } 160 | } 161 | weight_decay: 0.0001 162 | } 163 | use_moving_average: false 164 | 165 | } 166 | inter_op_parallelism_threads: 4 167 | intra_op_parallelism_threads: 4 168 | steps: 198080 # 1238 * 120 169 | steps_per_eval: 6190 # 1238 * 5 170 | # steps: 296960 # 1238 * 120 171 | # steps_per_eval: 9280 # 1856 * 5 172 | save_checkpoints_secs : 1800 # half hour 173 | save_summary_steps : 10 174 | enable_mixed_precision: false # for fp16 training, but sparseconvnet don't support fp16 175 | loss_scale_factor : 512.0 176 | clear_metrics_every_epoch: false 177 | } 178 | 179 | eval_input_reader: { 180 | record_file_path: "/root/data/kitti/kitti_val.tfrecord" 181 | class_names: ["Car"] 182 | batch_size: 3 183 | max_num_epochs : 160 184 | prefetch_size : 25 185 | max_number_of_voxels: 20000 186 | shuffle_points: false 187 | num_workers: 3 188 | anchor_area_threshold: 1 189 | remove_environment: false 190 | kitti_info_path: "/root/data/kitti/kitti_infos_val.pkl" 191 | # kitti_info_path: "/root/data/kitti/kitti_infos_test.pkl" 192 | kitti_root_path: "/root/data/kitti" 193 | } 194 | -------------------------------------------------------------------------------- /second/configs/pointpillars/ped_cycle/xyres_20.proto: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | voxel_generator { 4 | point_cloud_range : [0, -20, -2.5, 48, 20, 0.5] 5 | voxel_size : [0.2, 0.2, 3] 6 | max_number_of_points_per_voxel : 100 7 | } 8 | num_class: 2 9 | voxel_feature_extractor: { 10 | module_class_name: "PillarFeatureNet" 11 | num_filters: [64] 12 | with_distance: false 13 | } 14 | middle_feature_extractor: { 15 | module_class_name: "PointPillarsScatter" 16 | } 17 | rpn: { 18 | module_class_name: "RPN" 19 | layer_nums: [3, 5, 5] 20 | layer_strides: [1, 2, 2] 21 | num_filters: [64, 128, 256] 22 | upsample_strides: [1, 2, 4] 23 | num_upsample_filters: [128, 128, 128] 24 | use_groupnorm: false 25 | num_groups: 32 26 | } 27 | loss: { 28 | classification_loss: { 29 | weighted_sigmoid_focal: { 30 | alpha: 0.25 31 | gamma: 2.0 32 | anchorwise_output: true 33 | } 34 | } 35 | localization_loss: { 36 | weighted_smooth_l1: { 37 | sigma: 3.0 38 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 39 | } 40 | } 41 | classification_weight: 1.0 42 | localization_weight: 2.0 43 | } 44 | # Outputs 45 | use_sigmoid_score: true 46 | encode_background_as_zeros: true 47 | encode_rad_error_by_sin: true 48 | 49 | use_direction_classifier: true 50 | direction_loss_weight: 0.2 51 | use_aux_classifier: false 52 | # Loss 53 | pos_class_weight: 1.0 54 | neg_class_weight: 1.0 55 | 56 | loss_norm_type: NormByNumPositives 57 | # Postprocess 58 | post_center_limit_range: [0, -20, -2.5, 48, 20, 0.5] 59 | use_rotate_nms: false 60 | use_multi_class_nms: false 61 | nms_pre_max_size: 1000 62 | nms_post_max_size: 300 63 | nms_score_threshold: 0.05 64 | nms_iou_threshold: 0.5 65 | 66 | use_bev: false 67 | num_point_features: 4 68 | without_reflectivity: false 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | anchor_generators: { 77 | anchor_generator_stride: { 78 | sizes: [0.6, 1.76, 1.73] # wlh 79 | strides: [0.2, 0.2, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.1, -19.9, -1.465] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | matched_threshold : 0.5 83 | unmatched_threshold : 0.35 84 | } 85 | } 86 | anchor_generators: { 87 | anchor_generator_stride: { 88 | sizes: [0.6, 0.8, 1.73] # wlh 89 | strides: [0.2, 0.2, 0.0] # if generate only 1 z_center, z_stride will be ignored 90 | offsets: [0.1, -19.9, -1.465] # origin_offset + strides / 2 91 | rotations: [0, 1.57] # 0, pi/2 92 | matched_threshold : 0.5 93 | unmatched_threshold : 0.35 94 | } 95 | } 96 | 97 | sample_positive_fraction : -1 98 | sample_size : 512 99 | region_similarity_calculator: { 100 | nearest_iou_similarity: { 101 | } 102 | } 103 | } 104 | } 105 | } 106 | 107 | 108 | train_input_reader: { 109 | record_file_path: "/data/sets/kitti_second/kitti_train.tfrecord" 110 | class_names: ["Cyclist", "Pedestrian"] 111 | max_num_epochs : 160 112 | batch_size: 2 113 | prefetch_size : 25 114 | max_number_of_voxels: 12000 115 | shuffle_points: true 116 | num_workers: 2 117 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 118 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 119 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 120 | global_scaling_uniform_noise: [0.95, 1.05] 121 | global_random_rotation_range_per_object: [0, 0] 122 | anchor_area_threshold: 1 123 | remove_points_after_sample: false 124 | groundtruth_points_drop_percentage: 0.0 125 | groundtruth_drop_max_keep_points: 15 126 | database_sampler { 127 | database_info_path: "/data/sets/kitti_second/kitti_dbinfos_train.pkl" 128 | sample_groups { 129 | name_to_max_num { 130 | key: "Cyclist" 131 | value: 8 132 | } 133 | } 134 | database_prep_steps { 135 | filter_by_min_num_points { 136 | min_num_point_pairs { 137 | key: "Cyclist" 138 | value: 5 139 | } 140 | } 141 | } 142 | database_prep_steps { 143 | filter_by_difficulty { 144 | removed_difficulties: [-1] 145 | } 146 | } 147 | global_random_rotation_range_per_object: [0, 0] 148 | rate: 1.0 149 | } 150 | 151 | remove_unknown_examples: false 152 | remove_environment: false 153 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_train.pkl" 154 | kitti_root_path: "/data/sets/kitti_second" 155 | } 156 | 157 | train_config: { 158 | optimizer: { 159 | adam_optimizer: { 160 | learning_rate: { 161 | exponential_decay_learning_rate: { 162 | initial_learning_rate: 0.0002 163 | decay_steps: 27840 # 1856 steps per epoch * 15 epochs 164 | decay_factor: 0.8 165 | staircase: true 166 | } 167 | } 168 | weight_decay: 0.0001 169 | } 170 | use_moving_average: false 171 | 172 | } 173 | inter_op_parallelism_threads: 4 174 | intra_op_parallelism_threads: 4 175 | steps: 296960 # 1856 steps per epoch * 160 epochs 176 | steps_per_eval: 9280 # 1856 steps per epoch * 5 epochs 177 | save_checkpoints_secs : 1800 # half hour 178 | save_summary_steps : 10 179 | enable_mixed_precision: false 180 | loss_scale_factor : 512.0 181 | clear_metrics_every_epoch: false 182 | } 183 | 184 | eval_input_reader: { 185 | record_file_path: "/data/sets/kitti_second/kitti_val.tfrecord" 186 | class_names: ["Cyclist", "Pedestrian"] 187 | batch_size: 2 188 | max_num_epochs : 160 189 | prefetch_size : 25 190 | max_number_of_voxels: 12000 191 | shuffle_points: false 192 | num_workers: 3 193 | anchor_area_threshold: 1 194 | remove_environment: false 195 | kitti_info_path: "/data/sets/kitti_second/kitti_infos_val.pkl" 196 | kitti_root_path: "/data/sets/kitti_second" 197 | } 198 | --------------------------------------------------------------------------------