├── __init__.py ├── second ├── __init__.py ├── builder │ ├── __init__.py │ ├── preprocess_builder.py │ ├── dbsampler_builder.py │ ├── similarity_calculator_builder.py │ ├── voxel_builder.py │ ├── anchor_generator_builder.py │ └── target_assigner_builder.py ├── framework │ ├── __init__.py │ └── test.py ├── protos │ ├── __init__.py │ ├── model.proto │ ├── voxel_generator.proto │ ├── sampler.proto │ ├── train.proto │ ├── preprocess.proto │ ├── anchors.proto │ ├── pipeline.proto │ ├── box_coder.proto │ ├── similarity.proto │ ├── target.proto │ ├── input_reader.proto │ ├── second.proto │ ├── optimizer.proto │ ├── model_pb2.py │ ├── pipeline_pb2.py │ ├── voxel_generator_pb2.py │ ├── losses.proto │ └── train_pb2.py ├── pytorch │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── box_coders.py │ │ └── ghm_loss.py │ ├── builder │ │ ├── __init__.py │ │ ├── box_coder_builder.py │ │ ├── input_reader_builder.py │ │ ├── lr_scheduler_builder.py │ │ └── optimizer_builder.py │ ├── models │ │ ├── __init__.py │ │ └── resnet.py │ ├── utils │ │ └── __init__.py │ └── inference.py ├── utils │ ├── __init__.py │ ├── timer.py │ ├── check.py │ ├── merge_result.py │ ├── model_tool.py │ ├── config_tool.py │ ├── loader.py │ ├── config_tool │ │ ├── train.py │ │ └── __init__.py │ └── log_tool.py ├── kittiviewer │ ├── __init__.py │ ├── backend │ │ └── __init__.py │ └── frontend │ │ ├── textures │ │ └── sprites │ │ │ └── disc.png │ │ ├── js │ │ ├── shaders │ │ │ ├── CopyShader.js │ │ │ ├── LuminosityHighPassShader.js │ │ │ ├── ConvolutionShader.js │ │ │ ├── FocusShader.js │ │ │ └── FilmShader.js │ │ ├── Toast.js │ │ ├── postprocessing │ │ │ ├── RenderPass.js │ │ │ ├── ShaderPass.js │ │ │ ├── FilmPass.js │ │ │ ├── MaskPass.js │ │ │ ├── BloomPass.js │ │ │ └── EffectComposer.js │ │ ├── libs │ │ │ └── js.cookie.min.js │ │ └── renderers │ │ │ └── CSS2DRenderer.js │ │ └── css │ │ └── main.css ├── data │ ├── __init__.py │ └── dataset.py ├── core │ ├── __init__.py │ ├── non_max_suppression │ │ ├── __init__.py │ │ └── nms_cpu.py │ ├── box_coders.py │ ├── anchor_generator.py │ ├── inference.py │ └── region_similarity.py ├── create_data.py ├── script.py └── configs │ └── pointpillars │ └── car │ ├── xyres_20.config │ ├── xyres_28.config │ └── xyres_24.config ├── torchplus ├── ops │ ├── __init__.py │ └── array_ops.py ├── nn │ ├── modules │ │ ├── __init__.py │ │ ├── normalization.py │ │ └── common.py │ ├── __init__.py │ └── functional.py ├── __init__.py ├── train │ ├── __init__.py │ ├── common.py │ └── optim.py └── tools.py ├── images ├── kittibox.png ├── viewerweb.png └── simpleguide.png ├── LICENSE ├── .gitignore ├── RELEASE.md ├── Dockerfile └── NUSCENES-GUIDE.md /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/builder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/framework/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/protos/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /torchplus/ops/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/kittiviewer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /torchplus/nn/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/kittiviewer/backend/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/builder/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /second/pytorch/models/__init__.py: -------------------------------------------------------------------------------- 1 | from . import net_multi_head -------------------------------------------------------------------------------- /second/data/__init__.py: -------------------------------------------------------------------------------- 1 | from . import kitti_dataset 2 | from . import nuscenes_dataset -------------------------------------------------------------------------------- /images/kittibox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pyaf/second.pytorch/HEAD/images/kittibox.png -------------------------------------------------------------------------------- /images/viewerweb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pyaf/second.pytorch/HEAD/images/viewerweb.png -------------------------------------------------------------------------------- /images/simpleguide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pyaf/second.pytorch/HEAD/images/simpleguide.png -------------------------------------------------------------------------------- /second/core/__init__.py: -------------------------------------------------------------------------------- 1 | # from . import box_np_ops, box_tf_ops, geometry, preprocess, non_max_suppression -------------------------------------------------------------------------------- /second/kittiviewer/frontend/textures/sprites/disc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pyaf/second.pytorch/HEAD/second/kittiviewer/frontend/textures/sprites/disc.png -------------------------------------------------------------------------------- /torchplus/nn/__init__.py: -------------------------------------------------------------------------------- 1 | from torchplus.nn.functional import one_hot 2 | from torchplus.nn.modules.common import Empty, Sequential 3 | from torchplus.nn.modules.normalization import GroupNorm 4 | -------------------------------------------------------------------------------- /second/utils/timer.py: -------------------------------------------------------------------------------- 1 | import time 2 | from contextlib import contextmanager 3 | 4 | @contextmanager 5 | def simple_timer(name=''): 6 | t = time.time() 7 | yield 8 | print(f"{name} exec time: {time.time() - t}") -------------------------------------------------------------------------------- /second/protos/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/second.proto"; 5 | message DetectionModel{ 6 | oneof model { 7 | VoxelNet second = 1; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /torchplus/__init__.py: -------------------------------------------------------------------------------- 1 | from . import train 2 | from . import nn 3 | from . import metrics 4 | from . import tools 5 | 6 | from .tools import change_default_args 7 | from torchplus.ops.array_ops import scatter_nd, gather_nd 8 | -------------------------------------------------------------------------------- /second/core/non_max_suppression/__init__.py: -------------------------------------------------------------------------------- 1 | from second.core.non_max_suppression.nms_cpu import nms_jit, soft_nms_jit 2 | from second.core.non_max_suppression.nms_gpu import (nms_gpu, rotate_iou_gpu, 3 | rotate_nms_gpu) 4 | -------------------------------------------------------------------------------- /second/pytorch/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import time 2 | import contextlib 3 | import torch 4 | 5 | @contextlib.contextmanager 6 | def torch_timer(name=''): 7 | torch.cuda.synchronize() 8 | t = time.time() 9 | yield 10 | torch.cuda.synchronize() 11 | print(name, "time:", time.time() - t) -------------------------------------------------------------------------------- /torchplus/nn/functional.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def one_hot(tensor, depth, dim=-1, on_value=1.0, dtype=torch.float32): 4 | tensor_onehot = torch.zeros( 5 | *list(tensor.shape), depth, dtype=dtype, device=tensor.device) 6 | tensor_onehot.scatter_(dim, tensor.unsqueeze(dim).long(), on_value) 7 | return tensor_onehot 8 | -------------------------------------------------------------------------------- /torchplus/nn/modules/normalization.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class GroupNorm(torch.nn.GroupNorm): 5 | def __init__(self, num_channels, num_groups, eps=1e-5, affine=True): 6 | super().__init__( 7 | num_groups=num_groups, 8 | num_channels=num_channels, 9 | eps=eps, 10 | affine=affine) 11 | -------------------------------------------------------------------------------- /torchplus/train/__init__.py: -------------------------------------------------------------------------------- 1 | from torchplus.train.checkpoint import (latest_checkpoint, restore, 2 | restore_latest_checkpoints, 3 | restore_models, save, save_models, 4 | try_restore_latest_checkpoints) 5 | from torchplus.train.common import create_folder 6 | from torchplus.train.optim import MixedPrecisionWrapper 7 | -------------------------------------------------------------------------------- /second/protos/voxel_generator.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | message VoxelGenerator{ 6 | repeated float voxel_size = 1; 7 | repeated float point_cloud_range = 2; 8 | uint32 max_number_of_points_per_voxel = 3; 9 | bool full_empty_part_with_mean = 4; 10 | bool block_filtering = 5; 11 | int64 block_factor = 6; 12 | int64 block_size = 7; 13 | float height_threshold = 8; 14 | } 15 | -------------------------------------------------------------------------------- /second/protos/sampler.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/preprocess.proto"; 5 | 6 | message Group{ 7 | map name_to_max_num = 1; 8 | } 9 | 10 | message Sampler{ 11 | string database_info_path = 1; 12 | repeated Group sample_groups = 2; 13 | repeated DatabasePreprocessingStep database_prep_steps = 3; 14 | repeated float global_random_rotation_range_per_object = 4; 15 | float rate = 5; 16 | } 17 | -------------------------------------------------------------------------------- /second/protos/train.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | import "second/protos/optimizer.proto"; 6 | import "second/protos/preprocess.proto"; 7 | 8 | message TrainConfig{ 9 | Optimizer optimizer = 1; 10 | uint32 steps = 2; 11 | uint32 steps_per_eval = 3; 12 | uint32 save_checkpoints_secs = 4; 13 | uint32 save_summary_steps = 5; 14 | bool enable_mixed_precision = 6; 15 | float loss_scale_factor = 7; 16 | bool clear_metrics_every_epoch = 8; 17 | } -------------------------------------------------------------------------------- /second/protos/preprocess.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | message DatabasePreprocessingStep { 6 | oneof database_preprocessing_step { 7 | DBFilterByDifficulty filter_by_difficulty = 1; 8 | DBFilterByMinNumPointInGroundTruth filter_by_min_num_points = 2; 9 | } 10 | } 11 | 12 | message DBFilterByDifficulty{ 13 | repeated int32 removed_difficulties = 1; 14 | } 15 | 16 | message DBFilterByMinNumPointInGroundTruth{ 17 | map min_num_point_pairs = 1; 18 | } 19 | -------------------------------------------------------------------------------- /second/protos/anchors.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | message AnchorGeneratorStride { 6 | repeated float sizes = 1; 7 | repeated float strides = 2; 8 | repeated float offsets = 3; 9 | repeated float rotations = 4; 10 | repeated float custom_values = 5; 11 | } 12 | 13 | message AnchorGeneratorRange { 14 | repeated float sizes = 1; 15 | repeated float anchor_ranges = 2; 16 | repeated float rotations = 3; 17 | repeated float custom_values = 4; 18 | } 19 | 20 | message NoAnchor { 21 | } 22 | 23 | -------------------------------------------------------------------------------- /second/utils/check.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def is_array_like(x): 4 | return isinstance(x, (list, tuple, np.ndarray)) 5 | 6 | def shape_mergeable(x, expected_shape): 7 | mergeable = True 8 | if is_array_like(x) and is_array_like(expected_shape): 9 | x = np.array(x) 10 | if len(x.shape) == len(expected_shape): 11 | for s, s_ex in zip(x.shape, expected_shape): 12 | if s_ex is not None and s != s_ex: 13 | mergeable = False 14 | break 15 | return mergeable -------------------------------------------------------------------------------- /second/protos/pipeline.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | import "second/protos/input_reader.proto"; 6 | import "second/protos/model.proto"; 7 | import "second/protos/train.proto"; 8 | // Convenience message for configuring a training and eval pipeline. Allows all 9 | // of the pipeline parameters to be configured from one file. 10 | message TrainEvalPipelineConfig { 11 | DetectionModel model = 1; 12 | InputReader train_input_reader = 2; 13 | TrainConfig train_config = 3; 14 | InputReader eval_input_reader = 4; 15 | } 16 | 17 | -------------------------------------------------------------------------------- /second/protos/box_coder.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Configuration proto for the box coder to be used in the object detection 6 | // pipeline. See core/box_coder.py for details. 7 | message BoxCoder { 8 | oneof box_coder { 9 | GroundBox3dCoder ground_box3d_coder = 1; 10 | BevBoxCoder bev_box_coder = 2; 11 | } 12 | } 13 | 14 | message GroundBox3dCoder { 15 | bool linear_dim = 1; 16 | bool encode_angle_vector = 2; 17 | } 18 | 19 | message BevBoxCoder { 20 | bool linear_dim = 1; 21 | bool encode_angle_vector = 2; 22 | float z_fixed = 3; 23 | float h_fixed = 4; 24 | } 25 | -------------------------------------------------------------------------------- /second/builder/preprocess_builder.py: -------------------------------------------------------------------------------- 1 | import second.core.preprocess as prep 2 | 3 | def build_db_preprocess(db_prep_config): 4 | prep_type = db_prep_config.WhichOneof('database_preprocessing_step') 5 | 6 | if prep_type == 'filter_by_difficulty': 7 | cfg = db_prep_config.filter_by_difficulty 8 | return prep.DBFilterByDifficulty(list(cfg.removed_difficulties)) 9 | elif prep_type == 'filter_by_min_num_points': 10 | cfg = db_prep_config.filter_by_min_num_points 11 | return prep.DBFilterByMinNumPoint(dict(cfg.min_num_point_pairs)) 12 | else: 13 | raise ValueError("unknown database prep type") 14 | 15 | -------------------------------------------------------------------------------- /second/utils/merge_result.py: -------------------------------------------------------------------------------- 1 | import fire 2 | from pathlib import Path 3 | import re 4 | 5 | def merge(path1, path2, output_path): 6 | filepaths1 = Path(path1).glob('*.txt') 7 | prog = re.compile(r'^\d{6}.txt$') 8 | filepaths1 = filter(lambda f: prog.match(f.name), filepaths1) 9 | for fp1 in list(filepaths1): 10 | with open(fp1) as f1: 11 | contents = f1.readlines() 12 | if len(contents) != 0: 13 | contents += "\n" 14 | with open(Path(path2) / f"{fp1.stem}.txt", 'r') as f2: 15 | contents += f2.readlines() 16 | with open(Path(output_path) / f"{fp1.stem}.txt", 'w') as f: 17 | f.writelines(contents) 18 | 19 | if __name__ == '__main__': 20 | fire.Fire() 21 | 22 | -------------------------------------------------------------------------------- /torchplus/train/common.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import shutil 4 | 5 | def create_folder(prefix, add_time=True, add_str=None, delete=False): 6 | additional_str = '' 7 | if delete is True: 8 | if os.path.exists(prefix): 9 | shutil.rmtree(prefix) 10 | os.makedirs(prefix) 11 | folder = prefix 12 | if add_time is True: 13 | # additional_str has a form such as '170903_220351' 14 | additional_str += datetime.datetime.now().strftime("%y%m%d_%H%M%S") 15 | if add_str is not None: 16 | folder += '/' + additional_str + '_' + add_str 17 | else: 18 | folder += '/' + additional_str 19 | if delete is True: 20 | if os.path.exists(folder): 21 | shutil.rmtree(folder) 22 | os.makedirs(folder) 23 | return folder -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/CopyShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Full-screen textured quad shader 5 | */ 6 | 7 | THREE.CopyShader = { 8 | 9 | uniforms: { 10 | 11 | "tDiffuse": { value: null }, 12 | "opacity": { value: 1.0 } 13 | 14 | }, 15 | 16 | vertexShader: [ 17 | 18 | "varying vec2 vUv;", 19 | 20 | "void main() {", 21 | 22 | "vUv = uv;", 23 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 24 | 25 | "}" 26 | 27 | ].join( "\n" ), 28 | 29 | fragmentShader: [ 30 | 31 | "uniform float opacity;", 32 | 33 | "uniform sampler2D tDiffuse;", 34 | 35 | "varying vec2 vUv;", 36 | 37 | "void main() {", 38 | 39 | "vec4 texel = texture2D( tDiffuse, vUv );", 40 | "gl_FragColor = opacity * texel;", 41 | 42 | "}" 43 | 44 | ].join( "\n" ) 45 | 46 | }; 47 | -------------------------------------------------------------------------------- /second/protos/similarity.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Configuration proto for region similarity calculators. See 6 | // core/region_similarity_calculator.py for details. 7 | message RegionSimilarityCalculator { 8 | oneof region_similarity { 9 | RotateIouSimilarity rotate_iou_similarity = 1; 10 | NearestIouSimilarity nearest_iou_similarity = 2; 11 | DistanceSimilarity distance_similarity = 3; 12 | } 13 | } 14 | 15 | // Configuration for intersection-over-union (IOU) similarity calculator. 16 | message RotateIouSimilarity { 17 | } 18 | 19 | // Configuration for intersection-over-union (IOU) similarity calculator. 20 | message NearestIouSimilarity { 21 | } 22 | 23 | // Configuration for intersection-over-union (IOU) similarity calculator. 24 | message DistanceSimilarity { 25 | float distance_norm = 1; 26 | bool with_rotation = 2; 27 | float rotation_alpha = 3; 28 | } -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/Toast.js: -------------------------------------------------------------------------------- 1 | var Toast = function(toasts, timeout = 3000){ 2 | this.toasts = toasts; 3 | this.type = ['error', 'message', 'warning', 'success']; 4 | this.timeout = timeout; 5 | } 6 | 7 | Toast.prototype = { 8 | _addToast : function(type, text){ 9 | var toast; 10 | toast = document.createElement('li'); 11 | toast.classList.add(type); 12 | setTimeout(function(){ 13 | toast.remove(); 14 | }, this.timeout); 15 | this.toasts.appendChild(toast); 16 | return toast.innerHTML = `${type}: ${text}`; 17 | }, 18 | 19 | message : function(text){ 20 | return this._addToast("message", text); 21 | }, 22 | warning : function(text){ 23 | return this._addToast("warning", text); 24 | }, 25 | error : function(text){ 26 | return this._addToast("error", text); 27 | }, 28 | success : function(text){ 29 | return this._addToast("success", text); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /second/utils/model_tool.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import re 3 | import shutil 4 | 5 | def rm_invalid_model_dir(directory, step_thresh=200): 6 | directory = Path(directory).resolve() 7 | ckpt_re = r"[a-zA-Z0-9_]+\-([0-9]+)\.tckpt" 8 | ckpt_re = re.compile(ckpt_re) 9 | for path in directory.rglob("*"): 10 | if path.is_dir(): 11 | pipeline_path = (path / "pipeline.config") 12 | log_path = (path / "log.txt") 13 | summary_path = (path / "summary") 14 | must_exists = [pipeline_path, log_path, summary_path] 15 | if not all([e.exists() for e in must_exists]): 16 | continue 17 | ckpts = [] 18 | for subpath in path.iterdir(): 19 | match = ckpt_re.search(subpath.name) 20 | if match is not None: 21 | ckpts.append(int(match.group(1))) 22 | if len(ckpts) == 0 or all([e < step_thresh for e in ckpts]): 23 | shutil.rmtree(str(path)) 24 | -------------------------------------------------------------------------------- /second/builder/dbsampler_builder.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import second.core.preprocess as prep 4 | from second.builder import preprocess_builder 5 | from second.core.preprocess import DataBasePreprocessor 6 | from second.core.sample_ops import DataBaseSamplerV2 7 | 8 | 9 | def build(sampler_config): 10 | #import pdb; pdb.set_trace() 11 | cfg = sampler_config 12 | groups = list(cfg.sample_groups) 13 | prepors = [ 14 | preprocess_builder.build_db_preprocess(c) 15 | for c in cfg.database_prep_steps 16 | ] 17 | db_prepor = DataBasePreprocessor(prepors) 18 | rate = cfg.rate 19 | grot_range = cfg.global_random_rotation_range_per_object 20 | groups = [dict(g.name_to_max_num) for g in groups] 21 | info_path = cfg.database_info_path 22 | with open(info_path, 'rb') as f: 23 | db_infos = pickle.load(f) 24 | 25 | grot_range = list(grot_range) 26 | if len(grot_range) == 0: 27 | grot_range = None 28 | sampler = DataBaseSamplerV2(db_infos, groups, db_prepor, rate, grot_range) 29 | return sampler 30 | -------------------------------------------------------------------------------- /second/pytorch/builder/box_coder_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.protos import box_coder_pb2 4 | from second.pytorch.core.box_coders import (BevBoxCoderTorch, 5 | GroundBox3dCoderTorch) 6 | 7 | 8 | def build(box_coder_config): 9 | """Create optimizer based on config. 10 | 11 | Args: 12 | optimizer_config: A Optimizer proto message. 13 | 14 | Returns: 15 | An optimizer and a list of variables for summary. 16 | 17 | Raises: 18 | ValueError: when using an unsupported input data type. 19 | """ 20 | box_coder_type = box_coder_config.WhichOneof('box_coder') 21 | if box_coder_type == 'ground_box3d_coder': 22 | cfg = box_coder_config.ground_box3d_coder 23 | return GroundBox3dCoderTorch(cfg.linear_dim, cfg.encode_angle_vector) 24 | elif box_coder_type == 'bev_box_coder': 25 | cfg = box_coder_config.bev_box_coder 26 | return BevBoxCoderTorch(cfg.linear_dim, cfg.encode_angle_vector, cfg.z_fixed, cfg.h_fixed) 27 | else: 28 | raise ValueError("unknown box_coder type") 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /torchplus/ops/array_ops.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import math 3 | import time 4 | import torch 5 | 6 | 7 | def scatter_nd(indices, updates, shape): 8 | """pytorch edition of tensorflow scatter_nd. 9 | this function don't contain except handle code. so use this carefully 10 | when indice repeats, don't support repeat add which is supported 11 | in tensorflow. 12 | """ 13 | ret = torch.zeros(*shape, dtype=updates.dtype, device=updates.device) 14 | ndim = indices.shape[-1] 15 | output_shape = list(indices.shape[:-1]) + shape[indices.shape[-1]:] 16 | flatted_indices = indices.view(-1, ndim) 17 | slices = [flatted_indices[:, i] for i in range(ndim)] 18 | slices += [Ellipsis] 19 | ret[slices] = updates.view(*output_shape) 20 | return ret 21 | 22 | 23 | def gather_nd(params, indices): 24 | # this function has a limit that MAX_ADVINDEX_CALC_DIMS=5 25 | ndim = indices.shape[-1] 26 | output_shape = list(indices.shape[:-1]) + list(params.shape[indices.shape[-1]:]) 27 | flatted_indices = indices.view(-1, ndim) 28 | slices = [flatted_indices[:, i] for i in range(ndim)] 29 | slices += [Ellipsis] 30 | return params[slices].view(*output_shape) 31 | -------------------------------------------------------------------------------- /second/builder/similarity_calculator_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.core import region_similarity 4 | from second.protos import similarity_pb2 5 | 6 | 7 | def build(similarity_config): 8 | """Create optimizer based on config. 9 | 10 | Args: 11 | optimizer_config: A Optimizer proto message. 12 | 13 | Returns: 14 | An optimizer and a list of variables for summary. 15 | 16 | Raises: 17 | ValueError: when using an unsupported input data type. 18 | """ 19 | similarity_type = similarity_config.WhichOneof('region_similarity') 20 | if similarity_type == 'rotate_iou_similarity': 21 | return region_similarity.RotateIouSimilarity() 22 | elif similarity_type == 'nearest_iou_similarity': 23 | return region_similarity.NearestIouSimilarity() 24 | elif similarity_type == 'distance_similarity': 25 | cfg = similarity_config.distance_similarity 26 | return region_similarity.DistanceSimilarity( 27 | distance_norm=cfg.distance_norm, 28 | with_rotation=cfg.with_rotation, 29 | rotation_alpha=cfg.rotation_alpha) 30 | else: 31 | raise ValueError("unknown similarity type") 32 | -------------------------------------------------------------------------------- /second/builder/voxel_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from spconv.utils import VoxelGeneratorV2 4 | from second.protos import voxel_generator_pb2 5 | 6 | 7 | def build(voxel_config): 8 | """Builds a tensor dictionary based on the InputReader config. 9 | 10 | Args: 11 | input_reader_config: A input_reader_pb2.InputReader object. 12 | 13 | Returns: 14 | A tensor dict based on the input_reader_config. 15 | 16 | Raises: 17 | ValueError: On invalid input reader proto. 18 | ValueError: If no input paths are specified. 19 | """ 20 | if not isinstance(voxel_config, (voxel_generator_pb2.VoxelGenerator)): 21 | raise ValueError('input_reader_config not of type ' 22 | 'input_reader_pb2.InputReader.') 23 | voxel_generator = VoxelGeneratorV2( 24 | voxel_size=list(voxel_config.voxel_size), 25 | point_cloud_range=list(voxel_config.point_cloud_range), 26 | max_num_points=voxel_config.max_number_of_points_per_voxel, 27 | max_voxels=20000, 28 | full_mean=voxel_config.full_empty_part_with_mean, 29 | block_filtering=voxel_config.block_filtering, 30 | block_factor=voxel_config.block_factor, 31 | block_size=voxel_config.block_size, 32 | height_threshold=voxel_config.height_threshold) 33 | return voxel_generator 34 | -------------------------------------------------------------------------------- /second/create_data.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from pathlib import Path 3 | import pickle 4 | 5 | import fire 6 | 7 | import second.data.kitti_dataset as kitti_ds 8 | import second.data.nuscenes_dataset as nu_ds 9 | from second.data.all_dataset import create_groundtruth_database 10 | 11 | def kitti_data_prep(root_path): 12 | kitti_ds.create_kitti_info_file(root_path) 13 | kitti_ds.create_reduced_point_cloud(root_path) 14 | create_groundtruth_database("KittiDataset", root_path, Path(root_path) / "kitti_infos_train.pkl") 15 | 16 | def nuscenes_data_prep(root_path, version, dataset_name, max_sweeps=10): 17 | nu_ds.create_nuscenes_infos(root_path, version=version, max_sweeps=max_sweeps) 18 | name = "infos_train.pkl" 19 | if version == "v1.0-test": 20 | name = "infos_test.pkl" 21 | return 22 | create_groundtruth_database(dataset_name, root_path, Path(root_path) / name) 23 | 24 | if __name__ == '__main__': 25 | import os 26 | os.environ["MKL_NUM_THREADS"] = "24" 27 | os.environ["NUMEXPR_NUM_THREADS"] = "24" 28 | os.environ["OMP_NUM_THREADS"] = "24" 29 | fire.Fire() 30 | 31 | 32 | ''' 33 | Run this script as, from parent directory only: 34 | python create_data.py nuscenes_data_prep --root_path=../../data/nuscenes/v1.0-mini --version="v1.0-mini" --dataset_name="NuScenesDataset" --max_sweeps=10 35 | 36 | 37 | ''' 38 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/css/main.css: -------------------------------------------------------------------------------- 1 | .btn { 2 | /*height: 100%;*/ 3 | text-align: center; 4 | position: absolute; 5 | top: 90%; 6 | left: 40%; 7 | z-index: 2; 8 | font-size: 40pt; 9 | color:#bbb; 10 | } 11 | 12 | .prev { 13 | left: 38%; 14 | /*transform: scale(3, 1);*/ 15 | } 16 | .imgidx { 17 | background: transparent; 18 | width: 120px; 19 | height: 40px; 20 | font-size: 12pt; 21 | left: 47%; 22 | 23 | border: 2px solid blue; 24 | border-radius: 5px; 25 | 26 | } 27 | 28 | .next { 29 | left: 58%; 30 | /*transform: scale(3, 1);*/ 31 | } 32 | 33 | 34 | .toasts { 35 | position: fixed; 36 | max-width: 100%; 37 | width: 250px; 38 | right: 0; 39 | bottom: 0; 40 | padding: 0 10px; 41 | box-sizing: border-box; 42 | transition: height 1s; 43 | z-index: 100; 44 | } 45 | 46 | .toasts li { 47 | width: 100%; 48 | list-style-type: none; 49 | box-sizing: border-box; 50 | font-family: sans-serif; 51 | padding: 15px 20px; 52 | background: #222; 53 | color: white; 54 | border-radius: 2px; 55 | margin: 10px 0; 56 | } 57 | 58 | .toasts li.message { 59 | background: #358; 60 | } 61 | 62 | .toasts li.error { 63 | background: brown; 64 | } 65 | 66 | .toasts li.warning { 67 | background: chocolate; 68 | } 69 | 70 | .toasts li.success { 71 | background: seagreen; 72 | } -------------------------------------------------------------------------------- /second/protos/target.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/anchors.proto"; 5 | import "second/protos/similarity.proto"; 6 | 7 | message ClassSetting { 8 | oneof anchor_generator { 9 | AnchorGeneratorStride anchor_generator_stride = 1; 10 | AnchorGeneratorRange anchor_generator_range = 2; 11 | NoAnchor no_anchor = 3; 12 | } 13 | RegionSimilarityCalculator region_similarity_calculator = 4; 14 | bool use_multi_class_nms = 5; 15 | bool use_rotate_nms = 6; 16 | int32 nms_pre_max_size = 7; 17 | int32 nms_post_max_size = 8; 18 | float nms_score_threshold = 9; 19 | float nms_iou_threshold = 10; 20 | float matched_threshold = 11; 21 | float unmatched_threshold = 12; 22 | string class_name = 13; 23 | repeated int64 feature_map_size = 14; // 3D zyx (DHW) size 24 | } 25 | 26 | message TargetAssigner { 27 | repeated ClassSetting class_settings = 1; 28 | float sample_positive_fraction = 2; 29 | uint32 sample_size = 3; 30 | bool assign_per_class = 4; 31 | repeated int64 nms_pre_max_sizes = 5; // this will override setting in ClassSettings if provide. 32 | repeated int64 nms_post_max_sizes = 6; // this will override setting in ClassSettings if provide. 33 | repeated int64 nms_score_thresholds = 7; // this will override setting in ClassSettings if provide. 34 | repeated int64 nms_iou_thresholds = 8; // this will override setting in ClassSettings if provide. 35 | } -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/LuminosityHighPassShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author bhouston / http://clara.io/ 3 | * 4 | * Luminosity 5 | * http://en.wikipedia.org/wiki/Luminosity 6 | */ 7 | 8 | THREE.LuminosityHighPassShader = { 9 | 10 | shaderID: "luminosityHighPass", 11 | 12 | uniforms: { 13 | 14 | "tDiffuse": { type: "t", value: null }, 15 | "luminosityThreshold": { type: "f", value: 1.0 }, 16 | "smoothWidth": { type: "f", value: 1.0 }, 17 | "defaultColor": { type: "c", value: new THREE.Color( 0x000000 ) }, 18 | "defaultOpacity": { type: "f", value: 0.0 } 19 | 20 | }, 21 | 22 | vertexShader: [ 23 | 24 | "varying vec2 vUv;", 25 | 26 | "void main() {", 27 | 28 | "vUv = uv;", 29 | 30 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 31 | 32 | "}" 33 | 34 | ].join("\n"), 35 | 36 | fragmentShader: [ 37 | 38 | "uniform sampler2D tDiffuse;", 39 | "uniform vec3 defaultColor;", 40 | "uniform float defaultOpacity;", 41 | "uniform float luminosityThreshold;", 42 | "uniform float smoothWidth;", 43 | 44 | "varying vec2 vUv;", 45 | 46 | "void main() {", 47 | 48 | "vec4 texel = texture2D( tDiffuse, vUv );", 49 | 50 | "vec3 luma = vec3( 0.299, 0.587, 0.114 );", 51 | 52 | "float v = dot( texel.xyz, luma );", 53 | 54 | "vec4 outputColor = vec4( defaultColor.rgb, defaultOpacity );", 55 | 56 | "float alpha = smoothstep( luminosityThreshold, luminosityThreshold + smoothWidth, v );", 57 | 58 | "gl_FragColor = mix( outputColor, texel, alpha );", 59 | 60 | "}" 61 | 62 | ].join("\n") 63 | 64 | }; 65 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/RenderPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.RenderPass = function ( scene, camera, overrideMaterial, clearColor, clearAlpha ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | this.scene = scene; 10 | this.camera = camera; 11 | 12 | this.overrideMaterial = overrideMaterial; 13 | 14 | this.clearColor = clearColor; 15 | this.clearAlpha = ( clearAlpha !== undefined ) ? clearAlpha : 0; 16 | 17 | this.clear = true; 18 | this.clearDepth = false; 19 | this.needsSwap = false; 20 | 21 | }; 22 | 23 | THREE.RenderPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 24 | 25 | constructor: THREE.RenderPass, 26 | 27 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 28 | 29 | var oldAutoClear = renderer.autoClear; 30 | renderer.autoClear = false; 31 | 32 | this.scene.overrideMaterial = this.overrideMaterial; 33 | 34 | var oldClearColor, oldClearAlpha; 35 | 36 | if ( this.clearColor ) { 37 | 38 | oldClearColor = renderer.getClearColor().getHex(); 39 | oldClearAlpha = renderer.getClearAlpha(); 40 | 41 | renderer.setClearColor( this.clearColor, this.clearAlpha ); 42 | 43 | } 44 | 45 | if ( this.clearDepth ) { 46 | 47 | renderer.clearDepth(); 48 | 49 | } 50 | 51 | renderer.render( this.scene, this.camera, this.renderToScreen ? null : readBuffer, this.clear ); 52 | 53 | if ( this.clearColor ) { 54 | 55 | renderer.setClearColor( oldClearColor, oldClearAlpha ); 56 | 57 | } 58 | 59 | this.scene.overrideMaterial = null; 60 | renderer.autoClear = oldAutoClear; 61 | } 62 | 63 | } ); 64 | -------------------------------------------------------------------------------- /second/protos/input_reader.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | import "second/protos/preprocess.proto"; 5 | import "second/protos/sampler.proto"; 6 | 7 | message InputReader { 8 | uint32 batch_size = 1; 9 | message Dataset { 10 | string kitti_info_path = 1; 11 | string kitti_root_path = 2; 12 | string dataset_class_name = 3; // support KittiDataset and NuScenesDataset 13 | } 14 | Dataset dataset = 2; 15 | message Preprocess { 16 | bool shuffle_points = 1; 17 | uint32 max_number_of_voxels = 2; 18 | repeated float groundtruth_localization_noise_std = 3; 19 | repeated float groundtruth_rotation_uniform_noise = 4; 20 | repeated float global_rotation_uniform_noise = 5; 21 | repeated float global_scaling_uniform_noise = 6; 22 | repeated float global_translate_noise_std = 7; 23 | bool remove_unknown_examples = 8; 24 | uint32 num_workers = 9; 25 | float anchor_area_threshold = 10; 26 | bool remove_points_after_sample = 11; 27 | float groundtruth_points_drop_percentage = 12; 28 | uint32 groundtruth_drop_max_keep_points = 13; 29 | bool remove_environment = 14; 30 | repeated float global_random_rotation_range_per_object = 15; 31 | repeated DatabasePreprocessingStep database_prep_steps = 16; 32 | Sampler database_sampler = 17; 33 | bool use_group_id = 18; // this will enable group sample and noise 34 | int64 min_num_of_points_in_gt = 19; // gt boxes contains less than this will be ignored. 35 | bool random_flip_x = 20; 36 | bool random_flip_y = 21; 37 | float sample_importance = 22; 38 | } 39 | Preprocess preprocess = 3; 40 | uint32 max_num_epochs = 4; // deprecated 41 | uint32 prefetch_size = 5; // deprecated 42 | } 43 | -------------------------------------------------------------------------------- /second/pytorch/core/box_coders.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from second.core.box_coders import BevBoxCoder, GroundBox3dCoder 4 | from second.pytorch.core import box_torch_ops 5 | 6 | 7 | class GroundBox3dCoderTorch(GroundBox3dCoder): 8 | def encode_torch(self, boxes, anchors): 9 | return box_torch_ops.second_box_encode(boxes, anchors, self.vec_encode, 10 | self.linear_dim) 11 | 12 | def decode_torch(self, boxes, anchors): 13 | return box_torch_ops.second_box_decode(boxes, anchors, self.vec_encode, 14 | self.linear_dim) 15 | 16 | 17 | class BevBoxCoderTorch(BevBoxCoder): 18 | def encode_torch(self, boxes, anchors): 19 | anchors = anchors[..., [0, 1, 3, 4, 6]] 20 | boxes = boxes[..., [0, 1, 3, 4, 6]] 21 | return box_torch_ops.bev_box_encode(boxes, anchors, self.vec_encode, 22 | self.linear_dim) 23 | 24 | def decode_torch(self, encodings, anchors): 25 | anchors = anchors[..., [0, 1, 3, 4, 6]] 26 | ret = box_torch_ops.bev_box_decode(encodings, anchors, self.vec_encode, 27 | self.linear_dim) 28 | z_fixed = torch.full([*ret.shape[:-1], 1], 29 | self.z_fixed, 30 | dtype=ret.dtype, 31 | device=ret.device) 32 | h_fixed = torch.full([*ret.shape[:-1], 1], 33 | self.h_fixed, 34 | dtype=ret.dtype, 35 | device=ret.device) 36 | return torch.cat( 37 | [ret[..., :2], z_fixed, ret[..., 2:4], h_fixed, ret[..., 4:]], 38 | dim=-1) 39 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/ShaderPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.ShaderPass = function ( shader, textureID ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | this.textureID = ( textureID !== undefined ) ? textureID : "tDiffuse"; 10 | 11 | if ( shader instanceof THREE.ShaderMaterial ) { 12 | 13 | this.uniforms = shader.uniforms; 14 | 15 | this.material = shader; 16 | 17 | } else if ( shader ) { 18 | 19 | this.uniforms = THREE.UniformsUtils.clone( shader.uniforms ); 20 | 21 | this.material = new THREE.ShaderMaterial( { 22 | 23 | defines: Object.assign( {}, shader.defines ), 24 | uniforms: this.uniforms, 25 | vertexShader: shader.vertexShader, 26 | fragmentShader: shader.fragmentShader 27 | 28 | } ); 29 | 30 | } 31 | 32 | this.camera = new THREE.OrthographicCamera( - 1, 1, 1, - 1, 0, 1 ); 33 | this.scene = new THREE.Scene(); 34 | 35 | this.quad = new THREE.Mesh( new THREE.PlaneBufferGeometry( 2, 2 ), null ); 36 | this.quad.frustumCulled = false; // Avoid getting clipped 37 | this.scene.add( this.quad ); 38 | 39 | }; 40 | 41 | THREE.ShaderPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 42 | 43 | constructor: THREE.ShaderPass, 44 | 45 | render: function( renderer, writeBuffer, readBuffer, delta, maskActive ) { 46 | 47 | if ( this.uniforms[ this.textureID ] ) { 48 | 49 | this.uniforms[ this.textureID ].value = readBuffer.texture; 50 | 51 | } 52 | 53 | this.quad.material = this.material; 54 | 55 | if ( this.renderToScreen ) { 56 | 57 | renderer.render( this.scene, this.camera ); 58 | 59 | } else { 60 | 61 | renderer.render( this.scene, this.camera, writeBuffer, this.clear ); 62 | 63 | } 64 | 65 | } 66 | 67 | } ); 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | pretrained_models/ 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | *.o 10 | *.out 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | .vscode 110 | -------------------------------------------------------------------------------- /torchplus/tools.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import inspect 3 | import sys 4 | from collections import OrderedDict 5 | 6 | import numba 7 | import numpy as np 8 | import torch 9 | 10 | 11 | def get_pos_to_kw_map(func): 12 | pos_to_kw = {} 13 | fsig = inspect.signature(func) 14 | pos = 0 15 | for name, info in fsig.parameters.items(): 16 | if info.kind is info.POSITIONAL_OR_KEYWORD: 17 | pos_to_kw[pos] = name 18 | pos += 1 19 | return pos_to_kw 20 | 21 | 22 | def get_kw_to_default_map(func): 23 | kw_to_default = {} 24 | fsig = inspect.signature(func) 25 | for name, info in fsig.parameters.items(): 26 | if info.kind is info.POSITIONAL_OR_KEYWORD: 27 | if info.default is not info.empty: 28 | kw_to_default[name] = info.default 29 | return kw_to_default 30 | 31 | 32 | def change_default_args(**kwargs): 33 | def layer_wrapper(layer_class): 34 | class DefaultArgLayer(layer_class): 35 | def __init__(self, *args, **kw): 36 | pos_to_kw = get_pos_to_kw_map(layer_class.__init__) 37 | kw_to_pos = {kw: pos for pos, kw in pos_to_kw.items()} 38 | for key, val in kwargs.items(): 39 | if key not in kw and kw_to_pos[key] > len(args): 40 | kw[key] = val 41 | super().__init__(*args, **kw) 42 | 43 | return DefaultArgLayer 44 | 45 | return layer_wrapper 46 | 47 | def torch_to_np_dtype(ttype): 48 | type_map = { 49 | torch.float16: np.dtype(np.float16), 50 | torch.float32: np.dtype(np.float32), 51 | torch.float16: np.dtype(np.float64), 52 | torch.int32: np.dtype(np.int32), 53 | torch.int64: np.dtype(np.int64), 54 | torch.uint8: np.dtype(np.uint8), 55 | } 56 | return type_map[ttype] 57 | -------------------------------------------------------------------------------- /second/builder/anchor_generator_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from second.protos import box_coder_pb2 4 | from second.core.anchor_generator import (AnchorGeneratorStride, 5 | AnchorGeneratorRange) 6 | 7 | 8 | def build(class_cfg): 9 | """Create optimizer based on config. 10 | 11 | Args: 12 | optimizer_config: A Optimizer proto message. 13 | 14 | Returns: 15 | An optimizer and a list of variables for summary. 16 | 17 | Raises: 18 | ValueError: when using an unsupported input data type. 19 | """ 20 | ag_type = class_cfg.WhichOneof('anchor_generator') 21 | 22 | if ag_type == 'anchor_generator_stride': 23 | config = class_cfg.anchor_generator_stride 24 | ag = AnchorGeneratorStride( 25 | sizes=list(config.sizes), 26 | anchor_strides=list(config.strides), 27 | anchor_offsets=list(config.offsets), 28 | rotations=list(config.rotations), 29 | match_threshold=class_cfg.matched_threshold, 30 | unmatch_threshold=class_cfg.unmatched_threshold, 31 | class_name=class_cfg.class_name, 32 | custom_values=list(config.custom_values)) 33 | return ag 34 | elif ag_type == 'anchor_generator_range': 35 | config = class_cfg.anchor_generator_range 36 | ag = AnchorGeneratorRange( 37 | sizes=list(config.sizes), 38 | anchor_ranges=list(config.anchor_ranges), 39 | rotations=list(config.rotations), 40 | match_threshold=class_cfg.matched_threshold, 41 | unmatch_threshold=class_cfg.unmatched_threshold, 42 | class_name=class_cfg.class_name, 43 | custom_values=list(config.custom_values)) 44 | return ag 45 | elif ag_type == 'no_anchor': 46 | return None 47 | else: 48 | raise ValueError(" unknown anchor generator type") -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/FilmPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.FilmPass = function ( noiseIntensity, scanlinesIntensity, scanlinesCount, grayscale ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | if ( THREE.FilmShader === undefined ) 10 | console.error( "THREE.FilmPass relies on THREE.FilmShader" ); 11 | 12 | var shader = THREE.FilmShader; 13 | 14 | this.uniforms = THREE.UniformsUtils.clone( shader.uniforms ); 15 | 16 | this.material = new THREE.ShaderMaterial( { 17 | 18 | uniforms: this.uniforms, 19 | vertexShader: shader.vertexShader, 20 | fragmentShader: shader.fragmentShader 21 | 22 | } ); 23 | 24 | if ( grayscale !== undefined ) this.uniforms.grayscale.value = grayscale; 25 | if ( noiseIntensity !== undefined ) this.uniforms.nIntensity.value = noiseIntensity; 26 | if ( scanlinesIntensity !== undefined ) this.uniforms.sIntensity.value = scanlinesIntensity; 27 | if ( scanlinesCount !== undefined ) this.uniforms.sCount.value = scanlinesCount; 28 | 29 | this.camera = new THREE.OrthographicCamera( - 1, 1, 1, - 1, 0, 1 ); 30 | this.scene = new THREE.Scene(); 31 | 32 | this.quad = new THREE.Mesh( new THREE.PlaneBufferGeometry( 2, 2 ), null ); 33 | this.quad.frustumCulled = false; // Avoid getting clipped 34 | this.scene.add( this.quad ); 35 | 36 | }; 37 | 38 | THREE.FilmPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 39 | 40 | constructor: THREE.FilmPass, 41 | 42 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 43 | 44 | this.uniforms[ "tDiffuse" ].value = readBuffer.texture; 45 | this.uniforms[ "time" ].value += delta; 46 | 47 | this.quad.material = this.material; 48 | 49 | if ( this.renderToScreen ) { 50 | 51 | renderer.render( this.scene, this.camera ); 52 | 53 | } else { 54 | 55 | renderer.render( this.scene, this.camera, writeBuffer, this.clear ); 56 | 57 | } 58 | 59 | } 60 | 61 | } ); 62 | -------------------------------------------------------------------------------- /second/script.py: -------------------------------------------------------------------------------- 1 | from second.pytorch.train import train, evaluate 2 | from google.protobuf import text_format 3 | from second.protos import pipeline_pb2 4 | from pathlib import Path 5 | from second.utils import config_tool 6 | import warnings 7 | warnings.filterwarnings("ignore") 8 | 9 | def train_multi_rpn_layer_num(): 10 | config_path = "./configs/nuscenes/all.fhd.config" 11 | model_root = Path.home() / "second_test" # don't forget to change this. 12 | config = pipeline_pb2.TrainEvalPipelineConfig() 13 | with open(config_path, "r") as f: 14 | proto_str = f.read() 15 | text_format.Merge(proto_str, config) 16 | input_cfg = config.eval_input_reader 17 | model_cfg = config.model.second 18 | layer_nums = [2, 4, 7, 9] 19 | for l in layer_nums: 20 | model_dir = str(model_root / f"all_fhd_{l}") 21 | model_cfg.rpn.layer_nums[:] = [l] 22 | train(config, model_dir, resume=True) 23 | 24 | 25 | def eval_multi_threshold(): 26 | config_path = "./configs/nuscenes/all.fhd.config" 27 | ckpt_name = "/home/ags/second_test/all_fhd_2/" # don't forget to change this. 28 | #assert "/path/to/your" not in ckpt_name 29 | config = pipeline_pb2.TrainEvalPipelineConfig() 30 | with open(config_path, "r") as f: 31 | proto_str = f.read() 32 | text_format.Merge(proto_str, config) 33 | model_cfg = config.model.second 34 | 35 | #model_cfg['nms_score_threshold'] = 0.3 ### extra added by ags 36 | #import pdb; pdb.set_trace() 37 | threshs = [0.3] 38 | for thresh in threshs: 39 | model_cfg.nms_score_threshold = thresh 40 | # don't forget to change this. 41 | result_path = Path.home() / f"second_test_eval_{thresh:.2f}" 42 | evaluate( 43 | config, 44 | result_path=result_path, 45 | ckpt_path=str(ckpt_name), 46 | batch_size=1, 47 | measure_time=True) 48 | 49 | 50 | if __name__ == "__main__": 51 | #eval_multi_threshold() 52 | train_multi_rpn_layer_num() 53 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/libs/js.cookie.min.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Minified by jsDelivr using Terser v3.14.1. 3 | * Original file: /npm/js-cookie@2.2.0/src/js.cookie.js 4 | * 5 | * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files 6 | */ 7 | !function(e){var n=!1;if("function"==typeof define&&define.amd&&(define(e),n=!0),"object"==typeof exports&&(module.exports=e(),n=!0),!n){var o=window.Cookies,t=window.Cookies=e();t.noConflict=function(){return window.Cookies=o,t}}}(function(){function e(){for(var e=0,n={};e1){if("number"==typeof(i=e({path:"/"},t.defaults,i)).expires){var a=new Date;a.setMilliseconds(a.getMilliseconds()+864e5*i.expires),i.expires=a}i.expires=i.expires?i.expires.toUTCString():"";try{c=JSON.stringify(r),/^[\{\[]/.test(c)&&(r=c)}catch(e){}r=o.write?o.write(r,n):encodeURIComponent(String(r)).replace(/%(23|24|26|2B|3A|3C|3E|3D|2F|3F|40|5B|5D|5E|60|7B|7D|7C)/g,decodeURIComponent),n=(n=(n=encodeURIComponent(String(n))).replace(/%(23|24|26|2B|5E|60|7C)/g,decodeURIComponent)).replace(/[\(\)]/g,escape);var s="";for(var f in i)i[f]&&(s+="; "+f,!0!==i[f]&&(s+="="+i[f]));return document.cookie=n+"="+r+s}n||(c={});for(var p=document.cookie?document.cookie.split("; "):[],d=/(%[0-9A-Z]{2})+/g,u=0;u kMaxKernelSize ) kernelSize = kMaxKernelSize; 82 | halfWidth = ( kernelSize - 1 ) * 0.5; 83 | 84 | values = new Array( kernelSize ); 85 | sum = 0.0; 86 | for ( i = 0; i < kernelSize; ++ i ) { 87 | 88 | values[ i ] = gauss( i - halfWidth, sigma ); 89 | sum += values[ i ]; 90 | 91 | } 92 | 93 | // normalize the kernel 94 | 95 | for ( i = 0; i < kernelSize; ++ i ) values[ i ] /= sum; 96 | 97 | return values; 98 | 99 | } 100 | 101 | }; 102 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release 1.6.0alpha 2 | 3 | ## Major Features and Improvements 4 | 1. New dataset API (unstable during alpha), almost completely remove kitti-specific code. you can add your custom dataset by following steps: 5 | (1): implement all Dataset API functions 6 | (2): use web visualization tool to check whether the box is correct. 7 | (3): add your dataset to all_dataset.py, change the dataset_class_name in config file. 8 | 9 | 2. Add [NuScenes](https://www.nuscenes.org) dataset support (incomplete in 1.6.0alpha), I plan to reproduce the NDS score in their paper. 10 | 11 | 3. Add [pointpillars](https://github.com/nutonomy/second.pytorch) to this repo. 12 | 13 | 4. Full Tensorboard support. 14 | 15 | 5. FP16 and multi-gpu (need test, I only have one gpu) support. 16 | 17 | ## Minor Improvements and Bug fixes 18 | 19 | 1. Move all data-specific functions to their corresponding dataset file. 20 | 21 | 2. Improved config file structure, remove some unused item. 22 | 23 | 3. remove much unused and deprecated code. 24 | 25 | 4. add two learning rate scheduler: exp decay and manual step 26 | 27 | # Release 1.5.1 28 | 29 | ## Minor Improvements and Bug fixes 30 | 31 | 1. Better support for custom lidar data. You need to check KittiDataset for more details. (no test yet, I don't have custom data) 32 | * Change all box to center format. 33 | * Change kitti info format, now you need to regenerate kitti infos and gt database. 34 | * Eval functions now support custom data evaluation. you need to specify z_center and z_axis in eval function. 35 | 2. Better RPN, you can add custom block by inherit RPNBase and implement _make_layer method. 36 | 3. Update pretrained model. 37 | 4. Add a simple inference notebook. everyone should start this project by that notebook. 38 | 5. Add windows support. Training on windows is slow than linux. 39 | 40 | # Release 1.5 41 | 42 | ## Major Features and Improvements 43 | 44 | 1. New sparse convolution based models. VFE-based old models are deprecated. Now the model looks like this: 45 | points([N, 4])->voxels([N, 5, 4])->Features([N, 4])->Sparse Convolution Networks->RPN. See [this](https://github.com/traveller59/second.pytorch/blob/master/second/pytorch/models/middle.py) for more details of sparse conv networks. 46 | 2. The [SparseConvNet](https://github.com/facebookresearch/SparseConvNet) is deprecated. New library [spconv](https://github.com/traveller59/spconv) is introduced. 47 | 3. Super converge (from fastai) is implemented. Now all network can converge to a good result with only 50~80 epoch. For example. ```car.fhd.config``` only needs 50 epochs to reach 78.3 AP (car mod 3d). 48 | 4. Target assigner now works correctly when using multi-class. 49 | 50 | -------------------------------------------------------------------------------- /second/utils/config_tool.py: -------------------------------------------------------------------------------- 1 | # This file contains some config modification function. 2 | # some functions should be only used for KITTI dataset. 3 | 4 | from google.protobuf import text_format 5 | from second.protos import pipeline_pb2, second_pb2 6 | from pathlib import Path 7 | import numpy as np 8 | 9 | 10 | def change_detection_range(model_config, new_range): 11 | assert len(new_range) == 4, "you must provide a list such as [-50, -50, 50, 50]" 12 | old_pc_range = list(model_config.voxel_generator.point_cloud_range) 13 | old_pc_range[:2] = new_range[:2] 14 | old_pc_range[3:5] = new_range[2:] 15 | model_config.voxel_generator.point_cloud_range[:] = old_pc_range 16 | for anchor_generator in model_config.target_assigner.anchor_generators: 17 | a_type = anchor_generator.WhichOneof('anchor_generator') 18 | if a_type == "anchor_generator_range": 19 | a_cfg = anchor_generator.anchor_generator_range 20 | old_a_range = list(a_cfg.anchor_ranges) 21 | old_a_range[:2] = new_range[:2] 22 | old_a_range[3:5] = new_range[2:] 23 | a_cfg.anchor_ranges[:] = old_a_range 24 | elif a_type == "anchor_generator_stride": 25 | a_cfg = anchor_generator.anchor_generator_stride 26 | old_offset = list(a_cfg.offsets) 27 | stride = list(a_cfg.strides) 28 | old_offset[0] = new_range[0] + stride[0] / 2 29 | old_offset[1] = new_range[1] + stride[1] / 2 30 | a_cfg.offsets[:] = old_offset 31 | else: 32 | raise ValueError("unknown") 33 | old_post_range = list(model_config.post_center_limit_range) 34 | old_post_range[:2] = new_range[:2] 35 | old_post_range[3:5] = new_range[2:] 36 | model_config.post_center_limit_range[:] = old_post_range 37 | 38 | def get_downsample_factor(model_config): 39 | downsample_factor = np.prod(model_config.rpn.layer_strides) 40 | if len(model_config.rpn.upsample_strides) > 0: 41 | downsample_factor /= model_config.rpn.upsample_strides[-1] 42 | downsample_factor *= model_config.middle_feature_extractor.downsample_factor 43 | downsample_factor = int(downsample_factor) 44 | assert downsample_factor > 0 45 | return downsample_factor 46 | 47 | 48 | if __name__ == "__main__": 49 | config_path = "/home/yy/deeplearning/deeplearning/mypackages/second/configs/car.lite.1.config" 50 | config = pipeline_pb2.TrainEvalPipelineConfig() 51 | 52 | with open(config_path, "r") as f: 53 | proto_str = f.read() 54 | text_format.Merge(proto_str, config) 55 | 56 | change_detection_range(config, [-50, -50, 50, 50]) 57 | proto_str = text_format.MessageToString(config, indent=2) 58 | print(proto_str) 59 | 60 | -------------------------------------------------------------------------------- /second/pytorch/builder/input_reader_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Input reader builder. 16 | 17 | Creates data sources for DetectionModels from an InputReader config. See 18 | input_reader.proto for options. 19 | 20 | Note: If users wishes to also use their own InputReaders with the Object 21 | Detection configuration framework, they should define their own builder function 22 | that wraps the build function. 23 | """ 24 | 25 | from torch.utils.data import Dataset 26 | 27 | from second.builder import dataset_builder 28 | from second.protos import input_reader_pb2 29 | 30 | 31 | class DatasetWrapper(Dataset): 32 | """ convert our dataset to Dataset class in pytorch. 33 | """ 34 | 35 | def __init__(self, dataset): 36 | self._dataset = dataset 37 | 38 | def __len__(self): 39 | return len(self._dataset) 40 | 41 | def __getitem__(self, idx): 42 | return self._dataset[idx] 43 | 44 | @property 45 | def dataset(self): 46 | return self._dataset 47 | 48 | 49 | def build(input_reader_config, 50 | model_config, 51 | training, 52 | voxel_generator, 53 | target_assigner=None, 54 | multi_gpu=False) -> DatasetWrapper: 55 | """Builds a tensor dictionary based on the InputReader config. 56 | 57 | Args: 58 | input_reader_config: A input_reader_pb2.InputReader object. 59 | 60 | Returns: 61 | A tensor dict based on the input_reader_config. 62 | 63 | Raises: 64 | ValueError: On invalid input reader proto. 65 | ValueError: If no input paths are specified. 66 | """ 67 | if not isinstance(input_reader_config, input_reader_pb2.InputReader): 68 | raise ValueError('input_reader_config not of type ' 69 | 'input_reader_pb2.InputReader.') 70 | dataset = dataset_builder.build( 71 | input_reader_config, 72 | model_config, 73 | training, 74 | voxel_generator, 75 | target_assigner, 76 | multi_gpu=multi_gpu) 77 | dataset = DatasetWrapper(dataset) 78 | return dataset 79 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/FocusShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Focus shader 5 | * based on PaintEffect postprocess from ro.me 6 | * http://code.google.com/p/3-dreams-of-black/source/browse/deploy/js/effects/PaintEffect.js 7 | */ 8 | 9 | THREE.FocusShader = { 10 | 11 | uniforms : { 12 | 13 | "tDiffuse": { value: null }, 14 | "screenWidth": { value: 1024 }, 15 | "screenHeight": { value: 1024 }, 16 | "sampleDistance": { value: 0.94 }, 17 | "waveFactor": { value: 0.00125 } 18 | 19 | }, 20 | 21 | vertexShader: [ 22 | 23 | "varying vec2 vUv;", 24 | 25 | "void main() {", 26 | 27 | "vUv = uv;", 28 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 29 | 30 | "}" 31 | 32 | ].join( "\n" ), 33 | 34 | fragmentShader: [ 35 | 36 | "uniform float screenWidth;", 37 | "uniform float screenHeight;", 38 | "uniform float sampleDistance;", 39 | "uniform float waveFactor;", 40 | 41 | "uniform sampler2D tDiffuse;", 42 | 43 | "varying vec2 vUv;", 44 | 45 | "void main() {", 46 | 47 | "vec4 color, org, tmp, add;", 48 | "float sample_dist, f;", 49 | "vec2 vin;", 50 | "vec2 uv = vUv;", 51 | 52 | "add = color = org = texture2D( tDiffuse, uv );", 53 | 54 | "vin = ( uv - vec2( 0.5 ) ) * vec2( 1.4 );", 55 | "sample_dist = dot( vin, vin ) * 2.0;", 56 | 57 | "f = ( waveFactor * 100.0 + sample_dist ) * sampleDistance * 4.0;", 58 | 59 | "vec2 sampleSize = vec2( 1.0 / screenWidth, 1.0 / screenHeight ) * vec2( f );", 60 | 61 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.111964, 0.993712 ) * sampleSize );", 62 | "if( tmp.b < color.b ) color = tmp;", 63 | 64 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.846724, 0.532032 ) * sampleSize );", 65 | "if( tmp.b < color.b ) color = tmp;", 66 | 67 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.943883, -0.330279 ) * sampleSize );", 68 | "if( tmp.b < color.b ) color = tmp;", 69 | 70 | "add += tmp = texture2D( tDiffuse, uv + vec2( 0.330279, -0.943883 ) * sampleSize );", 71 | "if( tmp.b < color.b ) color = tmp;", 72 | 73 | "add += tmp = texture2D( tDiffuse, uv + vec2( -0.532032, -0.846724 ) * sampleSize );", 74 | "if( tmp.b < color.b ) color = tmp;", 75 | 76 | "add += tmp = texture2D( tDiffuse, uv + vec2( -0.993712, -0.111964 ) * sampleSize );", 77 | "if( tmp.b < color.b ) color = tmp;", 78 | 79 | "add += tmp = texture2D( tDiffuse, uv + vec2( -0.707107, 0.707107 ) * sampleSize );", 80 | "if( tmp.b < color.b ) color = tmp;", 81 | 82 | "color = color * vec4( 2.0 ) - ( add / vec4( 8.0 ) );", 83 | "color = color + ( add / vec4( 8.0 ) - color ) * ( vec4( 1.0 ) - vec4( sample_dist * 0.5 ) );", 84 | 85 | "gl_FragColor = vec4( color.rgb * color.rgb * vec3( 0.95 ) + color.rgb, 1.0 );", 86 | 87 | "}" 88 | 89 | 90 | ].join( "\n" ) 91 | }; 92 | -------------------------------------------------------------------------------- /second/core/box_coders.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from abc import abstractproperty 4 | from second.core import box_np_ops 5 | import numpy as np 6 | 7 | class BoxCoder(object): 8 | """Abstract base class for box coder.""" 9 | __metaclass__ = ABCMeta 10 | 11 | @abstractproperty 12 | def code_size(self): 13 | pass 14 | 15 | def encode(self, boxes, anchors): 16 | return self._encode(boxes, anchors) 17 | 18 | def decode(self, rel_codes, anchors): 19 | return self._decode(rel_codes, anchors) 20 | 21 | @abstractmethod 22 | def _encode(self, boxes, anchors): 23 | pass 24 | 25 | @abstractmethod 26 | def _decode(self, rel_codes, anchors): 27 | pass 28 | 29 | 30 | class GroundBox3dCoder(BoxCoder): 31 | def __init__(self, linear_dim=False, vec_encode=False, custom_ndim=0): 32 | super().__init__() 33 | self.linear_dim = linear_dim 34 | self.vec_encode = vec_encode 35 | self.custom_ndim = custom_ndim 36 | 37 | @property 38 | def code_size(self): 39 | res = 8 if self.vec_encode else 7 40 | return self.custom_ndim + res 41 | 42 | def _encode(self, boxes, anchors): 43 | return box_np_ops.second_box_encode(boxes, anchors, self.vec_encode, self.linear_dim) 44 | 45 | def _decode(self, encodings, anchors): 46 | return box_np_ops.second_box_decode(encodings, anchors, self.vec_encode, self.linear_dim) 47 | 48 | 49 | class BevBoxCoder(BoxCoder): 50 | """WARNING: this coder will return encoding with size=5, but 51 | takes size=7 boxes, anchors 52 | """ 53 | def __init__(self, linear_dim=False, vec_encode=False, z_fixed=-1.0, h_fixed=2.0, custom_ndim=0): 54 | super().__init__() 55 | self.linear_dim = linear_dim 56 | self.z_fixed = z_fixed 57 | self.h_fixed = h_fixed 58 | self.vec_encode = vec_encode 59 | self.custom_ndim = custom_ndim 60 | assert self.custom_ndim == 0 61 | 62 | @property 63 | def code_size(self): 64 | res = 6 if self.vec_encode else 5 65 | return self.custom_ndim + res 66 | 67 | def _encode(self, boxes, anchors): 68 | anchors = anchors[..., [0, 1, 3, 4, 6]] 69 | boxes = boxes[..., [0, 1, 3, 4, 6]] 70 | return box_np_ops.bev_box_encode(boxes, anchors, self.vec_encode, self.linear_dim) 71 | 72 | def _decode(self, encodings, anchors): 73 | anchors = anchors[..., [0, 1, 3, 4, 6]] 74 | ret = box_np_ops.bev_box_decode(encodings, anchors, self.vec_encode, self.linear_dim) 75 | z_fixed = np.full([*ret.shape[:-1], 1], self.z_fixed, dtype=ret.dtype) 76 | h_fixed = np.full([*ret.shape[:-1], 1], self.h_fixed, dtype=ret.dtype) 77 | return np.concatenate([ret[..., :2], z_fixed, ret[..., 2:4], h_fixed, ret[..., 4:]], axis=-1) 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /second/utils/loader.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from pathlib import Path 3 | import sys 4 | import os 5 | import logging 6 | logger = logging.getLogger('second.utils.loader') 7 | 8 | CUSTOM_LOADED_MODULES = {} 9 | 10 | 11 | def _get_possible_module_path(paths): 12 | ret = [] 13 | for p in paths: 14 | p = Path(p) 15 | for path in p.glob("*"): 16 | if path.suffix in ["py", ".so"] or (path.is_dir()): 17 | if path.stem.isidentifier(): 18 | ret.append(path) 19 | return ret 20 | 21 | 22 | def _get_regular_import_name(path, module_paths): 23 | path = Path(path) 24 | for mp in module_paths: 25 | mp = Path(mp) 26 | if mp == path: 27 | return path.stem 28 | try: 29 | relative_path = path.relative_to(Path(mp)) 30 | parts = list((relative_path.parent / relative_path.stem).parts) 31 | module_name = '.'.join([mp.stem] + parts) 32 | return module_name 33 | except: 34 | pass 35 | return None 36 | 37 | 38 | def import_file(path, name: str = None, add_to_sys=True, 39 | disable_warning=False): 40 | global CUSTOM_LOADED_MODULES 41 | path = Path(path) 42 | module_name = path.stem 43 | try: 44 | user_paths = os.environ['PYTHONPATH'].split(os.pathsep) 45 | except KeyError: 46 | user_paths = [] 47 | possible_paths = _get_possible_module_path(user_paths) 48 | model_import_name = _get_regular_import_name(path, possible_paths) 49 | if model_import_name is not None: 50 | return import_name(model_import_name) 51 | if name is not None: 52 | module_name = name 53 | spec = importlib.util.spec_from_file_location(module_name, path) 54 | module = importlib.util.module_from_spec(spec) 55 | spec.loader.exec_module(module) 56 | if not disable_warning: 57 | logger.warning(( 58 | f"Failed to perform regular import for file {path}. " 59 | "this means this file isn't in any folder in PYTHONPATH " 60 | "or don't have __init__.py in that project. " 61 | "directly file import may fail and some reflecting features are " 62 | "disabled even if import succeed. please add your project to PYTHONPATH " 63 | "or add __init__.py to ensure this file can be regularly imported. " 64 | )) 65 | 66 | if add_to_sys: # this will enable find objects defined in a file. 67 | # avoid replace system modules. 68 | if module_name in sys.modules and module_name not in CUSTOM_LOADED_MODULES: 69 | raise ValueError(f"{module_name} exists in system.") 70 | CUSTOM_LOADED_MODULES[module_name] = module 71 | sys.modules[module_name] = module 72 | return module 73 | 74 | 75 | def import_name(name, package=None): 76 | module = importlib.import_module(name, package) 77 | return module -------------------------------------------------------------------------------- /second/protos/optimizer.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Messages for configuring the optimizing strategy for training object 6 | // detection models. 7 | 8 | // Top level optimizer message. 9 | message Optimizer { 10 | oneof optimizer { 11 | RMSPropOptimizer rms_prop_optimizer = 1; 12 | MomentumOptimizer momentum_optimizer = 2; 13 | AdamOptimizer adam_optimizer = 3; 14 | } 15 | bool use_moving_average = 4; 16 | float moving_average_decay = 5; 17 | bool fixed_weight_decay = 6; // i.e. AdamW 18 | } 19 | 20 | // Configuration message for the RMSPropOptimizer 21 | // See: https://www.tensorflow.org/api_docs/python/tf/train/RMSPropOptimizer 22 | message RMSPropOptimizer { 23 | LearningRate learning_rate = 1; 24 | float momentum_optimizer_value = 2; 25 | float decay = 3; 26 | float epsilon = 4; 27 | float weight_decay = 5; 28 | } 29 | 30 | // Configuration message for the MomentumOptimizer 31 | // See: https://www.tensorflow.org/api_docs/python/tf/train/MomentumOptimizer 32 | message MomentumOptimizer { 33 | LearningRate learning_rate = 1; 34 | float momentum_optimizer_value = 2; 35 | float weight_decay = 3; 36 | } 37 | 38 | // Configuration message for the AdamOptimizer 39 | // See: https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer 40 | message AdamOptimizer { 41 | LearningRate learning_rate = 1; 42 | float weight_decay = 2; 43 | bool amsgrad = 3; 44 | } 45 | 46 | message LearningRate { 47 | oneof learning_rate { 48 | MultiPhase multi_phase = 1; 49 | OneCycle one_cycle = 2; 50 | ExponentialDecay exponential_decay = 3; 51 | ManualStepping manual_stepping = 4; 52 | } 53 | } 54 | 55 | message LearningRatePhase { 56 | float start = 1; 57 | string lambda_func = 2; 58 | string momentum_lambda_func = 3; 59 | } 60 | 61 | message MultiPhase { 62 | repeated LearningRatePhase phases = 1; 63 | } 64 | 65 | message OneCycle { 66 | float lr_max = 1; 67 | repeated float moms = 2; 68 | float div_factor = 3; 69 | float pct_start = 4; 70 | } 71 | 72 | /* 73 | ManualStepping example: 74 | initial_learning_rate = 0.001 75 | decay_length = 0.1 76 | decay_factor = 0.8 77 | staircase = True 78 | detail: 79 | progress 0%~10%, lr=0.001 80 | progress 10%~20%, lr=0.001 * 0.8 81 | progress 20%~30%, lr=0.001 * 0.8 * 0.8 82 | ...... 83 | */ 84 | 85 | 86 | message ExponentialDecay { 87 | float initial_learning_rate = 1; 88 | float decay_length = 2; // must in range (0, 1) 89 | float decay_factor = 3; 90 | bool staircase = 4; 91 | } 92 | 93 | /* 94 | ManualStepping example: 95 | boundaries = [0.8, 0.9] 96 | rates = [0.001, 0.002, 0.003] 97 | detail: 98 | progress 0%~80%, lr=0.001 99 | progress 80%~90%, lr=0.002 100 | progress 90%~100%, lr=0.003 101 | */ 102 | 103 | message ManualStepping { 104 | repeated float boundaries = 1; // must in range (0, 1) 105 | repeated float rates = 2; 106 | } -------------------------------------------------------------------------------- /second/protos/model_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/model.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from second.protos import second_pb2 as second_dot_protos_dot_second__pb2 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='second/protos/model.proto', 20 | package='second.protos', 21 | syntax='proto3', 22 | serialized_options=None, 23 | serialized_pb=_b('\n\x19second/protos/model.proto\x12\rsecond.protos\x1a\x1asecond/protos/second.proto\"D\n\x0e\x44\x65tectionModel\x12)\n\x06second\x18\x01 \x01(\x0b\x32\x17.second.protos.VoxelNetH\x00\x42\x07\n\x05modelb\x06proto3') 24 | , 25 | dependencies=[second_dot_protos_dot_second__pb2.DESCRIPTOR,]) 26 | 27 | 28 | 29 | 30 | _DETECTIONMODEL = _descriptor.Descriptor( 31 | name='DetectionModel', 32 | full_name='second.protos.DetectionModel', 33 | filename=None, 34 | file=DESCRIPTOR, 35 | containing_type=None, 36 | fields=[ 37 | _descriptor.FieldDescriptor( 38 | name='second', full_name='second.protos.DetectionModel.second', index=0, 39 | number=1, type=11, cpp_type=10, label=1, 40 | has_default_value=False, default_value=None, 41 | message_type=None, enum_type=None, containing_type=None, 42 | is_extension=False, extension_scope=None, 43 | serialized_options=None, file=DESCRIPTOR), 44 | ], 45 | extensions=[ 46 | ], 47 | nested_types=[], 48 | enum_types=[ 49 | ], 50 | serialized_options=None, 51 | is_extendable=False, 52 | syntax='proto3', 53 | extension_ranges=[], 54 | oneofs=[ 55 | _descriptor.OneofDescriptor( 56 | name='model', full_name='second.protos.DetectionModel.model', 57 | index=0, containing_type=None, fields=[]), 58 | ], 59 | serialized_start=72, 60 | serialized_end=140, 61 | ) 62 | 63 | _DETECTIONMODEL.fields_by_name['second'].message_type = second_dot_protos_dot_second__pb2._VOXELNET 64 | _DETECTIONMODEL.oneofs_by_name['model'].fields.append( 65 | _DETECTIONMODEL.fields_by_name['second']) 66 | _DETECTIONMODEL.fields_by_name['second'].containing_oneof = _DETECTIONMODEL.oneofs_by_name['model'] 67 | DESCRIPTOR.message_types_by_name['DetectionModel'] = _DETECTIONMODEL 68 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 69 | 70 | DetectionModel = _reflection.GeneratedProtocolMessageType('DetectionModel', (_message.Message,), dict( 71 | DESCRIPTOR = _DETECTIONMODEL, 72 | __module__ = 'second.protos.model_pb2' 73 | # @@protoc_insertion_point(class_scope:second.protos.DetectionModel) 74 | )) 75 | _sym_db.RegisterMessage(DetectionModel) 76 | 77 | 78 | # @@protoc_insertion_point(module_scope) 79 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/shaders/FilmShader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | * 4 | * Film grain & scanlines shader 5 | * 6 | * - ported from HLSL to WebGL / GLSL 7 | * http://www.truevision3d.com/forums/showcase/staticnoise_colorblackwhite_scanline_shaders-t18698.0.html 8 | * 9 | * Screen Space Static Postprocessor 10 | * 11 | * Produces an analogue noise overlay similar to a film grain / TV static 12 | * 13 | * Original implementation and noise algorithm 14 | * Pat 'Hawthorne' Shearon 15 | * 16 | * Optimized scanlines + noise version with intensity scaling 17 | * Georg 'Leviathan' Steinrohder 18 | * 19 | * This version is provided under a Creative Commons Attribution 3.0 License 20 | * http://creativecommons.org/licenses/by/3.0/ 21 | */ 22 | 23 | THREE.FilmShader = { 24 | 25 | uniforms: { 26 | 27 | "tDiffuse": { value: null }, 28 | "time": { value: 0.0 }, 29 | "nIntensity": { value: 0.5 }, 30 | "sIntensity": { value: 0.05 }, 31 | "sCount": { value: 4096 }, 32 | "grayscale": { value: 1 } 33 | 34 | }, 35 | 36 | vertexShader: [ 37 | 38 | "varying vec2 vUv;", 39 | 40 | "void main() {", 41 | 42 | "vUv = uv;", 43 | "gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );", 44 | 45 | "}" 46 | 47 | ].join( "\n" ), 48 | 49 | fragmentShader: [ 50 | 51 | "#include ", 52 | 53 | // control parameter 54 | "uniform float time;", 55 | 56 | "uniform bool grayscale;", 57 | 58 | // noise effect intensity value (0 = no effect, 1 = full effect) 59 | "uniform float nIntensity;", 60 | 61 | // scanlines effect intensity value (0 = no effect, 1 = full effect) 62 | "uniform float sIntensity;", 63 | 64 | // scanlines effect count value (0 = no effect, 4096 = full effect) 65 | "uniform float sCount;", 66 | 67 | "uniform sampler2D tDiffuse;", 68 | 69 | "varying vec2 vUv;", 70 | 71 | "void main() {", 72 | 73 | // sample the source 74 | "vec4 cTextureScreen = texture2D( tDiffuse, vUv );", 75 | 76 | // make some noise 77 | "float dx = rand( vUv + time );", 78 | 79 | // add noise 80 | "vec3 cResult = cTextureScreen.rgb + cTextureScreen.rgb * clamp( 0.1 + dx, 0.0, 1.0 );", 81 | 82 | // get us a sine and cosine 83 | "vec2 sc = vec2( sin( vUv.y * sCount ), cos( vUv.y * sCount ) );", 84 | 85 | // add scanlines 86 | "cResult += cTextureScreen.rgb * vec3( sc.x, sc.y, sc.x ) * sIntensity;", 87 | 88 | // interpolate between source and result by intensity 89 | "cResult = cTextureScreen.rgb + clamp( nIntensity, 0.0,1.0 ) * ( cResult - cTextureScreen.rgb );", 90 | 91 | // convert to grayscale if desired 92 | "if( grayscale ) {", 93 | 94 | "cResult = vec3( cResult.r * 0.3 + cResult.g * 0.59 + cResult.b * 0.11 );", 95 | 96 | "}", 97 | 98 | "gl_FragColor = vec4( cResult, cTextureScreen.a );", 99 | 100 | "}" 101 | 102 | ].join( "\n" ) 103 | 104 | }; 105 | -------------------------------------------------------------------------------- /torchplus/nn/modules/common.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from collections import OrderedDict 3 | 4 | import torch 5 | from torch.nn import functional as F 6 | 7 | class Empty(torch.nn.Module): 8 | def __init__(self, *args, **kwargs): 9 | super(Empty, self).__init__() 10 | 11 | def forward(self, *args, **kwargs): 12 | if len(args) == 1: 13 | return args[0] 14 | elif len(args) == 0: 15 | return None 16 | return args 17 | 18 | class Sequential(torch.nn.Module): 19 | r"""A sequential container. 20 | Modules will be added to it in the order they are passed in the constructor. 21 | Alternatively, an ordered dict of modules can also be passed in. 22 | 23 | To make it easier to understand, given is a small example:: 24 | 25 | # Example of using Sequential 26 | model = Sequential( 27 | nn.Conv2d(1,20,5), 28 | nn.ReLU(), 29 | nn.Conv2d(20,64,5), 30 | nn.ReLU() 31 | ) 32 | 33 | # Example of using Sequential with OrderedDict 34 | model = Sequential(OrderedDict([ 35 | ('conv1', nn.Conv2d(1,20,5)), 36 | ('relu1', nn.ReLU()), 37 | ('conv2', nn.Conv2d(20,64,5)), 38 | ('relu2', nn.ReLU()) 39 | ])) 40 | 41 | # Example of using Sequential with kwargs(python 3.6+) 42 | model = Sequential( 43 | conv1=nn.Conv2d(1,20,5), 44 | relu1=nn.ReLU(), 45 | conv2=nn.Conv2d(20,64,5), 46 | relu2=nn.ReLU() 47 | ) 48 | """ 49 | 50 | def __init__(self, *args, **kwargs): 51 | super(Sequential, self).__init__() 52 | if len(args) == 1 and isinstance(args[0], OrderedDict): 53 | for key, module in args[0].items(): 54 | self.add_module(key, module) 55 | else: 56 | for idx, module in enumerate(args): 57 | self.add_module(str(idx), module) 58 | for name, module in kwargs.items(): 59 | if sys.version_info < (3, 6): 60 | raise ValueError("kwargs only supported in py36+") 61 | if name in self._modules: 62 | raise ValueError("name exists.") 63 | self.add_module(name, module) 64 | 65 | def __getitem__(self, idx): 66 | if not (-len(self) <= idx < len(self)): 67 | raise IndexError('index {} is out of range'.format(idx)) 68 | if idx < 0: 69 | idx += len(self) 70 | it = iter(self._modules.values()) 71 | for i in range(idx): 72 | next(it) 73 | return next(it) 74 | 75 | def __len__(self): 76 | return len(self._modules) 77 | 78 | def add(self, module, name=None): 79 | if name is None: 80 | name = str(len(self._modules)) 81 | if name in self._modules: 82 | raise KeyError("name exists") 83 | self.add_module(name, module) 84 | 85 | def forward(self, input): 86 | # i = 0 87 | for module in self._modules.values(): 88 | # print(i) 89 | input = module(input) 90 | # i += 1 91 | return input -------------------------------------------------------------------------------- /second/pytorch/models/resnet.py: -------------------------------------------------------------------------------- 1 | import spconv 2 | from torch import nn 3 | from torch.nn import functional as F 4 | 5 | from torchplus.nn import Empty, GroupNorm, Sequential 6 | 7 | 8 | def conv3x3(in_planes, out_planes, stride=1, indice_key=None): 9 | """3x3 convolution with padding""" 10 | return spconv.SubMConv3d( 11 | in_planes, 12 | out_planes, 13 | kernel_size=3, 14 | stride=stride, 15 | padding=1, 16 | bias=False, 17 | indice_key=indice_key) 18 | 19 | 20 | def conv1x1(in_planes, out_planes, stride=1, indice_key=None): 21 | """1x1 convolution""" 22 | return spconv.SubMConv3d( 23 | in_planes, 24 | out_planes, 25 | kernel_size=1, 26 | stride=stride, 27 | padding=1, 28 | bias=False, 29 | indice_key=indice_key) 30 | 31 | 32 | class SparseBasicBlock(spconv.SparseModule): 33 | expansion = 1 34 | 35 | def __init__(self, 36 | inplanes, 37 | planes, 38 | stride=1, 39 | downsample=None, 40 | indice_key=None): 41 | super(SparseBasicBlock, self).__init__() 42 | self.conv1 = conv3x3(inplanes, planes, stride, indice_key=indice_key) 43 | self.bn1 = nn.BatchNorm1d(planes) 44 | self.relu = nn.ReLU() 45 | self.conv2 = conv3x3(planes, planes, indice_key=indice_key) 46 | self.bn2 = nn.BatchNorm1d(planes) 47 | self.downsample = downsample 48 | self.stride = stride 49 | 50 | def forward(self, x): 51 | identity = x 52 | 53 | out = self.conv1(x) 54 | out.features = self.bn1(out.features) 55 | out.features = self.relu(out.features) 56 | 57 | out = self.conv2(out) 58 | out.features = self.bn2(out.features) 59 | 60 | if self.downsample is not None: 61 | identity = self.downsample(x) 62 | 63 | out.features += identity.features 64 | out.features = self.relu(out.features) 65 | 66 | return out 67 | 68 | 69 | class SparseBottleneck(spconv.SparseModule): 70 | expansion = 4 71 | 72 | def __init__(self, 73 | inplanes, 74 | planes, 75 | stride=1, 76 | downsample=None, 77 | indice_key=None): 78 | super(SparseBottleneck, self).__init__() 79 | self.conv1 = conv1x1(inplanes, planes, indice_key=indice_key) 80 | self.bn1 = nn.BatchNorm1d(planes) 81 | self.conv2 = conv3x3(planes, planes, stride, indice_key=indice_key) 82 | self.bn2 = nn.BatchNorm1d(planes) 83 | self.conv3 = conv1x1( 84 | planes, planes * self.expansion, indice_key=indice_key) 85 | self.bn3 = nn.BatchNorm1d(planes * self.expansion) 86 | self.relu = nn.ReLU() 87 | self.downsample = downsample 88 | self.stride = stride 89 | 90 | def forward(self, x): 91 | identity = x 92 | 93 | out = self.conv1(x) 94 | out.features = self.bn1(out.features) 95 | out.features = self.relu(out.features) 96 | 97 | out = self.conv2(out) 98 | out.features = self.bn2(out.features) 99 | out.features = self.relu(out.features) 100 | 101 | out = self.conv3(out) 102 | out.features = self.bn3(out.features) 103 | 104 | if self.downsample is not None: 105 | identity = self.downsample(x) 106 | 107 | out += identity 108 | out.features = self.relu(out.features) 109 | 110 | return out 111 | -------------------------------------------------------------------------------- /second/pytorch/inference.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | import torch 5 | 6 | import torchplus 7 | from second.core import box_np_ops 8 | from second.core.inference import InferenceContext 9 | from second.builder import target_assigner_builder, voxel_builder 10 | from second.pytorch.builder import box_coder_builder, second_builder 11 | from second.pytorch.models.voxelnet import VoxelNet 12 | from second.pytorch.train import predict_to_kitti_label, example_convert_to_torch 13 | 14 | 15 | class TorchInferenceContext(InferenceContext): 16 | def __init__(self): 17 | super().__init__() 18 | self.net = None 19 | self.anchor_cache = None 20 | 21 | def _build(self): 22 | config = self.config 23 | input_cfg = config.eval_input_reader 24 | model_cfg = config.model.second 25 | train_cfg = config.train_config 26 | batch_size = 1 27 | voxel_generator = voxel_builder.build(model_cfg.voxel_generator) 28 | bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] 29 | grid_size = voxel_generator.grid_size 30 | self.voxel_generator = voxel_generator 31 | vfe_num_filters = list(model_cfg.voxel_feature_extractor.num_filters) 32 | 33 | box_coder = box_coder_builder.build(model_cfg.box_coder) 34 | target_assigner_cfg = model_cfg.target_assigner 35 | target_assigner = target_assigner_builder.build( 36 | target_assigner_cfg, bv_range, box_coder) 37 | self.target_assigner = target_assigner 38 | out_size_factor = model_cfg.rpn.layer_strides[0] / model_cfg.rpn.upsample_strides[0] 39 | out_size_factor *= model_cfg.middle_feature_extractor.downsample_factor 40 | out_size_factor = int(out_size_factor) 41 | self.net = second_builder.build(model_cfg, voxel_generator, 42 | target_assigner) 43 | self.net.cuda().eval() 44 | if train_cfg.enable_mixed_precision: 45 | self.net.half() 46 | self.net.metrics_to_float() 47 | self.net.convert_norm_to_float(self.net) 48 | feature_map_size = grid_size[:2] // out_size_factor 49 | feature_map_size = [*feature_map_size, 1][::-1] 50 | ret = target_assigner.generate_anchors(feature_map_size) 51 | anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) 52 | anchors = ret["anchors"] 53 | anchors = anchors.reshape([-1, 7]) 54 | matched_thresholds = ret["matched_thresholds"] 55 | unmatched_thresholds = ret["unmatched_thresholds"] 56 | anchors_bv = box_np_ops.rbbox2d_to_near_bbox( 57 | anchors[:, [0, 1, 3, 4, 6]]) 58 | anchor_cache = { 59 | "anchors": anchors, 60 | "anchors_bv": anchors_bv, 61 | "matched_thresholds": matched_thresholds, 62 | "unmatched_thresholds": unmatched_thresholds, 63 | "anchors_dict": anchors_dict, 64 | } 65 | self.anchor_cache = anchor_cache 66 | 67 | def _restore(self, ckpt_path): 68 | ckpt_path = Path(ckpt_path) 69 | assert ckpt_path.suffix == ".tckpt" 70 | torchplus.train.restore(str(ckpt_path), self.net) 71 | 72 | def _inference(self, example): 73 | train_cfg = self.config.train_config 74 | input_cfg = self.config.eval_input_reader 75 | model_cfg = self.config.model.second 76 | example_torch = example_convert_to_torch(example) 77 | result_annos = predict_to_kitti_label( 78 | self.net, example_torch, list( 79 | self.target_assigner.classes), 80 | model_cfg.post_center_limit_range, model_cfg.lidar_input) 81 | return result_annos 82 | 83 | def _ctx(self): 84 | return None 85 | -------------------------------------------------------------------------------- /second/utils/config_tool/train.py: -------------------------------------------------------------------------------- 1 | from second.protos.optimizer_pb2 import Optimizer, LearningRate, OneCycle, ManualStepping, ExponentialDecay 2 | from second.protos.sampler_pb2 import Sampler 3 | from second.utils.config_tool import read_config 4 | from pathlib import Path 5 | from google.protobuf import text_format 6 | from second.data.all_dataset import get_dataset_class 7 | 8 | def _get_optim_cfg(train_config, optim): 9 | if optim == "adam_optimizer": 10 | return train_config.optimizer.adam_optimizer 11 | elif optim == "rms_prop_optimizer": 12 | return train_config.optimizer.rms_prop_optimizer 13 | elif optim == "momentum_optimizer": 14 | return train_config.optimizer.momentum_optimizer 15 | else: 16 | raise NotImplementedError 17 | 18 | 19 | def manual_stepping(train_config, boundaries, rates, optim="adam_optimizer"): 20 | optim_cfg = _get_optim_cfg(train_config, optim) 21 | optim_cfg.learning_rate.manual_stepping.CopyFrom( 22 | ManualStepping(boundaries=boundaries, rates=rates)) 23 | 24 | 25 | def exp_decay(train_config, 26 | init_lr, 27 | decay_length, 28 | decay_factor, 29 | staircase=True, 30 | optim="adam_optimizer"): 31 | optim_cfg = _get_optim_cfg(train_config, optim) 32 | optim_cfg.learning_rate.exponential_decay.CopyFrom( 33 | ExponentialDecay( 34 | initial_learning_rate=init_lr, 35 | decay_length=decay_length, 36 | decay_factor=decay_factor, 37 | staircase=staircase)) 38 | 39 | 40 | def one_cycle(train_config, 41 | lr_max, 42 | moms, 43 | div_factor, 44 | pct_start, 45 | optim="adam_optimizer"): 46 | optim_cfg = _get_optim_cfg(train_config, optim) 47 | optim_cfg.learning_rate.one_cycle.CopyFrom( 48 | OneCycle( 49 | lr_max=lr_max, 50 | moms=moms, 51 | div_factor=div_factor, 52 | pct_start=pct_start)) 53 | 54 | def _div_up(a, b): 55 | return (a + b - 1) // b 56 | 57 | def set_train_step(config, 58 | epochs, 59 | eval_epoch): 60 | input_cfg = config.train_input_reader 61 | train_cfg = config.train_config 62 | batch_size = input_cfg.batch_size 63 | dataset_name = input_cfg.dataset.dataset_class_name 64 | ds = get_dataset_class(dataset_name)( 65 | root_path=input_cfg.dataset.kitti_root_path, 66 | info_path=input_cfg.dataset.kitti_info_path, 67 | ) 68 | num_examples_after_sample = len(ds) 69 | step_per_epoch = _div_up(num_examples_after_sample, batch_size) 70 | step_per_eval = step_per_epoch * eval_epoch 71 | total_step = step_per_epoch * epochs 72 | train_cfg.steps = total_step 73 | train_cfg.steps_per_eval = step_per_eval 74 | 75 | def disable_sample(config): 76 | input_cfg = config.train_input_reader 77 | input_cfg.database_sampler.CopyFrom(Sampler()) 78 | 79 | def disable_per_gt_aug(config): 80 | prep_cfg = config.train_input_reader.preprocess 81 | prep_cfg.groundtruth_localization_noise_std[:] = [0, 0, 0] 82 | prep_cfg.groundtruth_rotation_uniform_noise[:] = [0, 0] 83 | 84 | def disable_global_aug(config): 85 | prep_cfg = config.train_input_reader.preprocess 86 | prep_cfg.global_rotation_uniform_noise[:] = [0, 0] 87 | prep_cfg.global_scaling_uniform_noise[:] = [0, 0] 88 | prep_cfg.global_random_rotation_range_per_object[:] = [0, 0] 89 | prep_cfg.global_translate_noise_std[:] = [0, 0, 0] 90 | 91 | if __name__ == "__main__": 92 | path = Path(__file__).resolve().parents[2] / "configs/car.lite.config" 93 | config = read_config(path) 94 | manual_stepping(config.train_config, [0.8, 0.9], [1e-4, 1e-5, 1e-6]) 95 | 96 | print(text_format.MessageToString(config, indent=2)) -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # From https://github.com/ufoym/deepo/blob/master/docker/Dockerfile.pytorch-py36-cu90 2 | 3 | # ================================================================== 4 | # module list 5 | # ------------------------------------------------------------------ 6 | # python 3.6 (apt) 7 | # pytorch latest (pip) 8 | # ================================================================== 9 | 10 | FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 11 | RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ 12 | PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ 13 | GIT_CLONE="git clone --depth 10" && \ 14 | rm -rf /var/lib/apt/lists/* \ 15 | /etc/apt/sources.list.d/cuda.list \ 16 | /etc/apt/sources.list.d/nvidia-ml.list && \ 17 | apt-get update && \ 18 | # ================================================================== 19 | # tools 20 | # ------------------------------------------------------------------ 21 | DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ 22 | build-essential \ 23 | ca-certificates \ 24 | cmake \ 25 | wget \ 26 | git \ 27 | vim \ 28 | fish \ 29 | libsparsehash-dev \ 30 | && \ 31 | # ================================================================== 32 | # python 33 | # ------------------------------------------------------------------ 34 | DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ 35 | software-properties-common \ 36 | && \ 37 | add-apt-repository ppa:deadsnakes/ppa && \ 38 | apt-get update && \ 39 | DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ 40 | python3.6 \ 41 | python3.6-dev \ 42 | && \ 43 | wget -O ~/get-pip.py \ 44 | https://bootstrap.pypa.io/get-pip.py && \ 45 | python3.6 ~/get-pip.py && \ 46 | ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \ 47 | ln -s /usr/bin/python3.6 /usr/local/bin/python && \ 48 | $PIP_INSTALL \ 49 | setuptools \ 50 | && \ 51 | $PIP_INSTALL \ 52 | numpy \ 53 | scipy \ 54 | matplotlib \ 55 | Cython \ 56 | && \ 57 | # ================================================================== 58 | # pytorch 59 | # ------------------------------------------------------------------ 60 | $PIP_INSTALL \ 61 | torch_nightly -f \ 62 | https://download.pytorch.org/whl/nightly/cu90/torch_nightly.html \ 63 | && \ 64 | $PIP_INSTALL \ 65 | torchvision_nightly \ 66 | && \ 67 | # ================================================================== 68 | # config & cleanup 69 | # ------------------------------------------------------------------ 70 | ldconfig && \ 71 | apt-get clean && \ 72 | apt-get autoremove && \ 73 | rm -rf /var/lib/apt/lists/* /tmp/* ~/* 74 | 75 | RUN PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ 76 | $PIP_INSTALL \ 77 | shapely fire pybind11 tensorboardX protobuf \ 78 | scikit-image numba pillow 79 | 80 | WORKDIR /root 81 | RUN wget https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz 82 | RUN tar xzvf boost_1_68_0.tar.gz 83 | RUN cp -r ./boost_1_68_0/boost /usr/include 84 | RUN rm -rf ./boost_1_68_0 85 | RUN rm -rf ./boost_1_68_0.tar.gz 86 | RUN git clone https://github.com/traveller59/second.pytorch.git --depth 10 87 | RUN git clone https://github.com/traveller59/SparseConvNet.git --depth 10 88 | RUN cd ./SparseConvNet && python setup.py install && cd .. && rm -rf SparseConvNet 89 | ENV NUMBAPRO_CUDA_DRIVER=/usr/lib/x86_64-linux-gnu/libcuda.so 90 | ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so 91 | ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice 92 | ENV PYTHONPATH=/root/second.pytorch 93 | 94 | VOLUME ["/root/data"] 95 | VOLUME ["/root/model"] 96 | WORKDIR /root/second.pytorch/second 97 | 98 | ENTRYPOINT ["fish"] 99 | -------------------------------------------------------------------------------- /second/pytorch/builder/lr_scheduler_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Functions to build DetectionModel training optimizers.""" 17 | 18 | from torchplus.train import learning_schedules_fastai as lsf 19 | import torch 20 | 21 | def build(optimizer_config, optimizer, total_step): 22 | """Create lr scheduler based on config. note that 23 | lr_scheduler must accept a optimizer that has been restored. 24 | 25 | Args: 26 | optimizer_config: A Optimizer proto message. 27 | 28 | Returns: 29 | An optimizer and a list of variables for summary. 30 | 31 | Raises: 32 | ValueError: when using an unsupported input data type. 33 | """ 34 | optimizer_type = optimizer_config.WhichOneof('optimizer') 35 | 36 | if optimizer_type == 'rms_prop_optimizer': 37 | config = optimizer_config.rms_prop_optimizer 38 | lr_scheduler = _create_learning_rate_scheduler( 39 | config.learning_rate, optimizer, total_step=total_step) 40 | 41 | if optimizer_type == 'momentum_optimizer': 42 | config = optimizer_config.momentum_optimizer 43 | lr_scheduler = _create_learning_rate_scheduler( 44 | config.learning_rate, optimizer, total_step=total_step) 45 | 46 | if optimizer_type == 'adam_optimizer': 47 | config = optimizer_config.adam_optimizer 48 | lr_scheduler = _create_learning_rate_scheduler( 49 | config.learning_rate, optimizer, total_step=total_step) 50 | 51 | return lr_scheduler 52 | 53 | def _create_learning_rate_scheduler(learning_rate_config, optimizer, total_step): 54 | """Create optimizer learning rate scheduler based on config. 55 | 56 | Args: 57 | learning_rate_config: A LearningRate proto message. 58 | 59 | Returns: 60 | A learning rate. 61 | 62 | Raises: 63 | ValueError: when using an unsupported input data type. 64 | """ 65 | lr_scheduler = None 66 | learning_rate_type = learning_rate_config.WhichOneof('learning_rate') 67 | if learning_rate_type == 'multi_phase': 68 | config = learning_rate_config.multi_phase 69 | lr_phases = [] 70 | mom_phases = [] 71 | for phase_cfg in config.phases: 72 | lr_phases.append((phase_cfg.start, phase_cfg.lambda_func)) 73 | mom_phases.append((phase_cfg.start, phase_cfg.momentum_lambda_func)) 74 | lr_scheduler = lsf.LRSchedulerStep( 75 | optimizer,total_step, lr_phases, mom_phases) 76 | 77 | if learning_rate_type == 'one_cycle': 78 | config = learning_rate_config.one_cycle 79 | lr_scheduler = lsf.OneCycle( 80 | optimizer, total_step, config.lr_max, list(config.moms), config.div_factor, config.pct_start) 81 | if learning_rate_type == 'exponential_decay': 82 | config = learning_rate_config.exponential_decay 83 | lr_scheduler = lsf.ExponentialDecay( 84 | optimizer, total_step, config.initial_learning_rate, config.decay_length, config.decay_factor, config.staircase) 85 | if learning_rate_type == 'manual_stepping': 86 | config = learning_rate_config.manual_stepping 87 | lr_scheduler = lsf.ManualStepping( 88 | optimizer, total_step, list(config.boundaries), list(config.rates)) 89 | 90 | if lr_scheduler is None: 91 | raise ValueError('Learning_rate %s not supported.' % learning_rate_type) 92 | 93 | return lr_scheduler -------------------------------------------------------------------------------- /second/pytorch/builder/optimizer_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Functions to build DetectionModel training optimizers.""" 16 | 17 | from torchplus.train import learning_schedules 18 | from torchplus.train import optim 19 | import torch 20 | from torch import nn 21 | from torchplus.train.fastai_optim import OptimWrapper, FastAIMixedOptim 22 | from functools import partial 23 | 24 | def children(m: nn.Module): 25 | "Get children of `m`." 26 | return list(m.children()) 27 | 28 | 29 | def num_children(m: nn.Module) -> int: 30 | "Get number of children modules in `m`." 31 | return len(children(m)) 32 | 33 | flatten_model = lambda m: sum(map(flatten_model,m.children()),[]) if num_children(m) else [m] 34 | 35 | get_layer_groups = lambda m: [nn.Sequential(*flatten_model(m))] 36 | 37 | 38 | def build(optimizer_config, net, name=None, mixed=False, loss_scale=512.0): 39 | """Create optimizer based on config. 40 | 41 | Args: 42 | optimizer_config: A Optimizer proto message. 43 | 44 | Returns: 45 | An optimizer and a list of variables for summary. 46 | 47 | Raises: 48 | ValueError: when using an unsupported input data type. 49 | """ 50 | optimizer_type = optimizer_config.WhichOneof('optimizer') 51 | optimizer = None 52 | 53 | if optimizer_type == 'rms_prop_optimizer': 54 | config = optimizer_config.rms_prop_optimizer 55 | optimizer_func = partial( 56 | torch.optim.RMSprop, 57 | alpha=config.decay, 58 | momentum=config.momentum_optimizer_value, 59 | eps=config.epsilon) 60 | 61 | if optimizer_type == 'momentum_optimizer': 62 | config = optimizer_config.momentum_optimizer 63 | optimizer_func = partial( 64 | torch.optim.SGD, 65 | momentum=config.momentum_optimizer_value, 66 | eps=config.epsilon) 67 | 68 | if optimizer_type == 'adam_optimizer': 69 | config = optimizer_config.adam_optimizer 70 | if optimizer_config.fixed_weight_decay: 71 | optimizer_func = partial( 72 | torch.optim.Adam, betas=(0.9, 0.99), amsgrad=config.amsgrad) 73 | else: 74 | # regular adam 75 | optimizer_func = partial( 76 | torch.optim.Adam, amsgrad=config.amsgrad) 77 | 78 | 79 | 80 | # optimizer = OptimWrapper(optimizer, true_wd=optimizer_config.fixed_weight_decay, wd=config.weight_decay) 81 | optimizer = OptimWrapper.create( 82 | optimizer_func, 83 | 3e-3, 84 | get_layer_groups(net), 85 | wd=config.weight_decay, 86 | true_wd=optimizer_config.fixed_weight_decay, 87 | bn_wd=True) 88 | print(hasattr(optimizer, "_amp_stash"), '_amp_stash') 89 | if optimizer is None: 90 | raise ValueError('Optimizer %s not supported.' % optimizer_type) 91 | 92 | if optimizer_config.use_moving_average: 93 | raise ValueError('torch don\'t support moving average') 94 | if name is None: 95 | # assign a name to optimizer for checkpoint system 96 | optimizer.name = optimizer_type 97 | else: 98 | optimizer.name = name 99 | return optimizer 100 | -------------------------------------------------------------------------------- /second/core/anchor_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from second.core import box_np_ops 3 | 4 | class AnchorGenerator: 5 | @property 6 | def class_name(self): 7 | raise NotImplementedError 8 | 9 | @property 10 | def num_anchors_per_localization(self): 11 | raise NotImplementedError 12 | 13 | def generate(self, feature_map_size): 14 | raise NotImplementedError 15 | 16 | @property 17 | def ndim(self): 18 | raise NotImplementedError 19 | 20 | 21 | class AnchorGeneratorStride(AnchorGenerator): 22 | def __init__(self, 23 | sizes=[1.6, 3.9, 1.56], 24 | anchor_strides=[0.4, 0.4, 1.0], 25 | anchor_offsets=[0.2, -39.8, -1.78], 26 | rotations=[0, np.pi / 2], 27 | class_name=None, 28 | match_threshold=-1, 29 | unmatch_threshold=-1, 30 | custom_values=(), 31 | dtype=np.float32): 32 | super().__init__() 33 | self._sizes = sizes 34 | self._anchor_strides = anchor_strides 35 | self._anchor_offsets = anchor_offsets 36 | self._rotations = rotations 37 | self._dtype = dtype 38 | self._class_name = class_name 39 | self.match_threshold = match_threshold 40 | self.unmatch_threshold = unmatch_threshold 41 | self._custom_values = custom_values 42 | 43 | @property 44 | def class_name(self): 45 | return self._class_name 46 | 47 | @property 48 | def num_anchors_per_localization(self): 49 | num_rot = len(self._rotations) 50 | num_size = np.array(self._sizes).reshape([-1, 3]).shape[0] 51 | return num_rot * num_size 52 | 53 | def generate(self, feature_map_size): 54 | res = box_np_ops.create_anchors_3d_stride( 55 | feature_map_size, self._sizes, self._anchor_strides, 56 | self._anchor_offsets, self._rotations, self._dtype) 57 | if len(self._custom_values) > 0: 58 | custom_ndim = len(self._custom_values) 59 | custom = np.zeros([*res.shape[:-1], custom_ndim]) 60 | custom[:] = self._custom_values 61 | res = np.concatenate([res, custom], axis=-1) 62 | return res 63 | 64 | @property 65 | def ndim(self): 66 | return 7 + len(self._custom_values) 67 | 68 | @property 69 | def custom_ndim(self): 70 | return len(self._custom_values) 71 | 72 | class AnchorGeneratorRange(AnchorGenerator): 73 | def __init__(self, 74 | anchor_ranges, 75 | sizes=[1.6, 3.9, 1.56], 76 | rotations=[0, np.pi / 2], 77 | class_name=None, 78 | match_threshold=-1, 79 | unmatch_threshold=-1, 80 | custom_values=(), 81 | dtype=np.float32): 82 | super().__init__() 83 | self._sizes = sizes 84 | self._anchor_ranges = anchor_ranges 85 | self._rotations = rotations 86 | self._dtype = dtype 87 | self._class_name = class_name 88 | self.match_threshold = match_threshold 89 | self.unmatch_threshold = unmatch_threshold 90 | self._custom_values = custom_values 91 | 92 | @property 93 | def class_name(self): 94 | return self._class_name 95 | 96 | @property 97 | def num_anchors_per_localization(self): 98 | num_rot = len(self._rotations) 99 | num_size = np.array(self._sizes).reshape([-1, 3]).shape[0] 100 | return num_rot * num_size 101 | 102 | def generate(self, feature_map_size): 103 | res = box_np_ops.create_anchors_3d_range( 104 | feature_map_size, self._anchor_ranges, self._sizes, 105 | self._rotations, self._dtype) 106 | 107 | if len(self._custom_values) > 0: 108 | custom_ndim = len(self._custom_values) 109 | custom = np.zeros([*res.shape[:-1], custom_ndim]) 110 | custom[:] = self._custom_values 111 | res = np.concatenate([res, custom], axis=-1) 112 | return res 113 | 114 | @property 115 | def ndim(self): 116 | return 7 + len(self._custom_values) 117 | 118 | @property 119 | def custom_ndim(self): 120 | return len(self._custom_values) -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/BloomPass.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.BloomPass = function ( strength, kernelSize, sigma, resolution ) { 6 | 7 | THREE.Pass.call( this ); 8 | 9 | strength = ( strength !== undefined ) ? strength : 1; 10 | kernelSize = ( kernelSize !== undefined ) ? kernelSize : 25; 11 | sigma = ( sigma !== undefined ) ? sigma : 4.0; 12 | resolution = ( resolution !== undefined ) ? resolution : 256; 13 | 14 | // render targets 15 | 16 | var pars = { minFilter: THREE.LinearFilter, magFilter: THREE.LinearFilter, format: THREE.RGBAFormat }; 17 | 18 | this.renderTargetX = new THREE.WebGLRenderTarget( resolution, resolution, pars ); 19 | this.renderTargetX.texture.name = "BloomPass.x"; 20 | this.renderTargetY = new THREE.WebGLRenderTarget( resolution, resolution, pars ); 21 | this.renderTargetY.texture.name = "BloomPass.y"; 22 | 23 | // copy material 24 | 25 | if ( THREE.CopyShader === undefined ) 26 | console.error( "THREE.BloomPass relies on THREE.CopyShader" ); 27 | 28 | var copyShader = THREE.CopyShader; 29 | 30 | this.copyUniforms = THREE.UniformsUtils.clone( copyShader.uniforms ); 31 | 32 | this.copyUniforms[ "opacity" ].value = strength; 33 | 34 | this.materialCopy = new THREE.ShaderMaterial( { 35 | 36 | uniforms: this.copyUniforms, 37 | vertexShader: copyShader.vertexShader, 38 | fragmentShader: copyShader.fragmentShader, 39 | blending: THREE.AdditiveBlending, 40 | transparent: true 41 | 42 | } ); 43 | 44 | // convolution material 45 | 46 | if ( THREE.ConvolutionShader === undefined ) 47 | console.error( "THREE.BloomPass relies on THREE.ConvolutionShader" ); 48 | 49 | var convolutionShader = THREE.ConvolutionShader; 50 | 51 | this.convolutionUniforms = THREE.UniformsUtils.clone( convolutionShader.uniforms ); 52 | 53 | this.convolutionUniforms[ "uImageIncrement" ].value = THREE.BloomPass.blurX; 54 | this.convolutionUniforms[ "cKernel" ].value = THREE.ConvolutionShader.buildKernel( sigma ); 55 | 56 | this.materialConvolution = new THREE.ShaderMaterial( { 57 | 58 | uniforms: this.convolutionUniforms, 59 | vertexShader: convolutionShader.vertexShader, 60 | fragmentShader: convolutionShader.fragmentShader, 61 | defines: { 62 | "KERNEL_SIZE_FLOAT": kernelSize.toFixed( 1 ), 63 | "KERNEL_SIZE_INT": kernelSize.toFixed( 0 ) 64 | } 65 | 66 | } ); 67 | 68 | this.needsSwap = false; 69 | 70 | this.camera = new THREE.OrthographicCamera( - 1, 1, 1, - 1, 0, 1 ); 71 | this.scene = new THREE.Scene(); 72 | 73 | this.quad = new THREE.Mesh( new THREE.PlaneBufferGeometry( 2, 2 ), null ); 74 | this.quad.frustumCulled = false; // Avoid getting clipped 75 | this.scene.add( this.quad ); 76 | 77 | }; 78 | 79 | THREE.BloomPass.prototype = Object.assign( Object.create( THREE.Pass.prototype ), { 80 | 81 | constructor: THREE.BloomPass, 82 | 83 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 84 | 85 | if ( maskActive ) renderer.context.disable( renderer.context.STENCIL_TEST ); 86 | 87 | // Render quad with blured scene into texture (convolution pass 1) 88 | 89 | this.quad.material = this.materialConvolution; 90 | 91 | this.convolutionUniforms[ "tDiffuse" ].value = readBuffer.texture; 92 | this.convolutionUniforms[ "uImageIncrement" ].value = THREE.BloomPass.blurX; 93 | 94 | renderer.render( this.scene, this.camera, this.renderTargetX, true ); 95 | 96 | 97 | // Render quad with blured scene into texture (convolution pass 2) 98 | 99 | this.convolutionUniforms[ "tDiffuse" ].value = this.renderTargetX.texture; 100 | this.convolutionUniforms[ "uImageIncrement" ].value = THREE.BloomPass.blurY; 101 | 102 | renderer.render( this.scene, this.camera, this.renderTargetY, true ); 103 | 104 | // Render original scene with superimposed blur to texture 105 | 106 | this.quad.material = this.materialCopy; 107 | 108 | this.copyUniforms[ "tDiffuse" ].value = this.renderTargetY.texture; 109 | 110 | if ( maskActive ) renderer.context.enable( renderer.context.STENCIL_TEST ); 111 | 112 | renderer.render( this.scene, this.camera, readBuffer, this.clear ); 113 | 114 | } 115 | 116 | } ); 117 | 118 | THREE.BloomPass.blurX = new THREE.Vector2( 0.001953125, 0.0 ); 119 | THREE.BloomPass.blurY = new THREE.Vector2( 0.0, 0.001953125 ); 120 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/renderers/CSS2DRenderer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author mrdoob / http://mrdoob.com/ 3 | */ 4 | 5 | THREE.CSS2DObject = function ( element ) { 6 | 7 | THREE.Object3D.call( this ); 8 | 9 | this.element = element; 10 | this.element.style.position = 'absolute'; 11 | 12 | this.addEventListener( 'removed', function ( event ) { 13 | 14 | if ( this.element.parentNode !== null ) { 15 | 16 | this.element.parentNode.removeChild( this.element ); 17 | 18 | } 19 | 20 | } ); 21 | 22 | }; 23 | 24 | THREE.CSS2DObject.prototype = Object.create( THREE.Object3D.prototype ); 25 | THREE.CSS2DObject.prototype.constructor = THREE.CSS2DObject; 26 | 27 | // 28 | 29 | THREE.CSS2DRenderer = function () { 30 | 31 | console.log( 'THREE.CSS2DRenderer', THREE.REVISION ); 32 | 33 | var _width, _height; 34 | var _widthHalf, _heightHalf; 35 | 36 | var vector = new THREE.Vector3(); 37 | var viewMatrix = new THREE.Matrix4(); 38 | var viewProjectionMatrix = new THREE.Matrix4(); 39 | 40 | var cache = { 41 | objects: new WeakMap() 42 | }; 43 | 44 | var domElement = document.createElement( 'div' ); 45 | domElement.style.overflow = 'hidden'; 46 | 47 | this.domElement = domElement; 48 | 49 | this.getSize = function () { 50 | 51 | return { 52 | width: _width, 53 | height: _height 54 | }; 55 | 56 | }; 57 | 58 | this.setSize = function ( width, height ) { 59 | 60 | _width = width; 61 | _height = height; 62 | 63 | _widthHalf = _width / 2; 64 | _heightHalf = _height / 2; 65 | 66 | domElement.style.width = width + 'px'; 67 | domElement.style.height = height + 'px'; 68 | 69 | }; 70 | 71 | var renderObject = function ( object, camera ) { 72 | 73 | if ( object instanceof THREE.CSS2DObject ) { 74 | 75 | vector.setFromMatrixPosition( object.matrixWorld ); 76 | vector.applyMatrix4( viewProjectionMatrix ); 77 | 78 | var element = object.element; 79 | var style = 'translate(-50%,-50%) translate(' + ( vector.x * _widthHalf + _widthHalf ) + 'px,' + ( - vector.y * _heightHalf + _heightHalf ) + 'px)'; 80 | 81 | element.style.WebkitTransform = style; 82 | element.style.MozTransform = style; 83 | element.style.oTransform = style; 84 | element.style.transform = style; 85 | 86 | var objectData = { 87 | distanceToCameraSquared: getDistanceToSquared( camera, object ) 88 | }; 89 | 90 | cache.objects.set( object, objectData ); 91 | 92 | if ( element.parentNode !== domElement ) { 93 | 94 | domElement.appendChild( element ); 95 | 96 | } 97 | 98 | } 99 | 100 | for ( var i = 0, l = object.children.length; i < l; i ++ ) { 101 | 102 | renderObject( object.children[ i ], camera ); 103 | 104 | } 105 | 106 | }; 107 | 108 | var getDistanceToSquared = function () { 109 | 110 | var a = new THREE.Vector3(); 111 | var b = new THREE.Vector3(); 112 | 113 | return function ( object1, object2 ) { 114 | 115 | a.setFromMatrixPosition( object1.matrixWorld ); 116 | b.setFromMatrixPosition( object2.matrixWorld ); 117 | 118 | return a.distanceToSquared( b ); 119 | 120 | }; 121 | 122 | }(); 123 | 124 | var filterAndFlatten = function ( scene ) { 125 | 126 | var result = []; 127 | 128 | scene.traverse( function ( object ) { 129 | 130 | if ( object instanceof THREE.CSS2DObject ) result.push( object ); 131 | 132 | } ); 133 | 134 | return result; 135 | 136 | }; 137 | 138 | var zOrder = function ( scene ) { 139 | 140 | var sorted = filterAndFlatten( scene ).sort( function ( a, b ) { 141 | 142 | var distanceA = cache.objects.get( a ).distanceToCameraSquared; 143 | var distanceB = cache.objects.get( b ).distanceToCameraSquared; 144 | 145 | return distanceA - distanceB; 146 | 147 | } ); 148 | 149 | var zMax = sorted.length; 150 | 151 | for ( var i = 0, l = sorted.length; i < l; i ++ ) { 152 | 153 | sorted[ i ].element.style.zIndex = zMax - i; 154 | 155 | } 156 | 157 | }; 158 | 159 | this.render = function ( scene, camera ) { 160 | 161 | scene.updateMatrixWorld(); 162 | 163 | if ( camera.parent === null ) camera.updateMatrixWorld(); 164 | 165 | viewMatrix.copy( camera.matrixWorldInverse ); 166 | viewProjectionMatrix.multiplyMatrices( camera.projectionMatrix, viewMatrix ); 167 | 168 | renderObject( scene, camera ); 169 | zOrder( scene ); 170 | 171 | }; 172 | 173 | }; 174 | -------------------------------------------------------------------------------- /second/data/dataset.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import pickle 3 | import time 4 | from functools import partial 5 | 6 | import numpy as np 7 | 8 | from second.core import box_np_ops 9 | from second.core import preprocess as prep 10 | from second.data import kitti_common as kitti 11 | 12 | REGISTERED_DATASET_CLASSES = {} 13 | 14 | def register_dataset(cls, name=None): 15 | global REGISTERED_DATASET_CLASSES 16 | if name is None: 17 | name = cls.__name__ 18 | assert name not in REGISTERED_DATASET_CLASSES, f"exist class: {REGISTERED_DATASET_CLASSES}" 19 | REGISTERED_DATASET_CLASSES[name] = cls 20 | return cls 21 | 22 | def get_dataset_class(name): 23 | #import pdb; pdb.set_trace() 24 | global REGISTERED_DATASET_CLASSES 25 | assert name in REGISTERED_DATASET_CLASSES, f"available class: {REGISTERED_DATASET_CLASSES}" 26 | return REGISTERED_DATASET_CLASSES[name] 27 | 28 | 29 | class Dataset(object): 30 | """An abstract class representing a pytorch-like Dataset. 31 | All other datasets should subclass it. All subclasses should override 32 | ``__len__``, that provides the size of the dataset, and ``__getitem__``, 33 | supporting integer indexing in range from 0 to len(self) exclusive. 34 | """ 35 | NumPointFeatures = -1 36 | def __getitem__(self, index): 37 | """This function is used for preprocess. 38 | you need to create a input dict in this function for network inference. 39 | format: { 40 | anchors 41 | voxels 42 | num_points 43 | coordinates 44 | if training: 45 | labels 46 | reg_targets 47 | [optional]anchors_mask, slow in SECOND v1.5, don't use this. 48 | [optional]metadata, in kitti, image index is saved in metadata 49 | } 50 | """ 51 | raise NotImplementedError 52 | 53 | def __len__(self): 54 | raise NotImplementedError 55 | 56 | def get_sensor_data(self, query): 57 | """Dataset must provide a unified function to get data. 58 | Args: 59 | query: int or dict. this param must support int for training. 60 | if dict, should have this format (no example yet): 61 | { 62 | sensor_name: { 63 | sensor_meta 64 | } 65 | } 66 | if int, will return all sensor data. 67 | (TODO: how to deal with unsynchronized data?) 68 | Returns: 69 | sensor_data: dict. 70 | if query is int (return all), return a dict with all sensors: 71 | { 72 | sensor_name: sensor_data 73 | ... 74 | metadata: ... (for kitti, contains image_idx) 75 | } 76 | 77 | if sensor is lidar (all lidar point cloud must be concatenated to one array): 78 | e.g. If your dataset have two lidar sensor, you need to return a single dict: 79 | { 80 | "lidar": { 81 | "points": ... 82 | ... 83 | } 84 | } 85 | sensor_data: { 86 | points: [N, 3+] 87 | [optional]annotations: { 88 | "boxes": [N, 7] locs, dims, yaw, in lidar coord system. must tested 89 | in provided visualization tools such as second.utils.simplevis 90 | or web tool. 91 | "names": array of string. 92 | } 93 | } 94 | if sensor is camera (not used yet): 95 | sensor_data: { 96 | data: image string (array is too large) 97 | [optional]annotations: { 98 | "boxes": [N, 4] 2d bbox 99 | "names": array of string. 100 | } 101 | } 102 | metadata: { 103 | # dataset-specific information. 104 | # for kitti, must have image_idx for label file generation. 105 | image_idx: ... 106 | } 107 | [optional]calib # only used for kitti 108 | """ 109 | raise NotImplementedError 110 | 111 | def evaluation(self, dt_annos, output_dir): 112 | """Dataset must provide a evaluation function to evaluate model.""" 113 | raise NotImplementedError 114 | -------------------------------------------------------------------------------- /second/utils/config_tool/__init__.py: -------------------------------------------------------------------------------- 1 | # This file contains some config modification function. 2 | # some functions should be only used for KITTI dataset. 3 | 4 | from google.protobuf import text_format 5 | from second.protos import pipeline_pb2, second_pb2 6 | from pathlib import Path 7 | import numpy as np 8 | 9 | def read_config(path): 10 | config = pipeline_pb2.TrainEvalPipelineConfig() 11 | 12 | with open(path, "r") as f: 13 | proto_str = f.read() 14 | text_format.Merge(proto_str, config) 15 | return config 16 | 17 | def change_detection_range(model_config, new_range): 18 | assert len(new_range) == 4, "you must provide a list such as [-50, -50, 50, 50]" 19 | old_pc_range = list(model_config.voxel_generator.point_cloud_range) 20 | old_pc_range[:2] = new_range[:2] 21 | old_pc_range[3:5] = new_range[2:] 22 | model_config.voxel_generator.point_cloud_range[:] = old_pc_range 23 | for anchor_generator in model_config.target_assigner.anchor_generators: 24 | a_type = anchor_generator.WhichOneof('anchor_generator') 25 | if a_type == "anchor_generator_range": 26 | a_cfg = anchor_generator.anchor_generator_range 27 | old_a_range = list(a_cfg.anchor_ranges) 28 | old_a_range[:2] = new_range[:2] 29 | old_a_range[3:5] = new_range[2:] 30 | a_cfg.anchor_ranges[:] = old_a_range 31 | elif a_type == "anchor_generator_stride": 32 | a_cfg = anchor_generator.anchor_generator_stride 33 | old_offset = list(a_cfg.offsets) 34 | stride = list(a_cfg.strides) 35 | old_offset[0] = new_range[0] + stride[0] / 2 36 | old_offset[1] = new_range[1] + stride[1] / 2 37 | a_cfg.offsets[:] = old_offset 38 | else: 39 | raise ValueError("unknown") 40 | old_post_range = list(model_config.post_center_limit_range) 41 | old_post_range[:2] = new_range[:2] 42 | old_post_range[3:5] = new_range[2:] 43 | model_config.post_center_limit_range[:] = old_post_range 44 | 45 | 46 | def change_detection_range_v2(model_config, new_range): 47 | assert len(new_range) == 4, "you must provide a list such as [-50, -50, 50, 50]" 48 | old_pc_range = list(model_config.voxel_generator.point_cloud_range) 49 | old_pc_range[:2] = new_range[:2] 50 | old_pc_range[3:5] = new_range[2:] 51 | model_config.voxel_generator.point_cloud_range[:] = old_pc_range 52 | for anchor_generator in model_config.target_assigner.class_settings: 53 | a_type = anchor_generator.WhichOneof('anchor_generator') 54 | if a_type == "anchor_generator_range": 55 | a_cfg = anchor_generator.anchor_generator_range 56 | old_a_range = list(a_cfg.anchor_ranges) 57 | old_a_range[:2] = new_range[:2] 58 | old_a_range[3:5] = new_range[2:] 59 | a_cfg.anchor_ranges[:] = old_a_range 60 | elif a_type == "anchor_generator_stride": 61 | a_cfg = anchor_generator.anchor_generator_stride 62 | old_offset = list(a_cfg.offsets) 63 | stride = list(a_cfg.strides) 64 | old_offset[0] = new_range[0] + stride[0] / 2 65 | old_offset[1] = new_range[1] + stride[1] / 2 66 | a_cfg.offsets[:] = old_offset 67 | else: 68 | raise ValueError("unknown") 69 | old_post_range = list(model_config.post_center_limit_range) 70 | old_post_range[:2] = new_range[:2] 71 | old_post_range[3:5] = new_range[2:] 72 | model_config.post_center_limit_range[:] = old_post_range 73 | 74 | 75 | 76 | 77 | def get_downsample_factor(model_config): 78 | downsample_factor = np.prod(model_config.rpn.layer_strides) 79 | if len(model_config.rpn.upsample_strides) > 0: 80 | downsample_factor /= model_config.rpn.upsample_strides[-1] 81 | downsample_factor *= model_config.middle_feature_extractor.downsample_factor 82 | downsample_factor = np.round(downsample_factor).astype(np.int64) 83 | assert downsample_factor > 0 84 | return downsample_factor 85 | 86 | 87 | if __name__ == "__main__": 88 | config_path = "/home/yy/deeplearning/deeplearning/mypackages/second/configs/car.lite.1.config" 89 | config = pipeline_pb2.TrainEvalPipelineConfig() 90 | 91 | with open(config_path, "r") as f: 92 | proto_str = f.read() 93 | text_format.Merge(proto_str, config) 94 | 95 | change_detection_range(config, [-50, -50, 50, 50]) 96 | proto_str = text_format.MessageToString(config, indent=2) 97 | print(proto_str) 98 | 99 | -------------------------------------------------------------------------------- /second/core/inference.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import contextlib 3 | 4 | import numpy as np 5 | from google.protobuf import text_format 6 | 7 | from second.data.preprocess import merge_second_batch, prep_pointcloud 8 | from second.protos import pipeline_pb2 9 | import second.data.kitti_common as kitti 10 | 11 | class InferenceContext: 12 | def __init__(self): 13 | self.config = None 14 | self.root_path = None 15 | self.target_assigner = None 16 | self.voxel_generator = None 17 | self.anchor_cache = None 18 | self.built = False 19 | 20 | def get_inference_input_dict(self, info, points): 21 | assert self.anchor_cache is not None 22 | assert self.target_assigner is not None 23 | assert self.voxel_generator is not None 24 | assert self.config is not None 25 | assert self.built is True 26 | kitti.convert_to_kitti_info_version2(info) 27 | pc_info = info["point_cloud"] 28 | image_info = info["image"] 29 | calib = info["calib"] 30 | 31 | rect = calib['R0_rect'] 32 | Trv2c = calib['Tr_velo_to_cam'] 33 | P2 = calib['P2'] 34 | 35 | input_cfg = self.config.eval_input_reader 36 | model_cfg = self.config.model.second 37 | 38 | input_dict = { 39 | 'points': points, 40 | "calib": { 41 | 'rect': rect, 42 | 'Trv2c': Trv2c, 43 | 'P2': P2, 44 | }, 45 | "image": { 46 | 'image_shape': np.array(image_info["image_shape"], dtype=np.int32), 47 | 'image_idx': image_info['image_idx'], 48 | 'image_path': image_info['image_path'], 49 | }, 50 | } 51 | out_size_factor = np.prod(model_cfg.rpn.layer_strides) 52 | if len(model_cfg.rpn.upsample_strides) > 0: 53 | out_size_factor /= model_cfg.rpn.upsample_strides[-1] 54 | out_size_factor *= model_cfg.middle_feature_extractor.downsample_factor 55 | out_size_factor = int(out_size_factor) 56 | example = prep_pointcloud( 57 | input_dict=input_dict, 58 | root_path=str(self.root_path), 59 | voxel_generator=self.voxel_generator, 60 | target_assigner=self.target_assigner, 61 | max_voxels=input_cfg.max_number_of_voxels, 62 | class_names=self.target_assigner.classes, 63 | training=False, 64 | create_targets=False, 65 | shuffle_points=input_cfg.shuffle_points, 66 | generate_bev=False, 67 | without_reflectivity=model_cfg.without_reflectivity, 68 | num_point_features=model_cfg.num_point_features, 69 | anchor_area_threshold=input_cfg.anchor_area_threshold, 70 | anchor_cache=self.anchor_cache, 71 | out_size_factor=out_size_factor, 72 | out_dtype=np.float32) 73 | example["metadata"] = {} 74 | if "image" in info: 75 | example["metadata"]["image"] = input_dict["image"] 76 | 77 | if "anchors_mask" in example: 78 | example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) 79 | ############# 80 | # convert example to batched example 81 | ############# 82 | example = merge_second_batch([example]) 83 | return example 84 | 85 | def get_config(self, path): 86 | config = pipeline_pb2.TrainEvalPipelineConfig() 87 | with open(path, "r") as f: 88 | proto_str = f.read() 89 | text_format.Merge(proto_str, config) 90 | return config 91 | 92 | @abc.abstractclassmethod 93 | def _build(self): 94 | raise NotImplementedError() 95 | 96 | def build(self, config_path): 97 | self.config = self.get_config(config_path) 98 | ret = self._build() 99 | self.built = True 100 | return ret 101 | 102 | @abc.abstractclassmethod 103 | def _inference(self, example): 104 | raise NotImplementedError() 105 | 106 | def inference(self, example): 107 | return self._inference(example) 108 | 109 | @abc.abstractclassmethod 110 | def _restore(self, ckpt_path): 111 | raise NotImplementedError() 112 | 113 | def restore(self, ckpt_path): 114 | return self._restore(ckpt_path) 115 | 116 | @abc.abstractclassmethod 117 | def _ctx(self): 118 | raise NotImplementedError() 119 | 120 | @contextlib.contextmanager 121 | def ctx(self): 122 | yield self._ctx() 123 | -------------------------------------------------------------------------------- /torchplus/train/optim.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict, Iterable 2 | 3 | import torch 4 | from copy import deepcopy 5 | from itertools import chain 6 | from torch.autograd import Variable 7 | 8 | required = object() 9 | 10 | def param_fp32_copy(params): 11 | param_copy = [ 12 | param.clone().type(torch.cuda.FloatTensor).detach() for param in params 13 | ] 14 | for param in param_copy: 15 | param.requires_grad = True 16 | return param_copy 17 | 18 | def set_grad(params, params_with_grad, scale=1.0): 19 | for param, param_w_grad in zip(params, params_with_grad): 20 | if param.grad is None: 21 | param.grad = torch.nn.Parameter( 22 | param.data.new().resize_(*param.data.size())) 23 | grad = param_w_grad.grad.data 24 | if scale is not None: 25 | grad /= scale 26 | if torch.isnan(grad).any() or torch.isinf(grad).any(): 27 | return True # invalid grad 28 | param.grad.data.copy_(grad) 29 | return False 30 | 31 | class MixedPrecisionWrapper(object): 32 | """mixed precision optimizer wrapper. 33 | Arguments: 34 | optimizer (torch.optim.Optimizer): an instance of 35 | :class:`torch.optim.Optimizer` 36 | scale: (float): a scalar for grad scale. 37 | auto_scale: (bool): whether enable auto scale. 38 | The algorihm of auto scale is discribled in 39 | http://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html 40 | """ 41 | 42 | def __init__(self, 43 | optimizer, 44 | scale=None, 45 | auto_scale=True, 46 | inc_factor=2.0, 47 | dec_factor=0.5, 48 | num_iters_be_stable=500): 49 | # if not isinstance(optimizer, torch.optim.Optimizer): 50 | # raise ValueError("must provide a torch.optim.Optimizer") 51 | self.optimizer = optimizer 52 | if hasattr(self.optimizer, 'name'): 53 | self.name = self.optimizer.name # for ckpt system 54 | param_groups_copy = [] 55 | for i, group in enumerate(optimizer.param_groups): 56 | group_copy = {n: v for n, v in group.items() if n != 'params'} 57 | group_copy['params'] = param_fp32_copy(group['params']) 58 | param_groups_copy.append(group_copy) 59 | 60 | # switch param_groups, may be dangerous 61 | self.param_groups = optimizer.param_groups 62 | optimizer.param_groups = param_groups_copy 63 | self.grad_scale = scale 64 | self.auto_scale = auto_scale 65 | self.inc_factor = inc_factor 66 | self.dec_factor = dec_factor 67 | self.stable_iter_count = 0 68 | self.num_iters_be_stable = num_iters_be_stable 69 | 70 | def __getstate__(self): 71 | return self.optimizer.__getstate__() 72 | 73 | def __setstate__(self, state): 74 | return self.optimizer.__setstate__(state) 75 | 76 | def __repr__(self): 77 | return self.optimizer.__repr__() 78 | 79 | def state_dict(self): 80 | return self.optimizer.state_dict() 81 | 82 | def load_state_dict(self, state_dict): 83 | return self.optimizer.load_state_dict(state_dict) 84 | 85 | def zero_grad(self): 86 | return self.optimizer.zero_grad() 87 | 88 | def step(self, closure=None): 89 | for g, g_copy in zip(self.param_groups, self.optimizer.param_groups): 90 | invalid = set_grad(g_copy['params'], g['params'], self.grad_scale) 91 | if invalid: 92 | if self.grad_scale is None or self.auto_scale is False: 93 | raise ValueError("nan/inf detected but auto_scale disabled.") 94 | self.grad_scale *= self.dec_factor 95 | print('scale decay to {}'.format(self.grad_scale)) 96 | return 97 | if self.auto_scale is True: 98 | self.stable_iter_count += 1 99 | if self.stable_iter_count > self.num_iters_be_stable: 100 | if self.grad_scale is not None: 101 | self.grad_scale *= self.inc_factor 102 | self.stable_iter_count = 0 103 | 104 | if closure is None: 105 | self.optimizer.step() 106 | else: 107 | self.optimizer.step(closure) 108 | for g, g_copy in zip(self.param_groups, self.optimizer.param_groups): 109 | for p_copy, p in zip(g_copy['params'], g['params']): 110 | p.data.copy_(p_copy.data) 111 | -------------------------------------------------------------------------------- /second/core/region_similarity.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Region Similarity Calculators for BoxLists. 17 | 18 | Region Similarity Calculators compare a pairwise measure of similarity 19 | between the boxes in two BoxLists. 20 | """ 21 | from abc import ABCMeta 22 | from abc import abstractmethod 23 | 24 | from second.core import box_np_ops 25 | 26 | class RegionSimilarityCalculator(object): 27 | """Abstract base class for 2d region similarity calculator.""" 28 | __metaclass__ = ABCMeta 29 | 30 | def compare(self, boxes1, boxes2): 31 | """Computes matrix of pairwise similarity between BoxLists. 32 | 33 | This op (to be overriden) computes a measure of pairwise similarity between 34 | the boxes in the given BoxLists. Higher values indicate more similarity. 35 | 36 | Note that this method simply measures similarity and does not explicitly 37 | perform a matching. 38 | 39 | Args: 40 | boxes1: [N, 5] [x,y,w,l,r] tensor. 41 | boxes2: [M, 5] [x,y,w,l,r] tensor. 42 | 43 | Returns: 44 | a (float32) tensor of shape [N, M] with pairwise similarity score. 45 | """ 46 | return self._compare(boxes1, boxes2) 47 | 48 | @abstractmethod 49 | def _compare(self, boxes1, boxes2): 50 | pass 51 | 52 | 53 | class RotateIouSimilarity(RegionSimilarityCalculator): 54 | """Class to compute similarity based on Intersection over Union (IOU) metric. 55 | 56 | This class computes pairwise similarity between two BoxLists based on IOU. 57 | """ 58 | 59 | def _compare(self, boxes1, boxes2): 60 | """Compute pairwise IOU similarity between the two BoxLists. 61 | 62 | Args: 63 | boxlist1: BoxList holding N boxes. 64 | boxlist2: BoxList holding M boxes. 65 | 66 | Returns: 67 | A tensor with shape [N, M] representing pairwise iou scores. 68 | """ 69 | 70 | return box_np_ops.riou_cc(boxes1, boxes2) 71 | 72 | 73 | class NearestIouSimilarity(RegionSimilarityCalculator): 74 | """Class to compute similarity based on the squared distance metric. 75 | 76 | This class computes pairwise similarity between two BoxLists based on the 77 | negative squared distance metric. 78 | """ 79 | 80 | def _compare(self, boxes1, boxes2): 81 | """Compute matrix of (negated) sq distances. 82 | 83 | Args: 84 | boxlist1: BoxList holding N boxes. 85 | boxlist2: BoxList holding M boxes. 86 | 87 | Returns: 88 | A tensor with shape [N, M] representing negated pairwise squared distance. 89 | """ 90 | boxes1_bv = box_np_ops.rbbox2d_to_near_bbox(boxes1) 91 | boxes2_bv = box_np_ops.rbbox2d_to_near_bbox(boxes2) 92 | ret = box_np_ops.iou_jit(boxes1_bv, boxes2_bv, eps=0.0) 93 | return ret 94 | 95 | 96 | class DistanceSimilarity(RegionSimilarityCalculator): 97 | """Class to compute similarity based on Intersection over Area (IOA) metric. 98 | 99 | This class computes pairwise similarity between two BoxLists based on their 100 | pairwise intersections divided by the areas of second BoxLists. 101 | """ 102 | def __init__(self, distance_norm, with_rotation=False, rotation_alpha=0.5): 103 | self._distance_norm = distance_norm 104 | self._with_rotation = with_rotation 105 | self._rotation_alpha = rotation_alpha 106 | 107 | def _compare(self, boxes1, boxes2): 108 | """Compute matrix of (negated) sq distances. 109 | 110 | Args: 111 | boxlist1: BoxList holding N boxes. 112 | boxlist2: BoxList holding M boxes. 113 | 114 | Returns: 115 | A tensor with shape [N, M] representing negated pairwise squared distance. 116 | """ 117 | return box_np_ops.distance_similarity( 118 | boxes1[..., [0, 1, -1]], 119 | boxes2[..., [0, 1, -1]], 120 | dist_norm=self._distance_norm, 121 | with_rotation=self._with_rotation, 122 | rot_alpha=self._rotation_alpha) 123 | -------------------------------------------------------------------------------- /second/kittiviewer/frontend/js/postprocessing/EffectComposer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @author alteredq / http://alteredqualia.com/ 3 | */ 4 | 5 | THREE.EffectComposer = function ( renderer, renderTarget ) { 6 | 7 | this.renderer = renderer; 8 | 9 | if ( renderTarget === undefined ) { 10 | 11 | var parameters = { 12 | minFilter: THREE.LinearFilter, 13 | magFilter: THREE.LinearFilter, 14 | format: THREE.RGBAFormat, 15 | stencilBuffer: false 16 | }; 17 | 18 | var size = renderer.getDrawingBufferSize(); 19 | renderTarget = new THREE.WebGLRenderTarget( size.width, size.height, parameters ); 20 | renderTarget.texture.name = 'EffectComposer.rt1'; 21 | 22 | } 23 | 24 | this.renderTarget1 = renderTarget; 25 | this.renderTarget2 = renderTarget.clone(); 26 | this.renderTarget2.texture.name = 'EffectComposer.rt2'; 27 | 28 | this.writeBuffer = this.renderTarget1; 29 | this.readBuffer = this.renderTarget2; 30 | 31 | this.passes = []; 32 | 33 | // dependencies 34 | 35 | if ( THREE.CopyShader === undefined ) { 36 | 37 | console.error( 'THREE.EffectComposer relies on THREE.CopyShader' ); 38 | 39 | } 40 | 41 | if ( THREE.ShaderPass === undefined ) { 42 | 43 | console.error( 'THREE.EffectComposer relies on THREE.ShaderPass' ); 44 | 45 | } 46 | 47 | this.copyPass = new THREE.ShaderPass( THREE.CopyShader ); 48 | 49 | }; 50 | 51 | Object.assign( THREE.EffectComposer.prototype, { 52 | 53 | swapBuffers: function () { 54 | 55 | var tmp = this.readBuffer; 56 | this.readBuffer = this.writeBuffer; 57 | this.writeBuffer = tmp; 58 | 59 | }, 60 | 61 | addPass: function ( pass ) { 62 | 63 | this.passes.push( pass ); 64 | 65 | var size = this.renderer.getDrawingBufferSize(); 66 | pass.setSize( size.width, size.height ); 67 | 68 | }, 69 | 70 | insertPass: function ( pass, index ) { 71 | 72 | this.passes.splice( index, 0, pass ); 73 | 74 | }, 75 | 76 | render: function ( delta ) { 77 | 78 | var maskActive = false; 79 | 80 | var pass, i, il = this.passes.length; 81 | 82 | for ( i = 0; i < il; i ++ ) { 83 | 84 | pass = this.passes[ i ]; 85 | 86 | if ( pass.enabled === false ) continue; 87 | 88 | pass.render( this.renderer, this.writeBuffer, this.readBuffer, delta, maskActive ); 89 | 90 | if ( pass.needsSwap ) { 91 | 92 | if ( maskActive ) { 93 | 94 | var context = this.renderer.context; 95 | 96 | context.stencilFunc( context.NOTEQUAL, 1, 0xffffffff ); 97 | 98 | this.copyPass.render( this.renderer, this.writeBuffer, this.readBuffer, delta ); 99 | 100 | context.stencilFunc( context.EQUAL, 1, 0xffffffff ); 101 | 102 | } 103 | 104 | this.swapBuffers(); 105 | 106 | } 107 | 108 | if ( THREE.MaskPass !== undefined ) { 109 | 110 | if ( pass instanceof THREE.MaskPass ) { 111 | 112 | maskActive = true; 113 | 114 | } else if ( pass instanceof THREE.ClearMaskPass ) { 115 | 116 | maskActive = false; 117 | 118 | } 119 | 120 | } 121 | 122 | } 123 | 124 | }, 125 | 126 | reset: function ( renderTarget ) { 127 | 128 | if ( renderTarget === undefined ) { 129 | 130 | var size = this.renderer.getDrawingBufferSize(); 131 | 132 | renderTarget = this.renderTarget1.clone(); 133 | renderTarget.setSize( size.width, size.height ); 134 | 135 | } 136 | 137 | this.renderTarget1.dispose(); 138 | this.renderTarget2.dispose(); 139 | this.renderTarget1 = renderTarget; 140 | this.renderTarget2 = renderTarget.clone(); 141 | 142 | this.writeBuffer = this.renderTarget1; 143 | this.readBuffer = this.renderTarget2; 144 | 145 | }, 146 | 147 | setSize: function ( width, height ) { 148 | 149 | this.renderTarget1.setSize( width, height ); 150 | this.renderTarget2.setSize( width, height ); 151 | 152 | for ( var i = 0; i < this.passes.length; i ++ ) { 153 | 154 | this.passes[ i ].setSize( width, height ); 155 | 156 | } 157 | 158 | } 159 | 160 | } ); 161 | 162 | 163 | THREE.Pass = function () { 164 | 165 | // if set to true, the pass is processed by the composer 166 | this.enabled = true; 167 | 168 | // if set to true, the pass indicates to swap read and write buffer after rendering 169 | this.needsSwap = true; 170 | 171 | // if set to true, the pass clears its buffer before rendering 172 | this.clear = false; 173 | 174 | // if set to true, the result of the pass is rendered to screen 175 | this.renderToScreen = false; 176 | 177 | }; 178 | 179 | Object.assign( THREE.Pass.prototype, { 180 | 181 | setSize: function ( width, height ) {}, 182 | 183 | render: function ( renderer, writeBuffer, readBuffer, delta, maskActive ) { 184 | 185 | console.error( 'THREE.Pass: .render() must be implemented in derived pass.' ); 186 | 187 | } 188 | 189 | } ); 190 | -------------------------------------------------------------------------------- /second/protos/pipeline_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/pipeline.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from second.protos import input_reader_pb2 as second_dot_protos_dot_input__reader__pb2 16 | from second.protos import model_pb2 as second_dot_protos_dot_model__pb2 17 | from second.protos import train_pb2 as second_dot_protos_dot_train__pb2 18 | 19 | 20 | DESCRIPTOR = _descriptor.FileDescriptor( 21 | name='second/protos/pipeline.proto', 22 | package='second.protos', 23 | syntax='proto3', 24 | serialized_options=None, 25 | serialized_pb=_b('\n\x1csecond/protos/pipeline.proto\x12\rsecond.protos\x1a second/protos/input_reader.proto\x1a\x19second/protos/model.proto\x1a\x19second/protos/train.proto\"\xe8\x01\n\x17TrainEvalPipelineConfig\x12,\n\x05model\x18\x01 \x01(\x0b\x32\x1d.second.protos.DetectionModel\x12\x36\n\x12train_input_reader\x18\x02 \x01(\x0b\x32\x1a.second.protos.InputReader\x12\x30\n\x0ctrain_config\x18\x03 \x01(\x0b\x32\x1a.second.protos.TrainConfig\x12\x35\n\x11\x65val_input_reader\x18\x04 \x01(\x0b\x32\x1a.second.protos.InputReaderb\x06proto3') 26 | , 27 | dependencies=[second_dot_protos_dot_input__reader__pb2.DESCRIPTOR,second_dot_protos_dot_model__pb2.DESCRIPTOR,second_dot_protos_dot_train__pb2.DESCRIPTOR,]) 28 | 29 | 30 | 31 | 32 | _TRAINEVALPIPELINECONFIG = _descriptor.Descriptor( 33 | name='TrainEvalPipelineConfig', 34 | full_name='second.protos.TrainEvalPipelineConfig', 35 | filename=None, 36 | file=DESCRIPTOR, 37 | containing_type=None, 38 | fields=[ 39 | _descriptor.FieldDescriptor( 40 | name='model', full_name='second.protos.TrainEvalPipelineConfig.model', index=0, 41 | number=1, type=11, cpp_type=10, label=1, 42 | has_default_value=False, default_value=None, 43 | message_type=None, enum_type=None, containing_type=None, 44 | is_extension=False, extension_scope=None, 45 | serialized_options=None, file=DESCRIPTOR), 46 | _descriptor.FieldDescriptor( 47 | name='train_input_reader', full_name='second.protos.TrainEvalPipelineConfig.train_input_reader', index=1, 48 | number=2, type=11, cpp_type=10, label=1, 49 | has_default_value=False, default_value=None, 50 | message_type=None, enum_type=None, containing_type=None, 51 | is_extension=False, extension_scope=None, 52 | serialized_options=None, file=DESCRIPTOR), 53 | _descriptor.FieldDescriptor( 54 | name='train_config', full_name='second.protos.TrainEvalPipelineConfig.train_config', index=2, 55 | number=3, type=11, cpp_type=10, label=1, 56 | has_default_value=False, default_value=None, 57 | message_type=None, enum_type=None, containing_type=None, 58 | is_extension=False, extension_scope=None, 59 | serialized_options=None, file=DESCRIPTOR), 60 | _descriptor.FieldDescriptor( 61 | name='eval_input_reader', full_name='second.protos.TrainEvalPipelineConfig.eval_input_reader', index=3, 62 | number=4, type=11, cpp_type=10, label=1, 63 | has_default_value=False, default_value=None, 64 | message_type=None, enum_type=None, containing_type=None, 65 | is_extension=False, extension_scope=None, 66 | serialized_options=None, file=DESCRIPTOR), 67 | ], 68 | extensions=[ 69 | ], 70 | nested_types=[], 71 | enum_types=[ 72 | ], 73 | serialized_options=None, 74 | is_extendable=False, 75 | syntax='proto3', 76 | extension_ranges=[], 77 | oneofs=[ 78 | ], 79 | serialized_start=136, 80 | serialized_end=368, 81 | ) 82 | 83 | _TRAINEVALPIPELINECONFIG.fields_by_name['model'].message_type = second_dot_protos_dot_model__pb2._DETECTIONMODEL 84 | _TRAINEVALPIPELINECONFIG.fields_by_name['train_input_reader'].message_type = second_dot_protos_dot_input__reader__pb2._INPUTREADER 85 | _TRAINEVALPIPELINECONFIG.fields_by_name['train_config'].message_type = second_dot_protos_dot_train__pb2._TRAINCONFIG 86 | _TRAINEVALPIPELINECONFIG.fields_by_name['eval_input_reader'].message_type = second_dot_protos_dot_input__reader__pb2._INPUTREADER 87 | DESCRIPTOR.message_types_by_name['TrainEvalPipelineConfig'] = _TRAINEVALPIPELINECONFIG 88 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 89 | 90 | TrainEvalPipelineConfig = _reflection.GeneratedProtocolMessageType('TrainEvalPipelineConfig', (_message.Message,), dict( 91 | DESCRIPTOR = _TRAINEVALPIPELINECONFIG, 92 | __module__ = 'second.protos.pipeline_pb2' 93 | # @@protoc_insertion_point(class_scope:second.protos.TrainEvalPipelineConfig) 94 | )) 95 | _sym_db.RegisterMessage(TrainEvalPipelineConfig) 96 | 97 | 98 | # @@protoc_insertion_point(module_scope) 99 | -------------------------------------------------------------------------------- /second/framework/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | 4 | 5 | class TestCase(unittest.TestCase): 6 | def _GetNdArray(self, a): 7 | if not isinstance(a, np.ndarray): 8 | a = np.array(a) 9 | return a 10 | 11 | def assertAllEqual(self, a, b): 12 | """Asserts that two numpy arrays have the same values. 13 | Args: 14 | a: the expected numpy ndarray or anything can be converted to one. 15 | b: the actual numpy ndarray or anything can be converted to one. 16 | """ 17 | a = self._GetNdArray(a) 18 | b = self._GetNdArray(b) 19 | self.assertEqual(a.shape, b.shape, 20 | "Shape mismatch: expected %s, got %s." % (a.shape, 21 | b.shape)) 22 | same = (a == b) 23 | 24 | if a.dtype == np.float32 or a.dtype == np.float64: 25 | same = np.logical_or(same, np.logical_and( 26 | np.isnan(a), np.isnan(b))) 27 | if not np.all(same): 28 | # Prints more details than np.testing.assert_array_equal. 29 | diff = np.logical_not(same) 30 | if a.ndim: 31 | x = a[np.where(diff)] 32 | y = b[np.where(diff)] 33 | print("not equal where = ", np.where(diff)) 34 | else: 35 | # np.where is broken for scalars 36 | x, y = a, b 37 | print("not equal lhs = ", x) 38 | print("not equal rhs = ", y) 39 | np.testing.assert_array_equal(a, b) 40 | 41 | def assertAllClose(self, a, b, rtol=1e-6, atol=1e-6): 42 | """Asserts that two numpy arrays, or dicts of same, have near values. 43 | This does not support nested dicts. 44 | Args: 45 | a: The expected numpy ndarray (or anything can be converted to one), or 46 | dict of same. Must be a dict iff `b` is a dict. 47 | b: The actual numpy ndarray (or anything can be converted to one), or 48 | dict of same. Must be a dict iff `a` is a dict. 49 | rtol: relative tolerance. 50 | atol: absolute tolerance. 51 | Raises: 52 | ValueError: if only one of `a` and `b` is a dict. 53 | """ 54 | is_a_dict = isinstance(a, dict) 55 | if is_a_dict != isinstance(b, dict): 56 | raise ValueError("Can't compare dict to non-dict, %s vs %s." % (a, 57 | b)) 58 | if is_a_dict: 59 | self.assertCountEqual( 60 | a.keys(), 61 | b.keys(), 62 | msg="mismatched keys, expected %s, got %s" % (a.keys(), 63 | b.keys())) 64 | for k in a: 65 | self._assertArrayLikeAllClose( 66 | a[k], 67 | b[k], 68 | rtol=rtol, 69 | atol=atol, 70 | msg="%s: expected %s, got %s." % (k, a, b)) 71 | else: 72 | self._assertArrayLikeAllClose(a, b, rtol=rtol, atol=atol) 73 | 74 | def _assertArrayLikeAllClose(self, a, b, rtol=1e-6, atol=1e-6, msg=None): 75 | a = self._GetNdArray(a) 76 | b = self._GetNdArray(b) 77 | self.assertEqual(a.shape, b.shape, 78 | "Shape mismatch: expected %s, got %s." % (a.shape, 79 | b.shape)) 80 | if not np.allclose(a, b, rtol=rtol, atol=atol): 81 | # Prints more details than np.testing.assert_allclose. 82 | # 83 | # NOTE: numpy.allclose (and numpy.testing.assert_allclose) 84 | # checks whether two arrays are element-wise equal within a 85 | # tolerance. The relative difference (rtol * abs(b)) and the 86 | # absolute difference atol are added together to compare against 87 | # the absolute difference between a and b. Here, we want to 88 | # print out which elements violate such conditions. 89 | cond = np.logical_or( 90 | np.abs(a - b) > atol + rtol * np.abs(b), 91 | np.isnan(a) != np.isnan(b)) 92 | if a.ndim: 93 | x = a[np.where(cond)] 94 | y = b[np.where(cond)] 95 | print("not close where = ", np.where(cond)) 96 | else: 97 | # np.where is broken for scalars 98 | x, y = a, b 99 | print("not close lhs = ", x) 100 | print("not close rhs = ", y) 101 | print("not close dif = ", np.abs(x - y)) 102 | print("not close tol = ", atol + rtol * np.abs(y)) 103 | print("dtype = %s, shape = %s" % (a.dtype, a.shape)) 104 | np.testing.assert_allclose(a, b, rtol=rtol, atol=atol, err_msg=msg) 105 | -------------------------------------------------------------------------------- /second/utils/log_tool.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from tensorboardX import SummaryWriter 3 | import json 4 | from pathlib import Path 5 | 6 | def _flat_nested_json_dict(json_dict, flatted, sep=".", start=""): 7 | for k, v in json_dict.items(): 8 | if isinstance(v, dict): 9 | _flat_nested_json_dict(v, flatted, sep, start + sep + str(k)) 10 | else: 11 | flatted[start + sep + str(k)] = v 12 | 13 | def flat_nested_json_dict(json_dict, sep=".") -> dict: 14 | """flat a nested json-like dict. this function make shadow copy. 15 | """ 16 | flatted = {} 17 | for k, v in json_dict.items(): 18 | if isinstance(v, dict): 19 | _flat_nested_json_dict(v, flatted, sep, str(k)) 20 | else: 21 | flatted[str(k)] = v 22 | return flatted 23 | 24 | def metric_to_str(metrics, sep='.'): 25 | flatted_metrics = flat_nested_json_dict(metrics, sep) 26 | metrics_str_list = [] 27 | for k, v in flatted_metrics.items(): 28 | if isinstance(v, float): 29 | metrics_str_list.append(f"{k}={v:.4}") 30 | elif isinstance(v, (list, tuple)): 31 | if v and isinstance(v[0], float): 32 | v_str = ', '.join([f"{e:.4}" for e in v]) 33 | metrics_str_list.append(f"{k}=[{v_str}]") 34 | else: 35 | metrics_str_list.append(f"{k}={v}") 36 | else: 37 | metrics_str_list.append(f"{k}={v}") 38 | return ', '.join(metrics_str_list) 39 | 40 | class SimpleModelLog: 41 | """For simple log. 42 | generate 4 kinds of log: 43 | 1. simple log.txt, all metric dicts are flattened to produce 44 | readable results. 45 | 2. TensorBoard scalars and texts 46 | 3. multi-line json file log.json.lst 47 | 4. tensorboard_scalars.json, all scalars are stored in this file 48 | in tensorboard json format. 49 | """ 50 | def __init__(self, model_dir): 51 | self.model_dir = Path(model_dir) 52 | self.log_file = None 53 | self.log_mjson_file = None 54 | self.summary_writter = None 55 | self.metrics = [] 56 | self._text_current_gstep = -1 57 | self._tb_texts = [] 58 | 59 | def open(self): 60 | model_dir = self.model_dir 61 | assert model_dir.exists() 62 | summary_dir = model_dir / 'summary' 63 | summary_dir.mkdir(parents=True, exist_ok=True) 64 | 65 | log_mjson_file_path = model_dir / f'log.json.lst' 66 | if log_mjson_file_path.exists(): 67 | with open(log_mjson_file_path, 'r') as f: 68 | for line in f.readlines(): 69 | try: 70 | self.metrics.append(json.loads(line)) 71 | except Exception as e: 72 | print(e) 73 | import pdb; pdb.set_trace() 74 | log_file_path = model_dir / f'log.txt' 75 | self.log_mjson_file = open(log_mjson_file_path, 'a') 76 | self.log_file = open(log_file_path, 'a') 77 | self.summary_writter = SummaryWriter(str(summary_dir)) 78 | return self 79 | 80 | def close(self): 81 | assert self.summary_writter is not None 82 | self.log_mjson_file.close() 83 | self.log_file.close() 84 | tb_json_path = str(self.model_dir / "tensorboard_scalars.json") 85 | self.summary_writter.export_scalars_to_json(tb_json_path) 86 | self.summary_writter.close() 87 | self.log_mjson_file = None 88 | self.log_file = None 89 | self.summary_writter = None 90 | 91 | def log_text(self, text, step, tag="regular log"): 92 | """This function only add text to log.txt and tensorboard texts 93 | """ 94 | print(text) 95 | print(text, file=self.log_file) 96 | if step > self._text_current_gstep and self._text_current_gstep != -1: 97 | total_text = '\n'.join(self._tb_texts) 98 | self.summary_writter.add_text(tag, total_text, global_step=step) 99 | self._tb_texts = [] 100 | self._text_current_gstep = step 101 | else: 102 | self._tb_texts.append(text) 103 | if self._text_current_gstep == -1: 104 | self._text_current_gstep = step 105 | 106 | 107 | def log_metrics(self, metrics: dict, step): 108 | flatted_summarys = flat_nested_json_dict(metrics, "/") 109 | for k, v in flatted_summarys.items(): 110 | if isinstance(v, (list, tuple)): 111 | if any([isinstance(e, str) for e in v]): 112 | continue 113 | v_dict = {str(i): e for i, e in enumerate(v)} 114 | for k1, v1 in v_dict.items(): 115 | self.summary_writter.add_scalar(k + "/" + k1, v1, step) 116 | else: 117 | if isinstance(v, str): 118 | continue 119 | self.summary_writter.add_scalar(k, v, step) 120 | log_str = metric_to_str(metrics) 121 | print(log_str) 122 | print(log_str, file=self.log_file) 123 | print(json.dumps(metrics), file=self.log_mjson_file) 124 | 125 | -------------------------------------------------------------------------------- /second/protos/voxel_generator_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/voxel_generator.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | 16 | 17 | DESCRIPTOR = _descriptor.FileDescriptor( 18 | name='second/protos/voxel_generator.proto', 19 | package='second.protos', 20 | syntax='proto3', 21 | serialized_options=None, 22 | serialized_pb=_b('\n#second/protos/voxel_generator.proto\x12\rsecond.protos\"\xe7\x01\n\x0eVoxelGenerator\x12\x12\n\nvoxel_size\x18\x01 \x03(\x02\x12\x19\n\x11point_cloud_range\x18\x02 \x03(\x02\x12&\n\x1emax_number_of_points_per_voxel\x18\x03 \x01(\r\x12!\n\x19\x66ull_empty_part_with_mean\x18\x04 \x01(\x08\x12\x17\n\x0f\x62lock_filtering\x18\x05 \x01(\x08\x12\x14\n\x0c\x62lock_factor\x18\x06 \x01(\x03\x12\x12\n\nblock_size\x18\x07 \x01(\x03\x12\x18\n\x10height_threshold\x18\x08 \x01(\x02\x62\x06proto3') 23 | ) 24 | 25 | 26 | 27 | 28 | _VOXELGENERATOR = _descriptor.Descriptor( 29 | name='VoxelGenerator', 30 | full_name='second.protos.VoxelGenerator', 31 | filename=None, 32 | file=DESCRIPTOR, 33 | containing_type=None, 34 | fields=[ 35 | _descriptor.FieldDescriptor( 36 | name='voxel_size', full_name='second.protos.VoxelGenerator.voxel_size', index=0, 37 | number=1, type=2, cpp_type=6, label=3, 38 | has_default_value=False, default_value=[], 39 | message_type=None, enum_type=None, containing_type=None, 40 | is_extension=False, extension_scope=None, 41 | serialized_options=None, file=DESCRIPTOR), 42 | _descriptor.FieldDescriptor( 43 | name='point_cloud_range', full_name='second.protos.VoxelGenerator.point_cloud_range', index=1, 44 | number=2, type=2, cpp_type=6, label=3, 45 | has_default_value=False, default_value=[], 46 | message_type=None, enum_type=None, containing_type=None, 47 | is_extension=False, extension_scope=None, 48 | serialized_options=None, file=DESCRIPTOR), 49 | _descriptor.FieldDescriptor( 50 | name='max_number_of_points_per_voxel', full_name='second.protos.VoxelGenerator.max_number_of_points_per_voxel', index=2, 51 | number=3, type=13, cpp_type=3, label=1, 52 | has_default_value=False, default_value=0, 53 | message_type=None, enum_type=None, containing_type=None, 54 | is_extension=False, extension_scope=None, 55 | serialized_options=None, file=DESCRIPTOR), 56 | _descriptor.FieldDescriptor( 57 | name='full_empty_part_with_mean', full_name='second.protos.VoxelGenerator.full_empty_part_with_mean', index=3, 58 | number=4, type=8, cpp_type=7, label=1, 59 | has_default_value=False, default_value=False, 60 | message_type=None, enum_type=None, containing_type=None, 61 | is_extension=False, extension_scope=None, 62 | serialized_options=None, file=DESCRIPTOR), 63 | _descriptor.FieldDescriptor( 64 | name='block_filtering', full_name='second.protos.VoxelGenerator.block_filtering', index=4, 65 | number=5, type=8, cpp_type=7, label=1, 66 | has_default_value=False, default_value=False, 67 | message_type=None, enum_type=None, containing_type=None, 68 | is_extension=False, extension_scope=None, 69 | serialized_options=None, file=DESCRIPTOR), 70 | _descriptor.FieldDescriptor( 71 | name='block_factor', full_name='second.protos.VoxelGenerator.block_factor', index=5, 72 | number=6, type=3, cpp_type=2, label=1, 73 | has_default_value=False, default_value=0, 74 | message_type=None, enum_type=None, containing_type=None, 75 | is_extension=False, extension_scope=None, 76 | serialized_options=None, file=DESCRIPTOR), 77 | _descriptor.FieldDescriptor( 78 | name='block_size', full_name='second.protos.VoxelGenerator.block_size', index=6, 79 | number=7, type=3, cpp_type=2, label=1, 80 | has_default_value=False, default_value=0, 81 | message_type=None, enum_type=None, containing_type=None, 82 | is_extension=False, extension_scope=None, 83 | serialized_options=None, file=DESCRIPTOR), 84 | _descriptor.FieldDescriptor( 85 | name='height_threshold', full_name='second.protos.VoxelGenerator.height_threshold', index=7, 86 | number=8, type=2, cpp_type=6, label=1, 87 | has_default_value=False, default_value=float(0), 88 | message_type=None, enum_type=None, containing_type=None, 89 | is_extension=False, extension_scope=None, 90 | serialized_options=None, file=DESCRIPTOR), 91 | ], 92 | extensions=[ 93 | ], 94 | nested_types=[], 95 | enum_types=[ 96 | ], 97 | serialized_options=None, 98 | is_extendable=False, 99 | syntax='proto3', 100 | extension_ranges=[], 101 | oneofs=[ 102 | ], 103 | serialized_start=55, 104 | serialized_end=286, 105 | ) 106 | 107 | DESCRIPTOR.message_types_by_name['VoxelGenerator'] = _VOXELGENERATOR 108 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 109 | 110 | VoxelGenerator = _reflection.GeneratedProtocolMessageType('VoxelGenerator', (_message.Message,), dict( 111 | DESCRIPTOR = _VOXELGENERATOR, 112 | __module__ = 'second.protos.voxel_generator_pb2' 113 | # @@protoc_insertion_point(class_scope:second.protos.VoxelGenerator) 114 | )) 115 | _sym_db.RegisterMessage(VoxelGenerator) 116 | 117 | 118 | # @@protoc_insertion_point(module_scope) 119 | -------------------------------------------------------------------------------- /second/pytorch/core/ghm_loss.py: -------------------------------------------------------------------------------- 1 | ##################### 2 | # THIS LOSS IS NOT WORKING!!!! 3 | ##################### 4 | 5 | """ 6 | The implementation of GHM-C and GHM-R losses. 7 | Details can be found in the paper `Gradient Harmonized Single-stage Detector`: 8 | https://arxiv.org/abs/1811.05181 9 | Copyright (c) 2018 Multimedia Laboratory, CUHK. 10 | Licensed under the MIT License (see LICENSE for details) 11 | Written by Buyu Li 12 | """ 13 | 14 | from second.pytorch.core.losses import Loss, _sigmoid_cross_entropy_with_logits 15 | import torch 16 | 17 | class GHMCLoss(Loss): 18 | def __init__(self, bins=10, momentum=0): 19 | self.bins = bins 20 | self.momentum = momentum 21 | self.edges = [float(x) / bins for x in range(bins+1)] 22 | self.edges[-1] += 1e-6 23 | if momentum > 0: 24 | self.acc_sum = [0.0 for _ in range(bins)] 25 | self.count = 50 26 | 27 | def _compute_loss(self, 28 | prediction_tensor, 29 | target_tensor, 30 | weights, 31 | class_indices=None): 32 | """ Args: 33 | input [batch_num, class_num]: 34 | The direct prediction of classification fc layer. 35 | target [batch_num, class_num]: 36 | Binary target (0 or 1) for each sample each class. The value is -1 37 | when the sample is ignored. 38 | """ 39 | input = prediction_tensor 40 | target = target_tensor 41 | batch_size = prediction_tensor.shape[0] 42 | num_anchors = prediction_tensor.shape[1] 43 | num_class = prediction_tensor.shape[2] 44 | edges = self.edges 45 | weights_ghm = torch.zeros_like(input).view(-1, num_class) 46 | per_entry_cross_ent = (_sigmoid_cross_entropy_with_logits( 47 | labels=target_tensor, logits=prediction_tensor)) 48 | # gradient length 49 | g = torch.abs(input.sigmoid().detach() - target).view(-1, num_class) 50 | valid = weights.view(-1, 1).expand(-1, num_class) > 0 51 | num_examples = max(valid.float().sum().item(), 1.0) 52 | num_valid_bins = 0 # n valid bins 53 | self.count -= 1 54 | num_bins = [] 55 | for i in range(self.bins): 56 | inds = (g >= edges[i]) & (g < edges[i+1]) & valid 57 | num_in_bin = inds.sum().item() 58 | num_bins.append(num_in_bin) 59 | if num_in_bin > 0: 60 | if self.momentum > 0: 61 | self.acc_sum[i] = self.momentum * self.acc_sum[i] \ 62 | + (1 - self.momentum) * num_in_bin 63 | weights_ghm[inds] = num_examples / self.acc_sum[i] 64 | else: 65 | weights_ghm[inds] = num_examples / num_in_bin 66 | num_valid_bins += 1 67 | if self.count <= 0: 68 | print("GHMC loss bins:", num_bins) 69 | self.count = 50 70 | if num_valid_bins > 0: 71 | weights_ghm = weights_ghm / num_valid_bins 72 | return per_entry_cross_ent * weights_ghm.view(batch_size, num_anchors, num_class) / num_examples 73 | 74 | 75 | class GHMRLoss(Loss): 76 | def __init__(self, mu=0.02, bins=10, momentum=0, code_weights=None): 77 | self.mu = mu 78 | self.bins = bins 79 | self.edges = [float(x) / bins for x in range(bins+1)] 80 | self.edges[-1] = 1e3 81 | self.momentum = momentum 82 | if momentum > 0: 83 | self.acc_sum = [0.0 for _ in range(bins)] 84 | self._codewise = True 85 | 86 | def _compute_loss(self, 87 | prediction_tensor, 88 | target_tensor, 89 | weights): 90 | """ Args: 91 | input [batch_num, class_num]: 92 | The direct prediction of classification fc layer. 93 | target [batch_num, class_num]: 94 | Binary target (0 or 1) for each sample each class. The value is -1 95 | when the sample is ignored. 96 | """ 97 | # ASL1 loss 98 | diff = prediction_tensor - target_tensor 99 | loss = torch.sqrt(diff * diff + self.mu * self.mu) - self.mu 100 | batch_size = prediction_tensor.shape[0] 101 | num_anchors = prediction_tensor.shape[1] 102 | num_codes = prediction_tensor.shape[2] 103 | 104 | # gradient length 105 | g = torch.abs(diff / torch.sqrt(self.mu * self.mu + diff * diff)).detach().view(-1, num_codes) 106 | weights_ghm = torch.zeros_like(g) 107 | 108 | valid = weights.view(-1, 1).expand(-1, num_codes) > 0 109 | # print(g.shape, prediction_tensor.shape, valid.shape) 110 | num_examples = max(valid.float().sum().item() / num_codes, 1.0) 111 | num_valid_bins = 0 # n: valid bins 112 | for i in range(self.bins): 113 | inds = (g >= self.edges[i]) & (g < self.edges[i+1]) & valid 114 | num_in_bin = inds.sum().item() 115 | if num_in_bin > 0: 116 | num_valid_bins += 1 117 | if self.momentum > 0: 118 | self.acc_sum[i] = self.momentum * self.acc_sum[i] \ 119 | + (1 - self.momentum) * num_in_bin 120 | weights_ghm[inds] = num_examples / self.acc_sum[i] 121 | else: 122 | weights_ghm[inds] = num_examples / num_in_bin 123 | if num_valid_bins > 0: 124 | weights_ghm /= num_valid_bins 125 | weights_ghm = weights_ghm.view(batch_size, num_anchors, num_codes) 126 | loss = loss * weights_ghm / num_examples 127 | return loss 128 | -------------------------------------------------------------------------------- /second/protos/losses.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package second.protos; 4 | 5 | // Message for configuring the localization loss, classification loss and hard 6 | // example miner used for training object detection models. See core/losses.py 7 | // for details 8 | message Loss { 9 | // Localization loss to use. 10 | LocalizationLoss localization_loss = 1; 11 | 12 | // Classification loss to use. 13 | ClassificationLoss classification_loss = 2; 14 | 15 | // If not left to default, applies hard example mining. 16 | HardExampleMiner hard_example_miner = 3; 17 | 18 | // Classification loss weight. 19 | float classification_weight = 4; 20 | 21 | // Localization loss weight. 22 | float localization_weight = 5; 23 | 24 | } 25 | 26 | // Configuration for bounding box localization loss function. 27 | message LocalizationLoss { 28 | oneof localization_loss { 29 | WeightedL2LocalizationLoss weighted_l2 = 1; 30 | WeightedSmoothL1LocalizationLoss weighted_smooth_l1 = 2; 31 | WeightedGHMLocalizationLoss weighted_ghm = 3; 32 | } 33 | bool encode_rad_error_by_sin = 4; 34 | } 35 | 36 | // L2 location loss: 0.5 * ||weight * (a - b)|| ^ 2 37 | message WeightedL2LocalizationLoss { 38 | // DEPRECATED, do not use. 39 | // Output loss per anchor. 40 | bool anchorwise_output = 1; 41 | repeated float code_weight = 2; 42 | } 43 | 44 | // SmoothL1 (Huber) location loss: .5 * x ^ 2 if |x| < 1 else |x| - .5 45 | message WeightedSmoothL1LocalizationLoss { 46 | // DEPRECATED, do not use. 47 | // Output loss per anchor. 48 | bool anchorwise_output = 1; 49 | float sigma = 2; 50 | repeated float code_weight = 3; 51 | } 52 | message WeightedGHMLocalizationLoss { 53 | // DEPRECATED, do not use. 54 | // Output loss per anchor. 55 | bool anchorwise_output = 1; 56 | float mu = 2; 57 | int32 bins = 3; 58 | float momentum = 4; 59 | repeated float code_weight = 5; 60 | } 61 | 62 | 63 | // Configuration for class prediction loss function. 64 | message ClassificationLoss { 65 | oneof classification_loss { 66 | WeightedSigmoidClassificationLoss weighted_sigmoid = 1; 67 | WeightedSoftmaxClassificationLoss weighted_softmax = 2; 68 | BootstrappedSigmoidClassificationLoss bootstrapped_sigmoid = 3; 69 | SigmoidFocalClassificationLoss weighted_sigmoid_focal = 4; 70 | SoftmaxFocalClassificationLoss weighted_softmax_focal = 5; 71 | GHMClassificationLoss weighted_ghm = 6; 72 | } 73 | } 74 | 75 | // Classification loss using a sigmoid function over class predictions. 76 | message WeightedSigmoidClassificationLoss { 77 | // DEPRECATED, do not use. 78 | // Output loss per anchor. 79 | bool anchorwise_output = 1; 80 | } 81 | 82 | // Sigmoid Focal cross entropy loss as described in 83 | // https://arxiv.org/abs/1708.02002 84 | message SigmoidFocalClassificationLoss { 85 | // DEPRECATED, do not use. 86 | bool anchorwise_output = 1; 87 | // modulating factor for the loss. 88 | float gamma = 2; 89 | // alpha weighting factor for the loss. 90 | float alpha = 3; 91 | } 92 | // Sigmoid Focal cross entropy loss as described in 93 | // https://arxiv.org/abs/1708.02002 94 | message SoftmaxFocalClassificationLoss { 95 | // DEPRECATED, do not use. 96 | bool anchorwise_output = 1; 97 | // modulating factor for the loss. 98 | float gamma = 2; 99 | // alpha weighting factor for the loss. 100 | float alpha = 3; 101 | } 102 | message GHMClassificationLoss { 103 | bool anchorwise_output = 1; 104 | int32 bins = 2; 105 | float momentum = 3; 106 | } 107 | // Classification loss using a softmax function over class predictions. 108 | message WeightedSoftmaxClassificationLoss { 109 | // DEPRECATED, do not use. 110 | // Output loss per anchor. 111 | bool anchorwise_output = 1; 112 | // Scale logit (input) value before calculating softmax classification loss. 113 | // Typically used for softmax distillation. 114 | float logit_scale = 2; 115 | } 116 | 117 | // Classification loss using a sigmoid function over the class prediction with 118 | // the highest prediction score. 119 | message BootstrappedSigmoidClassificationLoss { 120 | // Interpolation weight between 0 and 1. 121 | float alpha = 1; 122 | 123 | // Whether hard boot strapping should be used or not. If true, will only use 124 | // one class favored by model. Othewise, will use all predicted class 125 | // probabilities. 126 | bool hard_bootstrap = 2; 127 | 128 | // DEPRECATED, do not use. 129 | // Output loss per anchor. 130 | bool anchorwise_output = 3; 131 | } 132 | 133 | // Configuation for hard example miner. 134 | message HardExampleMiner { 135 | // Maximum number of hard examples to be selected per image (prior to 136 | // enforcing max negative to positive ratio constraint). If set to 0, 137 | // all examples obtained after NMS are considered. 138 | int32 num_hard_examples = 1; 139 | 140 | // Minimum intersection over union for an example to be discarded during NMS. 141 | float iou_threshold = 2; 142 | 143 | // Whether to use classification losses ('cls', default), localization losses 144 | // ('loc') or both losses ('both'). In the case of 'both', cls_loss_weight and 145 | // loc_loss_weight are used to compute weighted sum of the two losses. 146 | enum LossType { 147 | BOTH = 0; 148 | CLASSIFICATION = 1; 149 | LOCALIZATION = 2; 150 | } 151 | LossType loss_type = 3; 152 | 153 | // Maximum number of negatives to retain for each positive anchor. If 154 | // num_negatives_per_positive is 0 no prespecified negative:positive ratio is 155 | // enforced. 156 | int32 max_negatives_per_positive = 4; 157 | 158 | // Minimum number of negative anchors to sample for a given image. Setting 159 | // this to a positive number samples negatives in an image without any 160 | // positive anchors and thus not bias the model towards having at least one 161 | // detection per image. 162 | int32 min_negatives_per_image = 5; 163 | } 164 | -------------------------------------------------------------------------------- /second/protos/train_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: second/protos/train.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from second.protos import optimizer_pb2 as second_dot_protos_dot_optimizer__pb2 16 | from second.protos import preprocess_pb2 as second_dot_protos_dot_preprocess__pb2 17 | 18 | 19 | DESCRIPTOR = _descriptor.FileDescriptor( 20 | name='second/protos/train.proto', 21 | package='second.protos', 22 | syntax='proto3', 23 | serialized_options=None, 24 | serialized_pb=_b('\n\x19second/protos/train.proto\x12\rsecond.protos\x1a\x1dsecond/protos/optimizer.proto\x1a\x1esecond/protos/preprocess.proto\"\xfa\x01\n\x0bTrainConfig\x12+\n\toptimizer\x18\x01 \x01(\x0b\x32\x18.second.protos.Optimizer\x12\r\n\x05steps\x18\x02 \x01(\r\x12\x16\n\x0esteps_per_eval\x18\x03 \x01(\r\x12\x1d\n\x15save_checkpoints_secs\x18\x04 \x01(\r\x12\x1a\n\x12save_summary_steps\x18\x05 \x01(\r\x12\x1e\n\x16\x65nable_mixed_precision\x18\x06 \x01(\x08\x12\x19\n\x11loss_scale_factor\x18\x07 \x01(\x02\x12!\n\x19\x63lear_metrics_every_epoch\x18\x08 \x01(\x08\x62\x06proto3') 25 | , 26 | dependencies=[second_dot_protos_dot_optimizer__pb2.DESCRIPTOR,second_dot_protos_dot_preprocess__pb2.DESCRIPTOR,]) 27 | 28 | 29 | 30 | 31 | _TRAINCONFIG = _descriptor.Descriptor( 32 | name='TrainConfig', 33 | full_name='second.protos.TrainConfig', 34 | filename=None, 35 | file=DESCRIPTOR, 36 | containing_type=None, 37 | fields=[ 38 | _descriptor.FieldDescriptor( 39 | name='optimizer', full_name='second.protos.TrainConfig.optimizer', index=0, 40 | number=1, type=11, cpp_type=10, label=1, 41 | has_default_value=False, default_value=None, 42 | message_type=None, enum_type=None, containing_type=None, 43 | is_extension=False, extension_scope=None, 44 | serialized_options=None, file=DESCRIPTOR), 45 | _descriptor.FieldDescriptor( 46 | name='steps', full_name='second.protos.TrainConfig.steps', index=1, 47 | number=2, type=13, cpp_type=3, label=1, 48 | has_default_value=False, default_value=0, 49 | message_type=None, enum_type=None, containing_type=None, 50 | is_extension=False, extension_scope=None, 51 | serialized_options=None, file=DESCRIPTOR), 52 | _descriptor.FieldDescriptor( 53 | name='steps_per_eval', full_name='second.protos.TrainConfig.steps_per_eval', index=2, 54 | number=3, type=13, cpp_type=3, label=1, 55 | has_default_value=False, default_value=0, 56 | message_type=None, enum_type=None, containing_type=None, 57 | is_extension=False, extension_scope=None, 58 | serialized_options=None, file=DESCRIPTOR), 59 | _descriptor.FieldDescriptor( 60 | name='save_checkpoints_secs', full_name='second.protos.TrainConfig.save_checkpoints_secs', index=3, 61 | number=4, type=13, cpp_type=3, label=1, 62 | has_default_value=False, default_value=0, 63 | message_type=None, enum_type=None, containing_type=None, 64 | is_extension=False, extension_scope=None, 65 | serialized_options=None, file=DESCRIPTOR), 66 | _descriptor.FieldDescriptor( 67 | name='save_summary_steps', full_name='second.protos.TrainConfig.save_summary_steps', index=4, 68 | number=5, type=13, cpp_type=3, label=1, 69 | has_default_value=False, default_value=0, 70 | message_type=None, enum_type=None, containing_type=None, 71 | is_extension=False, extension_scope=None, 72 | serialized_options=None, file=DESCRIPTOR), 73 | _descriptor.FieldDescriptor( 74 | name='enable_mixed_precision', full_name='second.protos.TrainConfig.enable_mixed_precision', index=5, 75 | number=6, type=8, cpp_type=7, label=1, 76 | has_default_value=False, default_value=False, 77 | message_type=None, enum_type=None, containing_type=None, 78 | is_extension=False, extension_scope=None, 79 | serialized_options=None, file=DESCRIPTOR), 80 | _descriptor.FieldDescriptor( 81 | name='loss_scale_factor', full_name='second.protos.TrainConfig.loss_scale_factor', index=6, 82 | number=7, type=2, cpp_type=6, label=1, 83 | has_default_value=False, default_value=float(0), 84 | message_type=None, enum_type=None, containing_type=None, 85 | is_extension=False, extension_scope=None, 86 | serialized_options=None, file=DESCRIPTOR), 87 | _descriptor.FieldDescriptor( 88 | name='clear_metrics_every_epoch', full_name='second.protos.TrainConfig.clear_metrics_every_epoch', index=7, 89 | number=8, type=8, cpp_type=7, label=1, 90 | has_default_value=False, default_value=False, 91 | message_type=None, enum_type=None, containing_type=None, 92 | is_extension=False, extension_scope=None, 93 | serialized_options=None, file=DESCRIPTOR), 94 | ], 95 | extensions=[ 96 | ], 97 | nested_types=[], 98 | enum_types=[ 99 | ], 100 | serialized_options=None, 101 | is_extendable=False, 102 | syntax='proto3', 103 | extension_ranges=[], 104 | oneofs=[ 105 | ], 106 | serialized_start=108, 107 | serialized_end=358, 108 | ) 109 | 110 | _TRAINCONFIG.fields_by_name['optimizer'].message_type = second_dot_protos_dot_optimizer__pb2._OPTIMIZER 111 | DESCRIPTOR.message_types_by_name['TrainConfig'] = _TRAINCONFIG 112 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 113 | 114 | TrainConfig = _reflection.GeneratedProtocolMessageType('TrainConfig', (_message.Message,), dict( 115 | DESCRIPTOR = _TRAINCONFIG, 116 | __module__ = 'second.protos.train_pb2' 117 | # @@protoc_insertion_point(class_scope:second.protos.TrainConfig) 118 | )) 119 | _sym_db.RegisterMessage(TrainConfig) 120 | 121 | 122 | # @@protoc_insertion_point(module_scope) 123 | -------------------------------------------------------------------------------- /second/core/non_max_suppression/nms_cpu.py: -------------------------------------------------------------------------------- 1 | import math 2 | from pathlib import Path 3 | import numba 4 | import numpy as np 5 | from spconv.utils import ( 6 | non_max_suppression_cpu, rotate_non_max_suppression_cpu) 7 | from second.core import box_np_ops 8 | from second.core.non_max_suppression.nms_gpu import rotate_iou_gpu 9 | 10 | 11 | def nms_cc(dets, thresh): 12 | scores = dets[:, 4] 13 | order = scores.argsort()[::-1].astype(np.int32) # highest->lowest 14 | return non_max_suppression_cpu(dets, order, thresh, 1.0) 15 | 16 | 17 | def rotate_nms_cc(dets, thresh): 18 | scores = dets[:, 5] 19 | order = scores.argsort()[::-1].astype(np.int32) # highest->lowest 20 | dets_corners = box_np_ops.center_to_corner_box2d(dets[:, :2], dets[:, 2:4], 21 | dets[:, 4]) 22 | 23 | dets_standup = box_np_ops.corner_to_standup_nd(dets_corners) 24 | 25 | standup_iou = box_np_ops.iou_jit(dets_standup, dets_standup, eps=0.0) 26 | # print(dets_corners.shape, order.shape, standup_iou.shape) 27 | return rotate_non_max_suppression_cpu(dets_corners, order, standup_iou, 28 | thresh) 29 | 30 | @numba.jit(nopython=True) 31 | def nms_jit(dets, thresh, eps=0.0): 32 | x1 = dets[:, 0] 33 | y1 = dets[:, 1] 34 | x2 = dets[:, 2] 35 | y2 = dets[:, 3] 36 | scores = dets[:, 4] 37 | areas = (x2 - x1 + eps) * (y2 - y1 + eps) 38 | order = scores.argsort()[::-1].astype(np.int32) # highest->lowest 39 | ndets = dets.shape[0] 40 | suppressed = np.zeros((ndets), dtype=np.int32) 41 | keep = [] 42 | for _i in range(ndets): 43 | i = order[_i] # start with highest score box 44 | if suppressed[ 45 | i] == 1: # if any box have enough iou with this, remove it 46 | continue 47 | keep.append(i) 48 | for _j in range(_i + 1, ndets): 49 | j = order[_j] 50 | if suppressed[j] == 1: 51 | continue 52 | # calculate iou between i and j box 53 | w = max(min(x2[i], x2[j]) - max(x1[i], x1[j]) + eps, 0.0) 54 | h = max(min(y2[i], y2[j]) - max(y1[i], y1[j]) + eps, 0.0) 55 | inter = w * h 56 | ovr = inter / (areas[i] + areas[j] - inter) 57 | # ovr = inter / areas[j] 58 | if ovr >= thresh: 59 | suppressed[j] = 1 60 | return keep 61 | 62 | 63 | @numba.jit('float32[:, :], float32, float32, float32, uint32', nopython=True) 64 | def soft_nms_jit(boxes, sigma=0.5, Nt=0.3, threshold=0.001, method=0): 65 | N = boxes.shape[0] 66 | pos = 0 67 | maxscore = 0 68 | maxpos = 0 69 | for i in range(N): 70 | maxscore = boxes[i, 4] 71 | maxpos = i 72 | 73 | tx1 = boxes[i, 0] 74 | ty1 = boxes[i, 1] 75 | tx2 = boxes[i, 2] 76 | ty2 = boxes[i, 3] 77 | ts = boxes[i, 4] 78 | pos = i + 1 79 | # get max box 80 | while pos < N: 81 | if maxscore < boxes[pos, 4]: 82 | maxscore = boxes[pos, 4] 83 | maxpos = pos 84 | pos = pos + 1 85 | 86 | # add max box as a detection 87 | boxes[i, 0] = boxes[maxpos, 0] 88 | boxes[i, 1] = boxes[maxpos, 1] 89 | boxes[i, 2] = boxes[maxpos, 2] 90 | boxes[i, 3] = boxes[maxpos, 3] 91 | boxes[i, 4] = boxes[maxpos, 4] 92 | 93 | # swap ith box with position of max box 94 | boxes[maxpos, 0] = tx1 95 | boxes[maxpos, 1] = ty1 96 | boxes[maxpos, 2] = tx2 97 | boxes[maxpos, 3] = ty2 98 | boxes[maxpos, 4] = ts 99 | 100 | tx1 = boxes[i, 0] 101 | ty1 = boxes[i, 1] 102 | tx2 = boxes[i, 2] 103 | ty2 = boxes[i, 3] 104 | ts = boxes[i, 4] 105 | 106 | pos = i + 1 107 | # NMS iterations, note that N changes if detection boxes fall below threshold 108 | while pos < N: 109 | x1 = boxes[pos, 0] 110 | y1 = boxes[pos, 1] 111 | x2 = boxes[pos, 2] 112 | y2 = boxes[pos, 3] 113 | s = boxes[pos, 4] 114 | 115 | area = (x2 - x1 + 1) * (y2 - y1 + 1) 116 | iw = (min(tx2, x2) - max(tx1, x1) + 1) 117 | if iw > 0: 118 | ih = (min(ty2, y2) - max(ty1, y1) + 1) 119 | if ih > 0: 120 | ua = float((tx2 - tx1 + 1) * (ty2 - ty1 + 1) + area - 121 | iw * ih) 122 | ov = iw * ih / ua #iou between max box and detection box 123 | 124 | if method == 1: # linear 125 | if ov > Nt: 126 | weight = 1 - ov 127 | else: 128 | weight = 1 129 | elif method == 2: # gaussian 130 | weight = np.exp(-(ov * ov) / sigma) 131 | else: # original NMS 132 | if ov > Nt: 133 | weight = 0 134 | else: 135 | weight = 1 136 | 137 | boxes[pos, 4] = weight * boxes[pos, 4] 138 | 139 | # if box score falls below threshold, discard the box by swapping with last box 140 | # update N 141 | if boxes[pos, 4] < threshold: 142 | boxes[pos, 0] = boxes[N - 1, 0] 143 | boxes[pos, 1] = boxes[N - 1, 1] 144 | boxes[pos, 2] = boxes[N - 1, 2] 145 | boxes[pos, 3] = boxes[N - 1, 3] 146 | boxes[pos, 4] = boxes[N - 1, 4] 147 | N = N - 1 148 | pos = pos - 1 149 | 150 | pos = pos + 1 151 | 152 | keep = [i for i in range(N)] 153 | return keep 154 | 155 | -------------------------------------------------------------------------------- /NUSCENES-GUIDE.md: -------------------------------------------------------------------------------- 1 | # Nuscenes Train and Eval guide 2 | 3 | ## General Tips 4 | 5 | * Nuscenes dataset evaluation contains many hard examples, you need to modify nms parameters (decrease score threshold, increase max size). You can use ```v1.0-mini``` to tune them. 6 | 7 | * Nuscenes dataset contain sweeps. You need to use 10 sweeps if you want to get good detection scores. Key-frame only can't get good result, so I drop support for that. 8 | 9 | * Nuscenes dataset contains 28130 train samples and 6019 validation samples. Use Nuscenes mini train set (my custom split, ~3500 samples) when develop if you don't have 4+ GPUs. See ```NuscenesDatasetD8``` for more details. 10 | 11 | * Some data augmentation will harm detection performance such as global rotation if their value is too large. 12 | 13 | * Use KITTI pretrain model if possible. You can use a pointpillars xyres_16 car model in [google drive](https://drive.google.com/open?id=1YOpgRkBgmSAJwMknoXmitEArNitZz63C) as pretrained model. 14 | 15 | * Multi-class NMS can increase performance of large objects. The parameters of mcnms should be grid-searched, don't enable mcmns during develop. 16 | 17 | * Database sample can greatly increase performance of bus. Currently no effect on other class. 18 | 19 | ## Config Guide 20 | 21 | ### Anchor Generator 22 | 23 | 1. use ```get_all_box_mean``` in nuscenes_dataset.py to get mean values of all boxes for each class. 24 | 25 | 2. change ```size``` and z-center in ```anchor_ranges``` in ```anchor_generator_range```. 26 | 27 | 3. choose thresholds: use ```helper_tune_target_assigner``` to get instance count and assigned anchors count. Then tune them. 28 | 29 | 4. add ```region_similarity_calculator```. If your anchors are too sparse, you need to use ```distance_similarity``` instead of ```nearest_iou_similarity``` for small classes such as pedestrian. 30 | 31 | 5. If you want to train with velocity, add ```custom_values``` to anchor generator. you can add two zeros. After that, anchors' shape will become ```[N, 9]```. 32 | 33 | ### Preprocess 34 | 35 | 1. disable all ground-truth noise. 36 | 37 | 2. ```global_rotation_uniform_noise``` may decrease performance. 38 | 39 | 3. disable ```database_sampler``` by delete all content in ```database_sampler```. 40 | 41 | ### Train 42 | 43 | Use ```set_train_step``` in utils.config_tool.train if you don't want to calculate them manually. 44 | 45 | ## Develop Guide 46 | 47 | * uncomment vis functions in prep_pointcloud to see assigned anchors and point cloud after data augmentation to ensure no bug in preprocess. 48 | 49 | * use code such as code in script_server.py instead of use commands in terminal. 50 | 51 | ## Config reference 52 | 53 | * all.fhd.config 54 | 55 | 1. Block filtering is enabled by default to decrease number of voxels. 56 | 57 | 2. Use ```upsample_strides: [0.5]``` to decrease number of anchors and increase training speed. 58 | 59 | * all.pp.mhead.config 60 | 61 | This config use custom ```VoxelNet``` class. 62 | 63 | 1. Use ```VoxelNetNuscenesMultiHead``` and write a multi-head network to handle small objects. 64 | 65 | 2. Output of multi head network MUST match order of class settings in config file. 66 | 67 | 3. If you use custom VoxelNet class, you MUST set feature_map_size for every class in class settings. 68 | 69 | * all.pp.largea.config 70 | 71 | This config use only large anchors. 72 | 73 | 1. you can use ```no_anchor``` to disable anchor generation of a class. 74 | 75 | 2. you must set ```assign_per_class``` to false when use ```no_anchor```. 76 | 77 | 78 | ## Reference Performance (Single GPU) 79 | 80 | * all.pp.lowa.config: 30 epoch, 1/2 dataset (NuscenesDatasetD2), train speed: 12 sample/s, ~50000 anchors 81 | 82 | ``` 83 | car Nusc dist AP@0.5, 1.0, 2.0, 4.0 84 | 58.85, 76.12, 80.65, 82.49 85 | bicycle Nusc dist AP@0.5, 1.0, 2.0, 4.0 86 | 0.00, 0.00, 0.00, 0.00 87 | bus Nusc dist AP@0.5, 1.0, 2.0, 4.0 88 | 2.55, 15.42, 27.19, 32.03 89 | construction_vehicle Nusc dist AP@0.5, 1.0, 2.0, 4.0 90 | 0.00, 0.00, 0.02, 0.31 91 | motorcycle Nusc dist AP@0.5, 1.0, 2.0, 4.0 92 | 8.61, 14.30, 15.00, 15.53 93 | pedestrian Nusc dist AP@0.5, 1.0, 2.0, 4.0 94 | 39.14, 49.29, 53.50, 57.03 95 | traffic_cone Nusc dist AP@0.5, 1.0, 2.0, 4.0 96 | 12.58, 18.92, 22.79, 27.99 97 | trailer Nusc dist AP@0.5, 1.0, 2.0, 4.0 98 | 0.00, 1.10, 7.42, 20.91 99 | truck Nusc dist AP@0.5, 1.0, 2.0, 4.0 100 | 5.44, 15.78, 22.77, 27.05 101 | barrier Nusc dist AP@0.5, 1.0, 2.0, 4.0 102 | 7.54, 34.54, 44.52, 49.80 103 | ``` 104 | 105 | * all.pp.mida.config: 30 epoch, 1/2 dataset (NuscenesDatasetD2), train speed: 10 sample/s, ~200000 anchors 106 | ``` 107 | car Nusc dist AP@0.5, 1.0, 2.0, 4.0 108 | 61.91, 76.75, 80.94, 82.53 109 | bicycle Nusc dist AP@0.5, 1.0, 2.0, 4.0 110 | 0.00, 0.00, 0.00, 0.00 111 | bus Nusc dist AP@0.5, 1.0, 2.0, 4.0 112 | 8.72, 25.76, 39.13, 42.55 113 | construction_vehicle Nusc dist AP@0.5, 1.0, 2.0, 4.0 114 | 0.00, 0.00, 0.27, 1.18 115 | motorcycle Nusc dist AP@0.5, 1.0, 2.0, 4.0 116 | 12.62, 17.82, 18.31, 18.82 117 | pedestrian Nusc dist AP@0.5, 1.0, 2.0, 4.0 118 | 55.50, 58.74, 61.27, 63.93 119 | traffic_cone Nusc dist AP@0.5, 1.0, 2.0, 4.0 120 | 17.47, 20.30, 23.56, 28.63 121 | trailer Nusc dist AP@0.5, 1.0, 2.0, 4.0 122 | 0.00, 4.76, 18.82, 28.26 123 | truck Nusc dist AP@0.5, 1.0, 2.0, 4.0 124 | 8.39, 20.62, 27.37, 31.12 125 | barrier Nusc dist AP@0.5, 1.0, 2.0, 4.0 126 | 10.61, 31.31, 40.60, 46.33 127 | ``` 128 | * all.pp.config: 50 epoch, 1/8 dataset (NuscenesDatasetD8), train speed: 4 sample/s, ~1200000 anchors 129 | 130 | ``` 131 | car Nusc dist AP@0.5, 1.0, 2.0, 4.0 132 | 62.90, 73.07, 76.77, 78.79 133 | bicycle Nusc dist AP@0.5, 1.0, 2.0, 4.0 134 | 0.00, 0.00, 0.00, 0.00 135 | bus Nusc dist AP@0.5, 1.0, 2.0, 4.0 136 | 9.53, 26.17, 38.01, 40.60 137 | construction_vehicle Nusc dist AP@0.5, 1.0, 2.0, 4.0 138 | 0.00, 0.00, 0.44, 1.43 139 | motorcycle Nusc dist AP@0.5, 1.0, 2.0, 4.0 140 | 9.25, 12.90, 13.69, 14.11 141 | pedestrian Nusc dist AP@0.5, 1.0, 2.0, 4.0 142 | 61.44, 62.61, 64.09, 66.35 143 | traffic_cone Nusc dist AP@0.5, 1.0, 2.0, 4.0 144 | 11.63, 13.14, 15.81, 21.22 145 | trailer Nusc dist AP@0.5, 1.0, 2.0, 4.0 146 | 0.80, 9.90, 17.61, 23.26 147 | truck Nusc dist AP@0.5, 1.0, 2.0, 4.0 148 | 9.81, 21.40, 27.55, 30.34 149 | ``` 150 | 151 | * all.fhd.config: train speed: 5.5 sample/s. comming soon 152 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_20.config: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | network_class_name: "VoxelNet" 4 | voxel_generator { 5 | full_empty_part_with_mean: false 6 | point_cloud_range : [0, -40, -3, 70.4, 40, 1] 7 | voxel_size : [0.2, 0.2, 4] 8 | max_number_of_points_per_voxel : 100 9 | } 10 | voxel_feature_extractor: { 11 | module_class_name: "PillarFeatureNet" 12 | num_filters: [64] 13 | with_distance: false 14 | num_input_features: 4 15 | } 16 | middle_feature_extractor: { 17 | module_class_name: "PointPillarsScatter" 18 | downsample_factor: 1 19 | num_input_features: 64 20 | } 21 | rpn: { 22 | module_class_name: "RPNV2" 23 | layer_nums: [3, 5, 5] 24 | layer_strides: [2, 2, 2] 25 | num_filters: [64, 128, 256] 26 | upsample_strides: [1, 2, 4] 27 | num_upsample_filters: [128, 128, 128] 28 | use_groupnorm: false 29 | num_groups: 32 30 | num_input_features: 64 31 | } 32 | loss: { 33 | classification_loss: { 34 | weighted_sigmoid_focal: { 35 | alpha: 0.25 36 | gamma: 2.0 37 | anchorwise_output: true 38 | } 39 | } 40 | localization_loss: { 41 | weighted_smooth_l1: { 42 | sigma: 3.0 43 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 44 | } 45 | } 46 | classification_weight: 1.0 47 | localization_weight: 2.0 48 | } 49 | num_point_features: 4 # model's num point feature should be independent of dataset 50 | # Outputs 51 | use_sigmoid_score: true 52 | encode_background_as_zeros: true 53 | encode_rad_error_by_sin: true 54 | 55 | use_direction_classifier: true 56 | direction_loss_weight: 0.2 57 | num_direction_bins: 2 58 | direction_limit_offset: 1 59 | 60 | # Loss 61 | pos_class_weight: 1.0 62 | neg_class_weight: 1.0 63 | 64 | loss_norm_type: NormByNumPositives 65 | # Postprocess 66 | post_center_limit_range: [0, -40, -5, 70.4, 40, 5] 67 | 68 | nms_class_agnostic: false # only valid in multi-class nms 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | class_settings: { 77 | anchor_generator_stride: { 78 | sizes: [1.6, 3.9, 1.56] # wlh 79 | strides: [0.4, 0.4, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.2, -39.8, -1.78] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | } 83 | matched_threshold : 0.6 84 | unmatched_threshold : 0.45 85 | class_name: "Car" 86 | use_rotate_nms: false 87 | use_multi_class_nms: false 88 | nms_pre_max_size: 1000 89 | nms_post_max_size: 300 90 | nms_score_threshold: 0.05 91 | nms_iou_threshold: 0.5 92 | region_similarity_calculator: { 93 | nearest_iou_similarity: { 94 | } 95 | } 96 | } 97 | sample_positive_fraction : -1 98 | sample_size : 512 99 | assign_per_class: true 100 | } 101 | } 102 | } 103 | 104 | train_input_reader: { 105 | dataset: { 106 | dataset_class_name: "KittiDataset" 107 | kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_train.pkl" 108 | kitti_root_path: "/media/yy/960evo/datasets/kitti" 109 | } 110 | 111 | batch_size: 2 112 | preprocess: { 113 | max_number_of_voxels: 12000 114 | shuffle_points: true 115 | num_workers: 2 116 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 117 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 118 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 119 | global_scaling_uniform_noise: [0.95, 1.05] 120 | global_random_rotation_range_per_object: [0, 0] 121 | global_translate_noise_std: [0.2, 0.2, 0.2] 122 | anchor_area_threshold: 1 123 | remove_points_after_sample: false 124 | groundtruth_points_drop_percentage: 0.0 125 | groundtruth_drop_max_keep_points: 15 126 | remove_unknown_examples: false 127 | sample_importance: 1.0 128 | random_flip_x: false 129 | random_flip_y: true 130 | remove_environment: false 131 | database_sampler { 132 | database_info_path: "/media/yy/960evo/datasets/kitti/kitti_dbinfos_train.pkl" 133 | sample_groups { 134 | name_to_max_num { 135 | key: "Car" 136 | value: 15 137 | } 138 | } 139 | database_prep_steps { 140 | filter_by_min_num_points { 141 | min_num_point_pairs { 142 | key: "Car" 143 | value: 5 144 | } 145 | } 146 | } 147 | database_prep_steps { 148 | filter_by_difficulty { 149 | removed_difficulties: [-1] 150 | } 151 | } 152 | global_random_rotation_range_per_object: [0, 0] 153 | rate: 1.0 154 | } 155 | } 156 | } 157 | 158 | train_config: { 159 | optimizer: { 160 | adam_optimizer: { 161 | learning_rate: { 162 | exponential_decay: { 163 | initial_learning_rate: 0.0002 164 | decay_length: 0.1 165 | decay_factor: 0.8 166 | staircase: True 167 | } 168 | } 169 | weight_decay: 0.0001 170 | } 171 | fixed_weight_decay: false 172 | use_moving_average: false 173 | } 174 | steps: 296960 # 92800 # 1856 steps per epoch * 160 epochs 175 | steps_per_eval: 9280 # 9280 # 1856 steps per epoch * 5 epochs 176 | save_checkpoints_secs : 1800 # half hour 177 | save_summary_steps : 10 178 | enable_mixed_precision: false 179 | loss_scale_factor: -1 180 | clear_metrics_every_epoch: true 181 | } 182 | 183 | eval_input_reader: { 184 | dataset: { 185 | dataset_class_name: "KittiDataset" 186 | kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_val.pkl" 187 | # kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_test.pkl" 188 | kitti_root_path: "/media/yy/960evo/datasets/kitti" 189 | } 190 | batch_size: 2 191 | 192 | preprocess: { 193 | max_number_of_voxels: 12000 194 | shuffle_points: false 195 | num_workers: 3 196 | anchor_area_threshold: 1 197 | remove_environment: false 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_28.config: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | network_class_name: "VoxelNet" 4 | voxel_generator { 5 | full_empty_part_with_mean: false 6 | point_cloud_range : [0, -40.32, -3, 71.68, 40.32, 1] 7 | voxel_size : [0.28, 0.28, 4] 8 | max_number_of_points_per_voxel : 100 9 | } 10 | voxel_feature_extractor: { 11 | module_class_name: "PillarFeatureNet" 12 | num_filters: [64] 13 | with_distance: false 14 | num_input_features: 4 15 | } 16 | middle_feature_extractor: { 17 | module_class_name: "PointPillarsScatter" 18 | downsample_factor: 1 19 | num_input_features: 64 20 | } 21 | rpn: { 22 | module_class_name: "RPNV2" 23 | layer_nums: [3, 5, 5] 24 | layer_strides: [2, 2, 2] 25 | num_filters: [64, 128, 256] 26 | upsample_strides: [1, 2, 4] 27 | num_upsample_filters: [128, 128, 128] 28 | use_groupnorm: false 29 | num_groups: 32 30 | num_input_features: 64 31 | } 32 | loss: { 33 | classification_loss: { 34 | weighted_sigmoid_focal: { 35 | alpha: 0.25 36 | gamma: 2.0 37 | anchorwise_output: true 38 | } 39 | } 40 | localization_loss: { 41 | weighted_smooth_l1: { 42 | sigma: 3.0 43 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 44 | } 45 | } 46 | classification_weight: 1.0 47 | localization_weight: 2.0 48 | } 49 | num_point_features: 4 # model's num point feature should be independent of dataset 50 | # Outputs 51 | use_sigmoid_score: true 52 | encode_background_as_zeros: true 53 | encode_rad_error_by_sin: true 54 | 55 | use_direction_classifier: true 56 | direction_loss_weight: 0.2 57 | num_direction_bins: 2 58 | direction_limit_offset: 1 59 | 60 | # Loss 61 | pos_class_weight: 1.0 62 | neg_class_weight: 1.0 63 | 64 | loss_norm_type: NormByNumPositives 65 | # Postprocess 66 | post_center_limit_range: [0, -40.32, -5, 71.68, 40.32, 5] 67 | nms_class_agnostic: false # only valid in multi-class nms 68 | box_coder: { 69 | ground_box3d_coder: { 70 | linear_dim: false 71 | encode_angle_vector: false 72 | } 73 | } 74 | target_assigner: { 75 | class_settings: { 76 | anchor_generator_stride: { 77 | sizes: [1.6, 3.9, 1.56] # wlh 78 | strides: [0.56, 0.56, 0.0] # if generate only 1 z_center, z_stride will be ignored 79 | offsets: [0.28, -40.04, -1.78] # origin_offset + strides / 2 80 | rotations: [0, 1.57] # 0, pi/2 81 | } 82 | matched_threshold : 0.6 83 | unmatched_threshold : 0.45 84 | class_name: "Car" 85 | use_rotate_nms: false 86 | use_multi_class_nms: false 87 | nms_pre_max_size: 1000 88 | nms_post_max_size: 300 89 | nms_score_threshold: 0.05 90 | nms_iou_threshold: 0.5 91 | region_similarity_calculator: { 92 | nearest_iou_similarity: { 93 | } 94 | } 95 | } 96 | sample_positive_fraction : -1 97 | sample_size : 512 98 | assign_per_class: true 99 | } 100 | } 101 | } 102 | 103 | train_input_reader: { 104 | dataset: { 105 | dataset_class_name: "KittiDataset" 106 | kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_train.pkl" 107 | kitti_root_path: "/media/yy/960evo/datasets/kitti" 108 | } 109 | 110 | batch_size: 2 111 | preprocess: { 112 | max_number_of_voxels: 12000 113 | shuffle_points: true 114 | num_workers: 2 115 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 116 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 117 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 118 | global_scaling_uniform_noise: [0.95, 1.05] 119 | global_random_rotation_range_per_object: [0, 0] 120 | global_translate_noise_std: [0.2, 0.2, 0.2] 121 | anchor_area_threshold: 1 122 | remove_points_after_sample: false 123 | groundtruth_points_drop_percentage: 0.0 124 | groundtruth_drop_max_keep_points: 15 125 | remove_unknown_examples: false 126 | sample_importance: 1.0 127 | random_flip_x: false 128 | random_flip_y: true 129 | remove_environment: false 130 | database_sampler { 131 | database_info_path: "/media/yy/960evo/datasets/kitti/kitti_dbinfos_train.pkl" 132 | sample_groups { 133 | name_to_max_num { 134 | key: "Car" 135 | value: 15 136 | } 137 | } 138 | database_prep_steps { 139 | filter_by_min_num_points { 140 | min_num_point_pairs { 141 | key: "Car" 142 | value: 5 143 | } 144 | } 145 | } 146 | database_prep_steps { 147 | filter_by_difficulty { 148 | removed_difficulties: [-1] 149 | } 150 | } 151 | global_random_rotation_range_per_object: [0, 0] 152 | rate: 1.0 153 | } 154 | } 155 | } 156 | 157 | train_config: { 158 | optimizer: { 159 | adam_optimizer: { 160 | learning_rate: { 161 | exponential_decay: { 162 | initial_learning_rate: 0.0002 163 | decay_length: 0.1 164 | decay_factor: 0.8 165 | staircase: True 166 | } 167 | } 168 | weight_decay: 0.0001 169 | } 170 | fixed_weight_decay: false 171 | use_moving_average: false 172 | } 173 | steps: 296960 # 92800 # 1856 steps per epoch * 160 epochs 174 | steps_per_eval: 9280 # 9280 # 1856 steps per epoch * 5 epochs 175 | save_checkpoints_secs : 1800 # half hour 176 | save_summary_steps : 10 177 | enable_mixed_precision: false 178 | loss_scale_factor: -1 179 | clear_metrics_every_epoch: true 180 | } 181 | 182 | eval_input_reader: { 183 | dataset: { 184 | dataset_class_name: "KittiDataset" 185 | kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_val.pkl" 186 | # kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_test.pkl" 187 | kitti_root_path: "/media/yy/960evo/datasets/kitti" 188 | } 189 | batch_size: 2 190 | 191 | preprocess: { 192 | max_number_of_voxels: 12000 193 | shuffle_points: false 194 | num_workers: 3 195 | anchor_area_threshold: 1 196 | remove_environment: false 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /second/configs/pointpillars/car/xyres_24.config: -------------------------------------------------------------------------------- 1 | model: { 2 | second: { 3 | network_class_name: "VoxelNet" 4 | voxel_generator { 5 | full_empty_part_with_mean: false 6 | point_cloud_range : [0, -40.32, -3, 71.04, 40.32, 1] 7 | voxel_size : [0.24, 0.24, 4] 8 | max_number_of_points_per_voxel : 100 9 | } 10 | voxel_feature_extractor: { 11 | module_class_name: "PillarFeatureNet" 12 | num_filters: [64] 13 | with_distance: false 14 | num_input_features: 4 15 | } 16 | middle_feature_extractor: { 17 | module_class_name: "PointPillarsScatter" 18 | downsample_factor: 1 19 | num_input_features: 64 20 | } 21 | rpn: { 22 | module_class_name: "RPNV2" 23 | layer_nums: [3, 5, 5] 24 | layer_strides: [2, 2, 2] 25 | num_filters: [64, 128, 256] 26 | upsample_strides: [1, 2, 4] 27 | num_upsample_filters: [128, 128, 128] 28 | use_groupnorm: false 29 | num_groups: 32 30 | num_input_features: 64 31 | } 32 | loss: { 33 | classification_loss: { 34 | weighted_sigmoid_focal: { 35 | alpha: 0.25 36 | gamma: 2.0 37 | anchorwise_output: true 38 | } 39 | } 40 | localization_loss: { 41 | weighted_smooth_l1: { 42 | sigma: 3.0 43 | code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] 44 | } 45 | } 46 | classification_weight: 1.0 47 | localization_weight: 2.0 48 | } 49 | num_point_features: 4 # model's num point feature should be independent of dataset 50 | # Outputs 51 | use_sigmoid_score: true 52 | encode_background_as_zeros: true 53 | encode_rad_error_by_sin: true 54 | 55 | use_direction_classifier: true 56 | direction_loss_weight: 0.2 57 | num_direction_bins: 2 58 | direction_limit_offset: 1 59 | 60 | # Loss 61 | pos_class_weight: 1.0 62 | neg_class_weight: 1.0 63 | 64 | loss_norm_type: NormByNumPositives 65 | # Postprocess 66 | post_center_limit_range: [0, -40.32, -5, 71.04, 40.32, 5] 67 | 68 | nms_class_agnostic: false # only valid in multi-class nms 69 | box_coder: { 70 | ground_box3d_coder: { 71 | linear_dim: false 72 | encode_angle_vector: false 73 | } 74 | } 75 | target_assigner: { 76 | class_settings: { 77 | anchor_generator_stride: { 78 | sizes: [1.6, 3.9, 1.56] # wlh 79 | strides: [0.48, 0.48, 0.0] # if generate only 1 z_center, z_stride will be ignored 80 | offsets: [0.24, -40.08, -1.78] # origin_offset + strides / 2 81 | rotations: [0, 1.57] # 0, pi/2 82 | } 83 | matched_threshold : 0.6 84 | unmatched_threshold : 0.45 85 | class_name: "Car" 86 | use_rotate_nms: false 87 | use_multi_class_nms: false 88 | nms_pre_max_size: 1000 89 | nms_post_max_size: 300 90 | nms_score_threshold: 0.05 91 | nms_iou_threshold: 0.5 92 | 93 | region_similarity_calculator: { 94 | nearest_iou_similarity: { 95 | } 96 | } 97 | } 98 | sample_positive_fraction : -1 99 | sample_size : 512 100 | assign_per_class: true 101 | } 102 | } 103 | } 104 | 105 | train_input_reader: { 106 | dataset: { 107 | dataset_class_name: "KittiDataset" 108 | kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_train.pkl" 109 | kitti_root_path: "/media/yy/960evo/datasets/kitti" 110 | } 111 | 112 | batch_size: 2 113 | preprocess: { 114 | max_number_of_voxels: 12000 115 | shuffle_points: true 116 | num_workers: 2 117 | groundtruth_localization_noise_std: [0.25, 0.25, 0.25] 118 | groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267] 119 | global_rotation_uniform_noise: [-0.78539816, 0.78539816] 120 | global_scaling_uniform_noise: [0.95, 1.05] 121 | global_random_rotation_range_per_object: [0, 0] 122 | global_translate_noise_std: [0.2, 0.2, 0.2] 123 | anchor_area_threshold: 1 124 | remove_points_after_sample: false 125 | groundtruth_points_drop_percentage: 0.0 126 | groundtruth_drop_max_keep_points: 15 127 | remove_unknown_examples: false 128 | sample_importance: 1.0 129 | random_flip_x: false 130 | random_flip_y: true 131 | remove_environment: false 132 | database_sampler { 133 | database_info_path: "/media/yy/960evo/datasets/kitti/kitti_dbinfos_train.pkl" 134 | sample_groups { 135 | name_to_max_num { 136 | key: "Car" 137 | value: 15 138 | } 139 | } 140 | database_prep_steps { 141 | filter_by_min_num_points { 142 | min_num_point_pairs { 143 | key: "Car" 144 | value: 5 145 | } 146 | } 147 | } 148 | database_prep_steps { 149 | filter_by_difficulty { 150 | removed_difficulties: [-1] 151 | } 152 | } 153 | global_random_rotation_range_per_object: [0, 0] 154 | rate: 1.0 155 | } 156 | } 157 | } 158 | 159 | train_config: { 160 | optimizer: { 161 | adam_optimizer: { 162 | learning_rate: { 163 | exponential_decay: { 164 | initial_learning_rate: 0.0002 165 | decay_length: 0.1 166 | decay_factor: 0.8 167 | staircase: True 168 | } 169 | } 170 | weight_decay: 0.0001 171 | } 172 | fixed_weight_decay: false 173 | use_moving_average: false 174 | } 175 | steps: 296960 # 92800 # 1856 steps per epoch * 160 epochs 176 | steps_per_eval: 9280 # 9280 # 1856 steps per epoch * 5 epochs 177 | save_checkpoints_secs : 1800 # half hour 178 | save_summary_steps : 10 179 | enable_mixed_precision: false 180 | loss_scale_factor: -1 181 | clear_metrics_every_epoch: true 182 | } 183 | 184 | eval_input_reader: { 185 | dataset: { 186 | dataset_class_name: "KittiDataset" 187 | kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_val.pkl" 188 | # kitti_info_path: "/media/yy/960evo/datasets/kitti/kitti_infos_test.pkl" 189 | kitti_root_path: "/media/yy/960evo/datasets/kitti" 190 | } 191 | batch_size: 2 192 | 193 | preprocess: { 194 | max_number_of_voxels: 12000 195 | shuffle_points: false 196 | num_workers: 3 197 | anchor_area_threshold: 1 198 | remove_environment: false 199 | } 200 | } 201 | --------------------------------------------------------------------------------