├── LICENSE ├── README.md ├── docs ├── data_preparation.md ├── inference.md └── setup.md ├── figs └── framework.png ├── projects ├── configs │ └── PersPETR │ │ ├── .ipynb_checkpoints │ │ ├── persdetr3d_vov_800_bs2_seq_24e-checkpoint.py │ │ └── persdetr3d_vov_800_bs2_seq_24e_testing-checkpoint.py │ │ ├── persdetr3d_vov_800_bs2_seq_24e.py │ │ └── persdetr3d_vov_800_bs2_seq_24e_testing.py └── mmdet3d_plugin │ ├── __init__.py │ ├── __pycache__ │ └── __init__.cpython-38.pyc │ ├── core │ ├── apis │ │ ├── .ipynb_checkpoints │ │ │ └── test-checkpoint.py │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-38.pyc │ │ │ ├── mmdet_train.cpython-38.pyc │ │ │ ├── test.cpython-38.pyc │ │ │ └── train.cpython-38.pyc │ │ ├── mmdet_train.py │ │ ├── test.py │ │ └── train.py │ ├── bbox │ │ ├── __pycache__ │ │ │ └── util.cpython-38.pyc │ │ ├── assigners │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ ├── hungarian_assigner_2d.cpython-38.pyc │ │ │ │ └── hungarian_assigner_3d.cpython-38.pyc │ │ │ ├── hungarian_assigner_2d.py │ │ │ └── hungarian_assigner_3d.py │ │ ├── coders │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ └── nms_free_coder.cpython-38.pyc │ │ │ └── nms_free_coder.py │ │ ├── match_costs │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-38.pyc │ │ │ │ └── match_cost.cpython-38.pyc │ │ │ └── match_cost.py │ │ └── util.py │ └── evaluation │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ └── eval_hooks.cpython-38.pyc │ │ └── eval_hooks.py │ ├── datasets │ ├── .ipynb_checkpoints │ │ ├── builder-checkpoint.py │ │ └── nuscenes_dataset-checkpoint.py │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── builder.cpython-38.pyc │ │ └── nuscenes_dataset.cpython-38.pyc │ ├── builder.py │ ├── nuscenes_dataset.py │ ├── pipelines │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-38.pyc │ │ │ ├── formating.cpython-38.pyc │ │ │ └── transform_3d.cpython-38.pyc │ │ ├── formating.py │ │ └── transform_3d.py │ ├── recall_3d.py │ ├── results_save_total.pkl │ └── samplers │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── distributed_sampler.cpython-38.pyc │ │ ├── group_sampler.cpython-38.pyc │ │ └── sampler.cpython-38.pyc │ │ ├── distributed_sampler.py │ │ ├── group_sampler.py │ │ └── sampler.py │ └── models │ ├── backbones │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── vovnet.cpython-38.pyc │ │ └── vovnetcp.cpython-38.pyc │ ├── vovnet.py │ └── vovnetcp.py │ ├── dense_heads │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── focal_head.cpython-38.pyc │ │ ├── perspective_head.cpython-38.pyc │ │ ├── petr_head_dn.cpython-38.pyc │ │ ├── sparse_head.cpython-38.pyc │ │ ├── streampetr_head.cpython-38.pyc │ │ └── yolox_head.cpython-38.pyc │ ├── focal_head.py │ ├── perspective_head.py │ ├── petr_head_dn.py │ ├── sparse_head.py │ ├── streampetr_head.py │ └── yolox_head.py │ ├── detectors │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── persdetr3d.cpython-38.pyc │ │ ├── petr3d.cpython-38.pyc │ │ └── repdetr3d.cpython-38.pyc │ ├── persdetr3d.py │ ├── petr3d.py │ └── repdetr3d.py │ ├── necks │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ └── cp_fpn.cpython-38.pyc │ └── cp_fpn.py │ └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── attention.cpython-38.pyc │ ├── checkviews.cpython-38.pyc │ ├── detr3d_transformer.cpython-38.pyc │ ├── grid_mask.cpython-38.pyc │ ├── misc.cpython-38.pyc │ ├── multi_scale_deformable_attn_function.cpython-38.pyc │ ├── petr_transformer.cpython-38.pyc │ ├── positional_encoding.cpython-38.pyc │ ├── prompt.cpython-38.pyc │ └── proposals_generation.cpython-38.pyc │ ├── attention.py │ ├── checkviews.py │ ├── detr3d_transformer.py │ ├── grid_mask.py │ ├── misc.py │ ├── multi_scale_deformable_attn_function.py │ ├── petr_transformer.py │ ├── positional_encoding.py │ ├── prompt.py │ └── proposals_generation.py └── tools ├── .ipynb_checkpoints ├── dist_test-checkpoint.sh └── test-checkpoint.py ├── __pycache__ └── visual_nuscenes.cpython-38.pyc ├── benchmark.py ├── cc ├── 0.jpg ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg └── 5.jpg ├── cc_0.jpg ├── cc_1.jpg ├── cc_3.jpg ├── cc_5.jpg ├── create_data_nusc.py ├── data_converter ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── __init__.cpython-39.pyc │ ├── nuscenes_converter.cpython-38.pyc │ └── nuscenes_converter.cpython-39.pyc └── nuscenes_converter.py ├── dist_test.sh ├── dist_train.sh ├── multi_dist_train.sh ├── slurm_test.sh ├── slurm_train.sh ├── test.py ├── train.py ├── visual_nuscenes.py └── visualize.py /README.md: -------------------------------------------------------------------------------- 1 |
2 |

[CVPR 2024] Enhancing 3D Object Detection with 2D Detection-Guided Query Anchors

3 |
4 | 5 | 6 | 7 |
8 | 9 |

10 | 11 | 12 | ## Introduction 13 | 14 | This repository is the official implementation of our paper "Enhancing 3D Object Detection with 2D Detection-Guided Query Anchors, CVPR 2024 [![arXiv](https://img.shields.io/badge/arXiv-Paper-.svg)](https://arxiv.org/abs/2403.06093)". Our code is based on [StreamPETR](https://github.com/exiawsh/StreamPETR). 15 | 16 | ## Getting Started 17 | 18 | Please follow the docs below. 19 | 20 | 1. [**Environment Setup.**](./docs/setup.md) 21 | 2. [**Data Preparation.**](./docs/data_preparation.md) 22 | 3. [**Inference.**](./docs/inference.md) 23 | 24 | ## Results on NuScenes Val Set. 25 | 26 | | Methods | Backbone | Image Size | NDS | mAP | config | model | 27 | | ----------------------- | -------- | ---------- | ---- | ---- | ------------------------------------------------------------ | ------------------------------------------------------------ | 28 | | StreamPETR | V2-99 | 320×800 | 57.1 | 48.2 | - | - | 29 | | StreamPETR-QAF2D (Ours) | V2-99 | 320×800 | 58.8 | 49.8 | [config](projects/configs/PersPETR/persdetr3d_vov_800_bs2_seq_24e.py) | [model](https://drive.google.com/file/d/1JtGKOGjlOJe3yJyC58GASNE-uoa91ozn/view?usp=sharing) | 30 | 31 | Comparison of the base detectors and their QAF2D enhanced version on the nuScenes validation split. 32 | 33 | ### Note 34 | Due to some internal policies, we do not release the full codebase, and the current 2D detection results are read from a saved file. 35 | 36 | ## Citation 37 | 38 | If you find QAF2D useful in your research or applications, please consider citing it. Thank you. 39 | 40 | ```bibtex 41 | @inproceedings{ji2024enhancing, 42 | title={Enhancing 3D Object Detection with 2D Detection-Guided Query Anchors}, 43 | author={Ji, Haoxuanye and Liang, Pengpeng and Cheng, Erkang}, 44 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and 45 | Pattern Recognition}, 46 | pages={21178--21187}, 47 | year={2024} 48 | } 49 | ``` 50 | 51 | -------------------------------------------------------------------------------- /docs/data_preparation.md: -------------------------------------------------------------------------------- 1 | # Data Preparation 2 | 3 | ## Dataset 4 | **1. Download nuScenes** 5 | 6 | Download the [nuScenes dataset](https://www.nuscenes.org/download) to `./data/nuscenes`. 7 | 8 | ## 2. Creating infos file 9 | 10 | We modify data preparation in `MMDetection3D`, which addtionally creates 2D annotations and temporal information for training/evaluation. 11 | ```shell 12 | python tools/create_data_nusc.py --root-path ./data/nuscenes --out-dir ./data/nuscenes --extra-tag nuscenes2d --version v1.0 13 | ``` 14 | 15 | Using the above code will generate `nuscenes2d_temporal_infos_{train,val}.pkl`. 16 | We also privided the processed [train](https://github.com/exiawsh/storage/releases/download/v1.0/nuscenes2d_temporal_infos_train.pkl), [val](https://github.com/exiawsh/storage/releases/download/v1.0/nuscenes2d_temporal_infos_val.pkl) and [test](https://github.com/exiawsh/storage/releases/download/v1.0/nuscenes2d_temporal_infos_test.pkl) pkl. 17 | 18 | 19 | 20 | * After preparation, you will be able to see the following directory structure: 21 | 22 | **Folder structure** 23 | 24 | ``` 25 | QAF2D 26 | ├── projects/ 27 | ├── mmdetection3d/ 28 | ├── tools/ 29 | ├── data/ 30 | │ ├── nuscenes/ 31 | │ │ ├── maps/ 32 | │ │ ├── samples/ 33 | │ │ ├── sweeps/ 34 | │ │ ├── v1.0-test/ 35 | | | ├── v1.0-trainval/ 36 | | | ├── nuscenes2d_temporal_infos_train.pkl 37 | | | ├── nuscenes2d_temporal_infos_val.pkl 38 | ``` 39 | 40 | In addition, you also need to rename nuscenes2d_temporal_infos_train.pkl to nuscenes2d_temporal_infos_train_stream.pkl and rename nuscenes2d_temporal_infos_val.pkl to nuscenes2d_temporal_infos_val_stream.pkl -------------------------------------------------------------------------------- /docs/inference.md: -------------------------------------------------------------------------------- 1 | # Inference 2 | 3 | 4 | 5 | You can evaluate the detection model following: 6 | 7 | ```bash 8 | tools/dist_test.sh projects/configs/PersPETR/persdetr3d_vov_800_bs2_seq_24e.py training_best_model_prompt/new_matching/latest.pth 8 --eval bbox 9 | ``` 10 | 11 | -------------------------------------------------------------------------------- /docs/setup.md: -------------------------------------------------------------------------------- 1 | # Environment Setup 2 | 3 | ## Base Environments 4 | Python >= 3.8 \ 5 | CUDA == 11.2 \ 6 | PyTorch == 1.9.0 \ 7 | mmdet3d == 1.0.0rc6 \ 8 | [flash-attn](https://github.com/HazyResearch/flash-attention) == 0.2.2 9 | 10 | **Notes**: 11 | - [flash-attn](https://github.com/HazyResearch/flash-attention) is an optional requirement, which can speedup and save GPU memory. If your device (e.g. TESLA V100) is not compatible with the flash-attn, you can skip its installation and comment the relevant [code](../projects/mmdet3d_plugin/models/utils). 12 | - It is also possible to consider installing version 1.9.0 + of Pytorch, but you need to find the appropriate flash-attn version (e.g. 0.2.8 for CUDA 11.6 and pytorch 1.13). 13 | 14 | 15 | ## Step-by-step installation instructions 16 | 17 | Following https://mmdetection3d.readthedocs.io/en/latest/getting_started.html#installation 18 | 19 | 20 | **a. Create a conda virtual environment and activate it.** 21 | ```shell 22 | conda create -n QAF2D python=3.8 -y 23 | conda activate QAF2D 24 | ``` 25 | 26 | **b. Install PyTorch and torchvision following the [official instructions](https://pytorch.org/).** 27 | ```shell 28 | pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 29 | # Recommended torch>=1.9 30 | ``` 31 | **c. Install flash-attn (optional).** 32 | ``` 33 | pip install flash-attn==0.2.2 34 | ``` 35 | 36 | **d. Clone QAF2D.** 37 | 38 | ``` 39 | git clone https://github.com/yzf99/QAF2D 40 | ``` 41 | 42 | **e. Install mmdet3d.** 43 | 44 | ```shell 45 | pip install mmcv-full==1.6.0 -f https://download.openmmlab.com/mmcv/dist/cu111/torch1.9.0/index.html 46 | pip install mmdet==2.28.2 47 | pip install mmsegmentation==0.30.0 48 | cd ./QAF2D 49 | git clone https://github.com/open-mmlab/mmdetection3d.git 50 | cd mmdetection3d 51 | git checkout v1.0.0rc6 52 | pip install -e . 53 | ``` -------------------------------------------------------------------------------- /figs/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/figs/framework.png -------------------------------------------------------------------------------- /projects/configs/PersPETR/.ipynb_checkpoints/persdetr3d_vov_800_bs2_seq_24e-checkpoint.py: -------------------------------------------------------------------------------- 1 | _base_ = [ 2 | '../../../mmdetection3d/configs/_base_/datasets/nus-3d.py', 3 | '../../../mmdetection3d/configs/_base_/default_runtime.py' 4 | ] 5 | backbone_norm_cfg = dict(type='LN', requires_grad=True) 6 | plugin = True 7 | plugin_dir = 'projects/mmdet3d_plugin/' 8 | 9 | # If point cloud range is changed, the models should also change their point 10 | # cloud range accordingly 11 | point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0] 12 | voxel_size = [0.2, 0.2, 8] 13 | img_norm_cfg = dict( 14 | mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395], to_rgb=False) # fix img_norm 15 | # For nuScenes we usually do 10-class detection 16 | class_names = [ 17 | 'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier', 18 | 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone' 19 | ] 20 | 21 | num_gpus = 8 22 | batch_size = 1 # 2 23 | num_iters_per_epoch = 28130 // (num_gpus * batch_size) 24 | num_epochs = 24 25 | 26 | queue_length = 1 27 | num_frame_losses = 1 28 | collect_keys = ['lidar2img', 'intrinsics', 'extrinsics', 29 | 'timestamp', 'img_timestamp', 'ego_pose', 'ego_pose_inv'] 30 | input_modality = dict( 31 | use_lidar=False, 32 | use_camera=True, 33 | use_radar=False, 34 | use_map=False, 35 | use_external=True) 36 | model = dict( 37 | type='PersDetr3D', 38 | num_frame_head_grads=num_frame_losses, 39 | num_frame_backbone_grads=num_frame_losses, 40 | num_frame_losses=num_frame_losses, 41 | use_grid_mask=True, 42 | stride=[8, 16, 32, 64], 43 | position_level=[0, 1, 2, 3], 44 | img_backbone=dict( 45 | type='VoVNetCP', # use checkpoint to save memory 46 | spec_name='V-99-eSE', 47 | norm_eval=True, 48 | frozen_stages=-1, 49 | input_ch=3, 50 | out_features=('stage2', 'stage3', 'stage4', 'stage5',)), 51 | img_neck=dict( 52 | type='FPN', # remove unused parameters 53 | start_level=1, 54 | add_extra_convs='on_output', 55 | relu_before_extra_convs=True, 56 | in_channels=[256, 512, 768, 1024], 57 | out_channels=256, 58 | num_outs=4), 59 | img_roi_head=dict( 60 | type='YOLOXHeadCustom', 61 | num_classes=10, 62 | in_channels=256, 63 | strides=[8, 16, 32, 64], 64 | train_cfg=dict(assigner=dict( 65 | type='SimOTAAssigner', center_radius=2.5)), 66 | test_cfg=dict(score_thr=0.01, nms=dict( 67 | type='nms', iou_threshold=0.65)), 68 | ), 69 | pts_bbox_head=dict( 70 | type='PerspectiveHead', 71 | num_classes=10, 72 | in_channels=256, 73 | num_query=644, 74 | memory_len=1024, 75 | topk_proposals=256, 76 | num_propagated=256, 77 | scalar=10, # noise groups 78 | noise_scale=1.0, 79 | dn_weight=1.0, # dn loss weight 80 | split=0.75, # positive rate 81 | with_dn=True, 82 | with_ego_pos=True, 83 | match_with_velo=False, 84 | code_weights=[2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 85 | transformer=dict( 86 | type='Detr3DTransformer', 87 | decoder=dict( 88 | type='Detr3DTransformerDecoder', 89 | embed_dims=256, 90 | num_layers=6, 91 | transformerlayers=dict( 92 | type='Detr3DTemporalDecoderLayer', 93 | batch_first=True, 94 | attn_cfgs=[ 95 | dict( 96 | type='MultiheadAttention', 97 | embed_dims=256, 98 | num_heads=8, 99 | dropout=0.1), 100 | dict( 101 | type='DeformableFeatureAggregationCuda', 102 | embed_dims=256, 103 | num_groups=8, 104 | num_levels=4, 105 | num_cams=6, 106 | dropout=0.1, 107 | num_pts=13, 108 | bias=2.), 109 | ], 110 | feedforward_channels=2048, 111 | ffn_dropout=0.1, 112 | with_cp=True, # use checkpoint to save memory 113 | operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 114 | 'ffn', 'norm')), 115 | )), 116 | bbox_coder=dict( 117 | type='NMSFreeCoder', 118 | post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], 119 | pc_range=point_cloud_range, 120 | max_num=300, 121 | voxel_size=voxel_size, 122 | num_classes=10), 123 | loss_cls=dict( 124 | type='FocalLoss', 125 | use_sigmoid=True, 126 | gamma=2.0, 127 | alpha=0.25, 128 | loss_weight=2.0), 129 | loss_bbox=dict(type='L1Loss', loss_weight=0.25), 130 | loss_iou=dict(type='GIoULoss', loss_weight=0.0),), 131 | # model training and testing settings 132 | train_cfg=dict(pts=dict( 133 | grid_size=[512, 512, 1], 134 | voxel_size=voxel_size, 135 | point_cloud_range=point_cloud_range, 136 | out_size_factor=4, 137 | assigner=dict( 138 | type='HungarianAssigner3D', 139 | cls_cost=dict(type='FocalLossCost', weight=2.0), 140 | reg_cost=dict(type='BBox3DL1Cost', weight=0.25), 141 | # Fake cost. This is just to make it compatible with DETR head. 142 | iou_cost=dict(type='IoUCost', weight=0.0), 143 | pc_range=point_cloud_range),))) 144 | 145 | 146 | dataset_type = 'CustomNuScenesDataset' 147 | data_root = './data/nuscenes/' 148 | 149 | file_client_args = dict(backend='disk') 150 | 151 | 152 | ida_aug_conf = { 153 | "resize_lim": (0.47, 0.625), 154 | "final_dim": (320, 800), 155 | "bot_pct_lim": (0.0, 0.0), 156 | "rot_lim": (0.0, 0.0), 157 | "H": 900, 158 | "W": 1600, 159 | "rand_flip": True, 160 | } 161 | 162 | # ida_aug_conf = { 163 | # "resize_lim": (0.82, 1.25), 164 | # "final_dim": (640, 1600), 165 | # "bot_pct_lim": (0.0, 0.0), 166 | # "rot_lim": (0.0, 0.0), 167 | # "H": 900, 168 | # "W": 1600, 169 | # "rand_flip": True, 170 | # } 171 | 172 | train_pipeline = [ 173 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 174 | dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_bbox=True, 175 | with_label=True, with_bbox_depth=True), 176 | dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), 177 | dict(type='ObjectNameFilter', classes=class_names), 178 | dict(type='ResizeCropFlipRotImage', 179 | data_aug_conf=ida_aug_conf, training=True), 180 | dict(type='GlobalRotScaleTransImage', 181 | rot_range=[-0.3925, 0.3925], 182 | translation_std=[0, 0, 0], 183 | scale_ratio_range=[0.95, 1.05], 184 | reverse_angle=True, 185 | training=True 186 | ), 187 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 188 | dict(type='PadMultiViewImage', size_divisor=32), 189 | dict(type='PETRFormatBundle3D', class_names=class_names, 190 | collect_keys=collect_keys + ['prev_exists']), 191 | dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img', 'gt_bboxes', 'gt_labels', 'centers2d', 'depths', 'prev_exists'] + collect_keys, 192 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 193 | 'img_norm_cfg', 'scene_token', 'gt_bboxes_3d', 'gt_labels_3d', 'results_plane', 'intrinsics', 'extrinsics', 194 | 'proposals_intrinsics')) 195 | ] 196 | test_pipeline = [ 197 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 198 | dict(type='ResizeCropFlipRotImage', 199 | data_aug_conf=ida_aug_conf, training=False), 200 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 201 | dict(type='PadMultiViewImage', size_divisor=32), 202 | dict( 203 | type='MultiScaleFlipAug3D', 204 | img_scale=(1333, 800), 205 | pts_scale_ratio=1, 206 | flip=False, 207 | transforms=[ 208 | dict( 209 | type='PETRFormatBundle3D', 210 | collect_keys=collect_keys, 211 | class_names=class_names, 212 | with_label=False), 213 | dict(type='Collect3D', keys=['img'] + collect_keys, 214 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token', 215 | 'results_plane', 'intrinsics', 'extrinsics', 'proposals_intrinsics')) 216 | ]) 217 | ] 218 | 219 | data = dict( 220 | samples_per_gpu=batch_size, 221 | workers_per_gpu=4, 222 | train=dict( 223 | type=dataset_type, 224 | data_root=data_root, 225 | ann_file=data_root + 'nuscenes2d_temporal_infos_train_stream.pkl', 226 | num_frame_losses=num_frame_losses, 227 | seq_split_num=2, # streaming video training 228 | seq_mode=True, # streaming video training 229 | pipeline=train_pipeline, 230 | classes=class_names, 231 | modality=input_modality, 232 | collect_keys=collect_keys + ['img', 'prev_exists', 'img_metas'], 233 | queue_length=queue_length, 234 | test_mode=False, 235 | use_valid_flag=True, 236 | filter_empty_gt=False, 237 | box_type_3d='LiDAR'), 238 | val=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 239 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_val_stream.pkl', classes=class_names, modality=input_modality), 240 | test=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 241 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_val_stream.pkl', classes=class_names, modality=input_modality), 242 | shuffler_sampler=dict(type='InfiniteGroupEachSampleInBatchSampler'), 243 | nonshuffler_sampler=dict(type='DistributedSampler') 244 | ) 245 | 246 | 247 | optimizer = dict( 248 | type='AdamW', 249 | lr=4e-4, # bs 8: 2e-4 || bs 16: 4e-4 250 | paramwise_cfg=dict( 251 | custom_keys={ 252 | 'img_backbone': dict(lr_mult=0.25), 253 | }), 254 | weight_decay=0.01) 255 | 256 | optimizer_config = dict(type='Fp16OptimizerHook', 257 | loss_scale='dynamic', grad_clip=dict(max_norm=35, norm_type=2)) 258 | # learning policy 259 | lr_config = dict( 260 | policy='CosineAnnealing', 261 | warmup='linear', 262 | warmup_iters=500, 263 | warmup_ratio=1.0 / 3, 264 | min_lr_ratio=1e-3, 265 | ) 266 | 267 | # evaluation = dict(interval=num_iters_per_epoch * 268 | # num_epochs, pipeline=test_pipeline) 269 | evaluation = dict(interval=num_iters_per_epoch * 270 | num_epochs, pipeline=test_pipeline) 271 | # when use checkpoint, find_unused_parameters must be False 272 | find_unused_parameters = False 273 | checkpoint_config = dict(interval=num_iters_per_epoch, max_keep_ckpts=1) 274 | runner = dict( 275 | type='IterBasedRunner', max_iters=num_epochs * num_iters_per_epoch) 276 | load_from = '/opt/data/private/jihao/Project/StreamPETR-main-original/ckpts/fcos3d_vovnet_imgbackbone-remapped.pth' 277 | resume_from = None 278 | -------------------------------------------------------------------------------- /projects/configs/PersPETR/.ipynb_checkpoints/persdetr3d_vov_800_bs2_seq_24e_testing-checkpoint.py: -------------------------------------------------------------------------------- 1 | _base_ = [ 2 | '../../../mmdetection3d/configs/_base_/datasets/nus-3d.py', 3 | '../../../mmdetection3d/configs/_base_/default_runtime.py' 4 | ] 5 | backbone_norm_cfg = dict(type='LN', requires_grad=True) 6 | plugin = True 7 | plugin_dir = 'projects/mmdet3d_plugin/' 8 | 9 | # If point cloud range is changed, the models should also change their point 10 | # cloud range accordingly 11 | point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0] 12 | voxel_size = [0.2, 0.2, 8] 13 | img_norm_cfg = dict( 14 | mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395], to_rgb=False) # fix img_norm 15 | # For nuScenes we usually do 10-class detection 16 | class_names = [ 17 | 'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier', 18 | 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone' 19 | ] 20 | 21 | num_gpus = 8 22 | batch_size = 2 23 | num_iters_per_epoch = 34149 // (num_gpus * batch_size) 24 | num_epochs = 50 25 | 26 | queue_length = 1 27 | num_frame_losses = 1 28 | collect_keys = ['lidar2img', 'intrinsics', 'extrinsics', 29 | 'timestamp', 'img_timestamp', 'ego_pose', 'ego_pose_inv'] 30 | input_modality = dict( 31 | use_lidar=False, 32 | use_camera=True, 33 | use_radar=False, 34 | use_map=False, 35 | use_external=True) 36 | model = dict( 37 | type='RepDetr3D', 38 | num_frame_head_grads=num_frame_losses, 39 | num_frame_backbone_grads=num_frame_losses, 40 | num_frame_losses=num_frame_losses, 41 | use_grid_mask=True, 42 | stride=[8, 16, 32, 64], 43 | position_level=[0, 1, 2, 3], 44 | img_backbone=dict( 45 | type='VoVNetCP', # use checkpoint to save memory 46 | spec_name='V-99-eSE', 47 | norm_eval=True, 48 | frozen_stages=-1, 49 | input_ch=3, 50 | out_features=('stage2', 'stage3', 'stage4', 'stage5',)), 51 | img_neck=dict( 52 | type='FPN', # remove unused parameters 53 | start_level=1, 54 | add_extra_convs='on_output', 55 | relu_before_extra_convs=True, 56 | in_channels=[256, 512, 768, 1024], 57 | out_channels=256, 58 | num_outs=4), 59 | img_roi_head=dict( 60 | type='YOLOXHeadCustom', 61 | num_classes=10, 62 | in_channels=256, 63 | strides=[8, 16, 32, 64], 64 | train_cfg=dict(assigner=dict( 65 | type='SimOTAAssigner', center_radius=2.5)), 66 | test_cfg=dict(score_thr=0.01, nms=dict( 67 | type='nms', iou_threshold=0.65)), 68 | ), 69 | pts_bbox_head=dict( 70 | type='SparseHead', 71 | num_classes=10, 72 | in_channels=256, 73 | num_query=644, 74 | memory_len=1024, 75 | topk_proposals=256, 76 | num_propagated=256, 77 | scalar=10, # noise groups 78 | noise_scale=1.0, 79 | dn_weight=1.0, # dn loss weight 80 | split=0.75, # positive rate 81 | with_dn=True, 82 | with_ego_pos=True, 83 | match_with_velo=False, 84 | code_weights=[2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 85 | transformer=dict( 86 | type='Detr3DTransformer', 87 | decoder=dict( 88 | type='Detr3DTransformerDecoder', 89 | embed_dims=256, 90 | num_layers=6, 91 | transformerlayers=dict( 92 | type='Detr3DTemporalDecoderLayer', 93 | batch_first=True, 94 | attn_cfgs=[ 95 | dict( 96 | type='MultiheadAttention', 97 | embed_dims=256, 98 | num_heads=8, 99 | dropout=0.1), 100 | dict( 101 | type='DeformableFeatureAggregationCuda', 102 | embed_dims=256, 103 | num_groups=8, 104 | num_levels=4, 105 | num_cams=6, 106 | dropout=0.1, 107 | num_pts=13, 108 | bias=2.), 109 | ], 110 | feedforward_channels=2048, 111 | ffn_dropout=0.1, 112 | with_cp=True, # use checkpoint to save memory 113 | operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 114 | 'ffn', 'norm')), 115 | )), 116 | bbox_coder=dict( 117 | type='NMSFreeCoder', 118 | post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], 119 | pc_range=point_cloud_range, 120 | max_num=300, 121 | voxel_size=voxel_size, 122 | num_classes=10), 123 | loss_cls=dict( 124 | type='FocalLoss', 125 | use_sigmoid=True, 126 | gamma=2.0, 127 | alpha=0.25, 128 | loss_weight=2.0), 129 | loss_bbox=dict(type='L1Loss', loss_weight=0.25), 130 | loss_iou=dict(type='GIoULoss', loss_weight=0.0),), 131 | # model training and testing settings 132 | train_cfg=dict(pts=dict( 133 | grid_size=[512, 512, 1], 134 | voxel_size=voxel_size, 135 | point_cloud_range=point_cloud_range, 136 | out_size_factor=4, 137 | assigner=dict( 138 | type='HungarianAssigner3D', 139 | cls_cost=dict(type='FocalLossCost', weight=2.0), 140 | reg_cost=dict(type='BBox3DL1Cost', weight=0.25), 141 | # Fake cost. This is just to make it compatible with DETR head. 142 | iou_cost=dict(type='IoUCost', weight=0.0), 143 | pc_range=point_cloud_range),))) 144 | 145 | 146 | dataset_type = 'CustomNuScenesDataset' 147 | data_root = './data/nuscenes/' 148 | 149 | file_client_args = dict(backend='disk') 150 | 151 | 152 | ida_aug_conf = { 153 | "resize_lim": (0.94, 1.25), 154 | "final_dim": (640, 1600), 155 | "bot_pct_lim": (0.0, 0.0), 156 | "rot_lim": (0.0, 0.0), 157 | "H": 900, 158 | "W": 1600, 159 | "rand_flip": True, 160 | } 161 | 162 | # ida_aug_conf = { 163 | # "resize_lim": (1.175, 1.25), 164 | # "final_dim": (800, 1600), 165 | # "bot_pct_lim": (0.0, 0.0), 166 | # "rot_lim": (0.0, 0.0), 167 | # "H": 900, 168 | # "W": 1600, 169 | # "rand_flip": True, 170 | # } 171 | 172 | train_pipeline = [ 173 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 174 | dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_bbox=True, 175 | with_label=True, with_bbox_depth=True), 176 | dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), 177 | dict(type='ObjectNameFilter', classes=class_names), 178 | dict(type='ResizeCropFlipRotImage', 179 | data_aug_conf=ida_aug_conf, training=True), 180 | dict(type='GlobalRotScaleTransImage', 181 | rot_range=[-0.3925, 0.3925], 182 | translation_std=[0, 0, 0], 183 | scale_ratio_range=[0.95, 1.05], 184 | reverse_angle=True, 185 | training=True 186 | ), 187 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 188 | dict(type='PadMultiViewImage', size_divisor=32), 189 | dict(type='PETRFormatBundle3D', class_names=class_names, 190 | collect_keys=collect_keys + ['prev_exists']), 191 | dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img', 'gt_bboxes', 'gt_labels', 'centers2d', 'depths', 'prev_exists'] + collect_keys, 192 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 193 | 'img_norm_cfg', 'scene_token', 'gt_bboxes_3d', 'gt_labels_3d', 'results_plane', 'intrinsics', 'extrinsics', 194 | 'proposals_intrinsics')) 195 | ] 196 | test_pipeline = [ 197 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 198 | dict(type='ResizeCropFlipRotImage', 199 | data_aug_conf=ida_aug_conf, training=False), 200 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 201 | dict(type='PadMultiViewImage', size_divisor=32), 202 | dict( 203 | type='MultiScaleFlipAug3D', 204 | img_scale=(1333, 800), 205 | pts_scale_ratio=1, 206 | flip=False, 207 | transforms=[ 208 | dict( 209 | type='PETRFormatBundle3D', 210 | collect_keys=collect_keys, 211 | class_names=class_names, 212 | with_label=False), 213 | dict(type='Collect3D', keys=['img'] + collect_keys, 214 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token', 215 | 'results_plane', 'intrinsics', 'extrinsics', 'proposals_intrinsics')) 216 | ]) 217 | ] 218 | 219 | data = dict( 220 | samples_per_gpu=batch_size, 221 | workers_per_gpu=4, 222 | train=dict( 223 | type=dataset_type, 224 | data_root=data_root, 225 | ann_file=[data_root + 'nuscenes2d_temporal_infos_train_stream.pkl', 226 | data_root + 'nuscenes2d_temporal_infos_val_stream.pkl'], 227 | num_frame_losses=num_frame_losses, 228 | seq_split_num=2, # streaming video training 229 | seq_mode=True, # streaming video training 230 | pipeline=train_pipeline, 231 | classes=class_names, 232 | modality=input_modality, 233 | collect_keys=collect_keys + ['img', 'prev_exists', 'img_metas'], 234 | queue_length=queue_length, 235 | test_mode=False, 236 | use_valid_flag=True, 237 | filter_empty_gt=False, 238 | box_type_3d='LiDAR'), 239 | val=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 240 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_val_stream.pkl', classes=class_names, modality=input_modality), 241 | test=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 242 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_test_stream.pkl', classes=class_names, modality=input_modality), 243 | shuffler_sampler=dict(type='InfiniteGroupEachSampleInBatchSampler'), 244 | nonshuffler_sampler=dict(type='DistributedSampler') 245 | ) 246 | 247 | 248 | optimizer = dict( 249 | type='AdamW', 250 | lr=4e-4, # bs 8: 2e-4 || bs 16: 4e-4 251 | paramwise_cfg=dict( 252 | custom_keys={ 253 | 'img_backbone': dict(lr_mult=0.25), 254 | }), 255 | weight_decay=0.01) 256 | 257 | optimizer_config = dict(type='Fp16OptimizerHook', 258 | loss_scale='dynamic', grad_clip=dict(max_norm=35, norm_type=2)) 259 | # learning policy 260 | lr_config = dict( 261 | policy='CosineAnnealing', 262 | warmup='linear', 263 | warmup_iters=500, 264 | warmup_ratio=1.0 / 3, 265 | min_lr_ratio=1e-3, 266 | ) 267 | 268 | evaluation = dict(interval=num_iters_per_epoch * 269 | num_epochs, pipeline=test_pipeline) 270 | # when use checkpoint, find_unused_parameters must be False 271 | find_unused_parameters = False 272 | checkpoint_config = dict(interval=num_iters_per_epoch, max_keep_ckpts=1) 273 | runner = dict( 274 | type='IterBasedRunner', max_iters=num_epochs * num_iters_per_epoch) 275 | load_from = '/opt/data/private/jihao/Project/StreamPETR-main/ckpts/dd3d_det_final.pth' 276 | resume_from = None 277 | -------------------------------------------------------------------------------- /projects/configs/PersPETR/persdetr3d_vov_800_bs2_seq_24e.py: -------------------------------------------------------------------------------- 1 | _base_ = [ 2 | '../../../mmdetection3d/configs/_base_/datasets/nus-3d.py', 3 | '../../../mmdetection3d/configs/_base_/default_runtime.py' 4 | ] 5 | backbone_norm_cfg = dict(type='LN', requires_grad=True) 6 | plugin = True 7 | plugin_dir = 'projects/mmdet3d_plugin/' 8 | 9 | # If point cloud range is changed, the models should also change their point 10 | # cloud range accordingly 11 | point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0] 12 | voxel_size = [0.2, 0.2, 8] 13 | img_norm_cfg = dict( 14 | mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395], to_rgb=False) # fix img_norm 15 | # For nuScenes we usually do 10-class detection 16 | class_names = [ 17 | 'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier', 18 | 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone' 19 | ] 20 | 21 | num_gpus = 8 22 | batch_size = 1 # 2 23 | num_iters_per_epoch = 28130 // (num_gpus * batch_size) 24 | num_epochs = 24 25 | 26 | queue_length = 1 27 | num_frame_losses = 1 28 | collect_keys = ['lidar2img', 'intrinsics', 'extrinsics', 29 | 'timestamp', 'img_timestamp', 'ego_pose', 'ego_pose_inv'] 30 | input_modality = dict( 31 | use_lidar=False, 32 | use_camera=True, 33 | use_radar=False, 34 | use_map=False, 35 | use_external=True) 36 | model = dict( 37 | type='PersDetr3D', 38 | num_frame_head_grads=num_frame_losses, 39 | num_frame_backbone_grads=num_frame_losses, 40 | num_frame_losses=num_frame_losses, 41 | use_grid_mask=True, 42 | stride=[8, 16, 32, 64], 43 | position_level=[0, 1, 2, 3], 44 | img_backbone=dict( 45 | type='VoVNetCP', # use checkpoint to save memory 46 | spec_name='V-99-eSE', 47 | norm_eval=True, 48 | frozen_stages=-1, 49 | input_ch=3, 50 | out_features=('stage2', 'stage3', 'stage4', 'stage5',)), 51 | img_neck=dict( 52 | type='FPN', # remove unused parameters 53 | start_level=1, 54 | add_extra_convs='on_output', 55 | relu_before_extra_convs=True, 56 | in_channels=[256, 512, 768, 1024], 57 | out_channels=256, 58 | num_outs=4), 59 | img_roi_head=dict( 60 | type='YOLOXHeadCustom', 61 | num_classes=10, 62 | in_channels=256, 63 | strides=[8, 16, 32, 64], 64 | train_cfg=dict(assigner=dict( 65 | type='SimOTAAssigner', center_radius=2.5)), 66 | test_cfg=dict(score_thr=0.01, nms=dict( 67 | type='nms', iou_threshold=0.65)), 68 | ), 69 | pts_bbox_head=dict( 70 | type='PerspectiveHead', 71 | num_classes=10, 72 | in_channels=256, 73 | num_query=644, 74 | memory_len=1024, 75 | topk_proposals=256, 76 | num_propagated=256, 77 | scalar=10, # noise groups 78 | noise_scale=1.0, 79 | dn_weight=1.0, # dn loss weight 80 | split=0.75, # positive rate 81 | with_dn=True, 82 | with_ego_pos=True, 83 | match_with_velo=False, 84 | code_weights=[2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 85 | transformer=dict( 86 | type='Detr3DTransformer', 87 | decoder=dict( 88 | type='Detr3DTransformerDecoder', 89 | embed_dims=256, 90 | num_layers=6, 91 | transformerlayers=dict( 92 | type='Detr3DTemporalDecoderLayer', 93 | batch_first=True, 94 | attn_cfgs=[ 95 | dict( 96 | type='MultiheadAttention', 97 | embed_dims=256, 98 | num_heads=8, 99 | dropout=0.1), 100 | dict( 101 | type='DeformableFeatureAggregationCuda', 102 | embed_dims=256, 103 | num_groups=8, 104 | num_levels=4, 105 | num_cams=6, 106 | dropout=0.1, 107 | num_pts=13, 108 | bias=2.), 109 | ], 110 | feedforward_channels=2048, 111 | ffn_dropout=0.1, 112 | with_cp=True, # use checkpoint to save memory 113 | operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 114 | 'ffn', 'norm')), 115 | )), 116 | bbox_coder=dict( 117 | type='NMSFreeCoder', 118 | post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], 119 | pc_range=point_cloud_range, 120 | max_num=300, 121 | voxel_size=voxel_size, 122 | num_classes=10), 123 | loss_cls=dict( 124 | type='FocalLoss', 125 | use_sigmoid=True, 126 | gamma=2.0, 127 | alpha=0.25, 128 | loss_weight=2.0), 129 | loss_bbox=dict(type='L1Loss', loss_weight=0.25), 130 | loss_iou=dict(type='GIoULoss', loss_weight=0.0),), 131 | # model training and testing settings 132 | train_cfg=dict(pts=dict( 133 | grid_size=[512, 512, 1], 134 | voxel_size=voxel_size, 135 | point_cloud_range=point_cloud_range, 136 | out_size_factor=4, 137 | assigner=dict( 138 | type='HungarianAssigner3D', 139 | cls_cost=dict(type='FocalLossCost', weight=2.0), 140 | reg_cost=dict(type='BBox3DL1Cost', weight=0.25), 141 | # Fake cost. This is just to make it compatible with DETR head. 142 | iou_cost=dict(type='IoUCost', weight=0.0), 143 | pc_range=point_cloud_range),))) 144 | 145 | 146 | dataset_type = 'CustomNuScenesDataset' 147 | data_root = './data/nuscenes/' 148 | 149 | file_client_args = dict(backend='disk') 150 | 151 | 152 | ida_aug_conf = { 153 | "resize_lim": (0.47, 0.625), 154 | "final_dim": (320, 800), 155 | "bot_pct_lim": (0.0, 0.0), 156 | "rot_lim": (0.0, 0.0), 157 | "H": 900, 158 | "W": 1600, 159 | "rand_flip": True, 160 | } 161 | 162 | # ida_aug_conf = { 163 | # "resize_lim": (0.82, 1.25), 164 | # "final_dim": (640, 1600), 165 | # "bot_pct_lim": (0.0, 0.0), 166 | # "rot_lim": (0.0, 0.0), 167 | # "H": 900, 168 | # "W": 1600, 169 | # "rand_flip": True, 170 | # } 171 | 172 | train_pipeline = [ 173 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 174 | dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_bbox=True, 175 | with_label=True, with_bbox_depth=True), 176 | dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), 177 | dict(type='ObjectNameFilter', classes=class_names), 178 | dict(type='ResizeCropFlipRotImage', 179 | data_aug_conf=ida_aug_conf, training=True), 180 | dict(type='GlobalRotScaleTransImage', 181 | rot_range=[-0.3925, 0.3925], 182 | translation_std=[0, 0, 0], 183 | scale_ratio_range=[0.95, 1.05], 184 | reverse_angle=True, 185 | training=True 186 | ), 187 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 188 | dict(type='PadMultiViewImage', size_divisor=32), 189 | dict(type='PETRFormatBundle3D', class_names=class_names, 190 | collect_keys=collect_keys + ['prev_exists']), 191 | dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img', 'gt_bboxes', 'gt_labels', 'centers2d', 'depths', 'prev_exists'] + collect_keys, 192 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 193 | 'img_norm_cfg', 'scene_token', 'gt_bboxes_3d', 'gt_labels_3d', 'results_plane', 'intrinsics', 'extrinsics', 194 | 'proposals_intrinsics')) 195 | ] 196 | test_pipeline = [ 197 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 198 | dict(type='ResizeCropFlipRotImage', 199 | data_aug_conf=ida_aug_conf, training=False), 200 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 201 | dict(type='PadMultiViewImage', size_divisor=32), 202 | dict( 203 | type='MultiScaleFlipAug3D', 204 | img_scale=(1333, 800), 205 | pts_scale_ratio=1, 206 | flip=False, 207 | transforms=[ 208 | dict( 209 | type='PETRFormatBundle3D', 210 | collect_keys=collect_keys, 211 | class_names=class_names, 212 | with_label=False), 213 | dict(type='Collect3D', keys=['img'] + collect_keys, 214 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token', 215 | 'results_plane', 'intrinsics', 'extrinsics', 'proposals_intrinsics')) 216 | ]) 217 | ] 218 | 219 | data = dict( 220 | samples_per_gpu=batch_size, 221 | workers_per_gpu=4, 222 | train=dict( 223 | type=dataset_type, 224 | data_root=data_root, 225 | ann_file=data_root + 'nuscenes2d_temporal_infos_train_stream.pkl', 226 | num_frame_losses=num_frame_losses, 227 | seq_split_num=2, # streaming video training 228 | seq_mode=True, # streaming video training 229 | pipeline=train_pipeline, 230 | classes=class_names, 231 | modality=input_modality, 232 | collect_keys=collect_keys + ['img', 'prev_exists', 'img_metas'], 233 | queue_length=queue_length, 234 | test_mode=False, 235 | use_valid_flag=True, 236 | filter_empty_gt=False, 237 | box_type_3d='LiDAR'), 238 | val=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 239 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_val_stream.pkl', classes=class_names, modality=input_modality), 240 | test=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 241 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_val_stream.pkl', classes=class_names, modality=input_modality), 242 | shuffler_sampler=dict(type='InfiniteGroupEachSampleInBatchSampler'), 243 | nonshuffler_sampler=dict(type='DistributedSampler') 244 | ) 245 | 246 | 247 | optimizer = dict( 248 | type='AdamW', 249 | lr=4e-4, # bs 8: 2e-4 || bs 16: 4e-4 250 | paramwise_cfg=dict( 251 | custom_keys={ 252 | 'img_backbone': dict(lr_mult=0.25), 253 | }), 254 | weight_decay=0.01) 255 | 256 | optimizer_config = dict(type='Fp16OptimizerHook', 257 | loss_scale='dynamic', grad_clip=dict(max_norm=35, norm_type=2)) 258 | # learning policy 259 | lr_config = dict( 260 | policy='CosineAnnealing', 261 | warmup='linear', 262 | warmup_iters=500, 263 | warmup_ratio=1.0 / 3, 264 | min_lr_ratio=1e-3, 265 | ) 266 | 267 | # evaluation = dict(interval=num_iters_per_epoch * 268 | # num_epochs, pipeline=test_pipeline) 269 | evaluation = dict(interval=num_iters_per_epoch * 270 | num_epochs, pipeline=test_pipeline) 271 | # when use checkpoint, find_unused_parameters must be False 272 | find_unused_parameters = False 273 | checkpoint_config = dict(interval=num_iters_per_epoch, max_keep_ckpts=1) 274 | runner = dict( 275 | type='IterBasedRunner', max_iters=num_epochs * num_iters_per_epoch) 276 | load_from = '/opt/data/private/jihao/Project/StreamPETR-main-original/ckpts/fcos3d_vovnet_imgbackbone-remapped.pth' 277 | resume_from = None 278 | -------------------------------------------------------------------------------- /projects/configs/PersPETR/persdetr3d_vov_800_bs2_seq_24e_testing.py: -------------------------------------------------------------------------------- 1 | _base_ = [ 2 | '../../../mmdetection3d/configs/_base_/datasets/nus-3d.py', 3 | '../../../mmdetection3d/configs/_base_/default_runtime.py' 4 | ] 5 | backbone_norm_cfg = dict(type='LN', requires_grad=True) 6 | plugin = True 7 | plugin_dir = 'projects/mmdet3d_plugin/' 8 | 9 | # If point cloud range is changed, the models should also change their point 10 | # cloud range accordingly 11 | point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0] 12 | voxel_size = [0.2, 0.2, 8] 13 | img_norm_cfg = dict( 14 | mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395], to_rgb=False) # fix img_norm 15 | # For nuScenes we usually do 10-class detection 16 | class_names = [ 17 | 'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier', 18 | 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone' 19 | ] 20 | 21 | num_gpus = 8 22 | batch_size = 2 23 | num_iters_per_epoch = 34149 // (num_gpus * batch_size) 24 | num_epochs = 50 25 | 26 | queue_length = 1 27 | num_frame_losses = 1 28 | collect_keys = ['lidar2img', 'intrinsics', 'extrinsics', 29 | 'timestamp', 'img_timestamp', 'ego_pose', 'ego_pose_inv'] 30 | input_modality = dict( 31 | use_lidar=False, 32 | use_camera=True, 33 | use_radar=False, 34 | use_map=False, 35 | use_external=True) 36 | model = dict( 37 | type='RepDetr3D', 38 | num_frame_head_grads=num_frame_losses, 39 | num_frame_backbone_grads=num_frame_losses, 40 | num_frame_losses=num_frame_losses, 41 | use_grid_mask=True, 42 | stride=[8, 16, 32, 64], 43 | position_level=[0, 1, 2, 3], 44 | img_backbone=dict( 45 | type='VoVNetCP', # use checkpoint to save memory 46 | spec_name='V-99-eSE', 47 | norm_eval=True, 48 | frozen_stages=-1, 49 | input_ch=3, 50 | out_features=('stage2', 'stage3', 'stage4', 'stage5',)), 51 | img_neck=dict( 52 | type='FPN', # remove unused parameters 53 | start_level=1, 54 | add_extra_convs='on_output', 55 | relu_before_extra_convs=True, 56 | in_channels=[256, 512, 768, 1024], 57 | out_channels=256, 58 | num_outs=4), 59 | img_roi_head=dict( 60 | type='YOLOXHeadCustom', 61 | num_classes=10, 62 | in_channels=256, 63 | strides=[8, 16, 32, 64], 64 | train_cfg=dict(assigner=dict( 65 | type='SimOTAAssigner', center_radius=2.5)), 66 | test_cfg=dict(score_thr=0.01, nms=dict( 67 | type='nms', iou_threshold=0.65)), 68 | ), 69 | pts_bbox_head=dict( 70 | type='SparseHead', 71 | num_classes=10, 72 | in_channels=256, 73 | num_query=644, 74 | memory_len=1024, 75 | topk_proposals=256, 76 | num_propagated=256, 77 | scalar=10, # noise groups 78 | noise_scale=1.0, 79 | dn_weight=1.0, # dn loss weight 80 | split=0.75, # positive rate 81 | with_dn=True, 82 | with_ego_pos=True, 83 | match_with_velo=False, 84 | code_weights=[2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 85 | transformer=dict( 86 | type='Detr3DTransformer', 87 | decoder=dict( 88 | type='Detr3DTransformerDecoder', 89 | embed_dims=256, 90 | num_layers=6, 91 | transformerlayers=dict( 92 | type='Detr3DTemporalDecoderLayer', 93 | batch_first=True, 94 | attn_cfgs=[ 95 | dict( 96 | type='MultiheadAttention', 97 | embed_dims=256, 98 | num_heads=8, 99 | dropout=0.1), 100 | dict( 101 | type='DeformableFeatureAggregationCuda', 102 | embed_dims=256, 103 | num_groups=8, 104 | num_levels=4, 105 | num_cams=6, 106 | dropout=0.1, 107 | num_pts=13, 108 | bias=2.), 109 | ], 110 | feedforward_channels=2048, 111 | ffn_dropout=0.1, 112 | with_cp=True, # use checkpoint to save memory 113 | operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 114 | 'ffn', 'norm')), 115 | )), 116 | bbox_coder=dict( 117 | type='NMSFreeCoder', 118 | post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], 119 | pc_range=point_cloud_range, 120 | max_num=300, 121 | voxel_size=voxel_size, 122 | num_classes=10), 123 | loss_cls=dict( 124 | type='FocalLoss', 125 | use_sigmoid=True, 126 | gamma=2.0, 127 | alpha=0.25, 128 | loss_weight=2.0), 129 | loss_bbox=dict(type='L1Loss', loss_weight=0.25), 130 | loss_iou=dict(type='GIoULoss', loss_weight=0.0),), 131 | # model training and testing settings 132 | train_cfg=dict(pts=dict( 133 | grid_size=[512, 512, 1], 134 | voxel_size=voxel_size, 135 | point_cloud_range=point_cloud_range, 136 | out_size_factor=4, 137 | assigner=dict( 138 | type='HungarianAssigner3D', 139 | cls_cost=dict(type='FocalLossCost', weight=2.0), 140 | reg_cost=dict(type='BBox3DL1Cost', weight=0.25), 141 | # Fake cost. This is just to make it compatible with DETR head. 142 | iou_cost=dict(type='IoUCost', weight=0.0), 143 | pc_range=point_cloud_range),))) 144 | 145 | 146 | dataset_type = 'CustomNuScenesDataset' 147 | data_root = './data/nuscenes/' 148 | 149 | file_client_args = dict(backend='disk') 150 | 151 | 152 | ida_aug_conf = { 153 | "resize_lim": (0.94, 1.25), 154 | "final_dim": (640, 1600), 155 | "bot_pct_lim": (0.0, 0.0), 156 | "rot_lim": (0.0, 0.0), 157 | "H": 900, 158 | "W": 1600, 159 | "rand_flip": True, 160 | } 161 | 162 | # ida_aug_conf = { 163 | # "resize_lim": (1.175, 1.25), 164 | # "final_dim": (800, 1600), 165 | # "bot_pct_lim": (0.0, 0.0), 166 | # "rot_lim": (0.0, 0.0), 167 | # "H": 900, 168 | # "W": 1600, 169 | # "rand_flip": True, 170 | # } 171 | 172 | train_pipeline = [ 173 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 174 | dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_bbox=True, 175 | with_label=True, with_bbox_depth=True), 176 | dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), 177 | dict(type='ObjectNameFilter', classes=class_names), 178 | dict(type='ResizeCropFlipRotImage', 179 | data_aug_conf=ida_aug_conf, training=True), 180 | dict(type='GlobalRotScaleTransImage', 181 | rot_range=[-0.3925, 0.3925], 182 | translation_std=[0, 0, 0], 183 | scale_ratio_range=[0.95, 1.05], 184 | reverse_angle=True, 185 | training=True 186 | ), 187 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 188 | dict(type='PadMultiViewImage', size_divisor=32), 189 | dict(type='PETRFormatBundle3D', class_names=class_names, 190 | collect_keys=collect_keys + ['prev_exists']), 191 | dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img', 'gt_bboxes', 'gt_labels', 'centers2d', 'depths', 'prev_exists'] + collect_keys, 192 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 193 | 'img_norm_cfg', 'scene_token', 'gt_bboxes_3d', 'gt_labels_3d', 'results_plane', 'intrinsics', 'extrinsics', 194 | 'proposals_intrinsics')) 195 | ] 196 | test_pipeline = [ 197 | dict(type='LoadMultiViewImageFromFiles', to_float32=True), 198 | dict(type='ResizeCropFlipRotImage', 199 | data_aug_conf=ida_aug_conf, training=False), 200 | dict(type='NormalizeMultiviewImage', **img_norm_cfg), 201 | dict(type='PadMultiViewImage', size_divisor=32), 202 | dict( 203 | type='MultiScaleFlipAug3D', 204 | img_scale=(1333, 800), 205 | pts_scale_ratio=1, 206 | flip=False, 207 | transforms=[ 208 | dict( 209 | type='PETRFormatBundle3D', 210 | collect_keys=collect_keys, 211 | class_names=class_names, 212 | with_label=False), 213 | dict(type='Collect3D', keys=['img'] + collect_keys, 214 | meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token', 215 | 'results_plane', 'intrinsics', 'extrinsics', 'proposals_intrinsics')) 216 | ]) 217 | ] 218 | 219 | data = dict( 220 | samples_per_gpu=batch_size, 221 | workers_per_gpu=4, 222 | train=dict( 223 | type=dataset_type, 224 | data_root=data_root, 225 | ann_file=[data_root + 'nuscenes2d_temporal_infos_train_stream.pkl', 226 | data_root + 'nuscenes2d_temporal_infos_val_stream.pkl'], 227 | num_frame_losses=num_frame_losses, 228 | seq_split_num=2, # streaming video training 229 | seq_mode=True, # streaming video training 230 | pipeline=train_pipeline, 231 | classes=class_names, 232 | modality=input_modality, 233 | collect_keys=collect_keys + ['img', 'prev_exists', 'img_metas'], 234 | queue_length=queue_length, 235 | test_mode=False, 236 | use_valid_flag=True, 237 | filter_empty_gt=False, 238 | box_type_3d='LiDAR'), 239 | val=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 240 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_val_stream.pkl', classes=class_names, modality=input_modality), 241 | test=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + \ 242 | ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'nuscenes2d_temporal_infos_test_stream.pkl', classes=class_names, modality=input_modality), 243 | shuffler_sampler=dict(type='InfiniteGroupEachSampleInBatchSampler'), 244 | nonshuffler_sampler=dict(type='DistributedSampler') 245 | ) 246 | 247 | 248 | optimizer = dict( 249 | type='AdamW', 250 | lr=4e-4, # bs 8: 2e-4 || bs 16: 4e-4 251 | paramwise_cfg=dict( 252 | custom_keys={ 253 | 'img_backbone': dict(lr_mult=0.25), 254 | }), 255 | weight_decay=0.01) 256 | 257 | optimizer_config = dict(type='Fp16OptimizerHook', 258 | loss_scale='dynamic', grad_clip=dict(max_norm=35, norm_type=2)) 259 | # learning policy 260 | lr_config = dict( 261 | policy='CosineAnnealing', 262 | warmup='linear', 263 | warmup_iters=500, 264 | warmup_ratio=1.0 / 3, 265 | min_lr_ratio=1e-3, 266 | ) 267 | 268 | evaluation = dict(interval=num_iters_per_epoch * 269 | num_epochs, pipeline=test_pipeline) 270 | # when use checkpoint, find_unused_parameters must be False 271 | find_unused_parameters = False 272 | checkpoint_config = dict(interval=num_iters_per_epoch, max_keep_ckpts=1) 273 | runner = dict( 274 | type='IterBasedRunner', max_iters=num_epochs * num_iters_per_epoch) 275 | load_from = '/opt/data/private/jihao/Project/StreamPETR-main/ckpts/dd3d_det_final.pth' 276 | resume_from = None 277 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/__init__.py: -------------------------------------------------------------------------------- 1 | from .core.bbox.assigners.hungarian_assigner_3d import HungarianAssigner3D 2 | from .core.bbox.coders.nms_free_coder import NMSFreeCoder 3 | from .core.bbox.match_costs import BBox3DL1Cost 4 | from .datasets import CustomNuScenesDataset 5 | from .datasets.pipelines import * 6 | from .models.dense_heads import * 7 | from .models.detectors import * 8 | from .models.necks import * 9 | from .models.backbones import * 10 | 11 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/.ipynb_checkpoints/test-checkpoint.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | import os.path as osp 7 | import pickle 8 | import shutil 9 | import tempfile 10 | import time 11 | 12 | import mmcv 13 | import torch 14 | import torch.distributed as dist 15 | from mmcv.image import tensor2imgs 16 | from mmcv.runner import get_dist_info 17 | 18 | from mmdet.core import encode_mask_results 19 | 20 | 21 | import mmcv 22 | import numpy as np 23 | import pycocotools.mask as mask_util 24 | 25 | def custom_encode_mask_results(mask_results): 26 | """Encode bitmap mask to RLE code. Semantic Masks only 27 | Args: 28 | mask_results (list | tuple[list]): bitmap mask results. 29 | In mask scoring rcnn, mask_results is a tuple of (segm_results, 30 | segm_cls_score). 31 | Returns: 32 | list | tuple: RLE encoded mask. 33 | """ 34 | cls_segms = mask_results 35 | num_classes = len(cls_segms) 36 | encoded_mask_results = [] 37 | for i in range(len(cls_segms)): 38 | encoded_mask_results.append( 39 | mask_util.encode( 40 | np.array( 41 | cls_segms[i][:, :, np.newaxis], order='F', 42 | dtype='uint8'))[0]) # encoded with RLE 43 | return [encoded_mask_results] 44 | 45 | def custom_multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False): 46 | """Test model with multiple gpus. 47 | This method tests model with multiple gpus and collects the results 48 | under two different modes: gpu and cpu modes. By setting 'gpu_collect=True' 49 | it encodes results to gpu tensors and use gpu communication for results 50 | collection. On cpu mode it saves the results on different gpus to 'tmpdir' 51 | and collects them by the rank 0 worker. 52 | Args: 53 | model (nn.Module): Model to be tested. 54 | data_loader (nn.Dataloader): Pytorch data loader. 55 | tmpdir (str): Path of directory to save the temporary results from 56 | different gpus under cpu mode. 57 | gpu_collect (bool): Option to use either gpu or cpu to collect results. 58 | Returns: 59 | list: The prediction results. 60 | """ 61 | model.eval() 62 | bbox_results = [] 63 | mask_results = [] 64 | dataset = data_loader.dataset 65 | rank, world_size = get_dist_info() 66 | if rank == 0: 67 | prog_bar = mmcv.ProgressBar(len(dataset)) 68 | time.sleep(2) # This line can prevent deadlock problem in some cases. 69 | have_mask = False 70 | for i, data in enumerate(data_loader): 71 | with torch.no_grad(): 72 | result = model(return_loss=False, rescale=True, **data) 73 | # encode mask results 74 | if isinstance(result, dict): 75 | if 'bbox_results' in result.keys(): 76 | bbox_result = result['bbox_results'] 77 | batch_size = len(result['bbox_results']) 78 | bbox_results.extend(bbox_result) 79 | if 'mask_results' in result.keys() and result['mask_results'] is not None: 80 | mask_result = custom_encode_mask_results(result['mask_results']) 81 | mask_results.extend(mask_result) 82 | have_mask = True 83 | else: 84 | batch_size = len(result) 85 | bbox_results.extend(result) 86 | 87 | #if isinstance(result[0], tuple): 88 | # assert False, 'this code is for instance segmentation, which our code will not utilize.' 89 | # result = [(bbox_results, encode_mask_results(mask_results)) 90 | # for bbox_results, mask_results in result] 91 | if rank == 0: 92 | 93 | for _ in range(batch_size * world_size): 94 | prog_bar.update() 95 | 96 | # collect results from all ranks 97 | if gpu_collect: 98 | bbox_results = collect_results_gpu(bbox_results, len(dataset)) 99 | if have_mask: 100 | mask_results = collect_results_gpu(mask_results, len(dataset)) 101 | else: 102 | mask_results = None 103 | else: 104 | bbox_results = collect_results_cpu(bbox_results, len(dataset), tmpdir) 105 | tmpdir = tmpdir+'_mask' if tmpdir is not None else None 106 | if have_mask: 107 | mask_results = collect_results_cpu(mask_results, len(dataset), tmpdir) 108 | else: 109 | mask_results = None 110 | 111 | if mask_results is None: 112 | return bbox_results 113 | return {'bbox_results': bbox_results, 'mask_results': mask_results} 114 | 115 | 116 | def collect_results_cpu(result_part, size, tmpdir=None): 117 | rank, world_size = get_dist_info() 118 | # create a tmp dir if it is not specified 119 | if tmpdir is None: 120 | MAX_LEN = 512 121 | # 32 is whitespace 122 | dir_tensor = torch.full((MAX_LEN, ), 123 | 32, 124 | dtype=torch.uint8, 125 | device='cuda') 126 | if rank == 0: 127 | mmcv.mkdir_or_exist('.dist_test') 128 | tmpdir = tempfile.mkdtemp(dir='.dist_test') 129 | tmpdir = torch.tensor( 130 | bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') 131 | dir_tensor[:len(tmpdir)] = tmpdir 132 | dist.broadcast(dir_tensor, 0) 133 | tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() 134 | else: 135 | mmcv.mkdir_or_exist(tmpdir) 136 | # dump the part result to the dir 137 | mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl')) 138 | dist.barrier() 139 | # collect all parts 140 | if rank != 0: 141 | return None 142 | else: 143 | # load results of all parts from tmp dir 144 | part_list = [] 145 | for i in range(world_size): 146 | part_file = osp.join(tmpdir, f'part_{i}.pkl') 147 | part_list.append(mmcv.load(part_file)) 148 | # sort the results 149 | ordered_results = [] 150 | ''' 151 | bacause we change the sample of the evaluation stage to make sure that each gpu will handle continuous sample, 152 | ''' 153 | #for res in zip(*part_list): 154 | for res in part_list: 155 | ordered_results.extend(list(res)) 156 | # the dataloader may pad some samples 157 | ordered_results = ordered_results[:size] 158 | # remove tmp dir 159 | shutil.rmtree(tmpdir) 160 | return ordered_results 161 | 162 | 163 | def collect_results_gpu(result_part, size): 164 | collect_results_cpu(result_part, size) -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/__init__.py: -------------------------------------------------------------------------------- 1 | from .train import custom_train_model 2 | from .mmdet_train import custom_train_detector 3 | from .test import custom_multi_gpu_test -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/apis/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/__pycache__/mmdet_train.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/apis/__pycache__/mmdet_train.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/__pycache__/test.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/apis/__pycache__/test.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/__pycache__/train.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/apis/__pycache__/train.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/mmdet_train.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | # --------------------------------------------- 7 | # Modified by Shihao Wang 8 | # --------------------------------------------- 9 | import random 10 | import warnings 11 | 12 | import numpy as np 13 | import torch 14 | import torch.distributed as dist 15 | from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 16 | from mmcv.runner import (HOOKS, DistSamplerSeedHook, EpochBasedRunner, 17 | Fp16OptimizerHook, OptimizerHook, build_optimizer, 18 | build_runner, get_dist_info) 19 | from mmcv.utils import build_from_cfg 20 | 21 | from mmdet.core import EvalHook 22 | 23 | from mmdet.datasets import (build_dataset, 24 | replace_ImageToTensor) 25 | from mmdet.utils import get_root_logger 26 | import time 27 | import os.path as osp 28 | from projects.mmdet3d_plugin.datasets.builder import build_dataloader 29 | from projects.mmdet3d_plugin.core.evaluation.eval_hooks import CustomDistEvalHook 30 | from projects.mmdet3d_plugin.datasets import custom_build_dataset 31 | def custom_train_detector(model, 32 | dataset, 33 | cfg, 34 | distributed=False, 35 | validate=False, 36 | timestamp=None, 37 | eval_model=None, 38 | meta=None): 39 | logger = get_root_logger(cfg.log_level) 40 | 41 | # prepare data loaders 42 | 43 | dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] 44 | #assert len(dataset)==1s 45 | if 'imgs_per_gpu' in cfg.data: 46 | logger.warning('"imgs_per_gpu" is deprecated in MMDet V2.0. ' 47 | 'Please use "samples_per_gpu" instead') 48 | if 'samples_per_gpu' in cfg.data: 49 | logger.warning( 50 | f'Got "imgs_per_gpu"={cfg.data.imgs_per_gpu} and ' 51 | f'"samples_per_gpu"={cfg.data.samples_per_gpu}, "imgs_per_gpu"' 52 | f'={cfg.data.imgs_per_gpu} is used in this experiments') 53 | else: 54 | logger.warning( 55 | 'Automatically set "samples_per_gpu"="imgs_per_gpu"=' 56 | f'{cfg.data.imgs_per_gpu} in this experiments') 57 | cfg.data.samples_per_gpu = cfg.data.imgs_per_gpu 58 | 59 | data_loaders = [ 60 | build_dataloader( 61 | ds, 62 | cfg.data.samples_per_gpu, 63 | cfg.data.workers_per_gpu, 64 | # cfg.gpus will be ignored if distributed 65 | len(cfg.gpu_ids), 66 | dist=distributed, 67 | seed=cfg.seed, 68 | shuffler_sampler=cfg.data.shuffler_sampler, # dict(type='DistributedGroupSampler'), 69 | nonshuffler_sampler=cfg.data.nonshuffler_sampler, # dict(type='DistributedSampler'), 70 | runner_type=cfg.runner, 71 | ) for ds in dataset 72 | ] 73 | 74 | # put model on gpus 75 | if distributed: 76 | find_unused_parameters = cfg.get('find_unused_parameters', False) 77 | # Sets the `find_unused_parameters` parameter in 78 | # torch.nn.parallel.DistributedDataParallel 79 | model = MMDistributedDataParallel( 80 | model.cuda(), 81 | device_ids=[torch.cuda.current_device()], 82 | broadcast_buffers=False, 83 | find_unused_parameters=find_unused_parameters) 84 | if eval_model is not None: 85 | eval_model = MMDistributedDataParallel( 86 | eval_model.cuda(), 87 | device_ids=[torch.cuda.current_device()], 88 | broadcast_buffers=False, 89 | find_unused_parameters=find_unused_parameters) 90 | else: 91 | model = MMDataParallel( 92 | model.cuda(cfg.gpu_ids[0]), device_ids=cfg.gpu_ids) 93 | if eval_model is not None: 94 | eval_model = MMDataParallel( 95 | eval_model.cuda(cfg.gpu_ids[0]), device_ids=cfg.gpu_ids) 96 | 97 | 98 | # build runner 99 | optimizer = build_optimizer(model, cfg.optimizer) 100 | 101 | if 'runner' not in cfg: 102 | cfg.runner = { 103 | 'type': 'EpochBasedRunner', 104 | 'max_epochs': cfg.total_epochs 105 | } 106 | warnings.warn( 107 | 'config is now expected to have a `runner` section, ' 108 | 'please set `runner` in your config.', UserWarning) 109 | else: 110 | if 'total_epochs' in cfg: 111 | assert cfg.total_epochs == cfg.runner.max_epochs 112 | if eval_model is not None: 113 | runner = build_runner( 114 | cfg.runner, 115 | default_args=dict( 116 | model=model, 117 | eval_model=eval_model, 118 | optimizer=optimizer, 119 | work_dir=cfg.work_dir, 120 | logger=logger, 121 | meta=meta)) 122 | else: 123 | runner = build_runner( 124 | cfg.runner, 125 | default_args=dict( 126 | model=model, 127 | optimizer=optimizer, 128 | work_dir=cfg.work_dir, 129 | logger=logger, 130 | meta=meta)) 131 | 132 | # an ugly workaround to make .log and .log.json filenames the same 133 | runner.timestamp = timestamp 134 | 135 | # fp16 setting 136 | fp16_cfg = cfg.get('fp16', None) 137 | if fp16_cfg is not None: 138 | optimizer_config = Fp16OptimizerHook( 139 | **cfg.optimizer_config, **fp16_cfg, distributed=distributed) 140 | elif distributed and 'type' not in cfg.optimizer_config: 141 | optimizer_config = OptimizerHook(**cfg.optimizer_config) 142 | else: 143 | optimizer_config = cfg.optimizer_config 144 | 145 | # register hooks 146 | runner.register_training_hooks(cfg.lr_config, optimizer_config, 147 | cfg.checkpoint_config, cfg.log_config, 148 | cfg.get('momentum_config', None)) 149 | 150 | # register profiler hook 151 | #trace_config = dict(type='tb_trace', dir_name='work_dir') 152 | #profiler_config = dict(on_trace_ready=trace_config) 153 | #runner.register_profiler_hook(profiler_config) 154 | 155 | if distributed: 156 | if isinstance(runner, EpochBasedRunner): 157 | runner.register_hook(DistSamplerSeedHook()) 158 | 159 | # register eval hooks 160 | if validate: 161 | # Support batch_size > 1 in validation 162 | val_samples_per_gpu = cfg.data.val.pop('samples_per_gpu', 1) 163 | if val_samples_per_gpu > 1: 164 | assert False 165 | # Replace 'ImageToTensor' to 'DefaultFormatBundle' 166 | cfg.data.val.pipeline = replace_ImageToTensor( 167 | cfg.data.val.pipeline) 168 | val_dataset = custom_build_dataset(cfg.data.val, dict(test_mode=True)) 169 | 170 | val_dataloader = build_dataloader( 171 | val_dataset, 172 | samples_per_gpu=val_samples_per_gpu, 173 | workers_per_gpu=cfg.data.workers_per_gpu, 174 | dist=distributed, 175 | shuffle=False, 176 | shuffler_sampler=cfg.data.shuffler_sampler, # dict(type='DistributedGroupSampler'), 177 | nonshuffler_sampler=cfg.data.nonshuffler_sampler, # dict(type='DistributedSampler'), 178 | ) 179 | eval_cfg = cfg.get('evaluation', {}) 180 | eval_cfg['by_epoch'] = cfg.runner['type'] != 'IterBasedRunner' 181 | eval_cfg['jsonfile_prefix'] = osp.join('val', cfg.work_dir, time.ctime().replace(' ','_').replace(':','_')) 182 | eval_hook = CustomDistEvalHook if distributed else EvalHook 183 | runner.register_hook(eval_hook(val_dataloader, **eval_cfg)) 184 | 185 | # user-defined hooks 186 | if cfg.get('custom_hooks', None): 187 | custom_hooks = cfg.custom_hooks 188 | assert isinstance(custom_hooks, list), \ 189 | f'custom_hooks expect list type, but got {type(custom_hooks)}' 190 | for hook_cfg in cfg.custom_hooks: 191 | assert isinstance(hook_cfg, dict), \ 192 | 'Each item in custom_hooks expects dict type, but got ' \ 193 | f'{type(hook_cfg)}' 194 | hook_cfg = hook_cfg.copy() 195 | priority = hook_cfg.pop('priority', 'NORMAL') 196 | hook = build_from_cfg(hook_cfg, HOOKS) 197 | runner.register_hook(hook, priority=priority) 198 | 199 | if cfg.resume_from: 200 | runner.resume(cfg.resume_from) 201 | elif cfg.load_from: 202 | runner.load_checkpoint(cfg.load_from) 203 | runner.run(data_loaders, cfg.workflow) 204 | 205 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/test.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | import os.path as osp 7 | import pickle 8 | import shutil 9 | import tempfile 10 | import time 11 | 12 | import mmcv 13 | import torch 14 | import torch.distributed as dist 15 | from mmcv.image import tensor2imgs 16 | from mmcv.runner import get_dist_info 17 | 18 | from mmdet.core import encode_mask_results 19 | 20 | 21 | import mmcv 22 | import numpy as np 23 | import pycocotools.mask as mask_util 24 | 25 | def custom_encode_mask_results(mask_results): 26 | """Encode bitmap mask to RLE code. Semantic Masks only 27 | Args: 28 | mask_results (list | tuple[list]): bitmap mask results. 29 | In mask scoring rcnn, mask_results is a tuple of (segm_results, 30 | segm_cls_score). 31 | Returns: 32 | list | tuple: RLE encoded mask. 33 | """ 34 | cls_segms = mask_results 35 | num_classes = len(cls_segms) 36 | encoded_mask_results = [] 37 | for i in range(len(cls_segms)): 38 | encoded_mask_results.append( 39 | mask_util.encode( 40 | np.array( 41 | cls_segms[i][:, :, np.newaxis], order='F', 42 | dtype='uint8'))[0]) # encoded with RLE 43 | return [encoded_mask_results] 44 | 45 | def custom_multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False): 46 | """Test model with multiple gpus. 47 | This method tests model with multiple gpus and collects the results 48 | under two different modes: gpu and cpu modes. By setting 'gpu_collect=True' 49 | it encodes results to gpu tensors and use gpu communication for results 50 | collection. On cpu mode it saves the results on different gpus to 'tmpdir' 51 | and collects them by the rank 0 worker. 52 | Args: 53 | model (nn.Module): Model to be tested. 54 | data_loader (nn.Dataloader): Pytorch data loader. 55 | tmpdir (str): Path of directory to save the temporary results from 56 | different gpus under cpu mode. 57 | gpu_collect (bool): Option to use either gpu or cpu to collect results. 58 | Returns: 59 | list: The prediction results. 60 | """ 61 | model.eval() 62 | bbox_results = [] 63 | mask_results = [] 64 | dataset = data_loader.dataset 65 | rank, world_size = get_dist_info() 66 | if rank == 0: 67 | prog_bar = mmcv.ProgressBar(len(dataset)) 68 | time.sleep(2) # This line can prevent deadlock problem in some cases. 69 | have_mask = False 70 | for i, data in enumerate(data_loader): 71 | with torch.no_grad(): 72 | result = model(return_loss=False, rescale=True, **data) 73 | # encode mask results 74 | if isinstance(result, dict): 75 | if 'bbox_results' in result.keys(): 76 | bbox_result = result['bbox_results'] 77 | batch_size = len(result['bbox_results']) 78 | bbox_results.extend(bbox_result) 79 | if 'mask_results' in result.keys() and result['mask_results'] is not None: 80 | mask_result = custom_encode_mask_results(result['mask_results']) 81 | mask_results.extend(mask_result) 82 | have_mask = True 83 | else: 84 | batch_size = len(result) 85 | bbox_results.extend(result) 86 | 87 | #if isinstance(result[0], tuple): 88 | # assert False, 'this code is for instance segmentation, which our code will not utilize.' 89 | # result = [(bbox_results, encode_mask_results(mask_results)) 90 | # for bbox_results, mask_results in result] 91 | if rank == 0: 92 | 93 | for _ in range(batch_size * world_size): 94 | prog_bar.update() 95 | 96 | # collect results from all ranks 97 | if gpu_collect: 98 | bbox_results = collect_results_gpu(bbox_results, len(dataset)) 99 | if have_mask: 100 | mask_results = collect_results_gpu(mask_results, len(dataset)) 101 | else: 102 | mask_results = None 103 | else: 104 | bbox_results = collect_results_cpu(bbox_results, len(dataset), tmpdir) 105 | tmpdir = tmpdir+'_mask' if tmpdir is not None else None 106 | if have_mask: 107 | mask_results = collect_results_cpu(mask_results, len(dataset), tmpdir) 108 | else: 109 | mask_results = None 110 | 111 | if mask_results is None: 112 | return bbox_results 113 | return {'bbox_results': bbox_results, 'mask_results': mask_results} 114 | 115 | 116 | def collect_results_cpu(result_part, size, tmpdir=None): 117 | rank, world_size = get_dist_info() 118 | # create a tmp dir if it is not specified 119 | if tmpdir is None: 120 | MAX_LEN = 512 121 | # 32 is whitespace 122 | dir_tensor = torch.full((MAX_LEN, ), 123 | 32, 124 | dtype=torch.uint8, 125 | device='cuda') 126 | if rank == 0: 127 | mmcv.mkdir_or_exist('.dist_test') 128 | tmpdir = tempfile.mkdtemp(dir='.dist_test') 129 | tmpdir = torch.tensor( 130 | bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') 131 | dir_tensor[:len(tmpdir)] = tmpdir 132 | dist.broadcast(dir_tensor, 0) 133 | tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() 134 | else: 135 | mmcv.mkdir_or_exist(tmpdir) 136 | # dump the part result to the dir 137 | mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl')) 138 | dist.barrier() 139 | # collect all parts 140 | if rank != 0: 141 | return None 142 | else: 143 | # load results of all parts from tmp dir 144 | part_list = [] 145 | for i in range(world_size): 146 | part_file = osp.join(tmpdir, f'part_{i}.pkl') 147 | part_list.append(mmcv.load(part_file)) 148 | # sort the results 149 | ordered_results = [] 150 | ''' 151 | bacause we change the sample of the evaluation stage to make sure that each gpu will handle continuous sample, 152 | ''' 153 | #for res in zip(*part_list): 154 | for res in part_list: 155 | ordered_results.extend(list(res)) 156 | # the dataloader may pad some samples 157 | ordered_results = ordered_results[:size] 158 | # remove tmp dir 159 | shutil.rmtree(tmpdir) 160 | return ordered_results 161 | 162 | 163 | def collect_results_gpu(result_part, size): 164 | collect_results_cpu(result_part, size) -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/apis/train.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | # --------------------------------------------- 7 | # Modified by Shihao Wang 8 | # --------------------------------------------- 9 | 10 | from .mmdet_train import custom_train_detector 11 | from mmseg.apis import train_segmentor 12 | from mmdet.apis import train_detector 13 | 14 | def custom_train_model(model, 15 | dataset, 16 | cfg, 17 | distributed=False, 18 | validate=False, 19 | timestamp=None, 20 | eval_model=None, 21 | meta=None): 22 | """A function wrapper for launching model training according to cfg. 23 | 24 | Because we need different eval_hook in runner. Should be deprecated in the 25 | future. 26 | """ 27 | if cfg.model.type in ['EncoderDecoder3D']: 28 | assert False 29 | else: 30 | custom_train_detector( 31 | model, 32 | dataset, 33 | cfg, 34 | distributed=distributed, 35 | validate=validate, 36 | timestamp=timestamp, 37 | eval_model=eval_model, 38 | meta=meta) 39 | 40 | 41 | def train_model(model, 42 | dataset, 43 | cfg, 44 | distributed=False, 45 | validate=False, 46 | timestamp=None, 47 | meta=None): 48 | """A function wrapper for launching model training according to cfg. 49 | 50 | Because we need different eval_hook in runner. Should be deprecated in the 51 | future. 52 | """ 53 | if cfg.model.type in ['EncoderDecoder3D']: 54 | train_segmentor( 55 | model, 56 | dataset, 57 | cfg, 58 | distributed=distributed, 59 | validate=validate, 60 | timestamp=timestamp, 61 | meta=meta) 62 | else: 63 | train_detector( 64 | model, 65 | dataset, 66 | cfg, 67 | distributed=distributed, 68 | validate=validate, 69 | timestamp=timestamp, 70 | meta=meta) 71 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/__pycache__/util.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/__pycache__/util.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/assigners/__init__.py: -------------------------------------------------------------------------------- 1 | from .hungarian_assigner_3d import HungarianAssigner3D 2 | from .hungarian_assigner_2d import HungarianAssigner2D 3 | __all__ = ['HungarianAssigner3D', 'HungarianAssigner2D'] -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/assigners/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/assigners/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/assigners/__pycache__/hungarian_assigner_2d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/assigners/__pycache__/hungarian_assigner_2d.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/assigners/__pycache__/hungarian_assigner_3d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/assigners/__pycache__/hungarian_assigner_3d.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/assigners/hungarian_assigner_2d.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | # --------------------------------------------- 3 | # Modified by Shihao Wang 4 | # --------------------------------------------- 5 | import torch 6 | 7 | from mmdet.core.bbox.builder import BBOX_ASSIGNERS 8 | from mmdet.core.bbox.assigners import AssignResult 9 | from mmdet.core.bbox.assigners import BaseAssigner 10 | from mmdet.core.bbox.match_costs import build_match_cost 11 | from mmdet.core import bbox_cxcywh_to_xyxy 12 | 13 | try: 14 | from scipy.optimize import linear_sum_assignment 15 | except ImportError: 16 | linear_sum_assignment = None 17 | 18 | 19 | @BBOX_ASSIGNERS.register_module() 20 | class HungarianAssigner2D(BaseAssigner): 21 | """Computes one-to-one matching between predictions and ground truth. 22 | 23 | This class computes an assignment between the targets and the predictions 24 | based on the costs. The costs are weighted sum of three components: 25 | classification cost, regression L1 cost and regression iou cost. The 26 | targets don't include the no_object, so generally there are more 27 | predictions than targets. After the one-to-one matching, the un-matched 28 | are treated as backgrounds. Thus each query prediction will be assigned 29 | with `0` or a positive integer indicating the ground truth index: 30 | 31 | - 0: negative sample, no assigned gt 32 | - positive integer: positive sample, index (1-based) of assigned gt 33 | 34 | Args: 35 | cls_weight (int | float, optional): The scale factor for classification 36 | cost. Default 1.0. 37 | bbox_weight (int | float, optional): The scale factor for regression 38 | L1 cost. Default 1.0. 39 | iou_weight (int | float, optional): The scale factor for regression 40 | iou cost. Default 1.0. 41 | iou_calculator (dict | optional): The config for the iou calculation. 42 | Default type `BboxOverlaps2D`. 43 | iou_mode (str | optional): "iou" (intersection over union), "iof" 44 | (intersection over foreground), or "giou" (generalized 45 | intersection over union). Default "giou". 46 | """ 47 | 48 | def __init__(self, 49 | cls_cost=dict(type='ClassificationCost', weight=1.), 50 | reg_cost=dict(type='BBoxL1Cost', weight=1.0), 51 | iou_cost=dict(type='IoUCost', iou_mode='giou', weight=1.0), 52 | centers2d_cost=dict(type='BBox3DL1Cost', weight=1.0)): 53 | self.cls_cost = build_match_cost(cls_cost) 54 | self.reg_cost = build_match_cost(reg_cost) 55 | self.iou_cost = build_match_cost(iou_cost) 56 | self.centers2d_cost = build_match_cost(centers2d_cost) 57 | 58 | def assign(self, 59 | bbox_pred, 60 | cls_pred, 61 | pred_centers2d, 62 | gt_bboxes, 63 | gt_labels, 64 | centers2d, 65 | img_meta, 66 | gt_bboxes_ignore=None, 67 | eps=1e-7): 68 | """Computes one-to-one matching based on the weighted costs. 69 | 70 | This method assign each query prediction to a ground truth or 71 | background. The `assigned_gt_inds` with -1 means don't care, 72 | 0 means negative sample, and positive number is the index (1-based) 73 | of assigned gt. 74 | The assignment is done in the following steps, the order matters. 75 | 76 | 1. assign every prediction to -1 77 | 2. compute the weighted costs 78 | 3. do Hungarian matching on CPU based on the costs 79 | 4. assign all to 0 (background) first, then for each matched pair 80 | between predictions and gts, treat this prediction as foreground 81 | and assign the corresponding gt index (plus 1) to it. 82 | 83 | Args: 84 | bbox_pred (Tensor): Predicted boxes with normalized coordinates 85 | (cx, cy, w, h), which are all in range [0, 1]. Shape 86 | [num_query, 4]. 87 | cls_pred (Tensor): Predicted classification logits, shape 88 | [num_query, num_class]. 89 | gt_bboxes (Tensor): Ground truth boxes with unnormalized 90 | coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. 91 | gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). 92 | img_meta (dict): Meta information for current image. 93 | gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are 94 | labelled as `ignored`. Default None. 95 | eps (int | float, optional): A value added to the denominator for 96 | numerical stability. Default 1e-7. 97 | 98 | Returns: 99 | :obj:`AssignResult`: The assigned result. 100 | """ 101 | assert gt_bboxes_ignore is None, \ 102 | 'Only case when gt_bboxes_ignore is None is supported.' 103 | num_gts, num_bboxes = gt_bboxes.size(0), bbox_pred.size(0) 104 | 105 | # 1. assign -1 by default 106 | assigned_gt_inds = bbox_pred.new_full((num_bboxes, ), 107 | -1, 108 | dtype=torch.long) 109 | assigned_labels = bbox_pred.new_full((num_bboxes, ), 110 | -1, 111 | dtype=torch.long) 112 | if num_gts == 0 or num_bboxes == 0: 113 | # No ground truth or boxes, return empty assignment 114 | if num_gts == 0: 115 | # No ground truth, assign all to background 116 | assigned_gt_inds[:] = 0 117 | return AssignResult( 118 | num_gts, assigned_gt_inds, None, labels=assigned_labels) 119 | img_h, img_w, _ = img_meta['pad_shape'] 120 | factor = gt_bboxes.new_tensor([img_w, img_h, img_w, 121 | img_h]).unsqueeze(0) 122 | 123 | # 2. compute the weighted costs 124 | # classification and bboxcost. 125 | cls_cost = self.cls_cost(cls_pred, gt_labels) 126 | # regression L1 cost 127 | normalize_gt_bboxes = gt_bboxes / factor 128 | reg_cost = self.reg_cost(bbox_pred, normalize_gt_bboxes) 129 | # regression iou cost, defaultly giou is used in official DETR. 130 | bboxes = bbox_cxcywh_to_xyxy(bbox_pred) * factor 131 | iou_cost = self.iou_cost(bboxes, gt_bboxes) 132 | 133 | # center2d L1 cost 134 | normalize_centers2d = centers2d / factor[:, 0:2] 135 | centers2d_cost = self.centers2d_cost(pred_centers2d, normalize_centers2d) 136 | 137 | # weighted sum of above four costs 138 | cost = cls_cost + reg_cost + iou_cost + centers2d_cost 139 | cost = torch.nan_to_num(cost, nan=100.0, posinf=100.0, neginf=-100.0) 140 | # 3. do Hungarian matching on CPU using linear_sum_assignment 141 | cost = cost.detach().cpu() 142 | if linear_sum_assignment is None: 143 | raise ImportError('Please run "pip install scipy" ' 144 | 'to install scipy first.') 145 | matched_row_inds, matched_col_inds = linear_sum_assignment(cost) 146 | matched_row_inds = torch.from_numpy(matched_row_inds).to( 147 | bbox_pred.device) 148 | matched_col_inds = torch.from_numpy(matched_col_inds).to( 149 | bbox_pred.device) 150 | 151 | # 4. assign backgrounds and foregrounds 152 | # assign all indices to backgrounds first 153 | assigned_gt_inds[:] = 0 154 | # assign foregrounds based on matching results 155 | assigned_gt_inds[matched_row_inds] = matched_col_inds + 1 156 | assigned_labels[matched_row_inds] = gt_labels[matched_col_inds] 157 | return AssignResult( 158 | num_gts, assigned_gt_inds, None, labels=assigned_labels) 159 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/assigners/hungarian_assigner_3d.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------ 2 | # Modified from DETR3D (https://github.com/WangYueFt/detr3d) 3 | # Copyright (c) 2021 Wang, Yue 4 | # ------------------------------------------------------------------------ 5 | import torch 6 | from mmdet.core.bbox.builder import BBOX_ASSIGNERS 7 | from mmdet.core.bbox.assigners import AssignResult 8 | from mmdet.core.bbox.assigners import BaseAssigner 9 | from mmdet.core.bbox.match_costs import build_match_cost 10 | from projects.mmdet3d_plugin.core.bbox.util import normalize_bbox 11 | 12 | try: 13 | from scipy.optimize import linear_sum_assignment 14 | except ImportError: 15 | linear_sum_assignment = None 16 | 17 | @BBOX_ASSIGNERS.register_module() 18 | class HungarianAssigner3D(BaseAssigner): 19 | def __init__(self, 20 | cls_cost=dict(type='ClassificationCost', weight=1.), 21 | reg_cost=dict(type='BBoxL1Cost', weight=1.0), 22 | iou_cost=dict(type='IoUCost', weight=0.0), 23 | pc_range=None): 24 | self.cls_cost = build_match_cost(cls_cost) 25 | self.reg_cost = build_match_cost(reg_cost) 26 | self.iou_cost = build_match_cost(iou_cost) 27 | self.pc_range = pc_range 28 | 29 | def assign(self, 30 | bbox_pred, 31 | cls_pred, 32 | gt_bboxes, 33 | gt_labels, 34 | gt_bboxes_ignore=None, 35 | code_weights=None, 36 | with_velo=False, 37 | eps=1e-7): 38 | assert gt_bboxes_ignore is None, \ 39 | 'Only case when gt_bboxes_ignore is None is supported.' 40 | num_gts, num_bboxes = gt_bboxes.size(0), bbox_pred.size(0) 41 | # 1. assign -1 by default 42 | assigned_gt_inds = bbox_pred.new_full((num_bboxes, ), 43 | -1, 44 | dtype=torch.long) 45 | assigned_labels = bbox_pred.new_full((num_bboxes, ), 46 | -1, 47 | dtype=torch.long) 48 | if num_gts == 0 or num_bboxes == 0: 49 | # No ground truth or boxes, return empty assignment 50 | if num_gts == 0: 51 | # No ground truth, assign all to background 52 | assigned_gt_inds[:] = 0 53 | return AssignResult( 54 | num_gts, assigned_gt_inds, None, labels=assigned_labels) 55 | # 2. compute the weighted costs 56 | # classification and bboxcost. 57 | cls_cost = self.cls_cost(cls_pred, gt_labels) 58 | # regression L1 cost 59 | normalized_gt_bboxes = normalize_bbox(gt_bboxes, self.pc_range) 60 | if code_weights is not None: 61 | bbox_pred = bbox_pred * code_weights 62 | normalized_gt_bboxes = normalized_gt_bboxes * code_weights 63 | 64 | if with_velo: 65 | reg_cost = self.reg_cost(bbox_pred, normalized_gt_bboxes) 66 | else: 67 | reg_cost = self.reg_cost(bbox_pred[:, :8], normalized_gt_bboxes[:, :8]) 68 | 69 | # weighted sum of above two costs 70 | cost = cls_cost + reg_cost 71 | 72 | # 3. do Hungarian matching on CPU using linear_sum_assignment 73 | cost = cost.detach().cpu() 74 | if linear_sum_assignment is None: 75 | raise ImportError('Please run "pip install scipy" ' 76 | 'to install scipy first.') 77 | cost = torch.nan_to_num(cost, nan=100.0, posinf=100.0, neginf=-100.0) 78 | matched_row_inds, matched_col_inds = linear_sum_assignment(cost) 79 | matched_row_inds = torch.from_numpy(matched_row_inds).to( 80 | bbox_pred.device) 81 | matched_col_inds = torch.from_numpy(matched_col_inds).to( 82 | bbox_pred.device) 83 | 84 | # 4. assign backgrounds and foregrounds 85 | # assign all indices to backgrounds first 86 | assigned_gt_inds[:] = 0 87 | # assign foregrounds based on matching results 88 | assigned_gt_inds[matched_row_inds] = matched_col_inds + 1 89 | assigned_labels[matched_row_inds] = gt_labels[matched_col_inds] 90 | return AssignResult( 91 | num_gts, assigned_gt_inds, None, labels=assigned_labels) -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/coders/__init__.py: -------------------------------------------------------------------------------- 1 | from .nms_free_coder import NMSFreeCoder 2 | __all__ = ['NMSFreeCoder'] 3 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/coders/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/coders/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/coders/__pycache__/nms_free_coder.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/coders/__pycache__/nms_free_coder.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/coders/nms_free_coder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from mmdet.core.bbox import BaseBBoxCoder 4 | from mmdet.core.bbox.builder import BBOX_CODERS 5 | from projects.mmdet3d_plugin.core.bbox.util import denormalize_bbox 6 | 7 | 8 | @BBOX_CODERS.register_module() 9 | class NMSFreeCoder(BaseBBoxCoder): 10 | """Bbox coder for NMS-free detector. 11 | Args: 12 | pc_range (list[float]): Range of point cloud. 13 | post_center_range (list[float]): Limit of the center. 14 | Default: None. 15 | max_num (int): Max number to be kept. Default: 100. 16 | score_threshold (float): Threshold to filter boxes based on score. 17 | Default: None. 18 | code_size (int): Code size of bboxes. Default: 9 19 | """ 20 | 21 | def __init__(self, 22 | pc_range, 23 | voxel_size=None, 24 | post_center_range=None, 25 | max_num=100, 26 | score_threshold=None, 27 | num_classes=10): 28 | 29 | self.pc_range = pc_range 30 | self.voxel_size = voxel_size 31 | self.post_center_range = post_center_range 32 | self.max_num = max_num 33 | self.score_threshold = score_threshold 34 | self.num_classes = num_classes 35 | 36 | def encode(self): 37 | pass 38 | 39 | def decode_single(self, cls_scores, bbox_preds): 40 | """Decode bboxes. 41 | Args: 42 | cls_scores (Tensor): Outputs from the classification head, \ 43 | shape [num_query, cls_out_channels]. Note \ 44 | cls_out_channels should includes background. 45 | bbox_preds (Tensor): Outputs from the regression \ 46 | head with normalized coordinate format (cx, cy, w, l, cz, h, rot_sine, rot_cosine, vx, vy). \ 47 | Shape [num_query, 9]. 48 | Returns: 49 | list[dict]: Decoded boxes. 50 | """ 51 | max_num = self.max_num 52 | 53 | cls_scores = cls_scores.sigmoid() 54 | scores, indexs = cls_scores.view(-1).topk(max_num) 55 | labels = indexs % self.num_classes 56 | bbox_index = torch.div(indexs, self.num_classes, rounding_mode='floor') 57 | bbox_preds = bbox_preds[bbox_index] 58 | 59 | final_box_preds = denormalize_bbox(bbox_preds, self.pc_range) 60 | final_scores = scores 61 | final_preds = labels 62 | 63 | # use score threshold 64 | if self.score_threshold is not None: 65 | thresh_mask = final_scores >= self.score_threshold 66 | if self.post_center_range is not None: 67 | self.post_center_range = torch.tensor(self.post_center_range, device=scores.device) 68 | 69 | mask = (final_box_preds[..., :3] >= 70 | self.post_center_range[:3]).all(1) 71 | mask &= (final_box_preds[..., :3] <= 72 | self.post_center_range[3:]).all(1) 73 | 74 | if self.score_threshold: 75 | mask &= thresh_mask 76 | 77 | boxes3d = final_box_preds[mask] 78 | scores = final_scores[mask] 79 | labels = final_preds[mask] 80 | predictions_dict = { 81 | 'bboxes': boxes3d, 82 | 'scores': scores, 83 | 'labels': labels 84 | } 85 | 86 | else: 87 | raise NotImplementedError( 88 | 'Need to reorganize output as a batch, only ' 89 | 'support post_center_range is not None for now!') 90 | return predictions_dict 91 | 92 | def decode(self, preds_dicts): 93 | """Decode bboxes. 94 | Args: 95 | all_cls_scores (Tensor): Outputs from the classification head, \ 96 | shape [nb_dec, bs, num_query, cls_out_channels]. Note \ 97 | cls_out_channels should includes background. 98 | all_bbox_preds (Tensor): Sigmoid outputs from the regression \ 99 | head with normalized coordinate format (cx, cy, w, l, cz, h, rot_sine, rot_cosine, vx, vy). \ 100 | Shape [nb_dec, bs, num_query, 9]. 101 | Returns: 102 | list[dict]: Decoded boxes. 103 | """ 104 | all_cls_scores = preds_dicts['all_cls_scores'][-1] 105 | all_bbox_preds = preds_dicts['all_bbox_preds'][-1] 106 | 107 | batch_size = all_cls_scores.size()[0] 108 | predictions_list = [] 109 | for i in range(batch_size): 110 | predictions_list.append(self.decode_single(all_cls_scores[i], all_bbox_preds[i])) 111 | return predictions_list 112 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/match_costs/__init__.py: -------------------------------------------------------------------------------- 1 | from mmdet.core.bbox.match_costs import build_match_cost 2 | from .match_cost import BBox3DL1Cost 3 | 4 | __all__ = ['build_match_cost', 'BBox3DL1Cost'] -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/match_costs/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/match_costs/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/match_costs/__pycache__/match_cost.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/bbox/match_costs/__pycache__/match_cost.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/match_costs/match_cost.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from mmdet.core.bbox.match_costs.builder import MATCH_COST 3 | 4 | @MATCH_COST.register_module() 5 | class BBox3DL1Cost(object): 6 | """BBox3DL1Cost. 7 | Args: 8 | weight (int | float, optional): loss_weight 9 | """ 10 | 11 | def __init__(self, weight=1.): 12 | self.weight = weight 13 | 14 | def __call__(self, bbox_pred, gt_bboxes): 15 | """ 16 | Args: 17 | bbox_pred (Tensor): Predicted boxes with normalized coordinates 18 | (cx, cy, w, h), which are all in range [0, 1]. Shape 19 | [num_query, 4]. 20 | gt_bboxes (Tensor): Ground truth boxes with normalized 21 | coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. 22 | Returns: 23 | torch.Tensor: bbox_cost value with weight 24 | """ 25 | bbox_cost = torch.cdist(bbox_pred, gt_bboxes, p=1) 26 | return bbox_cost * self.weight 27 | 28 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/bbox/util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def normalize_bbox(bboxes, pc_range): 4 | cx = bboxes[..., 0:1] 5 | cy = bboxes[..., 1:2] 6 | cz = bboxes[..., 2:3] 7 | w = bboxes[..., 3:4].log() 8 | l = bboxes[..., 4:5].log() 9 | h = bboxes[..., 5:6].log() 10 | 11 | rot = bboxes[..., 6:7] 12 | if bboxes.size(-1) > 7: 13 | vx = bboxes[..., 7:8] 14 | vy = bboxes[..., 8:9] 15 | normalized_bboxes = torch.cat( 16 | (cx, cy, cz, w, l, h, rot.sin(), rot.cos(), vx, vy), dim=-1 17 | ) 18 | else: 19 | normalized_bboxes = torch.cat( 20 | (cx, cy, cz, w, l, h, rot.sin(), rot.cos()), dim=-1 21 | ) 22 | return normalized_bboxes 23 | 24 | def denormalize_bbox(normalized_bboxes, pc_range): 25 | # rotation 26 | rot_sine = normalized_bboxes[..., 6:7] 27 | 28 | rot_cosine = normalized_bboxes[..., 7:8] 29 | rot = torch.atan2(rot_sine, rot_cosine) 30 | 31 | # center in the bev 32 | cx = normalized_bboxes[..., 0:1] 33 | cy = normalized_bboxes[..., 1:2] 34 | cz = normalized_bboxes[..., 2:3] 35 | 36 | # size 37 | w = normalized_bboxes[..., 3:4] 38 | l = normalized_bboxes[..., 4:5] 39 | h = normalized_bboxes[..., 5:6] 40 | 41 | w = w.exp() 42 | l = l.exp() 43 | h = h.exp() 44 | if normalized_bboxes.size(-1) > 8: 45 | # velocity 46 | vx = normalized_bboxes[:, 8:9] 47 | vy = normalized_bboxes[:, 9:10] 48 | denormalized_bboxes = torch.cat([cx, cy, cz, w, l, h, rot, vx, vy], dim=-1) 49 | else: 50 | denormalized_bboxes = torch.cat([cx, cy, cz, w, l, h, rot], dim=-1) 51 | return denormalized_bboxes -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .eval_hooks import CustomDistEvalHook -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/evaluation/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/evaluation/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/evaluation/__pycache__/eval_hooks.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/core/evaluation/__pycache__/eval_hooks.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/core/evaluation/eval_hooks.py: -------------------------------------------------------------------------------- 1 | 2 | # Note: Considering that MMCV's EvalHook updated its interface in V1.3.16, 3 | # in order to avoid strong version dependency, we did not directly 4 | # inherit EvalHook but BaseDistEvalHook. 5 | 6 | import bisect 7 | import os.path as osp 8 | 9 | import mmcv 10 | import torch.distributed as dist 11 | from mmcv.runner import DistEvalHook as BaseDistEvalHook 12 | from mmcv.runner import EvalHook as BaseEvalHook 13 | from torch.nn.modules.batchnorm import _BatchNorm 14 | from mmdet.core.evaluation.eval_hooks import DistEvalHook 15 | 16 | 17 | def _calc_dynamic_intervals(start_interval, dynamic_interval_list): 18 | assert mmcv.is_list_of(dynamic_interval_list, tuple) 19 | 20 | dynamic_milestones = [0] 21 | dynamic_milestones.extend( 22 | [dynamic_interval[0] for dynamic_interval in dynamic_interval_list]) 23 | dynamic_intervals = [start_interval] 24 | dynamic_intervals.extend( 25 | [dynamic_interval[1] for dynamic_interval in dynamic_interval_list]) 26 | return dynamic_milestones, dynamic_intervals 27 | 28 | 29 | class CustomDistEvalHook(BaseDistEvalHook): 30 | 31 | def __init__(self, *args, dynamic_intervals=None, **kwargs): 32 | super(CustomDistEvalHook, self).__init__(*args, **kwargs) 33 | self.use_dynamic_intervals = dynamic_intervals is not None 34 | if self.use_dynamic_intervals: 35 | self.dynamic_milestones, self.dynamic_intervals = \ 36 | _calc_dynamic_intervals(self.interval, dynamic_intervals) 37 | 38 | def _decide_interval(self, runner): 39 | if self.use_dynamic_intervals: 40 | progress = runner.epoch if self.by_epoch else runner.iter 41 | step = bisect.bisect(self.dynamic_milestones, (progress + 1)) 42 | # Dynamically modify the evaluation interval 43 | self.interval = self.dynamic_intervals[step - 1] 44 | 45 | def before_train_epoch(self, runner): 46 | """Evaluate the model only at the start of training by epoch.""" 47 | self._decide_interval(runner) 48 | super().before_train_epoch(runner) 49 | 50 | def before_train_iter(self, runner): 51 | self._decide_interval(runner) 52 | super().before_train_iter(runner) 53 | 54 | def _do_evaluate(self, runner): 55 | """perform evaluation and save ckpt.""" 56 | # Synchronization of BatchNorm's buffer (running_mean 57 | # and running_var) is not supported in the DDP of pytorch, 58 | # which may cause the inconsistent performance of models in 59 | # different ranks, so we broadcast BatchNorm's buffers 60 | # of rank 0 to other ranks to avoid this. 61 | if self.broadcast_bn_buffer: 62 | model = runner.model 63 | for name, module in model.named_modules(): 64 | if isinstance(module, 65 | _BatchNorm) and module.track_running_stats: 66 | dist.broadcast(module.running_var, 0) 67 | dist.broadcast(module.running_mean, 0) 68 | 69 | if not self._should_evaluate(runner): 70 | return 71 | 72 | tmpdir = self.tmpdir 73 | if tmpdir is None: 74 | tmpdir = osp.join(runner.work_dir, '.eval_hook') 75 | 76 | from projects.mmdet3d_plugin.core.apis.test import custom_multi_gpu_test # to solve circlur import 77 | 78 | results = custom_multi_gpu_test( 79 | runner.model, 80 | self.dataloader, 81 | tmpdir=tmpdir, 82 | gpu_collect=self.gpu_collect) 83 | if runner.rank == 0: 84 | print('\n') 85 | runner.log_buffer.output['eval_iter_num'] = len(self.dataloader) 86 | 87 | key_score = self.evaluate(runner, results) 88 | 89 | if self.save_best: 90 | self._save_ckpt(runner, key_score) 91 | 92 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/.ipynb_checkpoints/builder-checkpoint.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Shihao Wang 5 | # --------------------------------------------- 6 | import copy 7 | import platform 8 | import random 9 | from functools import partial 10 | 11 | import numpy as np 12 | from mmcv.parallel import collate 13 | from mmcv.runner import get_dist_info 14 | from mmcv.utils import Registry, build_from_cfg 15 | from torch.utils.data import DataLoader 16 | 17 | from mmdet.datasets.samplers import GroupSampler 18 | from projects.mmdet3d_plugin.datasets.samplers.group_sampler import DistributedGroupSampler 19 | from projects.mmdet3d_plugin.datasets.samplers.distributed_sampler import DistributedSampler 20 | from projects.mmdet3d_plugin.datasets.samplers.group_sampler import InfiniteGroupEachSampleInBatchSampler 21 | from projects.mmdet3d_plugin.datasets.samplers.sampler import build_sampler 22 | 23 | def build_dataloader(dataset, 24 | samples_per_gpu, 25 | workers_per_gpu, 26 | num_gpus=1, 27 | dist=True, 28 | shuffle=True, 29 | seed=None, 30 | shuffler_sampler=None, 31 | nonshuffler_sampler=None, 32 | runner_type=dict(type='EpochBasedRunner'), 33 | **kwargs): 34 | """Build PyTorch DataLoader. 35 | In distributed training, each GPU/process has a dataloader. 36 | In non-distributed training, there is only one dataloader for all GPUs. 37 | Args: 38 | dataset (Dataset): A PyTorch dataset. 39 | samples_per_gpu (int): Number of training samples on each GPU, i.e., 40 | batch size of each GPU. 41 | workers_per_gpu (int): How many subprocesses to use for data loading 42 | for each GPU. 43 | num_gpus (int): Number of GPUs. Only used in non-distributed training. 44 | dist (bool): Distributed training/test or not. Default: True. 45 | shuffle (bool): Whether to shuffle the data at every epoch. 46 | Default: True. 47 | kwargs: any keyword argument to be used to initialize DataLoader 48 | Returns: 49 | DataLoader: A PyTorch dataloader. 50 | """ 51 | rank, world_size = get_dist_info() 52 | 53 | if dist: 54 | # DistributedGroupSampler will definitely shuffle the data to satisfy 55 | # that images on each GPU are in the same group 56 | if shuffle: 57 | sampler = build_sampler(shuffler_sampler if shuffler_sampler is not None else dict(type='DistributedGroupSampler'), 58 | dict( 59 | dataset=dataset, 60 | samples_per_gpu=samples_per_gpu, 61 | num_replicas=world_size, 62 | rank=rank, 63 | seed=seed) 64 | ) 65 | 66 | else: 67 | sampler = build_sampler(nonshuffler_sampler if nonshuffler_sampler is not None else dict(type='DistributedSampler'), 68 | dict( 69 | dataset=dataset, 70 | num_replicas=world_size, 71 | rank=rank, 72 | shuffle=shuffle, 73 | seed=seed) 74 | ) 75 | 76 | batch_size = samples_per_gpu 77 | num_workers = workers_per_gpu 78 | batch_sampler = None 79 | 80 | 81 | else: 82 | # assert False, 'not support in bevformer' 83 | print('WARNING!!!!, Only can be used for obtain inference speed!!!!') 84 | sampler = GroupSampler(dataset, samples_per_gpu) if shuffle else None 85 | batch_size = num_gpus * samples_per_gpu 86 | num_workers = num_gpus * workers_per_gpu 87 | batch_sampler = None 88 | 89 | 90 | if runner_type['type'] == 'IterBasedRunner' and shuffler_sampler['type'] =='InfiniteGroupEachSampleInBatchSampler': 91 | # TODO: original has more options, but I'm not using them 92 | # https://github.com/open-mmlab/mmdetection/blob/3b72b12fe9b14de906d1363982b9fba05e7d47c1/mmdet/datasets/builder.py#L145-L157 93 | batch_sampler = InfiniteGroupEachSampleInBatchSampler( 94 | dataset, 95 | samples_per_gpu, 96 | world_size, 97 | rank, 98 | seed=seed) 99 | batch_size = 1 100 | sampler = None 101 | 102 | 103 | init_fn = partial( 104 | worker_init_fn, num_workers=num_workers, rank=rank, 105 | seed=seed) if seed is not None else None 106 | 107 | data_loader = DataLoader( 108 | dataset, 109 | batch_size=batch_size, 110 | sampler=sampler, 111 | batch_sampler=batch_sampler, 112 | num_workers=num_workers, 113 | collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), 114 | pin_memory=False, 115 | worker_init_fn=init_fn, 116 | **kwargs) 117 | 118 | return data_loader 119 | 120 | 121 | def worker_init_fn(worker_id, num_workers, rank, seed): 122 | # The seed of each worker equals to 123 | # num_worker * rank + worker_id + user_seed 124 | worker_seed = num_workers * rank + worker_id + seed 125 | np.random.seed(worker_seed) 126 | random.seed(worker_seed) 127 | 128 | 129 | # Copyright (c) OpenMMLab. All rights reserved. 130 | import platform 131 | from mmcv.utils import Registry, build_from_cfg 132 | 133 | from mmdet.datasets import DATASETS 134 | from mmdet.datasets.builder import _concat_dataset 135 | 136 | if platform.system() != 'Windows': 137 | # https://github.com/pytorch/pytorch/issues/973 138 | import resource 139 | rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) 140 | base_soft_limit = rlimit[0] 141 | hard_limit = rlimit[1] 142 | soft_limit = min(max(4096, base_soft_limit), hard_limit) 143 | resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) 144 | 145 | OBJECTSAMPLERS = Registry('Object sampler') 146 | 147 | 148 | def custom_build_dataset(cfg, default_args=None): 149 | from mmdet3d.datasets.dataset_wrappers import CBGSDataset 150 | from mmdet.datasets.dataset_wrappers import (ClassBalancedDataset, 151 | ConcatDataset, RepeatDataset) 152 | if isinstance(cfg, (list, tuple)): 153 | dataset = ConcatDataset([custom_build_dataset(c, default_args) for c in cfg]) 154 | elif cfg['type'] == 'ConcatDataset': 155 | dataset = ConcatDataset( 156 | [custom_build_dataset(c, default_args) for c in cfg['datasets']], 157 | cfg.get('separate_eval', True)) 158 | elif cfg['type'] == 'RepeatDataset': 159 | dataset = RepeatDataset( 160 | custom_build_dataset(cfg['dataset'], default_args), cfg['times']) 161 | elif cfg['type'] == 'ClassBalancedDataset': 162 | dataset = ClassBalancedDataset( 163 | custom_build_dataset(cfg['dataset'], default_args), cfg['oversample_thr']) 164 | elif cfg['type'] == 'CBGSDataset': 165 | dataset = CBGSDataset(custom_build_dataset(cfg['dataset'], default_args)) 166 | elif isinstance(cfg.get('ann_file'), (list, tuple)): 167 | dataset = _concat_dataset(cfg, default_args) 168 | else: 169 | dataset = build_from_cfg(cfg, DATASETS, default_args) 170 | 171 | return dataset -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .nuscenes_dataset import CustomNuScenesDataset 2 | from .builder import custom_build_dataset 3 | 4 | __all__ = [ 5 | 'CustomNuScenesDataset' 6 | ] 7 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/__pycache__/builder.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/__pycache__/builder.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/__pycache__/nuscenes_dataset.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/__pycache__/nuscenes_dataset.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/builder.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Shihao Wang 5 | # --------------------------------------------- 6 | import copy 7 | import platform 8 | import random 9 | from functools import partial 10 | 11 | import numpy as np 12 | from mmcv.parallel import collate 13 | from mmcv.runner import get_dist_info 14 | from mmcv.utils import Registry, build_from_cfg 15 | from torch.utils.data import DataLoader 16 | 17 | from mmdet.datasets.samplers import GroupSampler 18 | from projects.mmdet3d_plugin.datasets.samplers.group_sampler import DistributedGroupSampler 19 | from projects.mmdet3d_plugin.datasets.samplers.distributed_sampler import DistributedSampler 20 | from projects.mmdet3d_plugin.datasets.samplers.group_sampler import InfiniteGroupEachSampleInBatchSampler 21 | from projects.mmdet3d_plugin.datasets.samplers.sampler import build_sampler 22 | 23 | def build_dataloader(dataset, 24 | samples_per_gpu, 25 | workers_per_gpu, 26 | num_gpus=1, 27 | dist=True, 28 | shuffle=True, 29 | seed=None, 30 | shuffler_sampler=None, 31 | nonshuffler_sampler=None, 32 | runner_type=dict(type='EpochBasedRunner'), 33 | **kwargs): 34 | """Build PyTorch DataLoader. 35 | In distributed training, each GPU/process has a dataloader. 36 | In non-distributed training, there is only one dataloader for all GPUs. 37 | Args: 38 | dataset (Dataset): A PyTorch dataset. 39 | samples_per_gpu (int): Number of training samples on each GPU, i.e., 40 | batch size of each GPU. 41 | workers_per_gpu (int): How many subprocesses to use for data loading 42 | for each GPU. 43 | num_gpus (int): Number of GPUs. Only used in non-distributed training. 44 | dist (bool): Distributed training/test or not. Default: True. 45 | shuffle (bool): Whether to shuffle the data at every epoch. 46 | Default: True. 47 | kwargs: any keyword argument to be used to initialize DataLoader 48 | Returns: 49 | DataLoader: A PyTorch dataloader. 50 | """ 51 | rank, world_size = get_dist_info() 52 | 53 | if dist: 54 | # DistributedGroupSampler will definitely shuffle the data to satisfy 55 | # that images on each GPU are in the same group 56 | if shuffle: 57 | sampler = build_sampler(shuffler_sampler if shuffler_sampler is not None else dict(type='DistributedGroupSampler'), 58 | dict( 59 | dataset=dataset, 60 | samples_per_gpu=samples_per_gpu, 61 | num_replicas=world_size, 62 | rank=rank, 63 | seed=seed) 64 | ) 65 | 66 | else: 67 | sampler = build_sampler(nonshuffler_sampler if nonshuffler_sampler is not None else dict(type='DistributedSampler'), 68 | dict( 69 | dataset=dataset, 70 | num_replicas=world_size, 71 | rank=rank, 72 | shuffle=shuffle, 73 | seed=seed) 74 | ) 75 | 76 | batch_size = samples_per_gpu 77 | num_workers = workers_per_gpu 78 | batch_sampler = None 79 | 80 | 81 | else: 82 | # assert False, 'not support in bevformer' 83 | print('WARNING!!!!, Only can be used for obtain inference speed!!!!') 84 | sampler = GroupSampler(dataset, samples_per_gpu) if shuffle else None 85 | batch_size = num_gpus * samples_per_gpu 86 | num_workers = num_gpus * workers_per_gpu 87 | batch_sampler = None 88 | 89 | 90 | if runner_type['type'] == 'IterBasedRunner' and shuffler_sampler['type'] =='InfiniteGroupEachSampleInBatchSampler': 91 | # TODO: original has more options, but I'm not using them 92 | # https://github.com/open-mmlab/mmdetection/blob/3b72b12fe9b14de906d1363982b9fba05e7d47c1/mmdet/datasets/builder.py#L145-L157 93 | batch_sampler = InfiniteGroupEachSampleInBatchSampler( 94 | dataset, 95 | samples_per_gpu, 96 | world_size, 97 | rank, 98 | seed=seed) 99 | batch_size = 1 100 | sampler = None 101 | 102 | 103 | init_fn = partial( 104 | worker_init_fn, num_workers=num_workers, rank=rank, 105 | seed=seed) if seed is not None else None 106 | 107 | data_loader = DataLoader( 108 | dataset, 109 | batch_size=batch_size, 110 | sampler=sampler, 111 | batch_sampler=batch_sampler, 112 | num_workers=num_workers, 113 | collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), 114 | pin_memory=False, 115 | worker_init_fn=init_fn, 116 | **kwargs) 117 | 118 | return data_loader 119 | 120 | 121 | def worker_init_fn(worker_id, num_workers, rank, seed): 122 | # The seed of each worker equals to 123 | # num_worker * rank + worker_id + user_seed 124 | worker_seed = num_workers * rank + worker_id + seed 125 | np.random.seed(worker_seed) 126 | random.seed(worker_seed) 127 | 128 | 129 | # Copyright (c) OpenMMLab. All rights reserved. 130 | import platform 131 | from mmcv.utils import Registry, build_from_cfg 132 | 133 | from mmdet.datasets import DATASETS 134 | from mmdet.datasets.builder import _concat_dataset 135 | 136 | if platform.system() != 'Windows': 137 | # https://github.com/pytorch/pytorch/issues/973 138 | import resource 139 | rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) 140 | base_soft_limit = rlimit[0] 141 | hard_limit = rlimit[1] 142 | soft_limit = min(max(4096, base_soft_limit), hard_limit) 143 | resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) 144 | 145 | OBJECTSAMPLERS = Registry('Object sampler') 146 | 147 | 148 | def custom_build_dataset(cfg, default_args=None): 149 | from mmdet3d.datasets.dataset_wrappers import CBGSDataset 150 | from mmdet.datasets.dataset_wrappers import (ClassBalancedDataset, 151 | ConcatDataset, RepeatDataset) 152 | if isinstance(cfg, (list, tuple)): 153 | dataset = ConcatDataset([custom_build_dataset(c, default_args) for c in cfg]) 154 | elif cfg['type'] == 'ConcatDataset': 155 | dataset = ConcatDataset( 156 | [custom_build_dataset(c, default_args) for c in cfg['datasets']], 157 | cfg.get('separate_eval', True)) 158 | elif cfg['type'] == 'RepeatDataset': 159 | dataset = RepeatDataset( 160 | custom_build_dataset(cfg['dataset'], default_args), cfg['times']) 161 | elif cfg['type'] == 'ClassBalancedDataset': 162 | dataset = ClassBalancedDataset( 163 | custom_build_dataset(cfg['dataset'], default_args), cfg['oversample_thr']) 164 | elif cfg['type'] == 'CBGSDataset': 165 | dataset = CBGSDataset(custom_build_dataset(cfg['dataset'], default_args)) 166 | elif isinstance(cfg.get('ann_file'), (list, tuple)): 167 | dataset = _concat_dataset(cfg, default_args) 168 | else: 169 | dataset = build_from_cfg(cfg, DATASETS, default_args) 170 | 171 | return dataset -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | from .transform_3d import( 2 | PadMultiViewImage, 3 | NormalizeMultiviewImage, 4 | ResizeCropFlipRotImage, 5 | GlobalRotScaleTransImage, 6 | ) 7 | 8 | from .formating import( 9 | PETRFormatBundle3D, 10 | ) 11 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/pipelines/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/pipelines/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/pipelines/__pycache__/formating.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/pipelines/__pycache__/formating.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/pipelines/__pycache__/transform_3d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/pipelines/__pycache__/transform_3d.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/pipelines/formating.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------ 2 | # Modified from mmdetection3d (https://github.com/open-mmlab/mmdetection3d) 3 | # Copyright (c) OpenMMLab. All rights reserved. 4 | # ------------------------------------------------------------------------ 5 | # Modified by Shihao Wang 6 | # ------------------------------------------------------------------------ 7 | import numpy as np 8 | from mmdet.datasets.builder import PIPELINES 9 | from mmcv.parallel import DataContainer as DC 10 | from mmdet3d.core.points import BasePoints 11 | from mmdet.datasets.pipelines import to_tensor 12 | from mmdet3d.datasets.pipelines import DefaultFormatBundle 13 | 14 | @PIPELINES.register_module() 15 | class PETRFormatBundle3D(DefaultFormatBundle): 16 | """Default formatting bundle. 17 | 18 | It simplifies the pipeline of formatting common fields for voxels, 19 | including "proposals", "gt_bboxes", "gt_labels", "gt_masks" and 20 | "gt_semantic_seg". 21 | These fields are formatted as follows. 22 | 23 | - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) 24 | - proposals: (1)to tensor, (2)to DataContainer 25 | - gt_bboxes: (1)to tensor, (2)to DataContainer 26 | - gt_bboxes_ignore: (1)to tensor, (2)to DataContainer 27 | - gt_labels: (1)to tensor, (2)to DataContainer 28 | """ 29 | 30 | def __init__(self, class_names, collect_keys, with_gt=True, with_label=True): 31 | super(PETRFormatBundle3D, self).__init__() 32 | self.class_names = class_names 33 | self.with_gt = with_gt 34 | self.with_label = with_label 35 | self.collect_keys = collect_keys 36 | def __call__(self, results): 37 | """Call function to transform and format common fields in results. 38 | 39 | Args: 40 | results (dict): Result dict contains the data to convert. 41 | 42 | Returns: 43 | dict: The result dict contains the data that is formatted with 44 | default bundle. 45 | """ 46 | # Format 3D data 47 | if 'points' in results: 48 | assert isinstance(results['points'], BasePoints) 49 | results['points'] = DC(results['points'].tensor) 50 | 51 | for key in self.collect_keys: 52 | if key in ['timestamp', 'img_timestamp']: 53 | results[key] = DC(to_tensor(np.array(results[key], dtype=np.float64))) 54 | else: 55 | results[key] = DC(to_tensor(np.array(results[key], dtype=np.float32))) 56 | 57 | for key in ['voxels', 'coors', 'voxel_centers', 'num_points']: 58 | if key not in results: 59 | continue 60 | results[key] = DC(to_tensor(results[key]), stack=False) 61 | 62 | if self.with_gt: 63 | # Clean GT bboxes in the final 64 | if 'gt_bboxes_3d_mask' in results: 65 | gt_bboxes_3d_mask = results['gt_bboxes_3d_mask'] 66 | results['gt_bboxes_3d'] = results['gt_bboxes_3d'][ 67 | gt_bboxes_3d_mask] 68 | if 'gt_names_3d' in results: 69 | results['gt_names_3d'] = results['gt_names_3d'][ 70 | gt_bboxes_3d_mask] 71 | if 'centers2d' in results: 72 | results['centers2d'] = results['centers2d'][ 73 | gt_bboxes_3d_mask] 74 | if 'depths' in results: 75 | results['depths'] = results['depths'][gt_bboxes_3d_mask] 76 | if 'gt_bboxes_mask' in results: 77 | gt_bboxes_mask = results['gt_bboxes_mask'] 78 | if 'gt_bboxes' in results: 79 | results['gt_bboxes'] = results['gt_bboxes'][gt_bboxes_mask] 80 | results['gt_names'] = results['gt_names'][gt_bboxes_mask] 81 | if self.with_label: 82 | if 'gt_names' in results and len(results['gt_names']) == 0: 83 | results['gt_labels'] = np.array([], dtype=np.int64) 84 | results['attr_labels'] = np.array([], dtype=np.int64) 85 | elif 'gt_names' in results and isinstance( 86 | results['gt_names'][0], list): 87 | # gt_labels might be a list of list in multi-view setting 88 | results['gt_labels'] = [ 89 | np.array([self.class_names.index(n) for n in res], 90 | dtype=np.int64) for res in results['gt_names'] 91 | ] 92 | elif 'gt_names' in results: 93 | results['gt_labels'] = np.array([ 94 | self.class_names.index(n) for n in results['gt_names'] 95 | ], 96 | dtype=np.int64) 97 | # we still assume one pipeline for one frame LiDAR 98 | # thus, the 3D name is list[string] 99 | if 'gt_names_3d' in results: 100 | results['gt_labels_3d'] = np.array([ 101 | self.class_names.index(n) 102 | for n in results['gt_names_3d'] 103 | ], 104 | dtype=np.int64) 105 | results = super(PETRFormatBundle3D, self).__call__(results) 106 | return results 107 | 108 | def __repr__(self): 109 | """str: Return a string that describes the module.""" 110 | repr_str = self.__class__.__name__ 111 | repr_str += f'(class_names={self.class_names}, ' 112 | repr_str += f'collect_keys={self.collect_keys}, with_gt={self.with_gt}, with_label={self.with_label})' 113 | return repr_str -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/results_save_total.pkl: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a29a311289360e947238fb24c172780637689cc67e464acbcd21d56ed090fc5b 3 | size 142379641 4 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/__init__.py: -------------------------------------------------------------------------------- 1 | from .group_sampler import DistributedGroupSampler, InfiniteGroupEachSampleInBatchSampler 2 | from .distributed_sampler import DistributedSampler 3 | from .sampler import SAMPLER, build_sampler 4 | 5 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/samplers/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/__pycache__/distributed_sampler.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/samplers/__pycache__/distributed_sampler.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/__pycache__/group_sampler.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/samplers/__pycache__/group_sampler.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/__pycache__/sampler.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/datasets/samplers/__pycache__/sampler.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/distributed_sampler.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | # Modified by Shihao Wang 7 | # --------------------------------------------- 8 | import math 9 | import torch 10 | from torch.utils.data import DistributedSampler as _DistributedSampler 11 | from .sampler import SAMPLER 12 | 13 | 14 | @SAMPLER.register_module() 15 | class DistributedSampler(_DistributedSampler): 16 | 17 | def __init__(self, 18 | dataset=None, 19 | num_replicas=None, 20 | rank=None, 21 | shuffle=True, 22 | seed=0): 23 | super().__init__( 24 | dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle) 25 | # for the compatibility from PyTorch 1.3+ 26 | self.seed = seed if seed is not None else 0 27 | 28 | def __iter__(self): 29 | # deterministically shuffle based on epoch 30 | if self.shuffle: 31 | assert False 32 | else: 33 | indices = torch.arange(len(self.dataset)).tolist() 34 | 35 | # add extra samples to make it evenly divisible 36 | # in case that indices is shorter than half of total_size 37 | indices = (indices * 38 | math.ceil(self.total_size / len(indices)))[:self.total_size] 39 | assert len(indices) == self.total_size 40 | 41 | # subsample 42 | per_replicas = self.total_size//self.num_replicas 43 | # indices = indices[self.rank:self.total_size:self.num_replicas] 44 | indices = indices[self.rank*per_replicas:(self.rank+1)*per_replicas] 45 | assert len(indices) == self.num_samples 46 | 47 | return iter(indices) 48 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/group_sampler.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | # Modified by Shihao Wang 7 | # --------------------------------------------- 8 | import math 9 | import itertools 10 | import copy 11 | import torch.distributed as dist 12 | import numpy as np 13 | import torch 14 | from mmcv.runner import get_dist_info 15 | from torch.utils.data import Sampler 16 | from .sampler import SAMPLER 17 | import random 18 | from IPython import embed 19 | 20 | 21 | @SAMPLER.register_module() 22 | class DistributedGroupSampler(Sampler): 23 | """Sampler that restricts data loading to a subset of the dataset. 24 | It is especially useful in conjunction with 25 | :class:`torch.nn.parallel.DistributedDataParallel`. In such case, each 26 | process can pass a DistributedSampler instance as a DataLoader sampler, 27 | and load a subset of the original dataset that is exclusive to it. 28 | .. note:: 29 | Dataset is assumed to be of constant size. 30 | Arguments: 31 | dataset: Dataset used for sampling. 32 | num_replicas (optional): Number of processes participating in 33 | distributed training. 34 | rank (optional): Rank of the current process within num_replicas. 35 | seed (int, optional): random seed used to shuffle the sampler if 36 | ``shuffle=True``. This number should be identical across all 37 | processes in the distributed group. Default: 0. 38 | """ 39 | 40 | def __init__(self, 41 | dataset, 42 | samples_per_gpu=1, 43 | num_replicas=None, 44 | rank=None, 45 | seed=0): 46 | _rank, _num_replicas = get_dist_info() 47 | if num_replicas is None: 48 | num_replicas = _num_replicas 49 | if rank is None: 50 | rank = _rank 51 | self.dataset = dataset 52 | self.samples_per_gpu = samples_per_gpu 53 | self.num_replicas = num_replicas 54 | self.rank = rank 55 | self.epoch = 0 56 | self.seed = seed if seed is not None else 0 57 | 58 | assert hasattr(self.dataset, 'flag') 59 | self.flag = self.dataset.flag 60 | self.group_sizes = np.bincount(self.flag) 61 | 62 | self.num_samples = 0 63 | for i, j in enumerate(self.group_sizes): 64 | self.num_samples += int( 65 | math.ceil(self.group_sizes[i] * 1.0 / self.samples_per_gpu / 66 | self.num_replicas)) * self.samples_per_gpu 67 | self.total_size = self.num_samples * self.num_replicas 68 | 69 | def __iter__(self): 70 | # deterministically shuffle based on epoch 71 | g = torch.Generator() 72 | g.manual_seed(self.epoch + self.seed) 73 | 74 | indices = [] 75 | for i, size in enumerate(self.group_sizes): 76 | if size > 0: 77 | indice = np.where(self.flag == i)[0] 78 | assert len(indice) == size 79 | # add .numpy() to avoid bug when selecting indice in parrots. 80 | # TODO: check whether torch.randperm() can be replaced by 81 | # numpy.random.permutation(). 82 | indice = indice[list( 83 | torch.randperm(int(size), generator=g).numpy())].tolist() 84 | extra = int( 85 | math.ceil( 86 | size * 1.0 / self.samples_per_gpu / self.num_replicas) 87 | ) * self.samples_per_gpu * self.num_replicas - len(indice) 88 | # pad indice 89 | tmp = indice.copy() 90 | for _ in range(extra // size): 91 | indice.extend(tmp) 92 | indice.extend(tmp[:extra % size]) 93 | indices.extend(indice) 94 | 95 | assert len(indices) == self.total_size 96 | 97 | indices = [ 98 | indices[j] for i in list( 99 | torch.randperm( 100 | len(indices) // self.samples_per_gpu, generator=g)) 101 | for j in range(i * self.samples_per_gpu, (i + 1) * 102 | self.samples_per_gpu) 103 | ] 104 | 105 | # subsample 106 | offset = self.num_samples * self.rank 107 | indices = indices[offset:offset + self.num_samples] 108 | assert len(indices) == self.num_samples 109 | 110 | return iter(indices) 111 | 112 | def __len__(self): 113 | return self.num_samples 114 | 115 | def set_epoch(self, epoch): 116 | self.epoch = epoch 117 | 118 | 119 | def sync_random_seed(seed=None, device='cuda'): 120 | """Make sure different ranks share the same seed. 121 | All workers must call this function, otherwise it will deadlock. 122 | This method is generally used in `DistributedSampler`, 123 | because the seed should be identical across all processes 124 | in the distributed group. 125 | In distributed sampling, different ranks should sample non-overlapped 126 | data in the dataset. Therefore, this function is used to make sure that 127 | each rank shuffles the data indices in the same order based 128 | on the same seed. Then different ranks could use different indices 129 | to select non-overlapped data from the same data list. 130 | Args: 131 | seed (int, Optional): The seed. Default to None. 132 | device (str): The device where the seed will be put on. 133 | Default to 'cuda'. 134 | Returns: 135 | int: Seed to be used. 136 | """ 137 | if seed is None: 138 | seed = np.random.randint(2**31) 139 | assert isinstance(seed, int) 140 | 141 | rank, num_replicas = get_dist_info() 142 | 143 | if num_replicas == 1: 144 | return seed 145 | 146 | if rank == 0: 147 | random_num = torch.tensor(seed, dtype=torch.int32, device=device) 148 | else: 149 | random_num = torch.tensor(0, dtype=torch.int32, device=device) 150 | dist.broadcast(random_num, src=0) 151 | return random_num.item() 152 | 153 | @SAMPLER.register_module() 154 | class InfiniteGroupEachSampleInBatchSampler(Sampler): 155 | """ 156 | Pardon this horrendous name. Basically, we want every sample to be from its own group. 157 | If batch size is 4 and # of GPUs is 8, each sample of these 32 should be operating on 158 | its own group. 159 | Shuffling is only done for group order, not done within groups. 160 | """ 161 | 162 | def __init__(self, 163 | dataset, 164 | samples_per_gpu=1, 165 | num_replicas=None, 166 | rank=None, 167 | seed=0): 168 | 169 | _rank, _num_replicas = get_dist_info() 170 | if num_replicas is None: 171 | num_replicas = _num_replicas 172 | if rank is None: 173 | rank = _rank 174 | 175 | self.dataset = dataset 176 | self.batch_size = samples_per_gpu 177 | self.num_replicas = num_replicas 178 | self.rank = rank 179 | self.seed = sync_random_seed(seed) 180 | 181 | self.size = len(self.dataset) 182 | 183 | assert hasattr(self.dataset, 'flag') 184 | self.flag = self.dataset.flag 185 | self.group_sizes = np.bincount(self.flag) 186 | self.groups_num = len(self.group_sizes) 187 | self.global_batch_size = samples_per_gpu * num_replicas 188 | assert self.groups_num >= self.global_batch_size 189 | 190 | # Now, for efficiency, make a dict group_idx: List[dataset sample_idxs] 191 | self.group_idx_to_sample_idxs = { 192 | group_idx: np.where(self.flag == group_idx)[0].tolist() 193 | for group_idx in range(self.groups_num)} 194 | 195 | # Get a generator per sample idx. Considering samples over all 196 | # GPUs, each sample position has its own generator 197 | self.group_indices_per_global_sample_idx = [ 198 | self._group_indices_per_global_sample_idx(self.rank * self.batch_size + local_sample_idx) 199 | for local_sample_idx in range(self.batch_size)] 200 | 201 | # Keep track of a buffer of dataset sample idxs for each local sample idx 202 | self.buffer_per_local_sample = [[] for _ in range(self.batch_size)] 203 | 204 | def _infinite_group_indices(self): 205 | g = torch.Generator() 206 | g.manual_seed(self.seed) 207 | while True: 208 | yield from torch.randperm(self.groups_num, generator=g).tolist() 209 | 210 | def _group_indices_per_global_sample_idx(self, global_sample_idx): 211 | yield from itertools.islice(self._infinite_group_indices(), 212 | global_sample_idx, 213 | None, 214 | self.global_batch_size) 215 | 216 | def __iter__(self): 217 | while True: 218 | curr_batch = [] 219 | for local_sample_idx in range(self.batch_size): 220 | if len(self.buffer_per_local_sample[local_sample_idx]) == 0: 221 | # Finished current group, refill with next group 222 | new_group_idx = next(self.group_indices_per_global_sample_idx[local_sample_idx]) 223 | self.buffer_per_local_sample[local_sample_idx] = \ 224 | copy.deepcopy( 225 | self.group_idx_to_sample_idxs[new_group_idx]) 226 | 227 | curr_batch.append(self.buffer_per_local_sample[local_sample_idx].pop(0)) 228 | 229 | yield curr_batch 230 | 231 | def __len__(self): 232 | """Length of base dataset.""" 233 | return self.size 234 | 235 | def set_epoch(self, epoch): 236 | self.epoch = epoch -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/datasets/samplers/sampler.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | # Modified by Shihao Wang 7 | # --------------------------------------------- 8 | from mmcv.utils.registry import Registry, build_from_cfg 9 | 10 | SAMPLER = Registry('sampler') 11 | 12 | 13 | def build_sampler(cfg, default_args): 14 | return build_from_cfg(cfg, SAMPLER, default_args) 15 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/backbones/__init__.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------ 2 | # Copyright (c) 2022 megvii-model. All Rights Reserved. 3 | # ------------------------------------------------------------------------ 4 | # Modified from DETR3D (https://github.com/WangYueFt/detr3d) 5 | # Copyright (c) 2021 Wang, Yue 6 | # ------------------------------------------------------------------------ 7 | from .vovnet import VoVNet 8 | from .vovnetcp import VoVNetCP 9 | 10 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/backbones/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/backbones/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/backbones/__pycache__/vovnet.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/backbones/__pycache__/vovnet.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/backbones/__pycache__/vovnetcp.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/backbones/__pycache__/vovnetcp.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__init__.py: -------------------------------------------------------------------------------- 1 | from .focal_head import FocalHead 2 | from .petr_head_dn import PETRHeadDN 3 | from .streampetr_head import StreamPETRHead 4 | from .sparse_head import SparseHead 5 | from .yolox_head import YOLOXHeadCustom 6 | from .perspective_head import PerspectiveHead 7 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/focal_head.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/focal_head.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/perspective_head.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/perspective_head.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/petr_head_dn.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/petr_head_dn.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/sparse_head.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/sparse_head.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/streampetr_head.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/streampetr_head.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/dense_heads/__pycache__/yolox_head.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/dense_heads/__pycache__/yolox_head.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/detectors/__init__.py: -------------------------------------------------------------------------------- 1 | from .petr3d import Petr3D 2 | from .repdetr3d import RepDetr3D 3 | from .persdetr3d import PersDetr3D 4 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/detectors/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/detectors/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/detectors/__pycache__/persdetr3d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/detectors/__pycache__/persdetr3d.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/detectors/__pycache__/petr3d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/detectors/__pycache__/petr3d.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/detectors/__pycache__/repdetr3d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/detectors/__pycache__/repdetr3d.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/necks/__init__.py: -------------------------------------------------------------------------------- 1 | from .cp_fpn import CPFPN 2 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/necks/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/necks/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/necks/__pycache__/cp_fpn.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/necks/__pycache__/cp_fpn.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/necks/cp_fpn.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------ 2 | # Copyright (c) 2021 megvii-model. All Rights Reserved. 3 | # ------------------------------------------------------------------------ 4 | # Modified from mmdetection (https://github.com/open-mmlab/mmdetection) 5 | # Copyright (c) OpenMMLab. All rights reserved. 6 | # ------------------------------------------------------------------------ 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | from mmcv.cnn import ConvModule 10 | from mmcv.runner import BaseModule, auto_fp16 11 | 12 | from mmdet.models import NECKS 13 | 14 | ####This FPN remove the unused parameters which can used with checkpoint (with_cp = True in Backbone) 15 | @NECKS.register_module() 16 | class CPFPN(BaseModule): 17 | r"""Feature Pyramid Network. 18 | 19 | This is an implementation of paper `Feature Pyramid Networks for Object 20 | Detection `_. 21 | 22 | Args: 23 | in_channels (List[int]): Number of input channels per scale. 24 | out_channels (int): Number of output channels (used at each scale) 25 | num_outs (int): Number of output scales. 26 | start_level (int): Index of the start input backbone level used to 27 | build the feature pyramid. Default: 0. 28 | end_level (int): Index of the end input backbone level (exclusive) to 29 | build the feature pyramid. Default: -1, which means the last level. 30 | add_extra_convs (bool | str): If bool, it decides whether to add conv 31 | layers on top of the original feature maps. Default to False. 32 | If True, it is equivalent to `add_extra_convs='on_input'`. 33 | If str, it specifies the source feature map of the extra convs. 34 | Only the following options are allowed 35 | 36 | - 'on_input': Last feat map of neck inputs (i.e. backbone feature). 37 | - 'on_lateral': Last feature map after lateral convs. 38 | - 'on_output': The last output feature map after fpn convs. 39 | relu_before_extra_convs (bool): Whether to apply relu before the extra 40 | conv. Default: False. 41 | no_norm_on_lateral (bool): Whether to apply norm on lateral. 42 | Default: False. 43 | conv_cfg (dict): Config dict for convolution layer. Default: None. 44 | norm_cfg (dict): Config dict for normalization layer. Default: None. 45 | act_cfg (str): Config dict for activation layer in ConvModule. 46 | Default: None. 47 | upsample_cfg (dict): Config dict for interpolate layer. 48 | Default: `dict(mode='nearest')` 49 | init_cfg (dict or list[dict], optional): Initialization config dict. 50 | 51 | Example: 52 | >>> import torch 53 | >>> in_channels = [2, 3, 5, 7] 54 | >>> scales = [340, 170, 84, 43] 55 | >>> inputs = [torch.rand(1, c, s, s) 56 | ... for c, s in zip(in_channels, scales)] 57 | >>> self = FPN(in_channels, 11, len(in_channels)).eval() 58 | >>> outputs = self.forward(inputs) 59 | >>> for i in range(len(outputs)): 60 | ... print(f'outputs[{i}].shape = {outputs[i].shape}') 61 | outputs[0].shape = torch.Size([1, 11, 340, 340]) 62 | outputs[1].shape = torch.Size([1, 11, 170, 170]) 63 | outputs[2].shape = torch.Size([1, 11, 84, 84]) 64 | outputs[3].shape = torch.Size([1, 11, 43, 43]) 65 | """ 66 | 67 | def __init__(self, 68 | in_channels, 69 | out_channels, 70 | num_outs, 71 | start_level=0, 72 | end_level=-1, 73 | add_extra_convs=False, 74 | relu_before_extra_convs=False, 75 | no_norm_on_lateral=False, 76 | conv_cfg=None, 77 | norm_cfg=None, 78 | act_cfg=None, 79 | upsample_cfg=dict(mode='nearest'), 80 | init_cfg=dict( 81 | type='Xavier', layer='Conv2d', distribution='uniform')): 82 | super(CPFPN, self).__init__(init_cfg) 83 | assert isinstance(in_channels, list) 84 | self.in_channels = in_channels 85 | self.out_channels = out_channels 86 | self.num_ins = len(in_channels) 87 | self.num_outs = num_outs 88 | self.relu_before_extra_convs = relu_before_extra_convs 89 | self.no_norm_on_lateral = no_norm_on_lateral 90 | self.fp16_enabled = False 91 | self.upsample_cfg = upsample_cfg.copy() 92 | 93 | if end_level == -1: 94 | self.backbone_end_level = self.num_ins 95 | assert num_outs >= self.num_ins - start_level 96 | else: 97 | # if end_level < inputs, no extra level is allowed 98 | self.backbone_end_level = end_level 99 | assert end_level <= len(in_channels) 100 | assert num_outs == end_level - start_level 101 | self.start_level = start_level 102 | self.end_level = end_level 103 | self.add_extra_convs = add_extra_convs 104 | assert isinstance(add_extra_convs, (str, bool)) 105 | if isinstance(add_extra_convs, str): 106 | # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' 107 | assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') 108 | elif add_extra_convs: # True 109 | self.add_extra_convs = 'on_input' 110 | 111 | self.lateral_convs = nn.ModuleList() 112 | self.fpn_convs = nn.ModuleList() 113 | 114 | for i in range(self.start_level, self.backbone_end_level): 115 | l_conv = ConvModule( 116 | in_channels[i], 117 | out_channels, 118 | 1, 119 | conv_cfg=conv_cfg, 120 | norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, 121 | act_cfg=act_cfg, 122 | inplace=False) 123 | self.lateral_convs.append(l_conv) 124 | if i == 0 : 125 | fpn_conv = ConvModule( 126 | out_channels, 127 | out_channels, 128 | 3, 129 | padding=1, 130 | conv_cfg=conv_cfg, 131 | norm_cfg=norm_cfg, 132 | act_cfg=act_cfg, 133 | inplace=False) 134 | self.fpn_convs.append(fpn_conv) 135 | 136 | # add extra conv layers (e.g., RetinaNet) 137 | extra_levels = num_outs - self.backbone_end_level + self.start_level 138 | if self.add_extra_convs and extra_levels >= 1: 139 | for i in range(extra_levels): 140 | if i == 0 and self.add_extra_convs == 'on_input': 141 | in_channels = self.in_channels[self.backbone_end_level - 1] 142 | else: 143 | in_channels = out_channels 144 | extra_fpn_conv = ConvModule( 145 | in_channels, 146 | out_channels, 147 | 3, 148 | stride=2, 149 | padding=1, 150 | conv_cfg=conv_cfg, 151 | norm_cfg=norm_cfg, 152 | act_cfg=act_cfg, 153 | inplace=False) 154 | self.fpn_convs.append(extra_fpn_conv) 155 | 156 | @auto_fp16(out_fp32=True) 157 | def forward(self, inputs): 158 | """Forward function.""" 159 | assert len(inputs) == len(self.in_channels) 160 | 161 | # build laterals 162 | laterals = [ 163 | lateral_conv(inputs[i + self.start_level]) 164 | for i, lateral_conv in enumerate(self.lateral_convs) 165 | ] 166 | 167 | # build top-down path 168 | used_backbone_levels = len(laterals) 169 | for i in range(used_backbone_levels - 1, 0, -1): 170 | # In some cases, fixing `scale factor` (e.g. 2) is preferred, but 171 | # it cannot co-exist with `size` in `F.interpolate`. 172 | if 'scale_factor' in self.upsample_cfg: 173 | laterals[i - 1] += F.interpolate(laterals[i], 174 | **self.upsample_cfg) 175 | else: 176 | prev_shape = laterals[i - 1].shape[2:] 177 | laterals[i - 1] += F.interpolate( 178 | laterals[i], size=prev_shape, **self.upsample_cfg) 179 | 180 | # build outputs 181 | # part 1: from original levels 182 | outs = [ 183 | self.fpn_convs[i](laterals[i]) if i==0 else laterals[i] for i in range(used_backbone_levels) 184 | ] 185 | # part 2: add extra levels 186 | if self.num_outs > len(outs): 187 | # use max pool to get more levels on top of outputs 188 | # (e.g., Faster R-CNN, Mask R-CNN) 189 | if not self.add_extra_convs: 190 | for i in range(self.num_outs - used_backbone_levels): 191 | outs.append(F.max_pool2d(outs[-1], 1, stride=2)) 192 | # add conv layers on top of original feature maps (RetinaNet) 193 | else: 194 | if self.add_extra_convs == 'on_input': 195 | extra_source = inputs[self.backbone_end_level - 1] 196 | elif self.add_extra_convs == 'on_lateral': 197 | extra_source = laterals[-1] 198 | elif self.add_extra_convs == 'on_output': 199 | extra_source = outs[-1] 200 | else: 201 | raise NotImplementedError 202 | outs.append(self.fpn_convs[used_backbone_levels](extra_source)) 203 | for i in range(used_backbone_levels + 1, self.num_outs): 204 | if self.relu_before_extra_convs: 205 | outs.append(self.fpn_convs[i](F.relu(outs[-1]))) 206 | else: 207 | outs.append(self.fpn_convs[i](outs[-1])) 208 | return tuple(outs) 209 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .petr_transformer import PETRMultiheadAttention, PETRTransformerEncoder, PETRTemporalTransformer, PETRTemporalDecoderLayer, PETRMultiheadFlashAttention 2 | from .detr3d_transformer import DeformableFeatureAggregationCuda, Detr3DTransformer, Detr3DTransformerDecoder, Detr3DTemporalDecoderLayer 3 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/attention.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/attention.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/checkviews.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/checkviews.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/detr3d_transformer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/detr3d_transformer.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/grid_mask.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/grid_mask.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/misc.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/misc.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/multi_scale_deformable_attn_function.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/multi_scale_deformable_attn_function.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/petr_transformer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/petr_transformer.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/positional_encoding.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/positional_encoding.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/prompt.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/prompt.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/__pycache__/proposals_generation.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/projects/mmdet3d_plugin/models/utils/__pycache__/proposals_generation.cpython-38.pyc -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/checkviews.py: -------------------------------------------------------------------------------- 1 | from multiprocessing.spawn import import_main_path 2 | from random import sample 3 | import torch 4 | import numpy as np 5 | import cv2 6 | from torchvision import transforms 7 | 8 | normalize = transforms.Compose([ 9 | transforms.ToTensor(), 10 | transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) 11 | ]) 12 | 13 | 14 | def box_cxcywh_to_xyxy(x): 15 | x_c, y_c, w, h = x.unbind(-1) 16 | b = [(x_c - 0.5 * w), (y_c - 0.5 * h), 17 | (x_c + 0.5 * w), (y_c + 0.5 * h)] 18 | return torch.stack(b, dim=-1) 19 | 20 | 21 | def tensor_to_image(samples): 22 | std = [0.229, 0.224, 0.225] 23 | mean = [0.485, 0.456, 0.406] 24 | samp = samples.clone().cpu().squeeze(0) 25 | 26 | for i in range(len(mean)): 27 | samp[i] = samp[i].mul(std[i])+mean[i] 28 | 29 | img = transforms.ToPILImage()(samp) 30 | img = np.array(img) 31 | return img 32 | 33 | 34 | def check(samples, bboxs, idx): 35 | 36 | # b, c, h, w = samples.shape 37 | save_path_1 = '/opt/data/private/jihao/Project/dab-streampetr-baseline/tools/cc_' + \ 38 | str(idx)+'.jpg' 39 | 40 | img = tensor_to_image(samples) 41 | # samp = samples.clone().cpu().squeeze(0) 42 | # img = transforms.ToPILImage()(samp) 43 | # img = np.array(img) 44 | 45 | for bx in bboxs: 46 | # bx = box_cxcywh_to_xyxy(bx).tolist() 47 | x_1, y_1, x_2, y_2 = int(bx[0]), int(bx[1]), int(bx[2]), int(bx[3]) 48 | draw_2 = cv2.rectangle(img, (x_1, y_1), 49 | (x_2, y_2), (0, 255, 0), 1) 50 | cv2.imwrite(save_path_1, draw_2) 51 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/grid_mask.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from PIL import Image 5 | 6 | class Grid(object): 7 | def __init__(self, use_h, use_w, rotate = 1, offset=False, ratio = 0.5, mode=0, prob = 1.): 8 | self.use_h = use_h 9 | self.use_w = use_w 10 | self.rotate = rotate 11 | self.offset = offset 12 | self.ratio = ratio 13 | self.mode=mode 14 | self.st_prob = prob 15 | self.prob = prob 16 | 17 | def set_prob(self, epoch, max_epoch): 18 | self.prob = self.st_prob * epoch / max_epoch 19 | 20 | def __call__(self, img, label): 21 | if np.random.rand() > self.prob: 22 | return img, label 23 | h = img.size(1) 24 | w = img.size(2) 25 | self.d1 = 2 26 | self.d2 = min(h, w) 27 | hh = int(1.5*h) 28 | ww = int(1.5*w) 29 | d = np.random.randint(self.d1, self.d2) 30 | if self.ratio == 1: 31 | self.l = np.random.randint(1, d) 32 | else: 33 | self.l = min(max(int(d*self.ratio+0.5),1),d-1) 34 | mask = np.ones((hh, ww), np.float32) 35 | st_h = np.random.randint(d) 36 | st_w = np.random.randint(d) 37 | if self.use_h: 38 | for i in range(hh//d): 39 | s = d*i + st_h 40 | t = min(s+self.l, hh) 41 | mask[s:t,:] *= 0 42 | if self.use_w: 43 | for i in range(ww//d): 44 | s = d*i + st_w 45 | t = min(s+self.l, ww) 46 | mask[:,s:t] *= 0 47 | 48 | r = np.random.randint(self.rotate) 49 | mask = Image.fromarray(np.uint8(mask)) 50 | mask = mask.rotate(r) 51 | mask = np.asarray(mask) 52 | mask = mask[(hh-h)//2:(hh-h)//2+h, (ww-w)//2:(ww-w)//2+w] 53 | 54 | mask = torch.from_numpy(mask).float() 55 | if self.mode == 1: 56 | mask = 1-mask 57 | 58 | mask = mask.expand_as(img) 59 | if self.offset: 60 | offset = torch.from_numpy(2 * (np.random.rand(h,w) - 0.5)).float() 61 | offset = (1 - mask) * offset 62 | img = img * mask + offset 63 | else: 64 | img = img * mask 65 | 66 | return img, label 67 | 68 | 69 | class GridMask(nn.Module): 70 | def __init__(self, use_h, use_w, rotate = 1, offset=False, ratio = 0.5, mode=0, prob = 1.): 71 | super(GridMask, self).__init__() 72 | self.use_h = use_h 73 | self.use_w = use_w 74 | self.rotate = rotate 75 | self.offset = offset 76 | self.ratio = ratio 77 | self.mode = mode 78 | self.st_prob = prob 79 | self.prob = prob 80 | 81 | def set_prob(self, epoch, max_epoch): 82 | self.prob = self.st_prob * epoch / max_epoch #+ 1.#0.5 83 | 84 | def forward(self, x): 85 | if np.random.rand() > self.prob or not self.training: 86 | return x 87 | n,c,h,w = x.size() 88 | x = x.view(-1,h,w) 89 | hh = int(1.5*h) 90 | ww = int(1.5*w) 91 | d = np.random.randint(2, h) 92 | self.l = min(max(int(d*self.ratio+0.5),1),d-1) 93 | mask = np.ones((hh, ww), np.float32) 94 | st_h = np.random.randint(d) 95 | st_w = np.random.randint(d) 96 | if self.use_h: 97 | for i in range(hh//d): 98 | s = d*i + st_h 99 | t = min(s+self.l, hh) 100 | mask[s:t,:] *= 0 101 | if self.use_w: 102 | for i in range(ww//d): 103 | s = d*i + st_w 104 | t = min(s+self.l, ww) 105 | mask[:,s:t] *= 0 106 | 107 | r = np.random.randint(self.rotate) 108 | mask = Image.fromarray(np.uint8(mask)) 109 | mask = mask.rotate(r) 110 | mask = np.asarray(mask) 111 | mask = mask[(hh-h)//2:(hh-h)//2+h, (ww-w)//2:(ww-w)//2+w] 112 | 113 | mask = torch.from_numpy(mask).float().cuda() 114 | if self.mode == 1: 115 | mask = 1-mask 116 | mask = mask.expand_as(x) 117 | if self.offset: 118 | offset = torch.from_numpy(2 * (np.random.rand(h,w) - 0.5)).float().cuda() 119 | x = x * mask + offset * (1 - mask) 120 | else: 121 | x = x * mask 122 | 123 | return x.view(n,c,h,w) -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/misc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from mmdet.core import bbox_xyxy_to_cxcywh 5 | from mmdet.models.utils.transformer import inverse_sigmoid 6 | 7 | 8 | def memory_refresh(memory, prev_exist): 9 | memory_shape = memory.shape 10 | view_shape = [1 for _ in range(len(memory_shape))] 11 | prev_exist = prev_exist.view(-1, *view_shape[1:]) 12 | return memory * prev_exist 13 | 14 | 15 | def topk_gather(feat, topk_indexes): 16 | if topk_indexes is not None: 17 | feat_shape = feat.shape 18 | topk_shape = topk_indexes.shape 19 | 20 | view_shape = [1 for _ in range(len(feat_shape))] 21 | view_shape[:2] = topk_shape[:2] 22 | topk_indexes = topk_indexes.view(*view_shape) 23 | 24 | feat = torch.gather( 25 | feat, 1, topk_indexes.repeat(1, 1, *feat_shape[2:])) 26 | return feat 27 | 28 | 29 | def apply_ltrb(locations, pred_ltrb): 30 | """ 31 | :param locations: (1, H, W, 2) 32 | :param pred_ltrb: (N, H, W, 4) 33 | """ 34 | pred_boxes = torch.zeros_like(pred_ltrb) 35 | pred_boxes[..., 0] = (locations[..., 0] - pred_ltrb[..., 0]) # x1 36 | pred_boxes[..., 1] = (locations[..., 1] - pred_ltrb[..., 1]) # y1 37 | pred_boxes[..., 2] = (locations[..., 0] + pred_ltrb[..., 2]) # x2 38 | pred_boxes[..., 3] = (locations[..., 1] + pred_ltrb[..., 3]) # y2 39 | min_xy = pred_boxes[..., 0].new_tensor(0) 40 | max_xy = pred_boxes[..., 0].new_tensor(1) 41 | pred_boxes = torch.where(pred_boxes < min_xy, min_xy, pred_boxes) 42 | pred_boxes = torch.where(pred_boxes > max_xy, max_xy, pred_boxes) 43 | pred_boxes = bbox_xyxy_to_cxcywh(pred_boxes) 44 | 45 | return pred_boxes 46 | 47 | 48 | def apply_center_offset(locations, center_offset): 49 | """ 50 | :param locations: (1, H, W, 2) 51 | :param pred_ltrb: (N, H, W, 4) 52 | """ 53 | centers_2d = torch.zeros_like(center_offset) 54 | locations = inverse_sigmoid(locations) 55 | centers_2d[..., 0] = locations[..., 0] + center_offset[..., 0] # x1 56 | centers_2d[..., 1] = locations[..., 1] + center_offset[..., 1] # y1 57 | centers_2d = centers_2d.sigmoid() 58 | 59 | return centers_2d 60 | 61 | 62 | @torch.no_grad() 63 | def locations(features, stride, pad_h, pad_w): 64 | """ 65 | Arguments: 66 | features: (N, C, H, W) 67 | Return: 68 | locations: (H, W, 2) 69 | """ 70 | 71 | h, w = features.size()[-2:] 72 | device = features.device 73 | 74 | shifts_x = (torch.arange( 75 | 0, stride*w, step=stride, 76 | dtype=torch.float32, device=device 77 | ) + stride // 2) / pad_w 78 | shifts_y = (torch.arange( 79 | 0, h * stride, step=stride, 80 | dtype=torch.float32, device=device 81 | ) + stride // 2) / pad_h 82 | shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x) 83 | shift_x = shift_x.reshape(-1) 84 | shift_y = shift_y.reshape(-1) 85 | locations = torch.stack((shift_x, shift_y), dim=1) 86 | 87 | locations = locations.reshape(h, w, 2) 88 | 89 | return locations 90 | 91 | 92 | def gaussian_2d(shape, sigma=1.0): 93 | """Generate gaussian map. 94 | 95 | Args: 96 | shape (list[int]): Shape of the map. 97 | sigma (float, optional): Sigma to generate gaussian map. 98 | Defaults to 1. 99 | 100 | Returns: 101 | np.ndarray: Generated gaussian map. 102 | """ 103 | m, n = [(ss - 1.) / 2. for ss in shape] 104 | y, x = np.ogrid[-m:m + 1, -n:n + 1] 105 | 106 | h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) 107 | h[h < np.finfo(h.dtype).eps * h.max()] = 0 108 | return h 109 | 110 | 111 | def draw_heatmap_gaussian(heatmap, center, radius, k=1): 112 | """Get gaussian masked heatmap. 113 | 114 | Args: 115 | heatmap (torch.Tensor): Heatmap to be masked. 116 | center (torch.Tensor): Center coord of the heatmap. 117 | radius (int): Radius of gaussian. 118 | K (int, optional): Multiple of masked_gaussian. Defaults to 1. 119 | 120 | Returns: 121 | torch.Tensor: Masked heatmap. 122 | """ 123 | diameter = 2 * radius + 1 124 | gaussian = gaussian_2d((diameter, diameter), sigma=diameter / 6) 125 | 126 | x, y = int(center[0]), int(center[1]) 127 | 128 | height, width = heatmap.shape[0:2] 129 | 130 | left, right = min(x, radius), min(width - x, radius + 1) 131 | top, bottom = min(y, radius), min(height - y, radius + 1) 132 | 133 | masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] 134 | masked_gaussian = torch.from_numpy( 135 | gaussian[radius - top:radius + bottom, 136 | radius - left:radius + right]).to(heatmap.device, 137 | torch.float32) 138 | if min(masked_gaussian.shape) > 0 and min(masked_heatmap.shape) > 0: 139 | torch.max(masked_heatmap, masked_gaussian * k, out=masked_heatmap) 140 | return heatmap 141 | 142 | 143 | class SELayer_Linear(nn.Module): 144 | def __init__(self, channels, act_layer=nn.ReLU, gate_layer=nn.Sigmoid): 145 | super().__init__() 146 | self.conv_reduce = nn.Linear(channels, channels) 147 | self.act1 = act_layer() 148 | self.conv_expand = nn.Linear(channels, channels) 149 | self.gate = gate_layer() 150 | 151 | def forward(self, x, x_se): 152 | x_se = self.conv_reduce(x_se) 153 | x_se = self.act1(x_se) 154 | x_se = self.conv_expand(x_se) 155 | return x * self.gate(x_se) 156 | 157 | 158 | class MLN(nn.Module): 159 | ''' 160 | Args: 161 | c_dim (int): dimension of latent code c 162 | f_dim (int): feature dimension 163 | ''' 164 | 165 | def __init__(self, c_dim, f_dim=256): 166 | super().__init__() 167 | self.c_dim = c_dim 168 | self.f_dim = f_dim 169 | 170 | self.reduce = nn.Sequential( 171 | nn.Linear(c_dim, f_dim), 172 | nn.ReLU(), 173 | ) 174 | self.gamma = nn.Linear(f_dim, f_dim) 175 | self.beta = nn.Linear(f_dim, f_dim) 176 | self.ln = nn.LayerNorm(f_dim, elementwise_affine=False) 177 | self.reset_parameters() 178 | 179 | def reset_parameters(self): 180 | nn.init.zeros_(self.gamma.weight) 181 | nn.init.zeros_(self.beta.weight) 182 | nn.init.ones_(self.gamma.bias) 183 | nn.init.zeros_(self.beta.bias) 184 | 185 | def forward(self, x, c): 186 | x = self.ln(x) 187 | c = self.reduce(c) 188 | gamma = self.gamma(c) 189 | beta = self.beta(c) 190 | out = gamma * x + beta 191 | 192 | return out 193 | 194 | 195 | def transform_reference_points(reference_points, egopose, reverse=False, translation=True): 196 | reference_points = torch.cat( 197 | [reference_points, torch.ones_like(reference_points[..., 0:1])], dim=-1) 198 | if reverse: 199 | matrix = egopose.inverse() 200 | else: 201 | matrix = egopose 202 | if not translation: 203 | matrix[..., :3, 3] = 0.0 204 | reference_points = (matrix.unsqueeze( 205 | 1) @ reference_points.unsqueeze(-1)).squeeze(-1)[..., :3] 206 | return reference_points 207 | 208 | 209 | def normlize_boxes(bboxes, cam2lidar=None): 210 | if cam2lidar is None: 211 | centers_embed = bboxes[..., :3] 212 | sizes_embed = bboxes[..., 3:6].log() 213 | thetas_ = bboxes[..., -1:] 214 | theta_embed = torch.cat((thetas_.sin(), thetas_.cos()), dim=-1) 215 | bboxs_3ds = torch.cat( 216 | (centers_embed, sizes_embed, theta_embed), dim=-1) 217 | else: 218 | centers_embed = bboxes[..., :3] 219 | centers_masks = torch.ones_like(bboxes[..., :1]) 220 | centers_embed = torch.cat( 221 | (centers_embed, centers_masks), dim=-1).float() 222 | centers_embed = centers_embed @ torch.from_numpy( 223 | cam2lidar).t().to(centers_embed.device)[..., :3] 224 | 225 | sizes_embed = bboxes[..., 3:6] 226 | thetas_ = bboxes[..., -1:] 227 | thetas_ = (-1 * thetas_ - np.pi / 2) 228 | ws_, hs_, ls_ = sizes_embed[..., 229 | 0:1], sizes_embed[..., 1:2], sizes_embed[..., 2:3] 230 | 231 | ws_ = ws_.view(-1, 1) 232 | hs_ = hs_.view(-1, 1) 233 | ls_ = ls_.view(-1, 1) 234 | 235 | assert ws_.shape == hs_.shape and ls_.shape == hs_.shape 236 | theta_embed = torch.cat((thetas_.sin(), thetas_.cos()), dim=-1) 237 | sizes_embed = torch.cat((ws_, ls_, hs_), dim=-1).log() 238 | bboxs_3ds = torch.cat( 239 | (centers_embed, sizes_embed, theta_embed), dim=-1) 240 | 241 | return bboxs_3ds 242 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/multi_scale_deformable_attn_function.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | 7 | import torch 8 | from torch.cuda.amp import custom_bwd, custom_fwd 9 | from torch.autograd.function import Function, once_differentiable 10 | from mmcv.utils import ext_loader 11 | ext_module = ext_loader.load_ext( 12 | '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) 13 | 14 | 15 | class MultiScaleDeformableAttnFunction_fp16(Function): 16 | 17 | @staticmethod 18 | @custom_fwd(cast_inputs=torch.float16) 19 | def forward(ctx, value, value_spatial_shapes, value_level_start_index, 20 | sampling_locations, attention_weights, im2col_step): 21 | """GPU version of multi-scale deformable attention. 22 | 23 | Args: 24 | value (Tensor): The value has shape 25 | (bs, num_keys, mum_heads, embed_dims//num_heads) 26 | value_spatial_shapes (Tensor): Spatial shape of 27 | each feature map, has shape (num_levels, 2), 28 | last dimension 2 represent (h, w) 29 | sampling_locations (Tensor): The location of sampling points, 30 | has shape 31 | (bs ,num_queries, num_heads, num_levels, num_points, 2), 32 | the last dimension 2 represent (x, y). 33 | attention_weights (Tensor): The weight of sampling points used 34 | when calculate the attention, has shape 35 | (bs ,num_queries, num_heads, num_levels, num_points), 36 | im2col_step (Tensor): The step used in image to column. 37 | 38 | Returns: 39 | Tensor: has shape (bs, num_queries, embed_dims) 40 | """ 41 | ctx.im2col_step = im2col_step 42 | output = ext_module.ms_deform_attn_forward( 43 | value, 44 | value_spatial_shapes, 45 | value_level_start_index, 46 | sampling_locations, 47 | attention_weights, 48 | im2col_step=ctx.im2col_step) 49 | ctx.save_for_backward(value, value_spatial_shapes, 50 | value_level_start_index, sampling_locations, 51 | attention_weights) 52 | return output 53 | 54 | @staticmethod 55 | @once_differentiable 56 | @custom_bwd 57 | def backward(ctx, grad_output): 58 | """GPU version of backward function. 59 | 60 | Args: 61 | grad_output (Tensor): Gradient 62 | of output tensor of forward. 63 | 64 | Returns: 65 | Tuple[Tensor]: Gradient 66 | of input tensors in forward. 67 | """ 68 | value, value_spatial_shapes, value_level_start_index, \ 69 | sampling_locations, attention_weights = ctx.saved_tensors 70 | grad_value = torch.zeros_like(value) 71 | grad_sampling_loc = torch.zeros_like(sampling_locations) 72 | grad_attn_weight = torch.zeros_like(attention_weights) 73 | 74 | ext_module.ms_deform_attn_backward( 75 | value, 76 | value_spatial_shapes, 77 | value_level_start_index, 78 | sampling_locations, 79 | attention_weights, 80 | grad_output.contiguous(), 81 | grad_value, 82 | grad_sampling_loc, 83 | grad_attn_weight, 84 | im2col_step=ctx.im2col_step) 85 | 86 | return grad_value, None, None, \ 87 | grad_sampling_loc, grad_attn_weight, None 88 | 89 | 90 | class MultiScaleDeformableAttnFunction_fp32(Function): 91 | 92 | @staticmethod 93 | @custom_fwd(cast_inputs=torch.float32) 94 | def forward(ctx, value, value_spatial_shapes, value_level_start_index, 95 | sampling_locations, attention_weights, im2col_step): 96 | """GPU version of multi-scale deformable attention. 97 | 98 | Args: 99 | value (Tensor): The value has shape 100 | (bs, num_keys, mum_heads, embed_dims//num_heads) 101 | value_spatial_shapes (Tensor): Spatial shape of 102 | each feature map, has shape (num_levels, 2), 103 | last dimension 2 represent (h, w) 104 | sampling_locations (Tensor): The location of sampling points, 105 | has shape 106 | (bs ,num_queries, num_heads, num_levels, num_points, 2), 107 | the last dimension 2 represent (x, y). 108 | attention_weights (Tensor): The weight of sampling points used 109 | when calculate the attention, has shape 110 | (bs ,num_queries, num_heads, num_levels, num_points), 111 | im2col_step (Tensor): The step used in image to column. 112 | 113 | Returns: 114 | Tensor: has shape (bs, num_queries, embed_dims) 115 | """ 116 | 117 | ctx.im2col_step = im2col_step 118 | output = ext_module.ms_deform_attn_forward( 119 | value, 120 | value_spatial_shapes, 121 | value_level_start_index, 122 | sampling_locations, 123 | attention_weights, 124 | im2col_step=ctx.im2col_step) 125 | ctx.save_for_backward(value, value_spatial_shapes, 126 | value_level_start_index, sampling_locations, 127 | attention_weights) 128 | return output 129 | 130 | @staticmethod 131 | @once_differentiable 132 | @custom_bwd 133 | def backward(ctx, grad_output): 134 | """GPU version of backward function. 135 | 136 | Args: 137 | grad_output (Tensor): Gradient 138 | of output tensor of forward. 139 | 140 | Returns: 141 | Tuple[Tensor]: Gradient 142 | of input tensors in forward. 143 | """ 144 | value, value_spatial_shapes, value_level_start_index, \ 145 | sampling_locations, attention_weights = ctx.saved_tensors 146 | grad_value = torch.zeros_like(value) 147 | grad_sampling_loc = torch.zeros_like(sampling_locations) 148 | grad_attn_weight = torch.zeros_like(attention_weights) 149 | 150 | ext_module.ms_deform_attn_backward( 151 | value, 152 | value_spatial_shapes, 153 | value_level_start_index, 154 | sampling_locations, 155 | attention_weights, 156 | grad_output.contiguous(), 157 | grad_value, 158 | grad_sampling_loc, 159 | grad_attn_weight, 160 | im2col_step=ctx.im2col_step) 161 | 162 | return grad_value, None, None, \ 163 | grad_sampling_loc, grad_attn_weight, None 164 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/positional_encoding.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------ 2 | # Copyright (c) 2022 megvii-model. All Rights Reserved. 3 | # ------------------------------------------------------------------------ 4 | # Modified from mmdetection (https://github.com/open-mmlab/mmdetection) 5 | # Copyright (c) OpenMMLab. All rights reserved. 6 | # ------------------------------------------------------------------------ 7 | # Modified by Shihao Wang 8 | # ------------------------------------------------------------------------ 9 | import math 10 | import torch 11 | import torch.nn as nn 12 | import numpy as np 13 | 14 | 15 | def pos2posemb3d(pos, num_pos_feats=128, temperature=10000): 16 | scale = 2 * math.pi 17 | pos = pos * scale 18 | dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos.device) 19 | dim_t = temperature ** (2 * torch.div(dim_t, 2, 20 | rounding_mode='floor') / num_pos_feats) 21 | pos_x = pos[..., 0, None] / dim_t 22 | pos_y = pos[..., 1, None] / dim_t 23 | pos_z = pos[..., 2, None] / dim_t 24 | pos_x = torch.stack( 25 | (pos_x[..., 0::2].sin(), pos_x[..., 1::2].cos()), dim=-1).flatten(-2) 26 | pos_y = torch.stack( 27 | (pos_y[..., 0::2].sin(), pos_y[..., 1::2].cos()), dim=-1).flatten(-2) 28 | pos_z = torch.stack( 29 | (pos_z[..., 0::2].sin(), pos_z[..., 1::2].cos()), dim=-1).flatten(-2) 30 | posemb = torch.cat((pos_y, pos_x, pos_z), dim=-1) 31 | return posemb 32 | 33 | 34 | def pos2posemb1d(pos, num_pos_feats=256, temperature=10000): 35 | scale = 2 * math.pi 36 | pos = pos * scale 37 | dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos.device) 38 | dim_t = temperature ** (2 * torch.div(dim_t, 2, 39 | rounding_mode='floor') / num_pos_feats) 40 | pos_x = pos[..., 0, None] / dim_t 41 | 42 | pos_x = torch.stack( 43 | (pos_x[..., 0::2].sin(), pos_x[..., 1::2].cos()), dim=-1).flatten(-2) 44 | 45 | return pos_x 46 | 47 | 48 | def nerf_positional_encoding( 49 | tensor, num_encoding_functions=6, include_input=False, log_sampling=True 50 | ) -> torch.Tensor: 51 | r"""Apply positional encoding to the input. 52 | Args: 53 | tensor (torch.Tensor): Input tensor to be positionally encoded. 54 | encoding_size (optional, int): Number of encoding functions used to compute 55 | a positional encoding (default: 6). 56 | include_input (optional, bool): Whether or not to include the input in the 57 | positional encoding (default: True). 58 | Returns: 59 | (torch.Tensor): Positional encoding of the input tensor. 60 | """ 61 | # TESTED 62 | # Trivially, the input tensor is added to the positional encoding. 63 | encoding = [tensor] if include_input else [] 64 | frequency_bands = None 65 | if log_sampling: 66 | frequency_bands = 2.0 ** torch.linspace( 67 | 0.0, 68 | num_encoding_functions - 1, 69 | num_encoding_functions, 70 | dtype=tensor.dtype, 71 | device=tensor.device, 72 | ) 73 | else: 74 | frequency_bands = torch.linspace( 75 | 2.0 ** 0.0, 76 | 2.0 ** (num_encoding_functions - 1), 77 | num_encoding_functions, 78 | dtype=tensor.dtype, 79 | device=tensor.device, 80 | ) 81 | 82 | for freq in frequency_bands: 83 | for func in [torch.sin, torch.cos]: 84 | encoding.append(func(tensor * freq)) 85 | 86 | # Special case, for no positional encoding 87 | if len(encoding) == 1: 88 | return encoding[0] 89 | else: 90 | return torch.cat(encoding, dim=-1) 91 | 92 | 93 | def gen_sineembed_for_position(pos_tensor): 94 | # n_query, bs, _ = pos_tensor.size() 95 | # sineembed_tensor = torch.zeros(n_query, bs, 256) 96 | scale = 2 * math.pi 97 | dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device) 98 | dim_t = 10000 ** (2 * (dim_t // 2) / 128) 99 | 100 | # for x,y 101 | x_embed = pos_tensor[..., 0] * scale 102 | y_embed = pos_tensor[..., 1] * scale 103 | pos_x = x_embed[..., None] / dim_t 104 | pos_y = y_embed[..., None] / dim_t 105 | pos_x = torch.stack( 106 | (pos_x[..., 0::2].sin(), pos_x[..., 1::2].cos()), dim=3).flatten(-2) 107 | pos_y = torch.stack( 108 | (pos_y[..., 0::2].sin(), pos_y[..., 1::2].cos()), dim=3).flatten(-2) 109 | # for z 110 | z_embed = pos_tensor[..., 2] * scale 111 | pos_z = z_embed[..., None] / dim_t 112 | pos_z = torch.stack( 113 | (pos_z[..., 0::2].sin(), pos_z[..., 1::2].cos()), dim=3).flatten(-2) 114 | 115 | if pos_tensor.size(-1) == 2: 116 | pos = torch.cat((pos_y, pos_x), dim=2) 117 | elif pos_tensor.size(-1) == 4: 118 | w_embed = pos_tensor[:, :, 2] * scale 119 | pos_w = w_embed[:, :, None] / dim_t 120 | pos_w = torch.stack( 121 | (pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2) 122 | 123 | h_embed = pos_tensor[:, :, 3] * scale 124 | pos_h = h_embed[:, :, None] / dim_t 125 | pos_h = torch.stack( 126 | (pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2) 127 | 128 | pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) 129 | elif pos_tensor.size(-1) == 8: 130 | # for w 131 | w_embed = pos_tensor[..., 3] * scale 132 | pos_w = w_embed[..., None] / dim_t 133 | pos_w = torch.stack( 134 | (pos_w[..., 0::2].sin(), pos_w[..., 1::2].cos()), dim=3).flatten(-2) 135 | # for h 136 | h_embed = pos_tensor[..., 4] * scale 137 | pos_h = h_embed[..., None] / dim_t 138 | pos_h = torch.stack( 139 | (pos_h[..., 0::2].sin(), pos_h[..., 1::2].cos()), dim=3).flatten(-2) 140 | # for l 141 | l_embed = pos_tensor[..., 5] * scale 142 | pos_l = l_embed[..., None] / dim_t 143 | pos_l = torch.stack( 144 | (pos_l[..., 0::2].sin(), pos_l[..., 1::2].cos()), dim=3).flatten(-2) 145 | 146 | # for theta 147 | s_embed = pos_tensor[..., 6] * scale 148 | pos_s = s_embed[..., None] / dim_t 149 | pos_s = torch.stack( 150 | (pos_s[..., 0::2].sin(), pos_s[..., 1::2].cos()), dim=3).flatten(-2) 151 | c_embed = pos_tensor[..., 7] * scale 152 | pos_c = c_embed[..., None] / dim_t 153 | pos_c = torch.stack( 154 | (pos_c[..., 0::2].sin(), pos_c[..., 1::2].cos()), dim=3).flatten(-2) 155 | 156 | pos = torch.cat((pos_y, pos_x, pos_z, pos_w, pos_h, 157 | pos_l, pos_s, pos_c), dim=-1) 158 | else: 159 | raise ValueError( 160 | "Unknown pos_tensor shape(-1):{}".format(pos_tensor.size(-1))) 161 | return pos 162 | -------------------------------------------------------------------------------- /projects/mmdet3d_plugin/models/utils/prompt.py: -------------------------------------------------------------------------------- 1 | from turtle import forward 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | class PadPrompter(nn.Module): 7 | def __init__(self, c, w, h, eta=0.3): 8 | super(PadPrompter, self).__init__() 9 | pad_w = int(eta * w) + 1 10 | pad_h = int(eta * h) + 1 11 | feat_h = h 12 | feat_w = w 13 | 14 | self.base_h = feat_h - 2*pad_h 15 | self.base_w = feat_w - 2*pad_w 16 | self.base_c = c 17 | 18 | self.pad_up = nn.Parameter(torch.randn( 19 | [1, self.base_c, pad_h, feat_w])) 20 | self.pad_down = nn.Parameter(torch.randn( 21 | [1, self.base_c, pad_h, feat_w])) 22 | self.pad_left = nn.Parameter(torch.randn( 23 | [1, self.base_c, feat_h - 2*pad_h, pad_w])) 24 | self.pad_right = nn.Parameter(torch.randn( 25 | [1, self.base_c, feat_h - 2*pad_h, pad_w])) 26 | 27 | def forward(self, x): 28 | bs, t, n, c, hf, wf = x.size() 29 | x = x.reshape(bs*t*n, c, hf, wf) 30 | base = torch.zeros(1, c, self.base_h, self.base_w).to(x.device) 31 | prompt = torch.cat([self.pad_left, base, self.pad_right], dim=3) 32 | prompt = torch.cat([self.pad_up, prompt, self.pad_down], dim=2) 33 | prompt = torch.cat(x.size(0) * [prompt]) 34 | 35 | return (x + prompt).reshape(bs, t, n, c, hf, wf) 36 | -------------------------------------------------------------------------------- /tools/.ipynb_checkpoints/dist_test-checkpoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CONFIG=$1 4 | CHECKPOINT=$2 5 | GPUS=$3 6 | NNODES=${NNODES:-1} 7 | NODE_RANK=${NODE_RANK:-0} 8 | PORT=${PORT:-29500} 9 | MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} 10 | 11 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 12 | python -m torch.distributed.launch \ 13 | --nnodes=$NNODES \ 14 | --node_rank=$NODE_RANK \ 15 | --master_addr=$MASTER_ADDR \ 16 | --use_env \ 17 | --nproc_per_node=$GPUS \ 18 | --master_port=$PORT \ 19 | $(dirname "$0")/test.py \ 20 | $CONFIG \ 21 | $CHECKPOINT \ 22 | --launcher pytorch \ 23 | ${@:4} -------------------------------------------------------------------------------- /tools/__pycache__/visual_nuscenes.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/__pycache__/visual_nuscenes.cpython-38.pyc -------------------------------------------------------------------------------- /tools/benchmark.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------ 2 | # Modified from mmdetection3d (https://github.com/open-mmlab/mmdetection3d) 3 | # Copyright (c) OpenMMLab. All rights reserved. 4 | # ------------------------------------------------------------------------ 5 | # Modified by Shihao Wang 6 | # ------------------------------------------------------------------------ 7 | import argparse 8 | import time 9 | import torch 10 | from mmcv import Config 11 | from mmcv.parallel import MMDataParallel 12 | from mmcv.runner import load_checkpoint, wrap_fp16_model 13 | 14 | from mmdet3d.datasets import build_dataloader, build_dataset 15 | from mmdet3d.models import build_detector 16 | import os 17 | import sys 18 | sys.path.append('./') 19 | def parse_args(): 20 | parser = argparse.ArgumentParser(description='MMDet benchmark a model') 21 | parser.add_argument('config', help='test config file path') 22 | parser.add_argument('--checkpoint', help='checkpoint file') 23 | parser.add_argument('--samples', default=300, help='samples to benchmark') 24 | parser.add_argument( 25 | '--log-interval', default=50, help='interval of logging') 26 | args = parser.parse_args() 27 | return args 28 | 29 | 30 | def main(): 31 | args = parse_args() 32 | 33 | cfg = Config.fromfile(args.config) 34 | # set cudnn_benchmark 35 | if cfg.get('cudnn_benchmark', False): 36 | torch.backends.cudnn.benchmark = True 37 | cfg.model.pretrained = None 38 | cfg.data.test.test_mode = True 39 | 40 | if hasattr(cfg, 'plugin'): 41 | if cfg.plugin: 42 | import importlib 43 | if hasattr(cfg, 'plugin_dir'): 44 | plugin_dir = cfg.plugin_dir 45 | _module_dir = os.path.dirname(plugin_dir) 46 | _module_dir = _module_dir.split('/') 47 | _module_path = _module_dir[0] 48 | 49 | for m in _module_dir[1:]: 50 | _module_path = _module_path + '.' + m 51 | print(_module_path) 52 | plg_lib = importlib.import_module(_module_path) 53 | else: 54 | # import dir is the dirpath for the config file 55 | _module_dir = os.path.dirname(args.config) 56 | _module_dir = _module_dir.split('/') 57 | _module_path = _module_dir[0] 58 | for m in _module_dir[1:]: 59 | _module_path = _module_path + '.' + m 60 | print(_module_path) 61 | plg_lib = importlib.import_module(_module_path) 62 | # build the dataloader 63 | # TODO: support multiple images per gpu (only minor changes are needed) 64 | dataset = build_dataset(cfg.data.test) 65 | data_loader = build_dataloader( 66 | dataset, 67 | samples_per_gpu=1, 68 | workers_per_gpu=cfg.data.workers_per_gpu, 69 | dist=False, 70 | shuffle=False) 71 | 72 | # build the model and load checkpoint 73 | cfg.model.train_cfg = None 74 | model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) 75 | # fp16_cfg = cfg.get('fp16', None) 76 | # if fp16_cfg is not None: 77 | # wrap_fp16_model(model) 78 | # load_checkpoint(model, args.checkpoint, map_location='cpu') 79 | 80 | 81 | model = MMDataParallel(model, device_ids=[0]) 82 | 83 | model.eval() 84 | 85 | # the first several iterations may be very slow so skip them 86 | num_warmup = 5 87 | pure_inf_time = 0 88 | 89 | # benchmark with several samples and take the average 90 | for i, data in enumerate(data_loader): 91 | 92 | torch.cuda.synchronize() 93 | start_time = time.perf_counter() 94 | 95 | with torch.no_grad(): 96 | model(return_loss=False, rescale=True, **data) 97 | 98 | torch.cuda.synchronize() 99 | elapsed = time.perf_counter() - start_time 100 | 101 | if i >= num_warmup: 102 | pure_inf_time += elapsed 103 | if (i + 1) % args.log_interval == 0: 104 | fps = (i + 1 - num_warmup) / pure_inf_time 105 | print(f'Done image [{i + 1:<3}/ {args.samples}], ' 106 | f'fps: {fps:.1f} img / s') 107 | 108 | if (i + 1) == args.samples: 109 | pure_inf_time += elapsed 110 | fps = (i + 1 - num_warmup) / pure_inf_time 111 | print(f'Overall fps: {fps:.1f} img / s') 112 | break 113 | 114 | 115 | if __name__ == '__main__': 116 | main() 117 | -------------------------------------------------------------------------------- /tools/cc/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc/0.jpg -------------------------------------------------------------------------------- /tools/cc/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc/1.jpg -------------------------------------------------------------------------------- /tools/cc/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc/2.jpg -------------------------------------------------------------------------------- /tools/cc/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc/3.jpg -------------------------------------------------------------------------------- /tools/cc/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc/4.jpg -------------------------------------------------------------------------------- /tools/cc/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc/5.jpg -------------------------------------------------------------------------------- /tools/cc_0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc_0.jpg -------------------------------------------------------------------------------- /tools/cc_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc_1.jpg -------------------------------------------------------------------------------- /tools/cc_3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc_3.jpg -------------------------------------------------------------------------------- /tools/cc_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/cc_5.jpg -------------------------------------------------------------------------------- /tools/create_data_nusc.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | from os import path as osp 4 | 5 | from data_converter import nuscenes_converter as nuscenes_converter 6 | 7 | 8 | 9 | def nuscenes_data_prep(root_path, 10 | info_prefix, 11 | version, 12 | max_sweeps=10): 13 | """Prepare data related to nuScenes dataset. 14 | 15 | Related data consists of '.pkl' files recording basic infos, 16 | 2D annotations and groundtruth database. 17 | 18 | Args: 19 | root_path (str): Path of dataset root. 20 | info_prefix (str): The prefix of info filenames. 21 | version (str): Dataset version. 22 | dataset_name (str): The dataset class name. 23 | out_dir (str): Output directory of the groundtruth database info. 24 | max_sweeps (int, optional): Number of input consecutive frames. 25 | Default: 10 26 | """ 27 | nuscenes_converter.create_nuscenes_infos( 28 | root_path, info_prefix, version=version, max_sweeps=max_sweeps) 29 | 30 | 31 | parser = argparse.ArgumentParser(description='Data converter arg parser') 32 | parser.add_argument( 33 | '--root-path', 34 | type=str, 35 | default='./data/nuscenes', 36 | help='specify the root path of dataset') 37 | parser.add_argument( 38 | '--version', 39 | type=str, 40 | default='v1.0', 41 | required=False, 42 | help='specify the dataset version, no need for kitti') 43 | parser.add_argument( 44 | '--max-sweeps', 45 | type=int, 46 | default=10, 47 | required=False, 48 | help='specify sweeps of lidar per example') 49 | parser.add_argument( 50 | '--out-dir', 51 | type=str, 52 | default='/data/nuscenes', 53 | required=False, 54 | help='name of info pkl') 55 | parser.add_argument('--extra-tag', type=str, default='nuscenes2d') 56 | args = parser.parse_args() 57 | 58 | if __name__ == '__main__': 59 | if args.version != 'v1.0-mini': 60 | train_version = f'{args.version}-trainval' 61 | nuscenes_data_prep( 62 | root_path=args.root_path, 63 | info_prefix=args.extra_tag, 64 | version=train_version, 65 | max_sweeps=args.max_sweeps) 66 | test_version = f'{args.version}-test' 67 | nuscenes_data_prep( 68 | root_path=args.root_path, 69 | info_prefix=args.extra_tag, 70 | version=test_version, 71 | max_sweeps=args.max_sweeps) 72 | elif args.version == 'v1.0-mini': 73 | train_version = f'{args.version}' 74 | nuscenes_data_prep( 75 | root_path=args.root_path, 76 | info_prefix=args.extra_tag, 77 | version=train_version, 78 | max_sweeps=args.max_sweeps) 79 | -------------------------------------------------------------------------------- /tools/data_converter/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | -------------------------------------------------------------------------------- /tools/data_converter/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/data_converter/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /tools/data_converter/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/data_converter/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /tools/data_converter/__pycache__/nuscenes_converter.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/data_converter/__pycache__/nuscenes_converter.cpython-38.pyc -------------------------------------------------------------------------------- /tools/data_converter/__pycache__/nuscenes_converter.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nullmax-vision/QAF2D/ed2b9559548a51ffb7ffa3a02ff4c693af9d702b/tools/data_converter/__pycache__/nuscenes_converter.cpython-39.pyc -------------------------------------------------------------------------------- /tools/dist_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CONFIG=$1 4 | CHECKPOINT=$2 5 | GPUS=$3 6 | NNODES=${NNODES:-1} 7 | NODE_RANK=${NODE_RANK:-0} 8 | PORT=${PORT:-29500} 9 | MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} 10 | 11 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 12 | python -m torch.distributed.launch \ 13 | --nnodes=$NNODES \ 14 | --node_rank=$NODE_RANK \ 15 | --master_addr=$MASTER_ADDR \ 16 | --use_env \ 17 | --nproc_per_node=$GPUS \ 18 | --master_port=$PORT \ 19 | $(dirname "$0")/test.py \ 20 | $CONFIG \ 21 | $CHECKPOINT \ 22 | --launcher pytorch \ 23 | ${@:4} -------------------------------------------------------------------------------- /tools/dist_train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CONFIG=$1 4 | GPUS=$2 5 | NNODES=${NNODES:-1} 6 | NODE_RANK=${NODE_RANK:-0} 7 | PORT=${PORT:-29500} 8 | MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} 9 | 10 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 11 | python -m torch.distributed.launch \ 12 | --nnodes=$NNODES \ 13 | --node_rank=$NODE_RANK \ 14 | --master_addr=$MASTER_ADDR \ 15 | --use_env \ 16 | --nproc_per_node=$GPUS \ 17 | --master_port=$PORT \ 18 | $(dirname "$0")/train.py \ 19 | $CONFIG \ 20 | --seed 0 \ 21 | --launcher pytorch ${@:3} 22 | -------------------------------------------------------------------------------- /tools/multi_dist_train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ### 3 | ### frist worker run: NNODES=2 NODE_RANK=0 rlaunch --cpu=20 --gpu=8 --max-wait-time=24h --memory=100000 -- tools/multi_dist_train.sh projects/configs/vovnet/petr_vov_gridmask_p4_noscale_320_allcp_2node.py 8 --work-dir work_dirs/petr_vov_gridmask_p4_noscale_320_allcp_2node 4 | ### second worker run: NNODES=2 NODE_RANK=1 rlaunch --cpu=20 --gpu=8 --max-wait-time=24h --memory=100000 -- tools/multi_dist_train.sh projects/configs/vovnet/petr_vov_gridmask_p4_noscale_320_allcp_2node.py 8 --work-dir work_dirs/petr_vov_gridmask_p4_noscale_320_allcp_2node 5 | 6 | NCCL_IB_HCA=$(pushd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; do cat $i/ports/1/gid_attrs/types/* 2>/dev/null | grep v >/dev/null && echo $i ; done; popd > /dev/null) 7 | # [ -z "$NCCL_IB_HCA"] && NCCL_IB_HCA=mlx4_1; 8 | export NCCL_IB_HCA 9 | export NCCL_IB_GID_INDEX=3 10 | export NCCL_IB_TC=106 11 | 12 | NNODES=${NNODES:-2} ##Node nums 13 | NODE_RANK=${NODE_RANK:-1} ##Node rank of different machine 14 | CONFIG=$1 15 | GPUS=$2 ##Num gpus of a worker 16 | 17 | PORT=${PORT:-29500} 18 | MASTER_ADDR=${MASTER_ADDR:-"10.124.227.158"} 19 | 20 | if [[ $NODE_RANK == 0 ]]; 21 | then 22 | echo "Write the ip address of node 0 to the hostfile.txt" 23 | ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:" > hostfile.txt 24 | fi 25 | MASTER_ADDR=$(cat hostfile.txt) 26 | echo "MASTER_ADDR is : $MASTER_ADDR" 27 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 28 | python3 -m torch.distributed.launch \ 29 | --nnodes=$NNODES \ 30 | --node_rank=$NODE_RANK \ 31 | --master_addr=$MASTER_ADDR \ 32 | --nproc_per_node=$GPUS \ 33 | --master_port=$PORT \ 34 | $(dirname "$0")/train.py \ 35 | $CONFIG \ 36 | --seed 0 \ 37 | --launcher pytorch ${@:3} -------------------------------------------------------------------------------- /tools/slurm_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -x 4 | 5 | PARTITION=$1 6 | JOB_NAME=$2 7 | CONFIG=$3 8 | CHECKPOINT=$4 9 | GPUS=${GPUS:-8} 10 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 11 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 12 | PY_ARGS=${@:5} 13 | SRUN_ARGS=${SRUN_ARGS:-""} 14 | 15 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 16 | srun -p ${PARTITION} \ 17 | --job-name=${JOB_NAME} \ 18 | --gres=gpu:${GPUS_PER_NODE} \ 19 | --ntasks=${GPUS} \ 20 | --ntasks-per-node=${GPUS_PER_NODE} \ 21 | --cpus-per-task=${CPUS_PER_TASK} \ 22 | --kill-on-bad-exit=1 \ 23 | ${SRUN_ARGS} \ 24 | python -u tools/test.py ${CONFIG} ${CHECKPOINT} --launcher="slurm" ${PY_ARGS} 25 | -------------------------------------------------------------------------------- /tools/slurm_train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -x 4 | 5 | PARTITION=$1 6 | JOB_NAME=$2 7 | CONFIG=$3 8 | WORK_DIR=$4 9 | GPUS=${GPUS:-8} 10 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 11 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 12 | SRUN_ARGS=${SRUN_ARGS:-""} 13 | PY_ARGS=${@:5} 14 | 15 | PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ 16 | srun -p ${PARTITION} \ 17 | --job-name=${JOB_NAME} \ 18 | --gres=gpu:${GPUS_PER_NODE} \ 19 | --ntasks=${GPUS} \ 20 | --ntasks-per-node=${GPUS_PER_NODE} \ 21 | --cpus-per-task=${CPUS_PER_TASK} \ 22 | --kill-on-bad-exit=1 \ 23 | ${SRUN_ARGS} \ 24 | python -u tools/train.py ${CONFIG} --work-dir=${WORK_DIR} --launcher="slurm" ${PY_ARGS} 25 | -------------------------------------------------------------------------------- /tools/train.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------- 2 | # Copyright (c) OpenMMLab. All rights reserved. 3 | # --------------------------------------------- 4 | # Modified by Zhiqi Li 5 | # --------------------------------------------- 6 | 7 | from __future__ import division 8 | 9 | import argparse 10 | import copy 11 | import mmcv 12 | import os 13 | import sys 14 | import time 15 | import torch 16 | import warnings 17 | from mmcv import Config, DictAction 18 | from mmcv.runner import get_dist_info, init_dist, wrap_fp16_model 19 | from os import path as osp 20 | 21 | from mmdet import __version__ as mmdet_version 22 | from mmdet3d import __version__ as mmdet3d_version 23 | 24 | from mmdet3d.datasets import build_dataset 25 | from mmdet3d.models import build_model 26 | from mmdet3d.utils import collect_env, get_root_logger 27 | from mmdet.apis import set_random_seed 28 | from mmseg import __version__ as mmseg_version 29 | from mmcv.utils import TORCH_VERSION, digit_version 30 | 31 | sys.path.append(os.path.split(os.path.abspath(__file__))[ 32 | 0].rsplit('/', 1)[0]) 33 | 34 | 35 | def parse_args(): 36 | parser = argparse.ArgumentParser(description='Train a detector') 37 | parser.add_argument('config', help='train config file path') 38 | parser.add_argument('--work-dir', help='the dir to save logs and models') 39 | parser.add_argument( 40 | '--resume-from', help='the checkpoint file to resume from') 41 | parser.add_argument( 42 | '--no-validate', 43 | action='store_true', 44 | help='whether not to evaluate the checkpoint during training') 45 | group_gpus = parser.add_mutually_exclusive_group() 46 | group_gpus.add_argument( 47 | '--gpus', 48 | type=int, 49 | help='number of gpus to use ' 50 | '(only applicable to non-distributed training)') 51 | group_gpus.add_argument( 52 | '--gpu-ids', 53 | type=int, 54 | nargs='+', 55 | help='ids of gpus to use ' 56 | '(only applicable to non-distributed training)') 57 | parser.add_argument('--seed', type=int, default=0, help='random seed') 58 | parser.add_argument( 59 | '--deterministic', 60 | action='store_true', 61 | help='whether to set deterministic options for CUDNN backend.') 62 | parser.add_argument( 63 | '--options', 64 | nargs='+', 65 | action=DictAction, 66 | help='override some settings in the used config, the key-value pair ' 67 | 'in xxx=yyy format will be merged into config file (deprecate), ' 68 | 'change to --cfg-options instead.') 69 | parser.add_argument( 70 | '--cfg-options', 71 | nargs='+', 72 | action=DictAction, 73 | help='override some settings in the used config, the key-value pair ' 74 | 'in xxx=yyy format will be merged into config file. If the value to ' 75 | 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 76 | 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 77 | 'Note that the quotation marks are necessary and that no white space ' 78 | 'is allowed.') 79 | parser.add_argument( 80 | '--launcher', 81 | choices=['none', 'pytorch', 'slurm', 'mpi'], 82 | default='none', 83 | help='job launcher') 84 | parser.add_argument('--local_rank', type=int, default=0) 85 | parser.add_argument( 86 | '--autoscale-lr', 87 | action='store_true', 88 | help='automatically scale lr with the number of gpus') 89 | args = parser.parse_args() 90 | if 'LOCAL_RANK' not in os.environ: 91 | os.environ['LOCAL_RANK'] = str(args.local_rank) 92 | 93 | if args.options and args.cfg_options: 94 | raise ValueError( 95 | '--options and --cfg-options cannot be both specified, ' 96 | '--options is deprecated in favor of --cfg-options') 97 | if args.options: 98 | warnings.warn('--options is deprecated in favor of --cfg-options') 99 | args.cfg_options = args.options 100 | 101 | return args 102 | 103 | 104 | def main(): 105 | args = parse_args() 106 | 107 | cfg = Config.fromfile(args.config) 108 | if args.cfg_options is not None: 109 | cfg.merge_from_dict(args.cfg_options) 110 | # import modules from string list. 111 | if cfg.get('custom_imports', None): 112 | from mmcv.utils import import_modules_from_strings 113 | import_modules_from_strings(**cfg['custom_imports']) 114 | 115 | # import modules from plguin/xx, registry will be updated 116 | if hasattr(cfg, 'plugin'): 117 | if cfg.plugin: 118 | import importlib 119 | if hasattr(cfg, 'plugin_dir'): 120 | plugin_dir = cfg.plugin_dir 121 | _module_dir = os.path.dirname(plugin_dir) 122 | _module_dir = _module_dir.split('/') 123 | _module_path = _module_dir[0] 124 | 125 | for m in _module_dir[1:]: 126 | _module_path = _module_path + '.' + m 127 | print(_module_path) 128 | plg_lib = importlib.import_module(_module_path) 129 | else: 130 | # import dir is the dirpath for the config file 131 | _module_dir = os.path.dirname(args.config) 132 | _module_dir = _module_dir.split('/') 133 | _module_path = _module_dir[0] 134 | for m in _module_dir[1:]: 135 | _module_path = _module_path + '.' + m 136 | print(_module_path) 137 | plg_lib = importlib.import_module(_module_path) 138 | 139 | from projects.mmdet3d_plugin.core.apis.train import custom_train_model 140 | # set cudnn_benchmark 141 | if cfg.get('cudnn_benchmark', False): 142 | torch.backends.cudnn.benchmark = True 143 | 144 | # work_dir is determined in this priority: CLI > segment in file > filename 145 | if args.work_dir is not None: 146 | # update configs according to CLI args if args.work_dir is not None 147 | cfg.work_dir = args.work_dir 148 | elif cfg.get('work_dir', None) is None: 149 | # use config filename as default work_dir if cfg.work_dir is None 150 | cfg.work_dir = osp.join('./work_dirs', 151 | osp.splitext(osp.basename(args.config))[0]) 152 | # if args.resume_from is not None: 153 | if args.resume_from is not None and osp.isfile(args.resume_from): 154 | cfg.resume_from = args.resume_from 155 | if args.gpu_ids is not None: 156 | cfg.gpu_ids = args.gpu_ids 157 | else: 158 | cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus) 159 | if digit_version(TORCH_VERSION) == digit_version('1.8.1') and cfg.optimizer['type'] == 'AdamW': 160 | cfg.optimizer['type'] = 'AdamW2' # fix bug in Adamw 161 | if args.autoscale_lr: 162 | # apply the linear scaling rule (https://arxiv.org/abs/1706.02677) 163 | cfg.optimizer['lr'] = cfg.optimizer['lr'] * len(cfg.gpu_ids) / 8 164 | 165 | # init distributed env first, since logger depends on the dist info. 166 | if args.launcher == 'none': 167 | distributed = False 168 | else: 169 | distributed = True 170 | init_dist(args.launcher, **cfg.dist_params) 171 | # re-set gpu_ids with distributed training mode 172 | _, world_size = get_dist_info() 173 | cfg.gpu_ids = range(world_size) 174 | 175 | # create work_dir 176 | mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) 177 | # dump config 178 | cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) 179 | # init the logger before other steps 180 | timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) 181 | log_file = osp.join(cfg.work_dir, f'{timestamp}.log') 182 | # specify logger name, if we still use 'mmdet', the output info will be 183 | # filtered and won't be saved in the log_file 184 | # TODO: ugly workaround to judge whether we are training det or seg model 185 | if cfg.model.type in ['EncoderDecoder3D']: 186 | logger_name = 'mmseg' 187 | else: 188 | logger_name = 'mmdet' 189 | logger = get_root_logger( 190 | log_file=log_file, log_level=cfg.log_level, name=logger_name) 191 | 192 | # init the meta dict to record some important information such as 193 | # environment info and seed, which will be logged 194 | meta = dict() 195 | # log env info 196 | env_info_dict = collect_env() 197 | env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()]) 198 | dash_line = '-' * 60 + '\n' 199 | logger.info('Environment info:\n' + dash_line + env_info + '\n' + 200 | dash_line) 201 | meta['env_info'] = env_info 202 | meta['config'] = cfg.pretty_text 203 | 204 | # log some basic info 205 | logger.info(f'Distributed training: {distributed}') 206 | logger.info(f'Config:\n{cfg.pretty_text}') 207 | 208 | # set random seeds 209 | if args.seed is not None: 210 | logger.info(f'Set random seed to {args.seed}, ' 211 | f'deterministic: {args.deterministic}') 212 | set_random_seed(args.seed, deterministic=args.deterministic) 213 | cfg.seed = args.seed 214 | meta['seed'] = args.seed 215 | meta['exp_name'] = osp.basename(args.config) 216 | 217 | model = build_model( 218 | cfg.model, 219 | train_cfg=cfg.get('train_cfg'), 220 | test_cfg=cfg.get('test_cfg')) 221 | 222 | model.init_weights() 223 | 224 | if cfg.get('SyncBN', False): 225 | import torch.nn as nn 226 | model = nn.SyncBatchNorm.convert_sync_batchnorm(model) 227 | logger.info("Using SyncBN") 228 | 229 | logger.info(f'Model:\n{model}') 230 | datasets = [build_dataset(cfg.data.train)] 231 | if len(cfg.workflow) == 2: 232 | val_dataset = copy.deepcopy(cfg.data.val) 233 | # in case we use a dataset wrapper 234 | if 'dataset' in cfg.data.train: 235 | val_dataset.pipeline = cfg.data.train.dataset.pipeline 236 | else: 237 | val_dataset.pipeline = cfg.data.train.pipeline 238 | # set test_mode=False here in deep copied config 239 | # which do not affect AP/AR calculation later 240 | # refer to https://mmdetection3d.readthedocs.io/en/latest/tutorials/customize_runtime.html#customize-workflow # noqa 241 | val_dataset.test_mode = False 242 | datasets.append(build_dataset(val_dataset)) 243 | if cfg.checkpoint_config is not None: 244 | # save mmdet version, config file content and class names in 245 | # checkpoints as meta data 246 | cfg.checkpoint_config.meta = dict( 247 | mmdet_version=mmdet_version, 248 | mmseg_version=mmseg_version, 249 | mmdet3d_version=mmdet3d_version, 250 | config=cfg.pretty_text, 251 | CLASSES=datasets[0].CLASSES, 252 | PALETTE=datasets[0].PALETTE # for segmentors 253 | if hasattr(datasets[0], 'PALETTE') else None) 254 | # add an attribute for visualization convenience 255 | model.CLASSES = datasets[0].CLASSES 256 | custom_train_model( 257 | model, 258 | datasets, 259 | cfg, 260 | distributed=distributed, 261 | validate=(not args.no_validate), 262 | timestamp=timestamp, 263 | meta=meta) 264 | 265 | 266 | if __name__ == '__main__': 267 | torch.multiprocessing.set_start_method('fork', force=True) 268 | main() 269 | -------------------------------------------------------------------------------- /tools/visualize.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tqdm 3 | import json 4 | from visual_nuscenes import NuScenes 5 | use_gt = False 6 | out_dir = './result_vis/' 7 | result_json = "work_dirs/pp-nus/results_eval/pts_bbox/results_nusc" 8 | dataroot = '/opt/data/private/jihao/BEVFormer/data/nuscenes' 9 | if not os.path.exists(out_dir): 10 | os.mkdir(out_dir) 11 | 12 | if use_gt: 13 | nusc = NuScenes(version='v1.0-trainval', dataroot=dataroot, 14 | verbose=True, pred=False, annotations="sample_annotation") 15 | else: 16 | nusc = NuScenes(version='v1.0-trainval', dataroot=dataroot, 17 | verbose=True, pred=True, annotations=result_json, score_thr=0.25) 18 | 19 | with open('{}.json'.format(result_json)) as f: 20 | table = json.load(f) 21 | tokens = list(table['results'].keys()) 22 | 23 | for token in tqdm.tqdm(tokens[:100]): 24 | if use_gt: 25 | nusc.render_sample(token, out_path="./result_vis/" + 26 | token+"_gt.png", verbose=False) 27 | else: 28 | nusc.render_sample(token, out_path="./result_vis/" + 29 | token+"_pred.png", verbose=False) 30 | --------------------------------------------------------------------------------