├── .gitignore ├── apis_common ├── __init__.py ├── test_fgsm_img.py ├── test_pgd_img.py ├── test_patch_instance.py ├── test_pgd_imgpoint.py ├── test_patch_overlap.py ├── test_patch_class.py └── test_patch_temporal.py ├── extend_common ├── time_counter.py ├── path_string_split.py ├── get_scene_start_idx.py ├── img_check.py └── patch_apply.py ├── .gitmodules ├── Code_demo.md ├── Install.md ├── README.md ├── LICENSE └── detr3d_changes.patch /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | -------------------------------------------------------------------------------- /apis_common/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /extend_common/time_counter.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | def time_counter(last_time, motion_str, show_flag): 4 | now_time = time.time() 5 | if show_flag: 6 | print(motion_str, ':', str(round(now_time - last_time,7)),'s') 7 | last_time = time.time() 8 | return last_time 9 | -------------------------------------------------------------------------------- /extend_common/path_string_split.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def split_path_string_to_multiname(path_string): 5 | # 使用os.path.split来拆分路径并创建一个列表 6 | folder_list = [] 7 | while True: 8 | head, tail = os.path.split(path_string) 9 | if not tail: 10 | break 11 | folder_list.insert(0, tail) # 将文件夹名插入到列表的开头 12 | path_string = head 13 | return folder_list 14 | 15 | if __name__ == '__main__': 16 | folder_list = split_path_to_multiname("./folder1/folder2/folder3/1.png") 17 | print(folder_list) 18 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "TransFusion"] 2 | path = TransFusion 3 | url = https://github.com/XuyangBai/TransFusion 4 | [submodule "bevfusion"] 5 | path = bevfusion 6 | url = https://github.com/mit-han-lab/bevfusion 7 | [submodule "detr3d"] 8 | path = detr3d 9 | url = https://github.com/WangYueFt/detr3d 10 | [submodule "BEVDet"] 11 | path = BEVDet 12 | url = https://github.com/HuangJunJie2017/BEVDet 13 | [submodule "BEVFormer"] 14 | path = BEVFormer 15 | url = https://github.com/fundamentalvision/BEVFormer 16 | [submodule "mmdetection3d"] 17 | path = mmdetection3d 18 | url = https://github.com/open-mmlab/mmdetection3d 19 | -------------------------------------------------------------------------------- /extend_common/get_scene_start_idx.py: -------------------------------------------------------------------------------- 1 | def get_scene_start_idx(): 2 | print('scene_start_idx can only be used for no-shuffled nus-3d dataset') 3 | print('CANNOT be used for nus-mono3d dataste! Because of different scene order!') 4 | scene_start_idx_list = [ 5 | 0, 39, 79, 119, 158, 197, 236, 276, 316, 356, 396, 6 | 436, 475, 515, 555, 595, 635, 675, 715, 755, 795, 835, 7 | 875, 915, 955, 994, 1034, 1073, 1112, 1151, 1190, 1229, 8 | 1269, 1309, 1349, 1389, 1428, 1467, 1507, 1547, 1586, 1626, 9 | 1666, 1706, 1746, 1785, 1825, 1864, 1903, 1942, 1982, 2023, 10 | 2064, 2105, 2145, 2186, 2227, 2267, 2307, 2347, 2388, 2428, 11 | 2469, 2510, 2550, 2590, 2631, 2671, 2712, 2753, 2794, 2834, 12 | 2874, 2915, 2956, 2996, 3036, 3076, 3116, 3156, 3196, 3236, 13 | 3276, 3317, 3357, 3398, 3438, 3478, 3518, 3558, 3598, 3638, 14 | 3679, 3720, 3760, 3801, 3841, 3882, 3922, 3962, 4002, 4042, 15 | 4082, 4123, 4164, 4205, 4245, 4286, 4327, 4368, 4408, 4448, 16 | 4489, 4529, 4569, 4609, 4649, 4689, 4729, 4769, 4810, 4851, 17 | 4892, 4933, 4974, 5015, 5056, 5096, 5136, 5176, 5217, 5257, 18 | 5297, 5337, 5377, 5417, 5457, 5497, 5538, 5578, 5618, 5658, 19 | 5698, 5738, 5778, 5818, 5858, 5899, 5939, 5979 20 | ] 21 | return scene_start_idx_list 22 | 23 | if __name__ == '__main__': 24 | print(get_scene_start_idx()) 25 | -------------------------------------------------------------------------------- /extend_common/img_check.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | 4 | 5 | 6 | def img_diff_print(img1, img2, img1_name, img2_name): 7 | assert len(img1.shape)==len(img2.shape), 'imgtensor shape length must be the same' 8 | assert img1.shape==img2.shape, 'imgtensor shape must be the same' 9 | 10 | name_len = max(len(img1_name), len(img2_name)) 11 | print( 12 | '\n'+img1_name.rjust(name_len,' ')+' range R:',round(float(img1[...,0,:,:].min()),3), round(float(img1[...,0,:,:].max()),3), 13 | '\n'+img2_name.rjust(name_len,' ')+' range R:',round(float(img2[...,0,:,:].min()),3), round(float(img2[...,0,:,:].max()),3), 14 | '\n'+img1_name.rjust(name_len,' ')+' range G:',round(float(img1[...,1,:,:].min()),3), round(float(img1[...,1,:,:].max()),3), 15 | '\n'+img2_name.rjust(name_len,' ')+' range G:',round(float(img2[...,1,:,:].min()),3), round(float(img2[...,1,:,:].max()),3), 16 | '\n'+img1_name.rjust(name_len,' ')+' range B:',round(float(img1[...,2,:,:].min()),3), round(float(img1[...,2,:,:].max()),3), 17 | '\n'+img2_name.rjust(name_len,' ')+' range B:',round(float(img2[...,2,:,:].min()),3), round(float(img2[...,2,:,:].max()),3), 18 | '\n'+img1_name.rjust(name_len,' ')+' shape:', img1.shape, 19 | '\n'+img2_name.rjust(name_len,' ')+' shape:', img2.shape, 20 | ) 21 | 22 | if __name__ == '__main__': 23 | a = torch.rand(3,10,10) 24 | b = torch.rand(3,10,10) 25 | 26 | img_diff_print(a,b,'aaa', 'd1asweq') -------------------------------------------------------------------------------- /Code_demo.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Patch Mask and Info Code 4 | For example, TransFusion 5 | ```bash 6 | cd TransFusion 7 | conda activate transfusion-adv 8 | ``` 9 | Save instance patch mask, for instance patch attack 10 | ```bash 11 | python tools/test_save_instance_patch_mask_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth patch_instance_mask_dir 12 | ``` 13 | Save patch info, for other patch attacks 14 | ```bash 15 | python tools/test_save_patch_info_launcher.py configs/transfusion_nusc_voxel_LC_instancetoken.py ckpt/transfusion_LC.pth patch_info_dir 16 | ``` 17 | 18 | 19 | ## Attack Code Demo 20 | TransFusion 21 | ```bash 22 | cd TransFusion 23 | conda activate transfusion-adv 24 | ``` 25 | 26 | fgsm eps=8 27 | ```bash 28 | CUDA_VISIBLE_DEVICES=0 python tools/test_fgsm_img_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_fgsm_img_results 8 29 | ``` 30 | 31 | pgd eps=8 step=10 32 | ```bash 33 | CUDA_VISIBLE_DEVICES=0 python tools/test_pgd_img_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_pgd_img_results 8 10 34 | ``` 35 | 36 | pgd img_eps=8 point_eps=0.5 step=10 37 | ```bash 38 | CUDA_VISIBLE_DEVICES=0 python tools/test_pgd_imgpoint_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_pgd_imgpoint_results 8 0.5 10 39 | ``` 40 | 41 | instance patch patch_size=040 (for 0.4, 001 002 005 010 020 040 are supported) attack_step=10 42 | ```bash 43 | CUDA_VISIBLE_DEVICES=0 python tools/test_patch_instance_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_instance_patch_results 040 10 44 | ``` 45 | 46 | class patch patch_size=0.05 lr=0.1. 47 | (please carefully SAVE the attack log, eval log is in the attack log.) 48 | ```bash 49 | CUDA_VISIBLE_DEVICES=0 nohup python -u tools/test_patch_class_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_class_patch 0.05 0.1 >> adv_class_patch_size0.05_lr0.1.log 2>&1 & 50 | ``` 51 | 52 | temporal patch patch_size=0.05 lr=0.1 step=3 53 | ```bash 54 | CUDA_VISIBLE_DEVICES=0 python tools/test_patch_temporal_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_temporal_patch_results 0.05 0.1 3 55 | ``` 56 | 57 | overlap patch patch_size=0.05 lr=0.1 step=20 58 | ```bash 59 | CUDA_VISIBLE_DEVICES=0 python tools/test_patch_overlap_launcher.py configs/transfusion_nusc_voxel_LC_adv.py ckpt/transfusion_LC.pth adv_overlap_patch_results 0.05 0.1 20 60 | ``` 61 | 62 | ## Eval Code Demo 63 | For all attacks except class patch and overlap patch, we use the following code to eval: 64 | 65 | for example: transfusion fgsm 66 | ```bash 67 | python tools/test_scatterd_eval.py configs/transfusion_nusc_voxel_LC_adv.py fgsm_img_resultseps8 -eval bbox 68 | ``` 69 | 70 | For class patch, eval is in the attack code, please carefully save the attack log and search for eval results. 71 | 72 | For overlap patch, please refer to BEVFormer repo to find the evaluation code for overlap object detection. 73 | 74 | -------------------------------------------------------------------------------- /Install.md: -------------------------------------------------------------------------------- 1 | 2 | # Env We Use 3 | We use python=3.8 pytorch==1.10.1 torchvision==0.11.2 cudatoolkit=11.3 in all conda envs. 4 | Here we give you the code we use to install the envs. Hope be helpful for you. 5 | 6 | * TransFusion 7 | ``` 8 | conda create -n transfusion-adv python=3.8 -y 9 | conda activate transfusion-adv 10 | conda install pytorch==1.10.1 torchvision==0.11.2 cudatoolkit=11.3 -c pytorch -c conda-forge 11 | 12 | cd TransFusion 13 | # download mmcv v1.3.10.zip and unzip 14 | # download mmdet v2.11.0.zip and unzip 15 | 16 | cd mmcv-1.3.10 17 | MMCV_WITH_OPS=1 FORCE_CUDA=1 pip install -e . 18 | cd .. 19 | 20 | cd mmdet-2.11.0 21 | FORCE_CUDA=1 pip install -e . 22 | cd .. 23 | 24 | 25 | pip install orjson 26 | pip install scipy 27 | pip install numba==0.48.0 llvmlite==0.31.0 lyft-dataset-sdk==0.0.8 28 | 29 | ``` 30 | !!!! attention here !!!! 31 | 32 | you must apply our patch file before 'python setup.py develop' to ensure that you will complie differential voxelization in next step! 33 | ``` 34 | 35 | python setup.py develop 36 | pip install numpy==1.21.5 37 | 38 | pip uninstall setuptools -y 39 | pip install setuptools==59.5.0 40 | ``` 41 | 42 | 43 | * BEVFusion 44 | ``` 45 | conda create -n bevfusion-adv python=3.8 -y 46 | conda activate bevfusion-adv 47 | conda install pytorch==1.10.1 torchvision==0.11.2 cudatoolkit=11.3 -c pytorch -c conda-forge 48 | 49 | pip install pillow==8.4.0 50 | 51 | pip install mmcv-full==1.4.0 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10/index.html 52 | pip install mmdet==2.20.0 53 | 54 | pip install torchpack tqdm 55 | pip install nuscenes-devkit numba 56 | pip install mpi4py 57 | pip install orjson 58 | 59 | ``` 60 | !!!! attention here !!!! 61 | 62 | you must apply our patch file before 'python setup.py develop' to ensure that you will complie differential voxelization in next step! 63 | ``` 64 | 65 | python setup.py develop 66 | pip install numpy==1.21.5 67 | ``` 68 | 69 | 70 | * BEVFormer 71 | ``` 72 | conda create -n bevformer-adv python=3.8 -y 73 | conda activate bevformer-adv 74 | conda install pytorch==1.10.1 torchvision==0.11.2 cudatoolkit=11.3 -c pytorch -c conda-forge -y 75 | 76 | pip install mmcv-full==1.4.0 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10/index.html 77 | pip install mmdet==2.14.0 78 | pip install mmsegmentation==0.14.1 79 | 80 | pip install scipy 81 | pip install numba==0.48.0 llvmlite==0.31.0 lyft-dataset-sdk==0.0.8 82 | pip install numpy==1.21.5 83 | pip install IPython 84 | pip install orjson 85 | 86 | git clone https://github.com/open-mmlab/mmdetection3d.git 87 | cd mmdetection3d 88 | git checkout v0.17.1 # Other versions may not be compatible. 89 | python setup.py develop 90 | 91 | ``` 92 | 93 | * DETR3D 94 | ``` 95 | 96 | conda create -n detr3d-adv python=3.8 -y 97 | conda activate detr3d-adv 98 | conda install pytorch==1.10.1 torchvision==0.11.2 cudatoolkit=11.3 -c pytorch -c conda-forge -y 99 | 100 | pip install mmcv-full==1.4.0 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10/index.html 101 | pip install mmdet==2.14.0 102 | pip install mmsegmentation==0.14.1 103 | 104 | 105 | pip install scipy 106 | pip install numba==0.48.0 llvmlite==0.31.0 lyft-dataset-sdk==0.0.8 107 | pip install numpy==1.21.5 108 | pip install IPython 109 | 110 | pip install orjson 111 | 112 | cd mmdetection3d 113 | python setup.py develop 114 | 115 | 116 | pip unistall scikit-image 117 | pip istall scikit-image 118 | ``` 119 | 120 | 121 | 122 | 123 | * BEVDet 124 | ``` 125 | conda create -n bevdet-adv python=3.8 -y 126 | conda activate bevdet-adv 127 | conda install pytorch==1.10.1 torchvision==0.11.2 cudatoolkit=11.3 -c pytorch -c conda-forge -y 128 | 129 | 130 | pip install mmcv-full==1.3.16 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10/index.html 131 | pip install mmdet==2.14.0 132 | pip install mmsegmentation==0.14.1 133 | 134 | 135 | 136 | pip install orjson 137 | pip install scipy 138 | 139 | python setup.py develop 140 | pip install numba==0.48.0 llvmlite==0.31.0 lyft-dataset-sdk==0.0.8 141 | pip install numpy==1.21.5 142 | 143 | pip uninstall setuptools -y 144 | pip install setuptools==59.5.0 145 | 146 | 147 | ``` 148 | 149 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BEV_Robust 2 | Offical code for CVPR 2023 Paper Understanding the Robustness of 3D Object Detection With Bird's-Eye-View Representations in Autonomous Driving ([cvpr_url](https://openaccess.thecvf.com/content/CVPR2023/html/Zhu_Understanding_the_Robustness_of_3D_Object_Detection_With_Birds-Eye-View_Representations_CVPR_2023_paper.html)) ([arXiv_url](https://arxiv.org/abs/2303.17297)) 3 | 4 | 5 | ## Repo Explaination 6 | For you and we can clearly see what have been changed by us, we link original repo as submodules and give you repo changes file: XXXX_changes.patch 7 | 8 | First of all, you need to download all submodules: 9 | ``` 10 | git submodule update --init --recursive 11 | ``` 12 | 13 | 14 | Then, you can use these patch files like this (transfusion for example): 15 | ```bash 16 | cd TransFusion 17 | patch -p1 < ../transfusion_changes.patch 18 | ``` 19 | 20 | By the way, we save .patch files by 21 | ```bash 22 | git add . # add changes for a short time 23 | git diff --cached --name-only # show added changes 24 | git diff --cached > ../transfusion_changes.patch # save changes to outside file 25 | git reset --mixed # recover changes 26 | ``` 27 | 28 | Common attack codes are in [`./apis_common`](./apis_common). Common tool codes are in [`./extend_common`](./extend_common). These codes are used in model repos by soft link. 29 | 30 | Special attack codes are in model repo's `mmdeted/apis`. Special tool codes are in model repo's `extend`. 31 | 32 | 33 | ## Patch Attack Mask and Info 34 | The mask and info of adversarial patches are obtained all in TransFusion repo. 35 | You should generate mask and info in TransFusion, and then modify the mask/info path in all patch attack codes. 36 | 37 | 38 | 39 | ## Common Attack Code and Unique Attack Code 40 | Most model+attack pairs share common attack code in [`./apis_common`](./apis_common), but there are some exception: BEVFormer, FCOS3D. 41 | 42 | 43 | ### BEVFormer 44 | * BEVFormer patch_class 45 | * BEVFormer patch_temporal 46 | 47 | BEVFormer uses history info like 'prev_bev' and others. We keep the update of 'prev_bev' in forward_test(), not include it in forward_train()(in which we get attack loss). So we should run an extra forward_test(), after every time we change the frame in temporal-related attacks (patch_class, patch_temporal). 48 | 49 | Though, BEVFormer is not special in other attack, like FGSM and PGD. Because we eval (forward_test) immediately after each frame training in these attacks. 50 | 51 | 52 | ### FCOS3D 53 | * FCOS3D patch_temporal 54 | 55 | FCOS3D uses nus-mono dataset which has different scenes order with nus dataset. So the scene change detection code is different with other models. We use a stand-alone code to do patch_temporal for FCOS3D 56 | 57 | ## Make Voxelization Differential 58 | We replace the voxelization code in BEVFusion and TransFusion with a differetial version. This voxelization is modified from a old version of voxelization. We 59 | 60 | ## Env We Use 61 | Please refer to [Install.md](./Install.md). 62 | 63 | ## Prepare dataset and checkpoint 64 | Dataset and checkpoints prepare please refer to each model repo. 65 | 66 | ## All Code Demo 67 | Please refer to [Code_demo.md](./Code_demo.md). 68 | 69 | ## Other Things 70 | To save the work during paper preparing, we use BEVDepth implementation in BEVDet repo. Because the pytorch_lightning used in original BEVDepth repo is too hard to modify to add adversarial features. 71 | Thanks to [BEVDet](https://github.com/HuangJunJie2017/BEVDet) which give us another implementation for BEVDepth. And we still thanks the bravo work of [BEVDepth](https://github.com/Megvii-BaseDetection/BEVDepth). 72 | 73 | ## Acknowledgement 74 | Many thanks to these excellent projects: 75 | - [TransFusion](https://github.com/XuyangBai/TransFusion) 76 | - [BEVFusion](https://github.com/mit-han-lab/bevfusion) 77 | - [BEVFormer](https://github.com/fundamentalvision/BEVFormer) 78 | - [DETR3D](https://github.com/WangYueFt/detr3d) 79 | - [BEVDet](https://github.com/HuangJunJie2017/BEVDet) 80 | - [BEVDepth](https://github.com/Megvii-BaseDetection/BEVDepth) 81 | - [MMDetection3D](https://github.com/open-mmlab/mmdetection3d) 82 | 83 | 84 | ## Bibtex 85 | If this work is helpful for your research, please consider citing the following BibTeX entry. 86 | 87 | ``` 88 | @inproceedings{zhu2023understanding, 89 | title={Understanding the Robustness of 3D Object Detection With Bird's-Eye-View Representations in Autonomous Driving}, 90 | author={Zhu, Zijian and Zhang, Yichi and Chen, Hai and Dong, Yinpeng and Zhao, Shu and Ding, Wenbo and Zhong, Jiachen and Zheng, Shibao}, 91 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 92 | pages={21600--21610}, 93 | year={2023} 94 | } 95 | ``` 96 | -------------------------------------------------------------------------------- /apis_common/test_fgsm_img.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | from extend.custom_func import * 13 | from extend_common.img_check import img_diff_print 14 | from extend_common.time_counter import time_counter 15 | 16 | def single_gpu_test(model, data_loader, 17 | scattered_result_prefix=None, 18 | eps255=None 19 | ): 20 | 21 | model.eval() 22 | results = [] 23 | dataset = data_loader.dataset 24 | 25 | scattered_result_dir = scattered_result_prefix + 'eps'+eps255 26 | eps255 = float(eps255) 27 | 28 | 29 | 30 | os.makedirs(scattered_result_dir, exist_ok=True) 31 | device = model.src_device_obj 32 | 33 | time_test_flag = False 34 | prog_bar = mmcv.ProgressBar(len(dataset)) 35 | for data_i, data_out in enumerate(data_loader): 36 | 37 | results_name = str(data_i) + '.pkl' 38 | scattered_result_path = os.path.join(scattered_result_dir, results_name) 39 | 40 | if os.path.exists(scattered_result_path): 41 | print(scattered_result_path, 'exists! pass!') 42 | prog_bar.update() 43 | continue 44 | else: 45 | mmcv.dump(' ', scattered_result_path) 46 | 47 | 48 | last_time = time.time() 49 | 50 | # 1. data processing(customed) 51 | data_out = custom_data_preprocess(data_out) 52 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_out) 53 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 54 | last_time = time_counter(last_time, 'data load', time_test_flag) 55 | 56 | 57 | # FGSM all region attack, do not need patch info 58 | 59 | imgs_noisy = img_tensor_ncam.clone().detach() 60 | orig_imgs_input = img_tensor_ncam.clone().detach() 61 | 62 | eps01 = eps255 / 255. 63 | lr = eps01 64 | max_step = 1 65 | 66 | # to avoid no_gt 67 | if gt_labels_3d.shape[0] != 0: 68 | for step in range(max_step): 69 | imgs_noisy = imgs_noisy.clone().detach().requires_grad_(True) 70 | 71 | ############ resize norm pad 72 | image_ready = custom_differentiable_transform( 73 | img_tensor_rgb_6chw_0to1=imgs_noisy, 74 | img_metas=img_metas, 75 | ) 76 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 77 | 78 | if image_ready.isnan().sum()>0: 79 | print('nan in input image please check!') 80 | if data_i < 3 and step < 10: 81 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 82 | 83 | 84 | data_give = custom_image_data_give(data_out, image_ready) 85 | result = model(return_loss=True, **data_give) 86 | last_time = time_counter(last_time, 'model forward', time_test_flag) 87 | 88 | loss = 0 89 | for key in result: 90 | if 'loss' in key: 91 | loss = loss + result[key] 92 | advloss = - loss 93 | advloss.backward() 94 | 95 | # attack.step img 96 | imgs_noisy_grad = imgs_noisy.grad.detach() 97 | imgs_noisy = imgs_noisy - lr * imgs_noisy_grad.sign() 98 | 99 | # attack.project img 100 | diff = imgs_noisy - orig_imgs_input 101 | diff = torch.clamp(diff, - eps01, eps01) 102 | imgs_noisy = torch.clamp(diff + orig_imgs_input, 0, 1) 103 | 104 | last_time = time_counter(last_time, 'model backward', time_test_flag) 105 | try: 106 | print('attack step:', step, 107 | 'model_loss:',round(float(loss),5), 108 | 'fgsm_eps:', round(float(eps01),5), 109 | ) 110 | except Exception as e: 111 | print('print_func error:',e) 112 | else: 113 | # No gt bbox, directly test 114 | pass 115 | 116 | ############################################ 117 | # final eval！ 118 | with torch.no_grad(): 119 | imgs_noisy = imgs_noisy.clone().detach() 120 | ############ resize norm pad 121 | image_ready = custom_differentiable_transform( 122 | img_tensor_rgb_6chw_0to1=imgs_noisy, 123 | img_metas=img_metas, 124 | ) 125 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 126 | if image_ready.isnan().sum()>0: 127 | print('nan in input image please check!') 128 | 129 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 130 | 131 | data_give = custom_image_data_give(data_out, image_ready) 132 | data_give = custom_data_postprocess_eval(data_give) 133 | result = model(return_loss=False, rescale=True, **data_give) 134 | result = custom_result_postprocess(result) 135 | 136 | last_time = time_counter(last_time, 'eval forward', time_test_flag) 137 | if 'pts_bbox' in result[0]: 138 | _scores = result[0]['pts_bbox']['scores_3d'] 139 | elif 'img_bbox' in result[0]: 140 | _scores = result[0]['img_bbox']['scores_3d'] 141 | else: 142 | _scores = result[0]['scores_3d'] 143 | if len(_scores)>0: 144 | print('max conf:', round(float(_scores.max()),3)) 145 | else: 146 | print('nothing detected') 147 | 148 | mmcv.dump(result, scattered_result_path) 149 | results.extend(result) 150 | prog_bar.update() 151 | 152 | return results 153 | -------------------------------------------------------------------------------- /apis_common/test_pgd_img.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | from extend.custom_func import * 13 | from extend_common.img_check import img_diff_print 14 | from extend_common.time_counter import time_counter 15 | 16 | def single_gpu_test(model, data_loader, 17 | scattered_result_prefix=None, 18 | eps255=None, 19 | max_step=None 20 | ): 21 | 22 | model.eval() 23 | results = [] 24 | 25 | scattered_result_dir = scattered_result_prefix +'_eps'+eps255+'_step'+max_step 26 | eps255 = float(eps255) 27 | max_step = int(max_step) 28 | 29 | os.makedirs(scattered_result_dir, exist_ok=True) 30 | device = model.src_device_obj 31 | 32 | time_test_flag = False 33 | prog_bar = mmcv.ProgressBar(len(data_loader.dataset)) 34 | for data_i, data_out in enumerate(data_loader): 35 | 36 | results_name = str(data_i) + '.pkl' 37 | scattered_result_path = os.path.join(scattered_result_dir, results_name) 38 | 39 | if os.path.exists(scattered_result_path): 40 | print(scattered_result_path, 'exists! pass!') 41 | prog_bar.update() 42 | continue 43 | else: 44 | mmcv.dump(' ', scattered_result_path) 45 | 46 | 47 | last_time = time.time() 48 | 49 | # 1. data processing(customed) 50 | data_out = custom_data_preprocess(data_out) 51 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_out) 52 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 53 | last_time = time_counter(last_time, 'data load', time_test_flag) 54 | 55 | 56 | # PGD all region attack, do not need patch info 57 | 58 | imgs_noisy = img_tensor_ncam.clone().detach() 59 | orig_imgs_input = img_tensor_ncam.clone().detach() 60 | 61 | 62 | eps01 = eps255 / 255. 63 | lr = eps01 / (max_step-2) 64 | 65 | # pgd 是随机初始化的 random start 66 | delta = torch.rand_like(orig_imgs_input)*2*eps01 - eps01 67 | imgs_noisy = orig_imgs_input + delta 68 | imgs_noisy = torch.clamp(imgs_noisy, 0, 1) 69 | 70 | 71 | # to avoid no_gt 72 | if gt_labels_3d.shape[0] != 0: 73 | for step in range(max_step): 74 | imgs_noisy = imgs_noisy.clone().detach().requires_grad_(True) 75 | 76 | ############ resize norm pad 77 | image_ready = custom_differentiable_transform( 78 | img_tensor_rgb_6chw_0to1=imgs_noisy, 79 | img_metas=img_metas, 80 | ) 81 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 82 | 83 | if image_ready.isnan().sum()>0: 84 | print('nan in input image please check!') 85 | if data_i < 3 and step < 10: 86 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 87 | 88 | 89 | data_give = custom_image_data_give(data_out, image_ready) 90 | result = model(return_loss=True, **data_give) 91 | last_time = time_counter(last_time, 'model forward', time_test_flag) 92 | 93 | loss = 0 94 | for key in result: 95 | if 'loss' in key: 96 | loss = loss + result[key] 97 | advloss = - loss 98 | advloss.backward() 99 | 100 | # attack.step img 101 | imgs_noisy_grad = imgs_noisy.grad.detach() 102 | imgs_noisy = imgs_noisy - lr * imgs_noisy_grad.sign() 103 | 104 | # attack.project img 105 | diff = imgs_noisy - orig_imgs_input 106 | diff = torch.clamp(diff, - eps01, eps01) 107 | imgs_noisy = torch.clamp(diff + orig_imgs_input, 0, 1) 108 | 109 | 110 | last_time = time_counter(last_time, 'model backward', time_test_flag) 111 | try: 112 | print('attack step:', step, 113 | 'model_loss:', round(float(loss),3), 114 | 'pgd_eps:', round(float(eps01),5), 115 | 'pgd_lr:', round(float(lr),5), 116 | ) 117 | except Exception as e: 118 | print('print_func error:',e) 119 | else: 120 | # No gt bbox, directly test 121 | pass 122 | 123 | ############################################ 124 | # final eval！ 125 | with torch.no_grad(): 126 | ############ resize norm pad 127 | image_ready = custom_differentiable_transform( 128 | img_tensor_rgb_6chw_0to1=imgs_noisy, 129 | img_metas=img_metas, 130 | ) 131 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 132 | if image_ready.isnan().sum()>0: 133 | print('nan in input image please check!') 134 | 135 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 136 | 137 | data_give = custom_image_data_give(data_out, image_ready) 138 | data_give = custom_data_postprocess_eval(data_give) 139 | result = model(return_loss=False, rescale=True, **data_give) 140 | result = custom_result_postprocess(result) 141 | imgs_noisy = imgs_noisy.clone().detach() 142 | 143 | last_time = time_counter(last_time, 'eval forward', time_test_flag) 144 | if 'pts_bbox' in result[0]: 145 | _scores = result[0]['pts_bbox']['scores_3d'] 146 | elif 'img_bbox' in result[0]: 147 | _scores = result[0]['img_bbox']['scores_3d'] 148 | else: 149 | _scores = result[0]['scores_3d'] 150 | if len(_scores)>0: 151 | print('max conf:', round(float(_scores.max()),3)) 152 | else: 153 | print('nothing detected') 154 | 155 | mmcv.dump(result, scattered_result_path) 156 | results.extend(result) 157 | prog_bar.update() 158 | 159 | return results 160 | -------------------------------------------------------------------------------- /apis_common/test_patch_instance.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | from extend.custom_func import * 13 | from extend_common.img_check import img_diff_print 14 | from extend_common.time_counter import time_counter 15 | from extend_common.path_string_split import split_path_string_to_multiname 16 | 17 | 18 | def single_gpu_test(model, data_loader, 19 | scattered_result_prefix=None, 20 | mask_code=None, 21 | max_step=None 22 | ): 23 | 24 | model.eval() 25 | results = [] 26 | 27 | scattered_result_dir = scattered_result_prefix +'_area'+mask_code+'_step'+max_step 28 | max_step = int(max_step) 29 | 30 | os.makedirs(scattered_result_dir, exist_ok=True) 31 | device = model.src_device_obj 32 | 33 | time_test_flag = False 34 | prog_bar = mmcv.ProgressBar(len(data_loader.dataset)) 35 | for data_i, data_out in enumerate(data_loader): 36 | 37 | results_name = str(data_i) + '.pkl' 38 | scattered_result_path = os.path.join(scattered_result_dir, results_name) 39 | 40 | if os.path.exists(scattered_result_path): 41 | print(scattered_result_path, 'exists! pass!') 42 | prog_bar.update() 43 | continue 44 | else: 45 | mmcv.dump(' ', scattered_result_path) 46 | 47 | 48 | last_time = time.time() 49 | 50 | # 1. data processing(customed) 51 | data_out = custom_data_preprocess(data_out) 52 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_out) 53 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 54 | last_time = time_counter(last_time, 'data load', time_test_flag) 55 | 56 | 57 | # 2. read mask 58 | # instance patch need read mask 59 | mask_tensor_list = [] 60 | for img_path in img_path_list: 61 | file_name_valid_list = split_path_string_to_multiname(img_path)[-3:] 62 | file_name_valid_list.insert(0, '/data/zijian/mycode/BEV_Robust/TransFusion/patch_instance_mask_dir/'+mask_code) 63 | mask_path = os.path.join(*file_name_valid_list) 64 | mask_np = cv2.imread(mask_path) 65 | mask_tensor = torch.from_numpy(mask_np).permute(2,0,1)[[2,1,0]].float()/255. 66 | mask_tensor = (mask_tensor>0.5).float() 67 | mask_tensor_list.append(mask_tensor) 68 | mask_tensor_ncam = torch.stack(mask_tensor_list).to(device) 69 | 70 | 71 | 72 | 73 | orig_imgs_input = img_tensor_ncam.clone().detach() 74 | 75 | 76 | 77 | 78 | # to avoid no_gt 79 | if gt_labels_3d.shape[0] != 0: 80 | 81 | # random init noise layer 82 | noise_layer = torch.rand_like(orig_imgs_input) 83 | noise_layer.requires_grad_(True) 84 | optimizer = torch.optim.Adam([noise_layer], lr=0.1) 85 | 86 | 87 | for step in range(max_step): 88 | ############ apply patch 89 | imgs_noisy = torch.where(mask_tensor_ncam>0, noise_layer, orig_imgs_input) 90 | 91 | ############ resize norm pad 92 | image_ready = custom_differentiable_transform( 93 | img_tensor_rgb_6chw_0to1=imgs_noisy, 94 | img_metas=img_metas, 95 | ) 96 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 97 | 98 | if image_ready.isnan().sum()>0: 99 | print('nan in input image please check!') 100 | if data_i < 3 and step < 10: 101 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 102 | 103 | 104 | data_give = custom_image_data_give(data_out, image_ready) 105 | result = model(return_loss=True, **data_give) 106 | last_time = time_counter(last_time, 'model forward', time_test_flag) 107 | 108 | loss = 0 109 | for key in result: 110 | if 'loss' in key: 111 | loss = loss + result[key] 112 | advloss = - loss 113 | 114 | 115 | # attack.step img 116 | optimizer.zero_grad() 117 | advloss.backward() 118 | optimizer.step() 119 | 120 | # attack.project img 121 | noise_layer.data = torch.clamp(noise_layer, 0, 1) 122 | last_time = time_counter(last_time, 'model backward', time_test_flag) 123 | 124 | try: 125 | print('attack step:', step, 126 | 'model_loss:', round(float(loss),3), 127 | ) 128 | except Exception as e: 129 | print('print_func error:',e) 130 | else: 131 | # No gt bbox, directly test 132 | pass 133 | 134 | ############################################ 135 | # final eval！ 136 | with torch.no_grad(): 137 | ############ apply patch 138 | noise_layer = noise_layer.clone().detach() 139 | imgs_noisy = torch.where(mask_tensor_ncam>0, noise_layer, orig_imgs_input) 140 | 141 | ############ resize norm pad 142 | image_ready = custom_differentiable_transform( 143 | img_tensor_rgb_6chw_0to1=imgs_noisy, 144 | img_metas=img_metas, 145 | ) 146 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 147 | if image_ready.isnan().sum()>0: 148 | print('nan in input image please check!') 149 | 150 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 151 | 152 | data_give = custom_image_data_give(data_out, image_ready) 153 | data_give = custom_data_postprocess_eval(data_give) 154 | result = model(return_loss=False, rescale=True, **data_give) 155 | result = custom_result_postprocess(result) 156 | 157 | last_time = time_counter(last_time, 'eval forward', time_test_flag) 158 | 159 | if 'pts_bbox' in result[0]: 160 | _scores = result[0]['pts_bbox']['scores_3d'] 161 | elif 'img_bbox' in result[0]: 162 | _scores = result[0]['img_bbox']['scores_3d'] 163 | else: 164 | _scores = result[0]['scores_3d'] 165 | if len(_scores)>0: 166 | print('max conf:', round(float(_scores.max()),3)) 167 | else: 168 | print('nothing detected') 169 | 170 | mmcv.dump(result, scattered_result_path) 171 | results.extend(result) 172 | prog_bar.update() 173 | 174 | return results 175 | -------------------------------------------------------------------------------- /apis_common/test_pgd_imgpoint.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | from extend.custom_func import * 13 | from extend_common.img_check import img_diff_print 14 | from extend_common.time_counter import time_counter 15 | 16 | 17 | def single_gpu_test(model, data_loader, 18 | scattered_result_prefix=None, 19 | img_eps255=None, 20 | point_eps_m=None, 21 | max_step=None, 22 | ): 23 | 24 | model.eval() 25 | results = [] 26 | dataset = data_loader.dataset 27 | 28 | scattered_result_dir = scattered_result_prefix +'_imgeps'+img_eps255 +'_pointeps'+point_eps_m +'_step'+max_step 29 | img_eps255 = float(img_eps255) 30 | point_eps_m = float(point_eps_m) 31 | max_step = int(max_step) 32 | 33 | os.makedirs(scattered_result_dir, exist_ok=True) 34 | device = model.src_device_obj 35 | 36 | time_test_flag = False 37 | prog_bar = mmcv.ProgressBar(len(dataset)) 38 | for data_i, data_out in enumerate(data_loader): 39 | 40 | results_name = str(data_i) + '.pkl' 41 | scattered_result_path = os.path.join(scattered_result_dir, results_name) 42 | 43 | if os.path.exists(scattered_result_path): 44 | print(scattered_result_path, 'exists! pass!') 45 | prog_bar.update() 46 | continue 47 | else: 48 | mmcv.dump(' ', scattered_result_path) 49 | 50 | 51 | last_time = time.time() 52 | 53 | # 1. data processing(customed) 54 | data_out = custom_data_preprocess(data_out) 55 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d, points_tensor = custom_data_work_point(data_out) 56 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 57 | last_time = time_counter(last_time, 'data load', time_test_flag) 58 | 59 | 60 | # PGD all region attack, do not need patch info 61 | 62 | orig_imgs_input = img_tensor_ncam.clone().detach() 63 | orig_pts_input = points_tensor.clone().detach() 64 | 65 | 66 | img_eps01 = img_eps255 / 255. 67 | point_eps_m = point_eps_m 68 | img_lr = img_eps01 / (max_step-2) 69 | point_lr = point_eps_m / (max_step-2) 70 | 71 | # pgd random start 72 | delta = torch.rand_like(orig_imgs_input) * 2 * img_eps01 - img_eps01 73 | imgs_noisy = orig_imgs_input + delta 74 | imgs_noisy = torch.clamp(imgs_noisy, 0, 1) 75 | 76 | 77 | # pgd random start 78 | delta = torch.rand_like(orig_pts_input) * 2 * point_eps_m - point_eps_m 79 | pts_noisy = orig_pts_input + delta 80 | # points do not need clamp to [0,1], 81 | # but we only change xyz-channel of points 82 | pts_noisy[:,3:] = orig_pts_input[:,3:] 83 | 84 | 85 | 86 | 87 | # to avoid no_gt 88 | if gt_labels_3d.shape[0] != 0: 89 | for step in range(max_step): 90 | imgs_noisy = imgs_noisy.clone().detach().requires_grad_(True) 91 | pts_noisy = pts_noisy.clone().detach().requires_grad_(True) 92 | 93 | ############ resize norm pad 94 | image_ready = custom_differentiable_transform( 95 | img_tensor_rgb_6chw_0to1=imgs_noisy, 96 | img_metas=img_metas, 97 | ) 98 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 99 | if image_ready.isnan().sum()>0: 100 | print('nan in input image please check!') 101 | if data_i < 3 and step < 10: 102 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 103 | 104 | points_ready = pts_noisy 105 | 106 | data_give = custom_image_data_give_point(data_out, image_ready, points_ready) 107 | result = model(return_loss=True, **data_give) 108 | last_time = time_counter(last_time, 'model forward', time_test_flag) 109 | 110 | loss = 0 111 | for key in result: 112 | if 'loss' in key: 113 | loss = loss + result[key] 114 | advloss = - loss 115 | advloss.backward() 116 | 117 | # attack.step img 118 | imgs_noisy_grad = imgs_noisy.grad.detach() 119 | imgs_noisy = imgs_noisy - img_lr * imgs_noisy_grad.sign() 120 | # attack.project img 121 | diff = imgs_noisy - orig_imgs_input 122 | diff = torch.clamp(diff, -img_eps01, img_eps01) 123 | imgs_noisy = torch.clamp(diff + orig_imgs_input, 0, 1) 124 | 125 | 126 | # attack.step point 127 | pts_noisy_grad = pts_noisy.grad.detach() 128 | pts_noisy = pts_noisy - point_lr * pts_noisy_grad.sign() 129 | # attack.project point 130 | diff = pts_noisy - orig_pts_input 131 | diff = torch.clamp(diff, - point_eps_m, point_eps_m) 132 | pts_noisy = diff + orig_pts_input 133 | # the channel other than xyz cannot be changed 134 | pts_noisy[:,3:] = orig_pts_input[:,3:] 135 | 136 | 137 | last_time = time_counter(last_time, 'model backward', time_test_flag) 138 | 139 | try: 140 | print('step:', step, 141 | 'model_loss:', round(float(loss),3), 142 | 'img_eps:', round(float(img_eps01),5), 143 | 'img_lr:', round(float(img_lr),5), 144 | 'point_eps:', round(float(point_eps_m),5), 145 | 'point_lr:', round(float(point_lr),5), 146 | ) 147 | except Exception as e: 148 | print('print_func error:',e) 149 | else: 150 | # No gt bbox, directly test 151 | pass 152 | 153 | ############################################ 154 | # final eval！ 155 | with torch.no_grad(): 156 | imgs_noisy = imgs_noisy.clone().detach() 157 | ############ resize norm pad 158 | image_ready = custom_differentiable_transform( 159 | img_tensor_rgb_6chw_0to1=imgs_noisy, 160 | img_metas=img_metas, 161 | ) 162 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 163 | if image_ready.isnan().sum()>0: 164 | print('nan in input image please check!') 165 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 166 | 167 | points_ready = pts_noisy 168 | 169 | data_give = custom_image_data_give_point(data_out, image_ready, points_ready) 170 | data_give = custom_data_postprocess_eval(data_give) 171 | result = model(return_loss=False, rescale=True, **data_give) 172 | result = custom_result_postprocess(result) 173 | results.extend(result) 174 | mmcv.dump(result, scattered_result_path) 175 | 176 | last_time = time_counter(last_time, 'eval forward', time_test_flag) 177 | if 'pts_bbox' in result[0]: 178 | print('max conf:', round(float(result[0]['pts_bbox']['scores_3d'].max()),3)) 179 | else: 180 | print('max conf:', round(float(result[0]['scores_3d'].max()),3)) 181 | 182 | mmcv.dump(result, scattered_result_path) 183 | results.extend(result) 184 | prog_bar.update() 185 | 186 | return results 187 | -------------------------------------------------------------------------------- /apis_common/test_patch_overlap.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | import pickle 13 | from extend.custom_func import * 14 | from extend_common.img_check import img_diff_print 15 | from extend_common.time_counter import time_counter 16 | from extend_common.patch_apply import apply_patches_by_info_ovlp 17 | from extend_common.path_string_split import split_path_string_to_multiname 18 | 19 | 20 | def single_gpu_test(model, data_loader, 21 | scattered_result_prefix=None, 22 | area_rate_str=None, 23 | optim_lr=None, 24 | optim_step=None, 25 | ): 26 | 27 | model.eval() 28 | device = model.src_device_obj 29 | 30 | scattered_result_dir = scattered_result_prefix + '_rate'+area_rate_str + '_lr' + str(optim_lr)+'_step' + str(optim_step) 31 | os.makedirs(scattered_result_dir, exist_ok=True) 32 | 33 | optim_lr = float(optim_lr) 34 | optim_step = int(optim_step) 35 | 36 | 37 | ''' 38 | 这里定义一下整体的流程 39 | for循环遍历 40 | 为每一帧的每一个overlap的物体，定义一个patch的tensor，这个tensor会投影到两个图像上， 41 | 所有的 overlap的物体的tensor投影完成之后，通过for循环，adam lr=0.1循环优化20step， 42 | 当场检测这张攻击后的图，获得pkl文件，保存。 43 | 如果没有overlap物体，则直接原图eval。 44 | 45 | 为了保证2视角下，对抗优化的像素的一致性，保证两个视角下的patch视觉相似，patch的tensor尺寸设计为50x50 46 | 47 | 总体上思路和 instance-level 的攻击类似 48 | 在全部pkl保存完之后，使用 overlap专用的eval方法进行eval 49 | ''' 50 | 51 | 52 | max_step = optim_step 53 | cam_num = 6 54 | results = [] 55 | 56 | prog_bar = mmcv.ProgressBar(len(data_loader.dataset)) 57 | last_time = time.time() 58 | time_test_flag = False 59 | for data_i, data_out in enumerate(data_loader): 60 | 61 | results_name = str(data_i) + '.pkl' 62 | scattered_result_path = os.path.join(scattered_result_dir, results_name) 63 | 64 | if os.path.exists(scattered_result_path): 65 | print(scattered_result_path, 'exists! pass!') 66 | prog_bar.update() 67 | continue 68 | else: 69 | mmcv.dump(' ', scattered_result_path) 70 | 71 | 72 | #### 1. data processing(customed) 73 | data_out = custom_data_preprocess(data_out) 74 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_out) 75 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 76 | last_time = time_counter(last_time, 'data load', time_test_flag) 77 | cam_num = len(img_path_list) 78 | 79 | 80 | 81 | #### 2. read patch info from file 82 | patch_info_list = [] 83 | for cams_i in range(cam_num): 84 | img_path = img_path_list[cams_i] 85 | file_name_valid_list = split_path_string_to_multiname(img_path)[-3:] 86 | file_name_valid_list.insert(0, '/data/zijian/mycode/BEV_Robust/TransFusion/patch_info_2d3d3dt_square_dir/all') 87 | info_path = os.path.join(*file_name_valid_list) 88 | info_path = info_path.replace('.jpg', '.pkl') 89 | info_i = pickle.load(open(info_path, 'rb')) 90 | 91 | 92 | 93 | 94 | patch_info_list.append(info_i) 95 | last_time = time_counter(last_time, 'read pkl', time_test_flag) 96 | 97 | 98 | 99 | 100 | #### 3. judge overlap object exists or not 101 | # 同时存在gt和ovlp物体，才进行攻击， 否则直接eval 102 | attack_flag = False 103 | has_gt_flag = gt_labels_3d.shape[0] != 0 104 | if has_gt_flag: 105 | ovlp_flag = patch_info_list[0]['objects_info']['is_overlap'] 106 | ovlp_num = ovlp_flag.sum() 107 | ovlp_exist = (ovlp_num > 0) 108 | if ovlp_exist: 109 | attack_flag = True 110 | 111 | if attack_flag: 112 | patch_w = 50 113 | patch_h = 50 114 | ovlp_inst_patch_tensor = torch.rand(ovlp_num, 3, patch_h, patch_w).to(device) 115 | ovlp_inst_patch_tensor.requires_grad_() 116 | optimizer = torch.optim.Adam([ovlp_inst_patch_tensor], lr=optim_lr) 117 | 118 | 119 | 120 | for step in range(max_step): 121 | #### apply patch 122 | patched_img_tensor_ncam = img_tensor_ncam.clone() 123 | for cams_i in range(cam_num): 124 | patch_info_in_cami = patch_info_list[cams_i] 125 | patched_img_tensor_ncam[cams_i] = apply_patches_by_info_ovlp( 126 | info=patch_info_in_cami, 127 | image=patched_img_tensor_ncam[cams_i], 128 | patch_book=ovlp_inst_patch_tensor, 129 | area_str=area_rate_str, 130 | ) 131 | 132 | #### resize norm pad 133 | image_ready = custom_differentiable_transform( 134 | img_tensor_rgb_6chw_0to1=patched_img_tensor_ncam, 135 | img_metas=img_metas, 136 | ) 137 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 138 | if image_ready.isnan().sum()>0: 139 | print('nan in input image please check!') 140 | if data_i < 3 and step < 10: 141 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 142 | 143 | 144 | data_give = custom_image_data_give(data_out, image_ready) 145 | result = model(return_loss=True, **data_give) 146 | last_time = time_counter(last_time, 'model forward', time_test_flag) 147 | loss = 0 148 | for key in result: 149 | if 'loss' in key: 150 | loss = loss + result[key] 151 | advloss = - loss 152 | 153 | # attack.step img 154 | optimizer.zero_grad() 155 | advloss.backward() 156 | optimizer.step() 157 | 158 | # attack.project img 159 | ovlp_inst_patch_tensor.data = torch.clamp(ovlp_inst_patch_tensor, 0, 1) 160 | last_time = time_counter(last_time, 'model backward', time_test_flag) 161 | print('attack step:', step, 162 | 'model_loss:',round(float(loss),5), 163 | ) 164 | 165 | 166 | ######################### 167 | ##### 攻击结束，最后粘贴patch，准备eval 168 | #### apply patch 169 | patched_img_tensor_ncam = img_tensor_ncam.clone() 170 | for cams_i in range(cam_num): 171 | patch_info_in_cami = patch_info_list[cams_i] 172 | patched_img_tensor_ncam[cams_i] = apply_patches_by_info_ovlp( 173 | info=patch_info_in_cami, 174 | image=patched_img_tensor_ncam[cams_i], 175 | patch_book=ovlp_inst_patch_tensor, 176 | area_str=area_rate_str, 177 | ) 178 | if data_i < 30: 179 | save_image(patched_img_tensor_ncam, os.path.join(scattered_result_dir, str(data_i)+'.png')) 180 | else: 181 | # 同时存在gt和ovlp物体，才进行攻击， 否则直接eval 182 | patched_img_tensor_ncam = img_tensor_ncam.clone() 183 | 184 | 185 | 186 | ############################################ 187 | # 结尾 eval！ 188 | with torch.no_grad(): 189 | ############ resize norm pad 190 | image_ready = custom_differentiable_transform( 191 | img_tensor_rgb_6chw_0to1=patched_img_tensor_ncam, 192 | img_metas=img_metas, 193 | ) 194 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 195 | if image_ready.isnan().sum()>0: 196 | print('nan in input image please check!') 197 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 198 | 199 | data_give = custom_image_data_give(data_out, image_ready) 200 | data_give = custom_data_postprocess_eval(data_give) 201 | result = model(return_loss=False, rescale=True, **data_give) 202 | result = custom_result_postprocess(result) 203 | results.extend(result) 204 | mmcv.dump(result, scattered_result_path) 205 | 206 | prog_bar.update() 207 | return results 208 | 209 | 210 | 211 | -------------------------------------------------------------------------------- /apis_common/test_patch_class.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | import pickle 13 | from extend.custom_func import * 14 | from extend_common.img_check import img_diff_print 15 | from extend_common.time_counter import time_counter 16 | from extend_common.patch_apply import apply_patches_by_info 17 | from extend_common.path_string_split import split_path_string_to_multiname 18 | 19 | 20 | 21 | def single_gpu_test(model, data_loader, 22 | patch_save_prefix=None, 23 | area_rate_str=None, 24 | optim_lr=None 25 | ): 26 | 27 | model.eval() 28 | dataset = data_loader.dataset 29 | device = model.src_device_obj 30 | 31 | patch_save_dir = patch_save_prefix +'_area'+area_rate_str+'_lr'+optim_lr 32 | os.makedirs(patch_save_dir, exist_ok=True) 33 | 34 | optim_lr = float(optim_lr) 35 | 36 | 37 | # 为每一个类别定义一个patch 38 | # define one patch for evey class 39 | class_names_list = [ 40 | 'car', 'truck', 'construction_vehicle', 41 | 'bus', 'trailer', 'barrier', 42 | 'motorcycle', 'bicycle', 'pedestrian', 43 | 'traffic_cone' 44 | ] 45 | patch_w = 100 46 | patch_h = 100 47 | class_patches_tensor = torch.rand(len(class_names_list),3, patch_h, patch_w).to(device) 48 | class_patches_tensor.requires_grad_() 49 | optimizer = torch.optim.Adam([class_patches_tensor], lr=optim_lr) 50 | 51 | 52 | time_test_flag = False 53 | 54 | epoch_max = 3 55 | patch_info_list_database = {} 56 | 57 | # epoch 0 2 4 6 ... for train 58 | # epoch 1 3 5 7 ... for eval 59 | for epoch_d in range(epoch_max*2+1): 60 | epoch = int(epoch_d/2) 61 | patch_is_training = (epoch_d % 2 == 0) 62 | 63 | if patch_is_training: 64 | print('=============================') 65 | print('======= epoch',epoch,' train start =========') 66 | print('=============================') 67 | else: 68 | print('=============================') 69 | print('======= epoch',epoch,'eval start =========') 70 | print('=============================') 71 | results = [] 72 | 73 | prog_bar = mmcv.ProgressBar(len(dataset)) 74 | last_time = time.time() 75 | for data_i, data_out in enumerate(data_loader): 76 | 77 | 78 | #### 1. data processing(customed) 79 | data_out = custom_data_preprocess(data_out) 80 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_out) 81 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 82 | last_time = time_counter(last_time, 'data load', time_test_flag) 83 | cam_num = len(img_path_list) 84 | 85 | #### 2. read patch info from file/database 86 | if not str(data_i) in patch_info_list_database: 87 | patch_info_list = [] 88 | for cams_i in range(cam_num): 89 | img_path = img_path_list[cams_i] 90 | file_name_valid_list = split_path_string_to_multiname(img_path)[-3:] 91 | file_name_valid_list.insert(0, '/data/zijian/mycode/BEV_Robust/TransFusion/patch_info_2d3d3dt_square_dir/all') 92 | info_path = os.path.join(*file_name_valid_list) 93 | info_path = info_path.replace('.jpg', '.pkl') 94 | info_i = pickle.load(open(info_path, 'rb')) 95 | patch_info_list.append(info_i) 96 | patch_info_list_database[str(data_i)] = patch_info_list 97 | else: 98 | patch_info_list = patch_info_list_database[str(data_i)] 99 | last_time = time_counter(last_time, 'read pkl', time_test_flag) 100 | 101 | 102 | #### 3. apply patch 103 | patched_img_tensor_ncam = img_tensor_ncam.clone() 104 | # to avoid no_gt 105 | has_gt_flag = gt_labels_3d.shape[0] != 0 106 | if has_gt_flag: 107 | if patch_is_training: 108 | # for patch training 109 | for cams_i in range(cam_num): 110 | patch_info_in_cami = patch_info_list[cams_i] 111 | patched_img_tensor_ncam[cams_i] = apply_patches_by_info( 112 | info=patch_info_in_cami, 113 | image=patched_img_tensor_ncam[cams_i], 114 | patch_book=class_patches_tensor, 115 | area_str=area_rate_str, 116 | ) 117 | else: 118 | with torch.no_grad(): 119 | # for patch eval 120 | for cams_i in range(cam_num): 121 | patch_info_in_cami = patch_info_list[cams_i] 122 | patched_img_tensor_ncam[cams_i] = apply_patches_by_info( 123 | info=patch_info_in_cami, 124 | image=patched_img_tensor_ncam[cams_i], 125 | patch_book=class_patches_tensor, 126 | area_str=area_rate_str, 127 | ) 128 | else: # 没有gt 图像不做改变 if no gt donot change images 129 | pass 130 | 131 | if not has_gt_flag and patch_is_training: 132 | # 训练时，无gt的图直接跳过 133 | # when training, img with no gt will be skip 134 | # 测试时，正常测试，不跳过 135 | # when evaluating, img with no gt will still be evaluated 136 | continue 137 | 138 | 139 | # save for watch 140 | if patch_is_training and data_i % 100 == 0: 141 | save_image(patched_img_tensor_ncam, os.path.join(patch_save_dir, str(data_i)+'.png')) 142 | 143 | 144 | 145 | #### 4. resize norm pad 146 | image_ready = custom_differentiable_transform( 147 | img_tensor_rgb_6chw_0to1=patched_img_tensor_ncam, 148 | img_metas=img_metas, 149 | ) 150 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 151 | 152 | if image_ready.isnan().sum()>0: 153 | print('nan in input image please check!') 154 | if data_i < 10: 155 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 156 | 157 | 158 | #### 5. update patch or evaluate 159 | 160 | 161 | 162 | if patch_is_training: # 在训练 更新patch 163 | data_give = custom_image_data_give(data_out, image_ready) 164 | result = model(return_loss=True, **data_give) 165 | last_time = time_counter(last_time, 'model forward', time_test_flag) 166 | 167 | loss = 0 168 | for key in result: 169 | if 'loss' in key: 170 | loss = loss + result[key] 171 | advloss = - loss 172 | 173 | # attack.step img 174 | optimizer.zero_grad() 175 | advloss.backward() 176 | optimizer.step() 177 | 178 | # attack.project img 179 | class_patches_tensor.data = torch.clamp(class_patches_tensor, 0, 1) 180 | last_time = time_counter(last_time, 'model backward', time_test_flag) 181 | print('attack step:', data_i, 182 | 'model_loss:',round(float(loss),5), 183 | ) 184 | 185 | 186 | else: 187 | with torch.no_grad(): 188 | data_give = custom_image_data_give(data_out, image_ready) 189 | data_give = custom_data_postprocess_eval(data_give) 190 | result = model(return_loss=False, rescale=True, **data_give) 191 | result = custom_result_postprocess(result) 192 | results.extend(result) 193 | last_time = time_counter(last_time, 'model forward', time_test_flag) 194 | 195 | prog_bar.update() 196 | 197 | 198 | #### After one (train or val) epoch_d 199 | if not patch_is_training: 200 | print(dataset.evaluate(results,)) # eval_kwargs 在DETR3d里面，不是必须用到 201 | # class patch is evaluated during training. All evaluation scores are saved in nohup-log. 202 | 203 | ################################## 204 | # save 205 | ################################## 206 | if patch_is_training: 207 | print('=============================') 208 | print('======= epoch',epoch,'save =========') 209 | print('=============================') 210 | save_class_patches_path = os.path.join(patch_save_dir, 'epoch_'+str(epoch)+'class_patches.pkl') 211 | pickle.dump(class_patches_tensor.cpu(), open(save_class_patches_path, 'wb')) 212 | 213 | return results 214 | 215 | 216 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [Zijian Zhu] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /apis_common/test_patch_temporal.py: -------------------------------------------------------------------------------- 1 | import mmcv 2 | import torch 3 | import numpy as np 4 | import PIL.Image as Image 5 | import torchvision.transforms as transforms 6 | import torch.nn.functional as F 7 | import torchvision 8 | from torchvision.utils import save_image 9 | import cv2 10 | import time 11 | import os 12 | import pickle 13 | from extend.custom_func import * 14 | from extend_common.img_check import img_diff_print 15 | from extend_common.time_counter import time_counter 16 | from extend_common.patch_apply import apply_patches_by_info_4side 17 | from extend_common.path_string_split import split_path_string_to_multiname 18 | from extend_common.get_scene_start_idx import get_scene_start_idx 19 | 20 | 21 | def single_gpu_test(model, data_loader, 22 | scattered_result_prefix=None, 23 | area_rate_str=None, 24 | optim_lr=None, 25 | optim_step=None, 26 | index_min = None, 27 | index_max = None, 28 | ): 29 | 30 | model.eval() 31 | dataset = data_loader.dataset 32 | device = model.src_device_obj 33 | 34 | scattered_result_dir = scattered_result_prefix +'_area'+area_rate_str+'_lr'+optim_lr+'_step' + optim_step 35 | os.makedirs(scattered_result_dir, exist_ok=True) 36 | 37 | optim_lr = float(optim_lr) 38 | optim_step = int(optim_step) 39 | 40 | scene_start_idx_list = get_scene_start_idx() 41 | max_epoch_local = optim_step 42 | 43 | patch_info_list_database = {} 44 | time_test_flag = False 45 | 46 | 47 | results = [] 48 | prog_bar = mmcv.ProgressBar(len(dataset)) 49 | last_time = time.time() 50 | for data_i, data_out in enumerate(data_loader): 51 | if data_i < index_min: 52 | prog_bar.update() 53 | continue 54 | if data_i > index_max: 55 | break 56 | 57 | #### 1. data processing(customed) 58 | data_out = custom_data_preprocess(data_out) 59 | _, img_path_list, _, _, _ = custom_data_work(data_out) 60 | last_time = time_counter(last_time, 'data load', time_test_flag) 61 | cam_num = len(img_path_list) 62 | 63 | 64 | #### 2. read patch info from file/database 65 | if not str(data_i) in patch_info_list_database: 66 | patch_info_list = [] 67 | for cams_i in range(cam_num): 68 | img_path = img_path_list[cams_i] 69 | file_name_valid_list = split_path_string_to_multiname(img_path)[-3:] 70 | file_name_valid_list.insert(0, '/data/zijian/mycode/BEV_Robust/TransFusion/patch_info_2d3d3dt_square_dir/all') 71 | info_path = os.path.join(*file_name_valid_list) 72 | info_path = info_path.replace('.jpg', '.pkl') 73 | info_i = pickle.load(open(info_path, 'rb')) 74 | patch_info_list.append(info_i) 75 | patch_info_list_database[str(data_i)] = patch_info_list 76 | else: 77 | patch_info_list = patch_info_list_database[str(data_i)] 78 | last_time = time_counter(last_time, 'read pkl', time_test_flag) 79 | 80 | 81 | ''' 82 | 由于我们要一个场景（大概40帧左右），一起进行攻击 83 | 所以我需要先遍历数据集，把这一个场景的数据先拿出来，统计里面instance的数量，构建一个 patch 库 84 | 然后再在读取出的这一个场景的数据里做攻击 85 | 86 | 如果是场景的第0帧 87 | 则开始遍历当前场景，直到下一个第0帧的出现，这时候暂存下一个第0帧 88 | 遍历场景时，存下所有的注释信息， 89 | 并从之前存好的 patch info 中 获取 instance_token 90 | ''' 91 | 92 | 93 | scene_start_here_flag = (data_i in scene_start_idx_list) 94 | 95 | go_to_training_flag = False 96 | 97 | if data_i == 0: 98 | # 第0帧 99 | # start new 100 | data_in_scene_list = [] 101 | patch_info_in_scene_list = [] 102 | data_i_list = [] 103 | data_in_scene_list.append(data_out) 104 | patch_info_in_scene_list.append(patch_info_list) 105 | data_i_list.append(data_i) 106 | elif scene_start_here_flag and data_i > 0: 107 | # 之后的每一个首帧 108 | # 存一个连续场景的全部 data 和 patch_info 109 | # end old 110 | try: 111 | data_in_scene_list_full = data_in_scene_list 112 | patch_info_in_scene_list_full = patch_info_in_scene_list 113 | data_i_list_full = data_i_list 114 | go_to_training_flag = True 115 | except: 116 | print('start from data_i:', data_i) 117 | # start new 118 | data_in_scene_list = [] 119 | patch_info_in_scene_list = [] 120 | data_i_list = [] 121 | data_in_scene_list.append(data_out) 122 | patch_info_in_scene_list.append(patch_info_list) 123 | data_i_list.append(data_i) 124 | elif data_i == len(dataset)-1: 125 | data_in_scene_list.append(data_out) 126 | patch_info_in_scene_list.append(patch_info_list) 127 | data_i_list.append(data_i) 128 | # 最后一帧 129 | # end old 130 | data_in_scene_list_full = data_in_scene_list 131 | patch_info_in_scene_list_full = patch_info_in_scene_list 132 | data_i_list_full = data_i_list 133 | go_to_training_flag = True 134 | else: 135 | data_in_scene_list.append(data_out) 136 | patch_info_in_scene_list.append(patch_info_list) 137 | data_i_list.append(data_i) 138 | prog_bar.update() 139 | 140 | if go_to_training_flag: 141 | # local dataset: data_in_scene_list_full 142 | # local dataset: patch_info_in_scene_list_full 143 | # local dataset: data_i_list_full 144 | scene_length = len(data_in_scene_list_full) 145 | 146 | ###### 1.构建patch库 Establish local-scene patchbook 147 | # 每个物体的4个面，都放patch， 148 | # patchtensor的形状, 由实际的patchsize确定，兼容正方形patch 149 | instance_token_list = [] 150 | patch_4side_book_list = [] 151 | for i_local in range(scene_length): 152 | # 1.把数据拿出来，处理数据 153 | data_local = data_in_scene_list_full[i_local] 154 | patch_info_local = patch_info_in_scene_list_full[i_local] 155 | _, _, _, _, gt_labels_3d = custom_data_work(data_local) 156 | # 2.判断有没有gt 157 | # 防止出现 no_gt 158 | has_gt_flag = (gt_labels_3d.shape[0] != 0) and (type(patch_info_local[0]) != str) 159 | if has_gt_flag: 160 | scene_name = patch_info_local[0]['scene_info']['scene_name'] 161 | instance_tokens_i = patch_info_local[0]['objects_info']['instance_tokens'] 162 | for inst_tk_idx in range(len(instance_tokens_i)): 163 | instance_token = instance_tokens_i[inst_tk_idx] 164 | if not instance_token in instance_token_list: 165 | # 添加patch 166 | # 根据最先出现的patch，标注的信息，添加4个patch 167 | for j_cam_1frame in range(cam_num): 168 | if patch_info_local[j_cam_1frame]['patch_visible_bigger'][inst_tk_idx]: 169 | # 如果可以被，当前的camera看到，则添加，否则不添加 170 | patch_3d_wh = patch_info_local[j_cam_1frame]['patch_3d_temporal']['patch_3d_wh'][inst_tk_idx] 171 | patch_3d_wh_use = patch_3d_wh[area_rate_str] 172 | 173 | patch_4side_ = [] 174 | for j_side in range(4): 175 | patch_w_real, patch_h_real = patch_3d_wh_use[j_side] 176 | # 遵循每1m 100pix的密度 177 | patch_w_tensor = int(patch_w_real*100) 178 | patch_h_tensor = int(patch_h_real*100) 179 | patch_jside_ = torch.rand(3, patch_h_tensor, patch_w_tensor).to(device) 180 | patch_jside_.requires_grad_() 181 | patch_4side_.append(patch_jside_) 182 | 183 | instance_token_list.append(instance_token) 184 | patch_4side_book_list.extend(patch_4side_) 185 | 186 | # 为这些patch定义 优化器 187 | optimizer = torch.optim.Adam(patch_4side_book_list, lr=optim_lr) 188 | 189 | # 以后每一次取用，都需要，结合instance_token_list获取 token对应的index，再用 190 | 191 | 192 | for epoch_local in range(max_epoch_local): 193 | print('scene_name:', scene_name,'start epoch_local', epoch_local,'training') 194 | for i_local in range(scene_length): 195 | 196 | ############## 把数据拿出来，处理数据 Take out the data and process the data 197 | data_local = data_in_scene_list_full[i_local] 198 | patch_info_local = patch_info_in_scene_list_full[i_local] 199 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_local) 200 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 201 | last_time = time_counter(last_time, 'data process', time_test_flag) 202 | 203 | ############## apply patch 204 | patched_img_tensor_ncam = img_tensor_ncam.clone() 205 | # in case of no_gt 206 | has_gt_flag = (gt_labels_3d.shape[0] != 0) and (type(patch_info_local[0]) != str) 207 | if has_gt_flag: 208 | # apply patch 209 | for cams_i in range(cam_num): 210 | patch_info_in_cami = patch_info_local[cams_i] 211 | patched_img_tensor_ncam[cams_i] = apply_patches_by_info_4side( 212 | info=patch_info_in_cami, 213 | image=patched_img_tensor_ncam[cams_i], 214 | instance_token_book=instance_token_list, 215 | patch_book_4side=patch_4side_book_list, 216 | area_str=area_rate_str, 217 | ) 218 | # patched_img_tensor_ncam[cams_i] = (patched_img_tensor_ncam[cams_i] + patch_4side_book_list[0].mean()/1000).clamp(0,1) 219 | else: # no gt，图像不做改变，也不必优化patch 220 | continue 221 | 222 | last_time = time_counter(last_time, 'apply patch', time_test_flag) 223 | 224 | ############ resize norm pad 225 | image_ready = custom_differentiable_transform( 226 | img_tensor_rgb_6chw_0to1=patched_img_tensor_ncam, 227 | img_metas=img_metas, 228 | ) 229 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 230 | 231 | 232 | if image_ready.isnan().sum()>0: 233 | print('nan in input image please check!') 234 | 235 | data_i_actual = data_i_list_full[i_local] 236 | if data_i_actual < 100 and epoch_local < 3 and i_local < 3: 237 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 238 | 239 | 240 | data_give = custom_image_data_give(data_local, image_ready) 241 | result = model(return_loss=True, **data_give) # 经过model， data中的img会被修改为[6,3,H,W] 242 | last_time = time_counter(last_time, 'model forward', time_test_flag) 243 | loss = 0 244 | for key in result: 245 | if 'loss' in key: 246 | loss = loss + result[key] 247 | advloss = - loss 248 | optimizer.zero_grad() 249 | advloss.backward() 250 | optimizer.step() 251 | optimizer.zero_grad() 252 | 253 | last_time = time_counter(last_time, 'model backward', time_test_flag) 254 | 255 | for _patch_i in range(len(patch_4side_book_list)): 256 | patch_4side_book_list[_patch_i].data = torch.clamp(patch_4side_book_list[_patch_i], 0, 1) 257 | last_time = time_counter(last_time, 'patch clamp', time_test_flag) 258 | print('attack step:', i_local, 259 | 'model_loss:',round(float(loss),5), 260 | ) 261 | ######################### 262 | ##### 攻击结束，最后再遍历一遍，粘贴patch，eval 263 | print('scene_name:', scene_name,'start eval') 264 | prog_bar_local_eval = mmcv.ProgressBar(scene_length) 265 | with torch.no_grad(): 266 | for i_local in range(scene_length): 267 | 268 | ################# 把数据拿出来，处理数据 269 | data_local = data_in_scene_list_full[i_local] 270 | patch_info_local = patch_info_in_scene_list_full[i_local] 271 | img_metas, img_path_list, img_org_np, img_processed, gt_labels_3d = custom_data_work(data_local) 272 | img_tensor_ncam = custom_img_read_from_img_org(img_org_np, device) 273 | 274 | ################ 安装patch 275 | patched_img_tensor_ncam = img_tensor_ncam.clone() 276 | # 防止出现 no_gt 277 | has_gt_flag = (gt_labels_3d.shape[0] != 0) and (type(patch_info_local[0]) != str) 278 | if has_gt_flag: 279 | # apply patch 280 | for cams_i in range(cam_num): 281 | patch_info_in_cami = patch_info_local[cams_i] 282 | patched_img_tensor_ncam[cams_i] = apply_patches_by_info_4side( 283 | info=patch_info_in_cami, 284 | image=patched_img_tensor_ncam[cams_i], 285 | instance_token_book=instance_token_list, 286 | patch_book_4side=patch_4side_book_list, 287 | area_str=area_rate_str, 288 | ) 289 | else: # 没有gt，图像不做改变，直接eval 290 | pass 291 | 292 | ############ resize norm pad 293 | image_ready = custom_differentiable_transform( 294 | img_tensor_rgb_6chw_0to1=patched_img_tensor_ncam, 295 | img_metas=img_metas, 296 | ) 297 | last_time = time_counter(last_time, 'img rsnmpd', time_test_flag) 298 | if image_ready.isnan().sum()>0: 299 | print('nan in input image please check!') 300 | if i_local < 3: 301 | img_diff_print(img_processed, image_ready,'img_processed','image_ready') 302 | 303 | data_give = custom_image_data_give(data_local, image_ready) 304 | data_give = custom_data_postprocess_eval(data_give) 305 | result = model(return_loss=False, rescale=True, **data_give) 306 | result = custom_result_postprocess(result) 307 | results.extend(result) 308 | 309 | data_i_actual = data_i_list_full[i_local] 310 | scattered_result_path = os.path.join(scattered_result_dir, str(data_i_actual)+'.pkl') 311 | mmcv.dump(result, scattered_result_path) 312 | if data_i_actual < 100: 313 | save_image(patched_img_tensor_ncam, os.path.join(scattered_result_dir, str(data_i_actual)+'.png')) 314 | prog_bar_local_eval.update() 315 | print() 316 | return results 317 | 318 | 319 | 320 | -------------------------------------------------------------------------------- /extend_common/patch_apply.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torchvision.transforms as transforms 4 | import torch.nn.functional as F 5 | from torchvision.utils import save_image 6 | 7 | 8 | def proj_3d_to_2d(points_3d_batch, lidar2img, img_org_shape): 9 | points_4d_batch = torch.cat([points_3d_batch, torch.ones_like(points_3d_batch[:,0:1])], dim=-1) 10 | points_2d_batch = points_4d_batch @ lidar2img.t() 11 | points_2d_batch_depth = points_2d_batch[:,2] 12 | points_2d_batch_depth_positive = torch.clamp(points_2d_batch_depth, min=1e-5) 13 | points_2d_batch[:,0] /= points_2d_batch_depth_positive 14 | points_2d_batch[:,1] /= points_2d_batch_depth_positive 15 | on_the_image_depth = points_2d_batch_depth > 0 16 | coor_x, coor_y = points_2d_batch[:,0], points_2d_batch[:,1] 17 | h, w = img_org_shape 18 | on_the_image = (coor_x > 0) * (coor_x < w) * (coor_y > 0) * (coor_y < h) 19 | on_the_image = on_the_image * on_the_image_depth 20 | return points_2d_batch[:,:2], on_the_image 21 | 22 | 23 | 24 | def safe_img_patch_apply(image, patch, x1, y1): 25 | assert len(image.shape) == len(patch.shape) 26 | assert image.shape[0] == patch.shape[0] 27 | 28 | try: 29 | w_patch = patch.shape[-1] 30 | h_patch = patch.shape[-2] 31 | x2 = x1 + w_patch 32 | y2 = y1 + h_patch 33 | w_img = image.shape[-1] 34 | h_img = image.shape[-2] 35 | 36 | x1_use = x1 37 | x2_use = x2 38 | y1_use = y1 39 | y2_use = y2 40 | patch_use = patch 41 | 42 | if x1 < 0: 43 | x1_use = 0 44 | patch_use = patch_use[:, :, -x1:] 45 | elif x1 > w_img-1: 46 | return image 47 | 48 | if x2 > w_img: 49 | x2_use = w_img 50 | patch_use = patch_use[:, :, :-(x2 - w_img)] 51 | elif x2 < 0: 52 | return image 53 | 54 | if y1 < 0: 55 | y1_use = 0 56 | patch_use = patch_use[:, -y1:, :] 57 | elif y1 > h_img - 1: 58 | return image 59 | 60 | if y2 > h_img: 61 | y2_use = h_img 62 | patch_use = patch_use[:, :-(y2 - h_img), :] 63 | elif y2 < 0: 64 | return image 65 | 66 | image[:, y1_use:y2_use, x1_use:x2_use] = patch_use 67 | except: 68 | return image 69 | print('safe_img_patch_apply failed, skip this patch') 70 | return image 71 | 72 | 73 | 74 | 75 | def apply_patches_by_info(info, image, patch_book, area_str): 76 | local_classname_list = info['objects_info']['class_name'] 77 | local_classidx_list = info['objects_info']['class_idx'] 78 | 79 | # 重新排序，按照画面深度uvd的d排序，以保证从远到近粘贴 80 | _d = info['objects_info']['uvd_center'][:, 2] 81 | d_index = _d.sort(descending=True)[1] 82 | 83 | d_index_list = d_index.tolist() 84 | 85 | for i in d_index_list: 86 | if info['patch_visible_bigger'][i]: # 可见 87 | assert info['patch_2d']['patch_vertices'][i] != None 88 | use_patch_vertices_uv = info['patch_2d']['patch_vertices'][i][area_str][:,:2] 89 | use_patch_classname = local_classidx_list[i] 90 | use_patch = patch_book[use_patch_classname] 91 | patch_use = use_patch 92 | patch_h, patch_w = patch_use.shape[-2:] 93 | img_h, img_w = image.shape[-2:] 94 | 95 | if len(use_patch_vertices_uv.unique()) <= 4: 96 | # 低级抠图 97 | u1 = int(use_patch_vertices_uv[:,0].min()) 98 | u2 = int(use_patch_vertices_uv[:,0].max()) 99 | v1 = int(use_patch_vertices_uv[:,1].min()) 100 | v2 = int(use_patch_vertices_uv[:,1].max()) 101 | 102 | need_patch_h = v2 - v1 103 | need_patch_w = u2 - u1 104 | 105 | if need_patch_h > 0 and need_patch_w > 0: 106 | patch_use = F.interpolate(patch_use[None], (need_patch_h, need_patch_w), mode='bilinear', align_corners=False)[0] 107 | image = safe_img_patch_apply( 108 | image=image, 109 | patch=patch_use, 110 | x1=u1, 111 | y1=v1 112 | ) 113 | else: 114 | # 高级变换 115 | 116 | patch_pad = F.pad( 117 | patch_use, (0, img_w-patch_w, 0, img_h-patch_h), 118 | ) 119 | patch_mask_pad = torch.zeros_like(image) 120 | patch_mask_pad[:,:patch_h,:patch_w] = 1 121 | startpoints = [ 122 | (0, 0), 123 | (patch_w-1, 0), 124 | (patch_w-1, patch_h-1), 125 | (0, patch_h-1), 126 | ] 127 | endpoints = use_patch_vertices_uv 128 | patch_pad_trans = transforms.functional.perspective( 129 | patch_pad, 130 | startpoints=startpoints, 131 | endpoints=endpoints 132 | ) 133 | patch_mask_pad_trans = transforms.functional.perspective( 134 | patch_mask_pad, 135 | startpoints=startpoints, 136 | endpoints=endpoints 137 | ) 138 | image = torch.where(patch_mask_pad_trans > 0.5, patch_pad_trans, image) 139 | return image 140 | 141 | 142 | 143 | def apply_patches_by_info_ovlp(info, image, patch_book, area_str): 144 | local_classname_list = info['objects_info']['class_name'] 145 | local_classidx_list = info['objects_info']['class_idx'] 146 | 147 | local_ovlp_flag = info['objects_info']['is_overlap'] 148 | 149 | # 重新排序，按照画面深度uvd的d排序，以保证从远到近粘贴 150 | _d = info['objects_info']['uvd_center'][:, 2] 151 | d_index = _d.sort(descending=True)[1] 152 | 153 | # 需要判断，当前的obj是ovlp的第几个，粘贴对应的patch 154 | 155 | d_index_list = d_index.tolist() 156 | 157 | for i in d_index_list: 158 | if info['patch_visible_bigger'][i] and local_ovlp_flag[i]: # 可见 且 重叠 159 | assert info['patch_3d']['patch_vertices'][i] != None 160 | use_patch_vertices_uv = info['patch_3d']['patch_vertices'][i][area_str][:,:2] 161 | # 需要判断，当前的obj是ovlp的第几个，粘贴对应的patch 162 | use_patch_idx = local_ovlp_flag[:(i+1)].sum() - 1 163 | use_patch = patch_book[use_patch_idx] 164 | patch_use = use_patch 165 | patch_h, patch_w = patch_use.shape[-2:] 166 | img_h, img_w = image.shape[-2:] 167 | 168 | if len(use_patch_vertices_uv.unique()) <= 4: 169 | # 低级抠图 170 | u1 = int(use_patch_vertices_uv[:,0].min()) 171 | u2 = int(use_patch_vertices_uv[:,0].max()) 172 | v1 = int(use_patch_vertices_uv[:,1].min()) 173 | v2 = int(use_patch_vertices_uv[:,1].max()) 174 | 175 | need_patch_h = v2 - v1 176 | need_patch_w = u2 - u1 177 | 178 | if need_patch_h > 0 and need_patch_w > 0: 179 | patch_use = F.interpolate(patch_use[None], (need_patch_h, need_patch_w), mode='bilinear', align_corners=False)[0] 180 | image = safe_img_patch_apply( 181 | image=image, 182 | patch=patch_use, 183 | x1=u1, 184 | y1=v1 185 | ) 186 | else: 187 | # 高级变换 188 | 189 | patch_pad = F.pad( 190 | patch_use, (0, img_w-patch_w, 0, img_h-patch_h), 191 | ) 192 | patch_mask_pad = torch.zeros_like(image) 193 | patch_mask_pad[:,:patch_h,:patch_w] = 1 194 | startpoints = [ 195 | (0, 0), 196 | (patch_w-1, 0), 197 | (patch_w-1, patch_h-1), 198 | (0, patch_h-1), 199 | ] 200 | endpoints = use_patch_vertices_uv 201 | patch_pad_trans = transforms.functional.perspective( 202 | patch_pad, 203 | startpoints=startpoints, 204 | endpoints=endpoints 205 | ) 206 | patch_mask_pad_trans = transforms.functional.perspective( 207 | patch_mask_pad, 208 | startpoints=startpoints, 209 | endpoints=endpoints 210 | ) 211 | image = torch.where(patch_mask_pad_trans > 0.5, patch_pad_trans, image) 212 | return image 213 | 214 | 215 | 216 | 217 | def apply_patches_by_info_4side(info, image, instance_token_book, patch_book_4side, area_str): 218 | 219 | # 重新排序，按照画面深度uvd的d排序，以保证从远到近粘贴 220 | _d = info['objects_info']['uvd_center'][:, 2] 221 | d_index = _d.sort(descending=True)[1] 222 | 223 | d_index_list = d_index.tolist() 224 | 225 | for i_obj in d_index_list: 226 | if info['patch_visible_bigger'][i_obj]: # 可见 227 | assert info['patch_3d_temporal']['patch_vertices'][i_obj] != None 228 | use_patch_vertices_uv_4side = info['patch_3d_temporal']['patch_vertices'][i_obj][area_str] 229 | use_patch_vertices_uv_side_visible = info['patch_3d_temporal']['boxside_visiblie'][i_obj] 230 | 231 | for side_j in range(4): 232 | try: 233 | if use_patch_vertices_uv_side_visible[side_j]: 234 | 235 | use_patch_instance_token = info['objects_info']['instance_tokens'][i_obj] 236 | # 注意！ 每一个token对应4个patch，所以， 237 | # 找到index之后要*4， 238 | # 再加 0,1,2,3 239 | patch_4_start_idx = instance_token_book.index(use_patch_instance_token) * 4 240 | use_patch_vertices_uv = use_patch_vertices_uv_4side[side_j][:,:2].cuda() 241 | 242 | use_patch = patch_book_4side[patch_4_start_idx + side_j] 243 | patch_use = use_patch 244 | patch_h, patch_w = torch.tensor(patch_use.shape[-2:]).cuda().long() 245 | img_h, img_w = torch.tensor(image.shape[-2:]).cuda().long() 246 | 247 | if len(use_patch_vertices_uv.unique()) <= 4: 248 | # 低级抠图 249 | u1 = (use_patch_vertices_uv[:,0].min()).long() 250 | u2 = (use_patch_vertices_uv[:,0].max()).long() 251 | v1 = (use_patch_vertices_uv[:,1].min()).long() 252 | v2 = (use_patch_vertices_uv[:,1].max()).long() 253 | 254 | need_patch_h = v2 - v1 255 | need_patch_w = u2 - u1 256 | 257 | if need_patch_h > 0 and need_patch_w > 0: 258 | patch_use = F.interpolate(patch_use[None], (need_patch_h, need_patch_w), mode='bilinear', align_corners=False)[0] 259 | image = safe_img_patch_apply( 260 | image=image, 261 | patch=patch_use, 262 | x1=u1, 263 | y1=v1 264 | ) 265 | else: 266 | endpoints = use_patch_vertices_uv 267 | # 判断 endpoints 是否都在图像中，如果是，使用节约显存版本 268 | # 留一点边 269 | edge_width = 2 270 | u_in = (edge_width < endpoints[:,0]) * (endpoints[:,0] < img_w - edge_width - 1) 271 | v_in = (edge_width < endpoints[:,1]) * (endpoints[:,1] < img_h - edge_width - 1) 272 | 273 | endpoints_allinimage = u_in.all() * v_in.all() 274 | 275 | 276 | need_old_version = ~endpoints_allinimage 277 | 278 | if need_old_version == False: 279 | # 高级变换 (节约显存版本) 280 | # 思路： 只使用小图像进行变换，变换后抠图粘到大图中 281 | u_min = endpoints[:,0].min() 282 | u_max = endpoints[:,0].max() 283 | v_min = endpoints[:,1].min() 284 | v_max = endpoints[:,1].max() 285 | 286 | u_min_int = (u_min.floor() - 1).long() 287 | u_max_int = (u_max.ceil() + 1).long() 288 | v_min_int = (v_min.floor() - 1).long() 289 | v_max_int = (v_max.ceil() + 1).long() 290 | 291 | small_region_w = u_max_int - u_min_int 292 | small_region_h = v_max_int - v_min_int 293 | 294 | # patch粘贴区域，可能比patch更大，也可能更小，我们取大值进行绘图 295 | 296 | small_region_w_share = max(small_region_w, patch_w) 297 | small_region_h_share = max(small_region_h, patch_h) 298 | u_max_int_real = u_min_int + small_region_w_share 299 | v_max_int_real = v_min_int + small_region_h_share 300 | 301 | patch_pad = patch_use.new_zeros(3, small_region_h_share, small_region_w_share) 302 | patch_pad[:,:patch_h,:patch_w] = patch_use 303 | 304 | # 注意，如果patch更大，贴回到原图的时候，上下左右边界就都不同了，要重新计算 305 | 306 | # 注意，如果patch更大，可能patch左上角对齐粘贴处，尺寸就又超出去了，需要回到老方法 307 | need_old_version = False 308 | if u_max_int_real > img_w - edge_width - 1 or v_max_int_real > img_h - edge_width - 1: 309 | need_old_version = True 310 | 311 | 312 | if need_old_version == False: 313 | patch_mask_pad = torch.zeros_like(patch_pad) 314 | patch_mask_pad[:,:patch_h,:patch_w] = 1 315 | startpoints = torch.cuda.FloatTensor([ 316 | (0, 0), 317 | (patch_w-1, 0), 318 | (patch_w-1, patch_h-1), 319 | (0, patch_h-1), 320 | ]) 321 | 322 | endpoints_small_region = torch.zeros_like(endpoints) 323 | endpoints_small_region[:,0] = endpoints[:,0] - u_min_int 324 | endpoints_small_region[:,1] = endpoints[:,1] - v_min_int 325 | 326 | patch_pad_trans = transforms.functional.perspective( 327 | patch_pad, 328 | startpoints=startpoints.cpu(), 329 | endpoints=endpoints_small_region.cpu() 330 | ) # 输出居然有可能 = 1 + 1.19e-7 大于1了！ 331 | patch_mask_pad_trans = transforms.functional.perspective( 332 | patch_mask_pad, 333 | startpoints=startpoints.cpu(), 334 | endpoints=endpoints_small_region.cpu() 335 | ) 336 | small_region = torch.where( 337 | patch_mask_pad_trans > 0.5, 338 | patch_pad_trans, 339 | image[:, v_min_int:v_max_int_real, u_min_int:u_max_int_real] 340 | ) 341 | 342 | image[:, v_min_int:v_max_int_real, u_min_int:u_max_int_real] = small_region.clamp(0,1) 343 | else: 344 | # 高级变换 (原始版本) 345 | patch_pad = F.pad( 346 | patch_use, (0, img_w-patch_w, 0, img_h-patch_h), 347 | ) 348 | patch_mask_pad = torch.zeros_like(image) 349 | patch_mask_pad[:,:patch_h,:patch_w] = 1 350 | startpoints = torch.cuda.FloatTensor([ 351 | (0, 0), 352 | (patch_w-1, 0), 353 | (patch_w-1, patch_h-1), 354 | (0, patch_h-1), 355 | ]) 356 | endpoints = use_patch_vertices_uv 357 | patch_pad_trans = transforms.functional.perspective( 358 | patch_pad, 359 | startpoints=startpoints.cpu(), 360 | endpoints=endpoints.cpu() 361 | ) 362 | patch_mask_pad_trans = transforms.functional.perspective( 363 | patch_mask_pad, 364 | startpoints=startpoints.cpu(), 365 | endpoints=endpoints.cpu() 366 | ) 367 | image = torch.where(patch_mask_pad_trans > 0.5, patch_pad_trans, image).clamp(0,1) 368 | except: 369 | print('error happend, skip this side\'s patch') 370 | return image 371 | 372 | 373 | -------------------------------------------------------------------------------- /detr3d_changes.patch: -------------------------------------------------------------------------------- 1 | diff --git a/extend/__init__.py b/extend/__init__.py 2 | new file mode 100644 3 | index 0000000..e69de29 4 | diff --git a/extend/custom_func.py b/extend/custom_func.py 5 | new file mode 100644 6 | index 0000000..85e248f 7 | --- /dev/null 8 | +++ b/extend/custom_func.py 9 | @@ -0,0 +1,85 @@ 10 | +# only for detr3d 11 | + 12 | +import torch 13 | +import torchvision 14 | +import torchvision.transforms as transforms 15 | +import torch.nn.functional as F 16 | +from torchvision.utils import save_image 17 | +import copy 18 | + 19 | +def custom_data_preprocess(data): 20 | + for _key in data: 21 | + data[_key] = data[_key][0] 22 | + return data 23 | + 24 | +def custom_data_postprocess_eval(data): 25 | + data.pop('gt_bboxes_3d', None) 26 | + data.pop('gt_labels_3d', None) 27 | + for _key in data: 28 | + data[_key] = [data[_key]] 29 | + return data 30 | + 31 | +def custom_data_work(data): 32 | + metas = data['img_metas']._data[0][0] 33 | + img_path_list = metas['filename'] 34 | + img_org_np = metas['img_org'] 35 | + img_processed = data['img']._data[0].clone() 36 | + gt_labels_3d = data['gt_labels_3d']._data[0][0] 37 | + return metas, img_path_list, img_org_np, img_processed, gt_labels_3d 38 | + 39 | + 40 | + 41 | +def custom_result_postprocess(result): 42 | + result[0]['pts_bbox']['boxes_3d'].tensor = result[0]['pts_bbox']['boxes_3d'].tensor.cpu() 43 | + result[0]['pts_bbox']['scores_3d'] = result[0]['pts_bbox']['scores_3d'].cpu() 44 | + result[0]['pts_bbox']['labels_3d'] = result[0]['pts_bbox']['labels_3d'].cpu() 45 | + return result 46 | + 47 | + 48 | +def custom_img_read_from_img_org(img_org_np, device): 49 | + img_org_np_255_bgr_hwcn_uint8 = img_org_np # mmcv 读取 BGR 转 numpy 50 | + img_org_tensor_bgr_255_hwcn = torch.from_numpy(img_org_np_255_bgr_hwcn_uint8).float() 51 | + img_org_tensor_bgr_255 = img_org_tensor_bgr_255_hwcn.permute(3,2,0,1) 52 | + img_org_tensor_bgr = (img_org_tensor_bgr_255/255.).to(device) # 6chw 53 | + img_org_tensor_rgb = img_org_tensor_bgr[:,[2,1,0]] 54 | + img_tensor_rgb_6chw_0to1 = img_org_tensor_rgb 55 | + return img_tensor_rgb_6chw_0to1 56 | + 57 | + 58 | +def custom_differentiable_transform(img_tensor_rgb_6chw_0to1, img_metas): 59 | + 60 | + """Alternative Data Preparation for Original Model 61 | + 62 | + Args: 63 | + img_tensor (torch.tensor): (6xCxHxW), tensors of original imgs 64 | + """ 65 | + 66 | + assert len(img_tensor_rgb_6chw_0to1.shape) == 4 67 | + assert img_tensor_rgb_6chw_0to1.shape[0] == 6 68 | + assert img_tensor_rgb_6chw_0to1.shape[1] == 3 69 | + assert img_tensor_rgb_6chw_0to1.max() <= 1. 70 | + assert img_tensor_rgb_6chw_0to1.min() >= 0. 71 | + assert img_tensor_rgb_6chw_0to1.dtype == torch.float32 72 | + assert img_tensor_rgb_6chw_0to1.is_cuda 73 | + device = img_tensor_rgb_6chw_0to1.device 74 | + 75 | + # detr3d 用的是 bgr 76 | + # {'mean': array([103.53 , 116.28 , 123.675], dtype=float32), 77 | + # 'std': array([1., 1., 1.], dtype=float32), 'to_rgb': False} 78 | + img_tensor_255 = img_tensor_rgb_6chw_0to1[:,[2,1,0]] * 255. 79 | + 80 | + mean_tensor = torch.tensor(img_metas['img_norm_cfg']['mean'],dtype=torch.float32).unsqueeze(0).unsqueeze(-1).unsqueeze(-1).to(device) 81 | + std_tensor = torch.tensor(img_metas['img_norm_cfg']['std'], dtype=torch.float32).unsqueeze(0).unsqueeze(-1).unsqueeze(-1).to(device) 82 | + normalized_img_tensor = (img_tensor_255 - mean_tensor) / std_tensor 83 | + # for detr3d, there is no resize operation 84 | + img_h, img_w = img_metas['ori_shape'][:2] 85 | + pad_h, pad_w = img_metas['pad_shape'][0][:2] 86 | + PadModule = torch.nn.ZeroPad2d(padding=(0, pad_w-img_w, 0, pad_h-img_h)) 87 | + padded_img_tensor = PadModule(normalized_img_tensor.unsqueeze(0)) 88 | + return padded_img_tensor 89 | + 90 | + 91 | +def custom_image_data_give(data, image_ready): 92 | + data_copy = copy.deepcopy(data) 93 | + data_copy['img']._data[0] = image_ready 94 | + return data_copy 95 | diff --git a/extend_common b/extend_common 96 | new file mode 120000 97 | index 0000000..0633dd9 98 | --- /dev/null 99 | +++ b/extend_common 100 | @@ -0,0 +1 @@ 101 | +../extend_common/ 102 | \ No newline at end of file 103 | diff --git a/projects/__init__.py b/projects/__init__.py 104 | new file mode 100644 105 | index 0000000..e69de29 106 | diff --git a/projects/configs/detr3d/detr3d_res101_gridmask_adv.py b/projects/configs/detr3d/detr3d_res101_gridmask_adv.py 107 | new file mode 100644 108 | index 0000000..9ffda2d 109 | --- /dev/null 110 | +++ b/projects/configs/detr3d/detr3d_res101_gridmask_adv.py 111 | @@ -0,0 +1,243 @@ 112 | +_base_ = [ 113 | + '../../../mmdetection3d/configs/_base_/datasets/nus-3d.py', 114 | + '../../../mmdetection3d/configs/_base_/default_runtime.py' 115 | +] 116 | + 117 | +# add fp16 setting for save GPU memory 118 | +fp16 = dict(loss_scale=32.) 119 | + 120 | +plugin=True 121 | +plugin_dir='projects/mmdet3d_plugin/' 122 | + 123 | +# If point cloud range is changed, the models should also change their point 124 | +# cloud range accordingly 125 | +point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0] 126 | +voxel_size = [0.2, 0.2, 8] 127 | + 128 | +img_norm_cfg = dict( 129 | + mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) 130 | +# For nuScenes we usually do 10-class detection 131 | +class_names = [ 132 | + 'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier', 133 | + 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone' 134 | +] 135 | + 136 | +input_modality = dict( 137 | + use_lidar=False, 138 | + use_camera=True, 139 | + use_radar=False, 140 | + use_map=False, 141 | + use_external=False) 142 | + 143 | +model = dict( 144 | + type='Detr3D', 145 | + use_grid_mask=True, 146 | + img_backbone=dict( 147 | + type='ResNet', 148 | + depth=101, 149 | + num_stages=4, 150 | + out_indices=(0, 1, 2, 3), 151 | + frozen_stages=1, 152 | + norm_cfg=dict(type='BN2d', requires_grad=False), 153 | + norm_eval=True, 154 | + style='caffe', 155 | + dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False), 156 | + stage_with_dcn=(False, False, True, True)), 157 | + img_neck=dict( 158 | + type='FPN', 159 | + in_channels=[256, 512, 1024, 2048], 160 | + out_channels=256, 161 | + start_level=1, 162 | + add_extra_convs='on_output', 163 | + num_outs=4, 164 | + relu_before_extra_convs=True), 165 | + pts_bbox_head=dict( 166 | + type='Detr3DHead', 167 | + num_query=900, 168 | + num_classes=10, 169 | + in_channels=256, 170 | + sync_cls_avg_factor=True, 171 | + with_box_refine=True, 172 | + as_two_stage=False, 173 | + transformer=dict( 174 | + type='Detr3DTransformer', 175 | + decoder=dict( 176 | + type='Detr3DTransformerDecoder', 177 | + num_layers=6, 178 | + return_intermediate=True, 179 | + transformerlayers=dict( 180 | + type='DetrTransformerDecoderLayer', 181 | + attn_cfgs=[ 182 | + dict( 183 | + type='MultiheadAttention', 184 | + embed_dims=256, 185 | + num_heads=8, 186 | + dropout=0.1), 187 | + dict( 188 | + type='Detr3DCrossAtten', 189 | + pc_range=point_cloud_range, 190 | + num_points=1, 191 | + embed_dims=256) 192 | + ], 193 | + feedforward_channels=512, 194 | + ffn_dropout=0.1, 195 | + operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 196 | + 'ffn', 'norm')))), 197 | + bbox_coder=dict( 198 | + type='NMSFreeCoder', 199 | + post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], 200 | + pc_range=point_cloud_range, 201 | + max_num=300, 202 | + voxel_size=voxel_size, 203 | + num_classes=10), 204 | + positional_encoding=dict( 205 | + type='SinePositionalEncoding', 206 | + num_feats=128, 207 | + normalize=True, 208 | + offset=-0.5), 209 | + loss_cls=dict( 210 | + type='FocalLoss', 211 | + use_sigmoid=True, 212 | + gamma=2.0, 213 | + alpha=0.25, 214 | + loss_weight=2.0), 215 | + loss_bbox=dict(type='L1Loss', loss_weight=0.25), 216 | + loss_iou=dict(type='GIoULoss', loss_weight=0.0)), 217 | + # model training and testing settings 218 | + train_cfg=dict(pts=dict( 219 | + grid_size=[512, 512, 1], 220 | + voxel_size=voxel_size, 221 | + point_cloud_range=point_cloud_range, 222 | + out_size_factor=4, 223 | + assigner=dict( 224 | + type='HungarianAssigner3D', 225 | + cls_cost=dict(type='FocalLossCost', weight=2.0), 226 | + reg_cost=dict(type='BBox3DL1Cost', weight=0.25), 227 | + iou_cost=dict(type='IoUCost', weight=0.0), # Fake cost. This is just to make it compatible with DETR head. 228 | + pc_range=point_cloud_range)))) 229 | + 230 | +dataset_type = 'CustomNuScenesDataset' 231 | +data_root = 'data/nuscenes/' 232 | + 233 | +file_client_args = dict(backend='disk') 234 | + 235 | +db_sampler = dict( 236 | + data_root=data_root, 237 | + info_path=data_root + 'nuscenes_dbinfos_train.pkl', 238 | + rate=1.0, 239 | + prepare=dict( 240 | + filter_by_difficulty=[-1], 241 | + filter_by_min_points=dict( 242 | + car=5, 243 | + truck=5, 244 | + bus=5, 245 | + trailer=5, 246 | + construction_vehicle=5, 247 | + traffic_cone=5, 248 | + barrier=5, 249 | + motorcycle=5, 250 | + bicycle=5, 251 | + pedestrian=5)), 252 | + classes=class_names, 253 | + sample_groups=dict( 254 | + car=2, 255 | + truck=3, 256 | + construction_vehicle=7, 257 | + bus=4, 258 | + trailer=6, 259 | + barrier=2, 260 | + motorcycle=6, 261 | + bicycle=6, 262 | + pedestrian=2, 263 | + traffic_cone=2), 264 | + points_loader=dict( 265 | + type='LoadPointsFromFile', 266 | + coord_type='LIDAR', 267 | + load_dim=5, 268 | + use_dim=[0, 1, 2, 3, 4], 269 | + file_client_args=file_client_args)) 270 | + 271 | +train_pipeline = [ 272 | + dict(type='LoadMultiViewImageFromFiles', to_float32=True), 273 | + dict(type='PhotoMetricDistortionMultiViewImage'), 274 | + dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_attr_label=False), 275 | + dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), 276 | + dict(type='ObjectNameFilter', classes=class_names), 277 | + dict(type='NormalizeMultiviewImage', **img_norm_cfg), 278 | + dict(type='PadMultiViewImage', size_divisor=32), 279 | + dict(type='DefaultFormatBundle3D', class_names=class_names), 280 | + dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img']) 281 | +] 282 | +test_pipeline = [ 283 | + dict(type='LoadMultiViewImageFromFilesImgOrg', to_float32=True), 284 | + dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_attr_label=False), 285 | + dict(type='NormalizeMultiviewImage', **img_norm_cfg), 286 | + dict(type='PadMultiViewImage', size_divisor=32), 287 | + dict( 288 | + type='MultiScaleFlipAug3D', 289 | + img_scale=(1333, 800), 290 | + pts_scale_ratio=1, 291 | + flip=False, 292 | + transforms=[ 293 | + dict( 294 | + type='DefaultFormatBundle3D', 295 | + class_names=class_names, 296 | + with_label=True), 297 | + dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img'], 298 | + meta_keys=('filename', 'ori_shape', 'img_shape', 'lidar2img', 299 | + 'depth2img', 'cam2img', 'pad_shape', 300 | + 'scale_factor', 'flip', 'pcd_horizontal_flip', 301 | + 'pcd_vertical_flip', 'box_mode_3d', 'box_type_3d', 302 | + 'img_norm_cfg', 'pcd_trans', 'sample_idx', 303 | + 'pcd_scale_factor', 'pcd_rotation', 'pts_filename', 304 | + 'transformation_3d_flow', 305 | + 'img_org' 306 | + ) 307 | + ) 308 | + ]) 309 | +] 310 | + 311 | + 312 | +data = dict( 313 | + samples_per_gpu=1, 314 | + workers_per_gpu=4, 315 | + train=dict( 316 | + type=dataset_type, 317 | + data_root=data_root, 318 | + ann_file=data_root + 'nuscenes_infos_train.pkl', 319 | + pipeline=train_pipeline, 320 | + classes=class_names, 321 | + modality=input_modality, 322 | + test_mode=False, 323 | + use_valid_flag=True, 324 | + # we use box_type_3d='LiDAR' in kitti and nuscenes dataset 325 | + # and box_type_3d='Depth' in sunrgbd and scannet dataset. 326 | + box_type_3d='LiDAR'), 327 | + val=dict( 328 | + type=dataset_type, 329 | + pipeline=test_pipeline, classes=class_names, modality=input_modality), 330 | + test=dict( 331 | + type=dataset_type, 332 | + pipeline=test_pipeline, classes=class_names, modality=input_modality)) 333 | + 334 | +optimizer = dict( 335 | + type='AdamW', 336 | + lr=2e-4, 337 | + paramwise_cfg=dict( 338 | + custom_keys={ 339 | + 'img_backbone': dict(lr_mult=0.1), 340 | + }), 341 | + weight_decay=0.01) 342 | +optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) 343 | +# learning policy 344 | +lr_config = dict( 345 | + policy='CosineAnnealing', 346 | + warmup='linear', 347 | + warmup_iters=500, 348 | + warmup_ratio=1.0 / 3, 349 | + min_lr_ratio=1e-3) 350 | +total_epochs = 24 351 | +evaluation = dict(interval=2, pipeline=test_pipeline) 352 | + 353 | +runner = dict(type='EpochBasedRunner', max_epochs=total_epochs) 354 | +load_from='ckpts/fcos3d.pth' 355 | diff --git a/projects/mmdet3d_plugin/apis_common b/projects/mmdet3d_plugin/apis_common 356 | new file mode 120000 357 | index 0000000..d498da4 358 | --- /dev/null 359 | +++ b/projects/mmdet3d_plugin/apis_common 360 | @@ -0,0 +1 @@ 361 | +../../../apis_common/ 362 | \ No newline at end of file 363 | diff --git a/projects/mmdet3d_plugin/datasets/nuscenes_dataset.py b/projects/mmdet3d_plugin/datasets/nuscenes_dataset.py 364 | index 1ec4cf2..02514dd 100644 365 | --- a/projects/mmdet3d_plugin/datasets/nuscenes_dataset.py 366 | +++ b/projects/mmdet3d_plugin/datasets/nuscenes_dataset.py 367 | @@ -1,6 +1,9 @@ 368 | import numpy as np 369 | from mmdet.datasets import DATASETS 370 | from mmdet3d.datasets import NuScenesDataset 371 | +import orjson 372 | +import time 373 | +import json 374 | 375 | 376 | @DATASETS.register_module() 377 | @@ -72,5 +75,89 @@ class CustomNuScenesDataset(NuScenesDataset): 378 | if not self.test_mode: 379 | annos = self.get_ann_info(index) 380 | input_dict['ann_info'] = annos 381 | + else: 382 | + annos = self.get_ann_info(index) 383 | + input_dict['ann_info'] = annos 384 | 385 | return input_dict 386 | + 387 | + 388 | + def _format_bbox(self, results, jsonfile_prefix=None): 389 | + """Convert the results to the standard format. 390 | + 391 | + Args: 392 | + results (list[dict]): Testing results of the dataset. 393 | + jsonfile_prefix (str): The prefix of the output jsonfile. 394 | + You can specify the output directory/filename by 395 | + modifying the jsonfile_prefix. Default: None. 396 | + 397 | + Returns: 398 | + str: Path of the output json file. 399 | + """ 400 | + nusc_annos = {} 401 | + mapped_class_names = self.CLASSES 402 | + 403 | + 404 | + 405 | + from mmdet3d.datasets.nuscenes_dataset import output_to_nusc_box,lidar_nusc_box_to_global 406 | + 407 | + print('Start to convert detection format...') 408 | + for sample_id, det in enumerate(mmcv.track_iter_progress(results)): 409 | + annos = [] 410 | + boxes = output_to_nusc_box(det) 411 | + sample_token = self.data_infos[sample_id]['token'] 412 | + boxes = lidar_nusc_box_to_global(self.data_infos[sample_id], boxes, 413 | + mapped_class_names, 414 | + self.eval_detection_configs, 415 | + self.eval_version) 416 | + for i, box in enumerate(boxes): 417 | + name = mapped_class_names[box.label] 418 | + if np.sqrt(box.velocity[0]**2 + box.velocity[1]**2) > 0.2: 419 | + if name in [ 420 | + 'car', 421 | + 'construction_vehicle', 422 | + 'bus', 423 | + 'truck', 424 | + 'trailer', 425 | + ]: 426 | + attr = 'vehicle.moving' 427 | + elif name in ['bicycle', 'motorcycle']: 428 | + attr = 'cycle.with_rider' 429 | + else: 430 | + attr = NuScenesDataset.DefaultAttribute[name] 431 | + else: 432 | + if name in ['pedestrian']: 433 | + attr = 'pedestrian.standing' 434 | + elif name in ['bus']: 435 | + attr = 'vehicle.stopped' 436 | + else: 437 | + attr = NuScenesDataset.DefaultAttribute[name] 438 | + 439 | + nusc_anno = dict( 440 | + sample_token=sample_token, 441 | + translation=box.center.tolist(), 442 | + size=box.wlh.tolist(), 443 | + rotation=box.orientation.elements.tolist(), 444 | + velocity=box.velocity[:2].tolist(), 445 | + detection_name=name, 446 | + detection_score=box.score, 447 | + attribute_name=attr) 448 | + annos.append(nusc_anno) 449 | + nusc_annos[sample_token] = annos 450 | + nusc_submissions = { 451 | + 'meta': self.modality, 452 | + 'results': nusc_annos, 453 | + } 454 | + 455 | + mmcv.mkdir_or_exist(jsonfile_prefix) 456 | + res_path = osp.join(jsonfile_prefix, 'results_nusc.json') 457 | + # print('Results writes to', res_path) 458 | + # mmcv.dump(nusc_submissions, res_path) 459 | + 460 | + print('Results writes to', res_path,'by orjson') 461 | + start = time.time() 462 | + with open(res_path, "wb") as f: 463 | + f.write(orjson.dumps(nusc_submissions)) 464 | + print("by orjson in", time.time()-start,'s') 465 | + 466 | + return res_path 467 | \ No newline at end of file 468 | diff --git a/projects/mmdet3d_plugin/datasets/pipelines/__init__.py b/projects/mmdet3d_plugin/datasets/pipelines/__init__.py 469 | index a246a61..228772c 100755 470 | --- a/projects/mmdet3d_plugin/datasets/pipelines/__init__.py 471 | +++ b/projects/mmdet3d_plugin/datasets/pipelines/__init__.py 472 | @@ -3,6 +3,7 @@ from .transform_3d import ( 473 | PhotoMetricDistortionMultiViewImage, CropMultiViewImage, 474 | RandomScaleImageMultiViewImage, 475 | HorizontalRandomFlipMultiViewImage) 476 | +from .loading import LoadMultiViewImageFromFilesImgOrg 477 | 478 | __all__ = [ 479 | 'PadMultiViewImage', 'NormalizeMultiviewImage', 480 | diff --git a/projects/mmdet3d_plugin/datasets/pipelines/loading.py b/projects/mmdet3d_plugin/datasets/pipelines/loading.py 481 | new file mode 100644 482 | index 0000000..0764550 483 | --- /dev/null 484 | +++ b/projects/mmdet3d_plugin/datasets/pipelines/loading.py 485 | @@ -0,0 +1,73 @@ 486 | +# Copyright (c) OpenMMLab. All rights reserved. 487 | +import mmcv 488 | +import numpy as np 489 | + 490 | +from mmdet3d.core.points import BasePoints, get_points_type 491 | +from mmdet.datasets.builder import PIPELINES 492 | +from mmdet.datasets.pipelines import LoadAnnotations, LoadImageFromFile 493 | + 494 | + 495 | +# copy from mmdet3d LoadMultiViewImageFromFiles 496 | +@PIPELINES.register_module() 497 | +class LoadMultiViewImageFromFilesImgOrg(object): 498 | + """Load multi channel images from a list of separate channel files. 499 | + 500 | + Expects results['img_filename'] to be a list of filenames. 501 | + 502 | + Args: 503 | + to_float32 (bool): Whether to convert the img to float32. 504 | + Defaults to False. 505 | + color_type (str): Color type of the file. Defaults to 'unchanged'. 506 | + """ 507 | + 508 | + def __init__(self, to_float32=False, color_type='unchanged'): 509 | + self.to_float32 = to_float32 510 | + self.color_type = color_type 511 | + 512 | + def __call__(self, results): 513 | + """Call function to load multi-view image from files. 514 | + 515 | + Args: 516 | + results (dict): Result dict containing multi-view image filenames. 517 | + 518 | + Returns: 519 | + dict: The result dict containing the multi-view image data. \ 520 | + Added keys and values are described below. 521 | + 522 | + - filename (str): Multi-view image filenames. 523 | + - img (np.ndarray): Multi-view image arrays. 524 | + - img_shape (tuple[int]): Shape of multi-view image arrays. 525 | + - ori_shape (tuple[int]): Shape of original image arrays. 526 | + - pad_shape (tuple[int]): Shape of padded image arrays. 527 | + - scale_factor (float): Scale factor. 528 | + - img_norm_cfg (dict): Normalization configuration of images. 529 | + """ 530 | + filename = results['img_filename'] 531 | + # img is of shape (h, w, c, num_views) 532 | + img = np.stack( 533 | + [mmcv.imread(name, self.color_type) for name in filename], axis=-1) 534 | + if self.to_float32: 535 | + img = img.astype(np.float32) 536 | + results['filename'] = filename 537 | + # unravel to list, see `DefaultFormatBundle` in formating.py 538 | + # which will transpose each image separately and then stack into array 539 | + results['img'] = [img[..., i] for i in range(img.shape[-1])] 540 | + results['img_shape'] = img.shape 541 | + results['ori_shape'] = img.shape 542 | + # Set initial values for default meta_keys 543 | + results['pad_shape'] = img.shape 544 | + results['scale_factor'] = 1.0 545 | + num_channels = 1 if len(img.shape) < 3 else img.shape[2] 546 | + results['img_norm_cfg'] = dict( 547 | + mean=np.zeros(num_channels, dtype=np.float32), 548 | + std=np.ones(num_channels, dtype=np.float32), 549 | + to_rgb=False) 550 | + results['img_org'] = img 551 | + return results 552 | + 553 | + def __repr__(self): 554 | + """str: Return a string that describes the module.""" 555 | + repr_str = self.__class__.__name__ 556 | + repr_str += f'(to_float32={self.to_float32}, ' 557 | + repr_str += f"color_type='{self.color_type}')" 558 | + return repr_str 559 | diff --git a/tools/test_fgsm_img_launcher.py b/tools/test_fgsm_img_launcher.py 560 | new file mode 100755 561 | index 0000000..7399ebb 562 | --- /dev/null 563 | +++ b/tools/test_fgsm_img_launcher.py 564 | @@ -0,0 +1,254 @@ 565 | +# Copyright (c) OpenMMLab. All rights reserved. 566 | +import argparse 567 | +import mmcv 568 | +import os 569 | +import sys 570 | +sys_path = os.path.abspath(".") 571 | +sys.path.append(sys_path) 572 | +import torch 573 | +import warnings 574 | +from mmcv import Config, DictAction 575 | +from mmcv.cnn import fuse_conv_bn 576 | +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 577 | +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, 578 | + wrap_fp16_model) 579 | + 580 | +from projects.mmdet3d_plugin.apis_common.test_fgsm_img import single_gpu_test 581 | +from mmdet3d.datasets import build_dataloader, build_dataset 582 | +from mmdet3d.models import build_model 583 | +from mmdet.apis import multi_gpu_test, set_random_seed 584 | +from mmdet.datasets import replace_ImageToTensor 585 | + 586 | + 587 | +def parse_args(): 588 | + parser = argparse.ArgumentParser( 589 | + description='MMDet test (and eval) a model') 590 | + parser.add_argument('config', help='test config file path') 591 | + parser.add_argument('checkpoint', help='checkpoint file') 592 | + parser.add_argument('scattered_result_prefix', help='save scattered_result file dir') 593 | + parser.add_argument('eps255', help='eps of fgsm in 0-255') 594 | + parser.add_argument('--out', help='output result file in pickle format') 595 | + parser.add_argument( 596 | + '--fuse-conv-bn', 597 | + action='store_true', 598 | + help='Whether to fuse conv and bn, this will slightly increase' 599 | + 'the inference speed') 600 | + parser.add_argument( 601 | + '--format-only', 602 | + action='store_true', 603 | + help='Format the output results without perform evaluation. It is' 604 | + 'useful when you want to format the result to a specific format and ' 605 | + 'submit it to the test server') 606 | + parser.add_argument( 607 | + '--eval', 608 | + type=str, 609 | + nargs='+', 610 | + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' 611 | + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') 612 | + parser.add_argument('--show', action='store_true', help='show results') 613 | + parser.add_argument( 614 | + '--show-dir', help='directory where results will be saved') 615 | + parser.add_argument( 616 | + '--gpu-collect', 617 | + action='store_true', 618 | + help='whether to use gpu to collect results.') 619 | + parser.add_argument( 620 | + '--tmpdir', 621 | + help='tmp directory used for collecting results from multiple ' 622 | + 'workers, available when gpu-collect is not specified') 623 | + parser.add_argument('--seed', type=int, default=0, help='random seed') 624 | + parser.add_argument( 625 | + '--deterministic', 626 | + action='store_true', 627 | + help='whether to set deterministic options for CUDNN backend.') 628 | + parser.add_argument( 629 | + '--cfg-options', 630 | + nargs='+', 631 | + action=DictAction, 632 | + help='override some settings in the used config, the key-value pair ' 633 | + 'in xxx=yyy format will be merged into config file. If the value to ' 634 | + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 635 | + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 636 | + 'Note that the quotation marks are necessary and that no white space ' 637 | + 'is allowed.') 638 | + parser.add_argument( 639 | + '--options', 640 | + nargs='+', 641 | + action=DictAction, 642 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 643 | + 'format will be kwargs for dataset.evaluate() function (deprecate), ' 644 | + 'change to --eval-options instead.') 645 | + parser.add_argument( 646 | + '--eval-options', 647 | + nargs='+', 648 | + action=DictAction, 649 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 650 | + 'format will be kwargs for dataset.evaluate() function') 651 | + parser.add_argument( 652 | + '--launcher', 653 | + choices=['none', 'pytorch', 'slurm', 'mpi'], 654 | + default='none', 655 | + help='job launcher') 656 | + parser.add_argument('--local_rank', type=int, default=0) 657 | + args = parser.parse_args() 658 | + if 'LOCAL_RANK' not in os.environ: 659 | + os.environ['LOCAL_RANK'] = str(args.local_rank) 660 | + 661 | + if args.options and args.eval_options: 662 | + raise ValueError( 663 | + '--options and --eval-options cannot be both specified, ' 664 | + '--options is deprecated in favor of --eval-options') 665 | + if args.options: 666 | + warnings.warn('--options is deprecated in favor of --eval-options') 667 | + args.eval_options = args.options 668 | + return args 669 | + 670 | + 671 | +def main(): 672 | + args = parse_args() 673 | + 674 | + # assert args.out or args.eval or args.format_only or args.show \ 675 | + # or args.show_dir, \ 676 | + # ('Please specify at least one operation (save/eval/format/show the ' 677 | + # 'results / save the results) with the argument "--out", "--eval"' 678 | + # ', "--format-only", "--show" or "--show-dir"') 679 | + 680 | + if args.eval and args.format_only: 681 | + raise ValueError('--eval and --format_only cannot be both specified') 682 | + 683 | + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): 684 | + raise ValueError('The output file must be a pkl file.') 685 | + 686 | + cfg = Config.fromfile(args.config) 687 | + if args.cfg_options is not None: 688 | + cfg.merge_from_dict(args.cfg_options) 689 | + # import modules from string list. 690 | + if cfg.get('custom_imports', None): 691 | + from mmcv.utils import import_modules_from_strings 692 | + import_modules_from_strings(**cfg['custom_imports']) 693 | + 694 | + # import modules from plguin/xx, registry will be updated 695 | + if hasattr(cfg, 'plugin'): 696 | + if cfg.plugin: 697 | + import importlib 698 | + if hasattr(cfg, 'plugin_dir'): 699 | + plugin_dir = cfg.plugin_dir 700 | + _module_dir = os.path.dirname(plugin_dir) 701 | + _module_dir = _module_dir.split('/') 702 | + _module_path = _module_dir[0] 703 | + 704 | + for m in _module_dir[1:]: 705 | + _module_path = _module_path + '.' + m 706 | + print(_module_path) 707 | + plg_lib = importlib.import_module(_module_path) 708 | + else: 709 | + # import dir is the dirpath for the config file 710 | + _module_dir = os.path.dirname(args.config) 711 | + _module_dir = _module_dir.split('/') 712 | + _module_path = _module_dir[0] 713 | + for m in _module_dir[1:]: 714 | + _module_path = _module_path + '.' + m 715 | + print(_module_path) 716 | + plg_lib = importlib.import_module(_module_path) 717 | + 718 | + # set cudnn_benchmark 719 | + if cfg.get('cudnn_benchmark', False): 720 | + torch.backends.cudnn.benchmark = True 721 | + 722 | + cfg.model.pretrained = None 723 | + # in case the test dataset is concatenated 724 | + samples_per_gpu = 1 725 | + if isinstance(cfg.data.test, dict): 726 | + cfg.data.test.test_mode = True 727 | + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) 728 | + if samples_per_gpu > 1: 729 | + # Replace 'ImageToTensor' to 'DefaultFormatBundle' 730 | + cfg.data.test.pipeline = replace_ImageToTensor( 731 | + cfg.data.test.pipeline) 732 | + elif isinstance(cfg.data.test, list): 733 | + for ds_cfg in cfg.data.test: 734 | + ds_cfg.test_mode = True 735 | + samples_per_gpu = max( 736 | + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) 737 | + if samples_per_gpu > 1: 738 | + for ds_cfg in cfg.data.test: 739 | + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) 740 | + 741 | + # init distributed env first, since logger depends on the dist info. 742 | + if args.launcher == 'none': 743 | + distributed = False 744 | + else: 745 | + distributed = True 746 | + init_dist(args.launcher, **cfg.dist_params) 747 | + 748 | + # set random seeds 749 | + if args.seed is not None: 750 | + set_random_seed(args.seed, deterministic=args.deterministic) 751 | + 752 | + # build the dataloader 753 | + dataset = build_dataset(cfg.data.test) 754 | + data_loader = build_dataloader( 755 | + dataset, 756 | + samples_per_gpu=samples_per_gpu, 757 | + workers_per_gpu=cfg.data.workers_per_gpu, 758 | + dist=distributed, 759 | + shuffle=False) 760 | + 761 | + # build the model and load checkpoint 762 | + # cfg.model.train_cfg = None 763 | + model = build_model(cfg.model, test_cfg=cfg.get('test_cfg'), train_cfg=cfg.get('train_cfg')) 764 | + fp16_cfg = cfg.get('fp16', None) 765 | + if fp16_cfg is not None: 766 | + wrap_fp16_model(model) 767 | + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') 768 | + if args.fuse_conv_bn: 769 | + model = fuse_conv_bn(model) 770 | + # old versions did not save class info in checkpoints, this walkaround is 771 | + # for backward compatibility 772 | + if 'CLASSES' in checkpoint.get('meta', {}): 773 | + model.CLASSES = checkpoint['meta']['CLASSES'] 774 | + else: 775 | + model.CLASSES = dataset.CLASSES 776 | + # palette for visualization in segmentation tasks 777 | + if 'PALETTE' in checkpoint.get('meta', {}): 778 | + model.PALETTE = checkpoint['meta']['PALETTE'] 779 | + elif hasattr(dataset, 'PALETTE'): 780 | + # segmentation dataset has `PALETTE` attribute 781 | + model.PALETTE = dataset.PALETTE 782 | + 783 | + if not distributed: 784 | + model = MMDataParallel(model, device_ids=[0]) 785 | + outputs = single_gpu_test(model, data_loader, 786 | + args.scattered_result_prefix, 787 | + args.eps255, 788 | + ) 789 | + else: 790 | + model = MMDistributedDataParallel( 791 | + model.cuda(), 792 | + device_ids=[torch.cuda.current_device()], 793 | + broadcast_buffers=False) 794 | + outputs = multi_gpu_test(model, data_loader, args.tmpdir, 795 | + args.gpu_collect) 796 | + 797 | + rank, _ = get_dist_info() 798 | + if rank == 0: 799 | + if args.out: 800 | + print(f'\nwriting results to {args.out}') 801 | + mmcv.dump(outputs, args.out) 802 | + kwargs = {} if args.eval_options is None else args.eval_options 803 | + if args.format_only: 804 | + dataset.format_results(outputs, **kwargs) 805 | + if args.eval: 806 | + eval_kwargs = cfg.get('evaluation', {}).copy() 807 | + # hard-code way to remove EvalHook args 808 | + for key in [ 809 | + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 810 | + 'rule' 811 | + ]: 812 | + eval_kwargs.pop(key, None) 813 | + eval_kwargs.update(dict(metric=args.eval, **kwargs)) 814 | + print(dataset.evaluate(outputs, **eval_kwargs)) 815 | + 816 | + 817 | +if __name__ == '__main__': 818 | + main() 819 | diff --git a/tools/test_patch_class_launcher.py b/tools/test_patch_class_launcher.py 820 | new file mode 100755 821 | index 0000000..4cfb3fd 822 | --- /dev/null 823 | +++ b/tools/test_patch_class_launcher.py 824 | @@ -0,0 +1,256 @@ 825 | +# Copyright (c) OpenMMLab. All rights reserved. 826 | +import argparse 827 | +import mmcv 828 | +import os 829 | +import sys 830 | +sys_path = os.path.abspath(".") 831 | +sys.path.append(sys_path) 832 | +import torch 833 | +import warnings 834 | +from mmcv import Config, DictAction 835 | +from mmcv.cnn import fuse_conv_bn 836 | +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 837 | +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, 838 | + wrap_fp16_model) 839 | + 840 | +from projects.mmdet3d_plugin.apis_common.test_patch_class import single_gpu_test 841 | +from mmdet3d.datasets import build_dataloader, build_dataset 842 | +from mmdet3d.models import build_model 843 | +from mmdet.apis import multi_gpu_test, set_random_seed 844 | +from mmdet.datasets import replace_ImageToTensor 845 | + 846 | + 847 | +def parse_args(): 848 | + parser = argparse.ArgumentParser( 849 | + description='MMDet test (and eval) a model') 850 | + parser.add_argument('config', help='test config file path') 851 | + parser.add_argument('checkpoint', help='checkpoint file') 852 | + parser.add_argument('patch_save_prefix', help='save patch_save file dir') 853 | + parser.add_argument('area_rate_str', help='area rate of patch') 854 | + parser.add_argument('optim_lr', help='optim_lr of attack') 855 | + parser.add_argument('--out', help='output result file in pickle format') 856 | + parser.add_argument( 857 | + '--fuse-conv-bn', 858 | + action='store_true', 859 | + help='Whether to fuse conv and bn, this will slightly increase' 860 | + 'the inference speed') 861 | + parser.add_argument( 862 | + '--format-only', 863 | + action='store_true', 864 | + help='Format the output results without perform evaluation. It is' 865 | + 'useful when you want to format the result to a specific format and ' 866 | + 'submit it to the test server') 867 | + parser.add_argument( 868 | + '--eval', 869 | + type=str, 870 | + nargs='+', 871 | + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' 872 | + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') 873 | + parser.add_argument('--show', action='store_true', help='show results') 874 | + parser.add_argument( 875 | + '--show-dir', help='directory where results will be saved') 876 | + parser.add_argument( 877 | + '--gpu-collect', 878 | + action='store_true', 879 | + help='whether to use gpu to collect results.') 880 | + parser.add_argument( 881 | + '--tmpdir', 882 | + help='tmp directory used for collecting results from multiple ' 883 | + 'workers, available when gpu-collect is not specified') 884 | + parser.add_argument('--seed', type=int, default=0, help='random seed') 885 | + parser.add_argument( 886 | + '--deterministic', 887 | + action='store_true', 888 | + help='whether to set deterministic options for CUDNN backend.') 889 | + parser.add_argument( 890 | + '--cfg-options', 891 | + nargs='+', 892 | + action=DictAction, 893 | + help='override some settings in the used config, the key-value pair ' 894 | + 'in xxx=yyy format will be merged into config file. If the value to ' 895 | + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 896 | + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 897 | + 'Note that the quotation marks are necessary and that no white space ' 898 | + 'is allowed.') 899 | + parser.add_argument( 900 | + '--options', 901 | + nargs='+', 902 | + action=DictAction, 903 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 904 | + 'format will be kwargs for dataset.evaluate() function (deprecate), ' 905 | + 'change to --eval-options instead.') 906 | + parser.add_argument( 907 | + '--eval-options', 908 | + nargs='+', 909 | + action=DictAction, 910 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 911 | + 'format will be kwargs for dataset.evaluate() function') 912 | + parser.add_argument( 913 | + '--launcher', 914 | + choices=['none', 'pytorch', 'slurm', 'mpi'], 915 | + default='none', 916 | + help='job launcher') 917 | + parser.add_argument('--local_rank', type=int, default=0) 918 | + args = parser.parse_args() 919 | + if 'LOCAL_RANK' not in os.environ: 920 | + os.environ['LOCAL_RANK'] = str(args.local_rank) 921 | + 922 | + if args.options and args.eval_options: 923 | + raise ValueError( 924 | + '--options and --eval-options cannot be both specified, ' 925 | + '--options is deprecated in favor of --eval-options') 926 | + if args.options: 927 | + warnings.warn('--options is deprecated in favor of --eval-options') 928 | + args.eval_options = args.options 929 | + return args 930 | + 931 | + 932 | +def main(): 933 | + args = parse_args() 934 | + 935 | + # assert args.out or args.eval or args.format_only or args.show \ 936 | + # or args.show_dir, \ 937 | + # ('Please specify at least one operation (save/eval/format/show the ' 938 | + # 'results / save the results) with the argument "--out", "--eval"' 939 | + # ', "--format-only", "--show" or "--show-dir"') 940 | + 941 | + if args.eval and args.format_only: 942 | + raise ValueError('--eval and --format_only cannot be both specified') 943 | + 944 | + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): 945 | + raise ValueError('The output file must be a pkl file.') 946 | + 947 | + cfg = Config.fromfile(args.config) 948 | + if args.cfg_options is not None: 949 | + cfg.merge_from_dict(args.cfg_options) 950 | + # import modules from string list. 951 | + if cfg.get('custom_imports', None): 952 | + from mmcv.utils import import_modules_from_strings 953 | + import_modules_from_strings(**cfg['custom_imports']) 954 | + 955 | + # import modules from plguin/xx, registry will be updated 956 | + if hasattr(cfg, 'plugin'): 957 | + if cfg.plugin: 958 | + import importlib 959 | + if hasattr(cfg, 'plugin_dir'): 960 | + plugin_dir = cfg.plugin_dir 961 | + _module_dir = os.path.dirname(plugin_dir) 962 | + _module_dir = _module_dir.split('/') 963 | + _module_path = _module_dir[0] 964 | + 965 | + for m in _module_dir[1:]: 966 | + _module_path = _module_path + '.' + m 967 | + print(_module_path) 968 | + plg_lib = importlib.import_module(_module_path) 969 | + else: 970 | + # import dir is the dirpath for the config file 971 | + _module_dir = os.path.dirname(args.config) 972 | + _module_dir = _module_dir.split('/') 973 | + _module_path = _module_dir[0] 974 | + for m in _module_dir[1:]: 975 | + _module_path = _module_path + '.' + m 976 | + print(_module_path) 977 | + plg_lib = importlib.import_module(_module_path) 978 | + 979 | + # set cudnn_benchmark 980 | + if cfg.get('cudnn_benchmark', False): 981 | + torch.backends.cudnn.benchmark = True 982 | + 983 | + cfg.model.pretrained = None 984 | + # in case the test dataset is concatenated 985 | + samples_per_gpu = 1 986 | + if isinstance(cfg.data.test, dict): 987 | + cfg.data.test.test_mode = True 988 | + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) 989 | + if samples_per_gpu > 1: 990 | + # Replace 'ImageToTensor' to 'DefaultFormatBundle' 991 | + cfg.data.test.pipeline = replace_ImageToTensor( 992 | + cfg.data.test.pipeline) 993 | + elif isinstance(cfg.data.test, list): 994 | + for ds_cfg in cfg.data.test: 995 | + ds_cfg.test_mode = True 996 | + samples_per_gpu = max( 997 | + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) 998 | + if samples_per_gpu > 1: 999 | + for ds_cfg in cfg.data.test: 1000 | + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) 1001 | + 1002 | + # init distributed env first, since logger depends on the dist info. 1003 | + if args.launcher == 'none': 1004 | + distributed = False 1005 | + else: 1006 | + distributed = True 1007 | + init_dist(args.launcher, **cfg.dist_params) 1008 | + 1009 | + # set random seeds 1010 | + if args.seed is not None: 1011 | + set_random_seed(args.seed, deterministic=args.deterministic) 1012 | + 1013 | + # build the dataloader 1014 | + dataset = build_dataset(cfg.data.test) 1015 | + data_loader = build_dataloader( 1016 | + dataset, 1017 | + samples_per_gpu=samples_per_gpu, 1018 | + workers_per_gpu=cfg.data.workers_per_gpu, 1019 | + dist=distributed, 1020 | + shuffle=False) 1021 | + 1022 | + # build the model and load checkpoint 1023 | + # cfg.model.train_cfg = None 1024 | + model = build_model(cfg.model, test_cfg=cfg.get('test_cfg'), train_cfg=cfg.get('train_cfg')) 1025 | + fp16_cfg = cfg.get('fp16', None) 1026 | + if fp16_cfg is not None: 1027 | + wrap_fp16_model(model) 1028 | + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') 1029 | + if args.fuse_conv_bn: 1030 | + model = fuse_conv_bn(model) 1031 | + # old versions did not save class info in checkpoints, this walkaround is 1032 | + # for backward compatibility 1033 | + if 'CLASSES' in checkpoint.get('meta', {}): 1034 | + model.CLASSES = checkpoint['meta']['CLASSES'] 1035 | + else: 1036 | + model.CLASSES = dataset.CLASSES 1037 | + # palette for visualization in segmentation tasks 1038 | + if 'PALETTE' in checkpoint.get('meta', {}): 1039 | + model.PALETTE = checkpoint['meta']['PALETTE'] 1040 | + elif hasattr(dataset, 'PALETTE'): 1041 | + # segmentation dataset has `PALETTE` attribute 1042 | + model.PALETTE = dataset.PALETTE 1043 | + 1044 | + if not distributed: 1045 | + model = MMDataParallel(model, device_ids=[0]) 1046 | + outputs = single_gpu_test(model, data_loader, 1047 | + args.patch_save_prefix, 1048 | + args.area_rate_str, 1049 | + args.optim_lr 1050 | + ) 1051 | + else: 1052 | + model = MMDistributedDataParallel( 1053 | + model.cuda(), 1054 | + device_ids=[torch.cuda.current_device()], 1055 | + broadcast_buffers=False) 1056 | + outputs = multi_gpu_test(model, data_loader, args.tmpdir, 1057 | + args.gpu_collect) 1058 | + 1059 | + rank, _ = get_dist_info() 1060 | + if rank == 0: 1061 | + if args.out: 1062 | + print(f'\nwriting results to {args.out}') 1063 | + mmcv.dump(outputs, args.out) 1064 | + kwargs = {} if args.eval_options is None else args.eval_options 1065 | + if args.format_only: 1066 | + dataset.format_results(outputs, **kwargs) 1067 | + if args.eval: 1068 | + eval_kwargs = cfg.get('evaluation', {}).copy() 1069 | + # hard-code way to remove EvalHook args 1070 | + for key in [ 1071 | + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 1072 | + 'rule' 1073 | + ]: 1074 | + eval_kwargs.pop(key, None) 1075 | + eval_kwargs.update(dict(metric=args.eval, **kwargs)) 1076 | + print(dataset.evaluate(outputs, **eval_kwargs)) 1077 | + 1078 | + 1079 | +if __name__ == '__main__': 1080 | + main() 1081 | diff --git a/tools/test_patch_instance_launcher.py b/tools/test_patch_instance_launcher.py 1082 | new file mode 100755 1083 | index 0000000..22b9db5 1084 | --- /dev/null 1085 | +++ b/tools/test_patch_instance_launcher.py 1086 | @@ -0,0 +1,256 @@ 1087 | +# Copyright (c) OpenMMLab. All rights reserved. 1088 | +import argparse 1089 | +import mmcv 1090 | +import os 1091 | +import sys 1092 | +sys_path = os.path.abspath(".") 1093 | +sys.path.append(sys_path) 1094 | +import torch 1095 | +import warnings 1096 | +from mmcv import Config, DictAction 1097 | +from mmcv.cnn import fuse_conv_bn 1098 | +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 1099 | +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, 1100 | + wrap_fp16_model) 1101 | + 1102 | +from projects.mmdet3d_plugin.apis_common.test_patch_instance import single_gpu_test 1103 | +from mmdet3d.datasets import build_dataloader, build_dataset 1104 | +from mmdet3d.models import build_model 1105 | +from mmdet.apis import multi_gpu_test, set_random_seed 1106 | +from mmdet.datasets import replace_ImageToTensor 1107 | + 1108 | + 1109 | +def parse_args(): 1110 | + parser = argparse.ArgumentParser( 1111 | + description='MMDet test (and eval) a model') 1112 | + parser.add_argument('config', help='test config file path') 1113 | + parser.add_argument('checkpoint', help='checkpoint file') 1114 | + parser.add_argument('scattered_result_prefix', help='save scattered_result file dir') 1115 | + parser.add_argument('mask_code', help='mask area of instance patch') 1116 | + parser.add_argument('step', help='step of attack') 1117 | + parser.add_argument('--out', help='output result file in pickle format') 1118 | + parser.add_argument( 1119 | + '--fuse-conv-bn', 1120 | + action='store_true', 1121 | + help='Whether to fuse conv and bn, this will slightly increase' 1122 | + 'the inference speed') 1123 | + parser.add_argument( 1124 | + '--format-only', 1125 | + action='store_true', 1126 | + help='Format the output results without perform evaluation. It is' 1127 | + 'useful when you want to format the result to a specific format and ' 1128 | + 'submit it to the test server') 1129 | + parser.add_argument( 1130 | + '--eval', 1131 | + type=str, 1132 | + nargs='+', 1133 | + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' 1134 | + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') 1135 | + parser.add_argument('--show', action='store_true', help='show results') 1136 | + parser.add_argument( 1137 | + '--show-dir', help='directory where results will be saved') 1138 | + parser.add_argument( 1139 | + '--gpu-collect', 1140 | + action='store_true', 1141 | + help='whether to use gpu to collect results.') 1142 | + parser.add_argument( 1143 | + '--tmpdir', 1144 | + help='tmp directory used for collecting results from multiple ' 1145 | + 'workers, available when gpu-collect is not specified') 1146 | + parser.add_argument('--seed', type=int, default=0, help='random seed') 1147 | + parser.add_argument( 1148 | + '--deterministic', 1149 | + action='store_true', 1150 | + help='whether to set deterministic options for CUDNN backend.') 1151 | + parser.add_argument( 1152 | + '--cfg-options', 1153 | + nargs='+', 1154 | + action=DictAction, 1155 | + help='override some settings in the used config, the key-value pair ' 1156 | + 'in xxx=yyy format will be merged into config file. If the value to ' 1157 | + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 1158 | + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 1159 | + 'Note that the quotation marks are necessary and that no white space ' 1160 | + 'is allowed.') 1161 | + parser.add_argument( 1162 | + '--options', 1163 | + nargs='+', 1164 | + action=DictAction, 1165 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1166 | + 'format will be kwargs for dataset.evaluate() function (deprecate), ' 1167 | + 'change to --eval-options instead.') 1168 | + parser.add_argument( 1169 | + '--eval-options', 1170 | + nargs='+', 1171 | + action=DictAction, 1172 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1173 | + 'format will be kwargs for dataset.evaluate() function') 1174 | + parser.add_argument( 1175 | + '--launcher', 1176 | + choices=['none', 'pytorch', 'slurm', 'mpi'], 1177 | + default='none', 1178 | + help='job launcher') 1179 | + parser.add_argument('--local_rank', type=int, default=0) 1180 | + args = parser.parse_args() 1181 | + if 'LOCAL_RANK' not in os.environ: 1182 | + os.environ['LOCAL_RANK'] = str(args.local_rank) 1183 | + 1184 | + if args.options and args.eval_options: 1185 | + raise ValueError( 1186 | + '--options and --eval-options cannot be both specified, ' 1187 | + '--options is deprecated in favor of --eval-options') 1188 | + if args.options: 1189 | + warnings.warn('--options is deprecated in favor of --eval-options') 1190 | + args.eval_options = args.options 1191 | + return args 1192 | + 1193 | + 1194 | +def main(): 1195 | + args = parse_args() 1196 | + 1197 | + # assert args.out or args.eval or args.format_only or args.show \ 1198 | + # or args.show_dir, \ 1199 | + # ('Please specify at least one operation (save/eval/format/show the ' 1200 | + # 'results / save the results) with the argument "--out", "--eval"' 1201 | + # ', "--format-only", "--show" or "--show-dir"') 1202 | + 1203 | + if args.eval and args.format_only: 1204 | + raise ValueError('--eval and --format_only cannot be both specified') 1205 | + 1206 | + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): 1207 | + raise ValueError('The output file must be a pkl file.') 1208 | + 1209 | + cfg = Config.fromfile(args.config) 1210 | + if args.cfg_options is not None: 1211 | + cfg.merge_from_dict(args.cfg_options) 1212 | + # import modules from string list. 1213 | + if cfg.get('custom_imports', None): 1214 | + from mmcv.utils import import_modules_from_strings 1215 | + import_modules_from_strings(**cfg['custom_imports']) 1216 | + 1217 | + # import modules from plguin/xx, registry will be updated 1218 | + if hasattr(cfg, 'plugin'): 1219 | + if cfg.plugin: 1220 | + import importlib 1221 | + if hasattr(cfg, 'plugin_dir'): 1222 | + plugin_dir = cfg.plugin_dir 1223 | + _module_dir = os.path.dirname(plugin_dir) 1224 | + _module_dir = _module_dir.split('/') 1225 | + _module_path = _module_dir[0] 1226 | + 1227 | + for m in _module_dir[1:]: 1228 | + _module_path = _module_path + '.' + m 1229 | + print(_module_path) 1230 | + plg_lib = importlib.import_module(_module_path) 1231 | + else: 1232 | + # import dir is the dirpath for the config file 1233 | + _module_dir = os.path.dirname(args.config) 1234 | + _module_dir = _module_dir.split('/') 1235 | + _module_path = _module_dir[0] 1236 | + for m in _module_dir[1:]: 1237 | + _module_path = _module_path + '.' + m 1238 | + print(_module_path) 1239 | + plg_lib = importlib.import_module(_module_path) 1240 | + 1241 | + # set cudnn_benchmark 1242 | + if cfg.get('cudnn_benchmark', False): 1243 | + torch.backends.cudnn.benchmark = True 1244 | + 1245 | + cfg.model.pretrained = None 1246 | + # in case the test dataset is concatenated 1247 | + samples_per_gpu = 1 1248 | + if isinstance(cfg.data.test, dict): 1249 | + cfg.data.test.test_mode = True 1250 | + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) 1251 | + if samples_per_gpu > 1: 1252 | + # Replace 'ImageToTensor' to 'DefaultFormatBundle' 1253 | + cfg.data.test.pipeline = replace_ImageToTensor( 1254 | + cfg.data.test.pipeline) 1255 | + elif isinstance(cfg.data.test, list): 1256 | + for ds_cfg in cfg.data.test: 1257 | + ds_cfg.test_mode = True 1258 | + samples_per_gpu = max( 1259 | + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) 1260 | + if samples_per_gpu > 1: 1261 | + for ds_cfg in cfg.data.test: 1262 | + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) 1263 | + 1264 | + # init distributed env first, since logger depends on the dist info. 1265 | + if args.launcher == 'none': 1266 | + distributed = False 1267 | + else: 1268 | + distributed = True 1269 | + init_dist(args.launcher, **cfg.dist_params) 1270 | + 1271 | + # set random seeds 1272 | + if args.seed is not None: 1273 | + set_random_seed(args.seed, deterministic=args.deterministic) 1274 | + 1275 | + # build the dataloader 1276 | + dataset = build_dataset(cfg.data.test) 1277 | + data_loader = build_dataloader( 1278 | + dataset, 1279 | + samples_per_gpu=samples_per_gpu, 1280 | + workers_per_gpu=cfg.data.workers_per_gpu, 1281 | + dist=distributed, 1282 | + shuffle=False) 1283 | + 1284 | + # build the model and load checkpoint 1285 | + # cfg.model.train_cfg = None 1286 | + model = build_model(cfg.model, test_cfg=cfg.get('test_cfg'), train_cfg=cfg.get('train_cfg')) 1287 | + fp16_cfg = cfg.get('fp16', None) 1288 | + if fp16_cfg is not None: 1289 | + wrap_fp16_model(model) 1290 | + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') 1291 | + if args.fuse_conv_bn: 1292 | + model = fuse_conv_bn(model) 1293 | + # old versions did not save class info in checkpoints, this walkaround is 1294 | + # for backward compatibility 1295 | + if 'CLASSES' in checkpoint.get('meta', {}): 1296 | + model.CLASSES = checkpoint['meta']['CLASSES'] 1297 | + else: 1298 | + model.CLASSES = dataset.CLASSES 1299 | + # palette for visualization in segmentation tasks 1300 | + if 'PALETTE' in checkpoint.get('meta', {}): 1301 | + model.PALETTE = checkpoint['meta']['PALETTE'] 1302 | + elif hasattr(dataset, 'PALETTE'): 1303 | + # segmentation dataset has `PALETTE` attribute 1304 | + model.PALETTE = dataset.PALETTE 1305 | + 1306 | + if not distributed: 1307 | + model = MMDataParallel(model, device_ids=[0]) 1308 | + outputs = single_gpu_test(model, data_loader, 1309 | + args.scattered_result_prefix, 1310 | + args.mask_code, 1311 | + args.step 1312 | + ) 1313 | + else: 1314 | + model = MMDistributedDataParallel( 1315 | + model.cuda(), 1316 | + device_ids=[torch.cuda.current_device()], 1317 | + broadcast_buffers=False) 1318 | + outputs = multi_gpu_test(model, data_loader, args.tmpdir, 1319 | + args.gpu_collect) 1320 | + 1321 | + rank, _ = get_dist_info() 1322 | + if rank == 0: 1323 | + if args.out: 1324 | + print(f'\nwriting results to {args.out}') 1325 | + mmcv.dump(outputs, args.out) 1326 | + kwargs = {} if args.eval_options is None else args.eval_options 1327 | + if args.format_only: 1328 | + dataset.format_results(outputs, **kwargs) 1329 | + if args.eval: 1330 | + eval_kwargs = cfg.get('evaluation', {}).copy() 1331 | + # hard-code way to remove EvalHook args 1332 | + for key in [ 1333 | + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 1334 | + 'rule' 1335 | + ]: 1336 | + eval_kwargs.pop(key, None) 1337 | + eval_kwargs.update(dict(metric=args.eval, **kwargs)) 1338 | + print(dataset.evaluate(outputs, **eval_kwargs)) 1339 | + 1340 | + 1341 | +if __name__ == '__main__': 1342 | + main() 1343 | diff --git a/tools/test_patch_overlap_launcher.py b/tools/test_patch_overlap_launcher.py 1344 | new file mode 100755 1345 | index 0000000..c019bb7 1346 | --- /dev/null 1347 | +++ b/tools/test_patch_overlap_launcher.py 1348 | @@ -0,0 +1,260 @@ 1349 | +# Copyright (c) OpenMMLab. All rights reserved. 1350 | +import argparse 1351 | +import mmcv 1352 | +import os 1353 | +import sys 1354 | +sys_path = os.path.abspath(".") 1355 | +sys.path.append(sys_path) 1356 | +import torch 1357 | +import warnings 1358 | +from mmcv import Config, DictAction 1359 | +from mmcv.cnn import fuse_conv_bn 1360 | +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 1361 | +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, 1362 | + wrap_fp16_model) 1363 | + 1364 | +from projects.mmdet3d_plugin.apis_common.test_patch_overlap import single_gpu_test 1365 | +from mmdet3d.datasets import build_dataloader, build_dataset 1366 | +from mmdet3d.models import build_model 1367 | +from mmdet.apis import multi_gpu_test, set_random_seed 1368 | +from mmdet.datasets import replace_ImageToTensor 1369 | + 1370 | + 1371 | +def parse_args(): 1372 | + parser = argparse.ArgumentParser( 1373 | + description='MMDet test (and eval) a model') 1374 | + parser.add_argument('config', help='test config file path') 1375 | + parser.add_argument('checkpoint', help='checkpoint file') 1376 | + parser.add_argument('scattered_result_prefix', help='save scattered_result file dir') 1377 | + parser.add_argument('area_rate_str', help='area rate of patch') 1378 | + parser.add_argument('optim_lr', help='optim_lr of attack') 1379 | + parser.add_argument('optim_step', help='optim_lr of attack') 1380 | + parser.add_argument('--index-min', type=int, default=0) # for multi-gpu split dataset 1381 | + parser.add_argument('--index-max', type=int, default=100000) # for multi-gpu split dataset 1382 | + parser.add_argument('--out', help='output result file in pickle format') 1383 | + parser.add_argument( 1384 | + '--fuse-conv-bn', 1385 | + action='store_true', 1386 | + help='Whether to fuse conv and bn, this will slightly increase' 1387 | + 'the inference speed') 1388 | + parser.add_argument( 1389 | + '--format-only', 1390 | + action='store_true', 1391 | + help='Format the output results without perform evaluation. It is' 1392 | + 'useful when you want to format the result to a specific format and ' 1393 | + 'submit it to the test server') 1394 | + parser.add_argument( 1395 | + '--eval', 1396 | + type=str, 1397 | + nargs='+', 1398 | + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' 1399 | + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') 1400 | + parser.add_argument('--show', action='store_true', help='show results') 1401 | + parser.add_argument( 1402 | + '--show-dir', help='directory where results will be saved') 1403 | + parser.add_argument( 1404 | + '--gpu-collect', 1405 | + action='store_true', 1406 | + help='whether to use gpu to collect results.') 1407 | + parser.add_argument( 1408 | + '--tmpdir', 1409 | + help='tmp directory used for collecting results from multiple ' 1410 | + 'workers, available when gpu-collect is not specified') 1411 | + parser.add_argument('--seed', type=int, default=0, help='random seed') 1412 | + parser.add_argument( 1413 | + '--deterministic', 1414 | + action='store_true', 1415 | + help='whether to set deterministic options for CUDNN backend.') 1416 | + parser.add_argument( 1417 | + '--cfg-options', 1418 | + nargs='+', 1419 | + action=DictAction, 1420 | + help='override some settings in the used config, the key-value pair ' 1421 | + 'in xxx=yyy format will be merged into config file. If the value to ' 1422 | + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 1423 | + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 1424 | + 'Note that the quotation marks are necessary and that no white space ' 1425 | + 'is allowed.') 1426 | + parser.add_argument( 1427 | + '--options', 1428 | + nargs='+', 1429 | + action=DictAction, 1430 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1431 | + 'format will be kwargs for dataset.evaluate() function (deprecate), ' 1432 | + 'change to --eval-options instead.') 1433 | + parser.add_argument( 1434 | + '--eval-options', 1435 | + nargs='+', 1436 | + action=DictAction, 1437 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1438 | + 'format will be kwargs for dataset.evaluate() function') 1439 | + parser.add_argument( 1440 | + '--launcher', 1441 | + choices=['none', 'pytorch', 'slurm', 'mpi'], 1442 | + default='none', 1443 | + help='job launcher') 1444 | + parser.add_argument('--local_rank', type=int, default=0) 1445 | + args = parser.parse_args() 1446 | + if 'LOCAL_RANK' not in os.environ: 1447 | + os.environ['LOCAL_RANK'] = str(args.local_rank) 1448 | + 1449 | + if args.options and args.eval_options: 1450 | + raise ValueError( 1451 | + '--options and --eval-options cannot be both specified, ' 1452 | + '--options is deprecated in favor of --eval-options') 1453 | + if args.options: 1454 | + warnings.warn('--options is deprecated in favor of --eval-options') 1455 | + args.eval_options = args.options 1456 | + return args 1457 | + 1458 | + 1459 | +def main(): 1460 | + args = parse_args() 1461 | + 1462 | + # assert args.out or args.eval or args.format_only or args.show \ 1463 | + # or args.show_dir, \ 1464 | + # ('Please specify at least one operation (save/eval/format/show the ' 1465 | + # 'results / save the results) with the argument "--out", "--eval"' 1466 | + # ', "--format-only", "--show" or "--show-dir"') 1467 | + 1468 | + if args.eval and args.format_only: 1469 | + raise ValueError('--eval and --format_only cannot be both specified') 1470 | + 1471 | + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): 1472 | + raise ValueError('The output file must be a pkl file.') 1473 | + 1474 | + cfg = Config.fromfile(args.config) 1475 | + if args.cfg_options is not None: 1476 | + cfg.merge_from_dict(args.cfg_options) 1477 | + # import modules from string list. 1478 | + if cfg.get('custom_imports', None): 1479 | + from mmcv.utils import import_modules_from_strings 1480 | + import_modules_from_strings(**cfg['custom_imports']) 1481 | + 1482 | + # import modules from plguin/xx, registry will be updated 1483 | + if hasattr(cfg, 'plugin'): 1484 | + if cfg.plugin: 1485 | + import importlib 1486 | + if hasattr(cfg, 'plugin_dir'): 1487 | + plugin_dir = cfg.plugin_dir 1488 | + _module_dir = os.path.dirname(plugin_dir) 1489 | + _module_dir = _module_dir.split('/') 1490 | + _module_path = _module_dir[0] 1491 | + 1492 | + for m in _module_dir[1:]: 1493 | + _module_path = _module_path + '.' + m 1494 | + print(_module_path) 1495 | + plg_lib = importlib.import_module(_module_path) 1496 | + else: 1497 | + # import dir is the dirpath for the config file 1498 | + _module_dir = os.path.dirname(args.config) 1499 | + _module_dir = _module_dir.split('/') 1500 | + _module_path = _module_dir[0] 1501 | + for m in _module_dir[1:]: 1502 | + _module_path = _module_path + '.' + m 1503 | + print(_module_path) 1504 | + plg_lib = importlib.import_module(_module_path) 1505 | + 1506 | + # set cudnn_benchmark 1507 | + if cfg.get('cudnn_benchmark', False): 1508 | + torch.backends.cudnn.benchmark = True 1509 | + 1510 | + cfg.model.pretrained = None 1511 | + # in case the test dataset is concatenated 1512 | + samples_per_gpu = 1 1513 | + if isinstance(cfg.data.test, dict): 1514 | + cfg.data.test.test_mode = True 1515 | + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) 1516 | + if samples_per_gpu > 1: 1517 | + # Replace 'ImageToTensor' to 'DefaultFormatBundle' 1518 | + cfg.data.test.pipeline = replace_ImageToTensor( 1519 | + cfg.data.test.pipeline) 1520 | + elif isinstance(cfg.data.test, list): 1521 | + for ds_cfg in cfg.data.test: 1522 | + ds_cfg.test_mode = True 1523 | + samples_per_gpu = max( 1524 | + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) 1525 | + if samples_per_gpu > 1: 1526 | + for ds_cfg in cfg.data.test: 1527 | + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) 1528 | + 1529 | + # init distributed env first, since logger depends on the dist info. 1530 | + if args.launcher == 'none': 1531 | + distributed = False 1532 | + else: 1533 | + distributed = True 1534 | + init_dist(args.launcher, **cfg.dist_params) 1535 | + 1536 | + # set random seeds 1537 | + if args.seed is not None: 1538 | + set_random_seed(args.seed, deterministic=args.deterministic) 1539 | + 1540 | + # build the dataloader 1541 | + dataset = build_dataset(cfg.data.test) 1542 | + data_loader = build_dataloader( 1543 | + dataset, 1544 | + samples_per_gpu=samples_per_gpu, 1545 | + workers_per_gpu=cfg.data.workers_per_gpu, 1546 | + dist=distributed, 1547 | + shuffle=False) 1548 | + 1549 | + # build the model and load checkpoint 1550 | + # cfg.model.train_cfg = None 1551 | + model = build_model(cfg.model, test_cfg=cfg.get('test_cfg'), train_cfg=cfg.get('train_cfg')) 1552 | + fp16_cfg = cfg.get('fp16', None) 1553 | + if fp16_cfg is not None: 1554 | + wrap_fp16_model(model) 1555 | + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') 1556 | + if args.fuse_conv_bn: 1557 | + model = fuse_conv_bn(model) 1558 | + # old versions did not save class info in checkpoints, this walkaround is 1559 | + # for backward compatibility 1560 | + if 'CLASSES' in checkpoint.get('meta', {}): 1561 | + model.CLASSES = checkpoint['meta']['CLASSES'] 1562 | + else: 1563 | + model.CLASSES = dataset.CLASSES 1564 | + # palette for visualization in segmentation tasks 1565 | + if 'PALETTE' in checkpoint.get('meta', {}): 1566 | + model.PALETTE = checkpoint['meta']['PALETTE'] 1567 | + elif hasattr(dataset, 'PALETTE'): 1568 | + # segmentation dataset has `PALETTE` attribute 1569 | + model.PALETTE = dataset.PALETTE 1570 | + 1571 | + if not distributed: 1572 | + model = MMDataParallel(model, device_ids=[0]) 1573 | + outputs = single_gpu_test(model, data_loader, 1574 | + args.scattered_result_prefix, 1575 | + args.area_rate_str, 1576 | + args.optim_lr, 1577 | + args.optim_step 1578 | + ) 1579 | + else: 1580 | + model = MMDistributedDataParallel( 1581 | + model.cuda(), 1582 | + device_ids=[torch.cuda.current_device()], 1583 | + broadcast_buffers=False) 1584 | + outputs = multi_gpu_test(model, data_loader, args.tmpdir, 1585 | + args.gpu_collect) 1586 | + 1587 | + rank, _ = get_dist_info() 1588 | + if rank == 0: 1589 | + if args.out: 1590 | + print(f'\nwriting results to {args.out}') 1591 | + mmcv.dump(outputs, args.out) 1592 | + kwargs = {} if args.eval_options is None else args.eval_options 1593 | + if args.format_only: 1594 | + dataset.format_results(outputs, **kwargs) 1595 | + if args.eval: 1596 | + eval_kwargs = cfg.get('evaluation', {}).copy() 1597 | + # hard-code way to remove EvalHook args 1598 | + for key in [ 1599 | + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 1600 | + 'rule' 1601 | + ]: 1602 | + eval_kwargs.pop(key, None) 1603 | + eval_kwargs.update(dict(metric=args.eval, **kwargs)) 1604 | + print(dataset.evaluate(outputs, **eval_kwargs)) 1605 | + 1606 | + 1607 | +if __name__ == '__main__': 1608 | + main() 1609 | diff --git a/tools/test_patch_temporal_launcher.py b/tools/test_patch_temporal_launcher.py 1610 | new file mode 100755 1611 | index 0000000..d5ac1c3 1612 | --- /dev/null 1613 | +++ b/tools/test_patch_temporal_launcher.py 1614 | @@ -0,0 +1,262 @@ 1615 | +# Copyright (c) OpenMMLab. All rights reserved. 1616 | +import argparse 1617 | +import mmcv 1618 | +import os 1619 | +import sys 1620 | +sys_path = os.path.abspath(".") 1621 | +sys.path.append(sys_path) 1622 | +import torch 1623 | +import warnings 1624 | +from mmcv import Config, DictAction 1625 | +from mmcv.cnn import fuse_conv_bn 1626 | +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 1627 | +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, 1628 | + wrap_fp16_model) 1629 | + 1630 | +from projects.mmdet3d_plugin.apis_common.test_patch_temporal import single_gpu_test 1631 | +from mmdet3d.datasets import build_dataloader, build_dataset 1632 | +from mmdet3d.models import build_model 1633 | +from mmdet.apis import multi_gpu_test, set_random_seed 1634 | +from mmdet.datasets import replace_ImageToTensor 1635 | + 1636 | + 1637 | +def parse_args(): 1638 | + parser = argparse.ArgumentParser( 1639 | + description='MMDet test (and eval) a model') 1640 | + parser.add_argument('config', help='test config file path') 1641 | + parser.add_argument('checkpoint', help='checkpoint file') 1642 | + parser.add_argument('scattered_result_prefix', help='save scattered_result file dir') 1643 | + parser.add_argument('area_rate_str', help='area rate of patch') 1644 | + parser.add_argument('optim_lr', help='optim_lr of attack') 1645 | + parser.add_argument('optim_step', help='optim_lr of attack') 1646 | + parser.add_argument('--index-min', type=int, default=0) # for multi-gpu split dataset 1647 | + parser.add_argument('--index-max', type=int, default=100000) # for multi-gpu split dataset 1648 | + parser.add_argument('--out', help='output result file in pickle format') 1649 | + parser.add_argument( 1650 | + '--fuse-conv-bn', 1651 | + action='store_true', 1652 | + help='Whether to fuse conv and bn, this will slightly increase' 1653 | + 'the inference speed') 1654 | + parser.add_argument( 1655 | + '--format-only', 1656 | + action='store_true', 1657 | + help='Format the output results without perform evaluation. It is' 1658 | + 'useful when you want to format the result to a specific format and ' 1659 | + 'submit it to the test server') 1660 | + parser.add_argument( 1661 | + '--eval', 1662 | + type=str, 1663 | + nargs='+', 1664 | + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' 1665 | + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') 1666 | + parser.add_argument('--show', action='store_true', help='show results') 1667 | + parser.add_argument( 1668 | + '--show-dir', help='directory where results will be saved') 1669 | + parser.add_argument( 1670 | + '--gpu-collect', 1671 | + action='store_true', 1672 | + help='whether to use gpu to collect results.') 1673 | + parser.add_argument( 1674 | + '--tmpdir', 1675 | + help='tmp directory used for collecting results from multiple ' 1676 | + 'workers, available when gpu-collect is not specified') 1677 | + parser.add_argument('--seed', type=int, default=0, help='random seed') 1678 | + parser.add_argument( 1679 | + '--deterministic', 1680 | + action='store_true', 1681 | + help='whether to set deterministic options for CUDNN backend.') 1682 | + parser.add_argument( 1683 | + '--cfg-options', 1684 | + nargs='+', 1685 | + action=DictAction, 1686 | + help='override some settings in the used config, the key-value pair ' 1687 | + 'in xxx=yyy format will be merged into config file. If the value to ' 1688 | + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 1689 | + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 1690 | + 'Note that the quotation marks are necessary and that no white space ' 1691 | + 'is allowed.') 1692 | + parser.add_argument( 1693 | + '--options', 1694 | + nargs='+', 1695 | + action=DictAction, 1696 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1697 | + 'format will be kwargs for dataset.evaluate() function (deprecate), ' 1698 | + 'change to --eval-options instead.') 1699 | + parser.add_argument( 1700 | + '--eval-options', 1701 | + nargs='+', 1702 | + action=DictAction, 1703 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1704 | + 'format will be kwargs for dataset.evaluate() function') 1705 | + parser.add_argument( 1706 | + '--launcher', 1707 | + choices=['none', 'pytorch', 'slurm', 'mpi'], 1708 | + default='none', 1709 | + help='job launcher') 1710 | + parser.add_argument('--local_rank', type=int, default=0) 1711 | + args = parser.parse_args() 1712 | + if 'LOCAL_RANK' not in os.environ: 1713 | + os.environ['LOCAL_RANK'] = str(args.local_rank) 1714 | + 1715 | + if args.options and args.eval_options: 1716 | + raise ValueError( 1717 | + '--options and --eval-options cannot be both specified, ' 1718 | + '--options is deprecated in favor of --eval-options') 1719 | + if args.options: 1720 | + warnings.warn('--options is deprecated in favor of --eval-options') 1721 | + args.eval_options = args.options 1722 | + return args 1723 | + 1724 | + 1725 | +def main(): 1726 | + args = parse_args() 1727 | + 1728 | + # assert args.out or args.eval or args.format_only or args.show \ 1729 | + # or args.show_dir, \ 1730 | + # ('Please specify at least one operation (save/eval/format/show the ' 1731 | + # 'results / save the results) with the argument "--out", "--eval"' 1732 | + # ', "--format-only", "--show" or "--show-dir"') 1733 | + 1734 | + if args.eval and args.format_only: 1735 | + raise ValueError('--eval and --format_only cannot be both specified') 1736 | + 1737 | + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): 1738 | + raise ValueError('The output file must be a pkl file.') 1739 | + 1740 | + cfg = Config.fromfile(args.config) 1741 | + if args.cfg_options is not None: 1742 | + cfg.merge_from_dict(args.cfg_options) 1743 | + # import modules from string list. 1744 | + if cfg.get('custom_imports', None): 1745 | + from mmcv.utils import import_modules_from_strings 1746 | + import_modules_from_strings(**cfg['custom_imports']) 1747 | + 1748 | + # import modules from plguin/xx, registry will be updated 1749 | + if hasattr(cfg, 'plugin'): 1750 | + if cfg.plugin: 1751 | + import importlib 1752 | + if hasattr(cfg, 'plugin_dir'): 1753 | + plugin_dir = cfg.plugin_dir 1754 | + _module_dir = os.path.dirname(plugin_dir) 1755 | + _module_dir = _module_dir.split('/') 1756 | + _module_path = _module_dir[0] 1757 | + 1758 | + for m in _module_dir[1:]: 1759 | + _module_path = _module_path + '.' + m 1760 | + print(_module_path) 1761 | + plg_lib = importlib.import_module(_module_path) 1762 | + else: 1763 | + # import dir is the dirpath for the config file 1764 | + _module_dir = os.path.dirname(args.config) 1765 | + _module_dir = _module_dir.split('/') 1766 | + _module_path = _module_dir[0] 1767 | + for m in _module_dir[1:]: 1768 | + _module_path = _module_path + '.' + m 1769 | + print(_module_path) 1770 | + plg_lib = importlib.import_module(_module_path) 1771 | + 1772 | + # set cudnn_benchmark 1773 | + if cfg.get('cudnn_benchmark', False): 1774 | + torch.backends.cudnn.benchmark = True 1775 | + 1776 | + cfg.model.pretrained = None 1777 | + # in case the test dataset is concatenated 1778 | + samples_per_gpu = 1 1779 | + if isinstance(cfg.data.test, dict): 1780 | + cfg.data.test.test_mode = True 1781 | + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) 1782 | + if samples_per_gpu > 1: 1783 | + # Replace 'ImageToTensor' to 'DefaultFormatBundle' 1784 | + cfg.data.test.pipeline = replace_ImageToTensor( 1785 | + cfg.data.test.pipeline) 1786 | + elif isinstance(cfg.data.test, list): 1787 | + for ds_cfg in cfg.data.test: 1788 | + ds_cfg.test_mode = True 1789 | + samples_per_gpu = max( 1790 | + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) 1791 | + if samples_per_gpu > 1: 1792 | + for ds_cfg in cfg.data.test: 1793 | + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) 1794 | + 1795 | + # init distributed env first, since logger depends on the dist info. 1796 | + if args.launcher == 'none': 1797 | + distributed = False 1798 | + else: 1799 | + distributed = True 1800 | + init_dist(args.launcher, **cfg.dist_params) 1801 | + 1802 | + # set random seeds 1803 | + if args.seed is not None: 1804 | + set_random_seed(args.seed, deterministic=args.deterministic) 1805 | + 1806 | + # build the dataloader 1807 | + dataset = build_dataset(cfg.data.test) 1808 | + data_loader = build_dataloader( 1809 | + dataset, 1810 | + samples_per_gpu=samples_per_gpu, 1811 | + workers_per_gpu=cfg.data.workers_per_gpu, 1812 | + dist=distributed, 1813 | + shuffle=False) 1814 | + 1815 | + # build the model and load checkpoint 1816 | + # cfg.model.train_cfg = None 1817 | + model = build_model(cfg.model, test_cfg=cfg.get('test_cfg'), train_cfg=cfg.get('train_cfg')) 1818 | + fp16_cfg = cfg.get('fp16', None) 1819 | + if fp16_cfg is not None: 1820 | + wrap_fp16_model(model) 1821 | + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') 1822 | + if args.fuse_conv_bn: 1823 | + model = fuse_conv_bn(model) 1824 | + # old versions did not save class info in checkpoints, this walkaround is 1825 | + # for backward compatibility 1826 | + if 'CLASSES' in checkpoint.get('meta', {}): 1827 | + model.CLASSES = checkpoint['meta']['CLASSES'] 1828 | + else: 1829 | + model.CLASSES = dataset.CLASSES 1830 | + # palette for visualization in segmentation tasks 1831 | + if 'PALETTE' in checkpoint.get('meta', {}): 1832 | + model.PALETTE = checkpoint['meta']['PALETTE'] 1833 | + elif hasattr(dataset, 'PALETTE'): 1834 | + # segmentation dataset has `PALETTE` attribute 1835 | + model.PALETTE = dataset.PALETTE 1836 | + 1837 | + if not distributed: 1838 | + model = MMDataParallel(model, device_ids=[0]) 1839 | + outputs = single_gpu_test(model, data_loader, 1840 | + args.scattered_result_prefix, 1841 | + args.area_rate_str, 1842 | + args.optim_lr, 1843 | + args.optim_step, 1844 | + args.index_min, 1845 | + args.index_max, 1846 | + ) 1847 | + else: 1848 | + model = MMDistributedDataParallel( 1849 | + model.cuda(), 1850 | + device_ids=[torch.cuda.current_device()], 1851 | + broadcast_buffers=False) 1852 | + outputs = multi_gpu_test(model, data_loader, args.tmpdir, 1853 | + args.gpu_collect) 1854 | + 1855 | + rank, _ = get_dist_info() 1856 | + if rank == 0: 1857 | + if args.out: 1858 | + print(f'\nwriting results to {args.out}') 1859 | + mmcv.dump(outputs, args.out) 1860 | + kwargs = {} if args.eval_options is None else args.eval_options 1861 | + if args.format_only: 1862 | + dataset.format_results(outputs, **kwargs) 1863 | + if args.eval: 1864 | + eval_kwargs = cfg.get('evaluation', {}).copy() 1865 | + # hard-code way to remove EvalHook args 1866 | + for key in [ 1867 | + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 1868 | + 'rule' 1869 | + ]: 1870 | + eval_kwargs.pop(key, None) 1871 | + eval_kwargs.update(dict(metric=args.eval, **kwargs)) 1872 | + print(dataset.evaluate(outputs, **eval_kwargs)) 1873 | + 1874 | + 1875 | +if __name__ == '__main__': 1876 | + main() 1877 | diff --git a/tools/test_pgd_img_launcher.py b/tools/test_pgd_img_launcher.py 1878 | new file mode 100755 1879 | index 0000000..42d1f91 1880 | --- /dev/null 1881 | +++ b/tools/test_pgd_img_launcher.py 1882 | @@ -0,0 +1,256 @@ 1883 | +# Copyright (c) OpenMMLab. All rights reserved. 1884 | +import argparse 1885 | +import mmcv 1886 | +import os 1887 | +import sys 1888 | +sys_path = os.path.abspath(".") 1889 | +sys.path.append(sys_path) 1890 | +import torch 1891 | +import warnings 1892 | +from mmcv import Config, DictAction 1893 | +from mmcv.cnn import fuse_conv_bn 1894 | +from mmcv.parallel import MMDataParallel, MMDistributedDataParallel 1895 | +from mmcv.runner import (get_dist_info, init_dist, load_checkpoint, 1896 | + wrap_fp16_model) 1897 | + 1898 | +from projects.mmdet3d_plugin.apis_common.test_pgd_img import single_gpu_test 1899 | +from mmdet3d.datasets import build_dataloader, build_dataset 1900 | +from mmdet3d.models import build_model 1901 | +from mmdet.apis import multi_gpu_test, set_random_seed 1902 | +from mmdet.datasets import replace_ImageToTensor 1903 | + 1904 | + 1905 | +def parse_args(): 1906 | + parser = argparse.ArgumentParser( 1907 | + description='MMDet test (and eval) a model') 1908 | + parser.add_argument('config', help='test config file path') 1909 | + parser.add_argument('checkpoint', help='checkpoint file') 1910 | + parser.add_argument('scattered_result_prefix', help='save scattered_result file dir') 1911 | + parser.add_argument('eps255', help='eps of pgd in 0-255') 1912 | + parser.add_argument('step', help='step of pgd') 1913 | + parser.add_argument('--out', help='output result file in pickle format') 1914 | + parser.add_argument( 1915 | + '--fuse-conv-bn', 1916 | + action='store_true', 1917 | + help='Whether to fuse conv and bn, this will slightly increase' 1918 | + 'the inference speed') 1919 | + parser.add_argument( 1920 | + '--format-only', 1921 | + action='store_true', 1922 | + help='Format the output results without perform evaluation. It is' 1923 | + 'useful when you want to format the result to a specific format and ' 1924 | + 'submit it to the test server') 1925 | + parser.add_argument( 1926 | + '--eval', 1927 | + type=str, 1928 | + nargs='+', 1929 | + help='evaluation metrics, which depends on the dataset, e.g., "bbox",' 1930 | + ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC') 1931 | + parser.add_argument('--show', action='store_true', help='show results') 1932 | + parser.add_argument( 1933 | + '--show-dir', help='directory where results will be saved') 1934 | + parser.add_argument( 1935 | + '--gpu-collect', 1936 | + action='store_true', 1937 | + help='whether to use gpu to collect results.') 1938 | + parser.add_argument( 1939 | + '--tmpdir', 1940 | + help='tmp directory used for collecting results from multiple ' 1941 | + 'workers, available when gpu-collect is not specified') 1942 | + parser.add_argument('--seed', type=int, default=0, help='random seed') 1943 | + parser.add_argument( 1944 | + '--deterministic', 1945 | + action='store_true', 1946 | + help='whether to set deterministic options for CUDNN backend.') 1947 | + parser.add_argument( 1948 | + '--cfg-options', 1949 | + nargs='+', 1950 | + action=DictAction, 1951 | + help='override some settings in the used config, the key-value pair ' 1952 | + 'in xxx=yyy format will be merged into config file. If the value to ' 1953 | + 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 1954 | + 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 1955 | + 'Note that the quotation marks are necessary and that no white space ' 1956 | + 'is allowed.') 1957 | + parser.add_argument( 1958 | + '--options', 1959 | + nargs='+', 1960 | + action=DictAction, 1961 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1962 | + 'format will be kwargs for dataset.evaluate() function (deprecate), ' 1963 | + 'change to --eval-options instead.') 1964 | + parser.add_argument( 1965 | + '--eval-options', 1966 | + nargs='+', 1967 | + action=DictAction, 1968 | + help='custom options for evaluation, the key-value pair in xxx=yyy ' 1969 | + 'format will be kwargs for dataset.evaluate() function') 1970 | + parser.add_argument( 1971 | + '--launcher', 1972 | + choices=['none', 'pytorch', 'slurm', 'mpi'], 1973 | + default='none', 1974 | + help='job launcher') 1975 | + parser.add_argument('--local_rank', type=int, default=0) 1976 | + args = parser.parse_args() 1977 | + if 'LOCAL_RANK' not in os.environ: 1978 | + os.environ['LOCAL_RANK'] = str(args.local_rank) 1979 | + 1980 | + if args.options and args.eval_options: 1981 | + raise ValueError( 1982 | + '--options and --eval-options cannot be both specified, ' 1983 | + '--options is deprecated in favor of --eval-options') 1984 | + if args.options: 1985 | + warnings.warn('--options is deprecated in favor of --eval-options') 1986 | + args.eval_options = args.options 1987 | + return args 1988 | + 1989 | + 1990 | +def main(): 1991 | + args = parse_args() 1992 | + 1993 | + # assert args.out or args.eval or args.format_only or args.show \ 1994 | + # or args.show_dir, \ 1995 | + # ('Please specify at least one operation (save/eval/format/show the ' 1996 | + # 'results / save the results) with the argument "--out", "--eval"' 1997 | + # ', "--format-only", "--show" or "--show-dir"') 1998 | + 1999 | + if args.eval and args.format_only: 2000 | + raise ValueError('--eval and --format_only cannot be both specified') 2001 | + 2002 | + if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): 2003 | + raise ValueError('The output file must be a pkl file.') 2004 | + 2005 | + cfg = Config.fromfile(args.config) 2006 | + if args.cfg_options is not None: 2007 | + cfg.merge_from_dict(args.cfg_options) 2008 | + # import modules from string list. 2009 | + if cfg.get('custom_imports', None): 2010 | + from mmcv.utils import import_modules_from_strings 2011 | + import_modules_from_strings(**cfg['custom_imports']) 2012 | + 2013 | + # import modules from plguin/xx, registry will be updated 2014 | + if hasattr(cfg, 'plugin'): 2015 | + if cfg.plugin: 2016 | + import importlib 2017 | + if hasattr(cfg, 'plugin_dir'): 2018 | + plugin_dir = cfg.plugin_dir 2019 | + _module_dir = os.path.dirname(plugin_dir) 2020 | + _module_dir = _module_dir.split('/') 2021 | + _module_path = _module_dir[0] 2022 | + 2023 | + for m in _module_dir[1:]: 2024 | + _module_path = _module_path + '.' + m 2025 | + print(_module_path) 2026 | + plg_lib = importlib.import_module(_module_path) 2027 | + else: 2028 | + # import dir is the dirpath for the config file 2029 | + _module_dir = os.path.dirname(args.config) 2030 | + _module_dir = _module_dir.split('/') 2031 | + _module_path = _module_dir[0] 2032 | + for m in _module_dir[1:]: 2033 | + _module_path = _module_path + '.' + m 2034 | + print(_module_path) 2035 | + plg_lib = importlib.import_module(_module_path) 2036 | + 2037 | + # set cudnn_benchmark 2038 | + if cfg.get('cudnn_benchmark', False): 2039 | + torch.backends.cudnn.benchmark = True 2040 | + 2041 | + cfg.model.pretrained = None 2042 | + # in case the test dataset is concatenated 2043 | + samples_per_gpu = 1 2044 | + if isinstance(cfg.data.test, dict): 2045 | + cfg.data.test.test_mode = True 2046 | + samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1) 2047 | + if samples_per_gpu > 1: 2048 | + # Replace 'ImageToTensor' to 'DefaultFormatBundle' 2049 | + cfg.data.test.pipeline = replace_ImageToTensor( 2050 | + cfg.data.test.pipeline) 2051 | + elif isinstance(cfg.data.test, list): 2052 | + for ds_cfg in cfg.data.test: 2053 | + ds_cfg.test_mode = True 2054 | + samples_per_gpu = max( 2055 | + [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) 2056 | + if samples_per_gpu > 1: 2057 | + for ds_cfg in cfg.data.test: 2058 | + ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) 2059 | + 2060 | + # init distributed env first, since logger depends on the dist info. 2061 | + if args.launcher == 'none': 2062 | + distributed = False 2063 | + else: 2064 | + distributed = True 2065 | + init_dist(args.launcher, **cfg.dist_params) 2066 | + 2067 | + # set random seeds 2068 | + if args.seed is not None: 2069 | + set_random_seed(args.seed, deterministic=args.deterministic) 2070 | + 2071 | + # build the dataloader 2072 | + dataset = build_dataset(cfg.data.test) 2073 | + data_loader = build_dataloader( 2074 | + dataset, 2075 | + samples_per_gpu=samples_per_gpu, 2076 | + workers_per_gpu=cfg.data.workers_per_gpu, 2077 | + dist=distributed, 2078 | + shuffle=False) 2079 | + 2080 | + # build the model and load checkpoint 2081 | + # cfg.model.train_cfg = None 2082 | + model = build_model(cfg.model, test_cfg=cfg.get('test_cfg'), train_cfg=cfg.get('train_cfg')) 2083 | + fp16_cfg = cfg.get('fp16', None) 2084 | + if fp16_cfg is not None: 2085 | + wrap_fp16_model(model) 2086 | + checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu') 2087 | + if args.fuse_conv_bn: 2088 | + model = fuse_conv_bn(model) 2089 | + # old versions did not save class info in checkpoints, this walkaround is 2090 | + # for backward compatibility 2091 | + if 'CLASSES' in checkpoint.get('meta', {}): 2092 | + model.CLASSES = checkpoint['meta']['CLASSES'] 2093 | + else: 2094 | + model.CLASSES = dataset.CLASSES 2095 | + # palette for visualization in segmentation tasks 2096 | + if 'PALETTE' in checkpoint.get('meta', {}): 2097 | + model.PALETTE = checkpoint['meta']['PALETTE'] 2098 | + elif hasattr(dataset, 'PALETTE'): 2099 | + # segmentation dataset has `PALETTE` attribute 2100 | + model.PALETTE = dataset.PALETTE 2101 | + 2102 | + if not distributed: 2103 | + model = MMDataParallel(model, device_ids=[0]) 2104 | + outputs = single_gpu_test(model, data_loader, 2105 | + args.scattered_result_prefix, 2106 | + args.eps255, 2107 | + args.step 2108 | + ) 2109 | + else: 2110 | + model = MMDistributedDataParallel( 2111 | + model.cuda(), 2112 | + device_ids=[torch.cuda.current_device()], 2113 | + broadcast_buffers=False) 2114 | + outputs = multi_gpu_test(model, data_loader, args.tmpdir, 2115 | + args.gpu_collect) 2116 | + 2117 | + rank, _ = get_dist_info() 2118 | + if rank == 0: 2119 | + if args.out: 2120 | + print(f'\nwriting results to {args.out}') 2121 | + mmcv.dump(outputs, args.out) 2122 | + kwargs = {} if args.eval_options is None else args.eval_options 2123 | + if args.format_only: 2124 | + dataset.format_results(outputs, **kwargs) 2125 | + if args.eval: 2126 | + eval_kwargs = cfg.get('evaluation', {}).copy() 2127 | + # hard-code way to remove EvalHook args 2128 | + for key in [ 2129 | + 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 2130 | + 'rule' 2131 | + ]: 2132 | + eval_kwargs.pop(key, None) 2133 | + eval_kwargs.update(dict(metric=args.eval, **kwargs)) 2134 | + print(dataset.evaluate(outputs, **eval_kwargs)) 2135 | + 2136 | + 2137 | +if __name__ == '__main__': 2138 | + main() 2139 | --------------------------------------------------------------------------------