├── .gitattributes ├── .gitmodules ├── LICENSE ├── README.md ├── attacks.py ├── checkpoint ├── .gitkeep └── CIF10_CKPT │ └── wide_resnet_ckpt.pth ├── conf ├── __init__.py └── global_settings.py ├── data ├── attacks │ └── README ├── clean_data │ └── README ├── datasets │ └── .gitkeep ├── detection │ └── README ├── extracted_characteristics │ └── README └── weights │ └── .gitkeep ├── datasets ├── celbahq.py ├── get_dataloader.py └── smallimagenet.py ├── detect_adversarials.py ├── evaluate_detection.py ├── extract_characteristics.py ├── generate_clean_data.py ├── main_celebahq.sh ├── main_cif.sh ├── main_imagnet.sh ├── main_imagnet128.sh ├── main_imagnet3264.sh ├── models ├── .ipynb_checkpoints │ └── resnet-checkpoint.py ├── __pycache__ │ ├── orig_resnet.cpython-38.pyc │ ├── orig_resnet.cpython-39.pyc │ ├── resnet.cpython-37.pyc │ ├── resnet.cpython-38.pyc │ ├── vgg.cpython-37.pyc │ ├── vgg.cpython-38.pyc │ ├── vgg.cpython-39.pyc │ ├── vgg_cif10.cpython-39.pyc │ ├── wideresidual.cpython-38.pyc │ ├── wideresidual.cpython-39.pyc │ └── wideresidualorig.cpython-39.pyc ├── attention.py ├── densenet.py ├── googlenet.py ├── inceptionv3.py ├── inceptionv4.py ├── mobilenet.py ├── mobilenetv2.py ├── nasnet.py ├── orig_resnet.py ├── preactresnet.py ├── resnet.py ├── resnext.py ├── rir.py ├── senet.py ├── shufflenet.py ├── shufflenetv2.py ├── squeezenet.py ├── stochasticdepth.py ├── vgg.py ├── vgg_cif10.py ├── wideresidual.py ├── wideresidualorig.py └── xception.py ├── pics ├── .gitkeep └── overview_readme.png ├── requirements.yml ├── train.py ├── train_imagenet.py └── utils.py /.gitattributes: -------------------------------------------------------------------------------- 1 | checkpoint/CIF10_CKPT/wide_resnet_ckpt.pth filter=lfs diff=lfs merge=lfs -text 2 | *.pth filter=lfs diff=lfs merge=lfs -text 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "submodules/autoattack"] 2 | path = submodules/autoattack 3 | url = git@github.com:adverML/auto-attack.git 4 | branch = forspectraldefense 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Adversrial Machine Learning Benchmarks 2 | 3 | This code belongs to the papers: 4 | * `AAAI 22` [Is RobustBench/AutoAttack a suitable Benchmark for Adversarial Robustness?](https://openreview.net/forum?id=aLB3FaqoMBs) 5 | * `ICML 21` [Detecting AutoAttack Perturbations in the Frequency Domain](https://openreview.net/forum?id=8uWOTxbwo-Z) 6 | 7 | 8 | For this framework, please cite: 9 | ``` 10 | @inproceedings{ 11 | lorenz2022is, 12 | title={Is AutoAttack/AutoBench a suitable Benchmark for Adversarial Robustness?}, 13 | author={Peter Lorenz and Dominik Strassel and Margret Keuper and Janis Keuper}, 14 | booktitle={The AAAI-22 Workshop on Adversarial Machine Learning and Beyond}, 15 | year={2022}, 16 | url={https://openreview.net/forum?id=aLB3FaqoMBs} 17 | } 18 | ``` 19 | 20 | This repository is an expansion of [SpectralAdversarialDefense](https://github.com/paulaharder/SpectralAdversarialDefense), but has some **new features**: 21 | * Automatic logging. 22 | * Several runs can be saved for calculating the variance of the results. 23 | * new attack method: AutoAttack. 24 | * datasets: imagenet32, imagenet64, imagenet128, imagenet, celebahq32, celebahq64, and celebahq128. 25 | * new model: besides VGG-16 we trained a model WideResNet28-10, except for imagenet (used the standard pytorch model.) 26 | * bash scripts: Automatic starts various combination of input parameters 27 | * automatic .csv creation from all results. 28 | 29 | 30 | 31 | 32 | ## Overview 33 | 34 | ![overview](pics/overview_readme.png) 35 | 36 | This image shows the pipeline from training a model, generating adversarial examples to defend them. 37 | 38 | 1. **Training**: Models are trained. Pre-trained models are provided (WideResNet28-10: cif10, cif100, imagenet32, imagenet64, imagenet128, celebaHQ32, celebaHQ64, celebaHQ128; WideResNet51-2: ImageNet; VGG16: cif10 and cif100) 39 | 2. **Generate Clean Data**: Only correctly classfied samples are stored via `torch.save`. 40 | 3. **Attacks**: On this clean data severa atttacks can be executed: FGSM, BIM, AutoAttack (Std), PGD, DF and CW. 41 | 4. **Detect Feature**: Detectors try to distinguish between attacked and not-attacked images. 42 | 5. **Evaluation Detect**: Is the management script for handling several runs and extract the results to one `.csv` file. 43 | 44 | 45 | ## Requirements 46 | 47 | * GPUs: A100 (40GB), Titan V (12GB) or GTX 1080 (12GB) 48 | * CUDA 11.1 49 | * Python 3.9.5 50 | * PyTorch 1.9.0 51 | * cuDNN 8.0.5_0 52 | 53 | Clone the repository 54 | ```sh 55 | $ git clone --recurse-submodules https://github.com/adverML/SpectralDef_Framework 56 | $ cd SpectralDef_Framework 57 | ``` 58 | 59 | and install the requirements 60 | ```sh 61 | $ conda create --name cuda--11-1-1--pytorch--1-9-0 -f requirements.yml 62 | $ conda activate cuda--11-1-1--pytorch--1-9-0 63 | ``` 64 | 65 | There are two possiblities: Either use our data set with existing adversarial examples (not provided yet), in this case follow the instructions under 'Download' or generate the examples by yourself, by going threw 'Data generation'. For both possibilities conclude with 'Build a detector'. 66 | 67 | 68 | ### Download 69 | 70 | Download the adversarial examples (not provided yet) and their non-adversarial counterparts as well as the trained VGG-16 networks from: 71 | https://www.kaggle.com/j53t3r/weights. Extract the folders for the adversarial examples into /data and the models in the main directory. Afterwards continue with 'Build detector'. 72 | 73 | 90 | 91 | #### Datasets download 92 | 93 | These datasets are supported: 94 | * cifar10 95 | * cifar100 96 | * [ImageNet32x32](https://www.kaggle.com/j53t3r/imagenet32x32) 97 | * [ImageNet64x64](https://www.kaggle.com/j53t3r/imagenet64x64) 98 | * [ImageNet128x128](https://www.kaggle.com/j53t3r/imagenet128x128) 99 | * [ImageNet 2012](https://image-net.org/challenges/LSVRC/2012/2012-downloads.php) 100 | * [CelebaHQ 32x32 64x64 128x128 256x256](https://www.kaggle.com/j53t3r/celebahq) 101 | 102 | 103 | 104 | 105 | Download and copy the weights into `data/datasets/`. In case of troubles, adapt the paths in `conf/global_settings.py`. 106 | 107 | 108 | #### Model download 109 | 110 | To get the weights for all networks for CIFAR-10 (WideResNet 28-10 is already in this repository) and CIFAR-100, ImageNet and CelebaHQ download: 111 | 112 | 1. [Kaggle Download Weights](https://www.kaggle.com/j53t3r/weights) 113 | 2. Copy the weights into `checkpoint/`. 114 | 115 | 116 | In case of troubles, adapt the paths in `conf/global_settings.py`. You are welcome to create an issue on Github. 117 | 118 | ### Data generation 119 | 120 | Train the VGG16 on CIFAR-10: 121 | ```sh 122 | $ python train_cif10.py 123 | ``` 124 | 125 | or on CIFAR-100 126 | ```sh 127 | $ python train_cif100.py 128 | ``` 129 | 130 | 131 | 132 | 133 | The following skript will download the CIFAR-10/100 dataset and extract the CIFAR10/100 (imagenet32, imagenet64, imagenet128, celebAHQ32, ...) images, which are correctly classified by the network by running. Use --net cif10 for CIFAR-10 and --net cif100 for CIFAR-100 134 | ```sh 135 | $ # python generate_clean_data.py -h // for help 136 | $ python generate_clean_data.py --net cif10 137 | ``` 138 | 139 | Then generate the adversarial examples, argument can be fgsm (Fast Gradient Sign Method), bim (Basic Iterative Method), pgd (Projected Gradient Descent), [new] std (AutoAttack Standard), df (Deepfool), cw (Carlini and Wagner), : 140 | ```sh 141 | $ # python attack.py -h // for help 142 | $ python attack.py --attack fgsm 143 | ``` 144 | 145 | ### Build detector 146 | 147 | First extract the necessary characteristics to train a detector, choose a detector out of InputMFS (BlackBox - BB), InputPFS, LayerMFS (WhiteBox - WB), LayerPFS, LID, Mahalanobis adn an attack argument as before: 148 | 149 | ```sh 150 | ######## To Clarify from the Paper 151 | # InputMFS == BlackBox_MFS 152 | # InputPFS == BlackBox_PFS 153 | # LayerMFS == WhiteBox_MFS 154 | # LayerPFS == WhiteBox_PFS 155 | ``` 156 | 157 | Execute 158 | ```sh 159 | $ # python extract_characteristics.py -h // for help 160 | $ python extract_characteristics.py --attack fgsm --detector InputMFS 161 | ``` 162 | 163 | Then, train a classifier on the characteristics for a specific attack and detector: 164 | ```sh 165 | $ python detect_adversarials.py --attack fgsm --detector InputMFS 166 | ``` 167 | 168 | ### [new] Create csv file 169 | 170 | At the end of the file `evaluation_detection.py` different possibilities are shown: 171 | 172 | ```sh 173 | $ python evaluation_detection.py 174 | ``` 175 | 176 | Note that: `layers=False` for evaluating the detectors after the the right layers are selected. 177 | 178 | 179 | ## Other repositories used 180 | * For training the VGG-16 on CIFAR-10 we used: 181 | https://github.com/kuangliu/pytorch-cifar. 182 | * For training on CIFAR-100: 183 | https://github.com/weiaicunzai/pytorch-cifar100. 184 | * For training on imagenet32 (64 or 128) and celebaHQ32 (64 or 128) 185 | https://github.com/bearpaw/pytorch-classification. 186 | * For generating the adversarial examples we used the toolbox foolbox: 187 | https://github.com/bethgelab/foolbox. 188 | * For the LID detector we used: 189 | https://github.com/xingjunm/lid_adversarial_subspace_detection. 190 | * For the Mahalanobis detector we used: 191 | https://github.com/pokaxpoka/deep_Mahalanobis_detector. 192 | * For the AutoAttack detector we used: 193 | https://github.com/adverML/auto-attack/tree/forspectraldefense. This one is already added as: `git submodule add -b forspectraldefense git@github.com:adverML/auto-attack.git submodules/autoattack` 194 | * Other detectors: 195 | https://github.com/jayaram-r/adversarial-detection. 196 | -------------------------------------------------------------------------------- /attacks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ AutoAttack Foolbox 3 | 4 | author Peter Lorenz 5 | """ 6 | 7 | print('Load modules...') 8 | import os, sys 9 | import argparse 10 | import pdb 11 | import torch 12 | 13 | import numpy as np 14 | from tqdm import tqdm 15 | 16 | from conf import settings 17 | 18 | import foolbox 19 | from foolbox import PyTorchModel, accuracy, samples 20 | from foolbox.attacks import L2DeepFoolAttack, LinfBasicIterativeAttack, FGSM, L2CarliniWagnerAttack, FGM, PGD 21 | 22 | from utils import * 23 | 24 | 25 | if __name__ == '__main__': 26 | #processing the arguments 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument("--run_nr", default=1, type=int, help="Which run should be taken?") 29 | 30 | parser.add_argument("--attack", default='fgsm', help=settings.HELP_ATTACK) 31 | parser.add_argument("--net", default='cif10', help=settings.HELP_NET) 32 | parser.add_argument("--img_size", default='32', type=int, help=settings.HELP_IMG_SIZE) 33 | parser.add_argument("--num_classes", default='10', type=int, help=settings.HELP_NUM_CLASSES) 34 | parser.add_argument("--wanted_samples", default='2000', type=int, help=settings.HELP_WANTED_SAMPLES) 35 | parser.add_argument("--all_samples", default='4000', type=int, help="Samples from generate Clean data") 36 | 37 | 38 | # Only for Autoatack 39 | parser.add_argument('--norm', type=str, default='Linf') 40 | parser.add_argument('--eps', type=str, default='8./255.') 41 | parser.add_argument('--individual', action='store_true') 42 | parser.add_argument('--batch_size', type=int, default=1500) 43 | parser.add_argument('--log_path', type=str, default='log.txt') 44 | parser.add_argument('--version', type=str, default='standard') 45 | 46 | parser.add_argument('--net_normalization', action='store_false', help=settings.HELP_NET_NORMALIZATION) 47 | 48 | 49 | args = parser.parse_args() 50 | 51 | # output data 52 | output_path_dir = create_dir_attacks(args, root='./data/attacks/') 53 | 54 | save_args_to_file(args, output_path_dir) 55 | logger = Logger(output_path_dir + os.sep + 'log.txt') 56 | log_header(logger, args, output_path_dir, sys) # './data/attacks/imagenet32/wrn_28_10/fgsm' 57 | 58 | # check args 59 | args = check_args(args, logger) 60 | 61 | #load model 62 | logger.log('INFO: Load model...') 63 | model, preprocessing = load_model(args) 64 | 65 | model = model.eval() 66 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 67 | model = model.to(device) 68 | 69 | #load correctly classified data 70 | batch_size = 128 71 | if args.net == 'imagenet128' or args.net == 'celebaHQ128': 72 | batch_size = 64 73 | elif args.net == 'celebaHQ256': 74 | batch_size = 24 75 | elif args.net == 'imagenet': 76 | batch_size = 32 77 | 78 | # input data 79 | clean_data_path = create_dir_clean_data(args, root='./data/clean_data/') 80 | logger.log('INFO: clean data path: ' + clean_data_path) 81 | 82 | # set up final lists 83 | images = [] 84 | images_advs = [] 85 | 86 | success_counter = 0 87 | 88 | counter = 0 89 | success = [] 90 | success_rate = 0 91 | logger.log('INFO: Perform attacks...') 92 | 93 | testset = torch.load(clean_data_path + os.sep + 'clean_data')[:args.all_samples] 94 | logger.log("INFO: len(testset): {}".format(len(testset))) 95 | 96 | if args.attack == 'std' or args.attack == 'apgd-ce' or args.attack == 'apgd-t' or args.attack == 'fab-t' or args.attack == 'square': 97 | logger.log('INFO: Load data...') 98 | # testset = load_test_set(args) 99 | 100 | sys.path.append("./submodules/autoattack") 101 | from submodules.autoattack.autoattack import AutoAttack as AutoAttack_mod 102 | 103 | adversary = AutoAttack_mod(model, norm=args.norm, eps=epsilon_to_float(args.eps), log_path=output_path_dir + os.sep + 'log.txt', version=args.version) 104 | 105 | # run attack and save images 106 | with torch.no_grad(): 107 | if not args.individual: 108 | logger.log("INFO: mode: std; not individual") 109 | # raise NotImplementedError("mode: std; not individual") 110 | 111 | for x_test, y_test in testset: 112 | adv_complete, max_nr = adversary.run_standard_evaluation(x_test, y_test, bs=args.batch_size) 113 | 114 | tmp_images_advs = [] 115 | for it, img in enumerate(adv_complete): 116 | if not (np.abs(x_test[it] - img) <= 1e-5).all(): 117 | images.append(x_test[it]) 118 | tmp_images_advs.append(img) 119 | success_counter = success_counter + 1 120 | if (success_counter % 1000) == 0: 121 | get_debug_info( msg="success_counter " + str(success_counter) + " / " + str(args.wanted_samples) ) 122 | 123 | success.append( len(tmp_images_advs) / max_nr ) 124 | 125 | images_advs += tmp_images_advs 126 | tmp_images_advs = [] 127 | 128 | success_rate = np.mean(success) 129 | if success_counter >= args.wanted_samples: 130 | print( " success: {:2f}".format(success_rate) ) 131 | break 132 | else: 133 | logger.log("ERR: not implemented yet!") 134 | raise NotImplementedError("ERR: not implemented yet!") 135 | 136 | 137 | elif args.attack == 'fgsm' or args.attack == 'bim' or args.attack == 'pgd' or args.attack == 'df' or args.attack == 'cw': 138 | 139 | test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False) 140 | 141 | # total_len = len(testset) / batch_size 142 | 143 | #setup depending on attack 144 | if args.attack == 'fgsm': 145 | attack = FGSM() 146 | epsilons = [8./255.] 147 | elif args.attack == 'bim': 148 | attack = LinfBasicIterativeAttack() 149 | epsilons = [8./255.] 150 | elif args.attack == 'pgd': 151 | attack = PGD() 152 | epsilons = [8./255.] 153 | elif args.attack == 'df': 154 | attack = L2DeepFoolAttack() 155 | epsilons = None 156 | elif args.attack == 'cw': 157 | attack = L2CarliniWagnerAttack(steps=1000) 158 | epsilons = None 159 | elif args.attack == 'autoattack': 160 | logger.log("Err: Auttoattack is started from another script! attacks_autoattack.py") 161 | raise NotImplementedError("Err: Wrong Keyword use 'std' for 'ind' for AutoAttack!") 162 | else: 163 | logger.log('Err: unknown attack') 164 | raise NotImplementedError('Err: unknown attack') 165 | 166 | 167 | fmodel = PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing) 168 | 169 | logger.log('eps: {}'.format(epsilons)) 170 | 171 | multipl = 1 # 0.25 172 | # stop_round = round(total_len * 1) 173 | stop_round = args.wanted_samples 174 | 175 | 176 | for image, label in test_loader: 177 | 178 | # if counter > stop_round: 179 | # break 180 | # counter = counter + 1 181 | 182 | image = torch.squeeze(image) 183 | label = torch.squeeze(label) 184 | 185 | if batch_size == 1: 186 | image = torch.unsqueeze(image, 0) 187 | label = torch.unsqueeze(label, 0) 188 | 189 | image = image.cuda() 190 | label = label.cuda() 191 | 192 | _, adv, success = attack(fmodel, image, criterion=foolbox.criteria.Misclassification(label), epsilons=epsilons) 193 | 194 | if not (args.attack == 'cw' or args.attack == 'df'): 195 | adv = adv[0] # list to tensor 196 | success = success[0] 197 | for idx, suc in enumerate(success): 198 | counter = counter + 1 199 | if suc: 200 | images_advs.append(adv[idx].squeeze_(0)) 201 | images.append(image[idx].squeeze_(0)) 202 | success_counter = success_counter + 1 203 | 204 | # import pdb; pdb.set_trace() 205 | 206 | if success_counter >= args.wanted_samples: 207 | logger.log("INFO: wanted samples reached {}".format(args.wanted_samples)) 208 | break 209 | 210 | logger.log("INFO: len(testset): {}".format( len(testset) )) 211 | logger.log("INFO: success_counter {}".format(success_counter)) 212 | logger.log("INFO: images {}".format(len(images))) 213 | logger.log("INFO: images_advs {}".format(len(images_advs))) 214 | 215 | if args.attack == 'std' or args.individual: 216 | logger.log('INFO: attack success rate: {}'.format(success_rate) ) 217 | else: 218 | logger.log('INFO: attack success rate: {}'.format(success_counter / counter ) ) 219 | # logger.log('attack success rate: {}'.format(success_counter / len(data_loader.dataset)) ) 220 | # logger.log('attack success rate: {}'.format(success_counter / ((len(data_loader.dataset) - (len(data_loader.dataset) % batch_size)))) ) 221 | 222 | 223 | # create save dir 224 | images_path, images_advs_path = create_save_dir_path(output_path_dir, args) 225 | logger.log('images_path: ' + images_path) 226 | 227 | torch.save(images, images_path, pickle_protocol=4) 228 | torch.save(images_advs, images_advs_path, pickle_protocol=4) 229 | 230 | logger.log('INFO: Done performing attacks and adversarial examples are saved!') 231 | -------------------------------------------------------------------------------- /checkpoint/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/checkpoint/.gitkeep -------------------------------------------------------------------------------- /checkpoint/CIF10_CKPT/wide_resnet_ckpt.pth: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:93244b9cb746d585a5e321a251932f0c2654b528ed45f67f0eac3f26e1a94ccf 3 | size 146097950 4 | -------------------------------------------------------------------------------- /conf/__init__.py: -------------------------------------------------------------------------------- 1 | """ dynamically load settings 2 | 3 | author: Peter Lorenz 4 | """ 5 | import conf.global_settings as settings 6 | 7 | 8 | class Settings: 9 | def __init__(self, settings): 10 | 11 | for attr in dir(settings): 12 | if attr.isupper(): 13 | setattr(self, attr, getattr(settings, attr)) 14 | 15 | settings = Settings(settings) 16 | 17 | 18 | -------------------------------------------------------------------------------- /conf/global_settings.py: -------------------------------------------------------------------------------- 1 | """ configurations for this project 2 | 3 | from author baiyu 4 | extended by Peter Lorenz 5 | """ 6 | 7 | import os 8 | from datetime import datetime 9 | 10 | 11 | SHOW_DEBUG = True 12 | 13 | #tensorboard log dir 14 | LOG_DIR = 'runs' 15 | 16 | #save weights file per SAVE_EPOCH epoch 17 | SAVE_EPOCH = 10 18 | 19 | # Classes for each dataset 20 | MAX_CLASSES_IMAGENET = 1000 21 | MAX_CLASSES_CIF10 = 10 22 | MAX_CLASSES_CIF100 = 100 23 | MAX_CLASSES_CELEBAHQ = 4 24 | 25 | # Help Parameters Info 26 | HELP_ATTACK = "the attack method you want to use in order to create adversarial examples. Either fgsm, bim, pgd, df, cw or std (for AutoAttack: apgd-ce, apgd-t, fab-t and square.)" 27 | HELP_NET = "the dataset the net was trained on, einether cif10 or cif10vgg or cif100 or imagenet, imagenet32, imagenet64, imagenet128, celebaHQ32, celebaHQ64, celebaHQ128" 28 | HELP_NUM_CLASSES = "default: 1000; {10, 25, 50, 100, 250} only for imagenet32; 1000 classes for imagenet" 29 | HELP_WANTED_SAMPLES = "nr of samples to process" 30 | HELP_LAYER_NR = "Only for WhiteBox when you want to extract from a specific layer" 31 | HELP_DETECTOR = "The detector your want to use, out of InputMFS, InputPFS, LayerMFS, LayerPFS, LID, Mahalanobis" 32 | HELP_IMG_SIZE = "Default: 32x32 pixels; For Imagenet and CelebaHQ" 33 | HELP_MODE = "Choose test or validation case" 34 | HELP_BATCH_SIZE = "Number of batches per epoch" 35 | HELP_NET_NORMALIZATION = "Instead of data normalizeion. The data is normalized within the NN." 36 | HELP_CLF = "LR or RF" 37 | HELP_AA_EPSILONS = "epsilon: 8./255. 4/255, 3/255, 2/255, 1/255, 0.5/255" 38 | 39 | # Warnings 40 | WARN_DIR_EXISTS = "Directory already Exists! Do you want to continue?" 41 | 42 | 43 | # Weight Paths 44 | root_weights = "./checkpoint/" 45 | CIF10_CKPT = root_weights + 'CIF10_CKPT/wide_resnet_ckpt.pth' 46 | CIF10VGG_CKPT = root_weights + 'CIF10VGG_CKPT/vgg_cif10.pth' 47 | CIF100VGG_CKPT = root_weights + 'CIF100VGG_CKPT/vgg_cif100.pth' 48 | CIF100_CKPT = root_weights + 'CIF100_CKPT/model_best.pth.tar' 49 | 50 | IMAGENET32_CKPT_1000 = root_weights + 'IMAGENET32_CKPT_1000/model_best.pth.tar' # model_best.pth.tar 51 | IMAGENET32_CKPT_250 = root_weights + 'IMAGENET32_CKPT_250/model_best.pth.tar' 52 | IMAGENET32_CKPT_100 = root_weights + 'IMAGENET32_CKPT_100/model_best.pth.tar' 53 | IMAGENET32_CKPT_75 = root_weights + 'IMAGENET32_CKPT_75/model_best.pth.tar' 54 | IMAGENET32_CKPT_50 = root_weights + 'IMAGENET32_CKPT_50/model_best.pth.tar' 55 | IMAGENET32_CKPT_25 = root_weights + 'IMAGENET32_CKPT_25/model_best.pth.tar' 56 | IMAGENET32_CKPT_10 = root_weights + 'IMAGENET32_CKPT_10/model_best.pth.tar' 57 | 58 | IMAGENET64_CKPT_1000 = root_weights + 'IMAGENET64_CKPT_1000/model_best.pth.tar' 59 | IMAGENET128_CKPT_1000 = root_weights + 'IMAGENET128_CKPT_1000/model_best.pth.tar' 60 | 61 | # CELEBAHQ32_CKPT_2 = './checkpoint/wrn2810/32x32_64_0.1_Smiling_a100_Wednesday_18_August_2021_16h_02m_16s/wrn2810-175-best.pth' # '/home/lorenzp/adversialml/src/src/checkpoint/wrn2810/32x32_64_0.1_Smiling_a100_Wednesday_18_August_2021_16h_02m_16s/wrn2810-175-best.pth' 62 | CELEBAHQ32_CKPT_2 = root_weights + 'CELEBAHQ32_CKPT_2/wrn2810-161-best.pth' 63 | CELEBAHQ64_CKPT_2 = root_weights + 'CELEBAHQ64_CKPT_2/wrn2810-141-best.pth' 64 | CELEBAHQ128_CKPT_2 = root_weights + 'CELEBAHQ128_CKPT_2/wrn2810-140-best.pth' # '/home/lorenzp/adversialml/src/src/checkpoint/wrn2810/128x128_64_0.1_Smiling_a100_Sunday_22_August_2021_13h_01m_15s/wrn2810-80-regular.pth' 65 | 66 | CELEBAHQ32_CKPT_4 = root_weights + 'CELEBAHQ32_CKPT_4/wrn2810-200-best.pth' 67 | CELEBAHQ64_CKPT_4 = root_weights + 'CELEBAHQ64_CKPT_4/wrn2810-171-best.pth' 68 | CELEBAHQ128_CKPT_4 = root_weights + 'CELEBAHQ128_CKPT_4/wrn2810-100-regular.pth' # 89% 69 | CELEBAHQ256_CKPT_4 = root_weights + 'CELEBAHQ256_CKPT_4/wrn2810-70-regular.pth' # 78% 70 | 71 | 72 | # CIF10_CKPT = './checkpoint/wideresnet_2810/wide_resnet_ckpt.pth' 73 | # CIF10VGG_CKPT = './checkpoint/vgg16/original/models/vgg_cif10.pth' 74 | # CIF100VGG_CKPT = './checkpoint/vgg16/original/models/vgg_cif100.pth' 75 | # CIF100_CKPT = './../pytorch-classification/checkpoints/cifar100/wideresnet2810/model_best.pth.tar' 76 | 77 | # IMAGENET32_CKPT_1000 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810/model_best.pth.tar' # model_best.pth.tar 78 | # IMAGENET32_CKPT_250 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810_250/model_best.pth.tar' 79 | # IMAGENET32_CKPT_100 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810_100/model_best.pth.tar' 80 | # IMAGENET32_CKPT_75 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810_75/model_best.pth.tar' 81 | # IMAGENET32_CKPT_50 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810_50/model_best.pth.tar' 82 | # IMAGENET32_CKPT_25 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810_25/model_best.pth.tar' 83 | # IMAGENET32_CKPT_10 = './../pytorch-classification/checkpoints/imagenet32/wideresent2810_10/model_best.pth.tar' 84 | 85 | # IMAGENET64_CKPT_1000 = './../pytorch-classification/checkpoints/imagenet64/wideresent2810/model_best.pth.tar' 86 | # IMAGENET128_CKPT_1000 = './../pytorch-classification/checkpoints/imagenet128/wideresent2810/model_best.pth.tar' 87 | 88 | # # CELEBAHQ32_CKPT_2 = './checkpoint/wrn2810/32x32_64_0.1_Smiling_a100_Wednesday_18_August_2021_16h_02m_16s/wrn2810-175-best.pth' # '/home/lorenzp/adversialml/src/src/checkpoint/wrn2810/32x32_64_0.1_Smiling_a100_Wednesday_18_August_2021_16h_02m_16s/wrn2810-175-best.pth' 89 | # CELEBAHQ32_CKPT_2 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/32x32_128_0.1_Smiling_Thursday_30_September_2021_11h_01m_19s/wrn2810-161-best.pth' 90 | # CELEBAHQ64_CKPT_2 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/64x64_128_0.1_Smiling_Thursday_30_September_2021_15h_35m_05s/wrn2810-141-best.pth' 91 | # CELEBAHQ128_CKPT_2 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/128x128_64_0.1_Smiling_Thursday_30_September_2021_15h_37m_55s/wrn2810-140-best.pth' # '/home/lorenzp/adversialml/src/src/checkpoint/wrn2810/128x128_64_0.1_Smiling_a100_Sunday_22_August_2021_13h_01m_15s/wrn2810-80-regular.pth' 92 | 93 | # CELEBAHQ32_CKPT_4 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/32x32_64_0.1_Hair_Color_Thursday_04_November_2021_14h_35m_14s/wrn2810-200-best.pth' 94 | # CELEBAHQ64_CKPT_4 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/64x64_64_0.1_Hair_Color_Thursday_04_November_2021_17h_25m_16s/wrn2810-171-best.pth' 95 | # CELEBAHQ128_CKPT_4 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/128x128_64_0.1_Hair_Color_Thursday_04_November_2021_17h_38m_53s/wrn2810-100-regular.pth' # 89% 96 | # CELEBAHQ256_CKPT_4 = '/home/lorenzp/adversialml/src/pytorch-CelebAHQ/checkpoint/wrn2810/256x256_24_0.1_Hair_Color_Friday_05_November_2021_16h_44m_36s/wrn2810-70-regular.pth' # 78% 97 | 98 | # CSV Paths 99 | root_dataset= "./data/datasets/" 100 | # CELEBA_CSV_PATH = '../pytorch_ipynb/cnn/celeba-' # test_ 101 | CELEBA_CSV_PATH = root_dataset + 'CelebAHQ/cnn/celeba-' # test_ # https://www.kaggle.com/j53t3r/celebahq?select=cnn 102 | 103 | # Dataset Paths 104 | CIF10_PATH = root_dataset 105 | CIF100_PATH = root_dataset 106 | IMAGENET_PATH = root_dataset + "ImageNet" 107 | IMAGENET32_PATH = root_dataset + "Imagenet32x32" 108 | IMAGENET64_PATH = root_dataset + "Imagenet64x64" 109 | IMAGENET128_PATH = root_dataset + "Imagenet128x128" 110 | IMAGENET240_PATH = root_dataset + "Imagenet240x240" 111 | CELEBAHQ32_PATH = root_dataset + "CelebAHQ/Img/hq/data32x32" 112 | CELEBAHQ64_PATH = root_dataset + "CelebAHQ/Img/hq/data64x64" 113 | CELEBAHQ128_PATH = root_dataset + "CelebAHQ/Img/hq/data128x128" 114 | CELEBAHQ256_PATH = root_dataset + "CelebAHQ/Img/hq/data256x256" 115 | 116 | # CIF10_PATH = "./data" 117 | # CIF100_PATH = "./data" 118 | # IMAGENET_PATH = "/home/DATA/ITWM/ImageNet" 119 | # IMAGENET32_PATH = "/home/DATA/ITWM/Imagenet32x32" 120 | # IMAGENET64_PATH = "/home/DATA/ITWM/Imagenet64x64" 121 | # IMAGENET128_PATH = "/home/DATA/ITWM/Imagenet128x128" 122 | # IMAGENET240_PATH = "/home/DATA/ITWM/Imagenet240x240" 123 | # CELEBAHQ32_PATH = "/home/DATA/ITWM/CelebAHQ/Img/hq/data32x32" 124 | # CELEBAHQ64_PATH = "/home/DATA/ITWM/CelebAHQ/Img/hq/data64x64" 125 | # CELEBAHQ128_PATH = "/home/DATA/ITWM/CelebAHQ/Img/hq/data128x128" 126 | # CELEBAHQ256_PATH = "/home/DATA/ITWM/CelebAHQ/Img/hq/data256x256" 127 | 128 | # Detect Adversarials 129 | SAVE_CLASSIFIER = False 130 | 131 | # Extact Feaures 132 | ISSAMPLEMEANCALCULATED = False #True # Mahalannobis; set true if sample mean and precision are already calculated 133 | 134 | ATTACKS_LIST = ['fgsm', 'bim', 'pgd', 'std', 'df', 'cw'] 135 | DETECTOR_LIST_LAYERS = ['LayerPFS', 'LayerMFS'] 136 | # DETECTOR_LIST = ['InputPFS', 'InputMFS', 'LayerPFS', 'LayerMFS', 'LID', 'Mahalanobis'] 137 | DETECTOR_LIST = ['InputMFS', 'LayerMFS'] 138 | 139 | CLF = ['LR', 'RF'] 140 | 141 | -------------------------------------------------------------------------------- /data/attacks/README: -------------------------------------------------------------------------------- 1 | README 2 | 3 | 4 | Folder: attacks 5 | 6 | Script: attack_autoattack.py or attack_foolbox.py 7 | Previous script: generate_clean_data.py 8 | 9 | Correct classified images ares saved. Used: torch.save 10 | 11 | run = { run_1, run_2, run_3, run_4 } 12 | dataset = { cif10, cif10vgg, cif100, imagenet, imagenet32, imagenet64, imagenet128, celebaHQ32, celebaHQ64, celebaHQ128 } 13 | architecture = {wrn} 14 | depth = 28 15 | width = 10 16 | nrclasses = { 10, 25, 50, 75, 100, 250, 1000 } ... only for imagenet32 17 | attack = {fgsm, bim, pgd, std, df, cw } 18 | defense = {InputPFS, InputMFS, LayerPFS, LayerMFS, } 19 | 20 | 21 | Structure: 22 | 23 | attacks 24 | 25 | 26 | 27 | 28 | 29 | args.txt // the parsed arguments 30 | log.txt // logger.log, is appended; with date etc... 31 | clean_data // the data itself 32 | 33 | 34 | Example. ./data/attack/run_1/cif10/wrn_28_10_10/fgsm 35 | 36 | 37 | 38 | Debug Info: Accuracy: 39 | 40 | [x] cif10, 87% 41 | [x] cif10vgg, 82% 42 | [x] cif100, 79% 43 | [] cif100vgg, Not implemented! 44 | [x] imagenet, 77% 45 | [] imagenet32, 46 | [] imagenet32 10, 47 | [] imagenet32 25, 48 | [] imagenet32 50, 49 | [] imagenet32 75, 50 | [] imagenet32 100, 51 | [] imagenet32 250, 52 | [x] imagenet64, 70% 53 | [x] imagenet128, 79% 54 | [x] celebaHQ32, 90% not normalized 55 | [x] celebaHQ64, 91% not normalized 56 | [x] celebaHQ128 92% not normalized 57 | 58 | ... finished -------------------------------------------------------------------------------- /data/clean_data/README: -------------------------------------------------------------------------------- 1 | README 2 | 3 | 4 | Folder: clean_data 5 | 6 | Script: generate_clean_data.py 7 | Previous script: None 8 | 9 | Charactersitics are extracted from normal images and correct classified images. Used: torch.save 10 | 11 | run = { run_1, run_2, run_3, run_4 } 12 | dataset = { cif10, cif100, imagenet, imagenet32, imagenet64, imagenet128, celebaHQ32, celebaHQ64, celebaHQ128 } 13 | architecture = {wrn} 14 | depth = 28 15 | width = 10 16 | nrclasses = { 10, 25, 50, 75, 100, 250, 1000 } ... only for imagenet32 17 | attack = {fgsm, bim, pgd, std, df, cw } 18 | defense = {InputPFS, InputMFS, LayerPFS, LayerMFS, } 19 | 20 | 21 | Structure: 22 | 23 | clean_data 24 | 25 | 26 | 27 | 28 | args.txt // the parsed arguments 29 | log.txt // logger.log, is appended; with date etc... 30 | clean_data // the data itself 31 | 32 | Example. ./data/clean_data/cif10/wrn_28_10_10/ 33 | 34 | 35 | Debug Info: Accuracy: 36 | 37 | [x] cif10, 87% 38 | [x] cif10vgg, 82% 39 | [x] cif100, 79% 40 | [x] cif100vgg, 72% 41 | [x] imagenet, 77% 42 | [] imagenet32, 43 | [] imagenet32 10, 44 | [] imagenet32 25, 45 | [] imagenet32 50, 46 | [] imagenet32 75, 47 | [] imagenet32 100, 48 | [] imagenet32 250, 49 | [x] imagenet64, 70% 50 | [x] imagenet128, 79% 51 | [x] celebaHQ32, 90% 52 | [x] celebaHQ64, 91% 53 | [x] celebaHQ128 91% 54 | [x] celebaHQ256 86% 55 | 56 | ... finished 57 | -------------------------------------------------------------------------------- /data/datasets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/data/datasets/.gitkeep -------------------------------------------------------------------------------- /data/detection/README: -------------------------------------------------------------------------------- 1 | 2 | Folder: extracted_characteristics 3 | 4 | Script: detect_adversarials.py 5 | Previous script: extract_characteristics.py 6 | 7 | The data images are stored which classified correctly. Used: torch.save 8 | 9 | 10 | run = { run_1, run_2, run_3, run_4 } 11 | dataset = { cif10, cif100, imagenet, imagenet32, imagenet64, imagenet128, celebaHQ32, celebaHQ64, celebaHQ128 } 12 | architecture = {wrn} 13 | depth = 28 14 | width = 10 15 | attack = {fgsm, bim, pgd, std, df, cw } 16 | defense = {InputPFS, InputMFS, LayerPFS, LayerMFS, } 17 | clf = {LR, RF} 18 | 19 | 20 | Structure: 21 | 22 | extracted_characteristics 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | args.txt // the parsed arguments 31 | log.txt // logger.log, is appended; with date etc... 32 | .clf 33 | 34 | Example. ./data/detection/run_1/cif10/wrn_28_10_10/fgsm/InputPFS/LR 35 | 36 | Notes: 37 | 38 | run_0 git tag extract_withinputimg; has parameters with input image -------------------------------------------------------------------------------- /data/extracted_characteristics/README: -------------------------------------------------------------------------------- 1 | README 2 | 3 | 4 | Folder: extracted_characteristics 5 | 6 | Script: extract_characteristics.py 7 | Previous script: attack_autoattack.py or attack_foolbox.py 8 | 9 | The data images are stored which classified correctly. Used: torch.save 10 | 11 | 12 | run = { run_1, run_2, run_3, run_4 } 13 | dataset = { cif10, cif100, imagenet, imagenet32, imagenet64, imagenet128, celebaHQ32, celebaHQ64, celebaHQ128 } 14 | architecture = {wrn} 15 | depth = 28 16 | width = 10 17 | attack = {fgsm, bim, pgd, std, df, cw } 18 | defense = {InputPFS, InputMFS, LayerPFS, LayerMFS, } 19 | 20 | 21 | Structure: 22 | 23 | extracted_characteristics 24 | 25 | 26 | 27 | 28 | 29 | 30 | args.txt // the parsed arguments 31 | log.txt // logger.log, is appended; with date etc... 32 | images // the data itself 33 | images_adv // the data itself 34 | 35 | Example. ./data/extracted_characteristics/run_1/cif10/wrn_28_10_10/fgsm/LayerMFS/ 36 | 37 | 38 | 39 | 40 | 41 | run_0 git tag extract_withinputimg; has parameters with input image -------------------------------------------------------------------------------- /data/weights/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/data/weights/.gitkeep -------------------------------------------------------------------------------- /datasets/celbahq.py: -------------------------------------------------------------------------------- 1 | """ train and test dataset 2 | 3 | author baiyu 4 | """ 5 | import os 6 | import sys 7 | import pdb 8 | 9 | import pandas as pd 10 | from PIL import Image 11 | # import pickle 12 | 13 | # from skimage import io 14 | import matplotlib.pyplot as plt 15 | # import numpy 16 | import torch 17 | from torch.utils.data import Dataset 18 | 19 | 20 | class CelebaDataset(Dataset): 21 | """Custom Dataset for loading CelebA face images""" 22 | 23 | def __init__(self, csv_path, img_dir, data='Gender', transform=None): 24 | 25 | print("csv_path: ", csv_path) 26 | print("img_dir: ", img_dir) 27 | 28 | self.df = pd.read_csv(csv_path, index_col=0) 29 | self.img_dir = img_dir 30 | self.csv_path = csv_path 31 | self.img_names = self.df.index.values 32 | self.y = self.df[data].values 33 | self.transform = transform 34 | 35 | def __getitem__(self, index): 36 | img = Image.open(os.path.join(self.img_dir, self.img_names[index])) # R, G, B 37 | 38 | if self.transform is not None: 39 | img = self.transform(img) 40 | 41 | label = self.y[index] 42 | return img, label 43 | 44 | def __len__(self): 45 | return self.df.shape[0] 46 | 47 | 48 | class CelebaDatasetPath(Dataset): 49 | """Custom Dataset for loading CelebA face images""" 50 | 51 | def __init__(self, csv_path, img_dir, data='Gender', transform=None): 52 | 53 | print("csv_path: ", csv_path) 54 | print("img_dir: ", img_dir) 55 | 56 | self.df = pd.read_csv(csv_path, index_col=0) 57 | self.img_dir = img_dir 58 | self.csv_path = csv_path 59 | self.img_names = self.df.index.values 60 | self.y = self.df[data].values 61 | self.transform = transform 62 | 63 | def __getitem__(self, index): 64 | img = Image.open(os.path.join(self.img_dir, self.img_names[index])) 65 | 66 | if self.transform is not None: 67 | img = self.transform(img) 68 | 69 | label = self.y[index] 70 | return img, label, self.img_names[index], self.y 71 | 72 | def __len__(self): 73 | return self.df.shape[0] -------------------------------------------------------------------------------- /datasets/get_dataloader.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python3 3 | 4 | def get_dataloader(dataset_type, root_dir, is_train, batch_size, workers, resolution=32, classes=1000, **kwargs): 5 | # normalize = transforms.Normalize(mean=kwargs["mean"], std=kwargs["std"]) 6 | normalize = transforms.Normalize(mean=[0.4810, 0.4574, 0.4078], std=[0.2146, 0.2104, 0.2138]) 7 | 8 | transformations = [ 9 | transforms.RandomHorizontalFlip(), 10 | transforms.ToTensor(), 11 | normalize, 12 | ] if is_train else [ 13 | transforms.ToTensor(), 14 | normalize, 15 | ] 16 | trans = transforms.Compose(transformations) 17 | dataset = smallimagenet.SmallImagenet(root=root_dir, size=resolution, train=is_train, transform=trans, 18 | classes=range(classes)) if dataset_type == "SmallImageNet" else tinyimagenet.TinyImageNet( 19 | root=root_dir, train=is_train, transform=trans) 20 | shuffle = True # if is_train else False 21 | loader = torch.utils.data.DataLoader( 22 | dataset, batch_size=batch_size, shuffle=shuffle, 23 | num_workers=workers, pin_memory=True) 24 | return loader -------------------------------------------------------------------------------- /datasets/smallimagenet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import pdb 4 | 5 | import numpy as np 6 | from PIL import Image 7 | from torchvision.datasets.vision import VisionDataset 8 | 9 | 10 | class SmallImagenet(VisionDataset): 11 | train_list = ['train_data_batch_{}'.format(i + 1) for i in range(10)] 12 | val_list = ['val_data'] 13 | 14 | def __init__(self, root="data", size=32, train=True, transform=None, classes=None): 15 | super().__init__(root, transform=transform) 16 | file_list = self.train_list if train else self.val_list 17 | self.data = [] 18 | self.targets = [] 19 | for filename in file_list: 20 | filename = os.path.join(self.root, filename) 21 | 22 | with open(filename, 'rb') as f: 23 | entry = pickle.load(f) 24 | self.data.append(entry['data'].reshape(-1, 3, size, size)) 25 | self.targets.append(entry['labels']) 26 | 27 | self.data = np.vstack(self.data).transpose((0, 2, 3, 1)) 28 | self.targets = np.concatenate(self.targets).astype(int) - 1 29 | 30 | if classes is not None: 31 | classes = np.array(classes) 32 | filtered_data = [] 33 | filtered_targets = [] 34 | 35 | for l in classes: 36 | idxs = self.targets == l 37 | filtered_data.append(self.data[idxs]) 38 | filtered_targets.append(self.targets[idxs]) 39 | 40 | self.data = np.vstack(filtered_data) 41 | self.targets = np.concatenate(filtered_targets) 42 | 43 | def __len__(self): 44 | return len(self.data) 45 | 46 | def __getitem__(self, index): 47 | img, target = self.data[index], self.targets[index] 48 | img = Image.fromarray(img) 49 | if self.transform is not None: 50 | img = self.transform(img) 51 | return img, target 52 | -------------------------------------------------------------------------------- /detect_adversarials.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ Detect Adversarials 3 | 4 | author Peter Lorenz 5 | """ 6 | print('Load modules...') 7 | import numpy as np 8 | import pickle 9 | import torch 10 | import sys 11 | 12 | from conf import settings 13 | 14 | from utils import * 15 | 16 | from sklearn.model_selection import train_test_split 17 | from sklearn.linear_model import LogisticRegression 18 | 19 | from sklearn.ensemble import RandomForestClassifier 20 | from sklearn.metrics import confusion_matrix 21 | 22 | from sklearn.preprocessing import MinMaxScaler, StandardScaler 23 | from sklearn.metrics import roc_auc_score 24 | from sklearn.metrics import roc_curve 25 | from sklearn import svm 26 | import argparse 27 | 28 | import copy 29 | 30 | import pdb 31 | 32 | #processing the arguments 33 | parser = argparse.ArgumentParser() 34 | parser.add_argument("--run_nr", default=1, type=int, help="Which run should be taken?") 35 | 36 | parser.add_argument("--attack", default='fgsm', help=settings.HELP_ATTACK) 37 | parser.add_argument("--detector", default='InputMFS', help=settings.HELP_DETECTOR) 38 | parser.add_argument("--net", default='cif10', help=settings.HELP_NET) 39 | parser.add_argument("--mode", default='test', help="Choose test or validation case") 40 | parser.add_argument("--nr", default='-1', type=int, help=settings.HELP_LAYER_NR) 41 | parser.add_argument("--wanted_samples", default='1500', type=int, help=settings.HELP_WANTED_SAMPLES) 42 | parser.add_argument("--clf", default='LR', help="Logistic Regression (LR) or Random Forest (RF)") 43 | parser.add_argument("--num_classes", default='10', type=int, help=settings.HELP_NUM_CLASSES) 44 | 45 | # parser.add_argument("--eps", default='-1', help="epsilon: 4/255, 3/255, 2/255, 1/255, 0.5/255") 46 | parser.add_argument("--eps", default='8./255.', help=settings.HELP_AA_EPSILONS) 47 | # parser.add_argument("--eps", default='4./255.', help="epsilon: 4/255, 3/255, 2/255, 1/255, 0.5/255") 48 | # parser.add_argument("--eps", default='2./255.', help="epsilon: 4/255, 3/255, 2/255, 1/255, 0.5/255") 49 | # parser.add_argument("--eps", default='1./255.', help="epsilon: 4/255, 3/255, 2/255, 1/255, 0.5/255") 50 | # parser.add_argument("--eps", default='1./255.', help="epsilon: 4/255, 3/255, 2/255, 1/255, 0.5/255") 51 | # parser.add_argument("--eps", default='0.5/255.', help="epsilon: 4/255, 3/255, 2/255, 1/255, 0.5/255") 52 | 53 | args = parser.parse_args() 54 | 55 | # output data 56 | output_path_dir = create_dir_detection(args, root='./data/detection/') 57 | save_args_to_file(args, output_path_dir) 58 | logger = Logger(output_path_dir + os.sep + 'log.txt') 59 | log_header(logger, args, output_path_dir, sys) # './data/extracted_characteristics/imagenet32/wrn_28_10/std/8_255/LayerMFS' 60 | 61 | # load characteristics 62 | logger.log('INFO: Loading characteristics...') 63 | layer_name = layer_name_cif10 64 | if args.net == 'cif10vgg': 65 | layer_name = layer_name_cif10vgg 66 | 67 | 68 | # input data 69 | extracted_characteristics_path = create_dir_extracted_characteristics(args, root='./data/extracted_characteristics/', wait_input=False) 70 | characteristics_path, characteristics_advs_path = create_save_dir_path(extracted_characteristics_path, args, filename='characteristics' ) 71 | 72 | logger.log("characteristics_path: " + str(characteristics_path) ) 73 | logger.log("characteristics_advs_path: " + str(characteristics_advs_path) ) 74 | 75 | characteristics = torch.load(characteristics_path)[:args.wanted_samples] 76 | characteristics_adv = torch.load(characteristics_advs_path)[:args.wanted_samples] 77 | 78 | shape = np.shape(characteristics) 79 | logger.log("shape: " + str(shape)) 80 | 81 | if shape[0] < args.wanted_samples: 82 | logger.log("CAUTION: The actual number is smaller as the wanted samples!") 83 | 84 | k = shape[0] 85 | 86 | test_size = 0.2 87 | adv_X_train_val, adv_X_test, adv_y_train_val, adv_y_test = train_test_split(characteristics_adv, np.ones(k), test_size=test_size, random_state=42) 88 | b_X_train_val, b_X_test, b_y_train_val, b_y_test = train_test_split(characteristics, np.zeros(k), test_size=test_size, random_state=42) 89 | adv_X_train, adv_X_val, adv_y_train, adv_y_val = train_test_split(adv_X_train_val, adv_y_train_val, test_size=test_size, random_state=42) 90 | b_X_train, b_X_val, b_y_train, b_y_val = train_test_split(b_X_train_val, b_y_train_val, test_size=test_size, random_state=42) 91 | 92 | X_train = np.concatenate(( b_X_train, adv_X_train) ) 93 | y_train = np.concatenate(( b_y_train, adv_y_train) ) 94 | 95 | if args.mode == 'test': 96 | X_test = np.concatenate( (b_X_test, adv_X_test) ) 97 | y_test = np.concatenate( (b_y_test, adv_y_test) ) 98 | elif args.mode == 'validation': 99 | X_test = np.concatenate( (b_X_val, adv_X_val) ) 100 | y_test = np.concatenate( (b_y_val, adv_y_val) ) 101 | else: 102 | logger.log('Not a valid mode') 103 | 104 | logger.log("b_X_train" + str(b_X_train.shape) ) 105 | logger.log("adv_X_train" + str(adv_X_train.shape) ) 106 | 107 | logger.log("b_X_test" + str(b_X_test.shape) ) 108 | logger.log("adv_X_test" + str(adv_X_test.shape) ) 109 | 110 | #train classifier 111 | logger.log('Training classifier...') 112 | 113 | 114 | #special case 115 | # if (detector == 'LayerMFS'or detector =='LayerPFS') and net == 'imagenet33' and (attack_method=='std' or attack_method=='cw' or attack_method=='df'): 116 | # print("SVM") 117 | # # from cuml.svm import SVC 118 | # scaler = MinMaxScaler().fit(X_train) 119 | # X_train = scaler.transform(X_train) 120 | # X_test = scaler.transform(X_test) 121 | # if detector == 'LayerMFS': 122 | # gamma = 0.1 123 | # if attack_method == 'cw': 124 | # C=1 125 | # else: 126 | # C=10 127 | # else: 128 | # C=10 129 | # gamma = 0.01 130 | # # clf = SVC(probability=True, C=C, gamma=gamma) 131 | # clf = svm.SVC(probability=True, C=C, gamma=gamma ) # https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html 132 | # else: 133 | # # clf = RandomForestClassifier(max_depth=3, n_estimators=300) 134 | 135 | 136 | # C_list = [1, 5, 10, 15, 20] 137 | # gamma_list = [0.001, 0.01, 0.1, 1] 138 | 139 | # clf = LogisticRegression() 140 | 141 | # C_list = [1, 5, 10] 142 | # gamma_list = [0.01, 0.1] 143 | 144 | # C = 1 145 | # gamma = 0.01 146 | # clf = svm.SVC(probability=True, C=C, gamma=gamma ) 147 | 148 | 149 | # for c in C_list: 150 | # for g in gamma_list: 151 | # clf.set_params(C=c, gamma=g ) 152 | # clf.fit(X_train,y_train) 153 | # print ("C ", c, " gamma", g, "train error: ", clf.score(X_train, y_train) ) 154 | # print ("C ", c, " gamma", g, "test error: ", clf.score(X_test, y_test) ) 155 | 156 | 157 | if args.clf == 'LR' and settings.SAVE_CLASSIFIER: 158 | clf = LogisticRegression() 159 | logger.log(clf) 160 | clf.fit(X_train,y_train) 161 | logger.log("train score: " + str(clf.score(X_train, y_train)) ) 162 | logger.log("test score: " + str(clf.score(X_test, y_test)) ) 163 | 164 | if args.clf == 'RF' and settings.SAVE_CLASSIFIER: 165 | # trees = [100, 200, 300, 400, 500] 166 | # trees = [600, 700, 800, 900] 167 | # trees = [ 500 ] 168 | trees = [ 300 ] 169 | 170 | clf = RandomForestClassifier(n_estimators=100, n_jobs=-1) 171 | 172 | save_clf = copy.deepcopy(clf) 173 | test_score_save = 0 174 | 175 | for tr in trees: 176 | clf.set_params(n_estimators=tr) 177 | clf.fit(X_train, y_train) 178 | 179 | test_score = clf.score(X_test, y_test) 180 | logger.log("Tr "+ str(tr) + "train error: " + str(clf.score(X_train, y_train)) ) 181 | logger.log("Tr "+ str(tr) + "test error: " + str(test_score) ) 182 | if test_score > test_score_save: 183 | save_clf = copy.deepcopy(clf) 184 | clf = copy.deepcopy(save_clf) 185 | 186 | 187 | #save classifier 188 | classifier_pth = output_path_dir + os.sep + str(args.clf) + '.clf' 189 | if settings.SAVE_CLASSIFIER: 190 | torch.save(clf, classifier_pth) 191 | else: 192 | clf = torch.load(classifier_pth) 193 | 194 | 195 | logger.log('Evaluating classifier...') 196 | y_hat = clf.predict(X_test) 197 | y_hat_pr = clf.predict_proba(X_test)[:, 1] 198 | 199 | # logger.log( "train error: " + str(clf.score(X_train, y_train)) ) 200 | # logger.log( "test error: " + str(clf.score(X_test, y_test)) ) 201 | 202 | nr_not_detect_adv = 0 203 | 204 | benign_rate = 0 205 | benign_guesses = 0 206 | ad_guesses = 0 207 | ad_rate = 0 208 | for i in range(len(y_hat)): 209 | if y_hat[i] == 0: 210 | benign_guesses +=1 211 | if y_test[i]==0: 212 | benign_rate +=1 213 | else: 214 | ad_guesses +=1 215 | if y_test[i]==1: 216 | ad_rate +=1 217 | 218 | if y_test[i] == 1: 219 | if y_hat[i] == 0: 220 | nr_not_detect_adv +=1 221 | 222 | acc = (benign_rate+ad_rate)/len(y_hat) 223 | TP = 2*ad_rate/len(y_hat) 224 | TN = 2*benign_rate/len(y_hat) 225 | 226 | precision = ad_rate/ad_guesses 227 | 228 | TPR = 2 * ad_rate / len(y_hat) 229 | recall = round(100*TPR, 2) 230 | 231 | prec = precision 232 | rec = TPR 233 | 234 | 235 | auc = round(100*roc_auc_score(y_test, y_hat_pr), 2) 236 | acc = round(100*acc, 2) 237 | pre = round(100*precision, 1) 238 | tpr = round(100*TP, 2) 239 | f1 = round((2 * (prec*rec) / (prec+rec))*100, 2) 240 | fnr = round(100 - tpr, 2) 241 | 242 | 243 | 244 | logger.log('F1-Measure: ' + str(f1) ) 245 | logger.log('PREC: ' + str(pre) ) 246 | logger.log('ACC: ' + str(acc) ) 247 | logger.log('AUC: ' + str(auc) ) 248 | logger.log('TNR: ' + str(round(100*TN, 2)) ) # True negative rate/normal detetcion rate/selectivity is 249 | logger.log('TPR: ' + str(tpr) )# True positive rate/adversarial detetcion rate/recall/sensitivity is 250 | logger.log('FNR: ' + str(fnr) ) 251 | logger.log('RES:, AUC, ACC, PRE, TPR, F1, FNR' ) 252 | logger.log('RES:,' + str(auc) + ',' + str(acc) + ',' + str(pre) + ',' + str(tpr) + ',' + str(f1) + ',' + str(fnr) ) 253 | logger.log('<==========================================================================') 254 | 255 | -------------------------------------------------------------------------------- /generate_clean_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ Generate Clean Data 3 | 4 | author Peter Lorenz 5 | """ 6 | 7 | #this script extracts the correctly classified images 8 | print('INFO: Load modules...') 9 | import pdb 10 | import os, sys 11 | import json 12 | from conf import settings 13 | import argparse 14 | import datetime 15 | import torch 16 | import torchvision 17 | import torch.nn as nn 18 | import torchvision.transforms as transforms 19 | from collections import OrderedDict 20 | from torch.utils.data import DataLoader, TensorDataset 21 | import torchvision.datasets as datasets 22 | 23 | from utils import * 24 | 25 | from models.vgg_cif10 import VGG 26 | from models.wideresidual import WideResNet, WideBasic 27 | 28 | from datasets import smallimagenet 29 | 30 | 31 | if __name__ == '__main__': 32 | parser = argparse.ArgumentParser() 33 | parser.add_argument("--run_nr", default=1, type=int, help="Which run should be taken?") 34 | 35 | parser.add_argument("--net", default='cif10', help=settings.HELP_NET) 36 | parser.add_argument("--img_size", default=32, type=int, help=settings.HELP_IMG_SIZE) 37 | parser.add_argument("--num_classes", default=10, type=int, help=settings.HELP_NUM_CLASSES) 38 | parser.add_argument("--batch_size", default=1 , type=int, help=settings.HELP_BATCH_SIZE) 39 | parser.add_argument("--wanted_samples", default=4000, type=int, help=settings.HELP_WANTED_SAMPLES) 40 | 41 | parser.add_argument('--net_normalization', action='store_false', help=settings.HELP_NET_NORMALIZATION) 42 | 43 | args = parser.parse_args() 44 | 45 | 46 | if not args.batch_size == 1: 47 | get_debug_info(msg='Err: Batch size must be always 1!') 48 | assert True 49 | 50 | output_path_dir = create_dir_clean_data(args, root='./data/clean_data/') 51 | 52 | save_args_to_file(args, output_path_dir) 53 | logger = Logger(output_path_dir + os.sep + 'log.txt') 54 | log_header(logger, args, output_path_dir, sys) 55 | 56 | logger.log('INFO: Load model...') 57 | 58 | model, preprocessing = load_model(args) 59 | model.cuda() 60 | model.eval() 61 | 62 | 63 | logger.log('INFO: Load dataset...') 64 | test_loader = load_test_set(args, preprocessing=None) # Data Normalizations; No Net Normaliztion 65 | 66 | clean_dataset = [] 67 | correct = 0 68 | total = 0 69 | i = 0 70 | 71 | logger.log('INFO: Classify images...') 72 | 73 | for images, labels in test_loader: 74 | if i == 0: 75 | logger.log( "INFO: tensor size: " + str(images.size()) ) 76 | 77 | images = images.cuda() 78 | labels = labels.cuda() 79 | 80 | outputs = model(images) 81 | _, predicted = torch.max(outputs.data, 1) 82 | total += labels.size(0) 83 | 84 | correct += (predicted == labels).sum().item() 85 | if (predicted == labels): 86 | # clean_dataset.append(data) 87 | clean_dataset.append([images, labels]) 88 | 89 | i = i + 1 90 | if i % 500 == 0: 91 | acc = (args.wanted_samples, i, 100 * correct / total) 92 | logger.log('INFO: Accuracy of the network on the %d test images: %d, %d %%' % acc) 93 | 94 | if len(clean_dataset) >= args.wanted_samples: 95 | break 96 | 97 | # pdb.set_trace() 98 | logger.log("INFO: initial accuracy: {:.2f}".format(acc[-1])) 99 | logger.log("INFO: output_path_dir: " + output_path_dir + ", len(clean_dataset) " + str(len(clean_dataset)) ) 100 | 101 | torch.save(clean_dataset, output_path_dir + os.sep + 'clean_data', pickle_protocol=4) 102 | logger.log('INFO: Done extracting and saving correctly classified images!') 103 | -------------------------------------------------------------------------------- /main_imagnet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function log_msg { 4 | echo "`date` $@" 5 | } 6 | 7 | # DATASETS=(cif10 cif10vgg cif100 cif100vgg imagenet imagenet32 imagenet64 imagenet128 celebaHQ32 celebaHQ64 celebaHQ128) 8 | DATASETS="imagenet" 9 | ATTACKS="fgsm bim std pgd df cw" 10 | # ATTACKS="std" 11 | DETECTORS="InputPFS LayerPFS InputMFS LayerMFS LID Mahalanobis" 12 | CLF="LR RF" 13 | IMAGENET32CLASSES="25 50 100 250 1000" 14 | # NRSAMPLES="300 500 1000 1200 1500 2000" 15 | NRSAMPLES="1500" 16 | 17 | #----------------------------------------------------------------------------------------------------------------------------------- 18 | log_msg "Networks are already trained!" 19 | #----------------------------------------------------------------------------------------------------------------------------------- 20 | 21 | genereratecleandata () 22 | { 23 | log_msg "Generate Clean Data for Foolbox Attacks and Autoattack!" 24 | for net in $DATASETS; do 25 | if [ "$net" == imagenet ]; then 26 | python -u generate_clean_data.py --net "$net" --num_classes 1000 27 | fi 28 | done 29 | } 30 | 31 | #----------------------------------------------------------------------------------------------------------------------------------- 32 | attacks () 33 | { 34 | log_msg "Attack Clean Data with Foolbox Attacks and Autoattack!" 35 | 36 | for net in $DATASETS; do 37 | for att in $ATTACKS; do 38 | if [ "$net" == imagenet ]; then 39 | if [ "$att" == std ]; then 40 | python -u attacks.py --net "$net" --attack "$att" --img_size 32 --batch_size 128 --num_classes 1000 41 | else 42 | python -u attacks.py --net "$net" --attack "$att" --img_size 32 --batch_size 500 --num_classes 1000 43 | fi 44 | fi 45 | 46 | done 47 | done 48 | } 49 | 50 | #----------------------------------------------------------------------------------------------------------------------------------- 51 | extractcharacteristics () 52 | { 53 | log_msg "Extract Characteristics" 54 | 55 | for net in $DATASETS; do 56 | for att in $ATTACKS; do 57 | for det in $DETECTORS; do 58 | python -u extract_characteristics.py --net "$net" --attack "$att" --detector "$det" --num_classes 1000 59 | done 60 | done 61 | done 62 | } 63 | 64 | # #----------------------------------------------------------------------------------------------------------------------------------- 65 | detectadversarials () 66 | { 67 | log_msg "Detect Adversarials!" 68 | for net in $DATASETS; do 69 | for att in $ATTACKS; do 70 | for det in $DETECTORS; do 71 | for nrsamples in $NRSAMPLES; do 72 | for classifier in $CLF; do 73 | python -u detect_adversarials.py --net "$net" --attack "$att" --detector "$det" --wanted_samples "$nrsamples" --clf "$classifier" --num_classes 1000 74 | done 75 | done 76 | done 77 | done 78 | 79 | done 80 | } 81 | 82 | 83 | 84 | # for nr in {1,2,3,4}; do 85 | # echo "Generate Clean Data: run: $nr" 86 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/clean_data', net=['imagenet'] )" 87 | # genereratecleandata 88 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/clean_data', net=['imagenet'], dest='./log_evaluation/imagenet', run_nr=$nr)" 89 | # done 90 | 91 | 92 | # for nr in {1,2,3,4}; do 93 | # echo "Attacks: run: $nr" 94 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/attacks', net=['imagenet'] )" 95 | # attacks 96 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/attacks', net=['imagenet'], dest='./log_evaluation/imagenet', run_nr=$nr)" 97 | # done 98 | 99 | 100 | # for nr in 1; do 101 | # python -c "import evaluate_detection; evaluate_detection.copy_run_to_root(root='./data', net=['imagenet'], dest='./log_evaluation/imagenet', run_nr=$nr)" 102 | # extractcharacteristics 103 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/extracted_characteristics', net=['imagenet'], dest='./log_evaluation/imagenet', run_nr=$nr)" 104 | # done 105 | 106 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/clean_data', net=['imagenet'] )" 107 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/attacks', net=['imagenet'] )" 108 | 109 | # genereratecleandata 110 | # attacks 111 | 112 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/clean_data', net=['imagenet'], dest='./log_evaluation/imagenet', run_nr=2)" 113 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/attacks', net=['imagenet'], dest='./log_evaluation/imagenet', run_nr=2)" 114 | 115 | # extractcharacteristics 116 | # detectadversarials 117 | 118 | # #----------------------------------------------------------------------------------------------------------------------------------- 119 | log_msg "finished" 120 | exit 0 121 | 122 | # : <<'END' 123 | # just a comment! 124 | # END 125 | 126 | -------------------------------------------------------------------------------- /main_imagnet128.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | # bash main_imagnet128.sh &> log_evaluation/imagenet128/all.log 5 | 6 | function log_msg { 7 | echo "`date` $@" 8 | } 9 | 10 | # DATASETS=(cif10 cif10vgg cif100 cif100vgg imagenet imagenet32 imagenet64 imagenet128 celebaHQ32 celebaHQ64 celebaHQ128) 11 | DATASETS="imagenet128" 12 | # ATTACKS="fgsm bim pgd std df cw" 13 | ATTACKS="cw" 14 | # ATTACKS="fgsm bim pgd df cw" 15 | # ATTACKS="fgsm bim pgd std df" 16 | 17 | EPSILONS="4./255. 2./255. 1./255. 0.5/255." 18 | EPSILONS="8./255." 19 | 20 | 21 | # DETECTORS="InputPFS LayerPFS InputMFS LayerMFS LID Mahalanobis" 22 | DETECTORS="LayerMFS" 23 | CLF="LR" 24 | IMAGENET32CLASSES="25 50 100 250 1000" 25 | # NRSAMPLES="300 500 1000 1200 1500 2000" 26 | NRSAMPLES="1500" 27 | 28 | #----------------------------------------------------------------------------------------------------------------------------------- 29 | log_msg "Networks are already trained!" 30 | #----------------------------------------------------------------------------------------------------------------------------------- 31 | 32 | genereratecleandata () 33 | { 34 | log_msg "Generate Clean Data for Foolbox Attacks and Autoattack!" 35 | for net in $DATASETS; do 36 | 37 | if [ "$net" == imagenet128 ]; then 38 | python -u generate_clean_data.py --net "$net" --num_classes 1000 --img_size 128 39 | fi 40 | done 41 | } 42 | 43 | #----------------------------------------------------------------------------------------------------------------------------------- 44 | attacks () 45 | { 46 | log_msg "Attack Clean Data with Foolbox Attacks and Autoattack!" 47 | 48 | for net in $DATASETS; do 49 | 50 | for att in $ATTACKS; do 51 | 52 | if [ "$net" == imagenet128 ]; then 53 | python -u attacks.py --net "$net" --num_classes 1000 --attack "$att" --img_size 128 --batch_size 64 54 | fi 55 | 56 | done 57 | done 58 | } 59 | 60 | #----------------------------------------------------------------------------------------------------------------------------------- 61 | extractcharacteristics () 62 | { 63 | log_msg "Extract Characteristics" 64 | 65 | for net in $DATASETS; do 66 | for att in $ATTACKS; do 67 | for eps in $EPSILONS; do 68 | for det in $DETECTORS; do 69 | python -u extract_characteristics.py --net "$net" --attack "$att" --detector "$det" --num_classes 1000 --img_size 128 --wanted_samples 1600 --eps "$eps" 70 | done 71 | done 72 | done 73 | done 74 | } 75 | 76 | # #----------------------------------------------------------------------------------------------------------------------------------- 77 | detectadversarials () 78 | { 79 | log_msg "Detect Adversarials!" 80 | for net in $DATASETS; do 81 | for att in $ATTACKS; do 82 | for eps in $EPSILONS; do 83 | for det in $DETECTORS; do 84 | for nrsamples in $NRSAMPLES; do 85 | for classifier in $CLF; do 86 | python -u detect_adversarials.py --net "$net" --attack "$att" --detector "$det" --wanted_samples "$nrsamples" --clf "$classifier" --num_classes 1000 --eps "$eps" 87 | done 88 | done 89 | done 90 | done 91 | done 92 | done 93 | } 94 | 95 | # extractcharacteristics 96 | detectadversarials 97 | 98 | 99 | # python attacks.py --net imagenet128 --att std --batch_size 64 --num_classes 1000 --img_size 128 --eps 4./255. 100 | # python attacks.py --net imagenet128 --att std --batch_size 64 --num_classes 1000 --img_size 128 --eps 2./255. 101 | # python attacks.py --net imagenet128 --att std --batch_size 64 --num_classes 1000 --img_size 128 --eps 1./255. 102 | # python attacks.py --net imagenet128 --att std --batch_size 64 --num_classes 1000 --img_size 128 --eps 0.5/255. 103 | 104 | 105 | 106 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack fgsm 107 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack bim 108 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack std 109 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack pgd 110 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack df 111 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack cw 112 | 113 | 114 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack fgsm 115 | # sleep 3 116 | 117 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack bim 118 | # sleep 3 119 | 120 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack std 121 | # sleep 3 122 | 123 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack pgd 124 | # sleep 3 125 | 126 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack df 127 | # sleep 3 128 | 129 | # python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack cw 130 | 131 | 132 | 133 | # for nr in {1,2,3,4}; do 134 | # echo "Generate Clean Data: run: $nr" 135 | 136 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/clean_data', net=['imagenet128'] )" 137 | # genereratecleandata 138 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/clean_data', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=$nr)" 139 | 140 | # done 141 | 142 | 143 | # for nr in 4; do 144 | # echo "Attacks: run: $nr" 145 | # python -c "import evaluate_detection; evaluate_detection.copy_run_to_root(root='./data', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=$nr)" 146 | # attacks 147 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/attacks', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=$nr)" 148 | # done 149 | 150 | 151 | 152 | # imagenet128 153 | # for nr in 1; do 154 | # # python -c "import evaluate_detection; evaluate_detection.copy_run_to_root(root='./data', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=$nr)" 155 | # extractcharacteristics 156 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/extracted_characteristics', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=$nr)" 157 | # done 158 | 159 | # for nr in 1; do 160 | # detectadversarials 161 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/detection', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=$nr)" 162 | # done 163 | 164 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/clean_data', net=['imagenet128'] )" 165 | # python -c "import evaluate_detection; evaluate_detection.clean_root_folders( root='./data/attacks', net=['imagenet128'] )" 166 | 167 | # genereratecleandata 168 | # attacks 169 | 170 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/clean_data', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=2)" 171 | # python -c "import evaluate_detection; evaluate_detection.copy_run_dest(root='./data/attacks', net=['imagenet128'], dest='./log_evaluation/imagenet128', run_nr=2)" 172 | 173 | # extractcharacteristics 174 | # detectadversarials 175 | 176 | # #----------------------------------------------------------------------------------------------------------------------------------- 177 | log_msg "finished" 178 | exit 0 179 | 180 | # : <<'END' 181 | # just a comment! 182 | # END 183 | 184 | 185 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack fgsm 186 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack bim 187 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack std 188 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack pgd 189 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack df 190 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector InputMFS --img_size 128 --attack cw 191 | 192 | 193 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack fgsm 194 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack bim 195 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack std 196 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack pgd 197 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack df 198 | python -u extract_characteristics.py --net imagenet128 --num_classes 1000 --detector LayerMFS --img_size 128 --attack cw -------------------------------------------------------------------------------- /models/.ipynb_checkpoints/resnet-checkpoint.py: -------------------------------------------------------------------------------- 1 | """resnet in pytorch 2 | 3 | 4 | 5 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 6 | 7 | Deep Residual Learning for Image Recognition 8 | https://arxiv.org/abs/1512.03385v1 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | 14 | class BasicBlock(nn.Module): 15 | """Basic Block for resnet 18 and resnet 34 16 | 17 | """ 18 | 19 | #BasicBlock and BottleNeck block 20 | #have different output size 21 | #we use class attribute expansion 22 | #to distinct 23 | expansion = 1 24 | 25 | def __init__(self, in_channels, out_channels, stride=1): 26 | super().__init__() 27 | 28 | #residual function 29 | self.residual_function = nn.Sequential( 30 | nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False), 31 | nn.BatchNorm2d(out_channels), 32 | nn.ReLU(inplace=True), 33 | nn.Conv2d(out_channels, out_channels * BasicBlock.expansion, kernel_size=3, padding=1, bias=False), 34 | nn.BatchNorm2d(out_channels * BasicBlock.expansion) 35 | ) 36 | 37 | #shortcut 38 | self.shortcut = nn.Sequential() 39 | 40 | #the shortcut output dimension is not the same with residual function 41 | #use 1*1 convolution to match the dimension 42 | if stride != 1 or in_channels != BasicBlock.expansion * out_channels: 43 | self.shortcut = nn.Sequential( 44 | nn.Conv2d(in_channels, out_channels * BasicBlock.expansion, kernel_size=1, stride=stride, bias=False), 45 | nn.BatchNorm2d(out_channels * BasicBlock.expansion) 46 | ) 47 | 48 | def forward(self, x): 49 | return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) 50 | 51 | class BottleNeck(nn.Module): 52 | """Residual block for resnet over 50 layers 53 | 54 | """ 55 | expansion = 4 56 | def __init__(self, in_channels, out_channels, stride=1): 57 | super().__init__() 58 | self.residual_function = nn.Sequential( 59 | nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False), 60 | nn.BatchNorm2d(out_channels), 61 | nn.ReLU(inplace=True), 62 | nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False), 63 | nn.BatchNorm2d(out_channels), 64 | nn.ReLU(inplace=True), 65 | nn.Conv2d(out_channels, out_channels * BottleNeck.expansion, kernel_size=1, bias=False), 66 | nn.BatchNorm2d(out_channels * BottleNeck.expansion), 67 | ) 68 | 69 | self.shortcut = nn.Sequential() 70 | 71 | if stride != 1 or in_channels != out_channels * BottleNeck.expansion: 72 | self.shortcut = nn.Sequential( 73 | nn.Conv2d(in_channels, out_channels * BottleNeck.expansion, stride=stride, kernel_size=1, bias=False), 74 | nn.BatchNorm2d(out_channels * BottleNeck.expansion) 75 | ) 76 | 77 | def forward(self, x): 78 | return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) 79 | 80 | class ResNet(nn.Module): 81 | 82 | def __init__(self, block, num_block, num_classes=100): 83 | super().__init__() 84 | 85 | self.in_channels = 64 86 | 87 | self.conv1 = nn.Sequential( 88 | nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False), 89 | nn.BatchNorm2d(64), 90 | nn.ReLU(inplace=True)) 91 | #we use a different inputsize than the original paper 92 | #so conv2_x's stride is 1 93 | self.conv2_x = self._make_layer(block, 64, num_block[0], 1) 94 | self.conv3_x = self._make_layer(block, 128, num_block[1], 2) 95 | self.conv4_x = self._make_layer(block, 256, num_block[2], 2) 96 | self.conv5_x = self._make_layer(block, 512, num_block[3], 2) 97 | self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) 98 | self.fc = nn.Linear(512 * block.expansion, num_classes) 99 | 100 | def _make_layer(self, block, out_channels, num_blocks, stride): 101 | """make resnet layers(by layer i didnt mean this 'layer' was the 102 | same as a neuron netowork layer, ex. conv layer), one layer may 103 | contain more than one residual block 104 | 105 | Args: 106 | block: block type, basic block or bottle neck block 107 | out_channels: output depth channel number of this layer 108 | num_blocks: how many blocks per layer 109 | stride: the stride of the first block of this layer 110 | 111 | Return: 112 | return a resnet layer 113 | """ 114 | 115 | # we have num_block blocks per layer, the first block 116 | # could be 1 or 2, other blocks would always be 1 117 | strides = [stride] + [1] * (num_blocks - 1) 118 | layers = [] 119 | for stride in strides: 120 | layers.append(block(self.in_channels, out_channels, stride)) 121 | self.in_channels = out_channels * block.expansion 122 | 123 | return nn.Sequential(*layers) 124 | 125 | def forward(self, x): 126 | output = self.conv1(x) 127 | output = self.conv2_x(output) 128 | output = self.conv3_x(output) 129 | output = self.conv4_x(output) 130 | output = self.conv5_x(output) 131 | output = self.avg_pool(output) 132 | output = output.view(output.size(0), -1) 133 | output = self.fc(output) 134 | 135 | return output 136 | 137 | def resnet18(): 138 | """ return a ResNet 18 object 139 | """ 140 | return ResNet(BasicBlock, [2, 2, 2, 2]) 141 | 142 | def resnet34(): 143 | """ return a ResNet 34 object 144 | """ 145 | return ResNet(BasicBlock, [3, 4, 6, 3]) 146 | 147 | def resnet50(): 148 | """ return a ResNet 50 object 149 | """ 150 | return ResNet(BottleNeck, [3, 4, 6, 3]) 151 | 152 | def resnet101(): 153 | """ return a ResNet 101 object 154 | """ 155 | return ResNet(BottleNeck, [3, 4, 23, 3]) 156 | 157 | def resnet152(): 158 | """ return a ResNet 152 object 159 | """ 160 | return ResNet(BottleNeck, [3, 8, 36, 3]) 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /models/__pycache__/orig_resnet.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/orig_resnet.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/orig_resnet.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/orig_resnet.cpython-39.pyc -------------------------------------------------------------------------------- /models/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/resnet.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/resnet.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/vgg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/vgg.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/vgg.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/vgg.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/vgg.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/vgg.cpython-39.pyc -------------------------------------------------------------------------------- /models/__pycache__/vgg_cif10.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/vgg_cif10.cpython-39.pyc -------------------------------------------------------------------------------- /models/__pycache__/wideresidual.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/wideresidual.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/wideresidual.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/wideresidual.cpython-39.pyc -------------------------------------------------------------------------------- /models/__pycache__/wideresidualorig.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/models/__pycache__/wideresidualorig.cpython-39.pyc -------------------------------------------------------------------------------- /models/attention.py: -------------------------------------------------------------------------------- 1 | """residual attention network in pytorch 2 | 3 | 4 | 5 | [1] Fei Wang, Mengqing Jiang, Chen Qian, Shuo Yang, Cheng Li, Honggang Zhang, Xiaogang Wang, Xiaoou Tang 6 | 7 | Residual Attention Network for Image Classification 8 | https://arxiv.org/abs/1704.06904 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.functional as F 14 | 15 | #"""The Attention Module is built by pre-activation Residual Unit [11] with the 16 | #number of channels in each stage is the same as ResNet [10].""" 17 | 18 | class PreActResidualUnit(nn.Module): 19 | """PreAct Residual Unit 20 | Args: 21 | in_channels: residual unit input channel number 22 | out_channels: residual unit output channel numebr 23 | stride: stride of residual unit when stride = 2, downsample the featuremap 24 | """ 25 | 26 | def __init__(self, in_channels, out_channels, stride): 27 | super().__init__() 28 | 29 | bottleneck_channels = int(out_channels / 4) 30 | self.residual_function = nn.Sequential( 31 | #1x1 conv 32 | nn.BatchNorm2d(in_channels), 33 | nn.ReLU(inplace=True), 34 | nn.Conv2d(in_channels, bottleneck_channels, 1, stride), 35 | 36 | #3x3 conv 37 | nn.BatchNorm2d(bottleneck_channels), 38 | nn.ReLU(inplace=True), 39 | nn.Conv2d(bottleneck_channels, bottleneck_channels, 3, padding=1), 40 | 41 | #1x1 conv 42 | nn.BatchNorm2d(bottleneck_channels), 43 | nn.ReLU(inplace=True), 44 | nn.Conv2d(bottleneck_channels, out_channels, 1) 45 | ) 46 | 47 | self.shortcut = nn.Sequential() 48 | if stride != 2 or (in_channels != out_channels): 49 | self.shortcut = nn.Conv2d(in_channels, out_channels, 1, stride=stride) 50 | 51 | def forward(self, x): 52 | 53 | res = self.residual_function(x) 54 | shortcut = self.shortcut(x) 55 | 56 | return res + shortcut 57 | 58 | class AttentionModule1(nn.Module): 59 | 60 | def __init__(self, in_channels, out_channels, p=1, t=2, r=1): 61 | super().__init__() 62 | #"""The hyperparameter p denotes the number of preprocessing Residual 63 | #Units before splitting into trunk branch and mask branch. t denotes 64 | #the number of Residual Units in trunk branch. r denotes the number of 65 | #Residual Units between adjacent pooling layer in the mask branch.""" 66 | assert in_channels == out_channels 67 | 68 | self.pre = self._make_residual(in_channels, out_channels, p) 69 | self.trunk = self._make_residual(in_channels, out_channels, t) 70 | self.soft_resdown1 = self._make_residual(in_channels, out_channels, r) 71 | self.soft_resdown2 = self._make_residual(in_channels, out_channels, r) 72 | self.soft_resdown3 = self._make_residual(in_channels, out_channels, r) 73 | self.soft_resdown4 = self._make_residual(in_channels, out_channels, r) 74 | 75 | self.soft_resup1 = self._make_residual(in_channels, out_channels, r) 76 | self.soft_resup2 = self._make_residual(in_channels, out_channels, r) 77 | self.soft_resup3 = self._make_residual(in_channels, out_channels, r) 78 | self.soft_resup4 = self._make_residual(in_channels, out_channels, r) 79 | 80 | self.shortcut_short = PreActResidualUnit(in_channels, out_channels, 1) 81 | self.shortcut_long = PreActResidualUnit(in_channels, out_channels, 1) 82 | 83 | self.sigmoid = nn.Sequential( 84 | nn.BatchNorm2d(out_channels), 85 | nn.ReLU(inplace=True), 86 | nn.Conv2d(out_channels, out_channels, kernel_size=1), 87 | nn.BatchNorm2d(out_channels), 88 | nn.ReLU(inplace=True), 89 | nn.Conv2d(out_channels, out_channels, kernel_size=1), 90 | nn.Sigmoid() 91 | ) 92 | 93 | self.last = self._make_residual(in_channels, out_channels, p) 94 | 95 | def forward(self, x): 96 | ###We make the size of the smallest output map in each mask branch 7*7 to be consistent 97 | #with the smallest trunk output map size. 98 | ###Thus 3,2,1 max-pooling layers are used in mask branch with input size 56 * 56, 28 * 28, 14 * 14 respectively. 99 | x = self.pre(x) 100 | input_size = (x.size(2), x.size(3)) 101 | 102 | x_t = self.trunk(x) 103 | 104 | #first downsample out 28 105 | x_s = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) 106 | x_s = self.soft_resdown1(x_s) 107 | 108 | #28 shortcut 109 | shape1 = (x_s.size(2), x_s.size(3)) 110 | shortcut_long = self.shortcut_long(x_s) 111 | 112 | #seccond downsample out 14 113 | x_s = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) 114 | x_s = self.soft_resdown2(x_s) 115 | 116 | #14 shortcut 117 | shape2 = (x_s.size(2), x_s.size(3)) 118 | shortcut_short = self.soft_resdown3(x_s) 119 | 120 | #third downsample out 7 121 | x_s = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) 122 | x_s = self.soft_resdown3(x_s) 123 | 124 | #mid 125 | x_s = self.soft_resdown4(x_s) 126 | x_s = self.soft_resup1(x_s) 127 | 128 | #first upsample out 14 129 | x_s = self.soft_resup2(x_s) 130 | x_s = F.interpolate(x_s, size=shape2) 131 | x_s += shortcut_short 132 | 133 | #second upsample out 28 134 | x_s = self.soft_resup3(x_s) 135 | x_s = F.interpolate(x_s, size=shape1) 136 | x_s += shortcut_long 137 | 138 | #thrid upsample out 54 139 | x_s = self.soft_resup4(x_s) 140 | x_s = F.interpolate(x_s, size=input_size) 141 | 142 | x_s = self.sigmoid(x_s) 143 | x = (1 + x_s) * x_t 144 | x = self.last(x) 145 | 146 | return x 147 | 148 | def _make_residual(self, in_channels, out_channels, p): 149 | 150 | layers = [] 151 | for _ in range(p): 152 | layers.append(PreActResidualUnit(in_channels, out_channels, 1)) 153 | 154 | return nn.Sequential(*layers) 155 | 156 | class AttentionModule2(nn.Module): 157 | 158 | def __init__(self, in_channels, out_channels, p=1, t=2, r=1): 159 | super().__init__() 160 | #"""The hyperparameter p denotes the number of preprocessing Residual 161 | #Units before splitting into trunk branch and mask branch. t denotes 162 | #the number of Residual Units in trunk branch. r denotes the number of 163 | #Residual Units between adjacent pooling layer in the mask branch.""" 164 | assert in_channels == out_channels 165 | 166 | self.pre = self._make_residual(in_channels, out_channels, p) 167 | self.trunk = self._make_residual(in_channels, out_channels, t) 168 | self.soft_resdown1 = self._make_residual(in_channels, out_channels, r) 169 | self.soft_resdown2 = self._make_residual(in_channels, out_channels, r) 170 | self.soft_resdown3 = self._make_residual(in_channels, out_channels, r) 171 | 172 | self.soft_resup1 = self._make_residual(in_channels, out_channels, r) 173 | self.soft_resup2 = self._make_residual(in_channels, out_channels, r) 174 | self.soft_resup3 = self._make_residual(in_channels, out_channels, r) 175 | 176 | self.shortcut = PreActResidualUnit(in_channels, out_channels, 1) 177 | 178 | self.sigmoid = nn.Sequential( 179 | nn.BatchNorm2d(out_channels), 180 | nn.ReLU(inplace=True), 181 | nn.Conv2d(out_channels, out_channels, kernel_size=1), 182 | nn.BatchNorm2d(out_channels), 183 | nn.ReLU(inplace=True), 184 | nn.Conv2d(out_channels, out_channels, kernel_size=1), 185 | nn.Sigmoid() 186 | ) 187 | 188 | self.last = self._make_residual(in_channels, out_channels, p) 189 | 190 | def forward(self, x): 191 | x = self.pre(x) 192 | input_size = (x.size(2), x.size(3)) 193 | 194 | x_t = self.trunk(x) 195 | 196 | #first downsample out 14 197 | x_s = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) 198 | x_s = self.soft_resdown1(x_s) 199 | 200 | #14 shortcut 201 | shape1 = (x_s.size(2), x_s.size(3)) 202 | shortcut = self.shortcut(x_s) 203 | 204 | #seccond downsample out 7 205 | x_s = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) 206 | x_s = self.soft_resdown2(x_s) 207 | 208 | #mid 209 | x_s = self.soft_resdown3(x_s) 210 | x_s = self.soft_resup1(x_s) 211 | 212 | #first upsample out 14 213 | x_s = self.soft_resup2(x_s) 214 | x_s = F.interpolate(x_s, size=shape1) 215 | x_s += shortcut 216 | 217 | #second upsample out 28 218 | x_s = self.soft_resup3(x_s) 219 | x_s = F.interpolate(x_s, size=input_size) 220 | 221 | x_s = self.sigmoid(x_s) 222 | x = (1 + x_s) * x_t 223 | x = self.last(x) 224 | 225 | return x 226 | 227 | def _make_residual(self, in_channels, out_channels, p): 228 | 229 | layers = [] 230 | for _ in range(p): 231 | layers.append(PreActResidualUnit(in_channels, out_channels, 1)) 232 | 233 | return nn.Sequential(*layers) 234 | 235 | class AttentionModule3(nn.Module): 236 | 237 | def __init__(self, in_channels, out_channels, p=1, t=2, r=1): 238 | super().__init__() 239 | 240 | assert in_channels == out_channels 241 | 242 | self.pre = self._make_residual(in_channels, out_channels, p) 243 | self.trunk = self._make_residual(in_channels, out_channels, t) 244 | self.soft_resdown1 = self._make_residual(in_channels, out_channels, r) 245 | self.soft_resdown2 = self._make_residual(in_channels, out_channels, r) 246 | 247 | self.soft_resup1 = self._make_residual(in_channels, out_channels, r) 248 | self.soft_resup2 = self._make_residual(in_channels, out_channels, r) 249 | 250 | self.shortcut = PreActResidualUnit(in_channels, out_channels, 1) 251 | 252 | self.sigmoid = nn.Sequential( 253 | nn.BatchNorm2d(out_channels), 254 | nn.ReLU(inplace=True), 255 | nn.Conv2d(out_channels, out_channels, kernel_size=1), 256 | nn.BatchNorm2d(out_channels), 257 | nn.ReLU(inplace=True), 258 | nn.Conv2d(out_channels, out_channels, kernel_size=1), 259 | nn.Sigmoid() 260 | ) 261 | 262 | self.last = self._make_residual(in_channels, out_channels, p) 263 | 264 | def forward(self, x): 265 | x = self.pre(x) 266 | input_size = (x.size(2), x.size(3)) 267 | 268 | x_t = self.trunk(x) 269 | 270 | #first downsample out 14 271 | x_s = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) 272 | x_s = self.soft_resdown1(x_s) 273 | 274 | #mid 275 | x_s = self.soft_resdown2(x_s) 276 | x_s = self.soft_resup1(x_s) 277 | 278 | #first upsample out 14 279 | x_s = self.soft_resup2(x_s) 280 | x_s = F.interpolate(x_s, size=input_size) 281 | 282 | x_s = self.sigmoid(x_s) 283 | x = (1 + x_s) * x_t 284 | x = self.last(x) 285 | 286 | return x 287 | 288 | def _make_residual(self, in_channels, out_channels, p): 289 | 290 | layers = [] 291 | for _ in range(p): 292 | layers.append(PreActResidualUnit(in_channels, out_channels, 1)) 293 | 294 | return nn.Sequential(*layers) 295 | 296 | class Attention(nn.Module): 297 | """residual attention netowrk 298 | Args: 299 | block_num: attention module number for each stage 300 | """ 301 | 302 | def __init__(self, block_num, class_num=100): 303 | 304 | super().__init__() 305 | self.pre_conv = nn.Sequential( 306 | nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1), 307 | nn.BatchNorm2d(64), 308 | nn.ReLU(inplace=True) 309 | ) 310 | 311 | self.stage1 = self._make_stage(64, 256, block_num[0], AttentionModule1) 312 | self.stage2 = self._make_stage(256, 512, block_num[1], AttentionModule2) 313 | self.stage3 = self._make_stage(512, 1024, block_num[2], AttentionModule3) 314 | self.stage4 = nn.Sequential( 315 | PreActResidualUnit(1024, 2048, 2), 316 | PreActResidualUnit(2048, 2048, 1), 317 | PreActResidualUnit(2048, 2048, 1) 318 | ) 319 | self.avg = nn.AdaptiveAvgPool2d(1) 320 | self.linear = nn.Linear(2048, 100) 321 | 322 | def forward(self, x): 323 | x = self.pre_conv(x) 324 | x = self.stage1(x) 325 | x = self.stage2(x) 326 | x = self.stage3(x) 327 | x = self.stage4(x) 328 | x = self.avg(x) 329 | x = x.view(x.size(0), -1) 330 | x = self.linear(x) 331 | 332 | return x 333 | 334 | def _make_stage(self, in_channels, out_channels, num, block): 335 | 336 | layers = [] 337 | layers.append(PreActResidualUnit(in_channels, out_channels, 2)) 338 | 339 | for _ in range(num): 340 | layers.append(block(out_channels, out_channels)) 341 | 342 | return nn.Sequential(*layers) 343 | 344 | def attention56(): 345 | return Attention([1, 1, 1]) 346 | 347 | def attention92(): 348 | return Attention([1, 2, 3]) 349 | 350 | -------------------------------------------------------------------------------- /models/densenet.py: -------------------------------------------------------------------------------- 1 | """dense net in pytorch 2 | 3 | 4 | 5 | [1] Gao Huang, Zhuang Liu, Laurens van der Maaten, Kilian Q. Weinberger. 6 | 7 | Densely Connected Convolutional Networks 8 | https://arxiv.org/abs/1608.06993v5 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | 14 | 15 | 16 | #"""Bottleneck layers. Although each layer only produces k 17 | #output feature-maps, it typically has many more inputs. It 18 | #has been noted in [37, 11] that a 1×1 convolution can be in- 19 | #troduced as bottleneck layer before each 3×3 convolution 20 | #to reduce the number of input feature-maps, and thus to 21 | #improve computational efficiency.""" 22 | class Bottleneck(nn.Module): 23 | def __init__(self, in_channels, growth_rate): 24 | super().__init__() 25 | #"""In our experiments, we let each 1×1 convolution 26 | #produce 4k feature-maps.""" 27 | inner_channel = 4 * growth_rate 28 | 29 | #"""We find this design especially effective for DenseNet and 30 | #we refer to our network with such a bottleneck layer, i.e., 31 | #to the BN-ReLU-Conv(1×1)-BN-ReLU-Conv(3×3) version of H ` , 32 | #as DenseNet-B.""" 33 | self.bottle_neck = nn.Sequential( 34 | nn.BatchNorm2d(in_channels), 35 | nn.ReLU(inplace=True), 36 | nn.Conv2d(in_channels, inner_channel, kernel_size=1, bias=False), 37 | nn.BatchNorm2d(inner_channel), 38 | nn.ReLU(inplace=True), 39 | nn.Conv2d(inner_channel, growth_rate, kernel_size=3, padding=1, bias=False) 40 | ) 41 | 42 | def forward(self, x): 43 | return torch.cat([x, self.bottle_neck(x)], 1) 44 | 45 | #"""We refer to layers between blocks as transition 46 | #layers, which do convolution and pooling.""" 47 | class Transition(nn.Module): 48 | def __init__(self, in_channels, out_channels): 49 | super().__init__() 50 | #"""The transition layers used in our experiments 51 | #consist of a batch normalization layer and an 1×1 52 | #convolutional layer followed by a 2×2 average pooling 53 | #layer""". 54 | self.down_sample = nn.Sequential( 55 | nn.BatchNorm2d(in_channels), 56 | nn.Conv2d(in_channels, out_channels, 1, bias=False), 57 | nn.AvgPool2d(2, stride=2) 58 | ) 59 | 60 | def forward(self, x): 61 | return self.down_sample(x) 62 | 63 | #DesneNet-BC 64 | #B stands for bottleneck layer(BN-RELU-CONV(1x1)-BN-RELU-CONV(3x3)) 65 | #C stands for compression factor(0<=theta<=1) 66 | class DenseNet(nn.Module): 67 | def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_class=100): 68 | super().__init__() 69 | self.growth_rate = growth_rate 70 | 71 | #"""Before entering the first dense block, a convolution 72 | #with 16 (or twice the growth rate for DenseNet-BC) 73 | #output channels is performed on the input images.""" 74 | inner_channels = 2 * growth_rate 75 | 76 | #For convolutional layers with kernel size 3×3, each 77 | #side of the inputs is zero-padded by one pixel to keep 78 | #the feature-map size fixed. 79 | self.conv1 = nn.Conv2d(3, inner_channels, kernel_size=3, padding=1, bias=False) 80 | 81 | self.features = nn.Sequential() 82 | 83 | for index in range(len(nblocks) - 1): 84 | self.features.add_module("dense_block_layer_{}".format(index), self._make_dense_layers(block, inner_channels, nblocks[index])) 85 | inner_channels += growth_rate * nblocks[index] 86 | 87 | #"""If a dense block contains m feature-maps, we let the 88 | #following transition layer generate θm output feature- 89 | #maps, where 0 < θ ≤ 1 is referred to as the compression 90 | #fac-tor. 91 | out_channels = int(reduction * inner_channels) # int() will automatic floor the value 92 | self.features.add_module("transition_layer_{}".format(index), Transition(inner_channels, out_channels)) 93 | inner_channels = out_channels 94 | 95 | self.features.add_module("dense_block{}".format(len(nblocks) - 1), self._make_dense_layers(block, inner_channels, nblocks[len(nblocks)-1])) 96 | inner_channels += growth_rate * nblocks[len(nblocks) - 1] 97 | self.features.add_module('bn', nn.BatchNorm2d(inner_channels)) 98 | self.features.add_module('relu', nn.ReLU(inplace=True)) 99 | 100 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 101 | 102 | self.linear = nn.Linear(inner_channels, num_class) 103 | 104 | def forward(self, x): 105 | output = self.conv1(x) 106 | output = self.features(output) 107 | output = self.avgpool(output) 108 | output = output.view(output.size()[0], -1) 109 | output = self.linear(output) 110 | return output 111 | 112 | def _make_dense_layers(self, block, in_channels, nblocks): 113 | dense_block = nn.Sequential() 114 | for index in range(nblocks): 115 | dense_block.add_module('bottle_neck_layer_{}'.format(index), block(in_channels, self.growth_rate)) 116 | in_channels += self.growth_rate 117 | return dense_block 118 | 119 | def densenet121(): 120 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=32) 121 | 122 | def densenet169(): 123 | return DenseNet(Bottleneck, [6,12,32,32], growth_rate=32) 124 | 125 | def densenet201(): 126 | return DenseNet(Bottleneck, [6,12,48,32], growth_rate=32) 127 | 128 | def densenet161(): 129 | return DenseNet(Bottleneck, [6,12,36,24], growth_rate=48) 130 | 131 | -------------------------------------------------------------------------------- /models/googlenet.py: -------------------------------------------------------------------------------- 1 | """google net in pytorch 2 | 3 | 4 | 5 | [1] Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, 6 | Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. 7 | 8 | Going Deeper with Convolutions 9 | https://arxiv.org/abs/1409.4842v1 10 | """ 11 | 12 | import torch 13 | import torch.nn as nn 14 | 15 | class Inception(nn.Module): 16 | def __init__(self, input_channels, n1x1, n3x3_reduce, n3x3, n5x5_reduce, n5x5, pool_proj): 17 | super().__init__() 18 | 19 | #1x1conv branch 20 | self.b1 = nn.Sequential( 21 | nn.Conv2d(input_channels, n1x1, kernel_size=1), 22 | nn.BatchNorm2d(n1x1), 23 | nn.ReLU(inplace=True) 24 | ) 25 | 26 | #1x1conv -> 3x3conv branch 27 | self.b2 = nn.Sequential( 28 | nn.Conv2d(input_channels, n3x3_reduce, kernel_size=1), 29 | nn.BatchNorm2d(n3x3_reduce), 30 | nn.ReLU(inplace=True), 31 | nn.Conv2d(n3x3_reduce, n3x3, kernel_size=3, padding=1), 32 | nn.BatchNorm2d(n3x3), 33 | nn.ReLU(inplace=True) 34 | ) 35 | 36 | #1x1conv -> 5x5conv branch 37 | #we use 2 3x3 conv filters stacked instead 38 | #of 1 5x5 filters to obtain the same receptive 39 | #field with fewer parameters 40 | self.b3 = nn.Sequential( 41 | nn.Conv2d(input_channels, n5x5_reduce, kernel_size=1), 42 | nn.BatchNorm2d(n5x5_reduce), 43 | nn.ReLU(inplace=True), 44 | nn.Conv2d(n5x5_reduce, n5x5, kernel_size=3, padding=1), 45 | nn.BatchNorm2d(n5x5, n5x5), 46 | nn.ReLU(inplace=True), 47 | nn.Conv2d(n5x5, n5x5, kernel_size=3, padding=1), 48 | nn.BatchNorm2d(n5x5), 49 | nn.ReLU(inplace=True) 50 | ) 51 | 52 | #3x3pooling -> 1x1conv 53 | #same conv 54 | self.b4 = nn.Sequential( 55 | nn.MaxPool2d(3, stride=1, padding=1), 56 | nn.Conv2d(input_channels, pool_proj, kernel_size=1), 57 | nn.BatchNorm2d(pool_proj), 58 | nn.ReLU(inplace=True) 59 | ) 60 | 61 | def forward(self, x): 62 | return torch.cat([self.b1(x), self.b2(x), self.b3(x), self.b4(x)], dim=1) 63 | 64 | 65 | class GoogleNet(nn.Module): 66 | 67 | def __init__(self, num_class=100): 68 | super().__init__() 69 | self.prelayer = nn.Sequential( 70 | nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False), 71 | nn.BatchNorm2d(64), 72 | nn.ReLU(inplace=True), 73 | nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False), 74 | nn.BatchNorm2d(64), 75 | nn.ReLU(inplace=True), 76 | nn.Conv2d(64, 192, kernel_size=3, padding=1, bias=False), 77 | nn.BatchNorm2d(192), 78 | nn.ReLU(inplace=True), 79 | ) 80 | 81 | #although we only use 1 conv layer as prelayer, 82 | #we still use name a3, b3....... 83 | self.a3 = Inception(192, 64, 96, 128, 16, 32, 32) 84 | self.b3 = Inception(256, 128, 128, 192, 32, 96, 64) 85 | 86 | ##"""In general, an Inception network is a network consisting of 87 | ##modules of the above type stacked upon each other, with occasional 88 | ##max-pooling layers with stride 2 to halve the resolution of the 89 | ##grid""" 90 | self.maxpool = nn.MaxPool2d(3, stride=2, padding=1) 91 | 92 | self.a4 = Inception(480, 192, 96, 208, 16, 48, 64) 93 | self.b4 = Inception(512, 160, 112, 224, 24, 64, 64) 94 | self.c4 = Inception(512, 128, 128, 256, 24, 64, 64) 95 | self.d4 = Inception(512, 112, 144, 288, 32, 64, 64) 96 | self.e4 = Inception(528, 256, 160, 320, 32, 128, 128) 97 | 98 | self.a5 = Inception(832, 256, 160, 320, 32, 128, 128) 99 | self.b5 = Inception(832, 384, 192, 384, 48, 128, 128) 100 | 101 | #input feature size: 8*8*1024 102 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 103 | self.dropout = nn.Dropout2d(p=0.4) 104 | self.linear = nn.Linear(1024, num_class) 105 | 106 | def forward(self, x): 107 | x = self.prelayer(x) 108 | x = self.maxpool(x) 109 | x = self.a3(x) 110 | x = self.b3(x) 111 | 112 | x = self.maxpool(x) 113 | 114 | x = self.a4(x) 115 | x = self.b4(x) 116 | x = self.c4(x) 117 | x = self.d4(x) 118 | x = self.e4(x) 119 | 120 | x = self.maxpool(x) 121 | 122 | x = self.a5(x) 123 | x = self.b5(x) 124 | 125 | #"""It was found that a move from fully connected layers to 126 | #average pooling improved the top-1 accuracy by about 0.6%, 127 | #however the use of dropout remained essential even after 128 | #removing the fully connected layers.""" 129 | x = self.avgpool(x) 130 | x = self.dropout(x) 131 | x = x.view(x.size()[0], -1) 132 | x = self.linear(x) 133 | 134 | return x 135 | 136 | def googlenet(): 137 | return GoogleNet() 138 | 139 | 140 | -------------------------------------------------------------------------------- /models/inceptionv3.py: -------------------------------------------------------------------------------- 1 | """ inceptionv3 in pytorch 2 | 3 | 4 | [1] Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew Wojna 5 | 6 | Rethinking the Inception Architecture for Computer Vision 7 | https://arxiv.org/abs/1512.00567v3 8 | """ 9 | 10 | import torch 11 | import torch.nn as nn 12 | 13 | 14 | class BasicConv2d(nn.Module): 15 | 16 | def __init__(self, input_channels, output_channels, **kwargs): 17 | super().__init__() 18 | self.conv = nn.Conv2d(input_channels, output_channels, bias=False, **kwargs) 19 | self.bn = nn.BatchNorm2d(output_channels) 20 | self.relu = nn.ReLU(inplace=True) 21 | 22 | def forward(self, x): 23 | x = self.conv(x) 24 | x = self.bn(x) 25 | x = self.relu(x) 26 | 27 | return x 28 | 29 | #same naive inception module 30 | class InceptionA(nn.Module): 31 | 32 | def __init__(self, input_channels, pool_features): 33 | super().__init__() 34 | self.branch1x1 = BasicConv2d(input_channels, 64, kernel_size=1) 35 | 36 | self.branch5x5 = nn.Sequential( 37 | BasicConv2d(input_channels, 48, kernel_size=1), 38 | BasicConv2d(48, 64, kernel_size=5, padding=2) 39 | ) 40 | 41 | self.branch3x3 = nn.Sequential( 42 | BasicConv2d(input_channels, 64, kernel_size=1), 43 | BasicConv2d(64, 96, kernel_size=3, padding=1), 44 | BasicConv2d(96, 96, kernel_size=3, padding=1) 45 | ) 46 | 47 | self.branchpool = nn.Sequential( 48 | nn.AvgPool2d(kernel_size=3, stride=1, padding=1), 49 | BasicConv2d(input_channels, pool_features, kernel_size=3, padding=1) 50 | ) 51 | 52 | def forward(self, x): 53 | 54 | #x -> 1x1(same) 55 | branch1x1 = self.branch1x1(x) 56 | 57 | #x -> 1x1 -> 5x5(same) 58 | branch5x5 = self.branch5x5(x) 59 | #branch5x5 = self.branch5x5_2(branch5x5) 60 | 61 | #x -> 1x1 -> 3x3 -> 3x3(same) 62 | branch3x3 = self.branch3x3(x) 63 | 64 | #x -> pool -> 1x1(same) 65 | branchpool = self.branchpool(x) 66 | 67 | outputs = [branch1x1, branch5x5, branch3x3, branchpool] 68 | 69 | return torch.cat(outputs, 1) 70 | 71 | #downsample 72 | #Factorization into smaller convolutions 73 | class InceptionB(nn.Module): 74 | 75 | def __init__(self, input_channels): 76 | super().__init__() 77 | 78 | self.branch3x3 = BasicConv2d(input_channels, 384, kernel_size=3, stride=2) 79 | 80 | self.branch3x3stack = nn.Sequential( 81 | BasicConv2d(input_channels, 64, kernel_size=1), 82 | BasicConv2d(64, 96, kernel_size=3, padding=1), 83 | BasicConv2d(96, 96, kernel_size=3, stride=2) 84 | ) 85 | 86 | self.branchpool = nn.MaxPool2d(kernel_size=3, stride=2) 87 | 88 | def forward(self, x): 89 | 90 | #x - > 3x3(downsample) 91 | branch3x3 = self.branch3x3(x) 92 | 93 | #x -> 3x3 -> 3x3(downsample) 94 | branch3x3stack = self.branch3x3stack(x) 95 | 96 | #x -> avgpool(downsample) 97 | branchpool = self.branchpool(x) 98 | 99 | #"""We can use two parallel stride 2 blocks: P and C. P is a pooling 100 | #layer (either average or maximum pooling) the activation, both of 101 | #them are stride 2 the filter banks of which are concatenated as in 102 | #figure 10.""" 103 | outputs = [branch3x3, branch3x3stack, branchpool] 104 | 105 | return torch.cat(outputs, 1) 106 | 107 | #Factorizing Convolutions with Large Filter Size 108 | class InceptionC(nn.Module): 109 | def __init__(self, input_channels, channels_7x7): 110 | super().__init__() 111 | self.branch1x1 = BasicConv2d(input_channels, 192, kernel_size=1) 112 | 113 | c7 = channels_7x7 114 | 115 | #In theory, we could go even further and argue that one can replace any n × n 116 | #convolution by a 1 × n convolution followed by a n × 1 convolution and the 117 | #computational cost saving increases dramatically as n grows (see figure 6). 118 | self.branch7x7 = nn.Sequential( 119 | BasicConv2d(input_channels, c7, kernel_size=1), 120 | BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0)), 121 | BasicConv2d(c7, 192, kernel_size=(1, 7), padding=(0, 3)) 122 | ) 123 | 124 | self.branch7x7stack = nn.Sequential( 125 | BasicConv2d(input_channels, c7, kernel_size=1), 126 | BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0)), 127 | BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3)), 128 | BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0)), 129 | BasicConv2d(c7, 192, kernel_size=(1, 7), padding=(0, 3)) 130 | ) 131 | 132 | self.branch_pool = nn.Sequential( 133 | nn.AvgPool2d(kernel_size=3, stride=1, padding=1), 134 | BasicConv2d(input_channels, 192, kernel_size=1), 135 | ) 136 | 137 | def forward(self, x): 138 | 139 | #x -> 1x1(same) 140 | branch1x1 = self.branch1x1(x) 141 | 142 | #x -> 1layer 1*7 and 7*1 (same) 143 | branch7x7 = self.branch7x7(x) 144 | 145 | #x-> 2layer 1*7 and 7*1(same) 146 | branch7x7stack = self.branch7x7stack(x) 147 | 148 | #x-> avgpool (same) 149 | branchpool = self.branch_pool(x) 150 | 151 | outputs = [branch1x1, branch7x7, branch7x7stack, branchpool] 152 | 153 | return torch.cat(outputs, 1) 154 | 155 | class InceptionD(nn.Module): 156 | 157 | def __init__(self, input_channels): 158 | super().__init__() 159 | 160 | self.branch3x3 = nn.Sequential( 161 | BasicConv2d(input_channels, 192, kernel_size=1), 162 | BasicConv2d(192, 320, kernel_size=3, stride=2) 163 | ) 164 | 165 | self.branch7x7 = nn.Sequential( 166 | BasicConv2d(input_channels, 192, kernel_size=1), 167 | BasicConv2d(192, 192, kernel_size=(1, 7), padding=(0, 3)), 168 | BasicConv2d(192, 192, kernel_size=(7, 1), padding=(3, 0)), 169 | BasicConv2d(192, 192, kernel_size=3, stride=2) 170 | ) 171 | 172 | self.branchpool = nn.AvgPool2d(kernel_size=3, stride=2) 173 | 174 | def forward(self, x): 175 | 176 | #x -> 1x1 -> 3x3(downsample) 177 | branch3x3 = self.branch3x3(x) 178 | 179 | #x -> 1x1 -> 1x7 -> 7x1 -> 3x3 (downsample) 180 | branch7x7 = self.branch7x7(x) 181 | 182 | #x -> avgpool (downsample) 183 | branchpool = self.branchpool(x) 184 | 185 | outputs = [branch3x3, branch7x7, branchpool] 186 | 187 | return torch.cat(outputs, 1) 188 | 189 | 190 | #same 191 | class InceptionE(nn.Module): 192 | def __init__(self, input_channels): 193 | super().__init__() 194 | self.branch1x1 = BasicConv2d(input_channels, 320, kernel_size=1) 195 | 196 | self.branch3x3_1 = BasicConv2d(input_channels, 384, kernel_size=1) 197 | self.branch3x3_2a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1)) 198 | self.branch3x3_2b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0)) 199 | 200 | self.branch3x3stack_1 = BasicConv2d(input_channels, 448, kernel_size=1) 201 | self.branch3x3stack_2 = BasicConv2d(448, 384, kernel_size=3, padding=1) 202 | self.branch3x3stack_3a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1)) 203 | self.branch3x3stack_3b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0)) 204 | 205 | self.branch_pool = nn.Sequential( 206 | nn.AvgPool2d(kernel_size=3, stride=1, padding=1), 207 | BasicConv2d(input_channels, 192, kernel_size=1) 208 | ) 209 | 210 | def forward(self, x): 211 | 212 | #x -> 1x1 (same) 213 | branch1x1 = self.branch1x1(x) 214 | 215 | # x -> 1x1 -> 3x1 216 | # x -> 1x1 -> 1x3 217 | # concatenate(3x1, 1x3) 218 | #"""7. Inception modules with expanded the filter bank outputs. 219 | #This architecture is used on the coarsest (8 × 8) grids to promote 220 | #high dimensional representations, as suggested by principle 221 | #2 of Section 2.""" 222 | branch3x3 = self.branch3x3_1(x) 223 | branch3x3 = [ 224 | self.branch3x3_2a(branch3x3), 225 | self.branch3x3_2b(branch3x3) 226 | ] 227 | branch3x3 = torch.cat(branch3x3, 1) 228 | 229 | # x -> 1x1 -> 3x3 -> 1x3 230 | # x -> 1x1 -> 3x3 -> 3x1 231 | #concatenate(1x3, 3x1) 232 | branch3x3stack = self.branch3x3stack_1(x) 233 | branch3x3stack = self.branch3x3stack_2(branch3x3stack) 234 | branch3x3stack = [ 235 | self.branch3x3stack_3a(branch3x3stack), 236 | self.branch3x3stack_3b(branch3x3stack) 237 | ] 238 | branch3x3stack = torch.cat(branch3x3stack, 1) 239 | 240 | branchpool = self.branch_pool(x) 241 | 242 | outputs = [branch1x1, branch3x3, branch3x3stack, branchpool] 243 | 244 | return torch.cat(outputs, 1) 245 | 246 | class InceptionV3(nn.Module): 247 | 248 | def __init__(self, num_classes=100): 249 | super().__init__() 250 | self.Conv2d_1a_3x3 = BasicConv2d(3, 32, kernel_size=3, padding=1) 251 | self.Conv2d_2a_3x3 = BasicConv2d(32, 32, kernel_size=3, padding=1) 252 | self.Conv2d_2b_3x3 = BasicConv2d(32, 64, kernel_size=3, padding=1) 253 | self.Conv2d_3b_1x1 = BasicConv2d(64, 80, kernel_size=1) 254 | self.Conv2d_4a_3x3 = BasicConv2d(80, 192, kernel_size=3) 255 | 256 | #naive inception module 257 | self.Mixed_5b = InceptionA(192, pool_features=32) 258 | self.Mixed_5c = InceptionA(256, pool_features=64) 259 | self.Mixed_5d = InceptionA(288, pool_features=64) 260 | 261 | #downsample 262 | self.Mixed_6a = InceptionB(288) 263 | 264 | self.Mixed_6b = InceptionC(768, channels_7x7=128) 265 | self.Mixed_6c = InceptionC(768, channels_7x7=160) 266 | self.Mixed_6d = InceptionC(768, channels_7x7=160) 267 | self.Mixed_6e = InceptionC(768, channels_7x7=192) 268 | 269 | #downsample 270 | self.Mixed_7a = InceptionD(768) 271 | 272 | self.Mixed_7b = InceptionE(1280) 273 | self.Mixed_7c = InceptionE(2048) 274 | 275 | #6*6 feature size 276 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 277 | self.dropout = nn.Dropout2d() 278 | self.linear = nn.Linear(2048, num_classes) 279 | 280 | def forward(self, x): 281 | 282 | #32 -> 30 283 | x = self.Conv2d_1a_3x3(x) 284 | x = self.Conv2d_2a_3x3(x) 285 | x = self.Conv2d_2b_3x3(x) 286 | x = self.Conv2d_3b_1x1(x) 287 | x = self.Conv2d_4a_3x3(x) 288 | 289 | #30 -> 30 290 | x = self.Mixed_5b(x) 291 | x = self.Mixed_5c(x) 292 | x = self.Mixed_5d(x) 293 | 294 | #30 -> 14 295 | #Efficient Grid Size Reduction to avoid representation 296 | #bottleneck 297 | x = self.Mixed_6a(x) 298 | 299 | #14 -> 14 300 | #"""In practice, we have found that employing this factorization does not 301 | #work well on early layers, but it gives very good results on medium 302 | #grid-sizes (On m × m feature maps, where m ranges between 12 and 20). 303 | #On that level, very good results can be achieved by using 1 × 7 convolutions 304 | #followed by 7 × 1 convolutions.""" 305 | x = self.Mixed_6b(x) 306 | x = self.Mixed_6c(x) 307 | x = self.Mixed_6d(x) 308 | x = self.Mixed_6e(x) 309 | 310 | #14 -> 6 311 | #Efficient Grid Size Reduction 312 | x = self.Mixed_7a(x) 313 | 314 | #6 -> 6 315 | #We are using this solution only on the coarsest grid, 316 | #since that is the place where producing high dimensional 317 | #sparse representation is the most critical as the ratio of 318 | #local processing (by 1 × 1 convolutions) is increased compared 319 | #to the spatial aggregation.""" 320 | x = self.Mixed_7b(x) 321 | x = self.Mixed_7c(x) 322 | 323 | #6 -> 1 324 | x = self.avgpool(x) 325 | x = self.dropout(x) 326 | x = x.view(x.size(0), -1) 327 | x = self.linear(x) 328 | return x 329 | 330 | 331 | def inceptionv3(): 332 | return InceptionV3() 333 | 334 | 335 | 336 | -------------------------------------------------------------------------------- /models/mobilenet.py: -------------------------------------------------------------------------------- 1 | """mobilenet in pytorch 2 | 3 | 4 | 5 | [1] Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 6 | 7 | MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications 8 | https://arxiv.org/abs/1704.04861 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | 14 | 15 | class DepthSeperabelConv2d(nn.Module): 16 | 17 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 18 | super().__init__() 19 | self.depthwise = nn.Sequential( 20 | nn.Conv2d( 21 | input_channels, 22 | input_channels, 23 | kernel_size, 24 | groups=input_channels, 25 | **kwargs), 26 | nn.BatchNorm2d(input_channels), 27 | nn.ReLU(inplace=True) 28 | ) 29 | 30 | self.pointwise = nn.Sequential( 31 | nn.Conv2d(input_channels, output_channels, 1), 32 | nn.BatchNorm2d(output_channels), 33 | nn.ReLU(inplace=True) 34 | ) 35 | 36 | def forward(self, x): 37 | x = self.depthwise(x) 38 | x = self.pointwise(x) 39 | 40 | return x 41 | 42 | 43 | class BasicConv2d(nn.Module): 44 | 45 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 46 | 47 | super().__init__() 48 | self.conv = nn.Conv2d( 49 | input_channels, output_channels, kernel_size, **kwargs) 50 | self.bn = nn.BatchNorm2d(output_channels) 51 | self.relu = nn.ReLU(inplace=True) 52 | 53 | def forward(self, x): 54 | x = self.conv(x) 55 | x = self.bn(x) 56 | x = self.relu(x) 57 | 58 | return x 59 | 60 | 61 | class MobileNet(nn.Module): 62 | 63 | """ 64 | Args: 65 | width multipler: The role of the width multiplier α is to thin 66 | a network uniformly at each layer. For a given 67 | layer and width multiplier α, the number of 68 | input channels M becomes αM and the number of 69 | output channels N becomes αN. 70 | """ 71 | 72 | def __init__(self, width_multiplier=1, class_num=100): 73 | super().__init__() 74 | 75 | alpha = width_multiplier 76 | self.stem = nn.Sequential( 77 | BasicConv2d(3, int(32 * alpha), 3, padding=1, bias=False), 78 | DepthSeperabelConv2d( 79 | int(32 * alpha), 80 | int(64 * alpha), 81 | 3, 82 | padding=1, 83 | bias=False 84 | ) 85 | ) 86 | 87 | #downsample 88 | self.conv1 = nn.Sequential( 89 | DepthSeperabelConv2d( 90 | int(64 * alpha), 91 | int(128 * alpha), 92 | 3, 93 | stride=2, 94 | padding=1, 95 | bias=False 96 | ), 97 | DepthSeperabelConv2d( 98 | int(128 * alpha), 99 | int(128 * alpha), 100 | 3, 101 | padding=1, 102 | bias=False 103 | ) 104 | ) 105 | 106 | #downsample 107 | self.conv2 = nn.Sequential( 108 | DepthSeperabelConv2d( 109 | int(128 * alpha), 110 | int(256 * alpha), 111 | 3, 112 | stride=2, 113 | padding=1, 114 | bias=False 115 | ), 116 | DepthSeperabelConv2d( 117 | int(256 * alpha), 118 | int(256 * alpha), 119 | 3, 120 | padding=1, 121 | bias=False 122 | ) 123 | ) 124 | 125 | #downsample 126 | self.conv3 = nn.Sequential( 127 | DepthSeperabelConv2d( 128 | int(256 * alpha), 129 | int(512 * alpha), 130 | 3, 131 | stride=2, 132 | padding=1, 133 | bias=False 134 | ), 135 | 136 | DepthSeperabelConv2d( 137 | int(512 * alpha), 138 | int(512 * alpha), 139 | 3, 140 | padding=1, 141 | bias=False 142 | ), 143 | DepthSeperabelConv2d( 144 | int(512 * alpha), 145 | int(512 * alpha), 146 | 3, 147 | padding=1, 148 | bias=False 149 | ), 150 | DepthSeperabelConv2d( 151 | int(512 * alpha), 152 | int(512 * alpha), 153 | 3, 154 | padding=1, 155 | bias=False 156 | ), 157 | DepthSeperabelConv2d( 158 | int(512 * alpha), 159 | int(512 * alpha), 160 | 3, 161 | padding=1, 162 | bias=False 163 | ), 164 | DepthSeperabelConv2d( 165 | int(512 * alpha), 166 | int(512 * alpha), 167 | 3, 168 | padding=1, 169 | bias=False 170 | ) 171 | ) 172 | 173 | #downsample 174 | self.conv4 = nn.Sequential( 175 | DepthSeperabelConv2d( 176 | int(512 * alpha), 177 | int(1024 * alpha), 178 | 3, 179 | stride=2, 180 | padding=1, 181 | bias=False 182 | ), 183 | DepthSeperabelConv2d( 184 | int(1024 * alpha), 185 | int(1024 * alpha), 186 | 3, 187 | padding=1, 188 | bias=False 189 | ) 190 | ) 191 | 192 | self.fc = nn.Linear(int(1024 * alpha), class_num) 193 | self.avg = nn.AdaptiveAvgPool2d(1) 194 | 195 | def forward(self, x): 196 | x = self.stem(x) 197 | 198 | x = self.conv1(x) 199 | x = self.conv2(x) 200 | x = self.conv3(x) 201 | x = self.conv4(x) 202 | 203 | x = self.avg(x) 204 | x = x.view(x.size(0), -1) 205 | x = self.fc(x) 206 | return x 207 | 208 | 209 | def mobilenet(alpha=1, class_num=100): 210 | return MobileNet(alpha, class_num) 211 | 212 | -------------------------------------------------------------------------------- /models/mobilenetv2.py: -------------------------------------------------------------------------------- 1 | """mobilenetv2 in pytorch 2 | 3 | 4 | 5 | [1] Mark Sandler, Andrew Howard, Menglong Zhu, Andrey Zhmoginov, Liang-Chieh Chen 6 | 7 | MobileNetV2: Inverted Residuals and Linear Bottlenecks 8 | https://arxiv.org/abs/1801.04381 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.functional as F 14 | 15 | 16 | class LinearBottleNeck(nn.Module): 17 | 18 | def __init__(self, in_channels, out_channels, stride, t=6, class_num=100): 19 | super().__init__() 20 | 21 | self.residual = nn.Sequential( 22 | nn.Conv2d(in_channels, in_channels * t, 1), 23 | nn.BatchNorm2d(in_channels * t), 24 | nn.ReLU6(inplace=True), 25 | 26 | nn.Conv2d(in_channels * t, in_channels * t, 3, stride=stride, padding=1, groups=in_channels * t), 27 | nn.BatchNorm2d(in_channels * t), 28 | nn.ReLU6(inplace=True), 29 | 30 | nn.Conv2d(in_channels * t, out_channels, 1), 31 | nn.BatchNorm2d(out_channels) 32 | ) 33 | 34 | self.stride = stride 35 | self.in_channels = in_channels 36 | self.out_channels = out_channels 37 | 38 | def forward(self, x): 39 | 40 | residual = self.residual(x) 41 | 42 | if self.stride == 1 and self.in_channels == self.out_channels: 43 | residual += x 44 | 45 | return residual 46 | 47 | class MobileNetV2(nn.Module): 48 | 49 | def __init__(self, class_num=100): 50 | super().__init__() 51 | 52 | self.pre = nn.Sequential( 53 | nn.Conv2d(3, 32, 1, padding=1), 54 | nn.BatchNorm2d(32), 55 | nn.ReLU6(inplace=True) 56 | ) 57 | 58 | self.stage1 = LinearBottleNeck(32, 16, 1, 1) 59 | self.stage2 = self._make_stage(2, 16, 24, 2, 6) 60 | self.stage3 = self._make_stage(3, 24, 32, 2, 6) 61 | self.stage4 = self._make_stage(4, 32, 64, 2, 6) 62 | self.stage5 = self._make_stage(3, 64, 96, 1, 6) 63 | self.stage6 = self._make_stage(3, 96, 160, 1, 6) 64 | self.stage7 = LinearBottleNeck(160, 320, 1, 6) 65 | 66 | self.conv1 = nn.Sequential( 67 | nn.Conv2d(320, 1280, 1), 68 | nn.BatchNorm2d(1280), 69 | nn.ReLU6(inplace=True) 70 | ) 71 | 72 | self.conv2 = nn.Conv2d(1280, class_num, 1) 73 | 74 | def forward(self, x): 75 | x = self.pre(x) 76 | x = self.stage1(x) 77 | x = self.stage2(x) 78 | x = self.stage3(x) 79 | x = self.stage4(x) 80 | x = self.stage5(x) 81 | x = self.stage6(x) 82 | x = self.stage7(x) 83 | x = self.conv1(x) 84 | x = F.adaptive_avg_pool2d(x, 1) 85 | x = self.conv2(x) 86 | x = x.view(x.size(0), -1) 87 | 88 | return x 89 | 90 | def _make_stage(self, repeat, in_channels, out_channels, stride, t): 91 | 92 | layers = [] 93 | layers.append(LinearBottleNeck(in_channels, out_channels, stride, t)) 94 | 95 | while repeat - 1: 96 | layers.append(LinearBottleNeck(out_channels, out_channels, 1, t)) 97 | repeat -= 1 98 | 99 | return nn.Sequential(*layers) 100 | 101 | def mobilenetv2(): 102 | return MobileNetV2() -------------------------------------------------------------------------------- /models/nasnet.py: -------------------------------------------------------------------------------- 1 | """nasnet in pytorch 2 | 3 | 4 | 5 | [1] Barret Zoph, Vijay Vasudevan, Jonathon Shlens, Quoc V. Le 6 | 7 | Learning Transferable Architectures for Scalable Image Recognition 8 | https://arxiv.org/abs/1707.07012 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | 14 | class SeperableConv2d(nn.Module): 15 | 16 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 17 | 18 | super().__init__() 19 | self.depthwise = nn.Conv2d( 20 | input_channels, 21 | input_channels, 22 | kernel_size, 23 | groups=input_channels, 24 | **kwargs 25 | ) 26 | 27 | self.pointwise = nn.Conv2d( 28 | input_channels, 29 | output_channels, 30 | 1 31 | ) 32 | def forward(self, x): 33 | x = self.depthwise(x) 34 | x = self.pointwise(x) 35 | 36 | return x 37 | 38 | class SeperableBranch(nn.Module): 39 | 40 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 41 | """Adds 2 blocks of [relu-separable conv-batchnorm].""" 42 | super().__init__() 43 | self.block1 = nn.Sequential( 44 | nn.ReLU(), 45 | SeperableConv2d(input_channels, output_channels, kernel_size, **kwargs), 46 | nn.BatchNorm2d(output_channels) 47 | ) 48 | 49 | self.block2 = nn.Sequential( 50 | nn.ReLU(), 51 | SeperableConv2d(output_channels, output_channels, kernel_size, stride=1, padding=int(kernel_size / 2)), 52 | nn.BatchNorm2d(output_channels) 53 | ) 54 | 55 | def forward(self, x): 56 | x = self.block1(x) 57 | x = self.block2(x) 58 | 59 | return x 60 | 61 | class Fit(nn.Module): 62 | """Make the cell outputs compatible 63 | 64 | Args: 65 | prev_filters: filter number of tensor prev, needs to be modified 66 | filters: filter number of normal cell branch output filters 67 | """ 68 | 69 | def __init__(self, prev_filters, filters): 70 | super().__init__() 71 | self.relu = nn.ReLU() 72 | 73 | self.p1 = nn.Sequential( 74 | nn.AvgPool2d(1, stride=2), 75 | nn.Conv2d(prev_filters, int(filters / 2), 1) 76 | ) 77 | 78 | #make sure there is no information loss 79 | self.p2 = nn.Sequential( 80 | nn.ConstantPad2d((0, 1, 0, 1), 0), 81 | nn.ConstantPad2d((-1, 0, -1, 0), 0), #cropping 82 | nn.AvgPool2d(1, stride=2), 83 | nn.Conv2d(prev_filters, int(filters / 2), 1) 84 | ) 85 | 86 | self.bn = nn.BatchNorm2d(filters) 87 | 88 | self.dim_reduce = nn.Sequential( 89 | nn.ReLU(), 90 | nn.Conv2d(prev_filters, filters, 1), 91 | nn.BatchNorm2d(filters) 92 | ) 93 | 94 | self.filters = filters 95 | 96 | def forward(self, inputs): 97 | x, prev = inputs 98 | if prev is None: 99 | return x 100 | 101 | #image size does not match 102 | elif x.size(2) != prev.size(2): 103 | prev = self.relu(prev) 104 | p1 = self.p1(prev) 105 | p2 = self.p2(prev) 106 | prev = torch.cat([p1, p2], 1) 107 | prev = self.bn(prev) 108 | 109 | elif prev.size(1) != self.filters: 110 | prev = self.dim_reduce(prev) 111 | 112 | return prev 113 | 114 | 115 | class NormalCell(nn.Module): 116 | 117 | def __init__(self, x_in, prev_in, output_channels): 118 | super().__init__() 119 | 120 | self.dem_reduce = nn.Sequential( 121 | nn.ReLU(), 122 | nn.Conv2d(x_in, output_channels, 1, bias=False), 123 | nn.BatchNorm2d(output_channels) 124 | ) 125 | 126 | self.block1_left = SeperableBranch( 127 | output_channels, 128 | output_channels, 129 | kernel_size=3, 130 | padding=1, 131 | bias=False 132 | ) 133 | self.block1_right = nn.Sequential() 134 | 135 | self.block2_left = SeperableBranch( 136 | output_channels, 137 | output_channels, 138 | kernel_size=3, 139 | padding=1, 140 | bias=False 141 | ) 142 | self.block2_right = SeperableBranch( 143 | output_channels, 144 | output_channels, 145 | kernel_size=5, 146 | padding=2, 147 | bias=False 148 | ) 149 | 150 | self.block3_left = nn.AvgPool2d(3, stride=1, padding=1) 151 | self.block3_right = nn.Sequential() 152 | 153 | self.block4_left = nn.AvgPool2d(3, stride=1, padding=1) 154 | self.block4_right = nn.AvgPool2d(3, stride=1, padding=1) 155 | 156 | self.block5_left = SeperableBranch( 157 | output_channels, 158 | output_channels, 159 | kernel_size=5, 160 | padding=2, 161 | bias=False 162 | ) 163 | self.block5_right = SeperableBranch( 164 | output_channels, 165 | output_channels, 166 | kernel_size=3, 167 | padding=1, 168 | bias=False 169 | ) 170 | 171 | self.fit = Fit(prev_in, output_channels) 172 | 173 | def forward(self, x): 174 | x, prev = x 175 | 176 | #return transformed x as new x, and original x as prev 177 | #only prev tensor needs to be modified 178 | prev = self.fit((x, prev)) 179 | 180 | h = self.dem_reduce(x) 181 | 182 | x1 = self.block1_left(h) + self.block1_right(h) 183 | x2 = self.block2_left(prev) + self.block2_right(h) 184 | x3 = self.block3_left(h) + self.block3_right(h) 185 | x4 = self.block4_left(prev) + self.block4_right(prev) 186 | x5 = self.block5_left(prev) + self.block5_right(prev) 187 | 188 | return torch.cat([prev, x1, x2, x3, x4, x5], 1), x 189 | 190 | class ReductionCell(nn.Module): 191 | 192 | def __init__(self, x_in, prev_in, output_channels): 193 | super().__init__() 194 | 195 | self.dim_reduce = nn.Sequential( 196 | nn.ReLU(), 197 | nn.Conv2d(x_in, output_channels, 1), 198 | nn.BatchNorm2d(output_channels) 199 | ) 200 | 201 | #block1 202 | self.layer1block1_left = SeperableBranch(output_channels, output_channels, 7, stride=2, padding=3) 203 | self.layer1block1_right = SeperableBranch(output_channels, output_channels, 5, stride=2, padding=2) 204 | 205 | #block2 206 | self.layer1block2_left = nn.MaxPool2d(3, stride=2, padding=1) 207 | self.layer1block2_right = SeperableBranch(output_channels, output_channels, 7, stride=2, padding=3) 208 | 209 | #block3 210 | self.layer1block3_left = nn.AvgPool2d(3, 2, 1) 211 | self.layer1block3_right = SeperableBranch(output_channels, output_channels, 5, stride=2, padding=2) 212 | 213 | #block5 214 | self.layer2block1_left = nn.MaxPool2d(3, 2, 1) 215 | self.layer2block1_right = SeperableBranch(output_channels, output_channels, 3, stride=1, padding=1) 216 | 217 | #block4 218 | self.layer2block2_left = nn.AvgPool2d(3, 1, 1) 219 | self.layer2block2_right = nn.Sequential() 220 | 221 | self.fit = Fit(prev_in, output_channels) 222 | 223 | def forward(self, x): 224 | x, prev = x 225 | prev = self.fit((x, prev)) 226 | 227 | h = self.dim_reduce(x) 228 | 229 | layer1block1 = self.layer1block1_left(prev) + self.layer1block1_right(h) 230 | layer1block2 = self.layer1block2_left(h) + self.layer1block2_right(prev) 231 | layer1block3 = self.layer1block3_left(h) + self.layer1block3_right(prev) 232 | layer2block1 = self.layer2block1_left(h) + self.layer2block1_right(layer1block1) 233 | layer2block2 = self.layer2block2_left(layer1block1) + self.layer2block2_right(layer1block2) 234 | 235 | return torch.cat([ 236 | layer1block2, #https://github.com/keras-team/keras-applications/blob/master/keras_applications/nasnet.py line 739 237 | layer1block3, 238 | layer2block1, 239 | layer2block2 240 | ], 1), x 241 | 242 | 243 | class NasNetA(nn.Module): 244 | 245 | def __init__(self, repeat_cell_num, reduction_num, filters, stemfilter, class_num=100): 246 | super().__init__() 247 | 248 | self.stem = nn.Sequential( 249 | nn.Conv2d(3, stemfilter, 3, padding=1, bias=False), 250 | nn.BatchNorm2d(stemfilter) 251 | ) 252 | 253 | self.prev_filters = stemfilter 254 | self.x_filters = stemfilter 255 | self.filters = filters 256 | 257 | self.cell_layers = self._make_layers(repeat_cell_num, reduction_num) 258 | 259 | self.relu = nn.ReLU() 260 | self.avg = nn.AdaptiveAvgPool2d(1) 261 | self.fc = nn.Linear(self.filters * 6, class_num) 262 | 263 | 264 | def _make_normal(self, block, repeat, output): 265 | """make normal cell 266 | Args: 267 | block: cell type 268 | repeat: number of repeated normal cell 269 | output: output filters for each branch in normal cell 270 | Returns: 271 | stacked normal cells 272 | """ 273 | 274 | layers = [] 275 | for r in range(repeat): 276 | layers.append(block(self.x_filters, self.prev_filters, output)) 277 | self.prev_filters = self.x_filters 278 | self.x_filters = output * 6 #concatenate 6 branches 279 | 280 | return layers 281 | 282 | def _make_reduction(self, block, output): 283 | """make normal cell 284 | Args: 285 | block: cell type 286 | output: output filters for each branch in reduction cell 287 | Returns: 288 | reduction cell 289 | """ 290 | 291 | reduction = block(self.x_filters, self.prev_filters, output) 292 | self.prev_filters = self.x_filters 293 | self.x_filters = output * 4 #stack for 4 branches 294 | 295 | return reduction 296 | 297 | def _make_layers(self, repeat_cell_num, reduction_num): 298 | 299 | layers = [] 300 | for i in range(reduction_num): 301 | 302 | layers.extend(self._make_normal(NormalCell, repeat_cell_num, self.filters)) 303 | self.filters *= 2 304 | layers.append(self._make_reduction(ReductionCell, self.filters)) 305 | 306 | layers.extend(self._make_normal(NormalCell, repeat_cell_num, self.filters)) 307 | 308 | return nn.Sequential(*layers) 309 | 310 | 311 | def forward(self, x): 312 | 313 | x = self.stem(x) 314 | prev = None 315 | x, prev = self.cell_layers((x, prev)) 316 | x = self.relu(x) 317 | x = self.avg(x) 318 | x = x.view(x.size(0), -1) 319 | x = self.fc(x) 320 | 321 | return x 322 | 323 | 324 | def nasnet(): 325 | 326 | #stem filters must be 44, it's a pytorch workaround, cant change to other number 327 | return NasNetA(4, 2, 44, 44) 328 | 329 | -------------------------------------------------------------------------------- /models/preactresnet.py: -------------------------------------------------------------------------------- 1 | """preactresnet in pytorch 2 | 3 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 4 | 5 | Identity Mappings in Deep Residual Networks 6 | https://arxiv.org/abs/1603.05027 7 | """ 8 | 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | 13 | class PreActBasic(nn.Module): 14 | 15 | expansion = 1 16 | def __init__(self, in_channels, out_channels, stride): 17 | super().__init__() 18 | self.residual = nn.Sequential( 19 | nn.BatchNorm2d(in_channels), 20 | nn.ReLU(inplace=True), 21 | nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1), 22 | nn.BatchNorm2d(out_channels), 23 | nn.ReLU(inplace=True), 24 | nn.Conv2d(out_channels, out_channels * PreActBasic.expansion, kernel_size=3, padding=1) 25 | ) 26 | 27 | self.shortcut = nn.Sequential() 28 | if stride != 1 or in_channels != out_channels * PreActBasic.expansion: 29 | self.shortcut = nn.Conv2d(in_channels, out_channels * PreActBasic.expansion, 1, stride=stride) 30 | 31 | def forward(self, x): 32 | 33 | res = self.residual(x) 34 | shortcut = self.shortcut(x) 35 | 36 | return res + shortcut 37 | 38 | 39 | class PreActBottleNeck(nn.Module): 40 | 41 | expansion = 4 42 | def __init__(self, in_channels, out_channels, stride): 43 | super().__init__() 44 | 45 | self.residual = nn.Sequential( 46 | nn.BatchNorm2d(in_channels), 47 | nn.ReLU(inplace=True), 48 | nn.Conv2d(in_channels, out_channels, 1, stride=stride), 49 | 50 | nn.BatchNorm2d(out_channels), 51 | nn.ReLU(inplace=True), 52 | nn.Conv2d(out_channels, out_channels, 3, padding=1), 53 | 54 | nn.BatchNorm2d(out_channels), 55 | nn.ReLU(inplace=True), 56 | nn.Conv2d(out_channels, out_channels * PreActBottleNeck.expansion, 1) 57 | ) 58 | 59 | self.shortcut = nn.Sequential() 60 | 61 | if stride != 1 or in_channels != out_channels * PreActBottleNeck.expansion: 62 | self.shortcut = nn.Conv2d(in_channels, out_channels * PreActBottleNeck.expansion, 1, stride=stride) 63 | 64 | def forward(self, x): 65 | 66 | res = self.residual(x) 67 | shortcut = self.shortcut(x) 68 | 69 | return res + shortcut 70 | 71 | class PreActResNet(nn.Module): 72 | 73 | def __init__(self, block, num_block, class_num=100): 74 | super().__init__() 75 | self.input_channels = 64 76 | 77 | self.pre = nn.Sequential( 78 | nn.Conv2d(3, 64, 3, padding=1), 79 | nn.BatchNorm2d(64), 80 | nn.ReLU(inplace=True) 81 | ) 82 | 83 | self.stage1 = self._make_layers(block, num_block[0], 64, 1) 84 | self.stage2 = self._make_layers(block, num_block[1], 128, 2) 85 | self.stage3 = self._make_layers(block, num_block[2], 256, 2) 86 | self.stage4 = self._make_layers(block, num_block[3], 512, 2) 87 | 88 | self.linear = nn.Linear(self.input_channels, class_num) 89 | 90 | def _make_layers(self, block, block_num, out_channels, stride): 91 | layers = [] 92 | 93 | layers.append(block(self.input_channels, out_channels, stride)) 94 | self.input_channels = out_channels * block.expansion 95 | 96 | while block_num - 1: 97 | layers.append(block(self.input_channels, out_channels, 1)) 98 | self.input_channels = out_channels * block.expansion 99 | block_num -= 1 100 | 101 | return nn.Sequential(*layers) 102 | 103 | def forward(self, x): 104 | x = self.pre(x) 105 | 106 | x = self.stage1(x) 107 | x = self.stage2(x) 108 | x = self.stage3(x) 109 | x = self.stage4(x) 110 | 111 | x = F.adaptive_avg_pool2d(x, 1) 112 | x = x.view(x.size(0), -1) 113 | x = self.linear(x) 114 | 115 | return x 116 | 117 | def preactresnet18(): 118 | return PreActResNet(PreActBasic, [2, 2, 2, 2]) 119 | 120 | def preactresnet34(): 121 | return PreActResNet(PreActBasic, [3, 4, 6, 3]) 122 | 123 | def preactresnet50(): 124 | return PreActResNet(PreActBottleNeck, [3, 4, 6, 3]) 125 | 126 | def preactresnet101(): 127 | return PreActResNet(PreActBottleNeck, [3, 4, 23, 3]) 128 | 129 | def preactresnet152(): 130 | return PreActResNet(PreActBottleNeck, [3, 8, 36, 3]) 131 | 132 | -------------------------------------------------------------------------------- /models/resnet.py: -------------------------------------------------------------------------------- 1 | """resnet in pytorch 2 | 3 | 4 | 5 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 6 | 7 | Deep Residual Learning for Image Recognition 8 | https://arxiv.org/abs/1512.03385v1 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | from conf import settings 14 | 15 | class SelectiveSequential(nn.Module): 16 | def __init__(self, to_select, modules_dict): 17 | super(SelectiveSequential, self).__init__() 18 | for key, module in modules_dict.items(): 19 | self.add_module(key, module) 20 | self._to_select = to_select 21 | 22 | def forward(self, x): 23 | list = [] 24 | for name, module in self._modules.iteritems(): 25 | x = module(x) 26 | if name in self._to_select: 27 | list.append(x) 28 | return list 29 | 30 | class BasicBlock(nn.Module): 31 | """Basic Block for resnet 18 and resnet 34 32 | 33 | """ 34 | 35 | #BasicBlock and BottleNeck block 36 | #have different output size 37 | #we use class attribute expansion 38 | #to distinct 39 | expansion = 1 40 | 41 | def __init__(self, in_channels, out_channels, stride=1): 42 | super().__init__() 43 | 44 | # self.features 45 | 46 | #residual function 47 | self.residual_function = nn.Sequential( 48 | nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False), 49 | nn.BatchNorm2d(out_channels), 50 | nn.ReLU(inplace=True), 51 | nn.Conv2d(out_channels, out_channels * BasicBlock.expansion, kernel_size=3, padding=1, bias=False), 52 | nn.BatchNorm2d(out_channels * BasicBlock.expansion) 53 | ) 54 | 55 | #shortcut 56 | self.shortcut = nn.Sequential() 57 | 58 | #the shortcut output dimension is not the same with residual function 59 | #use 1*1 convolution to match the dimension 60 | if stride != 1 or in_channels != BasicBlock.expansion * out_channels: 61 | self.shortcut = nn.Sequential( 62 | nn.Conv2d(in_channels, out_channels * BasicBlock.expansion, kernel_size=1, stride=stride, bias=False), 63 | nn.BatchNorm2d(out_channels * BasicBlock.expansion) 64 | ) 65 | 66 | def forward(self, x): 67 | return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) 68 | 69 | class BottleNeck(nn.Module): 70 | """Residual block for resnet over 50 layers 71 | 72 | """ 73 | expansion = 4 74 | def __init__(self, in_channels, out_channels, stride=1): 75 | super().__init__() 76 | self.residual_function = nn.Sequential( 77 | nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False), 78 | nn.BatchNorm2d(out_channels), 79 | nn.ReLU(inplace=True), 80 | nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False), 81 | nn.BatchNorm2d(out_channels), 82 | nn.ReLU(inplace=True), 83 | nn.Conv2d(out_channels, out_channels * BottleNeck.expansion, kernel_size=1, bias=False), 84 | nn.BatchNorm2d(out_channels * BottleNeck.expansion), 85 | ) 86 | 87 | self.shortcut = nn.Sequential() 88 | 89 | if stride != 1 or in_channels != out_channels * BottleNeck.expansion: 90 | self.shortcut = nn.Sequential( 91 | nn.Conv2d(in_channels, out_channels * BottleNeck.expansion, stride=stride, kernel_size=1, bias=False), 92 | nn.BatchNorm2d(out_channels * BottleNeck.expansion) 93 | ) 94 | 95 | def forward(self, x): 96 | return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) 97 | 98 | class ResNet(nn.Module): 99 | 100 | # def __init__(self, features, block, num_block, num_classes=100): 101 | def __init__(self, block, num_block, num_classes=100): 102 | 103 | super().__init__() 104 | 105 | # self.features = features 106 | self.in_channels = 64 107 | 108 | # self.features = SelectiveSequential( 109 | # ['conv1', 'conv3'], 110 | # {'conv1': nn.Conv2d(1, 1, 3), 111 | # 'conv2': nn.Conv2d(1, 1, 3), 112 | # 'conv3': nn.Conv2d(1, 1, 3)} 113 | # ) 114 | 115 | self.conv1 = nn.Sequential( 116 | nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False), 117 | nn.BatchNorm2d(64), 118 | nn.ReLU(inplace=True)) 119 | 120 | #we use a different inputsize than the original paper 121 | #so conv2_x's stride is 1 122 | self.conv2_x = self._make_layer(block, 64, num_block[0], 1) 123 | self.conv3_x = self._make_layer(block, 128, num_block[1], 2) 124 | self.conv4_x = self._make_layer(block, 256, num_block[2], 2) 125 | self.conv5_x = self._make_layer(block, 512, num_block[3], 2) 126 | self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) 127 | self.fc = nn.Linear(512 * block.expansion, num_classes) 128 | 129 | 130 | def _make_layer(self, block, out_channels, num_blocks, stride): 131 | """make resnet layers(by layer i didnt mean this 'layer' was the 132 | same as a neuron netowork layer, ex. conv layer), one layer may 133 | contain more than one residual block 134 | 135 | Args: 136 | block: block type, basic block or bottle neck block 137 | out_channels: output depth channel number of this layer 138 | num_blocks: how many blocks per layer 139 | stride: the stride of the first block of this layer 140 | 141 | Return: 142 | return a resnet layer 143 | """ 144 | # we have num_block blocks per layer, the first block 145 | # could be 1 or 2, other blocks would always be 1 146 | strides = [stride] + [1] * (num_blocks - 1) 147 | layers = [] 148 | for stride in strides: 149 | layers.append(block(self.in_channels, out_channels, stride)) 150 | self.in_channels = out_channels * block.expansion 151 | 152 | return nn.Sequential(*layers) 153 | 154 | def forward(self, x): 155 | 156 | # output = self.features(x) 157 | output = self.conv1(x) 158 | output = self.conv2_x(output) 159 | output = self.conv3_x(output) 160 | output = self.conv4_x(output) 161 | output = self.conv5_x(output) 162 | output = self.avg_pool(output) 163 | output = output.view(output.size(0), -1) 164 | output = self.fc(output) 165 | 166 | return output 167 | 168 | def resnet18(num_classes=4): 169 | """ return a ResNet 18 object 170 | """ 171 | return ResNet(BasicBlock, [2, 2, 2, 2], num_classes) 172 | 173 | def resnet34(num_classes=4): 174 | """ return a ResNet 34 object 175 | """ 176 | return ResNet(BasicBlock, [3, 4, 6, 3], num_classes) 177 | 178 | def resnet50(num_classes=4): 179 | """ return a ResNet 50 object 180 | """ 181 | return ResNet(BottleNeck, [3, 4, 6, 3], num_classes) 182 | 183 | def resnet101(num_classes=4): 184 | """ return a ResNet 101 object 185 | """ 186 | return ResNet(BottleNeck, [3, 4, 23, 3], num_classes) 187 | 188 | def resnet152(num_classes=4): 189 | """ return a ResNet 152 object 190 | """ 191 | return ResNet(BottleNeck, [3, 8, 36, 3], num_classes) 192 | 193 | 194 | 195 | -------------------------------------------------------------------------------- /models/resnext.py: -------------------------------------------------------------------------------- 1 | """resnext in pytorch 2 | 3 | 4 | 5 | [1] Saining Xie, Ross Girshick, Piotr Dollár, Zhuowen Tu, Kaiming He. 6 | 7 | Aggregated Residual Transformations for Deep Neural Networks 8 | https://arxiv.org/abs/1611.05431 9 | """ 10 | 11 | import math 12 | import torch 13 | import torch.nn as nn 14 | import torch.nn.functional as F 15 | 16 | #only implements ResNext bottleneck c 17 | 18 | 19 | #"""This strategy exposes a new dimension, which we call “cardinality” 20 | #(the size of the set of transformations), as an essential factor 21 | #in addition to the dimensions of depth and width.""" 22 | CARDINALITY = 32 23 | DEPTH = 4 24 | BASEWIDTH = 64 25 | 26 | #"""The grouped convolutional layer in Fig. 3(c) performs 32 groups 27 | #of convolutions whose input and output channels are 4-dimensional. 28 | #The grouped convolutional layer concatenates them as the outputs 29 | #of the layer.""" 30 | 31 | class ResNextBottleNeckC(nn.Module): 32 | 33 | def __init__(self, in_channels, out_channels, stride): 34 | super().__init__() 35 | 36 | C = CARDINALITY #How many groups a feature map was splitted into 37 | 38 | #"""We note that the input/output width of the template is fixed as 39 | #256-d (Fig. 3), We note that the input/output width of the template 40 | #is fixed as 256-d (Fig. 3), and all widths are dou- bled each time 41 | #when the feature map is subsampled (see Table 1).""" 42 | D = int(DEPTH * out_channels / BASEWIDTH) #number of channels per group 43 | self.split_transforms = nn.Sequential( 44 | nn.Conv2d(in_channels, C * D, kernel_size=1, groups=C, bias=False), 45 | nn.BatchNorm2d(C * D), 46 | nn.ReLU(inplace=True), 47 | nn.Conv2d(C * D, C * D, kernel_size=3, stride=stride, groups=C, padding=1, bias=False), 48 | nn.BatchNorm2d(C * D), 49 | nn.ReLU(inplace=True), 50 | nn.Conv2d(C * D, out_channels * 4, kernel_size=1, bias=False), 51 | nn.BatchNorm2d(out_channels * 4), 52 | ) 53 | 54 | self.shortcut = nn.Sequential() 55 | 56 | if stride != 1 or in_channels != out_channels * 4: 57 | self.shortcut = nn.Sequential( 58 | nn.Conv2d(in_channels, out_channels * 4, stride=stride, kernel_size=1, bias=False), 59 | nn.BatchNorm2d(out_channels * 4) 60 | ) 61 | 62 | def forward(self, x): 63 | return F.relu(self.split_transforms(x) + self.shortcut(x)) 64 | 65 | class ResNext(nn.Module): 66 | 67 | def __init__(self, block, num_blocks, class_names=100): 68 | super().__init__() 69 | self.in_channels = 64 70 | 71 | self.conv1 = nn.Sequential( 72 | nn.Conv2d(3, 64, 3, stride=1, padding=1, bias=False), 73 | nn.BatchNorm2d(64), 74 | nn.ReLU(inplace=True) 75 | ) 76 | 77 | self.conv2 = self._make_layer(block, num_blocks[0], 64, 1) 78 | self.conv3 = self._make_layer(block, num_blocks[1], 128, 2) 79 | self.conv4 = self._make_layer(block, num_blocks[2], 256, 2) 80 | self.conv5 = self._make_layer(block, num_blocks[3], 512, 2) 81 | self.avg = nn.AdaptiveAvgPool2d((1, 1)) 82 | self.fc = nn.Linear(512 * 4, 100) 83 | 84 | def forward(self, x): 85 | x = self.conv1(x) 86 | x = self.conv2(x) 87 | x = self.conv3(x) 88 | x = self.conv4(x) 89 | x = self.conv5(x) 90 | x = self.avg(x) 91 | x = x.view(x.size(0), -1) 92 | x = self.fc(x) 93 | return x 94 | 95 | def _make_layer(self, block, num_block, out_channels, stride): 96 | """Building resnext block 97 | Args: 98 | block: block type(default resnext bottleneck c) 99 | num_block: number of blocks per layer 100 | out_channels: output channels per block 101 | stride: block stride 102 | 103 | Returns: 104 | a resnext layer 105 | """ 106 | strides = [stride] + [1] * (num_block - 1) 107 | layers = [] 108 | for stride in strides: 109 | layers.append(block(self.in_channels, out_channels, stride)) 110 | self.in_channels = out_channels * 4 111 | 112 | return nn.Sequential(*layers) 113 | 114 | def resnext50(): 115 | """ return a resnext50(c32x4d) network 116 | """ 117 | return ResNext(ResNextBottleNeckC, [3, 4, 6, 3]) 118 | 119 | def resnext101(): 120 | """ return a resnext101(c32x4d) network 121 | """ 122 | return ResNext(ResNextBottleNeckC, [3, 4, 23, 3]) 123 | 124 | def resnext152(): 125 | """ return a resnext101(c32x4d) network 126 | """ 127 | return ResNext(ResNextBottleNeckC, [3, 4, 36, 3]) 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /models/rir.py: -------------------------------------------------------------------------------- 1 | """resnet in resnet in pytorch 2 | 3 | 4 | 5 | [1] Sasha Targ, Diogo Almeida, Kevin Lyman. 6 | 7 | Resnet in Resnet: Generalizing Residual Architectures 8 | https://arxiv.org/abs/1603.08029v1 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | 14 | #geralized 15 | class ResnetInit(nn.Module): 16 | def __init__(self, in_channel, out_channel, stride): 17 | super().__init__() 18 | 19 | #"""The modular unit of the generalized residual network architecture is a 20 | #generalized residual block consisting of parallel states for a residual stream, 21 | #r, which contains identity shortcut connections and is similar to the structure 22 | #of a residual block from the original ResNet with a single convolutional layer 23 | #(parameters W l,r→r ) 24 | self.residual_stream_conv = nn.Conv2d(in_channel, out_channel, 3, padding=1, stride=stride) 25 | 26 | #"""and a transient stream, t, which is a standard convolutional layer 27 | #(W l,t→t ).""" 28 | self.transient_stream_conv = nn.Conv2d(in_channel, out_channel, 3, padding=1, stride=stride) 29 | 30 | #"""Two additional sets of convolutional filters in each block (W l,r→t , W l,t→r ) 31 | #also transfer information across streams.""" 32 | self.residual_stream_conv_across = nn.Conv2d(in_channel, out_channel, 3, padding=1, stride=stride) 33 | 34 | #"""We use equal numbers of filters for the residual and transient streams of the 35 | #generalized residual network, but optimizing this hyperparameter could lead to 36 | #further potential improvements.""" 37 | self.transient_stream_conv_across = nn.Conv2d(in_channel, out_channel, 3, padding=1, stride=stride) 38 | 39 | self.residual_bn_relu = nn.Sequential( 40 | nn.BatchNorm2d(out_channel), 41 | nn.ReLU(inplace=True) 42 | ) 43 | 44 | self.transient_bn_relu = nn.Sequential( 45 | nn.BatchNorm2d(out_channel), 46 | nn.ReLU(inplace=True) 47 | ) 48 | 49 | #"""The form of the shortcut connection can be an identity function with 50 | #the appropriate padding or a projection as in He et al. (2015b).""" 51 | self.short_cut = nn.Sequential() 52 | if in_channel != out_channel or stride != 1: 53 | self.short_cut = nn.Sequential( 54 | nn.Conv2d(in_channel, out_channel, kernel_size=1, stride=stride) 55 | ) 56 | 57 | 58 | def forward(self, x): 59 | x_residual, x_transient = x 60 | residual_r_r = self.residual_stream_conv(x_residual) 61 | residual_r_t = self.residual_stream_conv_across(x_residual) 62 | residual_shortcut = self.short_cut(x_residual) 63 | 64 | transient_t_t = self.transient_stream_conv(x_transient) 65 | transient_t_r = self.transient_stream_conv_across(x_transient) 66 | 67 | #transient_t_t = self.transient_stream_conv(x_residual) 68 | #transient_t_r = self.transient_stream_conv_across(x_residual) 69 | #"""Same-stream and cross-stream activations are summed (along with the 70 | #shortcut connection for the residual stream) before applying batch 71 | #normalization and ReLU nonlinearities (together σ) to get the output 72 | #states of the block (Equation 1) (Ioffe & Szegedy, 2015).""" 73 | x_residual = self.residual_bn_relu(residual_r_r + transient_t_r + residual_shortcut) 74 | x_transient = self.transient_bn_relu(residual_r_t + transient_t_t) 75 | 76 | return x_residual, x_transient 77 | 78 | 79 | 80 | class RiRBlock(nn.Module): 81 | def __init__(self, in_channel, out_channel, layer_num, stride, layer=ResnetInit): 82 | super().__init__() 83 | self.resnetinit = self._make_layers(in_channel, out_channel, layer_num, stride) 84 | 85 | #self.short_cut = nn.Sequential() 86 | #if stride != 1 or in_channel != out_channel: 87 | # self.short_cut = nn.Conv2d(in_channel, out_channel, kernel_size=1, stride=stride) 88 | 89 | def forward(self, x): 90 | x_residual, x_transient = self.resnetinit(x) 91 | #x_residual = x_residual + self.short_cut(x[0]) 92 | #x_transient = x_transient + self.short_cut(x[1]) 93 | 94 | return (x_residual, x_transient) 95 | 96 | #"""Replacing each of the convolutional layers within a residual 97 | #block from the original ResNet (Figure 1a) with a generalized residual block 98 | #(Figure 1b) leads us to a new architecture we call ResNet in ResNet (RiR) 99 | #(Figure 1d).""" 100 | def _make_layers(self, in_channel, out_channel, layer_num, stride, layer=ResnetInit): 101 | strides = [stride] + [1] * (layer_num - 1) 102 | layers = nn.Sequential() 103 | for index, s in enumerate(strides): 104 | layers.add_module("generalized layers{}".format(index), layer(in_channel, out_channel, s)) 105 | in_channel = out_channel 106 | 107 | return layers 108 | 109 | class ResnetInResneet(nn.Module): 110 | def __init__(self, num_classes=100): 111 | super().__init__() 112 | base = int(96 / 2) 113 | self.residual_pre_conv = nn.Sequential( 114 | nn.Conv2d(3, base, 3, padding=1), 115 | nn.BatchNorm2d(base), 116 | nn.ReLU(inplace=True) 117 | ) 118 | self.transient_pre_conv = nn.Sequential( 119 | nn.Conv2d(3, base, 3, padding=1), 120 | nn.BatchNorm2d(base), 121 | nn.ReLU(inplace=True) 122 | ) 123 | 124 | self.rir1 = RiRBlock(base, base, 2, 1) 125 | self.rir2 = RiRBlock(base, base, 2, 1) 126 | self.rir3 = RiRBlock(base, base * 2, 2, 2) 127 | self.rir4 = RiRBlock(base * 2, base * 2, 2, 1) 128 | self.rir5 = RiRBlock(base * 2, base * 2, 2, 1) 129 | self.rir6 = RiRBlock(base * 2, base * 4, 2, 2) 130 | self.rir7 = RiRBlock(base * 4, base * 4, 2, 1) 131 | self.rir8 = RiRBlock(base * 4, base * 4, 2, 1) 132 | 133 | self.conv1 = nn.Sequential( 134 | nn.Conv2d(384, num_classes, kernel_size=3, stride=2), #without this convolution, loss will soon be nan 135 | nn.BatchNorm2d(num_classes), 136 | nn.ReLU(inplace=True), 137 | ) 138 | 139 | self.classifier = nn.Sequential( 140 | nn.Linear(900, 450), 141 | nn.ReLU(), 142 | nn.Dropout(), 143 | nn.Linear(450, 100), 144 | ) 145 | 146 | self._weight_init() 147 | 148 | def forward(self, x): 149 | x_residual = self.residual_pre_conv(x) 150 | x_transient = self.transient_pre_conv(x) 151 | 152 | x_residual, x_transient = self.rir1((x_residual, x_transient)) 153 | x_residual, x_transient = self.rir2((x_residual, x_transient)) 154 | x_residual, x_transient = self.rir3((x_residual, x_transient)) 155 | x_residual, x_transient = self.rir4((x_residual, x_transient)) 156 | x_residual, x_transient = self.rir5((x_residual, x_transient)) 157 | x_residual, x_transient = self.rir6((x_residual, x_transient)) 158 | x_residual, x_transient = self.rir7((x_residual, x_transient)) 159 | x_residual, x_transient = self.rir8((x_residual, x_transient)) 160 | h = torch.cat([x_residual, x_transient], 1) 161 | h = self.conv1(h) 162 | h = h.view(h.size()[0], -1) 163 | h = self.classifier(h) 164 | 165 | return h 166 | 167 | def _weight_init(self): 168 | for m in self.modules(): 169 | if isinstance(m, nn.Conv2d): 170 | torch.nn.init.kaiming_normal(m.weight) 171 | m.bias.data.fill_(0.01) 172 | 173 | 174 | def resnet_in_resnet(): 175 | return ResnetInResneet() 176 | -------------------------------------------------------------------------------- /models/senet.py: -------------------------------------------------------------------------------- 1 | """senet in pytorch 2 | 3 | 4 | 5 | [1] Jie Hu, Li Shen, Samuel Albanie, Gang Sun, Enhua Wu 6 | 7 | Squeeze-and-Excitation Networks 8 | https://arxiv.org/abs/1709.01507 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.functional as F 14 | 15 | class BasicResidualSEBlock(nn.Module): 16 | 17 | expansion = 1 18 | 19 | def __init__(self, in_channels, out_channels, stride, r=16): 20 | super().__init__() 21 | 22 | self.residual = nn.Sequential( 23 | nn.Conv2d(in_channels, out_channels, 3, stride=stride, padding=1), 24 | nn.BatchNorm2d(out_channels), 25 | nn.ReLU(inplace=True), 26 | 27 | nn.Conv2d(out_channels, out_channels * self.expansion, 3, padding=1), 28 | nn.BatchNorm2d(out_channels * self.expansion), 29 | nn.ReLU(inplace=True) 30 | ) 31 | 32 | self.shortcut = nn.Sequential() 33 | if stride != 1 or in_channels != out_channels * self.expansion: 34 | self.shortcut = nn.Sequential( 35 | nn.Conv2d(in_channels, out_channels * self.expansion, 1, stride=stride), 36 | nn.BatchNorm2d(out_channels * self.expansion) 37 | ) 38 | 39 | self.squeeze = nn.AdaptiveAvgPool2d(1) 40 | self.excitation = nn.Sequential( 41 | nn.Linear(out_channels * self.expansion, out_channels * self.expansion // r), 42 | nn.ReLU(inplace=True), 43 | nn.Linear(out_channels * self.expansion // r, out_channels * self.expansion), 44 | nn.Sigmoid() 45 | ) 46 | 47 | def forward(self, x): 48 | shortcut = self.shortcut(x) 49 | residual = self.residual(x) 50 | 51 | squeeze = self.squeeze(residual) 52 | squeeze = squeeze.view(squeeze.size(0), -1) 53 | excitation = self.excitation(squeeze) 54 | excitation = excitation.view(residual.size(0), residual.size(1), 1, 1) 55 | 56 | x = residual * excitation.expand_as(residual) + shortcut 57 | 58 | return F.relu(x) 59 | 60 | class BottleneckResidualSEBlock(nn.Module): 61 | 62 | expansion = 4 63 | 64 | def __init__(self, in_channels, out_channels, stride, r=16): 65 | super().__init__() 66 | 67 | self.residual = nn.Sequential( 68 | nn.Conv2d(in_channels, out_channels, 1), 69 | nn.BatchNorm2d(out_channels), 70 | nn.ReLU(inplace=True), 71 | 72 | nn.Conv2d(out_channels, out_channels, 3, stride=stride, padding=1), 73 | nn.BatchNorm2d(out_channels), 74 | nn.ReLU(inplace=True), 75 | 76 | nn.Conv2d(out_channels, out_channels * self.expansion, 1), 77 | nn.BatchNorm2d(out_channels * self.expansion), 78 | nn.ReLU(inplace=True) 79 | ) 80 | 81 | self.squeeze = nn.AdaptiveAvgPool2d(1) 82 | self.excitation = nn.Sequential( 83 | nn.Linear(out_channels * self.expansion, out_channels * self.expansion // r), 84 | nn.ReLU(inplace=True), 85 | nn.Linear(out_channels * self.expansion // r, out_channels * self.expansion), 86 | nn.Sigmoid() 87 | ) 88 | 89 | self.shortcut = nn.Sequential() 90 | if stride != 1 or in_channels != out_channels * self.expansion: 91 | self.shortcut = nn.Sequential( 92 | nn.Conv2d(in_channels, out_channels * self.expansion, 1, stride=stride), 93 | nn.BatchNorm2d(out_channels * self.expansion) 94 | ) 95 | 96 | def forward(self, x): 97 | 98 | shortcut = self.shortcut(x) 99 | 100 | residual = self.residual(x) 101 | squeeze = self.squeeze(residual) 102 | squeeze = squeeze.view(squeeze.size(0), -1) 103 | excitation = self.excitation(squeeze) 104 | excitation = excitation.view(residual.size(0), residual.size(1), 1, 1) 105 | 106 | x = residual * excitation.expand_as(residual) + shortcut 107 | 108 | return F.relu(x) 109 | 110 | class SEResNet(nn.Module): 111 | 112 | def __init__(self, block, block_num, class_num=100): 113 | super().__init__() 114 | 115 | self.in_channels = 64 116 | 117 | self.pre = nn.Sequential( 118 | nn.Conv2d(3, 64, 3, padding=1), 119 | nn.BatchNorm2d(64), 120 | nn.ReLU(inplace=True) 121 | ) 122 | 123 | self.stage1 = self._make_stage(block, block_num[0], 64, 1) 124 | self.stage2 = self._make_stage(block, block_num[1], 128, 2) 125 | self.stage3 = self._make_stage(block, block_num[2], 256, 2) 126 | self.stage4 = self._make_stage(block, block_num[3], 512, 2) 127 | 128 | self.linear = nn.Linear(self.in_channels, class_num) 129 | 130 | def forward(self, x): 131 | x = self.pre(x) 132 | 133 | x = self.stage1(x) 134 | x = self.stage2(x) 135 | x = self.stage3(x) 136 | x = self.stage4(x) 137 | 138 | x = F.adaptive_avg_pool2d(x, 1) 139 | x = x.view(x.size(0), -1) 140 | 141 | x = self.linear(x) 142 | 143 | return x 144 | 145 | 146 | def _make_stage(self, block, num, out_channels, stride): 147 | 148 | layers = [] 149 | layers.append(block(self.in_channels, out_channels, stride)) 150 | self.in_channels = out_channels * block.expansion 151 | 152 | while num - 1: 153 | layers.append(block(self.in_channels, out_channels, 1)) 154 | num -= 1 155 | 156 | return nn.Sequential(*layers) 157 | 158 | def seresnet18(): 159 | return SEResNet(BasicResidualSEBlock, [2, 2, 2, 2]) 160 | 161 | def seresnet34(): 162 | return SEResNet(BasicResidualSEBlock, [3, 4, 6, 3]) 163 | 164 | def seresnet50(): 165 | return SEResNet(BottleneckResidualSEBlock, [3, 4, 6, 3]) 166 | 167 | def seresnet101(): 168 | return SEResNet(BottleneckResidualSEBlock, [3, 4, 23, 3]) 169 | 170 | def seresnet152(): 171 | return SEResNet(BottleneckResidualSEBlock, [3, 8, 36, 3]) 172 | -------------------------------------------------------------------------------- /models/shufflenet.py: -------------------------------------------------------------------------------- 1 | """shufflenet in pytorch 2 | 3 | 4 | 5 | [1] Xiangyu Zhang, Xinyu Zhou, Mengxiao Lin, Jian Sun. 6 | 7 | ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices 8 | https://arxiv.org/abs/1707.01083v2 9 | """ 10 | 11 | from functools import partial 12 | 13 | import torch 14 | import torch.nn as nn 15 | 16 | 17 | class BasicConv2d(nn.Module): 18 | 19 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 20 | super().__init__() 21 | self.conv = nn.Conv2d(input_channels, output_channels, kernel_size, **kwargs) 22 | self.bn = nn.BatchNorm2d(output_channels) 23 | self.relu = nn.ReLU(inplace=True) 24 | 25 | def forward(self, x): 26 | x = self.conv(x) 27 | x = self.bn(x) 28 | x = self.relu(x) 29 | return x 30 | 31 | class ChannelShuffle(nn.Module): 32 | 33 | def __init__(self, groups): 34 | super().__init__() 35 | self.groups = groups 36 | 37 | def forward(self, x): 38 | batchsize, channels, height, width = x.data.size() 39 | channels_per_group = int(channels / self.groups) 40 | 41 | #"""suppose a convolutional layer with g groups whose output has 42 | #g x n channels; we first reshape the output channel dimension 43 | #into (g, n)""" 44 | x = x.view(batchsize, self.groups, channels_per_group, height, width) 45 | 46 | #"""transposing and then flattening it back as the input of next layer.""" 47 | x = x.transpose(1, 2).contiguous() 48 | x = x.view(batchsize, -1, height, width) 49 | 50 | return x 51 | 52 | class DepthwiseConv2d(nn.Module): 53 | 54 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 55 | super().__init__() 56 | self.depthwise = nn.Sequential( 57 | nn.Conv2d(input_channels, output_channels, kernel_size, **kwargs), 58 | nn.BatchNorm2d(output_channels) 59 | ) 60 | 61 | def forward(self, x): 62 | return self.depthwise(x) 63 | 64 | class PointwiseConv2d(nn.Module): 65 | def __init__(self, input_channels, output_channels, **kwargs): 66 | super().__init__() 67 | self.pointwise = nn.Sequential( 68 | nn.Conv2d(input_channels, output_channels, 1, **kwargs), 69 | nn.BatchNorm2d(output_channels) 70 | ) 71 | 72 | def forward(self, x): 73 | return self.pointwise(x) 74 | 75 | class ShuffleNetUnit(nn.Module): 76 | 77 | def __init__(self, input_channels, output_channels, stage, stride, groups): 78 | super().__init__() 79 | 80 | #"""Similar to [9], we set the number of bottleneck channels to 1/4 81 | #of the output channels for each ShuffleNet unit.""" 82 | self.bottlneck = nn.Sequential( 83 | PointwiseConv2d( 84 | input_channels, 85 | int(output_channels / 4), 86 | groups=groups 87 | ), 88 | nn.ReLU(inplace=True) 89 | ) 90 | 91 | #"""Note that for Stage 2, we do not apply group convolution on the first pointwise 92 | #layer because the number of input channels is relatively small.""" 93 | if stage == 2: 94 | self.bottlneck = nn.Sequential( 95 | PointwiseConv2d( 96 | input_channels, 97 | int(output_channels / 4), 98 | groups=groups 99 | ), 100 | nn.ReLU(inplace=True) 101 | ) 102 | 103 | self.channel_shuffle = ChannelShuffle(groups) 104 | 105 | self.depthwise = DepthwiseConv2d( 106 | int(output_channels / 4), 107 | int(output_channels / 4), 108 | 3, 109 | groups=int(output_channels / 4), 110 | stride=stride, 111 | padding=1 112 | ) 113 | 114 | self.expand = PointwiseConv2d( 115 | int(output_channels / 4), 116 | output_channels, 117 | groups=groups 118 | ) 119 | 120 | self.relu = nn.ReLU(inplace=True) 121 | self.fusion = self._add 122 | self.shortcut = nn.Sequential() 123 | 124 | #"""As for the case where ShuffleNet is applied with stride, 125 | #we simply make two modifications (see Fig 2 (c)): 126 | #(i) add a 3 × 3 average pooling on the shortcut path; 127 | #(ii) replace the element-wise addition with channel concatenation, 128 | #which makes it easy to enlarge channel dimension with little extra 129 | #computation cost. 130 | if stride != 1 or input_channels != output_channels: 131 | self.shortcut = nn.AvgPool2d(3, stride=2, padding=1) 132 | 133 | self.expand = PointwiseConv2d( 134 | int(output_channels / 4), 135 | output_channels - input_channels, 136 | groups=groups 137 | ) 138 | 139 | self.fusion = self._cat 140 | 141 | def _add(self, x, y): 142 | return torch.add(x, y) 143 | 144 | def _cat(self, x, y): 145 | return torch.cat([x, y], dim=1) 146 | 147 | def forward(self, x): 148 | shortcut = self.shortcut(x) 149 | 150 | shuffled = self.bottlneck(x) 151 | shuffled = self.channel_shuffle(shuffled) 152 | shuffled = self.depthwise(shuffled) 153 | shuffled = self.expand(shuffled) 154 | 155 | output = self.fusion(shortcut, shuffled) 156 | output = self.relu(output) 157 | 158 | return output 159 | 160 | class ShuffleNet(nn.Module): 161 | 162 | def __init__(self, num_blocks, num_classes=100, groups=3): 163 | super().__init__() 164 | 165 | if groups == 1: 166 | out_channels = [24, 144, 288, 567] 167 | elif groups == 2: 168 | out_channels = [24, 200, 400, 800] 169 | elif groups == 3: 170 | out_channels = [24, 240, 480, 960] 171 | elif groups == 4: 172 | out_channels = [24, 272, 544, 1088] 173 | elif groups == 8: 174 | out_channels = [24, 384, 768, 1536] 175 | 176 | self.conv1 = BasicConv2d(3, out_channels[0], 3, padding=1, stride=1) 177 | self.input_channels = out_channels[0] 178 | 179 | self.stage2 = self._make_stage( 180 | ShuffleNetUnit, 181 | num_blocks[0], 182 | out_channels[1], 183 | stride=2, 184 | stage=2, 185 | groups=groups 186 | ) 187 | 188 | self.stage3 = self._make_stage( 189 | ShuffleNetUnit, 190 | num_blocks[1], 191 | out_channels[2], 192 | stride=2, 193 | stage=3, 194 | groups=groups 195 | ) 196 | 197 | self.stage4 = self._make_stage( 198 | ShuffleNetUnit, 199 | num_blocks[2], 200 | out_channels[3], 201 | stride=2, 202 | stage=4, 203 | groups=groups 204 | ) 205 | 206 | self.avg = nn.AdaptiveAvgPool2d((1, 1)) 207 | self.fc = nn.Linear(out_channels[3], num_classes) 208 | 209 | def forward(self, x): 210 | x = self.conv1(x) 211 | x = self.stage2(x) 212 | x = self.stage3(x) 213 | x = self.stage4(x) 214 | x = self.avg(x) 215 | x = x.view(x.size(0), -1) 216 | x = self.fc(x) 217 | 218 | return x 219 | 220 | def _make_stage(self, block, num_blocks, output_channels, stride, stage, groups): 221 | """make shufflenet stage 222 | 223 | Args: 224 | block: block type, shuffle unit 225 | out_channels: output depth channel number of this stage 226 | num_blocks: how many blocks per stage 227 | stride: the stride of the first block of this stage 228 | stage: stage index 229 | groups: group number of group convolution 230 | Return: 231 | return a shuffle net stage 232 | """ 233 | strides = [stride] + [1] * (num_blocks - 1) 234 | 235 | stage = [] 236 | 237 | for stride in strides: 238 | stage.append( 239 | block( 240 | self.input_channels, 241 | output_channels, 242 | stride=stride, 243 | stage=stage, 244 | groups=groups 245 | ) 246 | ) 247 | self.input_channels = output_channels 248 | 249 | return nn.Sequential(*stage) 250 | 251 | def shufflenet(): 252 | return ShuffleNet([4, 8, 4]) 253 | 254 | 255 | 256 | 257 | -------------------------------------------------------------------------------- /models/shufflenetv2.py: -------------------------------------------------------------------------------- 1 | """shufflenetv2 in pytorch 2 | 3 | 4 | 5 | [1] Ningning Ma, Xiangyu Zhang, Hai-Tao Zheng, Jian Sun 6 | 7 | ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design 8 | https://arxiv.org/abs/1807.11164 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.functional as F 14 | 15 | 16 | def channel_split(x, split): 17 | """split a tensor into two pieces along channel dimension 18 | Args: 19 | x: input tensor 20 | split:(int) channel size for each pieces 21 | """ 22 | assert x.size(1) == split * 2 23 | return torch.split(x, split, dim=1) 24 | 25 | def channel_shuffle(x, groups): 26 | """channel shuffle operation 27 | Args: 28 | x: input tensor 29 | groups: input branch number 30 | """ 31 | 32 | batch_size, channels, height, width = x.size() 33 | channels_per_group = int(channels // groups) 34 | 35 | x = x.view(batch_size, groups, channels_per_group, height, width) 36 | x = x.transpose(1, 2).contiguous() 37 | x = x.view(batch_size, -1, height, width) 38 | 39 | return x 40 | 41 | class ShuffleUnit(nn.Module): 42 | 43 | def __init__(self, in_channels, out_channels, stride): 44 | super().__init__() 45 | 46 | self.stride = stride 47 | self.in_channels = in_channels 48 | self.out_channels = out_channels 49 | 50 | if stride != 1 or in_channels != out_channels: 51 | self.residual = nn.Sequential( 52 | nn.Conv2d(in_channels, in_channels, 1), 53 | nn.BatchNorm2d(in_channels), 54 | nn.ReLU(inplace=True), 55 | nn.Conv2d(in_channels, in_channels, 3, stride=stride, padding=1, groups=in_channels), 56 | nn.BatchNorm2d(in_channels), 57 | nn.Conv2d(in_channels, int(out_channels / 2), 1), 58 | nn.BatchNorm2d(int(out_channels / 2)), 59 | nn.ReLU(inplace=True) 60 | ) 61 | 62 | self.shortcut = nn.Sequential( 63 | nn.Conv2d(in_channels, in_channels, 3, stride=stride, padding=1, groups=in_channels), 64 | nn.BatchNorm2d(in_channels), 65 | nn.Conv2d(in_channels, int(out_channels / 2), 1), 66 | nn.BatchNorm2d(int(out_channels / 2)), 67 | nn.ReLU(inplace=True) 68 | ) 69 | else: 70 | self.shortcut = nn.Sequential() 71 | 72 | in_channels = int(in_channels / 2) 73 | self.residual = nn.Sequential( 74 | nn.Conv2d(in_channels, in_channels, 1), 75 | nn.BatchNorm2d(in_channels), 76 | nn.ReLU(inplace=True), 77 | nn.Conv2d(in_channels, in_channels, 3, stride=stride, padding=1, groups=in_channels), 78 | nn.BatchNorm2d(in_channels), 79 | nn.Conv2d(in_channels, in_channels, 1), 80 | nn.BatchNorm2d(in_channels), 81 | nn.ReLU(inplace=True) 82 | ) 83 | 84 | 85 | def forward(self, x): 86 | 87 | if self.stride == 1 and self.out_channels == self.in_channels: 88 | shortcut, residual = channel_split(x, int(self.in_channels / 2)) 89 | else: 90 | shortcut = x 91 | residual = x 92 | 93 | shortcut = self.shortcut(shortcut) 94 | residual = self.residual(residual) 95 | x = torch.cat([shortcut, residual], dim=1) 96 | x = channel_shuffle(x, 2) 97 | 98 | return x 99 | 100 | class ShuffleNetV2(nn.Module): 101 | 102 | def __init__(self, ratio=1, class_num=100): 103 | super().__init__() 104 | if ratio == 0.5: 105 | out_channels = [48, 96, 192, 1024] 106 | elif ratio == 1: 107 | out_channels = [116, 232, 464, 1024] 108 | elif ratio == 1.5: 109 | out_channels = [176, 352, 704, 1024] 110 | elif ratio == 2: 111 | out_channels = [244, 488, 976, 2048] 112 | else: 113 | ValueError('unsupported ratio number') 114 | 115 | self.pre = nn.Sequential( 116 | nn.Conv2d(3, 24, 3, padding=1), 117 | nn.BatchNorm2d(24) 118 | ) 119 | 120 | self.stage2 = self._make_stage(24, out_channels[0], 3) 121 | self.stage3 = self._make_stage(out_channels[0], out_channels[1], 7) 122 | self.stage4 = self._make_stage(out_channels[1], out_channels[2], 3) 123 | self.conv5 = nn.Sequential( 124 | nn.Conv2d(out_channels[2], out_channels[3], 1), 125 | nn.BatchNorm2d(out_channels[3]), 126 | nn.ReLU(inplace=True) 127 | ) 128 | 129 | self.fc = nn.Linear(out_channels[3], class_num) 130 | 131 | def forward(self, x): 132 | x = self.pre(x) 133 | x = self.stage2(x) 134 | x = self.stage3(x) 135 | x = self.stage4(x) 136 | x = self.conv5(x) 137 | x = F.adaptive_avg_pool2d(x, 1) 138 | x = x.view(x.size(0), -1) 139 | x = self.fc(x) 140 | 141 | return x 142 | 143 | def _make_stage(self, in_channels, out_channels, repeat): 144 | layers = [] 145 | layers.append(ShuffleUnit(in_channels, out_channels, 2)) 146 | 147 | while repeat: 148 | layers.append(ShuffleUnit(out_channels, out_channels, 1)) 149 | repeat -= 1 150 | 151 | return nn.Sequential(*layers) 152 | 153 | def shufflenetv2(): 154 | return ShuffleNetV2() 155 | 156 | 157 | 158 | 159 | 160 | -------------------------------------------------------------------------------- /models/squeezenet.py: -------------------------------------------------------------------------------- 1 | """squeezenet in pytorch 2 | 3 | 4 | 5 | [1] Song Han, Jeff Pool, John Tran, William J. Dally 6 | 7 | squeezenet: Learning both Weights and Connections for Efficient Neural Networks 8 | https://arxiv.org/abs/1506.02626 9 | """ 10 | 11 | import torch 12 | import torch.nn as nn 13 | 14 | 15 | class Fire(nn.Module): 16 | 17 | def __init__(self, in_channel, out_channel, squzee_channel): 18 | 19 | super().__init__() 20 | self.squeeze = nn.Sequential( 21 | nn.Conv2d(in_channel, squzee_channel, 1), 22 | nn.BatchNorm2d(squzee_channel), 23 | nn.ReLU(inplace=True) 24 | ) 25 | 26 | self.expand_1x1 = nn.Sequential( 27 | nn.Conv2d(squzee_channel, int(out_channel / 2), 1), 28 | nn.BatchNorm2d(int(out_channel / 2)), 29 | nn.ReLU(inplace=True) 30 | ) 31 | 32 | self.expand_3x3 = nn.Sequential( 33 | nn.Conv2d(squzee_channel, int(out_channel / 2), 3, padding=1), 34 | nn.BatchNorm2d(int(out_channel / 2)), 35 | nn.ReLU(inplace=True) 36 | ) 37 | 38 | def forward(self, x): 39 | 40 | x = self.squeeze(x) 41 | x = torch.cat([ 42 | self.expand_1x1(x), 43 | self.expand_3x3(x) 44 | ], 1) 45 | 46 | return x 47 | 48 | class SqueezeNet(nn.Module): 49 | 50 | """mobile net with simple bypass""" 51 | def __init__(self, class_num=100): 52 | 53 | super().__init__() 54 | self.stem = nn.Sequential( 55 | nn.Conv2d(3, 96, 3, padding=1), 56 | nn.BatchNorm2d(96), 57 | nn.ReLU(inplace=True), 58 | nn.MaxPool2d(2, 2) 59 | ) 60 | 61 | self.fire2 = Fire(96, 128, 16) 62 | self.fire3 = Fire(128, 128, 16) 63 | self.fire4 = Fire(128, 256, 32) 64 | self.fire5 = Fire(256, 256, 32) 65 | self.fire6 = Fire(256, 384, 48) 66 | self.fire7 = Fire(384, 384, 48) 67 | self.fire8 = Fire(384, 512, 64) 68 | self.fire9 = Fire(512, 512, 64) 69 | 70 | self.conv10 = nn.Conv2d(512, class_num, 1) 71 | self.avg = nn.AdaptiveAvgPool2d(1) 72 | self.maxpool = nn.MaxPool2d(2, 2) 73 | 74 | def forward(self, x): 75 | x = self.stem(x) 76 | 77 | f2 = self.fire2(x) 78 | f3 = self.fire3(f2) + f2 79 | f4 = self.fire4(f3) 80 | f4 = self.maxpool(f4) 81 | 82 | f5 = self.fire5(f4) + f4 83 | f6 = self.fire6(f5) 84 | f7 = self.fire7(f6) + f6 85 | f8 = self.fire8(f7) 86 | f8 = self.maxpool(f8) 87 | 88 | f9 = self.fire9(f8) 89 | c10 = self.conv10(f9) 90 | 91 | x = self.avg(c10) 92 | x = x.view(x.size(0), -1) 93 | 94 | return x 95 | 96 | def squeezenet(class_num=100): 97 | return SqueezeNet(class_num=class_num) 98 | -------------------------------------------------------------------------------- /models/stochasticdepth.py: -------------------------------------------------------------------------------- 1 | """ 2 | resnet with stochastic depth 3 | 4 | [1] Gao Huang, Yu Sun, Zhuang Liu, Daniel Sedra, Kilian Weinberger 5 | Deep Networks with Stochastic Depth 6 | 7 | https://arxiv.org/abs/1603.09382v3 8 | """ 9 | import torch 10 | import torch.nn as nn 11 | from torch.distributions.bernoulli import Bernoulli 12 | import random 13 | 14 | 15 | class StochasticDepthBasicBlock(torch.jit.ScriptModule): 16 | 17 | expansion=1 18 | 19 | def __init__(self, p, in_channels, out_channels, stride=1): 20 | super().__init__() 21 | 22 | #self.p = torch.tensor(p).float() 23 | self.p = p 24 | self.residual = nn.Sequential( 25 | nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1), 26 | nn.BatchNorm2d(out_channels), 27 | nn.ReLU(inplace=True), 28 | nn.Conv2d(out_channels, out_channels * StochasticDepthBasicBlock.expansion, kernel_size=3, padding=1), 29 | nn.BatchNorm2d(out_channels) 30 | ) 31 | 32 | self.shortcut = nn.Sequential() 33 | 34 | if stride != 1 or in_channels != out_channels * StochasticDepthBasicBlock.expansion: 35 | self.shortcut = nn.Sequential( 36 | nn.Conv2d(in_channels, out_channels * StochasticDepthBasicBlock.expansion, kernel_size=1, stride=stride), 37 | nn.BatchNorm2d(out_channels) 38 | ) 39 | def survival(self): 40 | var = torch.bernoulli(torch.tensor(self.p).float()) 41 | return torch.equal(var, torch.tensor(1).float().to(var.device)) 42 | 43 | @torch.jit.script_method 44 | def forward(self, x): 45 | 46 | if self.training: 47 | if self.survival(): 48 | # official torch implementation 49 | # function ResidualDrop:updateOutput(input) 50 | # local skip_forward = self.skip:forward(input) 51 | # self.output:resizeAs(skip_forward):copy(skip_forward) 52 | # if self.train then 53 | # if self.gate then -- only compute convolutional output when gate is open 54 | # self.output:add(self.net:forward(input)) 55 | # end 56 | # else 57 | # self.output:add(self.net:forward(input):mul(1-self.deathRate)) 58 | # end 59 | # return self.output 60 | # end 61 | 62 | # paper: 63 | # Hl = ReLU(bl*fl(Hl−1) + id(Hl−1)). 64 | 65 | # paper and their official implementation are different 66 | # paper use relu after output 67 | # official implementation dosen't 68 | # 69 | # other implementions which use relu: 70 | # https://github.com/jiweeo/pytorch-stochastic-depth/blob/a6f95aaffee82d273c1cd73d9ed6ef0718c6683d/models/resnet.py 71 | # https://github.com/dblN/stochastic_depth_keras/blob/master/train.py 72 | 73 | # implementations which doesn't use relu: 74 | # https://github.com/transcranial/stochastic-depth/blob/master/stochastic-depth.ipynb 75 | # https://github.com/shamangary/Pytorch-Stochastic-Depth-Resnet/blob/master/TYY_stodepth_lineardecay.py 76 | 77 | # I will just stick with the official implementation, I think 78 | # whether add relu after residual won't effect the network 79 | # performance too much 80 | x = self.residual(x) + self.shortcut(x) 81 | else: 82 | # If bl = 0, the ResBlock reduces to the identity function 83 | x = self.shortcut(x) 84 | 85 | else: 86 | x = self.residual(x) * self.p + self.shortcut(x) 87 | 88 | return x 89 | 90 | 91 | class StochasticDepthBottleNeck(torch.jit.ScriptModule): 92 | """Residual block for resnet over 50 layers 93 | 94 | """ 95 | expansion = 4 96 | def __init__(self, p, in_channels, out_channels, stride=1): 97 | super().__init__() 98 | 99 | self.p = p 100 | self.residual = nn.Sequential( 101 | nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False), 102 | nn.BatchNorm2d(out_channels), 103 | nn.ReLU(inplace=True), 104 | nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False), 105 | nn.BatchNorm2d(out_channels), 106 | nn.ReLU(inplace=True), 107 | nn.Conv2d(out_channels, out_channels * StochasticDepthBottleNeck.expansion, kernel_size=1, bias=False), 108 | nn.BatchNorm2d(out_channels * StochasticDepthBottleNeck.expansion), 109 | ) 110 | 111 | self.shortcut = nn.Sequential() 112 | 113 | if stride != 1 or in_channels != out_channels * StochasticDepthBottleNeck.expansion: 114 | self.shortcut = nn.Sequential( 115 | nn.Conv2d(in_channels, out_channels * StochasticDepthBottleNeck.expansion, stride=stride, kernel_size=1, bias=False), 116 | nn.BatchNorm2d(out_channels * StochasticDepthBottleNeck.expansion) 117 | ) 118 | 119 | def survival(self): 120 | var = torch.bernoulli(torch.tensor(self.p).float()) 121 | return torch.equal(var, torch.tensor(1).float().to(var.device)) 122 | 123 | @torch.jit.script_method 124 | def forward(self, x): 125 | 126 | if self.training: 127 | if self.survival(): 128 | x = self.residual(x) + self.shortcut(x) 129 | else: 130 | x = self.shortcut(x) 131 | else: 132 | x = self.residual(x) * self.p + self.shortcut(x) 133 | 134 | return x 135 | 136 | class StochasticDepthResNet(nn.Module): 137 | 138 | def __init__(self, block, num_block, num_classes=100): 139 | super().__init__() 140 | 141 | self.in_channels = 64 142 | self.conv1 = nn.Sequential( 143 | nn.Conv2d(3, 64, kernel_size=3, padding=1), 144 | nn.BatchNorm2d(64), 145 | nn.ReLU(inplace=True) 146 | ) 147 | 148 | self.step = (1 - 0.5) / (sum(num_block) - 1) 149 | self.pl = 1 150 | self.conv2_x = self._make_layer(block, 64, num_block[0], 1) 151 | self.conv3_x = self._make_layer(block, 128, num_block[1], 2) 152 | self.conv4_x = self._make_layer(block, 256, num_block[2], 2) 153 | self.conv5_x = self._make_layer(block, 512, num_block[3], 2) 154 | self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) 155 | self.fc = nn.Linear(512 * block.expansion, num_classes) 156 | 157 | def _make_layer(self, block, out_channels, num_blocks, stride): 158 | 159 | strides = [stride] + [1] * (num_blocks - 1) 160 | layers = [] 161 | for stride in strides: 162 | layers.append(block(self.pl, self.in_channels, out_channels, stride)) 163 | self.in_channels = out_channels * block.expansion 164 | self.pl -= self.step 165 | 166 | return nn.Sequential(*layers) 167 | 168 | def forward(self, x): 169 | output = self.conv1(x) 170 | output = self.conv2_x(output) 171 | output = self.conv3_x(output) 172 | output = self.conv4_x(output) 173 | output = self.conv5_x(output) 174 | output = self.avg_pool(output) 175 | output = output.view(output.size(0), -1) 176 | output = self.fc(output) 177 | 178 | return output 179 | 180 | 181 | def stochastic_depth_resnet18(): 182 | """ return a ResNet 18 object 183 | """ 184 | return StochasticDepthResNet(StochasticDepthBasicBlock, [2, 2, 2, 2]) 185 | 186 | def stochastic_depth_resnet34(): 187 | """ return a ResNet 34 object 188 | """ 189 | return StochasticDepthResNet(StochasticDepthBasicBlock, [3, 4, 6, 3]) 190 | 191 | def stochastic_depth_resnet50(): 192 | 193 | """ return a ResNet 50 object 194 | """ 195 | return StochasticDepthResNet(StochasticDepthBottleNeck, [3, 4, 6, 3]) 196 | 197 | def stochastic_depth_resnet101(): 198 | """ return a ResNet 101 object 199 | """ 200 | return StochasticDepthResNet(StochasticDepthBottleNeck, [3, 4, 23, 3]) 201 | 202 | def stochastic_depth_resnet152(): 203 | """ return a ResNet 152 object 204 | """ 205 | return StochasticDepthResNet(StochasticDepthBottleNeck, [3, 8, 36, 3]) 206 | 207 | -------------------------------------------------------------------------------- /models/vgg.py: -------------------------------------------------------------------------------- 1 | """vgg in pytorch 2 | 3 | 4 | [1] Karen Simonyan, Andrew Zisserman 5 | 6 | Very Deep Convolutional Networks for Large-Scale Image Recognition. 7 | https://arxiv.org/abs/1409.1556v6 8 | """ 9 | '''VGG11/13/16/19 in Pytorch.''' 10 | 11 | import torch 12 | import torch.nn as nn 13 | from conf import settings 14 | 15 | cfg = { 16 | 'A' : [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 17 | 'B' : [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 18 | 'D' : [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 19 | 'E' : [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'] 20 | } 21 | 22 | class VGG(nn.Module): 23 | 24 | def __init__(self, features, num_class=100, preprocessing={}): 25 | super().__init__() 26 | self.features = features 27 | 28 | self.preprocessing = preprocessing 29 | 30 | if not len(self.preprocessing) == 0: 31 | self.mu = torch.tensor(preprocessing['mean']).float().view(3, 1, 1).cuda() 32 | self.sigma = torch.tensor(preprocessing['std']).float().view(3, 1, 1).cuda() 33 | 34 | self.classifier = nn.Sequential( 35 | nn.Linear(512, 4096), 36 | nn.ReLU(inplace=True), 37 | nn.Dropout(), 38 | nn.Linear(4096, 4096), 39 | nn.ReLU(inplace=True), 40 | nn.Dropout(), 41 | nn.Linear(4096, num_class) 42 | ) 43 | 44 | def forward(self, x): 45 | if not len(self.preprocessing) == 0: 46 | x = (x - self.mu) / self.sigma 47 | output = self.features(x) 48 | output = output.view(output.size()[0], -1) 49 | output = self.classifier(output) 50 | 51 | return output 52 | 53 | def make_layers(cfg, batch_norm=False): 54 | layers = [] 55 | 56 | input_channel = 3 57 | for l in cfg: 58 | if l == 'M': 59 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 60 | continue 61 | 62 | layers += [nn.Conv2d(input_channel, l, kernel_size=3, padding=1)] 63 | 64 | if batch_norm: 65 | layers += [nn.BatchNorm2d(l)] 66 | 67 | layers += [nn.ReLU(inplace=True)] 68 | input_channel = l 69 | 70 | return nn.Sequential(*layers) 71 | 72 | def vgg11_bn(num_class=100): 73 | return VGG(make_layers(cfg['A'], batch_norm=True), num_class) 74 | 75 | def vgg13_bn(num_class=100): 76 | return VGG(make_layers(cfg['B'], batch_norm=True), num_class) 77 | 78 | def vgg16_bn(num_class=100, preprocessing={}): 79 | return VGG(make_layers(cfg['D'], batch_norm=True), num_class, preprocessing=preprocessing) 80 | 81 | def vgg19_bn(num_class=100): 82 | return VGG(make_layers(cfg['E'], batch_norm=True), num_class) 83 | -------------------------------------------------------------------------------- /models/vgg_cif10.py: -------------------------------------------------------------------------------- 1 | '''VGG11/13/16/19 in Pytorch.''' 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | cfg = { 7 | 'VGG11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 8 | 'VGG13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 9 | 'VGG16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 10 | 'VGG19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 11 | } 12 | 13 | 14 | class VGG(nn.Module): 15 | def __init__(self, vgg_name, preprocessing={}): 16 | super(VGG, self).__init__() 17 | 18 | self.preprocessing = preprocessing 19 | 20 | if not len(self.preprocessing) == 0: 21 | self.mu = torch.tensor(preprocessing['mean']).float().view(3, 1, 1).cuda() 22 | self.sigma = torch.tensor(preprocessing['std']).float().view(3, 1, 1).cuda() 23 | 24 | self.features = self._make_layers(cfg[vgg_name]) 25 | self.classifier = nn.Linear(512, 10) 26 | 27 | def forward(self, x): 28 | if not len(self.preprocessing) == 0: 29 | x = (x - self.mu) / self.sigma 30 | out = self.features(x) 31 | out = out.view(out.size(0), -1) 32 | out = self.classifier(out) 33 | return out 34 | 35 | def _make_layers(self, cfg): 36 | layers = [] 37 | in_channels = 3 38 | for x in cfg: 39 | if x == 'M': 40 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 41 | else: 42 | layers += [nn.Conv2d(in_channels, x, kernel_size=3, padding=1), 43 | nn.BatchNorm2d(x), 44 | nn.ReLU(inplace=True)] 45 | in_channels = x 46 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 47 | return nn.Sequential(*layers) -------------------------------------------------------------------------------- /models/wideresidual.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.transforms as transforms 4 | 5 | 6 | class WideBasic(nn.Module): 7 | 8 | def __init__(self, in_channels, out_channels, stride=1): 9 | super().__init__() 10 | self.residual = nn.Sequential( 11 | nn.BatchNorm2d(in_channels), 12 | nn.ReLU(inplace=True), 13 | nn.Conv2d( 14 | in_channels, 15 | out_channels, 16 | kernel_size=3, 17 | stride=stride, 18 | padding=1 19 | ), 20 | nn.BatchNorm2d(out_channels), 21 | nn.ReLU(inplace=True), 22 | nn.Dropout(), 23 | nn.Conv2d( 24 | out_channels, 25 | out_channels, 26 | kernel_size=3, 27 | stride=1, 28 | padding=1 29 | ) 30 | ) 31 | 32 | self.shortcut = nn.Sequential() 33 | 34 | if in_channels != out_channels or stride != 1: 35 | self.shortcut = nn.Sequential( 36 | nn.Conv2d(in_channels, out_channels, 1, stride=stride) 37 | ) 38 | 39 | def forward(self, x): 40 | 41 | residual = self.residual(x) 42 | shortcut = self.shortcut(x) 43 | 44 | return residual + shortcut 45 | 46 | 47 | class WideResNet(nn.Module): 48 | def __init__(self, num_classes, block, depth=50, widen_factor=1, preprocessing={}): 49 | super().__init__() 50 | 51 | self.preprocessing = preprocessing 52 | if not len(preprocessing) == 0: 53 | print("mean: ", preprocessing['mean']) 54 | print("std : ", preprocessing['std']) 55 | 56 | self.mu = torch.tensor(preprocessing['mean']).float().view(3, 1, 1).cuda() 57 | self.sigma = torch.tensor(preprocessing['std'] ).float().view(3, 1, 1).cuda() 58 | 59 | # self.mu = torch.tensor([0.4914, 0.4822, 0.4465]).float().view(3, 1, 1).cuda() 60 | # self.sigma = torch.tensor([0.2023, 0.1994, 0.2010]).float().view(3, 1, 1).cuda() 61 | 62 | self.depth = depth 63 | k = widen_factor 64 | l = int((depth - 4) / 6) 65 | self.in_channels = 16 66 | self.init_conv = nn.Conv2d(3, self.in_channels, 3, 1, padding=1) 67 | self.conv2 = self._make_layer(block, 16 * k, l, 1) 68 | self.conv3 = self._make_layer(block, 32 * k, l, 2) 69 | self.conv4 = self._make_layer(block, 64 * k, l, 2) 70 | self.bn = nn.BatchNorm2d(64 * k) 71 | self.relu = nn.ReLU(inplace=True) 72 | self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) 73 | self.linear = nn.Linear(64 * k, num_classes) 74 | 75 | def forward(self, x): 76 | if not len(self.preprocessing) == 0: 77 | # print("normalize within the network!!") 78 | x = (x - self.mu) / self.sigma 79 | # torch.save(x, './data/normalization/imagenet32_gpu.pt') 80 | # import pdb; pdb.set_trace() 81 | # print('x shape', x.size()) 82 | x = self.init_conv(x) 83 | x = self.conv2(x) 84 | x = self.conv3(x) 85 | x = self.conv4(x) 86 | x = self.bn(x) 87 | x = self.relu(x) 88 | x = self.avg_pool(x) 89 | x = x.view(x.size(0), -1) 90 | x = self.linear(x) 91 | 92 | return x 93 | 94 | def _make_layer(self, block, out_channels, num_blocks, stride): 95 | """make resnet layers(by layer i didnt mean this 'layer' was the 96 | same as a neuron netowork layer, ex. conv layer), one layer may 97 | contain more than one residual block 98 | 99 | Args: 100 | block: block type, basic block or bottle neck block 101 | out_channels: output depth channel number of this layer 102 | num_blocks: how many blocks per layer 103 | stride: the stride of the first block of this layer 104 | 105 | Return: 106 | return a resnet layer 107 | """ 108 | 109 | # we have num_block blocks per layer, the first block 110 | # could be 1 or 2, other blocks would always be 1 111 | strides = [stride] + [1] * (num_blocks - 1) 112 | layers = [] 113 | for stride in strides: 114 | layers.append(block(self.in_channels, out_channels, stride)) 115 | self.in_channels = out_channels 116 | 117 | return nn.Sequential(*layers) 118 | 119 | 120 | # Table 9: Best WRN performance over various datasets, single run results. 121 | def wideresnet(num_classes=100, depth=40, widen_factor=10): 122 | net = WideResNet(num_classes, WideBasic, depth=depth, widen_factor=widen_factor) 123 | return net 124 | -------------------------------------------------------------------------------- /models/wideresidualorig.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | 6 | class WideBasic(nn.Module): 7 | 8 | def __init__(self, in_channels, out_channels, stride=1): 9 | super().__init__() 10 | self.residual = nn.Sequential( 11 | nn.BatchNorm2d(in_channels), 12 | nn.ReLU(inplace=True), 13 | nn.Conv2d( 14 | in_channels, 15 | out_channels, 16 | kernel_size=3, 17 | stride=stride, 18 | padding=1 19 | ), 20 | nn.BatchNorm2d(out_channels), 21 | nn.ReLU(inplace=True), 22 | nn.Dropout(), 23 | nn.Conv2d( 24 | out_channels, 25 | out_channels, 26 | kernel_size=3, 27 | stride=1, 28 | padding=1 29 | ) 30 | ) 31 | 32 | self.shortcut = nn.Sequential() 33 | 34 | if in_channels != out_channels or stride != 1: 35 | self.shortcut = nn.Sequential( 36 | nn.Conv2d(in_channels, out_channels, 1, stride=stride) 37 | ) 38 | 39 | def forward(self, x): 40 | 41 | residual = self.residual(x) 42 | shortcut = self.shortcut(x) 43 | 44 | return residual + shortcut 45 | 46 | class WideResNet(nn.Module): 47 | def __init__(self, num_classes, block, depth=50, widen_factor=1): 48 | super().__init__() 49 | 50 | # self.preprocessing = preprocessing 51 | # if not len(preprocessing) == 0: 52 | # self.mu = torch.tensor(preprocessing['mean']).float().view(3, 1, 1).cuda() 53 | # self.sigma = torch.tensor(preprocessing['std'] ).float().view(3, 1, 1).cuda() 54 | 55 | # self.mu = torch.tensor([0.4914, 0.4822, 0.4465]).float().view(3, 1, 1).cuda() 56 | # self.sigma = torch.tensor([0.2023, 0.1994, 0.2010]).float().view(3, 1, 1).cuda() 57 | 58 | self.depth = depth 59 | k = widen_factor 60 | l = int((depth - 4) / 6) 61 | self.in_channels = 16 62 | self.init_conv = nn.Conv2d(3, self.in_channels, 3, 1, padding=1) 63 | self.conv2 = self._make_layer(block, 16 * k, l, 1) 64 | self.conv3 = self._make_layer(block, 32 * k, l, 2) 65 | self.conv4 = self._make_layer(block, 64 * k, l, 2) 66 | self.bn = nn.BatchNorm2d(64 * k) 67 | self.relu = nn.ReLU(inplace=True) 68 | self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) 69 | self.linear = nn.Linear(64 * k, num_classes) 70 | 71 | def forward(self, x): 72 | # if not len(self.preprocessing) == 0: 73 | # x = (x - self.mu) / self.sigma 74 | x = self.init_conv(x) 75 | x = self.conv2(x) 76 | x = self.conv3(x) 77 | x = self.conv4(x) 78 | x = self.bn(x) 79 | x = self.relu(x) 80 | x = self.avg_pool(x) 81 | x = x.view(x.size(0), -1) 82 | x = self.linear(x) 83 | 84 | return x 85 | 86 | def _make_layer(self, block, out_channels, num_blocks, stride): 87 | """make resnet layers(by layer i didnt mean this 'layer' was the 88 | same as a neuron netowork layer, ex. conv layer), one layer may 89 | contain more than one residual block 90 | 91 | Args: 92 | block: block type, basic block or bottle neck block 93 | out_channels: output depth channel number of this layer 94 | num_blocks: how many blocks per layer 95 | stride: the stride of the first block of this layer 96 | 97 | Return: 98 | return a resnet layer 99 | """ 100 | 101 | # we have num_block blocks per layer, the first block 102 | # could be 1 or 2, other blocks would always be 1 103 | strides = [stride] + [1] * (num_blocks - 1) 104 | layers = [] 105 | for stride in strides: 106 | layers.append(block(self.in_channels, out_channels, stride)) 107 | self.in_channels = out_channels 108 | 109 | return nn.Sequential(*layers) 110 | 111 | 112 | # Table 9: Best WRN performance over various datasets, single run results. 113 | def wideresnet(num_classes=100, depth=40, widen_factor=10): 114 | net = WideResNet(num_classes, WideBasic, depth=depth, widen_factor=widen_factor) 115 | return net 116 | -------------------------------------------------------------------------------- /models/xception.py: -------------------------------------------------------------------------------- 1 | """xception in pytorch 2 | 3 | 4 | [1] François Chollet 5 | 6 | Xception: Deep Learning with Depthwise Separable Convolutions 7 | https://arxiv.org/abs/1610.02357 8 | """ 9 | 10 | import torch 11 | import torch.nn as nn 12 | 13 | class SeperableConv2d(nn.Module): 14 | 15 | #***Figure 4. An “extreme” version of our Inception module, 16 | #with one spatial convolution per output channel of the 1x1 17 | #convolution.""" 18 | def __init__(self, input_channels, output_channels, kernel_size, **kwargs): 19 | 20 | super().__init__() 21 | self.depthwise = nn.Conv2d( 22 | input_channels, 23 | input_channels, 24 | kernel_size, 25 | groups=input_channels, 26 | bias=False, 27 | **kwargs 28 | ) 29 | 30 | self.pointwise = nn.Conv2d(input_channels, output_channels, 1, bias=False) 31 | 32 | def forward(self, x): 33 | x = self.depthwise(x) 34 | x = self.pointwise(x) 35 | 36 | return x 37 | 38 | class EntryFlow(nn.Module): 39 | 40 | def __init__(self): 41 | 42 | super().__init__() 43 | self.conv1 = nn.Sequential( 44 | nn.Conv2d(3, 32, 3, padding=1, bias=False), 45 | nn.BatchNorm2d(32), 46 | nn.ReLU(inplace=True) 47 | ) 48 | 49 | self.conv2 = nn.Sequential( 50 | nn.Conv2d(32, 64, 3, padding=1, bias=False), 51 | nn.BatchNorm2d(64), 52 | nn.ReLU(inplace=True) 53 | ) 54 | 55 | self.conv3_residual = nn.Sequential( 56 | SeperableConv2d(64, 128, 3, padding=1), 57 | nn.BatchNorm2d(128), 58 | nn.ReLU(inplace=True), 59 | SeperableConv2d(128, 128, 3, padding=1), 60 | nn.BatchNorm2d(128), 61 | nn.MaxPool2d(3, stride=2, padding=1), 62 | ) 63 | 64 | self.conv3_shortcut = nn.Sequential( 65 | nn.Conv2d(64, 128, 1, stride=2), 66 | nn.BatchNorm2d(128), 67 | ) 68 | 69 | self.conv4_residual = nn.Sequential( 70 | nn.ReLU(inplace=True), 71 | SeperableConv2d(128, 256, 3, padding=1), 72 | nn.BatchNorm2d(256), 73 | nn.ReLU(inplace=True), 74 | SeperableConv2d(256, 256, 3, padding=1), 75 | nn.BatchNorm2d(256), 76 | nn.MaxPool2d(3, stride=2, padding=1) 77 | ) 78 | 79 | self.conv4_shortcut = nn.Sequential( 80 | nn.Conv2d(128, 256, 1, stride=2), 81 | nn.BatchNorm2d(256), 82 | ) 83 | 84 | #no downsampling 85 | self.conv5_residual = nn.Sequential( 86 | nn.ReLU(inplace=True), 87 | SeperableConv2d(256, 728, 3, padding=1), 88 | nn.BatchNorm2d(728), 89 | nn.ReLU(inplace=True), 90 | SeperableConv2d(728, 728, 3, padding=1), 91 | nn.BatchNorm2d(728), 92 | nn.MaxPool2d(3, 1, padding=1) 93 | ) 94 | 95 | #no downsampling 96 | self.conv5_shortcut = nn.Sequential( 97 | nn.Conv2d(256, 728, 1), 98 | nn.BatchNorm2d(728) 99 | ) 100 | 101 | def forward(self, x): 102 | x = self.conv1(x) 103 | x = self.conv2(x) 104 | residual = self.conv3_residual(x) 105 | shortcut = self.conv3_shortcut(x) 106 | x = residual + shortcut 107 | residual = self.conv4_residual(x) 108 | shortcut = self.conv4_shortcut(x) 109 | x = residual + shortcut 110 | residual = self.conv5_residual(x) 111 | shortcut = self.conv5_shortcut(x) 112 | x = residual + shortcut 113 | 114 | return x 115 | 116 | class MiddleFLowBlock(nn.Module): 117 | 118 | def __init__(self): 119 | super().__init__() 120 | 121 | self.shortcut = nn.Sequential() 122 | self.conv1 = nn.Sequential( 123 | nn.ReLU(inplace=True), 124 | SeperableConv2d(728, 728, 3, padding=1), 125 | nn.BatchNorm2d(728) 126 | ) 127 | self.conv2 = nn.Sequential( 128 | nn.ReLU(inplace=True), 129 | SeperableConv2d(728, 728, 3, padding=1), 130 | nn.BatchNorm2d(728) 131 | ) 132 | self.conv3 = nn.Sequential( 133 | nn.ReLU(inplace=True), 134 | SeperableConv2d(728, 728, 3, padding=1), 135 | nn.BatchNorm2d(728) 136 | ) 137 | 138 | def forward(self, x): 139 | residual = self.conv1(x) 140 | residual = self.conv2(residual) 141 | residual = self.conv3(residual) 142 | 143 | shortcut = self.shortcut(x) 144 | 145 | return shortcut + residual 146 | 147 | class MiddleFlow(nn.Module): 148 | def __init__(self, block): 149 | super().__init__() 150 | 151 | #"""then through the middle flow which is repeated eight times""" 152 | self.middel_block = self._make_flow(block, 8) 153 | 154 | def forward(self, x): 155 | x = self.middel_block(x) 156 | return x 157 | 158 | def _make_flow(self, block, times): 159 | flows = [] 160 | for i in range(times): 161 | flows.append(block()) 162 | 163 | return nn.Sequential(*flows) 164 | 165 | 166 | class ExitFLow(nn.Module): 167 | 168 | def __init__(self): 169 | super().__init__() 170 | self.residual = nn.Sequential( 171 | nn.ReLU(), 172 | SeperableConv2d(728, 728, 3, padding=1), 173 | nn.BatchNorm2d(728), 174 | nn.ReLU(), 175 | SeperableConv2d(728, 1024, 3, padding=1), 176 | nn.BatchNorm2d(1024), 177 | nn.MaxPool2d(3, stride=2, padding=1) 178 | ) 179 | 180 | self.shortcut = nn.Sequential( 181 | nn.Conv2d(728, 1024, 1, stride=2), 182 | nn.BatchNorm2d(1024) 183 | ) 184 | 185 | self.conv = nn.Sequential( 186 | SeperableConv2d(1024, 1536, 3, padding=1), 187 | nn.BatchNorm2d(1536), 188 | nn.ReLU(inplace=True), 189 | SeperableConv2d(1536, 2048, 3, padding=1), 190 | nn.BatchNorm2d(2048), 191 | nn.ReLU(inplace=True) 192 | ) 193 | 194 | self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) 195 | 196 | def forward(self, x): 197 | shortcut = self.shortcut(x) 198 | residual = self.residual(x) 199 | output = shortcut + residual 200 | output = self.conv(output) 201 | output = self.avgpool(output) 202 | 203 | return output 204 | 205 | class Xception(nn.Module): 206 | 207 | def __init__(self, block, num_class=100): 208 | super().__init__() 209 | self.entry_flow = EntryFlow() 210 | self.middel_flow = MiddleFlow(block) 211 | self.exit_flow = ExitFLow() 212 | 213 | self.fc = nn.Linear(2048, num_class) 214 | 215 | def forward(self, x): 216 | x = self.entry_flow(x) 217 | x = self.middel_flow(x) 218 | x = self.exit_flow(x) 219 | x = x.view(x.size(0), -1) 220 | x = self.fc(x) 221 | 222 | return x 223 | 224 | def xception(): 225 | return Xception(MiddleFLowBlock) 226 | 227 | 228 | -------------------------------------------------------------------------------- /pics/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/pics/.gitkeep -------------------------------------------------------------------------------- /pics/overview_readme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adverML/SpectralDef_Framework/bcad147381f7770b8fbb39e4a504a18126c3616a/pics/overview_readme.png -------------------------------------------------------------------------------- /requirements.yml: -------------------------------------------------------------------------------- 1 | name: cuda--11-1-1--pytorch--1-9-0 2 | channels: 3 | - pytorch 4 | - nvidia 5 | - conda-forge 6 | - defaults 7 | dependencies: 8 | - _libgcc_mutex=0.1=main 9 | - _openmp_mutex=4.5=1_gnu 10 | - anyio=3.3.4=py39hf3d152e_0 11 | - argon2-cffi=20.1.0=py39h27cfd23_1 12 | - astroid=2.6.6=py39h06a4308_0 13 | - async_generator=1.10=py_0 14 | - attrs=21.2.0=pyhd8ed1ab_0 15 | - babel=2.9.1=pyh44b312d_0 16 | - backcall=0.2.0=pyh9f0ad1d_0 17 | - backports=1.0=py_2 18 | - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 19 | - blas=1.0=mkl 20 | - bleach=4.1.0=pyhd8ed1ab_0 21 | - bottleneck=1.3.2=py39hdd57654_1 22 | - brotlipy=0.7.0=py39h3811e60_1001 23 | - bzip2=1.0.8=h7b6447c_0 24 | - ca-certificates=2021.10.26=h06a4308_2 25 | - certifi=2021.10.8=py39h06a4308_0 26 | - cffi=1.14.6=py39he32792d_0 27 | - chardet=4.0.0=py39hf3d152e_1 28 | - cryptography=35.0.0=py39hbca0aa6_0 29 | - cudatoolkit=11.1.1=h6406543_8 30 | - cycler=0.10.0=py_2 31 | - dbus=1.13.6=h48d8840_2 32 | - debugpy=1.4.1=py39he80948d_0 33 | - defusedxml=0.7.1=pyhd8ed1ab_0 34 | - entrypoints=0.3=pyhd8ed1ab_1003 35 | - expat=2.4.1=h9c3ff4c_0 36 | - faiss=1.7.1=py39cuda111he91d8c5_0_cuda 37 | - ffmpeg=4.2.2=h20bf706_0 38 | - fontconfig=2.13.1=hba837de_1005 39 | - freetype=2.10.4=h5ab3b9f_0 40 | - gettext=0.19.8.1=h0b5b191_1005 41 | - glib=2.68.3=h9c3ff4c_0 42 | - glib-tools=2.68.3=h9c3ff4c_0 43 | - gmp=6.2.1=h2531618_2 44 | - gnutls=3.6.15=he1e5248_0 45 | - gst-plugins-base=1.14.0=hbbd80ab_1 46 | - gstreamer=1.14.0=h28cd5cc_2 47 | - icu=58.2=hf484d3e_1000 48 | - importlib-metadata=4.8.1=py39hf3d152e_0 49 | - intel-openmp=2021.2.0=h06a4308_610 50 | - ipykernel=6.4.1=py39hef51801_0 51 | - ipython=7.28.0=py39hef51801_0 52 | - ipython_genutils=0.2.0=py_1 53 | - ipywidgets=7.6.5=pyhd8ed1ab_0 54 | - isort=5.9.3=pyhd3eb1b0_0 55 | - jedi=0.18.0=py39hf3d152e_2 56 | - jinja2=3.0.2=pyhd8ed1ab_0 57 | - joblib=1.0.1=pyhd3eb1b0_0 58 | - jpeg=9b=h024ee3a_2 59 | - json5=0.9.5=pyh9f0ad1d_0 60 | - jsonschema=4.1.0=pyhd8ed1ab_0 61 | - jupyter_client=7.0.6=pyhd8ed1ab_0 62 | - jupyter_core=4.8.1=py39hf3d152e_0 63 | - jupyter_server=1.11.1=pyhd8ed1ab_0 64 | - jupyterlab=3.2.0=pyhd8ed1ab_0 65 | - jupyterlab_pygments=0.1.2=pyh9f0ad1d_0 66 | - jupyterlab_server=2.8.2=pyhd8ed1ab_0 67 | - jupyterlab_widgets=1.0.2=pyhd8ed1ab_0 68 | - kiwisolver=1.3.1=py39h1a9c180_1 69 | - lame=3.100=h7b6447c_0 70 | - lazy-object-proxy=1.6.0=py39h27cfd23_0 71 | - lcms2=2.12=h3be6417_0 72 | - ld_impl_linux-64=2.35.1=h7274673_9 73 | - libblas=3.9.0=9_mkl 74 | - libfaiss=1.7.1=cuda111hf54f04a_0_cuda 75 | - libfaiss-avx2=1.7.1=cuda111h1234567_0_cuda 76 | - libffi=3.3=he6710b0_2 77 | - libgcc-ng=9.3.0=h5101ec6_17 78 | - libgfortran-ng=7.5.0=ha8ba4b0_17 79 | - libgfortran4=7.5.0=ha8ba4b0_17 80 | - libglib=2.68.3=h3e27bee_0 81 | - libgomp=9.3.0=h5101ec6_17 82 | - libiconv=1.16=h516909a_0 83 | - libidn2=2.3.1=h27cfd23_0 84 | - liblapack=3.9.0=9_mkl 85 | - libllvm11=11.1.0=h3826bc1_0 86 | - libopus=1.3.1=h7b6447c_0 87 | - libpng=1.6.37=hbc83047_0 88 | - libprotobuf=3.15.8=h780b84a_0 89 | - libsodium=1.0.18=h36c2ea0_1 90 | - libstdcxx-ng=9.3.0=hd4cf53a_17 91 | - libtasn1=4.16.0=h27cfd23_0 92 | - libtiff=4.2.0=h85742a9_0 93 | - libunistring=0.9.10=h27cfd23_0 94 | - libuuid=2.32.1=h7f98852_1000 95 | - libuv=1.40.0=h7b6447c_0 96 | - libvpx=1.7.0=h439df22_0 97 | - libwebp-base=1.2.0=h27cfd23_0 98 | - libxcb=1.13=h7f98852_1003 99 | - libxml2=2.9.12=h03d6c58_0 100 | - llvmlite=0.37.0=py39h295c915_1 101 | - lz4-c=1.9.3=h2531618_0 102 | - markupsafe=2.0.1=py39h3811e60_0 103 | - matplotlib=3.4.2=py39h06a4308_0 104 | - matplotlib-base=3.4.2=py39h2fa2bec_0 105 | - matplotlib-inline=0.1.3=pyhd8ed1ab_0 106 | - mccabe=0.6.1=py39h06a4308_1 107 | - mistune=0.8.4=py39h3811e60_1004 108 | - mkl=2021.2.0=h06a4308_296 109 | - mkl-service=2.3.0=py39h27cfd23_1 110 | - mkl_fft=1.3.0=py39h42c9631_2 111 | - mkl_random=1.2.1=py39ha9443f7_2 112 | - nbclassic=0.3.2=pyhd8ed1ab_0 113 | - nbclient=0.5.4=pyhd8ed1ab_0 114 | - nbconvert=6.2.0=py39hf3d152e_0 115 | - nbformat=5.1.3=pyhd8ed1ab_0 116 | - ncurses=6.2=he6710b0_1 117 | - nest-asyncio=1.5.1=pyhd8ed1ab_0 118 | - nettle=3.7.3=hbbd107a_1 119 | - ninja=1.10.2=hff7bd54_1 120 | - notebook=6.4.4=pyha770c72_0 121 | - numba=0.54.1=py39h51133e4_0 122 | - numexpr=2.7.3=py39h22e1b3c_1 123 | - numpy-base=1.20.2=py39hfae3a4d_0 124 | - olefile=0.46=py_0 125 | - openh264=2.1.0=hd408876_0 126 | - openjpeg=2.3.0=h05c96fa_1 127 | - openssl=1.1.1l=h7f8727e_0 128 | - packaging=21.0=pyhd8ed1ab_0 129 | - pandas=1.3.1=py39h8c16a72_0 130 | - pandoc=2.14.2=h7f98852_0 131 | - pandocfilters=1.5.0=pyhd8ed1ab_0 132 | - parso=0.8.2=pyhd8ed1ab_0 133 | - pcre=8.45=h9c3ff4c_0 134 | - pexpect=4.8.0=pyh9f0ad1d_2 135 | - pickleshare=0.7.5=py_1003 136 | - pillow=8.3.1=py39h2c7a002_0 137 | - pip=21.1.3=py39h06a4308_0 138 | - prometheus_client=0.11.0=pyhd8ed1ab_0 139 | - prompt-toolkit=3.0.20=pyha770c72_0 140 | - psutil=5.8.0=py39h3811e60_1 141 | - pthread-stubs=0.4=h36c2ea0_1001 142 | - ptyprocess=0.7.0=pyhd3deb0d_0 143 | - pycparser=2.20=pyh9f0ad1d_2 144 | - pygments=2.10.0=pyhd8ed1ab_0 145 | - pylint=2.9.6=py39h06a4308_1 146 | - pyopenssl=21.0.0=pyhd8ed1ab_0 147 | - pyparsing=2.4.7=pyh9f0ad1d_0 148 | - pyqt=5.9.2=py39h2531618_6 149 | - pyrsistent=0.17.3=py39h3811e60_2 150 | - pysocks=1.7.1=py39hf3d152e_3 151 | - python=3.9.5=h12debd9_4 152 | - python-dateutil=2.8.1=py_0 153 | - python_abi=3.9=2_cp39 154 | - pytorch=1.9.0=py3.9_cuda11.1_cudnn8.0.5_0 155 | - pytz=2021.1=pyhd3eb1b0_0 156 | - pyzmq=19.0.2=py39hb69f2a1_2 157 | - qt=5.9.7=h5867ecd_1 158 | - readline=8.1=h27cfd23_0 159 | - requests=2.26.0=pyhd8ed1ab_0 160 | - requests-unixsocket=0.2.0=py_0 161 | - scikit-learn=0.24.2=py39ha9443f7_0 162 | - scipy=1.6.2=py39had2a1c9_1 163 | - send2trash=1.8.0=pyhd8ed1ab_0 164 | - setuptools=52.0.0=py39h06a4308_0 165 | - sh=1.13.1=py39hf3d152e_2 166 | - sip=4.19.13=py39h2531618_0 167 | - sniffio=1.2.0=py39hf3d152e_1 168 | - sqlite=3.36.0=hc218d9a_0 169 | - tbb=2021.4.0=hd09550d_0 170 | - tensorboardx=2.4=pyhd8ed1ab_0 171 | - terminado=0.12.1=py39hf3d152e_0 172 | - testpath=0.5.0=pyhd8ed1ab_0 173 | - threadpoolctl=2.2.0=pyhb85f177_0 174 | - tk=8.6.10=hbc83047_0 175 | - toml=0.10.2=pyhd3eb1b0_0 176 | - tornado=6.1=py39h3811e60_1 177 | - tqdm=4.61.2=pyhd3eb1b0_1 178 | - traitlets=5.1.0=pyhd8ed1ab_0 179 | - typing_extensions=3.10.0.0=pyh06a4308_0 180 | - tzdata=2021a=h52ac0ba_0 181 | - wcwidth=0.2.5=pyh9f0ad1d_2 182 | - webencodings=0.5.1=py_1 183 | - websocket-client=0.57.0=py39hf3d152e_4 184 | - wheel=0.36.2=pyhd3eb1b0_0 185 | - widgetsnbextension=3.5.1=py39hf3d152e_4 186 | - wrapt=1.12.1=py39he8ac12f_1 187 | - x264=1!157.20191217=h7b6447c_0 188 | - xorg-libxau=1.0.9=h7f98852_0 189 | - xorg-libxdmcp=1.1.3=h7f98852_0 190 | - xz=5.2.5=h7b6447c_0 191 | - zeromq=4.3.4=h9c3ff4c_0 192 | - zipp=3.6.0=pyhd8ed1ab_0 193 | - zlib=1.2.11=h7b6447c_3 194 | - zstd=1.4.9=haebb681_0 195 | - pip: 196 | - absl-py==0.13.0 197 | - astunparse==1.6.3 198 | - autograd==1.3 199 | - cachetools==4.2.2 200 | - charset-normalizer==2.0.4 201 | - cleverhans==3.0.1 202 | - cloudpickle==1.6.0 203 | - decorator==5.0.9 204 | - dm-tree==0.1.6 205 | - eagerpy==0.29.0 206 | - easydict==1.9 207 | - falconn==1.3.1 208 | - filelock==3.2.0 209 | - flatbuffers==1.12 210 | - foolbox==3.3.1 211 | - freqopttest==0.1.0 212 | - future==0.18.2 213 | - gast==0.4.0 214 | - gdown==3.14.0 215 | - gitdb==4.0.7 216 | - gitpython==3.1.18 217 | - google-auth==1.34.0 218 | - google-auth-oauthlib==0.4.5 219 | - google-pasta==0.2.0 220 | - gputil==1.4.0 221 | - grpcio==1.34.1 222 | - h5py==3.1.0 223 | - idna==3.2 224 | - keras-nightly==2.5.0.dev2021032900 225 | - keras-preprocessing==1.1.2 226 | - markdown==3.3.4 227 | - mnist==0.2.2 228 | - nose==1.3.7 229 | - numpy==1.19.5 230 | - oauthlib==3.1.1 231 | - opt-einsum==3.3.0 232 | - progress==1.5 233 | - protobuf==3.17.3 234 | - pyasn1==0.4.8 235 | - pyasn1-modules==0.2.8 236 | - pycodestyle==2.7.0 237 | - requests-oauthlib==1.3.0 238 | - rsa==4.7.2 239 | - six==1.15.0 240 | - smmap==4.0.0 241 | - tensorboard==2.5.0 242 | - tensorboard-data-server==0.6.1 243 | - tensorboard-plugin-wit==1.8.0 244 | - tensorflow-estimator==2.5.0 245 | - tensorflow-probability==0.13.0 246 | - termcolor==1.1.0 247 | - theano==1.0.5 248 | - torchvision==0.10.0 249 | - typing-extensions==3.7.4.3 250 | - urllib3==1.26.6 251 | - werkzeug==2.0.1 252 | prefix: /home/lorenzp/.conda/envs/cuda--11-1-1--pytorch--1-9-0 253 | -------------------------------------------------------------------------------- /train_imagenet.py: -------------------------------------------------------------------------------- 1 | 2 | '''Train CIFAR10 with PyTorch.''' 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | import torch.nn.functional as F 7 | import torch.backends.cudnn as cudnn 8 | 9 | import torchvision 10 | import torchvision.transforms as transforms 11 | import torchvision.datasets as datasets 12 | 13 | import os 14 | import argparse 15 | 16 | # from models.vgg_cif10 import VGG 17 | 18 | from models.wideresidual import WideResNet, WideBasic 19 | 20 | from utils import progress_bar 21 | 22 | import pdb 23 | 24 | print("Train ImageNet ") 25 | 26 | 27 | parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') 28 | parser.add_argument('--lr', default=0.1, type=float, help='learning rate') 29 | parser.add_argument('--data', default='/home/lorenzp/datasets/ImageNet', help='path to dataset') 30 | parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint') 31 | 32 | parser.add_argument('-b', '--batch-size', default=16, type=int, 33 | metavar='N', 34 | help='mini-batch size (default: 256), this is the total ' 35 | 'batch size of all GPUs on the current node when ' 36 | 'using Data Parallel or Distributed Data Parallel') 37 | 38 | args = parser.parse_args() 39 | 40 | print("args", args) 41 | 42 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 43 | best_acc = 0 # best test accuracy 44 | start_epoch = 0 # start from epoch 0 or last checkpoint epoch 45 | 46 | # Data 47 | print('==> Preparing data..') 48 | 49 | train_sampler = None 50 | 51 | # Data loading code 52 | traindir = os.path.join(args.data, 'train') 53 | valdir = os.path.join(args.data, 'val') 54 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 55 | std=[0.229, 0.224, 0.225]) 56 | 57 | transform_train = transforms.Compose([ 58 | transforms.RandomResizedCrop(224), 59 | transforms.RandomHorizontalFlip(), 60 | transforms.ToTensor(), 61 | # normalize, 62 | ]) 63 | 64 | train_dataset = datasets.ImageFolder( 65 | traindir, 66 | transform_train 67 | ) 68 | trainloader = torch.utils.data.DataLoader( 69 | train_dataset, batch_size=args.batch_size, shuffle=True, 70 | num_workers=4, pin_memory=True, sampler=train_sampler) 71 | 72 | # trainset = torchvision.datasets.ImageNet( 73 | # root=args.data, train=True, download=False, transform=transform_train) 74 | 75 | # train_loader = torch.utils.data.DataLoader( 76 | # trainset, batch_size=128, shuffle=True, num_workers=4) 77 | 78 | 79 | val_transform = transforms.Compose([ 80 | # transforms.Resize(256), 81 | transforms.CenterCrop(224), 82 | transforms.ToTensor(), 83 | # normalize, 84 | ]) 85 | 86 | valloader = torch.utils.data.DataLoader( 87 | datasets.ImageFolder(valdir, val_transform), 88 | batch_size=args.batch_size, shuffle=False, 89 | num_workers=4) 90 | 91 | # testset = torchvision.datasets.ImageNet( 92 | # root=args.data, train=False, download=False, transform=test_transform) 93 | 94 | # test_loader = torch.utils.data.DataLoader( 95 | # testset, batch_size=128, shuffle=True, num_workers=4) 96 | 97 | 98 | 99 | # transform_train = transforms.Compose([ 100 | # transforms.RandomCrop(32, padding=4), 101 | # transforms.RandomHorizontalFlip(), 102 | # transforms.ToTensor(), 103 | # # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 104 | # ]) 105 | 106 | # transform_test = transforms.Compose([ 107 | # transforms.ToTensor(), 108 | # # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 109 | # ]) 110 | 111 | # trainset = torchvision.datasets.CIFAR10( 112 | # root='./data', train=True, download=True, transform=transform_train) 113 | # trainloader = torch.utils.data.DataLoader( 114 | # trainset, batch_size=128, shuffle=True, num_workers=2) 115 | 116 | # testset = torchvision.datasets.CIFAR10( 117 | # root='./data', train=False, download=True, transform=transform_test) 118 | # testloader = torch.utils.data.DataLoader( 119 | # testset, batch_size=100, shuffle=False, num_workers=2) 120 | 121 | # classes = ('plane', 'car', 'bird', 'cat', 'deer', 122 | # 'dog', 'frog', 'horse', 'ship', 'truck') 123 | 124 | # Model 125 | print('==> Building model..') 126 | 127 | # net = VGG('VGG16') 128 | 129 | depth=34 130 | widen_factor=10 131 | print("depth: ", depth, ", widen_factor", widen_factor) 132 | net = WideResNet(num_classes=1000, block=WideBasic, depth=depth, widen_factor=widen_factor) 133 | 134 | 135 | net = net.to(device) 136 | if device == 'cuda': 137 | net = torch.nn.DataParallel(net) 138 | cudnn.benchmark = True 139 | 140 | if args.resume: 141 | # Load checkpoint. 142 | print('==> Resuming from checkpoint..') 143 | assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!' 144 | checkpoint = torch.load('./checkpoint/ckpt.pth') 145 | net.load_state_dict(checkpoint['net']) 146 | best_acc = checkpoint['acc'] 147 | start_epoch = checkpoint['epoch'] 148 | 149 | criterion = nn.CrossEntropyLoss() 150 | optimizer = optim.SGD(net.parameters(), lr=args.lr, 151 | momentum=0.9, weight_decay=5e-4) 152 | 153 | 154 | # Training 155 | def train(epoch): 156 | print('\nEpoch: %d' % epoch) 157 | net.train() 158 | train_loss = 0 159 | correct = 0 160 | total = 0 161 | 162 | for batch_idx, (inputs, targets) in enumerate(trainloader): 163 | inputs, targets = inputs.to(device), targets.to(device) 164 | optimizer.zero_grad() 165 | outputs = net(inputs) 166 | 167 | loss = criterion(outputs, targets) 168 | loss.backward() 169 | optimizer.step() 170 | 171 | train_loss += loss.item() 172 | _, predicted = outputs.max(1) 173 | total += targets.size(0) 174 | correct += predicted.eq(targets).sum().item() 175 | 176 | if batch_idx 100 == 0: 177 | progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' 178 | % (train_loss/(batch_idx+1), 100.*correct/total, correct, total)) 179 | 180 | def test(epoch): 181 | global best_acc 182 | net.eval() 183 | test_loss = 0 184 | correct = 0 185 | total = 0 186 | 187 | with torch.no_grad(): 188 | for batch_idx, (inputs, targets) in enumerate(testloader): 189 | inputs, targets = inputs.to(device), targets.to(device) 190 | outputs = net(inputs) 191 | loss = criterion(outputs, targets) 192 | 193 | test_loss += loss.item() 194 | _, predicted = outputs.max(1) 195 | total += targets.size(0) 196 | correct += predicted.eq(targets).sum().item() 197 | 198 | if batch_idx % 100 == 0: 199 | progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' 200 | % (test_loss/(batch_idx+1), 100.*correct/total, correct, total)) 201 | 202 | 203 | # Save checkpoint. 204 | acc = 100.*correct/total 205 | if acc > best_acc: 206 | print('Saving..') 207 | state = { 208 | 'net': net.state_dict(), 209 | 'acc': acc, 210 | 'epoch': epoch, 211 | } 212 | 213 | 214 | path = 'checkpoint/wideresnet_' + str(depth) + str(widen_factor) 215 | if not os.path.isdir(path): 216 | os.mkdir(path) 217 | torch.save(state, './' + path + '/wide_resnet_imagenet_ckpt.pth') 218 | best_acc = acc 219 | 220 | 221 | for epoch in range(start_epoch, start_epoch+50): 222 | train(epoch) 223 | test(epoch) 224 | --------------------------------------------------------------------------------