├── .gitignore ├── README.md ├── attack.py ├── loaders.py ├── models ├── __init__.py ├── cifar │ ├── LICENSE │ ├── README.md │ ├── __init__.py │ ├── alexnet.py │ ├── gdas │ │ ├── LICENSE │ │ ├── README.md │ │ ├── __init__.py │ │ ├── configs │ │ │ ├── NAS-PTB-BASE.config │ │ │ ├── NAS-WT2-BASE.config │ │ │ ├── cos1800.config │ │ │ ├── cos600.config │ │ │ ├── nas-cifar-cos-cut.config │ │ │ ├── nas-cifar-cos-cutB128.config │ │ │ ├── nas-cifar-cos-cutB64.config │ │ │ ├── nas-cifar-cos-cutB96.config │ │ │ ├── nas-cifar-cos-cutW1.config │ │ │ ├── nas-cifar-cos-cutW3.config │ │ │ ├── nas-cifar-cos-cutW5.config │ │ │ ├── nas-cifar-cos-nocut.config │ │ │ ├── nas-imagenet-B128.config │ │ │ ├── nas-imagenet-B256.config │ │ │ ├── nas-imagenet.config │ │ │ ├── pyramidC10.config │ │ │ ├── pyramidC100.config │ │ │ ├── resnet165.config │ │ │ └── resnet200.config │ │ └── lib │ │ │ ├── __init__.py │ │ │ ├── datasets │ │ │ ├── LanguageDataset.py │ │ │ ├── MetaBatchSampler.py │ │ │ ├── TieredImageNet.py │ │ │ ├── __init__.py │ │ │ ├── get_dataset_with_transform.py │ │ │ ├── test_NLP.py │ │ │ └── test_dataset.py │ │ │ ├── nas │ │ │ ├── CifarNet.py │ │ │ ├── ImageNet.py │ │ │ ├── SE_Module.py │ │ │ ├── __init__.py │ │ │ ├── construct_utils.py │ │ │ ├── genotypes.py │ │ │ ├── head_utils.py │ │ │ ├── model_search.py │ │ │ └── operations.py │ │ │ ├── nas_rnn │ │ │ ├── __init__.py │ │ │ ├── basemodel.py │ │ │ ├── genotypes.py │ │ │ ├── model_search.py │ │ │ └── utils.py │ │ │ ├── scheduler │ │ │ ├── __init__.py │ │ │ ├── scheduler.py │ │ │ └── utils.py │ │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── draw_pts.py │ │ │ ├── evaluation_utils.py │ │ │ ├── flop_benchmark.py │ │ │ ├── gpu_manager.py │ │ │ ├── model_utils.py │ │ │ ├── save_meta.py │ │ │ └── utils.py │ ├── pyramidnet.py │ ├── utils.py │ ├── vgg.py │ └── wrn.py └── imagenet │ ├── LICENSE.txt │ ├── README.md │ ├── __init__.py │ ├── pnasnet.py │ ├── resnet.py │ ├── senet.py │ ├── torchvision_models.py │ ├── utils.py │ └── vgg.py └── subspace_attack_poster.pptx /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | wheels/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | MANIFEST 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | .pytest_cache/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | db.sqlite3 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # Environments 84 | .env 85 | .venv 86 | env/ 87 | venv/ 88 | ENV/ 89 | env.bak/ 90 | venv.bak/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | 105 | # input & output 106 | data/ 107 | output/ 108 | 109 | # PyCharm 110 | .idea/ 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # subspace-attack.pytorch 2 | Code for our NeurIPS 2019 paper [Subspace Attack: Exploiting Promising Subspaces for Query-Efficient Black-box Attacks](https://papers.nips.cc/paper/8638-subspace-attack-exploiting-promising-subspaces-for-query-efficient-black-box-attacks). 3 | 4 | Our paper is also available on [arxiv](https://arxiv.org/pdf/1906.04392.pdf). 5 | 6 | ## Environments 7 | * Python 3.5 8 | * PyTorch 1.1.0 9 | * torchvision 0.2.2 10 | * glog 0.3.1 11 | 12 | ## Datasets and Reference Models 13 | We use CIFAR-10 and ImageNet in our experiment, and these two datasets should be prepared into the following structure: 14 | 15 | ``` 16 | subspace-attack.pytorch 17 | └───data 18 | ├── cifar10 19 | │   ├── cifar-10-batches-py 20 | │   └── cifar-10-python.tar.gz 21 | └── imagenet 22 | ├── train 23 | └── val 24 | ``` 25 | 26 | For reference models, please download at [this link](https://drive.google.com/file/d/1aXTmN2AyNLdZ8zOeyLzpVbRHZRZD0fW0/view?usp=sharing), extract and put them into ```data/``` directory. 27 | 28 | ## Usage 29 | To mount an untargeted l-inf attack on a WRN victim model on CIFAR-10 using AlexNet+VGGNets as references (i.e., the CIFAR-10->WRN->Ours row in Table 1 of our paper) 30 | 31 | ``` 32 | python3 attack.py --arch wrn-28-10-drop --attack-type untargeted --norm-type linf --dataset cifar10 --ref-arch alexnet_bn vgg11_bn vgg13_bn vgg16_bn vgg19_bn --ref-arch-train-data cifar10.1 33 | ``` 34 | 35 | We also provide many logs for experiments used in our paper at [this link](https://drive.google.com/file/d/1hNpWgmlUGjEfFpNcPq1ULfyYHf6RkoM-/view?usp=sharing). 36 | 37 | ## Acknowledgements 38 | The following resources are very helpful for our work: 39 | 40 | * [Pretrained models and for ImageNet](https://github.com/Cadene/pretrained-models.pytorch) 41 | * [Pretrained models for CIFAR-10](https://github.com/bearpaw/pytorch-classification) 42 | * [GDAS](https://github.com/D-X-Y/GDAS) 43 | * [Official AutoAugment implementation](https://github.com/tensorflow/models/tree/master/research/autoaugment) 44 | * [ImageNet FGSM adversarially trained Inception-V3 model](https://github.com/tensorflow/models/tree/master/research/adv_imagenet_models) 45 | * [Carlini's CIFAR-10 ConvNet](https://github.com/carlini/nn_robust_attacks) 46 | 47 | ## Citation 48 | Please cite our work in your publications if it helps your research: 49 | 50 | ``` 51 | @inproceedings{subspaceattack, 52 | title={Subspace Attack: Exploiting Promising Subspaces for Query-Efficient Black-box Attacks}, 53 | author={Guo, Yiwen and Yan, Ziang and Zhang, Changshui}, 54 | booktitle={Advances in Neural Information Processing Systems}, 55 | pages={3820--3829}, 56 | year={2019} 57 | } 58 | ``` 59 | -------------------------------------------------------------------------------- /loaders.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import sys 4 | import pickle 5 | from PIL import Image 6 | from glob import glob 7 | import numpy as np 8 | import torch 9 | from torchvision import datasets, transforms 10 | from torchvision.datasets.utils import download_url, check_integrity 11 | 12 | 13 | class ImageNetIDDataset(torch.utils.data.Dataset): 14 | def __init__(self, root_dirname, phase, seed, size=224): 15 | super(ImageNetIDDataset, self).__init__() 16 | assert phase in ['train', 'val'] 17 | 18 | # get 1000 classes 19 | image_dirname = osp.join(root_dirname, phase) # e.g., data/imagenet/val 20 | classes = [d for d in os.listdir(image_dirname) if os.path.isdir(os.path.join(image_dirname, d))] 21 | classes.sort() 22 | self.class_to_idx = {classes[i]: i for i in range(len(classes))} 23 | assert len(self.class_to_idx) == 1000 24 | 25 | # get all images 26 | self.images_fname = glob('{}/*/*.JPEG'.format(image_dirname)) 27 | self.images_fname.sort() 28 | if phase == 'train': 29 | assert len(self.images_fname) == 1281167 30 | elif phase == 'val': 31 | assert len(self.images_fname) == 50000 32 | else: 33 | raise NotImplementedError('Unknown phase {} for imagenet support phases are: train/val'.format(phase)) 34 | 35 | # record the index of each image in the whole dataset, since we may select a subset later 36 | self.images_id = np.arange(len(self.images_fname)) 37 | 38 | # fix random seed 39 | # save previous RNG state 40 | state = np.random.get_state() 41 | # fix random seed, thus we have the same random status at each time 42 | np.random.seed(seed) 43 | perm = np.random.permutation(len(self.images_fname)) 44 | self.images_fname = list(np.array(self.images_fname)[perm]) 45 | self.images_id = list(np.array(self.images_id)[perm]) 46 | # restore previous RNG state for training && test 47 | np.random.set_state(state) 48 | 49 | # transform 50 | self.transform = transforms.Compose([transforms.Resize(int(size / 0.875)), 51 | transforms.CenterCrop(size), 52 | transforms.ToTensor()]) 53 | 54 | def __getitem__(self, index): 55 | # we always emit data in [0, 1] range to keep things simpler (normalization is performed in the main script). 56 | # image_id is the identifier of current image in the whole dataset 57 | # index is the index of current image in self.images_fname 58 | image_id = self.images_id[index] 59 | image_fname = self.images_fname[index] 60 | label = self.class_to_idx[image_fname.split('/')[-2]] 61 | with open(image_fname, 'rb') as f: 62 | with Image.open(f) as image: 63 | image = image.convert('RGB') 64 | 65 | # get standard format: 224x224, 0-1 range, RGB channel order 66 | image = self.transform(image) # [3, 224, 224] 67 | 68 | return image_id, image, label 69 | 70 | def __len__(self): 71 | return len(self.images_fname) 72 | 73 | 74 | class CIFAR10IDDataset(torch.utils.data.Dataset): 75 | # Adopted from torchvision 76 | base_folder = 'cifar-10-batches-py' 77 | url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 78 | filename = "cifar-10-python.tar.gz" 79 | tgz_md5 = 'c58f30108f718f92721af3b95e74349a' 80 | train_list = [ 81 | ['data_batch_1', 'c99cafc152244af753f735de768cd75f'], 82 | ['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'], 83 | ['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'], 84 | ['data_batch_4', '634d18415352ddfa80567beed471001a'], 85 | ['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'], 86 | ] 87 | 88 | test_list = [ 89 | ['test_batch', '40351d587109b95175f43aff81a1287e'], 90 | ] 91 | 92 | def __init__(self, root_dirname, phase, seed): 93 | self.root_dirname = os.path.expanduser(root_dirname) 94 | self.phase = phase # training set or test set 95 | assert self.phase in ['train', 'test'] 96 | 97 | # we load CIFAR-10 dataset into standard format (without any data augmentation) 98 | self.transform = transforms.ToTensor() 99 | 100 | # download if not exists, and check integrity 101 | self.download() 102 | if not self._check_integrity(): 103 | raise RuntimeError('Dataset not found or corrupted.' + 104 | ' You can use download=True to download it') 105 | 106 | # now load the picked numpy arrays 107 | if self.phase == 'train': 108 | self.images = [] 109 | self.labels = [] 110 | for fentry in self.train_list: 111 | f = fentry[0] 112 | file = os.path.join(self.root_dirname, self.base_folder, f) 113 | fo = open(file, 'rb') 114 | if sys.version_info[0] == 2: 115 | entry = pickle.load(fo) 116 | else: 117 | entry = pickle.load(fo, encoding='latin1') 118 | self.images.append(entry['data']) 119 | if 'labels' in entry: 120 | self.labels += entry['labels'] 121 | else: 122 | self.labels += entry['fine_labels'] 123 | fo.close() 124 | 125 | self.images = np.concatenate(self.images) 126 | self.images = self.images.reshape((50000, 3, 32, 32)) 127 | self.images = self.images.transpose((0, 2, 3, 1)) # convert to HWC 128 | else: 129 | f = self.test_list[0][0] 130 | file = os.path.join(self.root_dirname, self.base_folder, f) 131 | fo = open(file, 'rb') 132 | if sys.version_info[0] == 2: 133 | entry = pickle.load(fo) 134 | else: 135 | entry = pickle.load(fo, encoding='latin1') 136 | self.images = entry['data'] 137 | if 'labels' in entry: 138 | self.labels = entry['labels'] 139 | else: 140 | self.labels = entry['fine_labels'] 141 | fo.close() 142 | self.images = self.images.reshape((10000, 3, 32, 32)) 143 | self.images = self.images.transpose((0, 2, 3, 1)) # convert to HWC 144 | 145 | # record the index of each image in the whole dataset, since we may select a subset later 146 | self.images_id = np.arange(len(self.images)) 147 | 148 | # fix random seed 149 | # save previous RNG state 150 | state = np.random.get_state() 151 | # fix random seed, thus we have the same random status at each time 152 | np.random.seed(seed) 153 | perm = np.random.permutation(len(self.images)) 154 | self.images_id = list(np.array(self.images_id)[perm]) 155 | self.images = list(np.array(self.images)[perm]) 156 | self.labels = list(np.array(self.labels)[perm]) 157 | # restore previous RNG state for training && test 158 | np.random.set_state(state) 159 | 160 | def __getitem__(self, index): 161 | image_id, image, label = self.images_id[index], self.images[index], self.labels[index] 162 | 163 | # doing this so that it is consistent with all other datasets 164 | # to return a PIL Image 165 | image = Image.fromarray(image) 166 | 167 | image = self.transform(image) 168 | 169 | return image_id, image, label 170 | 171 | def __len__(self): 172 | return len(self.images) 173 | 174 | def _check_integrity(self): 175 | root = self.root_dirname 176 | for fentry in (self.train_list + self.test_list): 177 | filename, md5 = fentry[0], fentry[1] 178 | fpath = os.path.join(root, self.base_folder, filename) 179 | if not check_integrity(fpath, md5): 180 | return False 181 | return True 182 | 183 | def download(self): 184 | import tarfile 185 | 186 | if self._check_integrity(): 187 | print('Files already downloaded and verified') 188 | return 189 | 190 | root_dirname = self.root_dirname 191 | download_url(self.url, root_dirname, self.filename, self.tgz_md5) 192 | 193 | # extract file 194 | cwd = os.getcwd() 195 | tar = tarfile.open(os.path.join(root_dirname, self.filename), "r:gz") 196 | os.chdir(root_dirname) 197 | tar.extractall() 198 | tar.close() 199 | os.chdir(cwd) 200 | 201 | 202 | def make_loader(dataset, phase, batch_size=1, seed=0, **kwargs): 203 | """ 204 | Make loader. To make sure we use the same data for evaluation, 205 | these loaders will return (image_id, image, label) tuple instead of vanilla (image, label) tuple. 206 | :param dataset: mnist, cifar10 or imagenet. 207 | :param phase: train, val or test. 208 | :param batch_size: batch size. For imagenet we usually set batch size to 1. 209 | :param seed: random seed in selecting images. 210 | :param kwargs: for imagenet, kwargs could contain size (e.g., 224 or 299) 211 | :return: pytorch DataLoader object, could be used as iterator. 212 | """ 213 | 214 | if dataset == 'imagenet': 215 | assert phase in ['train', 'val'] 216 | loader = torch.utils.data.DataLoader(ImageNetIDDataset('data/imagenet', phase, seed, **kwargs), 217 | batch_size=batch_size, num_workers=0) 218 | loader.num_class = 1000 219 | 220 | elif dataset == 'cifar10': 221 | assert phase in ['train', 'test'] 222 | loader = torch.utils.data.DataLoader(CIFAR10IDDataset('data/cifar10', phase, seed), 223 | batch_size=batch_size, num_workers=0) 224 | loader.num_class = 10 225 | else: 226 | raise NotImplementedError('Unknown dataset {}, we only support cifar10 and imagenet now'.format(dataset)) 227 | 228 | return loader 229 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | import glog as log 2 | import os.path as osp 3 | import h5py 4 | import numpy as np 5 | import torch 6 | import models.cifar 7 | import models.imagenet 8 | 9 | 10 | def load_weight_from_pth_checkpoint(model, fname): 11 | log.info('Load weights from {}'.format(fname)) 12 | raw_state_dict = torch.load(fname, map_location='cpu')['state_dict'] 13 | state_dict = dict() 14 | for key, val in raw_state_dict.items(): 15 | new_key = key.replace('module.', '') 16 | state_dict[new_key] = val 17 | 18 | model.load_state_dict(state_dict) 19 | 20 | 21 | def make_model(dataset, arch, **kwargs): 22 | """ 23 | Make model, and load pre-trained weights. 24 | :param dataset: cifar10 or imagenet 25 | :param arch: arch name, e.g., alexnet_bn 26 | :return: model (in cpu and training mode) 27 | """ 28 | assert dataset in ['cifar10', 'imagenet'] 29 | if dataset == 'cifar10': 30 | if arch == 'gdas': 31 | assert kwargs['train_data'] == 'full' 32 | model = models.cifar.gdas('data/cifar10-models/gdas/seed-6293/checkpoint-cifar10-model.pth') 33 | model.mean = [125.3 / 255, 123.0 / 255, 113.9 / 255] 34 | model.std = [63.0 / 255, 62.1 / 255, 66.7 / 255] 35 | model.input_space = 'RGB' 36 | model.input_range = [0, 1] 37 | model.input_size = [3, 32, 32] 38 | elif arch == 'pyramidnet272': 39 | assert kwargs['train_data'] == 'full' 40 | model = models.cifar.pyramidnet272(num_classes=10) 41 | load_weight_from_pth_checkpoint(model, 'data/cifar10-models/pyramidnet272/checkpoint.pth') 42 | model.mean = [0.49139968, 0.48215841, 0.44653091] 43 | model.std = [0.24703223, 0.24348513, 0.26158784] 44 | model.input_space = 'RGB' 45 | model.input_range = [0, 1] 46 | model.input_size = [3, 32, 32] 47 | else: 48 | # decide weight filename prefix, suffix 49 | if kwargs['train_data'] in ['cifar10.1']: 50 | # use cifar10.1 (2,000 images) to train models 51 | if kwargs['train_data'] == 'cifar10.1': 52 | prefix = 'data/cifar10.1-models' 53 | else: 54 | raise NotImplementedError('Unknown train data {}'.format(kwargs['train_data'])) 55 | if kwargs['epoch'] == 'final': 56 | suffix = 'final.pth' 57 | elif kwargs['epoch'] == 'best': 58 | suffix = 'model_best.pth' 59 | else: 60 | raise NotImplementedError('Unknown epoch {} for train data {}'.format( 61 | kwargs['epoch'], kwargs['train_data'])) 62 | elif kwargs['train_data'] == 'full': 63 | # use full training set to train models 64 | prefix = 'data/cifar10-models' 65 | if kwargs['epoch'] == 'final': 66 | suffix = 'checkpoint.pth.tar' 67 | elif kwargs['epoch'] == 'best': 68 | suffix = 'model_best.pth.tar' 69 | else: 70 | raise NotImplementedError('Unknown epoch {} for train data {}'.format( 71 | kwargs['epoch'], kwargs['train_data'])) 72 | else: 73 | raise NotImplementedError('Unknown train data {}'.format(kwargs['train_data'])) 74 | 75 | if arch == 'alexnet_bn': 76 | model = models.cifar.alexnet_bn(num_classes=10) 77 | elif arch == 'vgg11_bn': 78 | model = models.cifar.vgg11_bn(num_classes=10) 79 | elif arch == 'vgg13_bn': 80 | model = models.cifar.vgg13_bn(num_classes=10) 81 | elif arch == 'vgg16_bn': 82 | model = models.cifar.vgg16_bn(num_classes=10) 83 | elif arch == 'vgg19_bn': 84 | model = models.cifar.vgg19_bn(num_classes=10) 85 | elif arch == 'wrn-28-10-drop': 86 | model = models.cifar.wrn(depth=28, widen_factor=10, dropRate=0.3, num_classes=10) 87 | else: 88 | raise NotImplementedError('Unknown arch {}'.format(arch)) 89 | 90 | # load weight 91 | load_weight_from_pth_checkpoint(model, osp.join(prefix, arch, suffix)) 92 | 93 | # assign meta info 94 | model.mean = [0.4914, 0.4822, 0.4465] 95 | model.std = [0.2023, 0.1994, 0.2010] 96 | model.input_space = 'RGB' 97 | model.input_range = [0, 1] 98 | model.input_size = [3, 32, 32] 99 | 100 | elif dataset == 'imagenet': 101 | 102 | model = eval('models.imagenet.{}(num_classes=1000, pretrained=\'imagenet\')'.format(arch)) 103 | 104 | if kwargs['train_data'] == 'full': 105 | # torchvision has load correct checkpoint automatically 106 | pass 107 | elif kwargs['train_data'] == 'imagenetv2-val': 108 | prefix = 'data/imagenetv2-v1val45000-models' 109 | if kwargs['epoch'] == 'final': 110 | suffix = 'checkpoint.pth.tar' 111 | elif kwargs['epoch'] == 'best': 112 | suffix = 'model_best.pth.tar' 113 | else: 114 | raise NotImplementedError('Unknown epoch {} for train data {}'.format( 115 | kwargs['epoch'], kwargs['train_data'])) 116 | 117 | # load weight 118 | load_weight_from_pth_checkpoint(model, osp.join(prefix, arch, suffix)) 119 | else: 120 | raise NotImplementedError('Unknown train data {}'.format(kwargs['train_data'])) 121 | else: 122 | raise NotImplementedError('Unknown dataset {}'.format(dataset)) 123 | 124 | return model 125 | -------------------------------------------------------------------------------- /models/cifar/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Wei Yang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /models/cifar/README.md: -------------------------------------------------------------------------------- 1 | # CIFAR-10 CNN models 2 | This folder contains a slightly modified version of the [pytorch-classification](https://github.com/bearpaw/pytorch-classification) project code (Copyright to [Wei Yang](https://github.com/bearpaw)). 3 | -------------------------------------------------------------------------------- /models/cifar/__init__.py: -------------------------------------------------------------------------------- 1 | """The models subpackage contains definitions for the following model for CIFAR10/CIFAR100 2 | architectures: 3 | 4 | - `AlexNet`_ 5 | - `VGG`_ 6 | - `ResNet`_ 7 | - `SqueezeNet`_ 8 | - `DenseNet`_ 9 | 10 | You can construct a model with random weights by calling its constructor: 11 | 12 | .. code:: python 13 | 14 | import torchvision.models as models 15 | resnet18 = models.resnet18() 16 | alexnet = models.alexnet() 17 | squeezenet = models.squeezenet1_0() 18 | densenet = models.densenet_161() 19 | 20 | We provide pre-trained models for the ResNet variants and AlexNet, using the 21 | PyTorch :mod:`torch.utils.model_zoo`. These can constructed by passing 22 | ``pretrained=True``: 23 | 24 | .. code:: python 25 | 26 | import torchvision.models as models 27 | resnet18 = models.resnet18(pretrained=True) 28 | alexnet = models.alexnet(pretrained=True) 29 | 30 | ImageNet 1-crop error rates (224x224) 31 | 32 | ======================== ============= ============= 33 | Network Top-1 error Top-5 error 34 | ======================== ============= ============= 35 | ResNet-18 30.24 10.92 36 | ResNet-34 26.70 8.58 37 | ResNet-50 23.85 7.13 38 | ResNet-101 22.63 6.44 39 | ResNet-152 21.69 5.94 40 | Inception v3 22.55 6.44 41 | AlexNet 43.45 20.91 42 | VGG-11 30.98 11.37 43 | VGG-13 30.07 10.75 44 | VGG-16 28.41 9.62 45 | VGG-19 27.62 9.12 46 | SqueezeNet 1.0 41.90 19.58 47 | SqueezeNet 1.1 41.81 19.38 48 | Densenet-121 25.35 7.83 49 | Densenet-169 24.00 7.00 50 | Densenet-201 22.80 6.43 51 | Densenet-161 22.35 6.20 52 | ======================== ============= ============= 53 | 54 | 55 | .. _AlexNet: https://arxiv.org/abs/1404.5997 56 | .. _VGG: https://arxiv.org/abs/1409.1556 57 | .. _ResNet: https://arxiv.org/abs/1512.03385 58 | .. _SqueezeNet: https://arxiv.org/abs/1602.07360 59 | .. _DenseNet: https://arxiv.org/abs/1608.06993 60 | """ 61 | 62 | from models.cifar.alexnet import * 63 | from models.cifar.vgg import * 64 | from models.cifar.wrn import * 65 | from models.cifar.gdas import * 66 | from models.cifar.pyramidnet import * 67 | -------------------------------------------------------------------------------- /models/cifar/alexnet.py: -------------------------------------------------------------------------------- 1 | '''AlexNet for CIFAR10. FC layers are removed. Paddings are adjusted. 2 | Without BN, the start learning rate should be 0.01 3 | (c) YANG, Wei 4 | ''' 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from .utils import DropoutConv2d 8 | 9 | __all__ = ['alexnet', 'alexnet_bn'] 10 | 11 | 12 | class AlexNet(nn.Module): 13 | 14 | def __init__(self, batch_norm=False, num_classes=10): 15 | super(AlexNet, self).__init__() 16 | layers = [ 17 | DropoutConv2d(3, 64, kernel_size=11, stride=4, padding=5) 18 | ] 19 | if batch_norm: 20 | layers += [nn.BatchNorm2d(64)] 21 | layers += [ 22 | nn.ReLU(inplace=True), 23 | nn.MaxPool2d(kernel_size=2, stride=2), 24 | DropoutConv2d(64, 192, kernel_size=5, padding=2), 25 | ] 26 | if batch_norm: 27 | layers += [nn.BatchNorm2d(192)] 28 | layers += [ 29 | nn.ReLU(inplace=True), 30 | nn.MaxPool2d(kernel_size=2, stride=2), 31 | DropoutConv2d(192, 384, kernel_size=3, padding=1), 32 | ] 33 | if batch_norm: 34 | layers += [nn.BatchNorm2d(384)] 35 | layers += [ 36 | nn.ReLU(inplace=True), 37 | DropoutConv2d(384, 256, kernel_size=3, padding=1), 38 | ] 39 | if batch_norm: 40 | layers += [nn.BatchNorm2d(256)] 41 | layers += [ 42 | nn.ReLU(inplace=True), 43 | DropoutConv2d(256, 256, kernel_size=3, padding=1), 44 | ] 45 | if batch_norm: 46 | layers += [nn.BatchNorm2d(256)] 47 | layers += [ 48 | nn.ReLU(inplace=True), 49 | nn.MaxPool2d(kernel_size=2, stride=2), 50 | ] 51 | self.features = nn.Sequential(*layers) 52 | self.classifier = nn.Linear(256, num_classes) 53 | 54 | # no dropout 55 | self.drop = 0 56 | 57 | def forward(self, x): 58 | for m in self.modules(): 59 | if isinstance(m, DropoutConv2d): 60 | m.drop = self.drop 61 | 62 | x = self.features(x) 63 | x = x.view(x.size(0), -1) 64 | x = F.dropout(x, p=self.drop, training=True) # force dropout 65 | x = self.classifier(x) 66 | return x 67 | 68 | 69 | def alexnet(**kwargs): 70 | r"""AlexNet model architecture from the 71 | `"One weird trick..." `_ paper. 72 | """ 73 | model = AlexNet(batch_norm=False, **kwargs) 74 | return model 75 | 76 | 77 | def alexnet_bn(**kwargs): 78 | r"""AlexNet model architecture from the 79 | `"One weird trick..." `_ paper. 80 | """ 81 | model = AlexNet(batch_norm=True, **kwargs) 82 | return model 83 | -------------------------------------------------------------------------------- /models/cifar/gdas/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Xuanyi Dong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /models/cifar/gdas/README.md: -------------------------------------------------------------------------------- 1 | # GDAS searched CNN architecture 2 | This folder contains a slightly modified version of the [GDAS](https://github.com/D-X-Y/GDAS) project code (Copyright to [Xuanyi Dong](https://github.com/D-X-Y)). 3 | 4 | I discard the architecture searching process but keep the network definition scripts unchanged, since only the searched architecture is relevant to our subspace based black box attack project. 5 | -------------------------------------------------------------------------------- /models/cifar/gdas/__init__.py: -------------------------------------------------------------------------------- 1 | '''GDAS net. 2 | Ported form 3 | https://github.com/D-X-Y/GDAS 4 | (c) Yuanyi Dong 5 | ''' 6 | import os 7 | import os.path as osp 8 | import torch 9 | 10 | from models.cifar.gdas.lib.scheduler import load_config 11 | from models.cifar.gdas.lib.scheduler import load_config 12 | from models.cifar.gdas.lib.nas import model_types 13 | from models.cifar.gdas.lib.nas import NetworkCIFAR as Network 14 | 15 | __all__ = ['gdas'] 16 | 17 | 18 | def gdas(checkpoint_fname): 19 | checkpoint = torch.load(checkpoint_fname, map_location='cpu') 20 | xargs = checkpoint['args'] 21 | config = load_config(os.path.join(osp.dirname(__file__), xargs.model_config)) 22 | genotype = model_types[xargs.arch] 23 | class_num = 10 24 | 25 | model = Network(xargs.init_channels, class_num, xargs.layers, config.auxiliary, genotype) 26 | model.load_state_dict(checkpoint['state_dict']) 27 | return model 28 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/NAS-PTB-BASE.config: -------------------------------------------------------------------------------- 1 | { 2 | "data_name" : ["str", "PTB"], 3 | "data_path" : ["str", "./data/data/penn"], 4 | "emsize" : ["int", 850], 5 | "nhid" : ["int", 850], 6 | "nhidlast" : ["int", 850], 7 | "LR" : ["float", 20], 8 | "clip" : ["float", 0.25], 9 | "epochs" : ["int", 3000], 10 | "train_batch": ["int", 64], 11 | "eval_batch": ["int", 10], 12 | "test_batch": ["int", 1], 13 | "bptt" : ["int", 35], 14 | 15 | "dropout" : ["float", 0.75], 16 | "dropouth" : ["float", 0.25], 17 | "dropoutx" : ["float", 0.75], 18 | "dropouti" : ["float", 0.2], 19 | "dropoute" : ["float", 0.1], 20 | 21 | "nonmono" : ["int", 5], 22 | "alpha" : ["float", 0], 23 | "beta" : ["float", 1e-3], 24 | "wdecay" : ["float", 8e-7], 25 | 26 | "max_seq_len_delta" : ["int", 20] 27 | } 28 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/NAS-WT2-BASE.config: -------------------------------------------------------------------------------- 1 | { 2 | "data_name" : ["str", "WT2"], 3 | "data_path" : ["str", "./data/data/wikitext-2"], 4 | "emsize" : ["int", 700], 5 | "nhid" : ["int", 700], 6 | "nhidlast" : ["int", 700], 7 | "LR" : ["float", 20], 8 | "clip" : ["float", 0.25], 9 | "epochs" : ["int", 3000], 10 | "train_batch": ["int", 64], 11 | "eval_batch": ["int", 10], 12 | "test_batch": ["int", 1], 13 | "bptt" : ["int", 35], 14 | 15 | "dropout" : ["float", 0.75], 16 | "dropouth" : ["float", 0.15], 17 | "dropoutx" : ["float", 0.75], 18 | "dropouti" : ["float", 0.2], 19 | "dropoute" : ["float", 0.1], 20 | 21 | "nonmono" : ["int", 5], 22 | "alpha" : ["float", 0], 23 | "beta" : ["float", 1e-3], 24 | "wdecay" : ["float", 5e-7], 25 | 26 | "max_seq_len_delta" : ["int", 20] 27 | } 28 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/cos1800.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 1800], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.2] 8 | } 9 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/cos600.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0005], 7 | "LR" : ["float", 0.2] 8 | } 9 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cut.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cutB128.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cutB64.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 64], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cutB96.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cutW1.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cutW3.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-cutW5.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0005], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-cifar-cos-nocut.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 0], 13 | "drop_path_prob" : ["float", 0.3] 14 | } 15 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-imagenet-B128.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "steplr"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 250], 5 | "decay_period": ["int", 1], 6 | "gamma" : ["float", 0.97], 7 | "momentum" : ["float", 0.9], 8 | "decay" : ["float", 0.00003], 9 | "LR" : ["float", 0.1], 10 | "label_smooth": ["float", 0.1], 11 | "auxiliary" : ["bool", 1], 12 | "auxiliary_weight" : ["float", 0.4], 13 | "grad_clip" : ["float", 5], 14 | "drop_path_prob" : ["float", 0] 15 | } 16 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-imagenet-B256.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "steplr"], 3 | "batch_size": ["int", 256], 4 | "epochs" : ["int", 250], 5 | "decay_period": ["int", 1], 6 | "gamma" : ["float", 0.97], 7 | "momentum" : ["float", 0.9], 8 | "decay" : ["float", 0.00003], 9 | "LR" : ["float", 0.1], 10 | "label_smooth": ["float", 0.1], 11 | "auxiliary" : ["bool", 1], 12 | "auxiliary_weight" : ["float", 0.4], 13 | "grad_clip" : ["float", 5], 14 | "drop_path_prob" : ["float", 0] 15 | } 16 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/nas-imagenet.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "steplr"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 250], 5 | "decay_period": ["int", 1], 6 | "gamma" : ["float", 0.97], 7 | "momentum" : ["float", 0.9], 8 | "decay" : ["float", 0.00003], 9 | "LR" : ["float", 0.1], 10 | "label_smooth": ["float", 0.1], 11 | "auxiliary" : ["bool", 1], 12 | "auxiliary_weight" : ["float", 0.4], 13 | "grad_clip" : ["float", 5], 14 | "drop_path_prob" : ["float", 0] 15 | } 16 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/pyramidC10.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 300], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.1], 8 | "milestones": ["int", [150, 225]], 9 | "gammas" : ["float", [0.1, 0.1]] 10 | } 11 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/pyramidC100.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 300], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.5], 8 | "milestones": ["int", [150, 225]], 9 | "gammas" : ["float", [0.1, 0.1]] 10 | } 11 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/resnet165.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 165], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.01], 8 | "milestones": ["int", [1, 83, 124]], 9 | "gammas" : ["float", [10, 0.1, 0.1]] 10 | } 11 | -------------------------------------------------------------------------------- /models/cifar/gdas/configs/resnet200.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 200], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0005], 7 | "LR" : ["float", 0.01], 8 | "milestones": ["int", [1 , 60, 120, 160]], 9 | "gammas" : ["float", [10, 0.2, 0.2, 0.2]] 10 | } 11 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZiangYan/subspace-attack.pytorch/96d323e43ad2718767713cfbf1172147ebecf645/models/cifar/gdas/lib/__init__.py -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/LanguageDataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | from collections import Counter 5 | 6 | 7 | class Dictionary(object): 8 | def __init__(self): 9 | self.word2idx = {} 10 | self.idx2word = [] 11 | self.counter = Counter() 12 | self.total = 0 13 | 14 | def add_word(self, word): 15 | if word not in self.word2idx: 16 | self.idx2word.append(word) 17 | self.word2idx[word] = len(self.idx2word) - 1 18 | token_id = self.word2idx[word] 19 | self.counter[token_id] += 1 20 | self.total += 1 21 | return self.word2idx[word] 22 | 23 | def __len__(self): 24 | return len(self.idx2word) 25 | 26 | 27 | class Corpus(object): 28 | def __init__(self, path): 29 | self.dictionary = Dictionary() 30 | self.train = self.tokenize(os.path.join(path, 'train.txt')) 31 | self.valid = self.tokenize(os.path.join(path, 'valid.txt')) 32 | self.test = self.tokenize(os.path.join(path, 'test.txt')) 33 | 34 | def tokenize(self, path): 35 | """Tokenizes a text file.""" 36 | assert os.path.exists(path) 37 | # Add words to the dictionary 38 | with open(path, 'r', encoding='utf-8') as f: 39 | tokens = 0 40 | for line in f: 41 | words = line.split() + [''] 42 | tokens += len(words) 43 | for word in words: 44 | self.dictionary.add_word(word) 45 | 46 | # Tokenize file content 47 | with open(path, 'r', encoding='utf-8') as f: 48 | ids = torch.LongTensor(tokens) 49 | token = 0 50 | for line in f: 51 | words = line.split() + [''] 52 | for word in words: 53 | ids[token] = self.dictionary.word2idx[word] 54 | token += 1 55 | 56 | return ids 57 | 58 | class SentCorpus(object): 59 | def __init__(self, path): 60 | self.dictionary = Dictionary() 61 | self.train = self.tokenize(os.path.join(path, 'train.txt')) 62 | self.valid = self.tokenize(os.path.join(path, 'valid.txt')) 63 | self.test = self.tokenize(os.path.join(path, 'test.txt')) 64 | 65 | def tokenize(self, path): 66 | """Tokenizes a text file.""" 67 | assert os.path.exists(path) 68 | # Add words to the dictionary 69 | with open(path, 'r', encoding='utf-8') as f: 70 | tokens = 0 71 | for line in f: 72 | words = line.split() + [''] 73 | tokens += len(words) 74 | for word in words: 75 | self.dictionary.add_word(word) 76 | 77 | # Tokenize file content 78 | sents = [] 79 | with open(path, 'r', encoding='utf-8') as f: 80 | for line in f: 81 | if not line: 82 | continue 83 | words = line.split() + [''] 84 | sent = torch.LongTensor(len(words)) 85 | for i, word in enumerate(words): 86 | sent[i] = self.dictionary.word2idx[word] 87 | sents.append(sent) 88 | 89 | return sents 90 | 91 | class BatchSentLoader(object): 92 | def __init__(self, sents, batch_size, pad_id=0, cuda=False, volatile=False): 93 | self.sents = sents 94 | self.batch_size = batch_size 95 | self.sort_sents = sorted(sents, key=lambda x: x.size(0)) 96 | self.cuda = cuda 97 | self.volatile = volatile 98 | self.pad_id = pad_id 99 | 100 | def __next__(self): 101 | if self.idx >= len(self.sort_sents): 102 | raise StopIteration 103 | 104 | batch_size = min(self.batch_size, len(self.sort_sents)-self.idx) 105 | batch = self.sort_sents[self.idx:self.idx+batch_size] 106 | max_len = max([s.size(0) for s in batch]) 107 | tensor = torch.LongTensor(max_len, batch_size).fill_(self.pad_id) 108 | for i in range(len(batch)): 109 | s = batch[i] 110 | tensor[:s.size(0),i].copy_(s) 111 | if self.cuda: 112 | tensor = tensor.cuda() 113 | 114 | self.idx += batch_size 115 | 116 | return tensor 117 | 118 | next = __next__ 119 | 120 | def __iter__(self): 121 | self.idx = 0 122 | return self 123 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/MetaBatchSampler.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import numpy as np 3 | import torch 4 | 5 | 6 | class MetaBatchSampler(object): 7 | 8 | def __init__(self, labels, classes_per_it, num_samples, iterations): 9 | ''' 10 | Initialize MetaBatchSampler 11 | Args: 12 | - labels: an iterable containing all the labels for the current dataset 13 | samples indexes will be infered from this iterable. 14 | - classes_per_it: number of random classes for each iteration 15 | - num_samples: number of samples for each iteration for each class (support + query) 16 | - iterations: number of iterations (episodes) per epoch 17 | ''' 18 | super(MetaBatchSampler, self).__init__() 19 | self.labels = labels.copy() 20 | self.classes_per_it = classes_per_it 21 | self.sample_per_class = num_samples 22 | self.iterations = iterations 23 | 24 | self.classes, self.counts = np.unique(self.labels, return_counts=True) 25 | assert len(self.classes) == np.max(self.classes) + 1 and np.min(self.classes) == 0 26 | assert classes_per_it < len(self.classes), '{:} vs. {:}'.format(classes_per_it, len(self.classes)) 27 | self.classes = torch.LongTensor(self.classes) 28 | 29 | # create a matrix, indexes, of dim: classes X max(elements per class) 30 | # fill it with nans 31 | # for every class c, fill the relative row with the indices samples belonging to c 32 | # in numel_per_class we store the number of samples for each class/row 33 | self.indexes = { x.item() : [] for x in self.classes } 34 | indexes = { x.item() : [] for x in self.classes } 35 | 36 | for idx, label in enumerate(self.labels): 37 | indexes[ label.item() ].append( idx ) 38 | for key, value in indexes.items(): 39 | self.indexes[ key ] = torch.LongTensor( value ) 40 | 41 | 42 | def __iter__(self): 43 | # yield a batch of indexes 44 | spc = self.sample_per_class 45 | cpi = self.classes_per_it 46 | 47 | for it in range(self.iterations): 48 | batch_size = spc * cpi 49 | batch = torch.LongTensor(batch_size) 50 | assert cpi < len(self.classes), '{:} vs. {:}'.format(cpi, len(self.classes)) 51 | c_idxs = torch.randperm(len(self.classes))[:cpi] 52 | 53 | for i, cls in enumerate(self.classes[c_idxs]): 54 | s = slice(i * spc, (i + 1) * spc) 55 | num = self.indexes[ cls.item() ].nelement() 56 | assert spc < num, '{:} vs. {:}'.format(spc, num) 57 | sample_idxs = torch.randperm( num )[:spc] 58 | batch[s] = self.indexes[ cls.item() ][sample_idxs] 59 | 60 | batch = batch[torch.randperm(len(batch))] 61 | yield batch 62 | 63 | def __len__(self): 64 | # returns the number of iterations (episodes) per epoch 65 | return self.iterations 66 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/TieredImageNet.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from PIL import Image 4 | import pickle as pkl 5 | import os, cv2, csv, glob 6 | import torch 7 | import torch.utils.data as data 8 | 9 | 10 | class TieredImageNet(data.Dataset): 11 | 12 | def __init__(self, root_dir, split, transform=None): 13 | self.split = split 14 | self.root_dir = root_dir 15 | self.transform = transform 16 | splits = split.split('-') 17 | 18 | images, labels, last = [], [], 0 19 | for split in splits: 20 | labels_name = '{:}/{:}_labels.pkl'.format(self.root_dir, split) 21 | images_name = '{:}/{:}_images.npz'.format(self.root_dir, split) 22 | # decompress images if npz not exits 23 | if not os.path.exists(images_name): 24 | png_pkl = images_name[:-4] + '_png.pkl' 25 | if os.path.exists(png_pkl): 26 | decompress(images_name, png_pkl) 27 | else: 28 | raise ValueError('png_pkl {:} not exits'.format( png_pkl )) 29 | assert os.path.exists(images_name) and os.path.exists(labels_name), '{:} & {:}'.format(images_name, labels_name) 30 | print ("Prepare {:} done".format(images_name)) 31 | try: 32 | with open(labels_name) as f: 33 | data = pkl.load(f) 34 | label_specific = data["label_specific"] 35 | except: 36 | with open(labels_name, 'rb') as f: 37 | data = pkl.load(f, encoding='bytes') 38 | label_specific = data[b'label_specific'] 39 | with np.load(images_name, mmap_mode="r", encoding='latin1') as data: 40 | image_data = data["images"] 41 | images.append( image_data ) 42 | label_specific = label_specific + last 43 | labels.append( label_specific ) 44 | last = np.max(label_specific) + 1 45 | print ("Load {:} done, with image shape = {:}, label shape = {:}, [{:} ~ {:}]".format(images_name, image_data.shape, label_specific.shape, np.min(label_specific), np.max(label_specific))) 46 | images, labels = np.concatenate(images), np.concatenate(labels) 47 | 48 | self.images = images 49 | self.labels = labels 50 | self.n_classes = int( np.max(labels) + 1 ) 51 | self.dict_index_label = {} 52 | for cls in range(self.n_classes): 53 | idxs = np.where(labels==cls)[0] 54 | self.dict_index_label[cls] = idxs 55 | self.length = len(labels) 56 | print ("There are {:} images, {:} labels [{:} ~ {:}]".format(images.shape, labels.shape, np.min(labels), np.max(labels))) 57 | 58 | 59 | def __repr__(self): 60 | return ('{name}(length={length}, classes={n_classes})'.format(name=self.__class__.__name__, **self.__dict__)) 61 | 62 | def __len__(self): 63 | return self.length 64 | 65 | def __getitem__(self, index): 66 | assert index >= 0 and index < self.length, 'invalid index = {:}'.format(index) 67 | image = self.images[index].copy() 68 | label = int(self.labels[index]) 69 | image = Image.fromarray(image[:,:,::-1].astype('uint8'), 'RGB') 70 | if self.transform is not None: 71 | image = self.transform( image ) 72 | return image, label 73 | 74 | 75 | 76 | 77 | def decompress(path, output): 78 | with open(output, 'rb') as f: 79 | array = pkl.load(f, encoding='bytes') 80 | images = np.zeros([len(array), 84, 84, 3], dtype=np.uint8) 81 | for ii, item in enumerate(array): 82 | im = cv2.imdecode(item, 1) 83 | images[ii] = im 84 | np.savez(path, images=images) 85 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .MetaBatchSampler import MetaBatchSampler 5 | from .TieredImageNet import TieredImageNet 6 | from .LanguageDataset import Corpus 7 | from .get_dataset_with_transform import get_datasets 8 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/get_dataset_with_transform.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, torch 5 | import os.path as osp 6 | import torchvision.datasets as dset 7 | import torch.backends.cudnn as cudnn 8 | import torchvision.transforms as transforms 9 | 10 | from ..utils import Cutout 11 | from .TieredImageNet import TieredImageNet 12 | 13 | 14 | Dataset2Class = {'cifar10' : 10, 15 | 'cifar100': 100, 16 | 'tiered' : -1, 17 | 'imagenet-1k' : 1000, 18 | 'imagenet-100': 100} 19 | 20 | 21 | def get_datasets(name, root, cutout): 22 | 23 | # Mean + Std 24 | if name == 'cifar10': 25 | mean = [x / 255 for x in [125.3, 123.0, 113.9]] 26 | std = [x / 255 for x in [63.0, 62.1, 66.7]] 27 | elif name == 'cifar100': 28 | mean = [x / 255 for x in [129.3, 124.1, 112.4]] 29 | std = [x / 255 for x in [68.2, 65.4, 70.4]] 30 | elif name == 'tiered': 31 | mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] 32 | elif name == 'imagenet-1k' or name == 'imagenet-100': 33 | mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] 34 | else: raise TypeError("Unknow dataset : {:}".format(name)) 35 | 36 | 37 | # Data Argumentation 38 | if name == 'cifar10' or name == 'cifar100': 39 | lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), 40 | transforms.Normalize(mean, std)] 41 | if cutout > 0 : lists += [Cutout(cutout)] 42 | train_transform = transforms.Compose(lists) 43 | test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)]) 44 | elif name == 'tiered': 45 | lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(80, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std)] 46 | if cutout > 0 : lists += [Cutout(cutout)] 47 | train_transform = transforms.Compose(lists) 48 | test_transform = transforms.Compose([transforms.CenterCrop(80), transforms.ToTensor(), transforms.Normalize(mean, std)]) 49 | elif name == 'imagenet-1k' or name == 'imagenet-100': 50 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) 51 | train_transform = transforms.Compose([ 52 | transforms.RandomResizedCrop(224), 53 | transforms.RandomHorizontalFlip(), 54 | transforms.ColorJitter( 55 | brightness=0.4, 56 | contrast=0.4, 57 | saturation=0.4, 58 | hue=0.2), 59 | transforms.ToTensor(), 60 | normalize, 61 | ]) 62 | test_transform = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize]) 63 | else: raise TypeError("Unknow dataset : {:}".format(name)) 64 | 65 | if name == 'cifar10': 66 | train_data = dset.CIFAR10 (root, train=True , transform=train_transform, download=True) 67 | test_data = dset.CIFAR10 (root, train=False, transform=test_transform , download=True) 68 | elif name == 'cifar100': 69 | train_data = dset.CIFAR100(root, train=True , transform=train_transform, download=True) 70 | test_data = dset.CIFAR100(root, train=False, transform=test_transform , download=True) 71 | elif name == 'imagenet-1k' or name == 'imagenet-100': 72 | train_data = dset.ImageFolder(osp.join(root, 'train'), train_transform) 73 | test_data = dset.ImageFolder(osp.join(root, 'val'), test_transform) 74 | else: raise TypeError("Unknow dataset : {:}".format(name)) 75 | 76 | class_num = Dataset2Class[name] 77 | return train_data, test_data, class_num 78 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/test_NLP.py: -------------------------------------------------------------------------------- 1 | import os, sys, torch 2 | 3 | from .LanguageDataset import SentCorpus, BatchSentLoader 4 | 5 | if __name__ == '__main__': 6 | path = '../../data/data/penn' 7 | corpus = SentCorpus( path ) 8 | loader = BatchSentLoader(corpus.test, 10) 9 | for i, d in enumerate(loader): 10 | print('{:} :: {:}'.format(i, d.size())) 11 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/datasets/test_dataset.py: -------------------------------------------------------------------------------- 1 | import os, sys, torch 2 | import torchvision.transforms as transforms 3 | 4 | from .TieredImageNet import TieredImageNet 5 | from .MetaBatchSampler import MetaBatchSampler 6 | 7 | root_dir = os.environ['TORCH_HOME'] + '/tiered-imagenet' 8 | print ('root : {:}'.format(root_dir)) 9 | means, stds = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] 10 | 11 | lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(84, padding=8), transforms.ToTensor(), transforms.Normalize(means, stds)] 12 | transform = transforms.Compose(lists) 13 | 14 | dataset = TieredImageNet(root_dir, 'val-test', transform) 15 | image, label = dataset[111] 16 | print ('image shape = {:}, label = {:}'.format(image.size(), label)) 17 | print ('image : min = {:}, max = {:} ||| label : {:}'.format(image.min(), image.max(), label)) 18 | 19 | 20 | sampler = MetaBatchSampler(dataset.labels, 250, 100, 10) 21 | 22 | dataloader = torch.utils.data.DataLoader(dataset, batch_sampler=sampler) 23 | 24 | print ('the length of dataset : {:}'.format( len(dataset) )) 25 | print ('the length of loader : {:}'.format( len(dataloader) )) 26 | 27 | for images, labels in dataloader: 28 | print ('images : {:}'.format( images.size() )) 29 | print ('labels : {:}'.format( labels.size() )) 30 | for i in range(3): 31 | print ('image-value-[{:}] : {:} ~ {:}, mean={:}, std={:}'.format(i, images[:,i].min(), images[:,i].max(), images[:,i].mean(), images[:,i].std())) 32 | 33 | print('-----') 34 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/CifarNet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .construct_utils import Cell, Transition 4 | 5 | class AuxiliaryHeadCIFAR(nn.Module): 6 | 7 | def __init__(self, C, num_classes): 8 | """assuming input size 8x8""" 9 | super(AuxiliaryHeadCIFAR, self).__init__() 10 | self.features = nn.Sequential( 11 | nn.ReLU(inplace=True), 12 | nn.AvgPool2d(5, stride=3, padding=0, count_include_pad=False), # image size = 2 x 2 13 | nn.Conv2d(C, 128, 1, bias=False), 14 | nn.BatchNorm2d(128), 15 | nn.ReLU(inplace=True), 16 | nn.Conv2d(128, 768, 2, bias=False), 17 | nn.BatchNorm2d(768), 18 | nn.ReLU(inplace=True) 19 | ) 20 | self.classifier = nn.Linear(768, num_classes) 21 | 22 | def forward(self, x): 23 | x = self.features(x) 24 | x = self.classifier(x.view(x.size(0),-1)) 25 | return x 26 | 27 | 28 | class NetworkCIFAR(nn.Module): 29 | 30 | def __init__(self, C, num_classes, layers, auxiliary, genotype): 31 | super(NetworkCIFAR, self).__init__() 32 | self._layers = layers 33 | 34 | stem_multiplier = 3 35 | C_curr = stem_multiplier*C 36 | self.stem = nn.Sequential( 37 | nn.Conv2d(3, C_curr, 3, padding=1, bias=False), 38 | nn.BatchNorm2d(C_curr) 39 | ) 40 | 41 | C_prev_prev, C_prev, C_curr = C_curr, C_curr, C 42 | self.cells = nn.ModuleList() 43 | reduction_prev = False 44 | for i in range(layers): 45 | if i in [layers//3, 2*layers//3]: 46 | C_curr *= 2 47 | reduction = True 48 | else: 49 | reduction = False 50 | if reduction and genotype.reduce is None: 51 | cell = Transition(C_prev_prev, C_prev, C_curr, reduction_prev) 52 | else: 53 | cell = Cell(genotype, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) 54 | reduction_prev = reduction 55 | self.cells.append( cell ) 56 | C_prev_prev, C_prev = C_prev, cell.multiplier*C_curr 57 | if i == 2*layers//3: 58 | C_to_auxiliary = C_prev 59 | 60 | if auxiliary: 61 | self.auxiliary_head = AuxiliaryHeadCIFAR(C_to_auxiliary, num_classes) 62 | else: 63 | self.auxiliary_head = None 64 | self.global_pooling = nn.AdaptiveAvgPool2d(1) 65 | self.classifier = nn.Linear(C_prev, num_classes) 66 | self.drop_path_prob = -1 67 | 68 | def update_drop_path(self, drop_path_prob): 69 | self.drop_path_prob = drop_path_prob 70 | 71 | def auxiliary_param(self): 72 | if self.auxiliary_head is None: return [] 73 | else: return list( self.auxiliary_head.parameters() ) 74 | 75 | def forward(self, inputs): 76 | s0 = s1 = self.stem(inputs) 77 | for i, cell in enumerate(self.cells): 78 | s0, s1 = s1, cell(s0, s1, self.drop_path_prob) 79 | if i == 2*self._layers//3: 80 | if self.auxiliary_head and self.training: 81 | logits_aux = self.auxiliary_head(s1) 82 | out = self.global_pooling(s1) 83 | out = out.view(out.size(0), -1) 84 | logits = self.classifier(out) 85 | 86 | if self.auxiliary_head and self.training: 87 | return logits, logits_aux 88 | else: 89 | return logits 90 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/ImageNet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .construct_utils import Cell, Transition 4 | 5 | class AuxiliaryHeadImageNet(nn.Module): 6 | 7 | def __init__(self, C, num_classes): 8 | """assuming input size 14x14""" 9 | super(AuxiliaryHeadImageNet, self).__init__() 10 | self.features = nn.Sequential( 11 | nn.ReLU(inplace=True), 12 | nn.AvgPool2d(5, stride=2, padding=0, count_include_pad=False), 13 | nn.Conv2d(C, 128, 1, bias=False), 14 | nn.BatchNorm2d(128), 15 | nn.ReLU(inplace=True), 16 | nn.Conv2d(128, 768, 2, bias=False), 17 | # NOTE: This batchnorm was omitted in my earlier implementation due to a typo. 18 | # Commenting it out for consistency with the experiments in the paper. 19 | # nn.BatchNorm2d(768), 20 | nn.ReLU(inplace=True) 21 | ) 22 | self.classifier = nn.Linear(768, num_classes) 23 | 24 | def forward(self, x): 25 | x = self.features(x) 26 | x = self.classifier(x.view(x.size(0),-1)) 27 | return x 28 | 29 | 30 | 31 | 32 | class NetworkImageNet(nn.Module): 33 | 34 | def __init__(self, C, num_classes, layers, auxiliary, genotype): 35 | super(NetworkImageNet, self).__init__() 36 | self._layers = layers 37 | 38 | self.stem0 = nn.Sequential( 39 | nn.Conv2d(3, C // 2, kernel_size=3, stride=2, padding=1, bias=False), 40 | nn.BatchNorm2d(C // 2), 41 | nn.ReLU(inplace=True), 42 | nn.Conv2d(C // 2, C, 3, stride=2, padding=1, bias=False), 43 | nn.BatchNorm2d(C), 44 | ) 45 | 46 | self.stem1 = nn.Sequential( 47 | nn.ReLU(inplace=True), 48 | nn.Conv2d(C, C, 3, stride=2, padding=1, bias=False), 49 | nn.BatchNorm2d(C), 50 | ) 51 | 52 | C_prev_prev, C_prev, C_curr = C, C, C 53 | 54 | self.cells = nn.ModuleList() 55 | reduction_prev = True 56 | for i in range(layers): 57 | if i in [layers // 3, 2 * layers // 3]: 58 | C_curr *= 2 59 | reduction = True 60 | else: 61 | reduction = False 62 | if reduction and genotype.reduce is None: 63 | cell = Transition(C_prev_prev, C_prev, C_curr, reduction_prev) 64 | else: 65 | cell = Cell(genotype, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) 66 | reduction_prev = reduction 67 | self.cells += [cell] 68 | C_prev_prev, C_prev = C_prev, cell.multiplier * C_curr 69 | if i == 2 * layers // 3: 70 | C_to_auxiliary = C_prev 71 | 72 | if auxiliary: 73 | self.auxiliary_head = AuxiliaryHeadImageNet(C_to_auxiliary, num_classes) 74 | else: 75 | self.auxiliary_head = None 76 | self.global_pooling = nn.AvgPool2d(7) 77 | self.classifier = nn.Linear(C_prev, num_classes) 78 | self.drop_path_prob = -1 79 | 80 | def update_drop_path(self, drop_path_prob): 81 | self.drop_path_prob = drop_path_prob 82 | 83 | def get_drop_path(self): 84 | return self.drop_path_prob 85 | 86 | def auxiliary_param(self): 87 | if self.auxiliary_head is None: return [] 88 | else: return list( self.auxiliary_head.parameters() ) 89 | 90 | def forward(self, input): 91 | s0 = self.stem0(input) 92 | s1 = self.stem1(s0) 93 | for i, cell in enumerate(self.cells): 94 | s0, s1 = s1, cell(s0, s1, self.drop_path_prob) 95 | #print ('{:} : {:} - {:}'.format(i, s0.size(), s1.size())) 96 | if i == 2 * self._layers // 3: 97 | if self.auxiliary_head and self.training: 98 | logits_aux = self.auxiliary_head(s1) 99 | out = self.global_pooling(s1) 100 | logits = self.classifier(out.view(out.size(0), -1)) 101 | if self.auxiliary_head and self.training: 102 | return logits, logits_aux 103 | else: 104 | return logits 105 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/SE_Module.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | # Squeeze and Excitation module 4 | 5 | class SqEx(nn.Module): 6 | 7 | def __init__(self, n_features, reduction=16): 8 | super(SqEx, self).__init__() 9 | 10 | if n_features % reduction != 0: 11 | raise ValueError('n_features must be divisible by reduction (default = 16)') 12 | 13 | self.linear1 = nn.Linear(n_features, n_features // reduction, bias=True) 14 | self.nonlin1 = nn.ReLU(inplace=True) 15 | self.linear2 = nn.Linear(n_features // reduction, n_features, bias=True) 16 | self.nonlin2 = nn.Sigmoid() 17 | 18 | def forward(self, x): 19 | 20 | y = F.avg_pool2d(x, kernel_size=x.size()[2:4]) 21 | y = y.permute(0, 2, 3, 1) 22 | y = self.nonlin1(self.linear1(y)) 23 | y = self.nonlin2(self.linear2(y)) 24 | y = y.permute(0, 3, 1, 2) 25 | y = x * y 26 | return y 27 | 28 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .model_search import Network 5 | from .CifarNet import NetworkCIFAR 6 | from .ImageNet import NetworkImageNet 7 | 8 | # genotypes 9 | from .genotypes import model_types 10 | 11 | from .construct_utils import return_alphas_str 12 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/construct_utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from .operations import OPS, FactorizedReduce, ReLUConvBN, Identity 6 | 7 | 8 | def random_select(length, ratio): 9 | clist = [] 10 | index = random.randint(0, length-1) 11 | for i in range(length): 12 | if i == index or random.random() < ratio: 13 | clist.append( 1 ) 14 | else: 15 | clist.append( 0 ) 16 | return clist 17 | 18 | 19 | def all_select(length): 20 | return [1 for i in range(length)] 21 | 22 | 23 | def drop_path(x, drop_prob): 24 | if drop_prob > 0.: 25 | keep_prob = 1. - drop_prob 26 | mask = x.new_zeros(x.size(0), 1, 1, 1) 27 | mask = mask.bernoulli_(keep_prob) 28 | x.div_(keep_prob) 29 | x.mul_(mask) 30 | return x 31 | 32 | 33 | def return_alphas_str(basemodel): 34 | string = 'normal : {:}'.format( F.softmax(basemodel.alphas_normal, dim=-1) ) 35 | if hasattr(basemodel, 'alphas_reduce'): 36 | string = string + '\nreduce : {:}'.format( F.softmax(basemodel.alphas_reduce, dim=-1) ) 37 | return string 38 | 39 | 40 | class Cell(nn.Module): 41 | 42 | def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev): 43 | super(Cell, self).__init__() 44 | # print(C_prev_prev, C_prev, C) 45 | 46 | if reduction_prev: 47 | self.preprocess0 = FactorizedReduce(C_prev_prev, C) 48 | else: 49 | self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0) 50 | self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0) 51 | 52 | if reduction: 53 | op_names, indices, values = zip(*genotype.reduce) 54 | concat = genotype.reduce_concat 55 | else: 56 | op_names, indices, values = zip(*genotype.normal) 57 | concat = genotype.normal_concat 58 | self._compile(C, op_names, indices, values, concat, reduction) 59 | 60 | def _compile(self, C, op_names, indices, values, concat, reduction): 61 | assert len(op_names) == len(indices) 62 | self._steps = len(op_names) // 2 63 | self._concat = concat 64 | self.multiplier = len(concat) 65 | 66 | self._ops = nn.ModuleList() 67 | for name, index in zip(op_names, indices): 68 | stride = 2 if reduction and index < 2 else 1 69 | op = OPS[name](C, stride, True) 70 | self._ops.append( op ) 71 | self._indices = indices 72 | self._values = values 73 | 74 | def forward(self, s0, s1, drop_prob): 75 | s0 = self.preprocess0(s0) 76 | s1 = self.preprocess1(s1) 77 | 78 | states = [s0, s1] 79 | for i in range(self._steps): 80 | h1 = states[self._indices[2*i]] 81 | h2 = states[self._indices[2*i+1]] 82 | op1 = self._ops[2*i] 83 | op2 = self._ops[2*i+1] 84 | h1 = op1(h1) 85 | h2 = op2(h2) 86 | if self.training and drop_prob > 0.: 87 | if not isinstance(op1, Identity): 88 | h1 = drop_path(h1, drop_prob) 89 | if not isinstance(op2, Identity): 90 | h2 = drop_path(h2, drop_prob) 91 | 92 | s = h1 + h2 93 | 94 | states += [s] 95 | return torch.cat([states[i] for i in self._concat], dim=1) 96 | 97 | 98 | 99 | class Transition(nn.Module): 100 | 101 | def __init__(self, C_prev_prev, C_prev, C, reduction_prev, multiplier=4): 102 | super(Transition, self).__init__() 103 | if reduction_prev: 104 | self.preprocess0 = FactorizedReduce(C_prev_prev, C) 105 | else: 106 | self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0) 107 | self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0) 108 | self.multiplier = multiplier 109 | 110 | self.reduction = True 111 | self.ops1 = nn.ModuleList( 112 | [nn.Sequential( 113 | nn.ReLU(inplace=False), 114 | nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), 115 | nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), 116 | nn.BatchNorm2d(C, affine=True), 117 | nn.ReLU(inplace=False), 118 | nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), 119 | nn.BatchNorm2d(C, affine=True)), 120 | nn.Sequential( 121 | nn.ReLU(inplace=False), 122 | nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), 123 | nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), 124 | nn.BatchNorm2d(C, affine=True), 125 | nn.ReLU(inplace=False), 126 | nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), 127 | nn.BatchNorm2d(C, affine=True))]) 128 | 129 | self.ops2 = nn.ModuleList( 130 | [nn.Sequential( 131 | nn.MaxPool2d(3, stride=2, padding=1), 132 | nn.BatchNorm2d(C, affine=True)), 133 | nn.Sequential( 134 | nn.MaxPool2d(3, stride=2, padding=1), 135 | nn.BatchNorm2d(C, affine=True))]) 136 | 137 | 138 | def forward(self, s0, s1, drop_prob = -1): 139 | s0 = self.preprocess0(s0) 140 | s1 = self.preprocess1(s1) 141 | 142 | X0 = self.ops1[0] (s0) 143 | X1 = self.ops1[1] (s1) 144 | if self.training and drop_prob > 0.: 145 | X0, X1 = drop_path(X0, drop_prob), drop_path(X1, drop_prob) 146 | 147 | #X2 = self.ops2[0] (X0+X1) 148 | X2 = self.ops2[0] (s0) 149 | X3 = self.ops2[1] (s1) 150 | if self.training and drop_prob > 0.: 151 | X2, X3 = drop_path(X2, drop_prob), drop_path(X3, drop_prob) 152 | return torch.cat([X0, X1, X2, X3], dim=1) 153 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/genotypes.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat') 4 | 5 | PRIMITIVES = [ 6 | 'none', 7 | 'max_pool_3x3', 8 | 'avg_pool_3x3', 9 | 'skip_connect', 10 | 'sep_conv_3x3', 11 | 'sep_conv_5x5', 12 | 'dil_conv_3x3', 13 | 'dil_conv_5x5' 14 | ] 15 | 16 | NASNet = Genotype( 17 | normal = [ 18 | ('sep_conv_5x5', 1, 1.0), 19 | ('sep_conv_3x3', 0, 1.0), 20 | ('sep_conv_5x5', 0, 1.0), 21 | ('sep_conv_3x3', 0, 1.0), 22 | ('avg_pool_3x3', 1, 1.0), 23 | ('skip_connect', 0, 1.0), 24 | ('avg_pool_3x3', 0, 1.0), 25 | ('avg_pool_3x3', 0, 1.0), 26 | ('sep_conv_3x3', 1, 1.0), 27 | ('skip_connect', 1, 1.0), 28 | ], 29 | normal_concat = [2, 3, 4, 5, 6], 30 | reduce = [ 31 | ('sep_conv_5x5', 1, 1.0), 32 | ('sep_conv_7x7', 0, 1.0), 33 | ('max_pool_3x3', 1, 1.0), 34 | ('sep_conv_7x7', 0, 1.0), 35 | ('avg_pool_3x3', 1, 1.0), 36 | ('sep_conv_5x5', 0, 1.0), 37 | ('skip_connect', 3, 1.0), 38 | ('avg_pool_3x3', 2, 1.0), 39 | ('sep_conv_3x3', 2, 1.0), 40 | ('max_pool_3x3', 1, 1.0), 41 | ], 42 | reduce_concat = [4, 5, 6], 43 | ) 44 | 45 | AmoebaNet = Genotype( 46 | normal = [ 47 | ('avg_pool_3x3', 0, 1.0), 48 | ('max_pool_3x3', 1, 1.0), 49 | ('sep_conv_3x3', 0, 1.0), 50 | ('sep_conv_5x5', 2, 1.0), 51 | ('sep_conv_3x3', 0, 1.0), 52 | ('avg_pool_3x3', 3, 1.0), 53 | ('sep_conv_3x3', 1, 1.0), 54 | ('skip_connect', 1, 1.0), 55 | ('skip_connect', 0, 1.0), 56 | ('avg_pool_3x3', 1, 1.0), 57 | ], 58 | normal_concat = [4, 5, 6], 59 | reduce = [ 60 | ('avg_pool_3x3', 0, 1.0), 61 | ('sep_conv_3x3', 1, 1.0), 62 | ('max_pool_3x3', 0, 1.0), 63 | ('sep_conv_7x7', 2, 1.0), 64 | ('sep_conv_7x7', 0, 1.0), 65 | ('avg_pool_3x3', 1, 1.0), 66 | ('max_pool_3x3', 0, 1.0), 67 | ('max_pool_3x3', 1, 1.0), 68 | ('conv_7x1_1x7', 0, 1.0), 69 | ('sep_conv_3x3', 5, 1.0), 70 | ], 71 | reduce_concat = [3, 4, 6] 72 | ) 73 | 74 | DARTS_V1 = Genotype( 75 | normal=[ 76 | ('sep_conv_3x3', 1, 1.0), 77 | ('sep_conv_3x3', 0, 1.0), 78 | ('skip_connect', 0, 1.0), 79 | ('sep_conv_3x3', 1, 1.0), 80 | ('skip_connect', 0, 1.0), 81 | ('sep_conv_3x3', 1, 1.0), 82 | ('sep_conv_3x3', 0, 1.0), 83 | ('skip_connect', 2, 1.0)], 84 | normal_concat=[2, 3, 4, 5], 85 | reduce=[ 86 | ('max_pool_3x3', 0, 1.0), 87 | ('max_pool_3x3', 1, 1.0), 88 | ('skip_connect', 2, 1.0), 89 | ('max_pool_3x3', 0, 1.0), 90 | ('max_pool_3x3', 0, 1.0), 91 | ('skip_connect', 2, 1.0), 92 | ('skip_connect', 2, 1.0), 93 | ('avg_pool_3x3', 0, 1.0)], 94 | reduce_concat=[2, 3, 4, 5] 95 | ) 96 | 97 | DARTS_V2 = Genotype( 98 | normal=[ 99 | ('sep_conv_3x3', 0, 1.0), 100 | ('sep_conv_3x3', 1, 1.0), 101 | ('sep_conv_3x3', 0, 1.0), 102 | ('sep_conv_3x3', 1, 1.0), 103 | ('sep_conv_3x3', 1, 1.0), 104 | ('skip_connect', 0, 1.0), 105 | ('skip_connect', 0, 1.0), 106 | ('dil_conv_3x3', 2, 1.0)], 107 | normal_concat=[2, 3, 4, 5], 108 | reduce=[ 109 | ('max_pool_3x3', 0, 1.0), 110 | ('max_pool_3x3', 1, 1.0), 111 | ('skip_connect', 2, 1.0), 112 | ('max_pool_3x3', 1, 1.0), 113 | ('max_pool_3x3', 0, 1.0), 114 | ('skip_connect', 2, 1.0), 115 | ('skip_connect', 2, 1.0), 116 | ('max_pool_3x3', 1, 1.0)], 117 | reduce_concat=[2, 3, 4, 5] 118 | ) 119 | 120 | PNASNet = Genotype( 121 | normal = [ 122 | ('sep_conv_5x5', 0, 1.0), 123 | ('max_pool_3x3', 0, 1.0), 124 | ('sep_conv_7x7', 1, 1.0), 125 | ('max_pool_3x3', 1, 1.0), 126 | ('sep_conv_5x5', 1, 1.0), 127 | ('sep_conv_3x3', 1, 1.0), 128 | ('sep_conv_3x3', 4, 1.0), 129 | ('max_pool_3x3', 1, 1.0), 130 | ('sep_conv_3x3', 0, 1.0), 131 | ('skip_connect', 1, 1.0), 132 | ], 133 | normal_concat = [2, 3, 4, 5, 6], 134 | reduce = [ 135 | ('sep_conv_5x5', 0, 1.0), 136 | ('max_pool_3x3', 0, 1.0), 137 | ('sep_conv_7x7', 1, 1.0), 138 | ('max_pool_3x3', 1, 1.0), 139 | ('sep_conv_5x5', 1, 1.0), 140 | ('sep_conv_3x3', 1, 1.0), 141 | ('sep_conv_3x3', 4, 1.0), 142 | ('max_pool_3x3', 1, 1.0), 143 | ('sep_conv_3x3', 0, 1.0), 144 | ('skip_connect', 1, 1.0), 145 | ], 146 | reduce_concat = [2, 3, 4, 5, 6], 147 | ) 148 | 149 | # https://arxiv.org/pdf/1802.03268.pdf 150 | ENASNet = Genotype( 151 | normal = [ 152 | ('sep_conv_3x3', 1, 1.0), 153 | ('skip_connect', 1, 1.0), 154 | ('sep_conv_5x5', 1, 1.0), 155 | ('skip_connect', 0, 1.0), 156 | ('avg_pool_3x3', 0, 1.0), 157 | ('sep_conv_3x3', 1, 1.0), 158 | ('sep_conv_3x3', 0, 1.0), 159 | ('avg_pool_3x3', 1, 1.0), 160 | ('sep_conv_5x5', 1, 1.0), 161 | ('avg_pool_3x3', 0, 1.0), 162 | ], 163 | normal_concat = [2, 3, 4, 5, 6], 164 | reduce = [ 165 | ('sep_conv_5x5', 0, 1.0), 166 | ('sep_conv_3x3', 1, 1.0), # 2 167 | ('sep_conv_3x3', 1, 1.0), 168 | ('avg_pool_3x3', 1, 1.0), # 3 169 | ('sep_conv_3x3', 1, 1.0), 170 | ('avg_pool_3x3', 1, 1.0), # 4 171 | ('avg_pool_3x3', 1, 1.0), 172 | ('sep_conv_5x5', 4, 1.0), # 5 173 | ('sep_conv_3x3', 5, 1.0), 174 | ('sep_conv_5x5', 0, 1.0), 175 | ], 176 | reduce_concat = [2, 3, 4, 5, 6], 177 | ) 178 | 179 | DARTS = DARTS_V2 180 | 181 | # Search by normal and reduce 182 | GDAS_V1 = Genotype( 183 | normal=[('skip_connect', 0, 0.13017432391643524), ('skip_connect', 1, 0.12947972118854523), ('skip_connect', 0, 0.13062666356563568), ('sep_conv_5x5', 2, 0.12980839610099792), ('sep_conv_3x3', 3, 0.12923765182495117), ('skip_connect', 0, 0.12901571393013), ('sep_conv_5x5', 4, 0.12938997149467468), ('sep_conv_3x3', 3, 0.1289220005273819)], 184 | normal_concat=range(2, 6), 185 | reduce=[('sep_conv_5x5', 0, 0.12862831354141235), ('sep_conv_3x3', 1, 0.12783904373645782), ('sep_conv_5x5', 2, 0.12725995481014252), ('sep_conv_5x5', 1, 0.12705285847187042), ('dil_conv_5x5', 2, 0.12797553837299347), ('sep_conv_3x3', 1, 0.12737272679805756), ('sep_conv_5x5', 0, 0.12833961844444275), ('sep_conv_5x5', 1, 0.12758426368236542)], 186 | reduce_concat=range(2, 6) 187 | ) 188 | 189 | # Search by normal and fixing reduction 190 | GDAS_F1 = Genotype( 191 | normal=[('skip_connect', 0, 0.16), ('skip_connect', 1, 0.13), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.16), ('sep_conv_3x3', 2, 0.15)], 192 | normal_concat=[2, 3, 4, 5], 193 | reduce=None, 194 | reduce_concat=[2, 3, 4, 5], 195 | ) 196 | 197 | # Combine DMS_V1 and DMS_F1 198 | GDAS_GF = Genotype( 199 | normal=[('skip_connect', 0, 0.13017432391643524), ('skip_connect', 1, 0.12947972118854523), ('skip_connect', 0, 0.13062666356563568), ('sep_conv_5x5', 2, 0.12980839610099792), ('sep_conv_3x3', 3, 0.12923765182495117), ('skip_connect', 0, 0.12901571393013), ('sep_conv_5x5', 4, 0.12938997149467468), ('sep_conv_3x3', 3, 0.1289220005273819)], 200 | normal_concat=range(2, 6), 201 | reduce=None, 202 | reduce_concat=range(2, 6) 203 | ) 204 | GDAS_FG = Genotype( 205 | normal=[('skip_connect', 0, 0.16), ('skip_connect', 1, 0.13), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.16), ('sep_conv_3x3', 2, 0.15)], 206 | normal_concat=range(2, 6), 207 | reduce=[('sep_conv_5x5', 0, 0.12862831354141235), ('sep_conv_3x3', 1, 0.12783904373645782), ('sep_conv_5x5', 2, 0.12725995481014252), ('sep_conv_5x5', 1, 0.12705285847187042), ('dil_conv_5x5', 2, 0.12797553837299347), ('sep_conv_3x3', 1, 0.12737272679805756), ('sep_conv_5x5', 0, 0.12833961844444275), ('sep_conv_5x5', 1, 0.12758426368236542)], 208 | reduce_concat=range(2, 6) 209 | ) 210 | 211 | model_types = {'DARTS_V1': DARTS_V1, 212 | 'DARTS_V2': DARTS_V2, 213 | 'NASNet' : NASNet, 214 | 'PNASNet' : PNASNet, 215 | 'AmoebaNet': AmoebaNet, 216 | 'ENASNet' : ENASNet, 217 | 'GDAS_V1' : GDAS_V1, 218 | 'GDAS_F1' : GDAS_F1, 219 | 'GDAS_GF' : GDAS_GF, 220 | 'GDAS_FG' : GDAS_FG} 221 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/head_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class ImageNetHEAD(nn.Sequential): 6 | def __init__(self, C, stride=2): 7 | super(ImageNetHEAD, self).__init__() 8 | self.add_module('conv1', nn.Conv2d(3, C // 2, kernel_size=3, stride=2, padding=1, bias=False)) 9 | self.add_module('bn1' , nn.BatchNorm2d(C // 2)) 10 | self.add_module('relu1', nn.ReLU(inplace=True)) 11 | self.add_module('conv2', nn.Conv2d(C // 2, C, kernel_size=3, stride=stride, padding=1, bias=False)) 12 | self.add_module('bn2' , nn.BatchNorm2d(C)) 13 | 14 | 15 | class CifarHEAD(nn.Sequential): 16 | def __init__(self, C): 17 | super(CifarHEAD, self).__init__() 18 | self.add_module('conv', nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False)) 19 | self.add_module('bn', nn.BatchNorm2d(C)) 20 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/model_search.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn.parameter import Parameter 5 | from .head_utils import CifarHEAD, ImageNetHEAD 6 | from .operations import OPS, FactorizedReduce, ReLUConvBN 7 | from .genotypes import PRIMITIVES, Genotype 8 | 9 | 10 | class MixedOp(nn.Module): 11 | 12 | def __init__(self, C, stride): 13 | super(MixedOp, self).__init__() 14 | self._ops = nn.ModuleList() 15 | for primitive in PRIMITIVES: 16 | op = OPS[primitive](C, stride, False) 17 | self._ops.append(op) 18 | 19 | def forward(self, x, weights): 20 | return sum(w * op(x) for w, op in zip(weights, self._ops)) 21 | 22 | 23 | class Cell(nn.Module): 24 | 25 | def __init__(self, steps, multiplier, C_prev_prev, C_prev, C, reduction, reduction_prev): 26 | super(Cell, self).__init__() 27 | self.reduction = reduction 28 | 29 | if reduction_prev: 30 | self.preprocess0 = FactorizedReduce(C_prev_prev, C, affine=False) 31 | else: 32 | self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0, affine=False) 33 | self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0, affine=False) 34 | self._steps = steps 35 | self._multiplier = multiplier 36 | 37 | self._ops = nn.ModuleList() 38 | for i in range(self._steps): 39 | for j in range(2+i): 40 | stride = 2 if reduction and j < 2 else 1 41 | op = MixedOp(C, stride) 42 | self._ops.append(op) 43 | 44 | def forward(self, s0, s1, weights): 45 | s0 = self.preprocess0(s0) 46 | s1 = self.preprocess1(s1) 47 | 48 | states = [s0, s1] 49 | offset = 0 50 | for i in range(self._steps): 51 | clist = [] 52 | for j, h in enumerate(states): 53 | x = self._ops[offset+j](h, weights[offset+j]) 54 | clist.append( x ) 55 | s = sum(clist) 56 | offset += len(states) 57 | states.append(s) 58 | 59 | return torch.cat(states[-self._multiplier:], dim=1) 60 | 61 | 62 | class Network(nn.Module): 63 | 64 | def __init__(self, C, num_classes, layers, steps=4, multiplier=4, stem_multiplier=3, head='cifar'): 65 | super(Network, self).__init__() 66 | self._C = C 67 | self._num_classes = num_classes 68 | self._layers = layers 69 | self._steps = steps 70 | self._multiplier = multiplier 71 | 72 | C_curr = stem_multiplier*C 73 | if head == 'cifar': 74 | self.stem = nn.Sequential( 75 | nn.Conv2d(3, C_curr, 3, padding=1, bias=False), 76 | nn.BatchNorm2d(C_curr) 77 | ) 78 | elif head == 'imagenet': 79 | self.stem = ImageNetHEAD(C_curr, stride=1) 80 | else: 81 | raise ValueError('Invalid head : {:}'.format(head)) 82 | 83 | C_prev_prev, C_prev, C_curr = C_curr, C_curr, C 84 | reduction_prev, cells = False, [] 85 | for i in range(layers): 86 | if i in [layers//3, 2*layers//3]: 87 | C_curr *= 2 88 | reduction = True 89 | else: 90 | reduction = False 91 | cell = Cell(steps, multiplier, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) 92 | reduction_prev = reduction 93 | cells.append( cell ) 94 | C_prev_prev, C_prev = C_prev, multiplier*C_curr 95 | self.cells = nn.ModuleList(cells) 96 | 97 | self.global_pooling = nn.AdaptiveAvgPool2d(1) 98 | self.classifier = nn.Linear(C_prev, num_classes) 99 | 100 | # initialize architecture parameters 101 | k = sum(1 for i in range(self._steps) for n in range(2+i)) 102 | num_ops = len(PRIMITIVES) 103 | 104 | self.alphas_normal = Parameter(torch.Tensor(k, num_ops)) 105 | self.alphas_reduce = Parameter(torch.Tensor(k, num_ops)) 106 | nn.init.normal_(self.alphas_normal, 0, 0.001) 107 | nn.init.normal_(self.alphas_reduce, 0, 0.001) 108 | 109 | def set_tau(self, tau): 110 | return -1 111 | 112 | def get_tau(self): 113 | return -1 114 | 115 | def arch_parameters(self): 116 | return [self.alphas_normal, self.alphas_reduce] 117 | 118 | def base_parameters(self): 119 | lists = list(self.stem.parameters()) + list(self.cells.parameters()) 120 | lists += list(self.global_pooling.parameters()) 121 | lists += list(self.classifier.parameters()) 122 | return lists 123 | 124 | def forward(self, inputs): 125 | batch, C, H, W = inputs.size() 126 | s0 = s1 = self.stem(inputs) 127 | for i, cell in enumerate(self.cells): 128 | if cell.reduction: 129 | weights = F.softmax(self.alphas_reduce, dim=-1) 130 | else: 131 | weights = F.softmax(self.alphas_normal, dim=-1) 132 | s0, s1 = s1, cell(s0, s1, weights) 133 | out = self.global_pooling(s1) 134 | out = out.view(batch, -1) 135 | logits = self.classifier(out) 136 | return logits 137 | 138 | def genotype(self): 139 | 140 | def _parse(weights): 141 | gene, n, start = [], 2, 0 142 | for i in range(self._steps): 143 | end = start + n 144 | W = weights[start:end].copy() 145 | edges = sorted(range(i + 2), key=lambda x: -max(W[x][k] for k in range(len(W[x])) if k != PRIMITIVES.index('none')))[:2] 146 | for j in edges: 147 | k_best = None 148 | for k in range(len(W[j])): 149 | if k != PRIMITIVES.index('none'): 150 | if k_best is None or W[j][k] > W[j][k_best]: 151 | k_best = k 152 | gene.append((PRIMITIVES[k_best], j, float(W[j][k_best]))) 153 | start = end 154 | n += 1 155 | return gene 156 | 157 | with torch.no_grad(): 158 | gene_normal = _parse(F.softmax(self.alphas_normal, dim=-1).cpu().numpy()) 159 | gene_reduce = _parse(F.softmax(self.alphas_reduce, dim=-1).cpu().numpy()) 160 | 161 | concat = range(2+self._steps-self._multiplier, self._steps+2) 162 | genotype = Genotype( 163 | normal=gene_normal, normal_concat=concat, 164 | reduce=gene_reduce, reduce_concat=concat 165 | ) 166 | return genotype 167 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas/operations.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | OPS = { 5 | 'none' : lambda C, stride, affine: Zero(stride), 6 | 'avg_pool_3x3' : lambda C, stride, affine: nn.Sequential( 7 | nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False), 8 | nn.BatchNorm2d(C, affine=False) ), 9 | 'max_pool_3x3' : lambda C, stride, affine: nn.Sequential( 10 | nn.MaxPool2d(3, stride=stride, padding=1), 11 | nn.BatchNorm2d(C, affine=False) ), 12 | 'skip_connect' : lambda C, stride, affine: Identity() if stride == 1 else FactorizedReduce(C, C, affine=affine), 13 | 'sep_conv_3x3' : lambda C, stride, affine: SepConv(C, C, 3, stride, 1, affine=affine), 14 | 'sep_conv_5x5' : lambda C, stride, affine: SepConv(C, C, 5, stride, 2, affine=affine), 15 | 'sep_conv_7x7' : lambda C, stride, affine: SepConv(C, C, 7, stride, 3, affine=affine), 16 | 'dil_conv_3x3' : lambda C, stride, affine: DilConv(C, C, 3, stride, 2, 2, affine=affine), 17 | 'dil_conv_5x5' : lambda C, stride, affine: DilConv(C, C, 5, stride, 4, 2, affine=affine), 18 | 'conv_7x1_1x7' : lambda C, stride, affine: Conv717(C, C, stride, affine), 19 | } 20 | 21 | class Conv717(nn.Module): 22 | 23 | def __init__(self, C_in, C_out, stride, affine): 24 | super(Conv717, self).__init__() 25 | self.op = nn.Sequential( 26 | nn.ReLU(inplace=False), 27 | nn.Conv2d(C_in , C_out, (1,7), stride=(1, stride), padding=(0, 3), bias=False), 28 | nn.Conv2d(C_out, C_out, (7,1), stride=(stride, 1), padding=(3, 0), bias=False), 29 | nn.BatchNorm2d(C_out, affine=affine) 30 | ) 31 | 32 | def forward(self, x): 33 | return self.op(x) 34 | 35 | 36 | class ReLUConvBN(nn.Module): 37 | 38 | def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): 39 | super(ReLUConvBN, self).__init__() 40 | self.op = nn.Sequential( 41 | nn.ReLU(inplace=False), 42 | nn.Conv2d(C_in, C_out, kernel_size, stride=stride, padding=padding, bias=False), 43 | nn.BatchNorm2d(C_out, affine=affine) 44 | ) 45 | 46 | def forward(self, x): 47 | return self.op(x) 48 | 49 | 50 | class DilConv(nn.Module): 51 | 52 | def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True): 53 | super(DilConv, self).__init__() 54 | self.op = nn.Sequential( 55 | nn.ReLU(inplace=False), 56 | nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=C_in, bias=False), 57 | nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False), 58 | nn.BatchNorm2d(C_out, affine=affine), 59 | ) 60 | 61 | def forward(self, x): 62 | return self.op(x) 63 | 64 | 65 | class SepConv(nn.Module): 66 | 67 | def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): 68 | super(SepConv, self).__init__() 69 | self.op = nn.Sequential( 70 | nn.ReLU(inplace=False), 71 | nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, groups=C_in, bias=False), 72 | nn.Conv2d(C_in, C_in, kernel_size=1, padding=0, bias=False), 73 | nn.BatchNorm2d(C_in, affine=affine), 74 | nn.ReLU(inplace=False), 75 | nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=1, padding=padding, groups=C_in, bias=False), 76 | nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False), 77 | nn.BatchNorm2d(C_out, affine=affine), 78 | ) 79 | 80 | def forward(self, x): 81 | return self.op(x) 82 | 83 | 84 | class Identity(nn.Module): 85 | 86 | def __init__(self): 87 | super(Identity, self).__init__() 88 | 89 | def forward(self, x): 90 | return x 91 | 92 | 93 | class Zero(nn.Module): 94 | 95 | def __init__(self, stride): 96 | super(Zero, self).__init__() 97 | self.stride = stride 98 | 99 | def forward(self, x): 100 | if self.stride == 1: 101 | return x.mul(0.) 102 | return x[:,:,::self.stride,::self.stride].mul(0.) 103 | 104 | 105 | class FactorizedReduce(nn.Module): 106 | 107 | def __init__(self, C_in, C_out, affine=True): 108 | super(FactorizedReduce, self).__init__() 109 | assert C_out % 2 == 0 110 | self.relu = nn.ReLU(inplace=False) 111 | self.conv_1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) 112 | self.conv_2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) 113 | self.bn = nn.BatchNorm2d(C_out, affine=affine) 114 | self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0) 115 | 116 | 117 | def forward(self, x): 118 | x = self.relu(x) 119 | y = self.pad(x) 120 | out = torch.cat([self.conv_1(x), self.conv_2(y[:,:,1:,1:])], dim=1) 121 | out = self.bn(out) 122 | return out 123 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas_rnn/__init__.py: -------------------------------------------------------------------------------- 1 | # utils 2 | from .utils import batchify, get_batch, repackage_hidden 3 | # models 4 | from .model_search import RNNModelSearch 5 | from .model_search import DARTSCellSearch 6 | from .basemodel import DARTSCell, RNNModel 7 | # architecture 8 | from .genotypes import DARTS_V1, DARTS_V2 9 | from .genotypes import GDAS 10 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas_rnn/basemodel.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from .genotypes import STEPS 6 | from .utils import mask2d, LockedDropout, embedded_dropout 7 | 8 | 9 | INITRANGE = 0.04 10 | 11 | def none_func(x): 12 | return x * 0 13 | 14 | 15 | class DARTSCell(nn.Module): 16 | 17 | def __init__(self, ninp, nhid, dropouth, dropoutx, genotype): 18 | super(DARTSCell, self).__init__() 19 | self.nhid = nhid 20 | self.dropouth = dropouth 21 | self.dropoutx = dropoutx 22 | self.genotype = genotype 23 | 24 | # genotype is None when doing arch search 25 | steps = len(self.genotype.recurrent) if self.genotype is not None else STEPS 26 | self._W0 = nn.Parameter(torch.Tensor(ninp+nhid, 2*nhid).uniform_(-INITRANGE, INITRANGE)) 27 | self._Ws = nn.ParameterList([ 28 | nn.Parameter(torch.Tensor(nhid, 2*nhid).uniform_(-INITRANGE, INITRANGE)) for i in range(steps) 29 | ]) 30 | 31 | def forward(self, inputs, hidden, arch_probs): 32 | T, B = inputs.size(0), inputs.size(1) 33 | 34 | if self.training: 35 | x_mask = mask2d(B, inputs.size(2), keep_prob=1.-self.dropoutx) 36 | h_mask = mask2d(B, hidden.size(2), keep_prob=1.-self.dropouth) 37 | else: 38 | x_mask = h_mask = None 39 | 40 | hidden = hidden[0] 41 | hiddens = [] 42 | for t in range(T): 43 | hidden = self.cell(inputs[t], hidden, x_mask, h_mask, arch_probs) 44 | hiddens.append(hidden) 45 | hiddens = torch.stack(hiddens) 46 | return hiddens, hiddens[-1].unsqueeze(0) 47 | 48 | def _compute_init_state(self, x, h_prev, x_mask, h_mask): 49 | if self.training: 50 | xh_prev = torch.cat([x * x_mask, h_prev * h_mask], dim=-1) 51 | else: 52 | xh_prev = torch.cat([x, h_prev], dim=-1) 53 | c0, h0 = torch.split(xh_prev.mm(self._W0), self.nhid, dim=-1) 54 | c0 = c0.sigmoid() 55 | h0 = h0.tanh() 56 | s0 = h_prev + c0 * (h0-h_prev) 57 | return s0 58 | 59 | def _get_activation(self, name): 60 | if name == 'tanh': 61 | f = torch.tanh 62 | elif name == 'relu': 63 | f = torch.relu 64 | elif name == 'sigmoid': 65 | f = torch.sigmoid 66 | elif name == 'identity': 67 | f = lambda x: x 68 | elif name == 'none': 69 | f = none_func 70 | else: 71 | raise NotImplementedError 72 | return f 73 | 74 | def cell(self, x, h_prev, x_mask, h_mask, _): 75 | s0 = self._compute_init_state(x, h_prev, x_mask, h_mask) 76 | 77 | states = [s0] 78 | for i, (name, pred) in enumerate(self.genotype.recurrent): 79 | s_prev = states[pred] 80 | if self.training: 81 | ch = (s_prev * h_mask).mm(self._Ws[i]) 82 | else: 83 | ch = s_prev.mm(self._Ws[i]) 84 | c, h = torch.split(ch, self.nhid, dim=-1) 85 | c = c.sigmoid() 86 | fn = self._get_activation(name) 87 | h = fn(h) 88 | s = s_prev + c * (h-s_prev) 89 | states += [s] 90 | output = torch.mean(torch.stack([states[i] for i in self.genotype.concat], -1), -1) 91 | return output 92 | 93 | 94 | class RNNModel(nn.Module): 95 | """Container module with an encoder, a recurrent module, and a decoder.""" 96 | def __init__(self, ntoken, ninp, nhid, nhidlast, 97 | dropout=0.5, dropouth=0.5, dropoutx=0.5, dropouti=0.5, dropoute=0.1, 98 | cell_cls=None, genotype=None): 99 | super(RNNModel, self).__init__() 100 | self.lockdrop = LockedDropout() 101 | self.encoder = nn.Embedding(ntoken, ninp) 102 | 103 | assert ninp == nhid == nhidlast 104 | if cell_cls == DARTSCell: 105 | assert genotype is not None 106 | rnns = [cell_cls(ninp, nhid, dropouth, dropoutx, genotype)] 107 | else: 108 | assert genotype is None 109 | rnns = [cell_cls(ninp, nhid, dropouth, dropoutx)] 110 | 111 | self.rnns = torch.nn.ModuleList(rnns) 112 | self.decoder = nn.Linear(ninp, ntoken) 113 | self.decoder.weight = self.encoder.weight 114 | self.init_weights() 115 | self.arch_weights = None 116 | 117 | self.ninp = ninp 118 | self.nhid = nhid 119 | self.nhidlast = nhidlast 120 | self.dropout = dropout 121 | self.dropouti = dropouti 122 | self.dropoute = dropoute 123 | self.ntoken = ntoken 124 | self.cell_cls = cell_cls 125 | # acceleration 126 | self.tau = None 127 | self.use_gumbel = False 128 | 129 | def set_gumbel(self, use_gumbel, set_check): 130 | self.use_gumbel = use_gumbel 131 | for i, rnn in enumerate(self.rnns): 132 | rnn.set_check(set_check) 133 | 134 | def set_tau(self, tau): 135 | self.tau = tau 136 | 137 | def get_tau(self): 138 | return self.tau 139 | 140 | def init_weights(self): 141 | self.encoder.weight.data.uniform_(-INITRANGE, INITRANGE) 142 | self.decoder.bias.data.fill_(0) 143 | self.decoder.weight.data.uniform_(-INITRANGE, INITRANGE) 144 | 145 | def forward(self, input, hidden, return_h=False): 146 | batch_size = input.size(1) 147 | 148 | emb = embedded_dropout(self.encoder, input, dropout=self.dropoute if self.training else 0) 149 | emb = self.lockdrop(emb, self.dropouti) 150 | 151 | raw_output = emb 152 | new_hidden = [] 153 | raw_outputs = [] 154 | outputs = [] 155 | if self.arch_weights is None: 156 | arch_probs = None 157 | else: 158 | if self.use_gumbel: arch_probs = F.gumbel_softmax(self.arch_weights, self.tau, False) 159 | else : arch_probs = F.softmax(self.arch_weights, dim=-1) 160 | 161 | for l, rnn in enumerate(self.rnns): 162 | current_input = raw_output 163 | raw_output, new_h = rnn(raw_output, hidden[l], arch_probs) 164 | new_hidden.append(new_h) 165 | raw_outputs.append(raw_output) 166 | hidden = new_hidden 167 | 168 | output = self.lockdrop(raw_output, self.dropout) 169 | outputs.append(output) 170 | 171 | logit = self.decoder(output.view(-1, self.ninp)) 172 | log_prob = nn.functional.log_softmax(logit, dim=-1) 173 | model_output = log_prob 174 | model_output = model_output.view(-1, batch_size, self.ntoken) 175 | 176 | if return_h: return model_output, hidden, raw_outputs, outputs 177 | else : return model_output, hidden 178 | 179 | def init_hidden(self, bsz): 180 | weight = next(self.parameters()).clone() 181 | return [weight.new(1, bsz, self.nhid).zero_()] 182 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas_rnn/genotypes.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | Genotype = namedtuple('Genotype', 'recurrent concat') 4 | 5 | PRIMITIVES = [ 6 | 'none', 7 | 'tanh', 8 | 'relu', 9 | 'sigmoid', 10 | 'identity' 11 | ] 12 | STEPS = 8 13 | CONCAT = 8 14 | 15 | ENAS = Genotype( 16 | recurrent = [ 17 | ('tanh', 0), 18 | ('tanh', 1), 19 | ('relu', 1), 20 | ('tanh', 3), 21 | ('tanh', 3), 22 | ('relu', 3), 23 | ('relu', 4), 24 | ('relu', 7), 25 | ('relu', 8), 26 | ('relu', 8), 27 | ('relu', 8), 28 | ], 29 | concat = [2, 5, 6, 9, 10, 11] 30 | ) 31 | 32 | DARTS_V1 = Genotype( 33 | recurrent = [ 34 | ('relu', 0), 35 | ('relu', 1), 36 | ('tanh', 2), 37 | ('relu', 3), ('relu', 4), ('identity', 1), ('relu', 5), ('relu', 1) 38 | ], 39 | concat=range(1, 9) 40 | ) 41 | 42 | DARTS_V2 = Genotype( 43 | recurrent = [ 44 | ('sigmoid', 0), ('relu', 1), ('relu', 1), 45 | ('identity', 1), ('tanh', 2), ('sigmoid', 5), 46 | ('tanh', 3), ('relu', 5) 47 | ], 48 | concat=range(1, 9) 49 | ) 50 | 51 | GDAS = Genotype( 52 | recurrent=[('relu', 0), ('relu', 0), ('identity', 1), ('relu', 1), ('tanh', 0), ('relu', 2), ('identity', 4), ('identity', 2)], 53 | concat=range(1, 9) 54 | ) 55 | 56 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas_rnn/model_search.py: -------------------------------------------------------------------------------- 1 | import copy, torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from collections import namedtuple 5 | from .genotypes import PRIMITIVES, STEPS, CONCAT, Genotype 6 | from .basemodel import DARTSCell, RNNModel 7 | 8 | 9 | class DARTSCellSearch(DARTSCell): 10 | 11 | def __init__(self, ninp, nhid, dropouth, dropoutx): 12 | super(DARTSCellSearch, self).__init__(ninp, nhid, dropouth, dropoutx, genotype=None) 13 | self.bn = nn.BatchNorm1d(nhid, affine=False) 14 | self.check_zero = False 15 | 16 | def set_check(self, check_zero): 17 | self.check_zero = check_zero 18 | 19 | def cell(self, x, h_prev, x_mask, h_mask, arch_probs): 20 | s0 = self._compute_init_state(x, h_prev, x_mask, h_mask) 21 | s0 = self.bn(s0) 22 | if self.check_zero: 23 | arch_probs_cpu = arch_probs.cpu().tolist() 24 | #arch_probs = F.softmax(self.weights, dim=-1) 25 | 26 | offset = 0 27 | states = s0.unsqueeze(0) 28 | for i in range(STEPS): 29 | if self.training: 30 | masked_states = states * h_mask.unsqueeze(0) 31 | else: 32 | masked_states = states 33 | ch = masked_states.view(-1, self.nhid).mm(self._Ws[i]).view(i+1, -1, 2*self.nhid) 34 | c, h = torch.split(ch, self.nhid, dim=-1) 35 | c = c.sigmoid() 36 | 37 | s = torch.zeros_like(s0) 38 | for k, name in enumerate(PRIMITIVES): 39 | if name == 'none': 40 | continue 41 | fn = self._get_activation(name) 42 | unweighted = states + c * (fn(h) - states) 43 | if self.check_zero: 44 | INDEX, INDDX = [], [] 45 | for jj in range(offset, offset+i+1): 46 | if arch_probs_cpu[jj][k] > 0: 47 | INDEX.append(jj) 48 | INDDX.append(jj-offset) 49 | if len(INDEX) == 0: continue 50 | s += torch.sum(arch_probs[INDEX, k].unsqueeze(-1).unsqueeze(-1) * unweighted[INDDX, :, :], dim=0) 51 | else: 52 | s += torch.sum(arch_probs[offset:offset+i+1, k].unsqueeze(-1).unsqueeze(-1) * unweighted, dim=0) 53 | s = self.bn(s) 54 | states = torch.cat([states, s.unsqueeze(0)], 0) 55 | offset += i+1 56 | output = torch.mean(states[-CONCAT:], dim=0) 57 | return output 58 | 59 | 60 | class RNNModelSearch(RNNModel): 61 | 62 | def __init__(self, *args): 63 | super(RNNModelSearch, self).__init__(*args) 64 | self._args = copy.deepcopy( args ) 65 | 66 | k = sum(i for i in range(1, STEPS+1)) 67 | self.arch_weights = nn.Parameter(torch.Tensor(k, len(PRIMITIVES))) 68 | nn.init.normal_(self.arch_weights, 0, 0.001) 69 | 70 | def base_parameters(self): 71 | lists = list(self.lockdrop.parameters()) 72 | lists += list(self.encoder.parameters()) 73 | lists += list(self.rnns.parameters()) 74 | lists += list(self.decoder.parameters()) 75 | return lists 76 | 77 | def arch_parameters(self): 78 | return [self.arch_weights] 79 | 80 | def genotype(self): 81 | 82 | def _parse(probs): 83 | gene = [] 84 | start = 0 85 | for i in range(STEPS): 86 | end = start + i + 1 87 | W = probs[start:end].copy() 88 | #j = sorted(range(i + 1), key=lambda x: -max(W[x][k] for k in range(len(W[x])) if k != PRIMITIVES.index('none')))[0] 89 | j = sorted(range(i + 1), key=lambda x: -max(W[x][k] for k in range(len(W[x])) ))[0] 90 | k_best = None 91 | for k in range(len(W[j])): 92 | #if k != PRIMITIVES.index('none'): 93 | # if k_best is None or W[j][k] > W[j][k_best]: 94 | # k_best = k 95 | if k_best is None or W[j][k] > W[j][k_best]: 96 | k_best = k 97 | gene.append((PRIMITIVES[k_best], j)) 98 | start = end 99 | return gene 100 | 101 | with torch.no_grad(): 102 | gene = _parse(F.softmax(self.arch_weights, dim=-1).cpu().numpy()) 103 | genotype = Genotype(recurrent=gene, concat=list(range(STEPS+1)[-CONCAT:])) 104 | return genotype 105 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/nas_rnn/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import os, shutil 4 | import numpy as np 5 | 6 | 7 | def repackage_hidden(h): 8 | if isinstance(h, torch.Tensor): 9 | return h.detach() 10 | else: 11 | return tuple(repackage_hidden(v) for v in h) 12 | 13 | 14 | def batchify(data, bsz, use_cuda): 15 | nbatch = data.size(0) // bsz 16 | data = data.narrow(0, 0, nbatch * bsz) 17 | data = data.view(bsz, -1).t().contiguous() 18 | if use_cuda: return data.cuda() 19 | else : return data 20 | 21 | 22 | def get_batch(source, i, seq_len): 23 | seq_len = min(seq_len, len(source) - 1 - i) 24 | data = source[i:i+seq_len].clone() 25 | target = source[i+1:i+1+seq_len].clone() 26 | return data, target 27 | 28 | 29 | 30 | def embedded_dropout(embed, words, dropout=0.1, scale=None): 31 | if dropout: 32 | mask = embed.weight.data.new().resize_((embed.weight.size(0), 1)).bernoulli_(1 - dropout).expand_as(embed.weight) / (1 - dropout) 33 | mask.requires_grad_(True) 34 | masked_embed_weight = mask * embed.weight 35 | else: 36 | masked_embed_weight = embed.weight 37 | if scale: 38 | masked_embed_weight = scale.expand_as(masked_embed_weight) * masked_embed_weight 39 | 40 | padding_idx = embed.padding_idx 41 | if padding_idx is None: 42 | padding_idx = -1 43 | X = torch.nn.functional.embedding( 44 | words, masked_embed_weight, 45 | padding_idx, embed.max_norm, embed.norm_type, 46 | embed.scale_grad_by_freq, embed.sparse) 47 | return X 48 | 49 | 50 | class LockedDropout(nn.Module): 51 | def __init__(self): 52 | super(LockedDropout, self).__init__() 53 | 54 | def forward(self, x, dropout=0.5): 55 | if not self.training or not dropout: 56 | return x 57 | m = x.data.new(1, x.size(1), x.size(2)).bernoulli_(1 - dropout) 58 | mask = m.div_(1 - dropout).detach() 59 | mask = mask.expand_as(x) 60 | return mask * x 61 | 62 | 63 | def mask2d(B, D, keep_prob, cuda=True): 64 | m = torch.floor(torch.rand(B, D) + keep_prob) / keep_prob 65 | if cuda: return m.cuda() 66 | else : return m 67 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .utils import load_config 5 | from .scheduler import MultiStepLR, obtain_scheduler 6 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/scheduler/scheduler.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import torch 5 | from bisect import bisect_right 6 | 7 | 8 | class MultiStepLR(torch.optim.lr_scheduler._LRScheduler): 9 | 10 | def __init__(self, optimizer, milestones, gammas, last_epoch=-1): 11 | if not list(milestones) == sorted(milestones): 12 | raise ValueError('Milestones should be a list of' 13 | ' increasing integers. Got {:}', milestones) 14 | assert len(milestones) == len(gammas), '{:} vs {:}'.format(milestones, gammas) 15 | self.milestones = milestones 16 | self.gammas = gammas 17 | super(MultiStepLR, self).__init__(optimizer, last_epoch) 18 | 19 | def get_lr(self): 20 | LR = 1 21 | for x in self.gammas[:bisect_right(self.milestones, self.last_epoch)]: LR = LR * x 22 | return [base_lr * LR for base_lr in self.base_lrs] 23 | 24 | 25 | def obtain_scheduler(config, optimizer): 26 | if config.type == 'multistep': 27 | scheduler = MultiStepLR(optimizer, milestones=config.milestones, gammas=config.gammas) 28 | elif config.type == 'cosine': 29 | scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, config.epochs) 30 | else: 31 | raise ValueError('Unknown learning rate scheduler type : {:}'.format(config.type)) 32 | return scheduler 33 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/scheduler/utils.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, json 5 | from pathlib import Path 6 | from collections import namedtuple 7 | 8 | support_types = ('str', 'int', 'bool', 'float') 9 | 10 | def convert_param(original_lists): 11 | assert isinstance(original_lists, list), 'The type is not right : {:}'.format(original_lists) 12 | ctype, value = original_lists[0], original_lists[1] 13 | assert ctype in support_types, 'Ctype={:}, support={:}'.format(ctype, support_types) 14 | is_list = isinstance(value, list) 15 | if not is_list: value = [value] 16 | outs = [] 17 | for x in value: 18 | if ctype == 'int': 19 | x = int(x) 20 | elif ctype == 'str': 21 | x = str(x) 22 | elif ctype == 'bool': 23 | x = bool(int(x)) 24 | elif ctype == 'float': 25 | x = float(x) 26 | else: 27 | raise TypeError('Does not know this type : {:}'.format(ctype)) 28 | outs.append(x) 29 | if not is_list: outs = outs[0] 30 | return outs 31 | 32 | def load_config(path): 33 | path = str(path) 34 | assert os.path.exists(path), 'Can not find {:}'.format(path) 35 | # Reading data back 36 | with open(path, 'r') as f: 37 | data = json.load(f) 38 | f.close() 39 | content = { k: convert_param(v) for k,v in data.items()} 40 | Arguments = namedtuple('Configure', ' '.join(content.keys())) 41 | content = Arguments(**content) 42 | return content 43 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .utils import AverageMeter, RecorderMeter, convert_secs2time 5 | from .utils import time_file_str, time_string 6 | from .utils import test_imagenet_data 7 | from .utils import print_log 8 | from .evaluation_utils import obtain_accuracy 9 | #from .draw_pts import draw_points 10 | from .gpu_manager import GPUManager 11 | 12 | from .save_meta import Save_Meta 13 | 14 | from .model_utils import count_parameters_in_MB 15 | from .model_utils import Cutout 16 | from .flop_benchmark import print_FLOPs 17 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/draw_pts.py: -------------------------------------------------------------------------------- 1 | import os, sys, time 2 | import numpy as np 3 | import matplotlib 4 | import random 5 | matplotlib.use('agg') 6 | import matplotlib.pyplot as plt 7 | import matplotlib.cm as cm 8 | 9 | def draw_points(points, labels, save_path): 10 | title = 'the visualized features' 11 | dpi = 100 12 | width, height = 1000, 1000 13 | legend_fontsize = 10 14 | figsize = width / float(dpi), height / float(dpi) 15 | fig = plt.figure(figsize=figsize) 16 | 17 | classes = np.unique(labels).tolist() 18 | colors = cm.rainbow(np.linspace(0, 1, len(classes))) 19 | 20 | legends = [] 21 | legendnames = [] 22 | 23 | for cls, c in zip(classes, colors): 24 | 25 | indexes = labels == cls 26 | ptss = points[indexes, :] 27 | x = ptss[:,0] 28 | y = ptss[:,1] 29 | if cls % 2 == 0: marker = 'x' 30 | else: marker = 'o' 31 | legend = plt.scatter(x, y, color=c, s=1, marker=marker) 32 | legendname = '{:02d}'.format(cls+1) 33 | legends.append( legend ) 34 | legendnames.append( legendname ) 35 | 36 | plt.legend(legends, legendnames, scatterpoints=1, ncol=5, fontsize=8) 37 | 38 | if save_path is not None: 39 | fig.savefig(save_path, dpi=dpi, bbox_inches='tight') 40 | print ('---- save figure {} into {}'.format(title, save_path)) 41 | plt.close(fig) 42 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/evaluation_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def obtain_accuracy(output, target, topk=(1,)): 4 | """Computes the precision@k for the specified values of k""" 5 | maxk = max(topk) 6 | batch_size = target.size(0) 7 | 8 | _, pred = output.topk(maxk, 1, True, True) 9 | pred = pred.t() 10 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 11 | 12 | res = [] 13 | for k in topk: 14 | correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 15 | res.append(correct_k.mul_(100.0 / batch_size)) 16 | return res 17 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/flop_benchmark.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | # modified from https://github.com/warmspringwinds/pytorch-segmentation-detection/blob/master/pytorch_segmentation_detection/utils/flops_benchmark.py 5 | import copy, torch 6 | 7 | def print_FLOPs(model, shape, logs): 8 | print_log, log = logs 9 | model = copy.deepcopy( model ) 10 | 11 | model = add_flops_counting_methods(model) 12 | model = model.cuda() 13 | model.eval() 14 | 15 | cache_inputs = torch.zeros(*shape).cuda() 16 | #print_log('In the calculating function : cache input size : {:}'.format(cache_inputs.size()), log) 17 | _ = model(cache_inputs) 18 | FLOPs = compute_average_flops_cost( model ) / 1e6 19 | print_log('FLOPs : {:} MB'.format(FLOPs), log) 20 | torch.cuda.empty_cache() 21 | 22 | 23 | # ---- Public functions 24 | def add_flops_counting_methods( model ): 25 | model.__batch_counter__ = 0 26 | add_batch_counter_hook_function( model ) 27 | model.apply( add_flops_counter_variable_or_reset ) 28 | model.apply( add_flops_counter_hook_function ) 29 | return model 30 | 31 | 32 | 33 | def compute_average_flops_cost(model): 34 | """ 35 | A method that will be available after add_flops_counting_methods() is called on a desired net object. 36 | Returns current mean flops consumption per image. 37 | """ 38 | batches_count = model.__batch_counter__ 39 | flops_sum = 0 40 | for module in model.modules(): 41 | if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear): 42 | flops_sum += module.__flops__ 43 | return flops_sum / batches_count 44 | 45 | 46 | # ---- Internal functions 47 | def pool_flops_counter_hook(pool_module, inputs, output): 48 | batch_size = inputs[0].size(0) 49 | kernel_size = pool_module.kernel_size 50 | out_C, output_height, output_width = output.shape[1:] 51 | assert out_C == inputs[0].size(1), '{:} vs. {:}'.format(out_C, inputs[0].size()) 52 | 53 | overall_flops = batch_size * out_C * output_height * output_width * kernel_size * kernel_size 54 | pool_module.__flops__ += overall_flops 55 | 56 | 57 | def fc_flops_counter_hook(fc_module, inputs, output): 58 | batch_size = inputs[0].size(0) 59 | xin, xout = fc_module.in_features, fc_module.out_features 60 | assert xin == inputs[0].size(1) and xout == output.size(1), 'IO=({:}, {:})'.format(xin, xout) 61 | overall_flops = batch_size * xin * xout 62 | if fc_module.bias is not None: 63 | overall_flops += batch_size * xout 64 | fc_module.__flops__ += overall_flops 65 | 66 | 67 | def conv_flops_counter_hook(conv_module, inputs, output): 68 | batch_size = inputs[0].size(0) 69 | output_height, output_width = output.shape[2:] 70 | 71 | kernel_height, kernel_width = conv_module.kernel_size 72 | in_channels = conv_module.in_channels 73 | out_channels = conv_module.out_channels 74 | groups = conv_module.groups 75 | conv_per_position_flops = kernel_height * kernel_width * in_channels * out_channels / groups 76 | 77 | active_elements_count = batch_size * output_height * output_width 78 | overall_flops = conv_per_position_flops * active_elements_count 79 | 80 | if conv_module.bias is not None: 81 | overall_flops += out_channels * active_elements_count 82 | conv_module.__flops__ += overall_flops 83 | 84 | 85 | def batch_counter_hook(module, inputs, output): 86 | # Can have multiple inputs, getting the first one 87 | inputs = inputs[0] 88 | batch_size = inputs.shape[0] 89 | module.__batch_counter__ += batch_size 90 | 91 | 92 | def add_batch_counter_hook_function(module): 93 | if not hasattr(module, '__batch_counter_handle__'): 94 | handle = module.register_forward_hook(batch_counter_hook) 95 | module.__batch_counter_handle__ = handle 96 | 97 | 98 | def add_flops_counter_variable_or_reset(module): 99 | if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear) \ 100 | or isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d): 101 | module.__flops__ = 0 102 | 103 | 104 | def add_flops_counter_hook_function(module): 105 | if isinstance(module, torch.nn.Conv2d): 106 | if not hasattr(module, '__flops_handle__'): 107 | handle = module.register_forward_hook(conv_flops_counter_hook) 108 | module.__flops_handle__ = handle 109 | elif isinstance(module, torch.nn.Linear): 110 | if not hasattr(module, '__flops_handle__'): 111 | handle = module.register_forward_hook(fc_flops_counter_hook) 112 | module.__flops_handle__ = handle 113 | elif isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d): 114 | if not hasattr(module, '__flops_handle__'): 115 | handle = module.register_forward_hook(pool_flops_counter_hook) 116 | module.__flops_handle__ = handle 117 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/gpu_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | class GPUManager(): 4 | queries = ('index', 'gpu_name', 'memory.free', 'memory.used', 'memory.total', 'power.draw', 'power.limit') 5 | 6 | def __init__(self): 7 | all_gpus = self.query_gpu(False) 8 | 9 | def get_info(self, ctype): 10 | cmd = 'nvidia-smi --query-gpu={} --format=csv,noheader'.format(ctype) 11 | lines = os.popen(cmd).readlines() 12 | lines = [line.strip('\n') for line in lines] 13 | return lines 14 | 15 | def query_gpu(self, show=True): 16 | num_gpus = len( self.get_info('index') ) 17 | all_gpus = [ {} for i in range(num_gpus) ] 18 | for query in self.queries: 19 | infos = self.get_info(query) 20 | for idx, info in enumerate(infos): 21 | all_gpus[idx][query] = info 22 | 23 | if 'CUDA_VISIBLE_DEVICES' in os.environ: 24 | CUDA_VISIBLE_DEVICES = os.environ['CUDA_VISIBLE_DEVICES'].split(',') 25 | selected_gpus = [] 26 | for idx, CUDA_VISIBLE_DEVICE in enumerate(CUDA_VISIBLE_DEVICES): 27 | find = False 28 | for gpu in all_gpus: 29 | if gpu['index'] == CUDA_VISIBLE_DEVICE: 30 | assert find==False, 'Duplicate cuda device index : {}'.format(CUDA_VISIBLE_DEVICE) 31 | find = True 32 | selected_gpus.append( gpu.copy() ) 33 | selected_gpus[-1]['index'] = '{}'.format(idx) 34 | assert find, 'Does not find the device : {}'.format(CUDA_VISIBLE_DEVICE) 35 | all_gpus = selected_gpus 36 | 37 | if show: 38 | allstrings = '' 39 | for gpu in all_gpus: 40 | string = '| ' 41 | for query in self.queries: 42 | if query.find('memory') == 0: xinfo = '{:>9}'.format(gpu[query]) 43 | else: xinfo = gpu[query] 44 | string = string + query + ' : ' + xinfo + ' | ' 45 | allstrings = allstrings + string + '\n' 46 | return allstrings 47 | else: 48 | return all_gpus 49 | 50 | def select_by_memory(self, numbers=1): 51 | all_gpus = self.query_gpu(False) 52 | assert numbers <= len(all_gpus), 'Require {} gpus more than you have'.format(numbers) 53 | alls = [] 54 | for idx, gpu in enumerate(all_gpus): 55 | free_memory = gpu['memory.free'] 56 | free_memory = free_memory.split(' ')[0] 57 | free_memory = int(free_memory) 58 | index = gpu['index'] 59 | alls.append((free_memory, index)) 60 | alls.sort(reverse = True) 61 | alls = [ int(alls[i][1]) for i in range(numbers) ] 62 | return sorted(alls) 63 | 64 | """ 65 | if __name__ == '__main__': 66 | manager = GPUManager() 67 | manager.query_gpu(True) 68 | indexes = manager.select_by_memory(3) 69 | print (indexes) 70 | """ 71 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/model_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | 5 | 6 | def count_parameters_in_MB(model): 7 | if isinstance(model, nn.Module): 8 | return np.sum(np.prod(v.size()) for v in model.parameters())/1e6 9 | else: 10 | return np.sum(np.prod(v.size()) for v in model)/1e6 11 | 12 | 13 | class Cutout(object): 14 | def __init__(self, length): 15 | self.length = length 16 | 17 | def __repr__(self): 18 | return ('{name}(length={length})'.format(name=self.__class__.__name__, **self.__dict__)) 19 | 20 | def __call__(self, img): 21 | h, w = img.size(1), img.size(2) 22 | mask = np.ones((h, w), np.float32) 23 | y = np.random.randint(h) 24 | x = np.random.randint(w) 25 | 26 | y1 = np.clip(y - self.length // 2, 0, h) 27 | y2 = np.clip(y + self.length // 2, 0, h) 28 | x1 = np.clip(x - self.length // 2, 0, w) 29 | x2 = np.clip(x + self.length // 2, 0, w) 30 | 31 | mask[y1: y2, x1: x2] = 0. 32 | mask = torch.from_numpy(mask) 33 | mask = mask.expand_as(img) 34 | img *= mask 35 | return img 36 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/save_meta.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import torch 5 | import os, sys 6 | import os.path as osp 7 | import numpy as np 8 | 9 | def tensor2np(x): 10 | if isinstance(x, np.ndarray): return x 11 | if x.is_cuda: x = x.cpu() 12 | return x.numpy() 13 | 14 | class Save_Meta(): 15 | 16 | def __init__(self): 17 | self.reset() 18 | 19 | def __repr__(self): 20 | return ('{name}'.format(name=self.__class__.__name__)+'(number of data = {})'.format(len(self))) 21 | 22 | def reset(self): 23 | self.predictions = [] 24 | self.groundtruth = [] 25 | 26 | def __len__(self): 27 | return len(self.predictions) 28 | 29 | def append(self, _pred, _ground): 30 | _pred, _ground = tensor2np(_pred), tensor2np(_ground) 31 | assert _ground.shape[0] == _pred.shape[0] and len(_pred.shape) == 2 and len(_ground.shape) == 1, 'The shapes are wrong : {} & {}'.format(_pred.shape, _ground.shape) 32 | self.predictions.append(_pred) 33 | self.groundtruth.append(_ground) 34 | 35 | def save(self, save_dir, filename, test=True): 36 | meta = {'predictions': self.predictions, 37 | 'groundtruth': self.groundtruth} 38 | filename = osp.join(save_dir, filename) 39 | torch.save(meta, filename) 40 | if test: 41 | predictions = np.concatenate(self.predictions) 42 | groundtruth = np.concatenate(self.groundtruth) 43 | predictions = np.argmax(predictions, axis=1) 44 | accuracy = np.sum(groundtruth==predictions) * 100.0 / predictions.size 45 | else: 46 | accuracy = None 47 | print ('save save_meta into {} with accuracy = {}'.format(filename, accuracy)) 48 | 49 | def load(self, filename): 50 | assert os.path.isfile(filename), '{} is not a file'.format(filename) 51 | checkpoint = torch.load(filename) 52 | self.predictions = checkpoint['predictions'] 53 | self.groundtruth = checkpoint['groundtruth'] 54 | -------------------------------------------------------------------------------- /models/cifar/gdas/lib/utils/utils.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, time 5 | import numpy as np 6 | import random 7 | 8 | class AverageMeter(object): 9 | """Computes and stores the average and current value""" 10 | def __init__(self): 11 | self.reset() 12 | 13 | def reset(self): 14 | self.val = 0 15 | self.avg = 0 16 | self.sum = 0 17 | self.count = 0 18 | 19 | def update(self, val, n=1): 20 | self.val = val 21 | self.sum += val * n 22 | self.count += n 23 | self.avg = self.sum / self.count 24 | 25 | 26 | class RecorderMeter(object): 27 | """Computes and stores the minimum loss value and its epoch index""" 28 | def __init__(self, total_epoch): 29 | self.reset(total_epoch) 30 | 31 | def reset(self, total_epoch): 32 | assert total_epoch > 0 33 | self.total_epoch = total_epoch 34 | self.current_epoch = 0 35 | self.epoch_losses = np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] 36 | self.epoch_losses = self.epoch_losses - 1 37 | 38 | self.epoch_accuracy= np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] 39 | self.epoch_accuracy= self.epoch_accuracy 40 | 41 | def update(self, idx, train_loss, train_acc, val_loss, val_acc): 42 | assert idx >= 0 and idx < self.total_epoch, 'total_epoch : {} , but update with the {} index'.format(self.total_epoch, idx) 43 | self.epoch_losses [idx, 0] = train_loss 44 | self.epoch_losses [idx, 1] = val_loss 45 | self.epoch_accuracy[idx, 0] = train_acc 46 | self.epoch_accuracy[idx, 1] = val_acc 47 | self.current_epoch = idx + 1 48 | return self.max_accuracy(False) == self.epoch_accuracy[idx, 1] 49 | 50 | def max_accuracy(self, istrain): 51 | if self.current_epoch <= 0: return 0 52 | if istrain: return self.epoch_accuracy[:self.current_epoch, 0].max() 53 | else: return self.epoch_accuracy[:self.current_epoch, 1].max() 54 | 55 | def plot_curve(self, save_path): 56 | import matplotlib 57 | matplotlib.use('agg') 58 | import matplotlib.pyplot as plt 59 | title = 'the accuracy/loss curve of train/val' 60 | dpi = 100 61 | width, height = 1600, 1000 62 | legend_fontsize = 10 63 | figsize = width / float(dpi), height / float(dpi) 64 | 65 | fig = plt.figure(figsize=figsize) 66 | x_axis = np.array([i for i in range(self.total_epoch)]) # epochs 67 | y_axis = np.zeros(self.total_epoch) 68 | 69 | plt.xlim(0, self.total_epoch) 70 | plt.ylim(0, 100) 71 | interval_y = 5 72 | interval_x = 5 73 | plt.xticks(np.arange(0, self.total_epoch + interval_x, interval_x)) 74 | plt.yticks(np.arange(0, 100 + interval_y, interval_y)) 75 | plt.grid() 76 | plt.title(title, fontsize=20) 77 | plt.xlabel('the training epoch', fontsize=16) 78 | plt.ylabel('accuracy', fontsize=16) 79 | 80 | y_axis[:] = self.epoch_accuracy[:, 0] 81 | plt.plot(x_axis, y_axis, color='g', linestyle='-', label='train-accuracy', lw=2) 82 | plt.legend(loc=4, fontsize=legend_fontsize) 83 | 84 | y_axis[:] = self.epoch_accuracy[:, 1] 85 | plt.plot(x_axis, y_axis, color='y', linestyle='-', label='valid-accuracy', lw=2) 86 | plt.legend(loc=4, fontsize=legend_fontsize) 87 | 88 | 89 | y_axis[:] = self.epoch_losses[:, 0] 90 | plt.plot(x_axis, y_axis*50, color='g', linestyle=':', label='train-loss-x50', lw=2) 91 | plt.legend(loc=4, fontsize=legend_fontsize) 92 | 93 | y_axis[:] = self.epoch_losses[:, 1] 94 | plt.plot(x_axis, y_axis*50, color='y', linestyle=':', label='valid-loss-x50', lw=2) 95 | plt.legend(loc=4, fontsize=legend_fontsize) 96 | 97 | if save_path is not None: 98 | fig.savefig(save_path, dpi=dpi, bbox_inches='tight') 99 | print ('---- save figure {} into {}'.format(title, save_path)) 100 | plt.close(fig) 101 | 102 | def print_log(print_string, log): 103 | print ("{:}".format(print_string)) 104 | if log is not None: 105 | log.write('{}\n'.format(print_string)) 106 | log.flush() 107 | 108 | def time_file_str(): 109 | ISOTIMEFORMAT='%Y-%m-%d' 110 | string = '{}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) 111 | return string + '-{}'.format(random.randint(1, 10000)) 112 | 113 | def time_string(): 114 | ISOTIMEFORMAT='%Y-%m-%d-%X' 115 | string = '[{}]'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) 116 | return string 117 | 118 | def convert_secs2time(epoch_time, return_str=False): 119 | need_hour = int(epoch_time / 3600) 120 | need_mins = int((epoch_time - 3600*need_hour) / 60) 121 | need_secs = int(epoch_time - 3600*need_hour - 60*need_mins) 122 | if return_str == False: 123 | return need_hour, need_mins, need_secs 124 | else: 125 | return '[Need: {:02d}:{:02d}:{:02d}]'.format(need_hour, need_mins, need_secs) 126 | 127 | def test_imagenet_data(imagenet): 128 | total_length = len(imagenet) 129 | assert total_length == 1281166 or total_length == 50000, 'The length of ImageNet is wrong : {}'.format(total_length) 130 | map_id = {} 131 | for index in range(total_length): 132 | path, target = imagenet.imgs[index] 133 | folder, image_name = os.path.split(path) 134 | _, folder = os.path.split(folder) 135 | if folder not in map_id: 136 | map_id[folder] = target 137 | else: 138 | assert map_id[folder] == target, 'Class : {} is not {}'.format(folder, target) 139 | assert image_name.find(folder) == 0, '{} is wrong.'.format(path) 140 | print ('Check ImageNet Dataset OK') 141 | -------------------------------------------------------------------------------- /models/cifar/pyramidnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import math 4 | 5 | __all__ = ['pyramidnet272'] 6 | 7 | 8 | def conv3x3(in_planes, out_planes, stride=1): 9 | "3x3 convolution with padding" 10 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 11 | padding=1, bias=False) 12 | 13 | 14 | def calc_prob(curr_layer, total_layers, p_l): 15 | """Calculates drop prob depending on the current layer.""" 16 | return 1 - (float(curr_layer) / total_layers) * p_l 17 | 18 | 19 | class Bottleneck(nn.Module): 20 | outchannel_ratio = 4 21 | 22 | def __init__(self, inplanes, planes, stride=1, downsample=None, prob=1.): 23 | super(Bottleneck, self).__init__() 24 | self.bn1 = nn.BatchNorm2d(inplanes) 25 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 26 | self.bn2 = nn.BatchNorm2d(planes) 27 | if stride == 1: 28 | self.conv2 = nn.Conv2d(planes, (planes * 1), kernel_size=3, stride=stride, 29 | padding=1, bias=False) 30 | else: 31 | self.conv2 = nn.Sequential(nn.ZeroPad2d((0, 1, 0, 1)), 32 | nn.Conv2d(planes, (planes * 1), kernel_size=3, stride=stride, 33 | padding=0, bias=False)) 34 | self.bn3 = nn.BatchNorm2d((planes * 1)) 35 | self.conv3 = nn.Conv2d((planes * 1), planes * Bottleneck.outchannel_ratio, kernel_size=1, bias=False) 36 | self.bn4 = nn.BatchNorm2d(planes * Bottleneck.outchannel_ratio) 37 | self.relu = nn.ReLU(inplace=True) 38 | self.downsample = downsample 39 | self.stride = stride 40 | self.prob = prob 41 | self.padding = None 42 | 43 | def forward(self, x): 44 | 45 | out = self.bn1(x) 46 | out = self.conv1(out) 47 | 48 | out = self.bn2(out) 49 | out = self.relu(out) 50 | out = self.conv2(out) 51 | 52 | out = self.bn3(out) 53 | out = self.relu(out) 54 | out = self.conv3(out) 55 | 56 | out = self.bn4(out) 57 | 58 | # shake drop inference 59 | # we may support shake drop training in a future version 60 | assert not self.training 61 | out = out * self.prob 62 | 63 | if self.downsample is not None: 64 | shortcut = self.downsample(x) 65 | featuremap_size = shortcut.size()[2:4] 66 | else: 67 | shortcut = x 68 | featuremap_size = out.size()[2:4] 69 | 70 | batch_size = out.size()[0] 71 | residual_channel = out.size()[1] 72 | shortcut_channel = shortcut.size()[1] 73 | if residual_channel != shortcut_channel: 74 | if self.padding is None: 75 | self.padding = torch.zeros(batch_size, residual_channel - shortcut_channel, 76 | featuremap_size[0], featuremap_size[1]) 77 | self.padding = self.padding.to(x.device) 78 | out += torch.cat((shortcut, self.padding), 1) 79 | else: 80 | out += shortcut 81 | 82 | return out 83 | 84 | 85 | class PyramidNet(nn.Module): 86 | 87 | def __init__(self, depth, alpha, num_classes): 88 | super(PyramidNet, self).__init__() 89 | self.inplanes = 16 90 | n = int((depth - 2) / 9) 91 | block = Bottleneck 92 | 93 | self.addrate = alpha / (3 * n * 1.0) 94 | 95 | self.input_featuremap_dim = self.inplanes 96 | self.conv1 = nn.Conv2d(3, self.input_featuremap_dim, kernel_size=3, stride=1, padding=1, bias=False) 97 | self.bn1 = nn.BatchNorm2d(self.input_featuremap_dim) 98 | 99 | self.featuremap_dim = self.input_featuremap_dim 100 | 101 | self.p_l = 0.5 102 | self.layer_num = 1 103 | self.total_layers = n * 3 104 | 105 | self.layer1 = self.pyramidal_make_layer(block, n) 106 | self.layer2 = self.pyramidal_make_layer(block, n, stride=2) 107 | self.layer3 = self.pyramidal_make_layer(block, n, stride=2) 108 | 109 | self.final_featuremap_dim = self.input_featuremap_dim 110 | self.bn_final = nn.BatchNorm2d(self.final_featuremap_dim) 111 | self.relu_final = nn.ReLU(inplace=True) 112 | self.avgpool = nn.AvgPool2d(8) 113 | self.fc = nn.Linear(self.final_featuremap_dim, num_classes) 114 | 115 | for m in self.modules(): 116 | if isinstance(m, nn.Conv2d): 117 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 118 | m.weight.data.normal_(0, math.sqrt(2. / n)) 119 | elif isinstance(m, nn.BatchNorm2d): 120 | m.weight.data.fill_(1) 121 | m.bias.data.zero_() 122 | 123 | def pyramidal_make_layer(self, block, block_depth, stride=1): 124 | downsample = None 125 | if stride != 1: # or self.inplanes != int(round(featuremap_dim_1st)) * block.outchannel_ratio: 126 | downsample = nn.AvgPool2d((2, 2), stride=(2, 2), ceil_mode=True) 127 | 128 | layers = [] 129 | self.featuremap_dim = self.featuremap_dim + self.addrate 130 | prob = calc_prob(self.layer_num, self.total_layers, self.p_l) 131 | layers.append(block(self.input_featuremap_dim, int(round(self.featuremap_dim)), stride, downsample, prob)) 132 | self.layer_num += 1 133 | for i in range(1, block_depth): 134 | temp_featuremap_dim = self.featuremap_dim + self.addrate 135 | prob = calc_prob(self.layer_num, self.total_layers, self.p_l) 136 | layers.append( 137 | block(int(round(self.featuremap_dim)) * block.outchannel_ratio, int(round(temp_featuremap_dim)), 1, 138 | prob=prob)) 139 | self.layer_num += 1 140 | self.featuremap_dim = temp_featuremap_dim 141 | self.input_featuremap_dim = int(round(self.featuremap_dim)) * block.outchannel_ratio 142 | 143 | return nn.Sequential(*layers) 144 | 145 | def forward(self, x): 146 | x = self.conv1(x) 147 | x = self.bn1(x) 148 | 149 | x = self.layer1(x) 150 | x = self.layer2(x) 151 | x = self.layer3(x) 152 | 153 | x = self.bn_final(x) 154 | x = self.relu_final(x) 155 | x = self.avgpool(x) 156 | x = x.view(x.size(0), -1) 157 | x = self.fc(x) 158 | 159 | return x 160 | 161 | 162 | def pyramidnet272(**kwargs): 163 | return PyramidNet(depth=272, alpha=200, **kwargs) 164 | -------------------------------------------------------------------------------- /models/cifar/utils.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | 4 | 5 | class DropoutConv2d(nn.Conv2d): 6 | def __init__(self, in_channels, out_channels, kernel_size, stride=1, 7 | padding=0, dilation=1, groups=1, bias=True, drop=0): 8 | super(DropoutConv2d, self).__init__( 9 | in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias) 10 | self.drop = drop 11 | 12 | def forward(self, x): 13 | x = super(DropoutConv2d, self).forward(x) 14 | x = F.dropout(x, p=self.drop, training=True) 15 | return x 16 | -------------------------------------------------------------------------------- /models/cifar/vgg.py: -------------------------------------------------------------------------------- 1 | '''VGG for CIFAR10. FC layers are removed. 2 | (c) YANG, Wei 3 | ''' 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from .utils import DropoutConv2d 7 | import math 8 | 9 | 10 | __all__ = [ 11 | 'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', 12 | 'vgg19_bn', 'vgg19', 13 | ] 14 | 15 | 16 | model_urls = { 17 | 'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth', 18 | 'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth', 19 | 'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth', 20 | 'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', 21 | } 22 | 23 | 24 | class VGG(nn.Module): 25 | 26 | def __init__(self, features, num_classes=1000): 27 | super(VGG, self).__init__() 28 | self.features = features 29 | self.classifier = nn.Linear(512, num_classes) 30 | self._initialize_weights() 31 | 32 | # no dropout 33 | self.drop = 0 34 | 35 | def forward(self, x): 36 | for m in self.modules(): 37 | if isinstance(m, DropoutConv2d): 38 | m.drop = self.drop 39 | 40 | x = self.features(x) 41 | x = x.view(x.size(0), -1) 42 | x = F.dropout(x, p=self.drop, training=True) # force dropout 43 | x = self.classifier(x) 44 | return x 45 | 46 | def _initialize_weights(self): 47 | for m in self.modules(): 48 | if isinstance(m, DropoutConv2d): 49 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 50 | m.weight.data.normal_(0, math.sqrt(2. / n)) 51 | if m.bias is not None: 52 | m.bias.data.zero_() 53 | elif isinstance(m, nn.BatchNorm2d): 54 | m.weight.data.fill_(1) 55 | m.bias.data.zero_() 56 | elif isinstance(m, nn.Linear): 57 | n = m.weight.size(1) 58 | m.weight.data.normal_(0, 0.01) 59 | m.bias.data.zero_() 60 | 61 | 62 | def make_layers(cfg, batch_norm=False): 63 | layers = [] 64 | in_channels = 3 65 | for v in cfg: 66 | if v == 'M': 67 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 68 | else: 69 | conv2d = DropoutConv2d(in_channels, v, kernel_size=3, padding=1) 70 | if batch_norm: 71 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 72 | else: 73 | layers += [conv2d, nn.ReLU(inplace=True)] 74 | in_channels = v 75 | return nn.Sequential(*layers) 76 | 77 | 78 | cfg = { 79 | 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 80 | 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 81 | 'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 82 | 'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 83 | } 84 | 85 | 86 | def vgg11(**kwargs): 87 | """VGG 11-layer model (configuration "A") 88 | 89 | Args: 90 | pretrained (bool): If True, returns a model pre-trained on ImageNet 91 | """ 92 | model = VGG(make_layers(cfg['A']), **kwargs) 93 | return model 94 | 95 | 96 | def vgg11_bn(**kwargs): 97 | """VGG 11-layer model (configuration "A") with batch normalization""" 98 | model = VGG(make_layers(cfg['A'], batch_norm=True), **kwargs) 99 | return model 100 | 101 | 102 | def vgg13(**kwargs): 103 | """VGG 13-layer model (configuration "B") 104 | 105 | Args: 106 | pretrained (bool): If True, returns a model pre-trained on ImageNet 107 | """ 108 | model = VGG(make_layers(cfg['B']), **kwargs) 109 | return model 110 | 111 | 112 | def vgg13_bn(**kwargs): 113 | """VGG 13-layer model (configuration "B") with batch normalization""" 114 | model = VGG(make_layers(cfg['B'], batch_norm=True), **kwargs) 115 | return model 116 | 117 | 118 | def vgg16(**kwargs): 119 | """VGG 16-layer model (configuration "D") 120 | 121 | Args: 122 | pretrained (bool): If True, returns a model pre-trained on ImageNet 123 | """ 124 | model = VGG(make_layers(cfg['D']), **kwargs) 125 | return model 126 | 127 | 128 | def vgg16_bn(**kwargs): 129 | """VGG 16-layer model (configuration "D") with batch normalization""" 130 | model = VGG(make_layers(cfg['D'], batch_norm=True), **kwargs) 131 | return model 132 | 133 | 134 | def vgg19(**kwargs): 135 | """VGG 19-layer model (configuration "E") 136 | 137 | Args: 138 | pretrained (bool): If True, returns a model pre-trained on ImageNet 139 | """ 140 | model = VGG(make_layers(cfg['E']), **kwargs) 141 | return model 142 | 143 | 144 | def vgg19_bn(**kwargs): 145 | """VGG 19-layer model (configuration 'E') with batch normalization""" 146 | model = VGG(make_layers(cfg['E'], batch_norm=True), **kwargs) 147 | return model 148 | -------------------------------------------------------------------------------- /models/cifar/wrn.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from .utils import DropoutConv2d 6 | import random 7 | 8 | __all__ = ['wrn'] 9 | 10 | 11 | class BasicBlock(nn.Module): 12 | def __init__(self, in_planes, out_planes, stride, dropRate=0.0): 13 | super(BasicBlock, self).__init__() 14 | self.bn1 = nn.BatchNorm2d(in_planes) 15 | self.relu1 = nn.ReLU(inplace=True) 16 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 17 | padding=1, bias=False) 18 | self.bn2 = nn.BatchNorm2d(out_planes) 19 | self.relu2 = nn.ReLU(inplace=True) 20 | self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, 21 | padding=1, bias=False) 22 | self.droprate = dropRate 23 | self.equalInOut = (in_planes == out_planes) 24 | self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, 25 | padding=0, bias=False) or None 26 | 27 | # no drop 28 | self.drop = 0 29 | 30 | def forward(self, x): 31 | if self.equalInOut and (random.random() < self.drop): 32 | return x 33 | else: 34 | if not self.equalInOut: 35 | x = self.relu1(self.bn1(x)) 36 | else: 37 | out = self.relu1(self.bn1(x)) 38 | out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) 39 | if self.droprate > 0: 40 | out = F.dropout(out, p=self.droprate, training=self.training) 41 | out = self.conv2(out) 42 | return torch.add(x if self.equalInOut else self.convShortcut(x), out) 43 | 44 | class NetworkBlock(nn.Module): 45 | def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): 46 | super(NetworkBlock, self).__init__() 47 | self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) 48 | def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): 49 | layers = [] 50 | for i in range(nb_layers): 51 | layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) 52 | return nn.Sequential(*layers) 53 | def forward(self, x): 54 | return self.layer(x) 55 | 56 | class WideResNet(nn.Module): 57 | def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0): 58 | super(WideResNet, self).__init__() 59 | nChannels = [16, 16*widen_factor, 32*widen_factor, 64*widen_factor] 60 | assert (depth - 4) % 6 == 0, 'depth should be 6n+4' 61 | n = (depth - 4) // 6 62 | block = BasicBlock 63 | # 1st conv before any network block 64 | self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, 65 | padding=1, bias=False) 66 | # 1st block 67 | self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) 68 | # 2nd block 69 | self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) 70 | # 3rd block 71 | self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) 72 | # global average pooling and classifier 73 | self.bn1 = nn.BatchNorm2d(nChannels[3]) 74 | self.relu = nn.ReLU(inplace=True) 75 | self.fc = nn.Linear(nChannels[3], num_classes) 76 | self.nChannels = nChannels[3] 77 | 78 | for m in self.modules(): 79 | if isinstance(m, nn.Conv2d): 80 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 81 | m.weight.data.normal_(0, math.sqrt(2. / n)) 82 | elif isinstance(m, nn.BatchNorm2d): 83 | m.weight.data.fill_(1) 84 | m.bias.data.zero_() 85 | elif isinstance(m, nn.Linear): 86 | m.bias.data.zero_() 87 | 88 | # no drop 89 | self.drop = 0 90 | 91 | def forward(self, x): 92 | for m in self.modules(): 93 | if isinstance(m, (DropoutConv2d, BasicBlock)): 94 | m.drop = self.drop 95 | 96 | out = self.conv1(x) 97 | out = self.block1(out) 98 | out = self.block2(out) 99 | out = self.block3(out) 100 | out = self.relu(self.bn1(out)) 101 | out = F.avg_pool2d(out, 8) 102 | out = out.view(-1, self.nChannels) 103 | out = F.dropout(out, p=self.drop, training=True) # force dropout 104 | return self.fc(out) 105 | 106 | def wrn(**kwargs): 107 | """ 108 | Constructs a Wide Residual Networks. 109 | """ 110 | model = WideResNet(**kwargs) 111 | return model 112 | -------------------------------------------------------------------------------- /models/imagenet/LICENSE.txt: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Remi Cadene 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /models/imagenet/README.md: -------------------------------------------------------------------------------- 1 | # ImageNet CNN models 2 | This folder contains a slightly modified version of the [pretrained-models.pytorch](https://github.com/Cadene/pretrained-models.pytorch) project code (Copyright to [Remi Cadene](https://github.com/Cadene)). 3 | -------------------------------------------------------------------------------- /models/imagenet/__init__.py: -------------------------------------------------------------------------------- 1 | from models.imagenet.torchvision_models import resnet18 2 | from models.imagenet.torchvision_models import resnet34 3 | from models.imagenet.torchvision_models import resnet50 4 | from models.imagenet.torchvision_models import inceptionv3 5 | 6 | from models.imagenet.senet import senet154 7 | 8 | from models.imagenet.pnasnet import pnasnet5large 9 | 10 | -------------------------------------------------------------------------------- /models/imagenet/pnasnet.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division, absolute_import 2 | from collections import OrderedDict 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.utils.model_zoo as model_zoo 7 | 8 | 9 | pretrained_settings = { 10 | 'pnasnet5large': { 11 | 'imagenet': { 12 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/pnasnet5large-bf079911.pth', 13 | 'input_space': 'RGB', 14 | 'input_size': [3, 331, 331], 15 | 'input_range': [0, 1], 16 | 'mean': [0.5, 0.5, 0.5], 17 | 'std': [0.5, 0.5, 0.5], 18 | 'num_classes': 1000 19 | }, 20 | 'imagenet+background': { 21 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/pnasnet5large-bf079911.pth', 22 | 'input_space': 'RGB', 23 | 'input_size': [3, 331, 331], 24 | 'input_range': [0, 1], 25 | 'mean': [0.5, 0.5, 0.5], 26 | 'std': [0.5, 0.5, 0.5], 27 | 'num_classes': 1001 28 | } 29 | } 30 | } 31 | 32 | 33 | class MaxPool(nn.Module): 34 | 35 | def __init__(self, kernel_size, stride=1, padding=1, zero_pad=False): 36 | super(MaxPool, self).__init__() 37 | self.zero_pad = nn.ZeroPad2d((1, 0, 1, 0)) if zero_pad else None 38 | self.pool = nn.MaxPool2d(kernel_size, stride=stride, padding=padding) 39 | 40 | def forward(self, x): 41 | if self.zero_pad: 42 | x = self.zero_pad(x) 43 | x = self.pool(x) 44 | if self.zero_pad: 45 | x = x[:, :, 1:, 1:] 46 | return x 47 | 48 | 49 | class SeparableConv2d(nn.Module): 50 | 51 | def __init__(self, in_channels, out_channels, dw_kernel_size, dw_stride, 52 | dw_padding): 53 | super(SeparableConv2d, self).__init__() 54 | self.depthwise_conv2d = nn.Conv2d(in_channels, in_channels, 55 | kernel_size=dw_kernel_size, 56 | stride=dw_stride, padding=dw_padding, 57 | groups=in_channels, bias=False) 58 | self.pointwise_conv2d = nn.Conv2d(in_channels, out_channels, 59 | kernel_size=1, bias=False) 60 | 61 | def forward(self, x): 62 | x = self.depthwise_conv2d(x) 63 | x = self.pointwise_conv2d(x) 64 | return x 65 | 66 | 67 | class BranchSeparables(nn.Module): 68 | 69 | def __init__(self, in_channels, out_channels, kernel_size, stride=1, 70 | stem_cell=False, zero_pad=False): 71 | super(BranchSeparables, self).__init__() 72 | padding = kernel_size // 2 73 | middle_channels = out_channels if stem_cell else in_channels 74 | self.zero_pad = nn.ZeroPad2d((1, 0, 1, 0)) if zero_pad else None 75 | self.relu_1 = nn.ReLU() 76 | self.separable_1 = SeparableConv2d(in_channels, middle_channels, 77 | kernel_size, dw_stride=stride, 78 | dw_padding=padding) 79 | self.bn_sep_1 = nn.BatchNorm2d(middle_channels, eps=0.001) 80 | self.relu_2 = nn.ReLU() 81 | self.separable_2 = SeparableConv2d(middle_channels, out_channels, 82 | kernel_size, dw_stride=1, 83 | dw_padding=padding) 84 | self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001) 85 | 86 | def forward(self, x): 87 | x = self.relu_1(x) 88 | if self.zero_pad: 89 | x = self.zero_pad(x) 90 | x = self.separable_1(x) 91 | if self.zero_pad: 92 | x = x[:, :, 1:, 1:].contiguous() 93 | x = self.bn_sep_1(x) 94 | x = self.relu_2(x) 95 | x = self.separable_2(x) 96 | x = self.bn_sep_2(x) 97 | return x 98 | 99 | 100 | class ReluConvBn(nn.Module): 101 | 102 | def __init__(self, in_channels, out_channels, kernel_size, stride=1): 103 | super(ReluConvBn, self).__init__() 104 | self.relu = nn.ReLU() 105 | self.conv = nn.Conv2d(in_channels, out_channels, 106 | kernel_size=kernel_size, stride=stride, 107 | bias=False) 108 | self.bn = nn.BatchNorm2d(out_channels, eps=0.001) 109 | 110 | def forward(self, x): 111 | x = self.relu(x) 112 | x = self.conv(x) 113 | x = self.bn(x) 114 | return x 115 | 116 | 117 | class FactorizedReduction(nn.Module): 118 | 119 | def __init__(self, in_channels, out_channels): 120 | super(FactorizedReduction, self).__init__() 121 | self.relu = nn.ReLU() 122 | self.path_1 = nn.Sequential(OrderedDict([ 123 | ('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False)), 124 | ('conv', nn.Conv2d(in_channels, out_channels // 2, 125 | kernel_size=1, bias=False)), 126 | ])) 127 | self.path_2 = nn.Sequential(OrderedDict([ 128 | ('pad', nn.ZeroPad2d((0, 1, 0, 1))), 129 | ('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False)), 130 | ('conv', nn.Conv2d(in_channels, out_channels // 2, 131 | kernel_size=1, bias=False)), 132 | ])) 133 | self.final_path_bn = nn.BatchNorm2d(out_channels, eps=0.001) 134 | 135 | def forward(self, x): 136 | x = self.relu(x) 137 | 138 | x_path1 = self.path_1(x) 139 | 140 | x_path2 = self.path_2.pad(x) 141 | x_path2 = x_path2[:, :, 1:, 1:] 142 | x_path2 = self.path_2.avgpool(x_path2) 143 | x_path2 = self.path_2.conv(x_path2) 144 | 145 | out = self.final_path_bn(torch.cat([x_path1, x_path2], 1)) 146 | return out 147 | 148 | 149 | class CellBase(nn.Module): 150 | 151 | def cell_forward(self, x_left, x_right): 152 | x_comb_iter_0_left = self.comb_iter_0_left(x_left) 153 | x_comb_iter_0_right = self.comb_iter_0_right(x_left) 154 | x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right 155 | 156 | x_comb_iter_1_left = self.comb_iter_1_left(x_right) 157 | x_comb_iter_1_right = self.comb_iter_1_right(x_right) 158 | x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right 159 | 160 | x_comb_iter_2_left = self.comb_iter_2_left(x_right) 161 | x_comb_iter_2_right = self.comb_iter_2_right(x_right) 162 | x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right 163 | 164 | x_comb_iter_3_left = self.comb_iter_3_left(x_comb_iter_2) 165 | x_comb_iter_3_right = self.comb_iter_3_right(x_right) 166 | x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right 167 | 168 | x_comb_iter_4_left = self.comb_iter_4_left(x_left) 169 | if self.comb_iter_4_right: 170 | x_comb_iter_4_right = self.comb_iter_4_right(x_right) 171 | else: 172 | x_comb_iter_4_right = x_right 173 | x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right 174 | 175 | x_out = torch.cat( 176 | [x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, 177 | x_comb_iter_4], 1) 178 | return x_out 179 | 180 | 181 | class CellStem0(CellBase): 182 | 183 | def __init__(self, in_channels_left, out_channels_left, in_channels_right, 184 | out_channels_right): 185 | super(CellStem0, self).__init__() 186 | self.conv_1x1 = ReluConvBn(in_channels_right, out_channels_right, 187 | kernel_size=1) 188 | self.comb_iter_0_left = BranchSeparables(in_channels_left, 189 | out_channels_left, 190 | kernel_size=5, stride=2, 191 | stem_cell=True) 192 | self.comb_iter_0_right = nn.Sequential(OrderedDict([ 193 | ('max_pool', MaxPool(3, stride=2)), 194 | ('conv', nn.Conv2d(in_channels_left, out_channels_left, 195 | kernel_size=1, bias=False)), 196 | ('bn', nn.BatchNorm2d(out_channels_left, eps=0.001)), 197 | ])) 198 | self.comb_iter_1_left = BranchSeparables(out_channels_right, 199 | out_channels_right, 200 | kernel_size=7, stride=2) 201 | self.comb_iter_1_right = MaxPool(3, stride=2) 202 | self.comb_iter_2_left = BranchSeparables(out_channels_right, 203 | out_channels_right, 204 | kernel_size=5, stride=2) 205 | self.comb_iter_2_right = BranchSeparables(out_channels_right, 206 | out_channels_right, 207 | kernel_size=3, stride=2) 208 | self.comb_iter_3_left = BranchSeparables(out_channels_right, 209 | out_channels_right, 210 | kernel_size=3) 211 | self.comb_iter_3_right = MaxPool(3, stride=2) 212 | self.comb_iter_4_left = BranchSeparables(in_channels_right, 213 | out_channels_right, 214 | kernel_size=3, stride=2, 215 | stem_cell=True) 216 | self.comb_iter_4_right = ReluConvBn(out_channels_right, 217 | out_channels_right, 218 | kernel_size=1, stride=2) 219 | 220 | def forward(self, x_left): 221 | x_right = self.conv_1x1(x_left) 222 | x_out = self.cell_forward(x_left, x_right) 223 | return x_out 224 | 225 | 226 | class Cell(CellBase): 227 | 228 | def __init__(self, in_channels_left, out_channels_left, in_channels_right, 229 | out_channels_right, is_reduction=False, zero_pad=False, 230 | match_prev_layer_dimensions=False): 231 | super(Cell, self).__init__() 232 | 233 | # If `is_reduction` is set to `True` stride 2 is used for 234 | # convolutional and pooling layers to reduce the spatial size of 235 | # the output of a cell approximately by a factor of 2. 236 | stride = 2 if is_reduction else 1 237 | 238 | # If `match_prev_layer_dimensions` is set to `True` 239 | # `FactorizedReduction` is used to reduce the spatial size 240 | # of the left input of a cell approximately by a factor of 2. 241 | self.match_prev_layer_dimensions = match_prev_layer_dimensions 242 | if match_prev_layer_dimensions: 243 | self.conv_prev_1x1 = FactorizedReduction(in_channels_left, 244 | out_channels_left) 245 | else: 246 | self.conv_prev_1x1 = ReluConvBn(in_channels_left, 247 | out_channels_left, kernel_size=1) 248 | 249 | self.conv_1x1 = ReluConvBn(in_channels_right, out_channels_right, 250 | kernel_size=1) 251 | self.comb_iter_0_left = BranchSeparables(out_channels_left, 252 | out_channels_left, 253 | kernel_size=5, stride=stride, 254 | zero_pad=zero_pad) 255 | self.comb_iter_0_right = MaxPool(3, stride=stride, zero_pad=zero_pad) 256 | self.comb_iter_1_left = BranchSeparables(out_channels_right, 257 | out_channels_right, 258 | kernel_size=7, stride=stride, 259 | zero_pad=zero_pad) 260 | self.comb_iter_1_right = MaxPool(3, stride=stride, zero_pad=zero_pad) 261 | self.comb_iter_2_left = BranchSeparables(out_channels_right, 262 | out_channels_right, 263 | kernel_size=5, stride=stride, 264 | zero_pad=zero_pad) 265 | self.comb_iter_2_right = BranchSeparables(out_channels_right, 266 | out_channels_right, 267 | kernel_size=3, stride=stride, 268 | zero_pad=zero_pad) 269 | self.comb_iter_3_left = BranchSeparables(out_channels_right, 270 | out_channels_right, 271 | kernel_size=3) 272 | self.comb_iter_3_right = MaxPool(3, stride=stride, zero_pad=zero_pad) 273 | self.comb_iter_4_left = BranchSeparables(out_channels_left, 274 | out_channels_left, 275 | kernel_size=3, stride=stride, 276 | zero_pad=zero_pad) 277 | if is_reduction: 278 | self.comb_iter_4_right = ReluConvBn(out_channels_right, 279 | out_channels_right, 280 | kernel_size=1, stride=stride) 281 | else: 282 | self.comb_iter_4_right = None 283 | 284 | def forward(self, x_left, x_right): 285 | x_left = self.conv_prev_1x1(x_left) 286 | x_right = self.conv_1x1(x_right) 287 | x_out = self.cell_forward(x_left, x_right) 288 | return x_out 289 | 290 | 291 | class PNASNet5Large(nn.Module): 292 | def __init__(self, num_classes=1001): 293 | super(PNASNet5Large, self).__init__() 294 | self.num_classes = num_classes 295 | self.conv_0 = nn.Sequential(OrderedDict([ 296 | ('conv', nn.Conv2d(3, 96, kernel_size=3, stride=2, bias=False)), 297 | ('bn', nn.BatchNorm2d(96, eps=0.001)) 298 | ])) 299 | self.cell_stem_0 = CellStem0(in_channels_left=96, out_channels_left=54, 300 | in_channels_right=96, 301 | out_channels_right=54) 302 | self.cell_stem_1 = Cell(in_channels_left=96, out_channels_left=108, 303 | in_channels_right=270, out_channels_right=108, 304 | match_prev_layer_dimensions=True, 305 | is_reduction=True) 306 | self.cell_0 = Cell(in_channels_left=270, out_channels_left=216, 307 | in_channels_right=540, out_channels_right=216, 308 | match_prev_layer_dimensions=True) 309 | self.cell_1 = Cell(in_channels_left=540, out_channels_left=216, 310 | in_channels_right=1080, out_channels_right=216) 311 | self.cell_2 = Cell(in_channels_left=1080, out_channels_left=216, 312 | in_channels_right=1080, out_channels_right=216) 313 | self.cell_3 = Cell(in_channels_left=1080, out_channels_left=216, 314 | in_channels_right=1080, out_channels_right=216) 315 | self.cell_4 = Cell(in_channels_left=1080, out_channels_left=432, 316 | in_channels_right=1080, out_channels_right=432, 317 | is_reduction=True, zero_pad=True) 318 | self.cell_5 = Cell(in_channels_left=1080, out_channels_left=432, 319 | in_channels_right=2160, out_channels_right=432, 320 | match_prev_layer_dimensions=True) 321 | self.cell_6 = Cell(in_channels_left=2160, out_channels_left=432, 322 | in_channels_right=2160, out_channels_right=432) 323 | self.cell_7 = Cell(in_channels_left=2160, out_channels_left=432, 324 | in_channels_right=2160, out_channels_right=432) 325 | self.cell_8 = Cell(in_channels_left=2160, out_channels_left=864, 326 | in_channels_right=2160, out_channels_right=864, 327 | is_reduction=True) 328 | self.cell_9 = Cell(in_channels_left=2160, out_channels_left=864, 329 | in_channels_right=4320, out_channels_right=864, 330 | match_prev_layer_dimensions=True) 331 | self.cell_10 = Cell(in_channels_left=4320, out_channels_left=864, 332 | in_channels_right=4320, out_channels_right=864) 333 | self.cell_11 = Cell(in_channels_left=4320, out_channels_left=864, 334 | in_channels_right=4320, out_channels_right=864) 335 | self.relu = nn.ReLU() 336 | self.avg_pool = nn.AvgPool2d(11, stride=1, padding=0) 337 | self.dropout = nn.Dropout(0.5) 338 | self.last_linear = nn.Linear(4320, num_classes) 339 | 340 | def features(self, x): 341 | x_conv_0 = self.conv_0(x) 342 | x_stem_0 = self.cell_stem_0(x_conv_0) 343 | x_stem_1 = self.cell_stem_1(x_conv_0, x_stem_0) 344 | x_cell_0 = self.cell_0(x_stem_0, x_stem_1) 345 | x_cell_1 = self.cell_1(x_stem_1, x_cell_0) 346 | x_cell_2 = self.cell_2(x_cell_0, x_cell_1) 347 | x_cell_3 = self.cell_3(x_cell_1, x_cell_2) 348 | x_cell_4 = self.cell_4(x_cell_2, x_cell_3) 349 | x_cell_5 = self.cell_5(x_cell_3, x_cell_4) 350 | x_cell_6 = self.cell_6(x_cell_4, x_cell_5) 351 | x_cell_7 = self.cell_7(x_cell_5, x_cell_6) 352 | x_cell_8 = self.cell_8(x_cell_6, x_cell_7) 353 | x_cell_9 = self.cell_9(x_cell_7, x_cell_8) 354 | x_cell_10 = self.cell_10(x_cell_8, x_cell_9) 355 | x_cell_11 = self.cell_11(x_cell_9, x_cell_10) 356 | return x_cell_11 357 | 358 | def logits(self, features): 359 | x = self.relu(features) 360 | x = self.avg_pool(x) 361 | x = x.view(x.size(0), -1) 362 | x = self.dropout(x) 363 | x = self.last_linear(x) 364 | return x 365 | 366 | def forward(self, input): 367 | x = self.features(input) 368 | x = self.logits(x) 369 | return x 370 | 371 | 372 | def pnasnet5large(num_classes=1001, pretrained='imagenet'): 373 | r"""PNASNet-5 model architecture from the 374 | `"Progressive Neural Architecture Search" 375 | `_ paper. 376 | """ 377 | if pretrained: 378 | settings = pretrained_settings['pnasnet5large'][pretrained] 379 | assert num_classes == settings[ 380 | 'num_classes'], 'num_classes should be {}, but is {}'.format( 381 | settings['num_classes'], num_classes) 382 | 383 | # both 'imagenet'&'imagenet+background' are loaded from same parameters 384 | model = PNASNet5Large(num_classes=1001) 385 | model.load_state_dict(model_zoo.load_url(settings['url'])) 386 | 387 | if pretrained == 'imagenet': 388 | new_last_linear = nn.Linear(model.last_linear.in_features, 1000) 389 | new_last_linear.weight.data = model.last_linear.weight.data[1:] 390 | new_last_linear.bias.data = model.last_linear.bias.data[1:] 391 | model.last_linear = new_last_linear 392 | 393 | model.input_space = settings['input_space'] 394 | model.input_size = settings['input_size'] 395 | model.input_range = settings['input_range'] 396 | 397 | model.mean = settings['mean'] 398 | model.std = settings['std'] 399 | else: 400 | model = PNASNet5Large(num_classes=num_classes) 401 | return model 402 | -------------------------------------------------------------------------------- /models/imagenet/resnet.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import math 4 | import random 5 | import torch.utils.model_zoo as model_zoo 6 | from .utils import DropoutConv2d 7 | 8 | 9 | __all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 10 | 'resnet152'] 11 | 12 | 13 | model_urls = { 14 | 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 15 | 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 16 | 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 17 | 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 18 | 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 19 | } 20 | 21 | 22 | def conv3x3(in_planes, out_planes, stride=1): 23 | """3x3 convolution with padding""" 24 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 25 | padding=1, bias=False) 26 | 27 | 28 | class BasicBlock(nn.Module): 29 | expansion = 1 30 | 31 | def __init__(self, inplanes, planes, stride=1, downsample=None): 32 | super(BasicBlock, self).__init__() 33 | self.conv1 = conv3x3(inplanes, planes, stride) 34 | self.bn1 = nn.BatchNorm2d(planes) 35 | self.relu = nn.ReLU(inplace=True) 36 | self.conv2 = conv3x3(planes, planes) 37 | self.bn2 = nn.BatchNorm2d(planes) 38 | self.downsample = downsample 39 | self.stride = stride 40 | 41 | # no drop 42 | self.drop = 0 43 | 44 | def forward(self, x): 45 | if (self.downsample is None) and (random.random() < self.drop): 46 | return x 47 | else: 48 | residual = x 49 | 50 | out = self.conv1(x) 51 | out = self.bn1(out) 52 | out = self.relu(out) 53 | 54 | out = self.conv2(out) 55 | out = self.bn2(out) 56 | 57 | if self.downsample is not None: 58 | residual = self.downsample(x) 59 | 60 | out += residual 61 | out = self.relu(out) 62 | 63 | return out 64 | 65 | 66 | class Bottleneck(nn.Module): 67 | expansion = 4 68 | 69 | def __init__(self, inplanes, planes, stride=1, downsample=None): 70 | super(Bottleneck, self).__init__() 71 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 72 | self.bn1 = nn.BatchNorm2d(planes) 73 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, 74 | padding=1, bias=False) 75 | self.bn2 = nn.BatchNorm2d(planes) 76 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 77 | self.bn3 = nn.BatchNorm2d(planes * 4) 78 | self.relu = nn.ReLU(inplace=True) 79 | self.downsample = downsample 80 | self.stride = stride 81 | 82 | # no drop 83 | self.drop = 0 84 | 85 | def forward(self, x): 86 | if (self.downsample is None) and (random.random() < self.drop): 87 | return x 88 | else: 89 | residual = x 90 | 91 | out = self.conv1(x) 92 | out = self.bn1(out) 93 | out = self.relu(out) 94 | 95 | out = self.conv2(out) 96 | out = self.bn2(out) 97 | out = self.relu(out) 98 | 99 | out = self.conv3(out) 100 | out = self.bn3(out) 101 | 102 | if self.downsample is not None: 103 | residual = self.downsample(x) 104 | 105 | out += residual 106 | out = self.relu(out) 107 | 108 | return out 109 | 110 | 111 | class ResNet(nn.Module): 112 | 113 | def __init__(self, block, layers, num_classes=1000): 114 | self.inplanes = 64 115 | super(ResNet, self).__init__() 116 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 117 | bias=False) 118 | self.bn1 = nn.BatchNorm2d(64) 119 | self.relu = nn.ReLU(inplace=True) 120 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 121 | self.layer1 = self._make_layer(block, 64, layers[0]) 122 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 123 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 124 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 125 | self.avgpool = nn.AvgPool2d(7, stride=1) 126 | self.fc = nn.Linear(512 * block.expansion, num_classes) 127 | 128 | for m in self.modules(): 129 | if isinstance(m, nn.Conv2d): 130 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 131 | m.weight.data.normal_(0, math.sqrt(2. / n)) 132 | elif isinstance(m, nn.BatchNorm2d): 133 | m.weight.data.fill_(1) 134 | m.bias.data.zero_() 135 | 136 | # no drop 137 | self.drop = 0 138 | 139 | def _make_layer(self, block, planes, blocks, stride=1): 140 | downsample = None 141 | if stride != 1 or self.inplanes != planes * block.expansion: 142 | downsample = nn.Sequential( 143 | nn.Conv2d(self.inplanes, planes * block.expansion, 144 | kernel_size=1, stride=stride, bias=False), 145 | nn.BatchNorm2d(planes * block.expansion), 146 | ) 147 | 148 | layers = [] 149 | layers.append(block(self.inplanes, planes, stride, downsample)) 150 | self.inplanes = planes * block.expansion 151 | for i in range(1, blocks): 152 | layers.append(block(self.inplanes, planes)) 153 | 154 | return nn.Sequential(*layers) 155 | 156 | def forward(self, x): 157 | for m in self.modules(): 158 | if isinstance(m, (DropoutConv2d, BasicBlock, Bottleneck)): 159 | m.drop = self.drop 160 | 161 | x = self.conv1(x) 162 | x = self.bn1(x) 163 | x = self.relu(x) 164 | x = self.maxpool(x) 165 | 166 | x = self.layer1(x) 167 | x = self.layer2(x) 168 | x = self.layer3(x) 169 | x = self.layer4(x) 170 | 171 | x = self.avgpool(x) 172 | x = x.view(x.size(0), -1) 173 | x = F.dropout(x, p=self.drop, training=True) # force dropout 174 | x = self.fc(x) 175 | 176 | return x 177 | 178 | 179 | def resnet18(pretrained=False, **kwargs): 180 | """Constructs a ResNet-18 model. 181 | 182 | Args: 183 | pretrained (bool): If True, returns a model pre-trained on ImageNet 184 | """ 185 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) 186 | if pretrained: 187 | model.load_state_dict(model_zoo.load_url(model_urls['resnet18'])) 188 | return model 189 | 190 | 191 | def resnet34(pretrained=False, **kwargs): 192 | """Constructs a ResNet-34 model. 193 | 194 | Args: 195 | pretrained (bool): If True, returns a model pre-trained on ImageNet 196 | """ 197 | model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) 198 | if pretrained: 199 | model.load_state_dict(model_zoo.load_url(model_urls['resnet34'])) 200 | return model 201 | 202 | 203 | def resnet50(pretrained=False, **kwargs): 204 | """Constructs a ResNet-50 model. 205 | 206 | Args: 207 | pretrained (bool): If True, returns a model pre-trained on ImageNet 208 | """ 209 | model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) 210 | if pretrained: 211 | model.load_state_dict(model_zoo.load_url(model_urls['resnet50'])) 212 | return model 213 | 214 | 215 | def resnet101(pretrained=False, **kwargs): 216 | """Constructs a ResNet-101 model. 217 | 218 | Args: 219 | pretrained (bool): If True, returns a model pre-trained on ImageNet 220 | """ 221 | model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) 222 | if pretrained: 223 | model.load_state_dict(model_zoo.load_url(model_urls['resnet101'])) 224 | return model 225 | 226 | 227 | def resnet152(pretrained=False, **kwargs): 228 | """Constructs a ResNet-152 model. 229 | 230 | Args: 231 | pretrained (bool): If True, returns a model pre-trained on ImageNet 232 | """ 233 | model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) 234 | if pretrained: 235 | model.load_state_dict(model_zoo.load_url(model_urls['resnet152'])) 236 | return model 237 | -------------------------------------------------------------------------------- /models/imagenet/senet.py: -------------------------------------------------------------------------------- 1 | """ 2 | ResNet code gently borrowed from 3 | https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py 4 | """ 5 | from __future__ import print_function, division, absolute_import 6 | from collections import OrderedDict 7 | import math 8 | 9 | import torch.nn as nn 10 | from torch.utils import model_zoo 11 | 12 | __all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152', 13 | 'se_resnext50_32x4d', 'se_resnext101_32x4d'] 14 | 15 | pretrained_settings = { 16 | 'senet154': { 17 | 'imagenet': { 18 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth', 19 | 'input_space': 'RGB', 20 | 'input_size': [3, 224, 224], 21 | 'input_range': [0, 1], 22 | 'mean': [0.485, 0.456, 0.406], 23 | 'std': [0.229, 0.224, 0.225], 24 | 'num_classes': 1000 25 | } 26 | }, 27 | 'se_resnet50': { 28 | 'imagenet': { 29 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth', 30 | 'input_space': 'RGB', 31 | 'input_size': [3, 224, 224], 32 | 'input_range': [0, 1], 33 | 'mean': [0.485, 0.456, 0.406], 34 | 'std': [0.229, 0.224, 0.225], 35 | 'num_classes': 1000 36 | } 37 | }, 38 | 'se_resnet101': { 39 | 'imagenet': { 40 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth', 41 | 'input_space': 'RGB', 42 | 'input_size': [3, 224, 224], 43 | 'input_range': [0, 1], 44 | 'mean': [0.485, 0.456, 0.406], 45 | 'std': [0.229, 0.224, 0.225], 46 | 'num_classes': 1000 47 | } 48 | }, 49 | 'se_resnet152': { 50 | 'imagenet': { 51 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth', 52 | 'input_space': 'RGB', 53 | 'input_size': [3, 224, 224], 54 | 'input_range': [0, 1], 55 | 'mean': [0.485, 0.456, 0.406], 56 | 'std': [0.229, 0.224, 0.225], 57 | 'num_classes': 1000 58 | } 59 | }, 60 | 'se_resnext50_32x4d': { 61 | 'imagenet': { 62 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth', 63 | 'input_space': 'RGB', 64 | 'input_size': [3, 224, 224], 65 | 'input_range': [0, 1], 66 | 'mean': [0.485, 0.456, 0.406], 67 | 'std': [0.229, 0.224, 0.225], 68 | 'num_classes': 1000 69 | } 70 | }, 71 | 'se_resnext101_32x4d': { 72 | 'imagenet': { 73 | 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth', 74 | 'input_space': 'RGB', 75 | 'input_size': [3, 224, 224], 76 | 'input_range': [0, 1], 77 | 'mean': [0.485, 0.456, 0.406], 78 | 'std': [0.229, 0.224, 0.225], 79 | 'num_classes': 1000 80 | } 81 | }, 82 | } 83 | 84 | 85 | class SEModule(nn.Module): 86 | 87 | def __init__(self, channels, reduction): 88 | super(SEModule, self).__init__() 89 | self.avg_pool = nn.AdaptiveAvgPool2d(1) 90 | self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1, 91 | padding=0) 92 | self.relu = nn.ReLU(inplace=True) 93 | self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1, 94 | padding=0) 95 | self.sigmoid = nn.Sigmoid() 96 | 97 | def forward(self, x): 98 | module_input = x 99 | x = self.avg_pool(x) 100 | x = self.fc1(x) 101 | x = self.relu(x) 102 | x = self.fc2(x) 103 | x = self.sigmoid(x) 104 | return module_input * x 105 | 106 | 107 | class Bottleneck(nn.Module): 108 | """ 109 | Base class for bottlenecks that implements `forward()` method. 110 | """ 111 | def forward(self, x): 112 | residual = x 113 | 114 | out = self.conv1(x) 115 | out = self.bn1(out) 116 | out = self.relu(out) 117 | 118 | out = self.conv2(out) 119 | out = self.bn2(out) 120 | out = self.relu(out) 121 | 122 | out = self.conv3(out) 123 | out = self.bn3(out) 124 | 125 | if self.downsample is not None: 126 | residual = self.downsample(x) 127 | 128 | out = self.se_module(out) + residual 129 | out = self.relu(out) 130 | 131 | return out 132 | 133 | 134 | class SEBottleneck(Bottleneck): 135 | """ 136 | Bottleneck for SENet154. 137 | """ 138 | expansion = 4 139 | 140 | def __init__(self, inplanes, planes, groups, reduction, stride=1, 141 | downsample=None): 142 | super(SEBottleneck, self).__init__() 143 | self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False) 144 | self.bn1 = nn.BatchNorm2d(planes * 2) 145 | self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3, 146 | stride=stride, padding=1, groups=groups, 147 | bias=False) 148 | self.bn2 = nn.BatchNorm2d(planes * 4) 149 | self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1, 150 | bias=False) 151 | self.bn3 = nn.BatchNorm2d(planes * 4) 152 | self.relu = nn.ReLU(inplace=True) 153 | self.se_module = SEModule(planes * 4, reduction=reduction) 154 | self.downsample = downsample 155 | self.stride = stride 156 | 157 | 158 | class SEResNetBottleneck(Bottleneck): 159 | """ 160 | ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe 161 | implementation and uses `stride=stride` in `conv1` and not in `conv2` 162 | (the latter is used in the torchvision implementation of ResNet). 163 | """ 164 | expansion = 4 165 | 166 | def __init__(self, inplanes, planes, groups, reduction, stride=1, 167 | downsample=None): 168 | super(SEResNetBottleneck, self).__init__() 169 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False, 170 | stride=stride) 171 | self.bn1 = nn.BatchNorm2d(planes) 172 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1, 173 | groups=groups, bias=False) 174 | self.bn2 = nn.BatchNorm2d(planes) 175 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 176 | self.bn3 = nn.BatchNorm2d(planes * 4) 177 | self.relu = nn.ReLU(inplace=True) 178 | self.se_module = SEModule(planes * 4, reduction=reduction) 179 | self.downsample = downsample 180 | self.stride = stride 181 | 182 | 183 | class SEResNeXtBottleneck(Bottleneck): 184 | """ 185 | ResNeXt bottleneck type C with a Squeeze-and-Excitation module. 186 | """ 187 | expansion = 4 188 | 189 | def __init__(self, inplanes, planes, groups, reduction, stride=1, 190 | downsample=None, base_width=4): 191 | super(SEResNeXtBottleneck, self).__init__() 192 | width = math.floor(planes * (base_width / 64)) * groups 193 | self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False, 194 | stride=1) 195 | self.bn1 = nn.BatchNorm2d(width) 196 | self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride, 197 | padding=1, groups=groups, bias=False) 198 | self.bn2 = nn.BatchNorm2d(width) 199 | self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False) 200 | self.bn3 = nn.BatchNorm2d(planes * 4) 201 | self.relu = nn.ReLU(inplace=True) 202 | self.se_module = SEModule(planes * 4, reduction=reduction) 203 | self.downsample = downsample 204 | self.stride = stride 205 | 206 | 207 | class SENet(nn.Module): 208 | 209 | def __init__(self, block, layers, groups, reduction, dropout_p=0.2, 210 | inplanes=128, input_3x3=True, downsample_kernel_size=3, 211 | downsample_padding=1, num_classes=1000): 212 | """ 213 | Parameters 214 | ---------- 215 | block (nn.Module): Bottleneck class. 216 | - For SENet154: SEBottleneck 217 | - For SE-ResNet models: SEResNetBottleneck 218 | - For SE-ResNeXt models: SEResNeXtBottleneck 219 | layers (list of ints): Number of residual blocks for 4 layers of the 220 | network (layer1...layer4). 221 | groups (int): Number of groups for the 3x3 convolution in each 222 | bottleneck block. 223 | - For SENet154: 64 224 | - For SE-ResNet models: 1 225 | - For SE-ResNeXt models: 32 226 | reduction (int): Reduction ratio for Squeeze-and-Excitation modules. 227 | - For all models: 16 228 | dropout_p (float or None): Drop probability for the Dropout layer. 229 | If `None` the Dropout layer is not used. 230 | - For SENet154: 0.2 231 | - For SE-ResNet models: None 232 | - For SE-ResNeXt models: None 233 | inplanes (int): Number of input channels for layer1. 234 | - For SENet154: 128 235 | - For SE-ResNet models: 64 236 | - For SE-ResNeXt models: 64 237 | input_3x3 (bool): If `True`, use three 3x3 convolutions instead of 238 | a single 7x7 convolution in layer0. 239 | - For SENet154: True 240 | - For SE-ResNet models: False 241 | - For SE-ResNeXt models: False 242 | downsample_kernel_size (int): Kernel size for downsampling convolutions 243 | in layer2, layer3 and layer4. 244 | - For SENet154: 3 245 | - For SE-ResNet models: 1 246 | - For SE-ResNeXt models: 1 247 | downsample_padding (int): Padding for downsampling convolutions in 248 | layer2, layer3 and layer4. 249 | - For SENet154: 1 250 | - For SE-ResNet models: 0 251 | - For SE-ResNeXt models: 0 252 | num_classes (int): Number of outputs in `last_linear` layer. 253 | - For all models: 1000 254 | """ 255 | super(SENet, self).__init__() 256 | self.inplanes = inplanes 257 | if input_3x3: 258 | layer0_modules = [ 259 | ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1, 260 | bias=False)), 261 | ('bn1', nn.BatchNorm2d(64)), 262 | ('relu1', nn.ReLU(inplace=True)), 263 | ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1, 264 | bias=False)), 265 | ('bn2', nn.BatchNorm2d(64)), 266 | ('relu2', nn.ReLU(inplace=True)), 267 | ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1, 268 | bias=False)), 269 | ('bn3', nn.BatchNorm2d(inplanes)), 270 | ('relu3', nn.ReLU(inplace=True)), 271 | ] 272 | else: 273 | layer0_modules = [ 274 | ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2, 275 | padding=3, bias=False)), 276 | ('bn1', nn.BatchNorm2d(inplanes)), 277 | ('relu1', nn.ReLU(inplace=True)), 278 | ] 279 | # To preserve compatibility with Caffe weights `ceil_mode=True` 280 | # is used instead of `padding=1`. 281 | layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2, 282 | ceil_mode=True))) 283 | self.layer0 = nn.Sequential(OrderedDict(layer0_modules)) 284 | self.layer1 = self._make_layer( 285 | block, 286 | planes=64, 287 | blocks=layers[0], 288 | groups=groups, 289 | reduction=reduction, 290 | downsample_kernel_size=1, 291 | downsample_padding=0 292 | ) 293 | self.layer2 = self._make_layer( 294 | block, 295 | planes=128, 296 | blocks=layers[1], 297 | stride=2, 298 | groups=groups, 299 | reduction=reduction, 300 | downsample_kernel_size=downsample_kernel_size, 301 | downsample_padding=downsample_padding 302 | ) 303 | self.layer3 = self._make_layer( 304 | block, 305 | planes=256, 306 | blocks=layers[2], 307 | stride=2, 308 | groups=groups, 309 | reduction=reduction, 310 | downsample_kernel_size=downsample_kernel_size, 311 | downsample_padding=downsample_padding 312 | ) 313 | self.layer4 = self._make_layer( 314 | block, 315 | planes=512, 316 | blocks=layers[3], 317 | stride=2, 318 | groups=groups, 319 | reduction=reduction, 320 | downsample_kernel_size=downsample_kernel_size, 321 | downsample_padding=downsample_padding 322 | ) 323 | self.avg_pool = nn.AvgPool2d(7, stride=1) 324 | self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None 325 | self.last_linear = nn.Linear(512 * block.expansion, num_classes) 326 | 327 | def _make_layer(self, block, planes, blocks, groups, reduction, stride=1, 328 | downsample_kernel_size=1, downsample_padding=0): 329 | downsample = None 330 | if stride != 1 or self.inplanes != planes * block.expansion: 331 | downsample = nn.Sequential( 332 | nn.Conv2d(self.inplanes, planes * block.expansion, 333 | kernel_size=downsample_kernel_size, stride=stride, 334 | padding=downsample_padding, bias=False), 335 | nn.BatchNorm2d(planes * block.expansion), 336 | ) 337 | 338 | layers = [] 339 | layers.append(block(self.inplanes, planes, groups, reduction, stride, 340 | downsample)) 341 | self.inplanes = planes * block.expansion 342 | for i in range(1, blocks): 343 | layers.append(block(self.inplanes, planes, groups, reduction)) 344 | 345 | return nn.Sequential(*layers) 346 | 347 | def features(self, x): 348 | x = self.layer0(x) 349 | x = self.layer1(x) 350 | x = self.layer2(x) 351 | x = self.layer3(x) 352 | x = self.layer4(x) 353 | return x 354 | 355 | def logits(self, x): 356 | x = self.avg_pool(x) 357 | if self.dropout is not None: 358 | x = self.dropout(x) 359 | x = x.view(x.size(0), -1) 360 | x = self.last_linear(x) 361 | return x 362 | 363 | def forward(self, x): 364 | x = self.features(x) 365 | x = self.logits(x) 366 | return x 367 | 368 | 369 | def initialize_pretrained_model(model, num_classes, settings): 370 | assert num_classes == settings['num_classes'], \ 371 | 'num_classes should be {}, but is {}'.format( 372 | settings['num_classes'], num_classes) 373 | model.load_state_dict(model_zoo.load_url(settings['url'])) 374 | model.input_space = settings['input_space'] 375 | model.input_size = settings['input_size'] 376 | model.input_range = settings['input_range'] 377 | model.mean = settings['mean'] 378 | model.std = settings['std'] 379 | 380 | 381 | def senet154(num_classes=1000, pretrained='imagenet'): 382 | model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16, 383 | dropout_p=0.2, num_classes=num_classes) 384 | if pretrained is not None: 385 | settings = pretrained_settings['senet154'][pretrained] 386 | initialize_pretrained_model(model, num_classes, settings) 387 | return model 388 | 389 | 390 | def se_resnet50(num_classes=1000, pretrained='imagenet'): 391 | model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16, 392 | dropout_p=None, inplanes=64, input_3x3=False, 393 | downsample_kernel_size=1, downsample_padding=0, 394 | num_classes=num_classes) 395 | if pretrained is not None: 396 | settings = pretrained_settings['se_resnet50'][pretrained] 397 | initialize_pretrained_model(model, num_classes, settings) 398 | return model 399 | 400 | 401 | def se_resnet101(num_classes=1000, pretrained='imagenet'): 402 | model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16, 403 | dropout_p=None, inplanes=64, input_3x3=False, 404 | downsample_kernel_size=1, downsample_padding=0, 405 | num_classes=num_classes) 406 | if pretrained is not None: 407 | settings = pretrained_settings['se_resnet101'][pretrained] 408 | initialize_pretrained_model(model, num_classes, settings) 409 | return model 410 | 411 | 412 | def se_resnet152(num_classes=1000, pretrained='imagenet'): 413 | model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16, 414 | dropout_p=None, inplanes=64, input_3x3=False, 415 | downsample_kernel_size=1, downsample_padding=0, 416 | num_classes=num_classes) 417 | if pretrained is not None: 418 | settings = pretrained_settings['se_resnet152'][pretrained] 419 | initialize_pretrained_model(model, num_classes, settings) 420 | return model 421 | 422 | 423 | def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'): 424 | model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16, 425 | dropout_p=None, inplanes=64, input_3x3=False, 426 | downsample_kernel_size=1, downsample_padding=0, 427 | num_classes=num_classes) 428 | if pretrained is not None: 429 | settings = pretrained_settings['se_resnext50_32x4d'][pretrained] 430 | initialize_pretrained_model(model, num_classes, settings) 431 | return model 432 | 433 | 434 | def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'): 435 | model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16, 436 | dropout_p=None, inplanes=64, input_3x3=False, 437 | downsample_kernel_size=1, downsample_padding=0, 438 | num_classes=num_classes) 439 | if pretrained is not None: 440 | settings = pretrained_settings['se_resnext101_32x4d'][pretrained] 441 | initialize_pretrained_model(model, num_classes, settings) 442 | return model 443 | -------------------------------------------------------------------------------- /models/imagenet/torchvision_models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function, division, absolute_import 3 | import torchvision.models as models 4 | import torch.utils.model_zoo as model_zoo 5 | import torch.nn.functional as F 6 | import types 7 | import re 8 | 9 | # drop-out/layer for vgg and resent 10 | from .utils import DropoutConv2d 11 | from . import vgg as drop_vgg 12 | from . import resnet as drop_resnet 13 | 14 | ################################################################# 15 | # You can find the definitions of those models here: 16 | # https://github.com/pytorch/vision/blob/master/torchvision/models 17 | # 18 | # To fit the API, we usually added/redefined some methods and 19 | # renamed some attributs (see below for each models). 20 | # 21 | # However, you usually do not need to see the original model 22 | # definition from torchvision. Just use `print(model)` to see 23 | # the modules and see bellow the `model.features` and 24 | # `model.classifier` definitions. 25 | ################################################################# 26 | 27 | __all__ = [ 28 | 'alexnet', 29 | 'densenet121', 'densenet169', 'densenet201', 'densenet161', 30 | 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', 31 | 'inceptionv3', 32 | 'squeezenet1_0', 'squeezenet1_1', 33 | 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', 34 | 'vgg19_bn', 'vgg19' 35 | ] 36 | 37 | model_urls = { 38 | 'alexnet': 'https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth', 39 | 'densenet121': 'http://data.lip6.fr/cadene/pretrainedmodels/densenet121-fbdb23505.pth', 40 | 'densenet169': 'http://data.lip6.fr/cadene/pretrainedmodels/densenet169-f470b90a4.pth', 41 | 'densenet201': 'http://data.lip6.fr/cadene/pretrainedmodels/densenet201-5750cbb1e.pth', 42 | 'densenet161': 'http://data.lip6.fr/cadene/pretrainedmodels/densenet161-347e6b360.pth', 43 | 'inceptionv3': 'https://download.pytorch.org/models/inception_v3_google-1a9a5a14.pth', 44 | 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 45 | 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 46 | 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 47 | 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 48 | 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 49 | 'squeezenet1_0': 'https://download.pytorch.org/models/squeezenet1_0-a815701f.pth', 50 | 'squeezenet1_1': 'https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth', 51 | 'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth', 52 | 'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth', 53 | 'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth', 54 | 'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', 55 | 'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth', 56 | 'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth', 57 | 'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth', 58 | 'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth', 59 | # 'vgg16_caffe': 'https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg16-00b39a1b.pth', 60 | # 'vgg19_caffe': 'https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth' 61 | } 62 | 63 | input_sizes = {} 64 | means = {} 65 | stds = {} 66 | 67 | for model_name in __all__: 68 | input_sizes[model_name] = [3, 224, 224] 69 | means[model_name] = [0.485, 0.456, 0.406] 70 | stds[model_name] = [0.229, 0.224, 0.225] 71 | 72 | for model_name in ['inceptionv3']: 73 | input_sizes[model_name] = [3, 299, 299] 74 | means[model_name] = [0.5, 0.5, 0.5] 75 | stds[model_name] = [0.5, 0.5, 0.5] 76 | 77 | pretrained_settings = {} 78 | 79 | for model_name in __all__: 80 | pretrained_settings[model_name] = { 81 | 'imagenet': { 82 | 'url': model_urls[model_name], 83 | 'input_space': 'RGB', 84 | 'input_size': input_sizes[model_name], 85 | 'input_range': [0, 1], 86 | 'mean': means[model_name], 87 | 'std': stds[model_name], 88 | 'num_classes': 1000 89 | } 90 | } 91 | 92 | # for model_name in ['vgg16', 'vgg19']: 93 | # pretrained_settings[model_name]['imagenet_caffe'] = { 94 | # 'url': model_urls[model_name + '_caffe'], 95 | # 'input_space': 'BGR', 96 | # 'input_size': input_sizes[model_name], 97 | # 'input_range': [0, 255], 98 | # 'mean': [103.939, 116.779, 123.68], 99 | # 'std': [1., 1., 1.], 100 | # 'num_classes': 1000 101 | # } 102 | 103 | def update_state_dict(state_dict): 104 | # '.'s are no longer allowed in module names, but pervious _DenseLayer 105 | # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. 106 | # They are also in the checkpoints in model_urls. This pattern is used 107 | # to find such keys. 108 | pattern = re.compile( 109 | r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') 110 | for key in list(state_dict.keys()): 111 | res = pattern.match(key) 112 | if res: 113 | new_key = res.group(1) + res.group(2) 114 | state_dict[new_key] = state_dict[key] 115 | del state_dict[key] 116 | return state_dict 117 | 118 | def load_pretrained(model, num_classes, settings): 119 | assert num_classes == settings['num_classes'], \ 120 | "num_classes should be {}, but is {}".format(settings['num_classes'], num_classes) 121 | state_dict = model_zoo.load_url(settings['url']) 122 | state_dict = update_state_dict(state_dict) 123 | model.load_state_dict(state_dict) 124 | model.input_space = settings['input_space'] 125 | model.input_size = settings['input_size'] 126 | model.input_range = settings['input_range'] 127 | model.mean = settings['mean'] 128 | model.std = settings['std'] 129 | return model 130 | 131 | ################################################################# 132 | # AlexNet 133 | 134 | def modify_alexnet(model): 135 | # Modify attributs 136 | model._features = model.features 137 | del model.features 138 | model.dropout0 = model.classifier[0] 139 | model.linear0 = model.classifier[1] 140 | model.relu0 = model.classifier[2] 141 | model.dropout1 = model.classifier[3] 142 | model.linear1 = model.classifier[4] 143 | model.relu1 = model.classifier[5] 144 | model.last_linear = model.classifier[6] 145 | del model.classifier 146 | 147 | def features(self, input): 148 | x = self._features(input) 149 | x = x.view(x.size(0), 256 * 6 * 6) 150 | x = self.dropout0(x) 151 | x = self.linear0(x) 152 | x = self.relu0(x) 153 | x = self.dropout1(x) 154 | x = self.linear1(x) 155 | return x 156 | 157 | def logits(self, features): 158 | x = self.relu1(features) 159 | x = self.last_linear(x) 160 | return x 161 | 162 | def forward(self, input): 163 | x = self.features(input) 164 | x = self.logits(x) 165 | return x 166 | 167 | # Modify methods 168 | model.features = types.MethodType(features, model) 169 | model.logits = types.MethodType(logits, model) 170 | model.forward = types.MethodType(forward, model) 171 | return model 172 | 173 | def alexnet(num_classes=1000, pretrained='imagenet'): 174 | r"""AlexNet model architecture from the 175 | `"One weird trick..." `_ paper. 176 | """ 177 | # https://github.com/pytorch/vision/blob/master/torchvision/models/alexnet.py 178 | model = models.alexnet(pretrained=False) 179 | if pretrained is not None: 180 | settings = pretrained_settings['alexnet'][pretrained] 181 | model = load_pretrained(model, num_classes, settings) 182 | model = modify_alexnet(model) 183 | return model 184 | 185 | ############################################################### 186 | # DenseNets 187 | 188 | def modify_densenets(model): 189 | # Modify attributs 190 | model.last_linear = model.classifier 191 | del model.classifier 192 | 193 | def logits(self, features): 194 | x = F.relu(features, inplace=True) 195 | x = F.avg_pool2d(x, kernel_size=7, stride=1) 196 | x = x.view(x.size(0), -1) 197 | x = self.last_linear(x) 198 | return x 199 | 200 | def forward(self, input): 201 | x = self.features(input) 202 | x = self.logits(x) 203 | return x 204 | 205 | # Modify methods 206 | model.logits = types.MethodType(logits, model) 207 | model.forward = types.MethodType(forward, model) 208 | return model 209 | 210 | def densenet121(num_classes=1000, pretrained='imagenet'): 211 | r"""Densenet-121 model from 212 | `"Densely Connected Convolutional Networks" ` 213 | """ 214 | model = models.densenet121(pretrained=False) 215 | if pretrained is not None: 216 | settings = pretrained_settings['densenet121'][pretrained] 217 | model = load_pretrained(model, num_classes, settings) 218 | model = modify_densenets(model) 219 | return model 220 | 221 | def densenet169(num_classes=1000, pretrained='imagenet'): 222 | r"""Densenet-169 model from 223 | `"Densely Connected Convolutional Networks" ` 224 | """ 225 | model = models.densenet169(pretrained=False) 226 | if pretrained is not None: 227 | settings = pretrained_settings['densenet169'][pretrained] 228 | model = load_pretrained(model, num_classes, settings) 229 | model = modify_densenets(model) 230 | return model 231 | 232 | def densenet201(num_classes=1000, pretrained='imagenet'): 233 | r"""Densenet-201 model from 234 | `"Densely Connected Convolutional Networks" ` 235 | """ 236 | model = models.densenet201(pretrained=False) 237 | if pretrained is not None: 238 | settings = pretrained_settings['densenet201'][pretrained] 239 | model = load_pretrained(model, num_classes, settings) 240 | model = modify_densenets(model) 241 | return model 242 | 243 | def densenet161(num_classes=1000, pretrained='imagenet'): 244 | r"""Densenet-161 model from 245 | `"Densely Connected Convolutional Networks" ` 246 | """ 247 | model = models.densenet161(pretrained=False) 248 | if pretrained is not None: 249 | settings = pretrained_settings['densenet161'][pretrained] 250 | model = load_pretrained(model, num_classes, settings) 251 | model = modify_densenets(model) 252 | return model 253 | 254 | ############################################################### 255 | # InceptionV3 256 | 257 | def inceptionv3(num_classes=1000, pretrained='imagenet'): 258 | r"""Inception v3 model architecture from 259 | `"Rethinking the Inception Architecture for Computer Vision" `_. 260 | """ 261 | model = models.inception_v3(pretrained=False) 262 | if pretrained is not None: 263 | settings = pretrained_settings['inceptionv3'][pretrained] 264 | model = load_pretrained(model, num_classes, settings) 265 | 266 | # Modify attributs 267 | model.last_linear = model.fc 268 | del model.fc 269 | 270 | def features(self, input): 271 | # 299 x 299 x 3 272 | x = self.Conv2d_1a_3x3(input) # 149 x 149 x 32 273 | x = self.Conv2d_2a_3x3(x) # 147 x 147 x 32 274 | x = self.Conv2d_2b_3x3(x) # 147 x 147 x 64 275 | x = F.max_pool2d(x, kernel_size=3, stride=2) # 73 x 73 x 64 276 | x = self.Conv2d_3b_1x1(x) # 73 x 73 x 80 277 | x = self.Conv2d_4a_3x3(x) # 71 x 71 x 192 278 | x = F.max_pool2d(x, kernel_size=3, stride=2) # 35 x 35 x 192 279 | x = self.Mixed_5b(x) # 35 x 35 x 256 280 | x = self.Mixed_5c(x) # 35 x 35 x 288 281 | x = self.Mixed_5d(x) # 35 x 35 x 288 282 | x = self.Mixed_6a(x) # 17 x 17 x 768 283 | x = self.Mixed_6b(x) # 17 x 17 x 768 284 | x = self.Mixed_6c(x) # 17 x 17 x 768 285 | x = self.Mixed_6d(x) # 17 x 17 x 768 286 | x = self.Mixed_6e(x) # 17 x 17 x 768 287 | if self.training and self.aux_logits: 288 | self._out_aux = self.AuxLogits(x) # 17 x 17 x 768 289 | x = self.Mixed_7a(x) # 8 x 8 x 1280 290 | x = self.Mixed_7b(x) # 8 x 8 x 2048 291 | x = self.Mixed_7c(x) # 8 x 8 x 2048 292 | return x 293 | 294 | def logits(self, features): 295 | x = F.avg_pool2d(features, kernel_size=8) # 1 x 1 x 2048 296 | x = F.dropout(x, training=self.training) # 1 x 1 x 2048 297 | x = x.view(x.size(0), -1) # 2048 298 | x = self.last_linear(x) # 1000 (num_classes) 299 | if self.training and self.aux_logits: 300 | aux = self._out_aux 301 | self._out_aux = None 302 | return x, aux 303 | return x 304 | 305 | def forward(self, input): 306 | x = self.features(input) 307 | x = self.logits(x) 308 | return x 309 | 310 | # Modify methods 311 | model.features = types.MethodType(features, model) 312 | model.logits = types.MethodType(logits, model) 313 | model.forward = types.MethodType(forward, model) 314 | return model 315 | 316 | ############################################################### 317 | # ResNets 318 | 319 | def modify_resnets(model): 320 | # Modify attributs 321 | model.last_linear = model.fc 322 | model.fc = None 323 | 324 | def features(self, input): 325 | x = self.conv1(input) 326 | x = self.bn1(x) 327 | x = self.relu(x) 328 | x = self.maxpool(x) 329 | 330 | x = self.layer1(x) 331 | x = self.layer2(x) 332 | x = self.layer3(x) 333 | x = self.layer4(x) 334 | return x 335 | 336 | def logits(self, features): 337 | x = self.avgpool(features) 338 | x = x.view(x.size(0), -1) 339 | x = F.dropout(x, p=self.drop, training=True) # force dropout 340 | x = self.last_linear(x) 341 | return x 342 | 343 | def forward(self, input): 344 | for m in self.modules(): 345 | if isinstance(m, (DropoutConv2d, drop_resnet.BasicBlock, drop_resnet.Bottleneck)): 346 | m.drop = self.drop 347 | 348 | x = self.features(input) 349 | x = self.logits(x) 350 | return x 351 | 352 | # Modify methods 353 | model.features = types.MethodType(features, model) 354 | model.logits = types.MethodType(logits, model) 355 | model.forward = types.MethodType(forward, model) 356 | return model 357 | 358 | def resnet18(num_classes=1000, pretrained='imagenet'): 359 | """Constructs a ResNet-18 model. 360 | """ 361 | model = drop_resnet.resnet18(pretrained=False) 362 | if pretrained is not None: 363 | settings = pretrained_settings['resnet18'][pretrained] 364 | model = load_pretrained(model, num_classes, settings) 365 | model = modify_resnets(model) 366 | return model 367 | 368 | def resnet34(num_classes=1000, pretrained='imagenet'): 369 | """Constructs a ResNet-34 model. 370 | """ 371 | model = drop_resnet.resnet34(pretrained=False) 372 | if pretrained is not None: 373 | settings = pretrained_settings['resnet34'][pretrained] 374 | model = load_pretrained(model, num_classes, settings) 375 | model = modify_resnets(model) 376 | return model 377 | 378 | def resnet50(num_classes=1000, pretrained='imagenet'): 379 | """Constructs a ResNet-50 model. 380 | """ 381 | model = drop_resnet.resnet50(pretrained=False) 382 | if pretrained is not None: 383 | settings = pretrained_settings['resnet50'][pretrained] 384 | model = load_pretrained(model, num_classes, settings) 385 | model = modify_resnets(model) 386 | return model 387 | 388 | def resnet101(num_classes=1000, pretrained='imagenet'): 389 | """Constructs a ResNet-101 model. 390 | """ 391 | model = drop_resnet.resnet101(pretrained=False) 392 | if pretrained is not None: 393 | settings = pretrained_settings['resnet101'][pretrained] 394 | model = load_pretrained(model, num_classes, settings) 395 | model = modify_resnets(model) 396 | return model 397 | 398 | def resnet152(num_classes=1000, pretrained='imagenet'): 399 | """Constructs a ResNet-152 model. 400 | """ 401 | model = drop_resnet.resnet152(pretrained=False) 402 | if pretrained is not None: 403 | settings = pretrained_settings['resnet152'][pretrained] 404 | model = load_pretrained(model, num_classes, settings) 405 | model = modify_resnets(model) 406 | return model 407 | 408 | ############################################################### 409 | # SqueezeNets 410 | 411 | def modify_squeezenets(model): 412 | # /!\ Beware squeezenets do not have any last_linear module 413 | 414 | # Modify attributs 415 | model.dropout = model.classifier[0] 416 | model.last_conv = model.classifier[1] 417 | model.relu = model.classifier[2] 418 | model.avgpool = model.classifier[3] 419 | del model.classifier 420 | 421 | def logits(self, features): 422 | x = self.dropout(features) 423 | x = self.last_conv(x) 424 | x = self.relu(x) 425 | x = self.avgpool(x) 426 | return x 427 | 428 | def forward(self, input): 429 | x = self.features(input) 430 | x = self.logits(x) 431 | return x 432 | 433 | # Modify methods 434 | model.logits = types.MethodType(logits, model) 435 | model.forward = types.MethodType(forward, model) 436 | return model 437 | 438 | def squeezenet1_0(num_classes=1000, pretrained='imagenet'): 439 | r"""SqueezeNet model architecture from the `"SqueezeNet: AlexNet-level 440 | accuracy with 50x fewer parameters and <0.5MB model size" 441 | `_ paper. 442 | """ 443 | model = models.squeezenet1_0(pretrained=False) 444 | if pretrained is not None: 445 | settings = pretrained_settings['squeezenet1_0'][pretrained] 446 | model = load_pretrained(model, num_classes, settings) 447 | model = modify_squeezenets(model) 448 | return model 449 | 450 | def squeezenet1_1(num_classes=1000, pretrained='imagenet'): 451 | r"""SqueezeNet 1.1 model from the `official SqueezeNet repo 452 | `_. 453 | SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters 454 | than SqueezeNet 1.0, without sacrificing accuracy. 455 | """ 456 | model = models.squeezenet1_1(pretrained=False) 457 | if pretrained is not None: 458 | settings = pretrained_settings['squeezenet1_1'][pretrained] 459 | model = load_pretrained(model, num_classes, settings) 460 | model = modify_squeezenets(model) 461 | return model 462 | 463 | ############################################################### 464 | # VGGs 465 | 466 | def modify_vggs(model): 467 | # Modify attributs 468 | model._features = model.features 469 | del model.features 470 | model.linear0 = model.classifier[0] 471 | model.relu0 = model.classifier[1] 472 | model.dropout0 = model.classifier[2] 473 | model.linear1 = model.classifier[3] 474 | model.relu1 = model.classifier[4] 475 | model.dropout1 = model.classifier[5] 476 | model.last_linear = model.classifier[6] 477 | del model.classifier 478 | 479 | def features(self, input): 480 | x = self._features(input) 481 | x = x.view(x.size(0), -1) 482 | x = F.dropout(x, p=self.drop, training=True) # force dropout 483 | x = self.linear0(x) 484 | x = self.relu0(x) 485 | x = self.dropout0(x) 486 | x = F.dropout(x, p=self.drop, training=True) # force dropout 487 | x = self.linear1(x) 488 | return x 489 | 490 | def logits(self, features): 491 | x = self.relu1(features) 492 | x = F.dropout(x, p=self.drop, training=True) # force dropout 493 | x = self.dropout1(x) 494 | x = self.last_linear(x) 495 | return x 496 | 497 | def forward(self, input): 498 | for m in self.modules(): 499 | if isinstance(m, DropoutConv2d): 500 | m.drop = self.drop 501 | 502 | x = self.features(input) 503 | x = self.logits(x) 504 | return x 505 | 506 | # Modify methods 507 | model.features = types.MethodType(features, model) 508 | model.logits = types.MethodType(logits, model) 509 | model.forward = types.MethodType(forward, model) 510 | return model 511 | 512 | def vgg11(num_classes=1000, pretrained='imagenet'): 513 | """VGG 11-layer model (configuration "A") 514 | """ 515 | model = drop_vgg.vgg11(pretrained=False) 516 | if pretrained is not None: 517 | settings = pretrained_settings['vgg11'][pretrained] 518 | model = load_pretrained(model, num_classes, settings) 519 | model = modify_vggs(model) 520 | return model 521 | 522 | def vgg11_bn(num_classes=1000, pretrained='imagenet'): 523 | """VGG 11-layer model (configuration "A") with batch normalization 524 | """ 525 | model = drop_vgg.vgg11_bn(pretrained=False) 526 | if pretrained is not None: 527 | settings = pretrained_settings['vgg11_bn'][pretrained] 528 | model = load_pretrained(model, num_classes, settings) 529 | model = modify_vggs(model) 530 | return model 531 | 532 | def vgg13(num_classes=1000, pretrained='imagenet'): 533 | """VGG 13-layer model (configuration "B") 534 | """ 535 | model = drop_vgg.vgg13(pretrained=False) 536 | if pretrained is not None: 537 | settings = pretrained_settings['vgg13'][pretrained] 538 | model = load_pretrained(model, num_classes, settings) 539 | model = modify_vggs(model) 540 | return model 541 | 542 | def vgg13_bn(num_classes=1000, pretrained='imagenet'): 543 | """VGG 13-layer model (configuration "B") with batch normalization 544 | """ 545 | model = drop_vgg.vgg13_bn(pretrained=False) 546 | if pretrained is not None: 547 | settings = pretrained_settings['vgg13_bn'][pretrained] 548 | model = load_pretrained(model, num_classes, settings) 549 | model = modify_vggs(model) 550 | return model 551 | 552 | def vgg16(num_classes=1000, pretrained='imagenet'): 553 | """VGG 16-layer model (configuration "D") 554 | """ 555 | model = drop_vgg.vgg16(pretrained=False) 556 | if pretrained is not None: 557 | settings = pretrained_settings['vgg16'][pretrained] 558 | model = load_pretrained(model, num_classes, settings) 559 | model = modify_vggs(model) 560 | return model 561 | 562 | def vgg16_bn(num_classes=1000, pretrained='imagenet'): 563 | """VGG 16-layer model (configuration "D") with batch normalization 564 | """ 565 | model = drop_vgg.vgg16_bn(pretrained=False) 566 | if pretrained is not None: 567 | settings = pretrained_settings['vgg16_bn'][pretrained] 568 | model = load_pretrained(model, num_classes, settings) 569 | model = modify_vggs(model) 570 | return model 571 | 572 | def vgg19(num_classes=1000, pretrained='imagenet'): 573 | """VGG 19-layer model (configuration "E") 574 | """ 575 | model = drop_vgg.vgg19(pretrained=False) 576 | if pretrained is not None: 577 | settings = pretrained_settings['vgg19'][pretrained] 578 | model = load_pretrained(model, num_classes, settings) 579 | model = modify_vggs(model) 580 | return model 581 | 582 | def vgg19_bn(num_classes=1000, pretrained='imagenet'): 583 | """VGG 19-layer model (configuration 'E') with batch normalization 584 | """ 585 | model = drop_vgg.vgg19_bn(pretrained=False) 586 | if pretrained is not None: 587 | settings = pretrained_settings['vgg19_bn'][pretrained] 588 | model = load_pretrained(model, num_classes, settings) 589 | model = modify_vggs(model) 590 | return model 591 | -------------------------------------------------------------------------------- /models/imagenet/utils.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | 4 | 5 | class DropoutConv2d(nn.Conv2d): 6 | def __init__(self, in_channels, out_channels, kernel_size, stride=1, 7 | padding=0, dilation=1, groups=1, bias=True, drop=0): 8 | super(DropoutConv2d, self).__init__( 9 | in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias) 10 | self.drop = drop 11 | 12 | def forward(self, x): 13 | x = super(DropoutConv2d, self).forward(x) 14 | x = F.dropout(x, p=self.drop, training=True) 15 | return x 16 | -------------------------------------------------------------------------------- /models/imagenet/vgg.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch.utils.model_zoo as model_zoo 4 | from .utils import DropoutConv2d 5 | import math 6 | 7 | 8 | __all__ = [ 9 | 'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', 10 | 'vgg19_bn', 'vgg19', 11 | ] 12 | 13 | 14 | model_urls = { 15 | 'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth', 16 | 'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth', 17 | 'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth', 18 | 'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', 19 | 'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth', 20 | 'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth', 21 | 'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth', 22 | 'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth', 23 | } 24 | 25 | 26 | class VGG(nn.Module): 27 | 28 | def __init__(self, features, num_classes=1000, init_weights=True): 29 | super(VGG, self).__init__() 30 | self.features = features 31 | self.classifier = nn.Sequential( 32 | nn.Linear(512 * 7 * 7, 4096), 33 | nn.ReLU(True), 34 | nn.Dropout(), 35 | nn.Linear(4096, 4096), 36 | nn.ReLU(True), 37 | nn.Dropout(), 38 | nn.Linear(4096, num_classes), 39 | ) 40 | if init_weights: 41 | self._initialize_weights() 42 | 43 | # no dropout 44 | self.drop = 0 45 | 46 | def forward(self, x): 47 | for m in self.modules(): 48 | if isinstance(m, DropoutConv2d): 49 | m.drop = self.drop 50 | 51 | x = self.features(x) 52 | x = x.view(x.size(0), -1) 53 | x = F.dropout(x, p=self.drop, training=True) # force dropout 54 | x = self.classifier(x) 55 | return x 56 | 57 | def _initialize_weights(self): 58 | for m in self.modules(): 59 | if isinstance(m, DropoutConv2d): 60 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 61 | m.weight.data.normal_(0, math.sqrt(2. / n)) 62 | if m.bias is not None: 63 | m.bias.data.zero_() 64 | elif isinstance(m, nn.BatchNorm2d): 65 | m.weight.data.fill_(1) 66 | m.bias.data.zero_() 67 | elif isinstance(m, nn.Linear): 68 | m.weight.data.normal_(0, 0.01) 69 | m.bias.data.zero_() 70 | 71 | 72 | def make_layers(cfg, batch_norm=False): 73 | layers = [] 74 | in_channels = 3 75 | for v in cfg: 76 | if v == 'M': 77 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 78 | else: 79 | conv2d = DropoutConv2d(in_channels, v, kernel_size=3, padding=1) 80 | if batch_norm: 81 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 82 | else: 83 | layers += [conv2d, nn.ReLU(inplace=True)] 84 | in_channels = v 85 | return nn.Sequential(*layers) 86 | 87 | 88 | cfg = { 89 | 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 90 | 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 91 | 'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 92 | 'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 93 | } 94 | 95 | 96 | def vgg11(pretrained=False, **kwargs): 97 | """VGG 11-layer model (configuration "A") 98 | 99 | Args: 100 | pretrained (bool): If True, returns a model pre-trained on ImageNet 101 | """ 102 | if pretrained: 103 | kwargs['init_weights'] = False 104 | model = VGG(make_layers(cfg['A']), **kwargs) 105 | if pretrained: 106 | model.load_state_dict(model_zoo.load_url(model_urls['vgg11'])) 107 | return model 108 | 109 | 110 | def vgg11_bn(pretrained=False, **kwargs): 111 | """VGG 11-layer model (configuration "A") with batch normalization 112 | 113 | Args: 114 | pretrained (bool): If True, returns a model pre-trained on ImageNet 115 | """ 116 | if pretrained: 117 | kwargs['init_weights'] = False 118 | model = VGG(make_layers(cfg['A'], batch_norm=True), **kwargs) 119 | if pretrained: 120 | model.load_state_dict(model_zoo.load_url(model_urls['vgg11_bn'])) 121 | return model 122 | 123 | 124 | def vgg13(pretrained=False, **kwargs): 125 | """VGG 13-layer model (configuration "B") 126 | 127 | Args: 128 | pretrained (bool): If True, returns a model pre-trained on ImageNet 129 | """ 130 | if pretrained: 131 | kwargs['init_weights'] = False 132 | model = VGG(make_layers(cfg['B']), **kwargs) 133 | if pretrained: 134 | model.load_state_dict(model_zoo.load_url(model_urls['vgg13'])) 135 | return model 136 | 137 | 138 | def vgg13_bn(pretrained=False, **kwargs): 139 | """VGG 13-layer model (configuration "B") with batch normalization 140 | 141 | Args: 142 | pretrained (bool): If True, returns a model pre-trained on ImageNet 143 | """ 144 | if pretrained: 145 | kwargs['init_weights'] = False 146 | model = VGG(make_layers(cfg['B'], batch_norm=True), **kwargs) 147 | if pretrained: 148 | model.load_state_dict(model_zoo.load_url(model_urls['vgg13_bn'])) 149 | return model 150 | 151 | 152 | def vgg16(pretrained=False, **kwargs): 153 | """VGG 16-layer model (configuration "D") 154 | 155 | Args: 156 | pretrained (bool): If True, returns a model pre-trained on ImageNet 157 | """ 158 | if pretrained: 159 | kwargs['init_weights'] = False 160 | model = VGG(make_layers(cfg['D']), **kwargs) 161 | if pretrained: 162 | model.load_state_dict(model_zoo.load_url(model_urls['vgg16'])) 163 | return model 164 | 165 | 166 | def vgg16_bn(pretrained=False, **kwargs): 167 | """VGG 16-layer model (configuration "D") with batch normalization 168 | 169 | Args: 170 | pretrained (bool): If True, returns a model pre-trained on ImageNet 171 | """ 172 | if pretrained: 173 | kwargs['init_weights'] = False 174 | model = VGG(make_layers(cfg['D'], batch_norm=True), **kwargs) 175 | if pretrained: 176 | model.load_state_dict(model_zoo.load_url(model_urls['vgg16_bn'])) 177 | return model 178 | 179 | 180 | def vgg19(pretrained=False, **kwargs): 181 | """VGG 19-layer model (configuration "E") 182 | 183 | Args: 184 | pretrained (bool): If True, returns a model pre-trained on ImageNet 185 | """ 186 | if pretrained: 187 | kwargs['init_weights'] = False 188 | model = VGG(make_layers(cfg['E']), **kwargs) 189 | if pretrained: 190 | model.load_state_dict(model_zoo.load_url(model_urls['vgg19'])) 191 | return model 192 | 193 | 194 | def vgg19_bn(pretrained=False, **kwargs): 195 | """VGG 19-layer model (configuration 'E') with batch normalization 196 | 197 | Args: 198 | pretrained (bool): If True, returns a model pre-trained on ImageNet 199 | """ 200 | if pretrained: 201 | kwargs['init_weights'] = False 202 | model = VGG(make_layers(cfg['E'], batch_norm=True), **kwargs) 203 | if pretrained: 204 | model.load_state_dict(model_zoo.load_url(model_urls['vgg19_bn'])) 205 | return model 206 | -------------------------------------------------------------------------------- /subspace_attack_poster.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZiangYan/subspace-attack.pytorch/96d323e43ad2718767713cfbf1172147ebecf645/subspace_attack_poster.pptx --------------------------------------------------------------------------------