├── opt ├── __init__.py ├── lf.py ├── utils.py └── process.py ├── utils ├── __init__.py ├── dataloader.py └── utils.py ├── pipeline.png ├── intermediate_results └── aux_cifar10_ResNet18_8_1000_128.txt ├── README.md ├── config.py ├── models ├── AugLag.py ├── vgg_imagenet.py ├── quan_vgg_cifar.py ├── resnet_cifar.py ├── quan_vgg_imagenet.py ├── vanilla_resnet.py ├── quantization.py ├── quan_resnet.py ├── quan_resnet_cifar.py ├── quan_resnet_imagenet.py └── resnet_imagenet.py ├── test_lambda.py ├── requirements.txt ├── LICENSE.txt └── atk_insts └── cifar10_atks.txt /opt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jianshuod/TBA/HEAD/pipeline.png -------------------------------------------------------------------------------- /intermediate_results/aux_cifar10_ResNet18_8_1000_128.txt: -------------------------------------------------------------------------------- 1 | 4403 2 | 3494 3 | 906 4 | 4938 5 | 4645 6 | 8058 7 | 5796 8 | 4405 9 | 3624 10 | 3629 11 | 4860 12 | 6009 13 | 8461 14 | 6111 15 | 7436 16 | 218 17 | 9166 18 | 783 19 | 3516 20 | 526 21 | 7883 22 | 5497 23 | 4639 24 | 1103 25 | 8251 26 | 9297 27 | 3756 28 | 322 29 | 9462 30 | 2810 31 | 3280 32 | 4421 33 | 3587 34 | 2367 35 | 5503 36 | 6853 37 | 9630 38 | 4753 39 | 9019 40 | 8974 41 | 6813 42 | 2665 43 | 2217 44 | 5839 45 | 8118 46 | 5991 47 | 6177 48 | 102 49 | 6063 50 | 9759 51 | 7168 52 | 4328 53 | 1788 54 | 4957 55 | 4158 56 | 4855 57 | 3029 58 | 4352 59 | 2758 60 | 7280 61 | 3319 62 | 1855 63 | 9419 64 | 4008 65 | 1056 66 | 1617 67 | 4484 68 | 3294 69 | 6795 70 | 1644 71 | 649 72 | 3555 73 | 3732 74 | 4047 75 | 3568 76 | 6612 77 | 5201 78 | 21 79 | 3105 80 | 2262 81 | 182 82 | 1996 83 | 1227 84 | 3887 85 | 7207 86 | 4657 87 | 4322 88 | 5876 89 | 9537 90 | 3074 91 | 4972 92 | 8688 93 | 2309 94 | 2434 95 | 2893 96 | 8297 97 | 7579 98 | 6946 99 | 2962 100 | 8330 101 | 7872 102 | 7889 103 | 2352 104 | 3486 105 | 2020 106 | 6752 107 | 4318 108 | 2501 109 | 2723 110 | 7133 111 | 8292 112 | 5037 113 | 770 114 | 7996 115 | 4811 116 | 5093 117 | 639 118 | 4179 119 | 5208 120 | 5150 121 | 1175 122 | 6950 123 | 6427 124 | 105 125 | 4428 126 | 6203 127 | 7135 128 | 1735 129 | -------------------------------------------------------------------------------- /utils/dataloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import torch 4 | import torchvision.transforms as transforms 5 | import torchvision.datasets as datasets 6 | 7 | from config import imagenet_root 8 | 9 | root = imagenet_root 10 | 11 | def data_loader(batch_size=256, workers=1, pin_memory=True): 12 | traindir = os.path.join(root, 'train') 13 | valdir = os.path.join(root, 'val') 14 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 15 | std=[0.229, 0.224, 0.225]) 16 | 17 | train_dataset = datasets.ImageFolder( 18 | traindir, 19 | transforms.Compose([ 20 | transforms.RandomResizedCrop(224), 21 | transforms.RandomHorizontalFlip(), 22 | transforms.ToTensor(), 23 | normalize 24 | ]) 25 | ) 26 | val_dataset = datasets.ImageFolder( 27 | valdir, 28 | transforms.Compose([ 29 | transforms.Resize(256), 30 | transforms.CenterCrop(224), 31 | transforms.ToTensor(), 32 | normalize 33 | ]) 34 | ) 35 | 36 | train_loader = torch.utils.data.DataLoader( 37 | train_dataset, 38 | batch_size=batch_size, 39 | shuffle=True, 40 | num_workers=workers, 41 | pin_memory=pin_memory, 42 | sampler=None 43 | ) 44 | val_loader = torch.utils.data.DataLoader( 45 | val_dataset, 46 | batch_size=batch_size, 47 | shuffle=False, 48 | num_workers=workers, 49 | pin_memory=pin_memory 50 | ) 51 | return train_loader, val_loader -------------------------------------------------------------------------------- /opt/lf.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | def loss_func_b_hat(output, labels, target_h_thr, s, t, b_rel, w, lam3, lam4, lam5, y3, y4, z3, z4, rho3, rho4): 5 | l1 = F.cross_entropy(output[:-1], labels[:-1]) * lam4 6 | 7 | l2_1 = torch.max(output[-1][s] - output[-1][t], torch.tensor(0.0).cuda()) 8 | l2_2 = torch.max(target_h_thr - output[-1][t], torch.tensor(0.0).cuda()) 9 | l2 = (l2_1 + l2_2) * lam5 10 | 11 | y3, y4, z3, z4= torch.tensor(y3).float().cuda(), torch.tensor(y4).float().cuda(),\ 12 | torch.tensor(z3).float().cuda(), torch.tensor(z4).float().cuda() 13 | 14 | b_rel = torch.tensor(b_rel).float().cuda() 15 | b_hat = torch.cat((w[s].view(-1), w[t].view(-1))) 16 | l3 = torch.norm(b_hat - b_rel) ** 2 * lam3 17 | 18 | l4 = (rho3/2) * torch.norm(b_hat - y3) ** 2 + (rho4/2) * torch.norm(b_hat - y4) ** 2\ 19 | + z3@(b_hat-y3) + z4@(b_hat-y4) 20 | 21 | return l1 + l2 + l3 + l4, [l1.item(), l2.item(), l3.item(), l4.item()] 22 | 23 | def loss_func_b_rel(output, labels, source_h_thr, s, t, b_hat, w, lam3, lam1, lam2, y1, y2, z1, z2, rho1, rho2): 24 | 25 | l1 = F.cross_entropy(output[:-1], labels[:-1]) * lam1 26 | 27 | l2_1 = torch.max(source_h_thr - output[-1][s], torch.tensor(0.0).cuda()) 28 | l2_2 = torch.max(output[-1][t] - output[-1][s], torch.tensor(0.0).cuda()) 29 | l2 = (l2_1 + l2_2) * lam2 30 | 31 | y1, y2, z1, z2= torch.tensor(y1).float().cuda(), torch.tensor(y2).float().cuda(),\ 32 | torch.tensor(z1).float().cuda(), torch.tensor(z2).float().cuda() 33 | 34 | b_hat = torch.tensor(b_hat).float().cuda() 35 | b_rel = torch.cat((w[s].view(-1), w[t].view(-1))) 36 | l3 = torch.norm(b_hat - b_rel)**2 * lam3 37 | 38 | l4 = (rho1/2) * torch.norm(b_rel - y1) ** 2 + (rho2/2) * torch.norm(b_rel - y2) ** 2\ 39 | + z1@(b_rel-y1) + z2@(b_rel-y2) 40 | 41 | return l1 + l2 + l3 + l4, [l1.item(), l2.item(), l3.item(), l4.item()] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # One-bit Flip is All You Need: When Bit-flip Attack Meets Model Training 2 | 3 | This is the official implementation of our paper [One-bit Flip is All You Need: When Bit-flip Attack Meets Model Training](), accepted by ICCV 2023. This research project is developed based on Python 3 and Pytorch. 4 | 5 | 6 | ## Citation 7 | 8 | If you think this work or our codes are useful for your research, please cite our paper via: 9 | 10 | ```bibtex 11 | @inproceedings{dong2023one, 12 | title={One-bit Flip is All You Need: When Bit-flip Attack Meets Model Training}, 13 | author={Dong, Jianshuo and Han, Qiu and Li, Yiming and Zhang, Tianwei and Li, Yuanjie and Lai, Zeqi and Zhang, Chao and Xia, Shu-Tao }, 14 | booktitle={ICCV}, 15 | year={2023} 16 | } 17 | ``` 18 | 19 | ## The Main Pipeline of Our Methods 20 | 21 | ![pipeline](./pipeline.png) 22 | 23 | ## Requirement Commands (Anaconda): 24 | 25 | Install by running the following cmd in the work directory 26 | 27 | ``` 28 | conda create --name tba --file ./requirements.txt 29 | ``` 30 | 31 | ## Running Examples 32 | 33 | Step 1: [Download](https://www.dropbox.com/s/ax24afm1vqs9k8m/176_95.25.pth?dl=0) the model checkpoint, and then place it in the directory "checkpoint/resnet18" 34 | 35 | Step 2: Fill out the path to this work directory in your server 36 | 37 | Step 3: configure the path to CIFAR-10 dataset in ``config.py`` 38 | 39 | Step 4: run the demo 40 | 41 | ```shell 42 | python ./test_lambda.py -dc cifar10 -mc ResNet18 -bw 8 --mannual -ri 1 -ro 30 43 | ``` 44 | 45 | 46 | ## Variables 47 | 48 | ### Task Specification 49 | ``` 50 | -bw: bit width (quantization config), 8 is provided in demo. 51 | 52 | -mc: model choice, ResNet18 is provided. 53 | 54 | -dc: dataset choice, cifar10 is provided. 55 | 56 | -tn: number of target instances, no effect in demo 57 | 58 | --rc: whether to randomly choose auxiliary samples 59 | ``` 60 | ### Hyperparameter Specification 61 | ``` 62 | --mannual: whether to mannually set hyperparameters (False means using default values defined in config.py) 63 | 64 | -bs_l: base lambda, set to 1 as default 65 | 66 | -ri: inner ratio, lambda_in in paper 67 | 68 | -ro: outer ratio, lambda_out in paper 69 | 70 | -lr: learning rate for updating parameters 71 | ``` 72 | 73 | ## Results 74 | The log for attacking 8-bit quantized ResNet-18 is provided. Please refer to `log_resnet18_8.txt` for our results. 75 | 76 | 77 | ## License 78 | 79 | This project is licensed under the terms of the Apache License 2.0. See the LICENSE file for the full text. 80 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | cifar_root = 'xxx' 4 | imagenet_root = 'xxx' 5 | 6 | default_path = 'xxx' 7 | model_root = os.path.join(default_path, 'checkpoints/model_ck') 8 | intermediate_results = os.path.join(default_path, 'intermediate_results') 9 | log_root = os.path.join(default_path, 'logs/tba') 10 | target_insts_path = os.path.join(default_path, 'atk_insts') 11 | attk_insts_path = os.path.join(default_path, 'atk_insts') 12 | seed = 512 13 | 14 | no_use_gpu_id = [] 15 | 16 | class args(): 17 | gpu_id = '1' 18 | attack_idx = 1317 19 | target_class = 0 20 | log_interval = 1 21 | 22 | base_lambda = 1 23 | ratio_in = 10 24 | ratio_out = 1.5 25 | 26 | margin = 10 27 | 28 | ext_max_iters = 2000 29 | ext_min_iters = 1000 30 | inn_max_iters = 3 31 | initial_rho1 = 0.0001 32 | initial_rho2 = 0.0001 33 | initial_rho3 = 0.0001 34 | initial_rho4 = 0.0001 35 | max_rho1 = 50 36 | max_rho2 = 50 37 | max_rho3 = 50 38 | max_rho4 = 50 39 | rho_fact = 1.01 40 | rho_refresh_int = 1 41 | stop_threshold = 1e-4 42 | counter_tolerance = 300 43 | projection_lp = 2 44 | 45 | def __init__(self, opt) -> None: 46 | self.dc = opt.dc 47 | self.rc = opt.rc 48 | self.arch = opt.mc 49 | self.bit_length = opt.bw 50 | self.ck_path = os.path.join(default_path, 51 | { 52 | 'cifar10': 53 | { 54 | 'ResNet18': 'checkpoint/resnet18/176_95.25.pth', 55 | 'vgg16':'checkpoint/vgg16/182_93.64.pth' 56 | }, 57 | 'imagenet': 58 | { 59 | 'ResNet34': 'checkpoint/model_pth_imagenet/resnet34-b627a593.pth', 60 | 'vgg19_bn':'checkpoint/model_pth_imagenet/vgg19_bn-c79401a0.pth' 61 | } 62 | }[self.dc][self.arch] 63 | ) 64 | if hasattr(opt, 'manual') and opt.manual: 65 | self.base_lambda = opt.bs_l 66 | self.ratio_in = opt.ri 67 | self.ratio_out = opt.ro 68 | self.inn_lr = opt.lr 69 | 70 | self.lam1 = self.base_lambda 71 | self.lam2 = self.ratio_in * self.lam1 72 | self.lam3 = self.ratio_out * self.lam1 73 | self.lam4 = self.base_lambda 74 | self.lam5 = self.ratio_in * self.lam4 75 | self.tn = opt.tn 76 | self.target_atk_insts = os.path.join(attk_insts_path, f'{self.dc}_atks.txt') 77 | 78 | self.n_aux = 128 if self.dc == 'cifar10' else 512 79 | self.inn_lr = 0.005 if self.dc == 'cifar10' else 0.01 80 | 81 | 82 | def show_args(self): 83 | property = [f"{p}:{getattr(self, p)}" for p in dir(self) if'__' not in p] 84 | print(', '.join(property)) 85 | 86 | 87 | -------------------------------------------------------------------------------- /opt/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import copy 3 | import numpy as np 4 | 5 | import torch 6 | 7 | import config 8 | 9 | def test_asr(auglag, all_data, target_idx, target_class): 10 | output_t = auglag(all_data[target_idx:target_idx+1]) 11 | _, pred_t = output_t.topk(1, 1, True, True) 12 | pred_t = pred_t.squeeze(1) 13 | return target_class == pred_t[0].item() 14 | 15 | def cal_acc(auglag, all_data, labels, target_idx, aux_idx, n_aux): 16 | output_a = auglag(all_data) 17 | _, pred_a = output_a.topk(1, 1, True, True) 18 | pred_a = pred_a.squeeze(1) 19 | pa_acc_a = len([i for i in range(len(output_a)) if labels[i] == pred_a[i] and i != target_idx and i not in aux_idx]) / \ 20 | (len(labels) - 1 - n_aux) 21 | return pa_acc_a 22 | 23 | def cal_acc_asr(auglag, all_data, labels, target_idx, aux_idx, n_aux, target_class): 24 | output_a = auglag(all_data) 25 | _, pred_a = output_a.topk(1, 1, True, True) 26 | pred_a = pred_a.squeeze(1) 27 | pa_acc_a = len([i for i in range(len(output_a)) if labels[i] == pred_a[i] and i != target_idx and i not in aux_idx]) / \ 28 | (len(labels) - 1 - n_aux) 29 | return pa_acc_a, target_class == pred_a[target_idx].item() 30 | 31 | def assemble_b(auglag, s_c, t_c): 32 | b_b_new_s = auglag.w_twos.data[s_c].view(-1).detach().cpu().numpy() 33 | b_b_new_t = auglag.w_twos.data[t_c].view(-1).detach().cpu().numpy() 34 | b_b = np.append(b_b_new_s, b_b_new_t) 35 | return b_b 36 | 37 | def project_box(x): 38 | xp = x 39 | xp[x>1]=1 40 | xp[x<0]=0 41 | 42 | return xp 43 | 44 | 45 | def project_shifted_Lp_ball(x, p): 46 | shift_vec = 1/2*np.ones(x.size) 47 | shift_x = x-shift_vec 48 | normp_shift = np.linalg.norm(shift_x, p) 49 | n = x.size 50 | xp = (n**(1/p)) * shift_x / (2*normp_shift) + shift_vec 51 | 52 | return xp 53 | 54 | 55 | def project_positive(x): 56 | xp = np.clip(x, 0, None) 57 | return xp 58 | 59 | def get_binary_v(auglag, previous_auglag): 60 | ''' 61 | 62 | :param auglag: 63 | :param previous_auglag: 64 | :return: 65 | flag denote whether the new version is the same with previous one 66 | True is different 67 | False is the same 68 | ''' 69 | auglag_p = copy.deepcopy(auglag) 70 | auglag_p.w_twos.data[auglag.w_twos.data > 0.5] = 1.0 71 | auglag_p.w_twos.data[auglag.w_twos.data < 0.5] = 0.0 72 | n_bit = torch.norm(auglag_p.w_twos.data.view(-1) - previous_auglag.w_twos.data.view(-1),p=0).item() 73 | dif_flag = False if n_bit < 1 else True 74 | return auglag_p, dif_flag 75 | 76 | 77 | def load_aux_idx(random_choice, attack_idx, total_len, args): 78 | if random_choice: 79 | return np.random.choice( 80 | [i for i in range(total_len) if i != attack_idx], args.n_aux, 81 | replace=False) 82 | else: 83 | # fix all baselines using the same auxiliary set 84 | return np.loadtxt(os.path.join( 85 | config.intermediate_results, f"aux_{args.dc}_{args.arch}_{args.bit_length}_{args.tn}_{args.n_aux}.txt" 86 | ), dtype=int) 87 | -------------------------------------------------------------------------------- /models/AugLag.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | import torch.nn.functional as F 5 | from bitstring import Bits 6 | 7 | 8 | class augLag_Base(nn.Module): 9 | def __init__(self, n_bits, w, b, step_size, init=False): 10 | super(augLag_Base, self).__init__() 11 | self.n_bits = n_bits 12 | self.b = nn.Parameter(torch.tensor(b).float(), requires_grad=True) 13 | self.w_twos = nn.Parameter(torch.zeros(list(w.shape) + [self.n_bits]), 14 | requires_grad=True) 15 | self.step_size = step_size 16 | self.w = w 17 | 18 | base = [2 ** i for i in range(self.n_bits - 1, -1, -1)] 19 | base[0] = -base[0] 20 | self.base = nn.Parameter(torch.tensor([[base]]).float()) 21 | 22 | if init: 23 | self.reset_w_twos() 24 | 25 | def reset_w_twos(self): 26 | for i in range(self.w.shape[0]): 27 | for j in range(self.w.shape[1]): 28 | self.w_twos.data[i][j] += \ 29 | torch.tensor([int(b) for b in Bits(int=int(self.w[i][j]), 30 | length=self.n_bits).bin]) 31 | 32 | 33 | class augLag_Linear(augLag_Base): 34 | 35 | def __init__(self, n_bits, w, b, step_size, init=False): 36 | super(augLag_Linear, self).__init__(n_bits, w, b, step_size, init) 37 | 38 | def forward(self, x): 39 | # covert w_twos to float 40 | w = self.w_twos * self.base 41 | w = torch.sum(w, 42 | dim=2) * self.step_size # step size is the delta in quantization process. this line corresponds to the dequantization process 43 | 44 | # calculate output 45 | x = F.linear(x, w, self.b) 46 | 47 | return x 48 | 49 | 50 | class augLag_Conv2(augLag_Base): 51 | def __init__(self, n_bits, w, b, step_size, stride, padding, dilation, 52 | groups, init=False): 53 | super(augLag_Conv2, self).__init__(n_bits, w, b, step_size, init) 54 | self.stride = stride 55 | self.padding = padding 56 | self.dilation = dilation 57 | self.groups = groups 58 | 59 | def forward(self, x): 60 | 61 | # covert w_twos to float 62 | w = self.w_twos * self.base 63 | w = torch.sum(w, 64 | dim=4) * self.step_size # step size is the delta in quantization process. this line corresponds to the dequantization process 65 | 66 | # calculate output 67 | x = F.conv2d(x, w, self.b, self.stride, 68 | self.padding, self.dilation, self.groups) 69 | 70 | return x 71 | 72 | def reset_w_twos(self): 73 | for i in range(self.w.shape[0]): 74 | for j in range(self.w.shape[1]): 75 | for k in range(self.w.shape[2]): 76 | for z in range(self.w.shape[3]): 77 | self.w_twos.data[i][j][k][z] += \ 78 | torch.tensor([int(b) for b in 79 | Bits(int=int(self.w[i][j][k][z]), 80 | length=self.n_bits).bin]) -------------------------------------------------------------------------------- /models/vgg_imagenet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.utils.model_zoo as model_zoo 4 | 5 | from typing import Union, List, Dict, Any, Optional, cast 6 | 7 | class FakeReLU(torch.autograd.Function): 8 | @staticmethod 9 | def forward(ctx, input): 10 | return input.clamp(min=0) 11 | 12 | @staticmethod 13 | def backward(ctx, grad_output): 14 | return grad_output 15 | 16 | class VGG(nn.Module): 17 | def __init__( 18 | self, features: nn.Module, num_classes: int = 1000, init_weights: bool = True, dropout: float = 0.5 19 | ) -> None: 20 | super().__init__() 21 | self.features = features 22 | self.avgpool = nn.AdaptiveAvgPool2d((7, 7)) 23 | self.classifier = nn.Sequential( 24 | nn.Linear(512 * 7 * 7, 4096), 25 | nn.ReLU(True), 26 | nn.Dropout(p=dropout), 27 | nn.Linear(4096, 4096), 28 | nn.ReLU(True), 29 | nn.Dropout(p=dropout), 30 | nn.Linear(4096, num_classes), 31 | ) 32 | if init_weights: 33 | for m in self.modules(): 34 | if isinstance(m, nn.Conv2d): 35 | nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") 36 | if m.bias is not None: 37 | nn.init.constant_(m.bias, 0) 38 | elif isinstance(m, nn.BatchNorm2d): 39 | nn.init.constant_(m.weight, 1) 40 | nn.init.constant_(m.bias, 0) 41 | elif isinstance(m, nn.Linear): 42 | nn.init.normal_(m.weight, 0, 0.01) 43 | nn.init.constant_(m.bias, 0) 44 | 45 | def forward(self, x: torch.Tensor, with_latent=False, fake_relu=False) -> torch.Tensor: 46 | feats = self.features(x) 47 | pooled = self.avgpool(feats) 48 | x = pooled.view(pooled.size(0), -1) 49 | x_latent = self.classifier[:4](x) 50 | x_relu = FakeReLU.apply(x_latent) if fake_relu \ 51 | else self.classifier[-3](x_latent) 52 | x_out = self.classifier[-2:](x_relu) 53 | 54 | if with_latent: 55 | return x_out, x_relu 56 | return x_out 57 | 58 | 59 | def make_layers(cfg: List[Union[str, int]], batch_norm: bool = False) -> nn.Sequential: 60 | layers: List[nn.Module] = [] 61 | in_channels = 3 62 | for v in cfg: 63 | if v == "M": 64 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 65 | else: 66 | v = cast(int, v) 67 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 68 | if batch_norm: 69 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 70 | else: 71 | layers += [conv2d, nn.ReLU(inplace=True)] 72 | in_channels = v 73 | return nn.Sequential(*layers) 74 | 75 | 76 | cfgs: Dict[str, List[Union[str, int]]] = { 77 | "A": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"], 78 | "B": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"], 79 | "D": [64, 64, "M", 128, 128, "M", 256, 256, 256, "M", 512, 512, 512, "M", 512, 512, 512, "M"], 80 | "E": [64, 64, "M", 128, 128, "M", 256, 256, 256, 256, "M", 512, 512, 512, 512, "M", 512, 512, 512, 512, "M"], 81 | } 82 | 83 | def vgg19_bn(): 84 | return VGG(make_layers(cfgs['E'], True)) -------------------------------------------------------------------------------- /models/quan_vgg_cifar.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from models.quantization import * 6 | 7 | cfg = { 8 | 'VGG11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 9 | 'VGG13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 10 | 'VGG16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 11 | 'VGG19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 12 | } 13 | 14 | class FakeReLU(torch.autograd.Function): 15 | @staticmethod 16 | def forward(ctx, input): 17 | return input.clamp(min=0) 18 | 19 | @staticmethod 20 | def backward(ctx, grad_output): 21 | return grad_output 22 | 23 | class VGG(nn.Module): 24 | def __init__(self, vgg_name, n_bits): 25 | super(VGG, self).__init__() 26 | self.n_bits = n_bits 27 | self.features = self._make_layers(cfg[vgg_name]) 28 | self.classifier = quan_Linear(512, 10, n_bits=n_bits) 29 | 30 | def forward(self, x, with_latent=False, fake_relu=False): 31 | out = self.features[0:-3](x) 32 | if fake_relu: 33 | out = FakeReLU.apply(out) 34 | else: out = self.features[-3](out) 35 | out = self.features[-2:](out) 36 | out = out.view(out.size(0), -1) 37 | final = self.classifier(out) 38 | if with_latent: 39 | return final, out 40 | else:return final 41 | 42 | 43 | def _make_layers(self, cfg): 44 | layers = [] 45 | in_channels = 3 46 | for x in cfg: 47 | if x == 'M': 48 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 49 | else: 50 | layers += [quan_Conv2d(in_channels, x, kernel_size=3, padding=1, n_bits=self.n_bits), 51 | nn.BatchNorm2d(x), 52 | nn.ReLU(inplace=True)] 53 | in_channels = x 54 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 55 | return nn.Sequential(*layers) 56 | 57 | class VGG_mid(nn.Module): 58 | def __init__(self, vgg_name, n_bits): 59 | super(VGG_mid, self).__init__() 60 | self.n_bits = n_bits 61 | self.features = self._make_layers(cfg[vgg_name]) 62 | self.classifier = quan_Linear(512, 10, n_bits=n_bits) 63 | 64 | def forward(self, x): 65 | out = self.features(x) 66 | out = out.view(out.size(0), -1) 67 | return out 68 | 69 | def _make_layers(self, cfg): 70 | layers = [] 71 | in_channels = 3 72 | for x in cfg: 73 | if x == 'M': 74 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 75 | else: 76 | layers += [quan_Conv2d(in_channels, x, kernel_size=3, padding=1, n_bits=self.n_bits), 77 | nn.BatchNorm2d(x), 78 | nn.ReLU(inplace=True)] 79 | in_channels = x 80 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 81 | return nn.Sequential(*layers) 82 | 83 | 84 | def vgg16_quan(bit_width): 85 | return VGG("VGG16", bit_width) 86 | 87 | def vgg16_quan_mid(bit_width): 88 | return VGG_mid("VGG16", bit_width) 89 | 90 | def vgg19_quan(bit_width): 91 | return VGG("VGG19", bit_width) 92 | 93 | def vgg19_quan_mid(bit_width): 94 | return VGG_mid("VGG19", bit_width) -------------------------------------------------------------------------------- /models/resnet_cifar.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | class BasicBlock(nn.Module): 7 | expansion = 1 8 | 9 | def __init__(self, in_planes, planes, stride=1): 10 | super(BasicBlock, self).__init__() 11 | self.conv1 = nn.Conv2d( 12 | in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 13 | self.bn1 = nn.BatchNorm2d(planes) 14 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, 15 | stride=1, padding=1, bias=False) 16 | self.bn2 = nn.BatchNorm2d(planes) 17 | 18 | self.shortcut = nn.Sequential() 19 | if stride != 1 or in_planes != self.expansion*planes: 20 | self.shortcut = nn.Sequential( 21 | nn.Conv2d(in_planes, self.expansion*planes, 22 | kernel_size=1, stride=stride, bias=False), 23 | nn.BatchNorm2d(self.expansion*planes) 24 | ) 25 | 26 | def forward(self, x): 27 | out = F.relu(self.bn1(self.conv1(x))) 28 | out = self.bn2(self.conv2(out)) 29 | out += self.shortcut(x) 30 | out = F.relu(out) 31 | return out 32 | 33 | 34 | class Bottleneck(nn.Module): 35 | expansion = 4 36 | 37 | def __init__(self, in_planes, planes, stride=1): 38 | super(Bottleneck, self).__init__() 39 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) 40 | self.bn1 = nn.BatchNorm2d(planes) 41 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, 42 | stride=stride, padding=1, bias=False) 43 | self.bn2 = nn.BatchNorm2d(planes) 44 | self.conv3 = nn.Conv2d(planes, self.expansion * 45 | planes, kernel_size=1, bias=False) 46 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 47 | 48 | self.shortcut = nn.Sequential() 49 | if stride != 1 or in_planes != self.expansion*planes: 50 | self.shortcut = nn.Sequential( 51 | nn.Conv2d(in_planes, self.expansion*planes, 52 | kernel_size=1, stride=stride, bias=False), 53 | nn.BatchNorm2d(self.expansion*planes) 54 | ) 55 | 56 | def forward(self, x): 57 | out = F.relu(self.bn1(self.conv1(x))) 58 | out = F.relu(self.bn2(self.conv2(out))) 59 | out = self.bn3(self.conv3(out)) 60 | out += self.shortcut(x) 61 | out = F.relu(out) 62 | return out 63 | 64 | 65 | class ResNet(nn.Module): 66 | def __init__(self, block, num_blocks, num_classes=10): 67 | super(ResNet, self).__init__() 68 | self.in_planes = 64 69 | 70 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, 71 | stride=1, padding=1, bias=False) 72 | self.bn1 = nn.BatchNorm2d(64) 73 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 74 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 75 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 76 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 77 | self.linear = nn.Linear(512*block.expansion, num_classes) 78 | 79 | def _make_layer(self, block, planes, num_blocks, stride): 80 | strides = [stride] + [1]*(num_blocks-1) 81 | layers = [] 82 | for stride in strides: 83 | layers.append(block(self.in_planes, planes, stride)) 84 | self.in_planes = planes * block.expansion 85 | return nn.Sequential(*layers) 86 | 87 | def forward(self, x): 88 | out = F.relu(self.bn1(self.conv1(x))) 89 | out = self.layer1(out) 90 | out = self.layer2(out) 91 | out = self.layer3(out) 92 | out = self.layer4(out) 93 | out = F.avg_pool2d(out, 4) 94 | out = out.view(out.size(0), -1) 95 | out = self.linear(out) 96 | return out 97 | 98 | 99 | def ResNet18(): 100 | return ResNet(BasicBlock, [2, 2, 2, 2]) 101 | 102 | 103 | def ResNet34(): 104 | return ResNet(BasicBlock, [3, 4, 6, 3]) 105 | 106 | 107 | def ResNet50(): 108 | return ResNet(Bottleneck, [3, 4, 6, 3]) 109 | 110 | 111 | def ResNet101(): 112 | return ResNet(Bottleneck, [3, 4, 23, 3]) 113 | 114 | 115 | def ResNet152(): 116 | return ResNet(Bottleneck, [3, 8, 36, 3]) 117 | 118 | 119 | def test(): 120 | net = ResNet18() 121 | y = net(torch.randn(1, 3, 32, 32)) 122 | print(y.size()) 123 | 124 | # test() 125 | -------------------------------------------------------------------------------- /models/quan_vgg_imagenet.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.utils.model_zoo as model_zoo 3 | from .quantization import * 4 | 5 | from typing import Union, List, Dict, Any, Optional, cast 6 | 7 | class VGG(nn.Module): 8 | def __init__( 9 | self, features: nn.Module, num_classes: int = 1000, init_weights: bool = True, dropout: float = 0.5, n_bits = 8 10 | ) -> None: 11 | super().__init__() 12 | self.n_bits = n_bits 13 | self.features = features 14 | self.avgpool = nn.AdaptiveAvgPool2d((7, 7)) 15 | self.classifier = nn.Sequential( 16 | quan_Linear(512 * 7 * 7, 4096, n_bits=n_bits), 17 | nn.ReLU(True), 18 | nn.Dropout(p=dropout), 19 | quan_Linear(4096, 4096, n_bits=n_bits), 20 | nn.ReLU(True), 21 | nn.Dropout(p=dropout), 22 | quan_Linear(4096, num_classes, n_bits=n_bits), 23 | ) 24 | if init_weights: 25 | for m in self.modules(): 26 | if isinstance(m, quan_Conv2d): 27 | nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") 28 | if m.bias is not None: 29 | nn.init.constant_(m.bias, 0) 30 | elif isinstance(m, nn.BatchNorm2d): 31 | nn.init.constant_(m.weight, 1) 32 | nn.init.constant_(m.bias, 0) 33 | elif isinstance(m, quan_Linear): 34 | nn.init.normal_(m.weight, 0, 0.01) 35 | nn.init.constant_(m.bias, 0) 36 | 37 | def forward(self, x: torch.Tensor) -> torch.Tensor: 38 | x = self.features(x) 39 | x = self.avgpool(x) 40 | x = torch.flatten(x, 1) 41 | x = self.classifier(x) 42 | return x 43 | 44 | class VGG_mid(nn.Module): 45 | def __init__( 46 | self, features: nn.Module, num_classes: int = 1000, init_weights: bool = True, dropout: float = 0.5, n_bits = 8 47 | ) -> None: 48 | super().__init__() 49 | self.n_bits = n_bits 50 | self.features = features 51 | self.avgpool = nn.AdaptiveAvgPool2d((7, 7)) 52 | self.classifier = nn.Sequential( 53 | quan_Linear(512 * 7 * 7, 4096, n_bits=n_bits), 54 | nn.ReLU(True), 55 | nn.Dropout(p=dropout), 56 | quan_Linear(4096, 4096, n_bits=n_bits), 57 | nn.ReLU(True), 58 | nn.Dropout(p=dropout), 59 | quan_Linear(4096, num_classes, n_bits=n_bits), 60 | ) 61 | if init_weights: 62 | for m in self.modules(): 63 | if isinstance(m, quan_Conv2d): 64 | nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") 65 | if m.bias is not None: 66 | nn.init.constant_(m.bias, 0) 67 | elif isinstance(m, nn.BatchNorm2d): 68 | nn.init.constant_(m.weight, 1) 69 | nn.init.constant_(m.bias, 0) 70 | elif isinstance(m, quan_Linear): 71 | nn.init.normal_(m.weight, 0, 0.01) 72 | nn.init.constant_(m.bias, 0) 73 | 74 | def forward(self, x: torch.Tensor) -> torch.Tensor: 75 | x = self.features(x) 76 | x = self.avgpool(x) 77 | x = torch.flatten(x, 1) 78 | x = self.classifier[:-1](x) 79 | return x 80 | 81 | 82 | 83 | def make_layers(cfg: List[Union[str, int]], batch_norm: bool = False, n_bits=8) -> nn.Sequential: 84 | layers: List[nn.Module] = [] 85 | in_channels = 3 86 | for v in cfg: 87 | if v == "M": 88 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 89 | else: 90 | v = cast(int, v) 91 | conv2d = quan_Conv2d(in_channels, v, kernel_size=3, padding=1, n_bits=n_bits) 92 | if batch_norm: 93 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 94 | else: 95 | layers += [conv2d, nn.ReLU(inplace=True)] 96 | in_channels = v 97 | return nn.Sequential(*layers) 98 | 99 | 100 | cfgs: Dict[str, List[Union[str, int]]] = { 101 | "A": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"], 102 | "B": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"], 103 | "D": [64, 64, "M", 128, 128, "M", 256, 256, 256, "M", 512, 512, 512, "M", 512, 512, 512, "M"], 104 | "E": [64, 64, "M", 128, 128, "M", 256, 256, 256, 256, "M", 512, 512, 512, 512, "M", 512, 512, 512, 512, "M"], 105 | } 106 | 107 | def vgg16_bn_quan(n_bits): 108 | return VGG(make_layers(cfgs['D'], True), n_bits=n_bits) 109 | 110 | def vgg16_bn_quan_mid(n_bits): 111 | return VGG_mid(make_layers(cfgs['D'], True), n_bits=n_bits) 112 | 113 | def vgg19_bn_quan(n_bits): 114 | return VGG(make_layers(cfgs['E'], True), n_bits=n_bits) 115 | 116 | def vgg19_bn_quan_mid(n_bits): 117 | return VGG_mid(make_layers(cfgs['E'], True), n_bits=n_bits) -------------------------------------------------------------------------------- /test_lambda.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | warnings.filterwarnings("ignore") 3 | import config 4 | 5 | import os 6 | print(f"pid {os.getpid()}") 7 | os.chdir(config.default_path) 8 | 9 | import time 10 | import argparse 11 | import numpy as np 12 | 13 | from models.quantization import * 14 | from opt.process import attack_v2 15 | from utils.utils import load_model, load_data, initialize_env, load_auglag_st, load_clean_output 16 | from opt.utils import load_aux_idx 17 | 18 | parser = argparse.ArgumentParser(description='TBA') 19 | parser.add_argument('-bw', default=8, type=int, help='bit width') 20 | parser.add_argument('-mc', default='ResNet18', type=str, help='Model choice') 21 | parser.add_argument('-dc', default='cifar10', type=str, help='Dataset choice') 22 | parser.add_argument('-tn', default=1000, type=int, help='number of target instances') 23 | parser.add_argument("--rc",action="store_true",help="randomly select aux set") 24 | 25 | parser.add_argument("--manual",action="store_true",help="Mannually set hyperparams") 26 | parser.add_argument("-bs_l", type=float, default=1, help="base lambda") 27 | parser.add_argument("-ri", type=float, default=1, help="ratio inner") 28 | parser.add_argument("-ro", type=float, default=15, help="ratio outer") 29 | parser.add_argument("-lr", type=float, default=0.005, help="inner learning rate") 30 | opt = parser.parse_args() 31 | 32 | def main(): 33 | initialize_env(config.seed) 34 | 35 | # prepare the data 36 | print("Prepare data ... ") 37 | 38 | args = config.args(opt) 39 | args.show_args() 40 | 41 | ck_path = args.ck_path 42 | arch = args.arch 43 | bit_length = args.bit_length 44 | 45 | weight, bias, step_size = load_model(args.dc, arch, bit_length, ck_path=ck_path) 46 | 47 | all_data, labels = load_data(args.dc, arch, bit_length, ck_path) 48 | labels_cuda = labels.cuda() 49 | 50 | clean_output = load_clean_output(bit_length, weight, bias, step_size, all_data, args) 51 | 52 | target_attk_insts_path = args.target_atk_insts 53 | 54 | attack_info = np.loadtxt(target_attk_insts_path).astype(int) 55 | total = 0 56 | suc_num = 0 57 | total_n_bit_ab = 0 58 | total_acc_a = 0 59 | total_acc_b = 0 60 | total_time = 0 61 | total_n_bit_a_ori = 0 62 | n_bit_abs = [] 63 | acc_as = [] 64 | acc_bs = [] 65 | n_bit_a_oris = [] 66 | time_costs = [] 67 | suc_insts = [] 68 | suc_targs = [] 69 | end_iters = [] 70 | total_len = len(attack_info) 71 | for i, (target_class, attack_idx) in enumerate(attack_info): 72 | print('--------------------------------------------') 73 | source_class = int(labels[attack_idx]) 74 | total += 1 75 | aux_idx = load_aux_idx(args.rc, attack_idx, len(labels), args) 76 | 77 | s_time = time.time() 78 | 79 | print("Prepare a start point") 80 | auglag_st = load_auglag_st(bit_length, weight, bias, step_size, args) 81 | l1, l2, l3, l4, l5 = (args.lam1, args.lam2, args.lam3, args.lam4, args.lam5) 82 | 83 | print("Attack_alpha Start") 84 | (select_iter, max_acc_a, select_n_bit_a_ori, max_acc_b, min_n_bit_ab), end_iter,_ \ 85 | = attack_v2(auglag_st, (source_class, target_class, attack_idx), all_data, labels_cuda, aux_idx, clean_output, (l1, l2, l3, l4, l5), args) 86 | 87 | this_time = time.time() - s_time 88 | total_time += this_time 89 | print(f"{target_class}, {attack_idx}, time_cost {this_time}") 90 | if select_iter != -1: 91 | suc_num += 1 92 | total_n_bit_ab += min_n_bit_ab 93 | total_n_bit_a_ori += select_n_bit_a_ori 94 | total_acc_a += max_acc_a 95 | total_acc_b += max_acc_b 96 | suc_insts.append(attack_idx) 97 | suc_targs.append(target_class) 98 | n_bit_abs.append(min_n_bit_ab) 99 | n_bit_a_oris.append(select_n_bit_a_ori) 100 | acc_as.append(max_acc_a * 100) 101 | acc_bs.append(max_acc_b * 100) 102 | time_costs.append(this_time) 103 | end_iters.append(end_iter) 104 | if suc_num == 0: continue 105 | else: 106 | print(f"Total {total}, suc {suc_num}, fail {total - suc_num}, avg_n_bit_a_ori {total_n_bit_a_ori / suc_num} avg_n_bit_if_suc {total_n_bit_ab / suc_num}, " 107 | f"avg_acc_a {total_acc_a / suc_num}, avg_acc_b {total_acc_b / suc_num}, avg_time_cost {total_time / total}") 108 | 109 | print(f'ASR {suc_num / total_len * 100}') 110 | print(f'N_bit_a_ori mean {np.average(n_bit_a_oris)} std {np.std(n_bit_a_oris)}') 111 | print(f'ACC_a mean {np.average(acc_as)} std {np.std(acc_as)}') 112 | print(f'N_bit_a_b mean {np.average(n_bit_abs)} std {np.std(n_bit_abs)}') 113 | print(f'ACC_b mean {np.average(acc_bs)} std {np.std(acc_bs)}') 114 | print(f'Exterior iterations mean {np.average(end_iters)}') 115 | print(f'Time Cost {np.average(time_costs)}') 116 | 117 | if __name__ == "__main__": 118 | main() 119 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # This file may be used to create an environment using: 2 | # $ conda create --name --file 3 | _libgcc_mutex=0.1=main 4 | _openmp_mutex=5.1=1_gnu 5 | backcall=0.2.0=pyh9f0ad1d_0 6 | backports=1.0=pyhd8ed1ab_3 7 | backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 8 | bitstring=4.0.1=pypi_0 9 | blas=1.0=mkl 10 | blessings=1.7=py37h06a4308_1002 11 | bottleneck=1.3.5=py37h7deecbd_0 12 | brotli=1.0.9=h5eee18b_7 13 | brotli-bin=1.0.9=h5eee18b_7 14 | brotlipy=0.7.0=py37h27cfd23_1003 15 | bzip2=1.0.8=h7b6447c_0 16 | ca-certificates=2023.01.10=h06a4308_0 17 | certifi=2022.12.7=py37h06a4308_0 18 | cffi=1.15.1=py37h5eee18b_3 19 | charset-normalizer=2.0.4=pyhd3eb1b0_0 20 | cryptography=38.0.4=py37h9ce1e76_0 21 | cudatoolkit=11.3.1=h2bc3f7f_2 22 | cycler=0.11.0=pyhd3eb1b0_0 23 | dbus=1.13.18=hb2f20db_0 24 | debugpy=1.5.1=py37h295c915_0 25 | decorator=5.1.1=pyhd8ed1ab_0 26 | dill=0.3.6=py37h06a4308_0 27 | entrypoints=0.4=pyhd8ed1ab_0 28 | expat=2.4.9=h6a678d5_0 29 | ffmpeg=4.3=hf484d3e_0 30 | fftw=3.3.9=h27cfd23_1 31 | flit-core=3.6.0=pyhd3eb1b0_0 32 | fontconfig=2.14.1=h52c9d5c_1 33 | fonttools=4.25.0=pyhd3eb1b0_0 34 | freetype=2.12.1=h4a9f257_0 35 | giflib=5.2.1=h7b6447c_0 36 | gitdb=4.0.10=pypi_0 37 | gitpython=3.1.31=pypi_0 38 | glib=2.69.1=he621ea3_2 39 | gmp=6.2.1=h295c915_3 40 | gnutls=3.6.15=he1e5248_0 41 | gpustat=0.6.0=pyhd3eb1b0_1 42 | grpcio=1.51.1=pypi_0 43 | gst-plugins-base=1.14.0=h8213a91_2 44 | gstreamer=1.14.0=h28cd5cc_2 45 | icu=58.2=he6710b0_3 46 | idna=3.4=py37h06a4308_0 47 | importlib-metadata=6.0.0=pypi_0 48 | intel-openmp=2021.4.0=h06a4308_3561 49 | ipykernel=6.15.0=pyh210e3f2_0 50 | ipython=7.33.0=py37h89c1867_0 51 | jedi=0.18.2=pyhd8ed1ab_0 52 | joblib=1.1.1=py37h06a4308_0 53 | jpeg=9e=h7f8727e_0 54 | jupyter_client=7.0.6=pyhd8ed1ab_0 55 | jupyter_core=4.11.1=py37h06a4308_0 56 | kiwisolver=1.4.4=py37h6a678d5_0 57 | krb5=1.19.4=h568e23c_0 58 | lame=3.100=h7b6447c_0 59 | lcms2=2.12=h3be6417_0 60 | ld_impl_linux-64=2.38=h1181459_1 61 | lerc=3.0=h295c915_0 62 | libbrotlicommon=1.0.9=h5eee18b_7 63 | libbrotlidec=1.0.9=h5eee18b_7 64 | libbrotlienc=1.0.9=h5eee18b_7 65 | libclang=10.0.1=default_hb85057a_2 66 | libdeflate=1.8=h7f8727e_5 67 | libedit=3.1.20221030=h5eee18b_0 68 | libevent=2.1.12=h8f2d780_0 69 | libffi=3.4.2=h6a678d5_6 70 | libgcc-ng=11.2.0=h1234567_1 71 | libgfortran-ng=11.2.0=h00389a5_1 72 | libgfortran5=11.2.0=h1234567_1 73 | libgomp=11.2.0=h1234567_1 74 | libiconv=1.16=h7f8727e_2 75 | libidn2=2.3.2=h7f8727e_0 76 | libllvm10=10.0.1=hbcb73fb_5 77 | libpng=1.6.37=hbc83047_0 78 | libpq=12.9=h16c4e8d_3 79 | libsodium=1.0.18=h36c2ea0_1 80 | libstdcxx-ng=11.2.0=h1234567_1 81 | libtasn1=4.16.0=h27cfd23_0 82 | libtiff=4.5.0=hecacb30_0 83 | libunistring=0.9.10=h27cfd23_0 84 | libuuid=1.41.5=h5eee18b_0 85 | libwebp=1.2.4=h11a3e52_0 86 | libwebp-base=1.2.4=h5eee18b_0 87 | libxcb=1.15=h7f8727e_0 88 | libxkbcommon=1.0.1=hfa300c1_0 89 | libxml2=2.9.14=h74e7548_0 90 | libxslt=1.1.35=h4e12654_0 91 | lz4-c=1.9.4=h6a678d5_0 92 | matplotlib=3.5.2=py37h06a4308_0 93 | matplotlib-base=3.5.2=py37hf590b9c_0 94 | matplotlib-inline=0.1.6=pyhd8ed1ab_0 95 | mkl=2021.4.0=h06a4308_640 96 | mkl-service=2.4.0=py37h7f8727e_0 97 | mkl_fft=1.3.1=py37hd3c417c_0 98 | mkl_random=1.2.2=py37h51133e4_0 99 | munkres=1.1.4=py_0 100 | ncurses=6.3=h5eee18b_3 101 | nest-asyncio=1.5.6=pyhd8ed1ab_0 102 | nettle=3.7.3=hbbd107a_1 103 | nspr=4.33=h295c915_0 104 | nss=3.74=h0370c37_0 105 | numexpr=2.8.4=py37he184ba9_0 106 | numpy=1.21.5=py37h6c91a56_3 107 | numpy-base=1.21.5=py37ha15fc14_3 108 | nvidia-ml=7.352.0=pyhd3eb1b0_0 109 | opencv-python=4.7.0.68=pypi_0 110 | openh264=2.1.1=h4ff587b_0 111 | openssl=1.1.1t=h7f8727e_0 112 | packaging=22.0=py37h06a4308_0 113 | pandas=1.3.5=py37h8c16a72_0 114 | parso=0.8.3=pyhd8ed1ab_0 115 | pcre=8.45=h295c915_0 116 | pexpect=4.8.0=pyh1a96a4e_2 117 | pickleshare=0.7.5=py_1003 118 | pillow=9.3.0=py37hace64e9_1 119 | pip=22.3.1=py37h06a4308_0 120 | ply=3.11=py37_0 121 | prettytable=3.6.0=pypi_0 122 | prompt-toolkit=3.0.36=pyha770c72_0 123 | psutil=5.9.0=py37h5eee18b_0 124 | ptyprocess=0.7.0=pyhd3deb0d_0 125 | py3nvml=0.2.7=pypi_0 126 | pycparser=2.21=pyhd3eb1b0_0 127 | pygments=2.14.0=pyhd8ed1ab_0 128 | pynvml=11.4.1=pypi_0 129 | pyopenssl=22.0.0=pyhd3eb1b0_0 130 | pyparsing=3.0.9=py37h06a4308_0 131 | pyqt=5.15.7=py37h6a678d5_1 132 | pyqt5-sip=12.11.0=py37h6a678d5_1 133 | pysocks=1.7.1=py37_1 134 | python=3.7.16=h7a1cb2a_0 135 | python-dateutil=2.8.2=pyhd3eb1b0_0 136 | python_abi=3.7=2_cp37m 137 | pytorch=1.12.1=py3.7_cuda11.3_cudnn8.3.2_0 138 | pytorch-mutex=1.0=cuda 139 | pytz=2022.7=py37h06a4308_0 140 | pyzmq=19.0.2=py37hac76be4_2 141 | qt-main=5.15.2=h327a75a_7 142 | qt-webengine=5.15.9=hd2b0992_4 143 | qtwebkit=5.212=h4eab89a_4 144 | readline=8.2=h5eee18b_0 145 | requests=2.28.1=py37h06a4308_0 146 | robustness=9=dev_0 147 | scikit-learn=1.0.2=py37h51133e4_1 148 | scipy=1.7.3=py37h6c91a56_2 149 | seaborn=0.12.2=py37h06a4308_0 150 | setuptools=65.6.3=py37h06a4308_0 151 | sip=6.6.2=py37h6a678d5_0 152 | six=1.16.0=pyhd3eb1b0_1 153 | sklearn=0.0.post1=pypi_0 154 | smmap=5.0.0=pypi_0 155 | sqlite=3.40.1=h5082296_0 156 | threadpoolctl=2.2.0=pyh0d69192_0 157 | tk=8.6.12=h1ccaba5_0 158 | toml=0.10.2=pyhd3eb1b0_0 159 | torchaudio=0.12.1=py37_cu113 160 | torchvision=0.13.1=py37_cu113 161 | tornado=6.2=py37h5eee18b_0 162 | tqdm=4.64.1=py37h06a4308_0 163 | traitlets=5.9.0=pyhd8ed1ab_0 164 | typing_extensions=4.4.0=py37h06a4308_0 165 | urllib3=1.26.14=py37h06a4308_0 166 | wcwidth=0.2.6=pyhd8ed1ab_0 167 | wheel=0.37.1=pyhd3eb1b0_0 168 | xlwt=1.3.0=pypi_0 169 | xmltodict=0.13.0=pypi_0 170 | xz=5.2.10=h5eee18b_0 171 | zeromq=4.3.4=h9c3ff4c_1 172 | zipp=3.11.0=pypi_0 173 | zlib=1.2.13=h5eee18b_0 174 | zstd=1.5.2=ha4553b6_0 175 | -------------------------------------------------------------------------------- /models/vanilla_resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import init 5 | import math 6 | 7 | 8 | __all__ = ['CifarResNet', 'resnet20', "resnet20_mid", 'resnet32', 'resnet44', 'resnet56', 'resnet110'] 9 | 10 | def _weights_init(m): 11 | classname = m.__class__.__name__ 12 | print(classname) 13 | if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d): 14 | init.kaiming_normal(m.weight) 15 | 16 | class LambdaLayer(nn.Module): 17 | def __init__(self, lambd): 18 | super(LambdaLayer, self).__init__() 19 | self.lambd = lambd 20 | 21 | def forward(self, x): 22 | return self.lambd(x) 23 | 24 | 25 | class BasicBlock(nn.Module): 26 | expansion = 1 27 | 28 | def __init__(self, in_planes, planes, stride=1, option='A'): 29 | super(BasicBlock, self).__init__() 30 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 31 | self.bn1 = nn.BatchNorm2d(planes) 32 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 33 | self.bn2 = nn.BatchNorm2d(planes) 34 | 35 | self.shortcut = nn.Sequential() 36 | if stride != 1 or in_planes != planes: 37 | if option == 'A': 38 | """ 39 | For CIFAR10 ResNet paper uses option A. 40 | """ 41 | self.shortcut = LambdaLayer(lambda x: 42 | F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), "constant", 0)) 43 | elif option == 'B': 44 | self.shortcut = nn.Sequential( 45 | nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), 46 | nn.BatchNorm2d(self.expansion * planes) 47 | ) 48 | 49 | def forward(self, x): 50 | out = F.relu(self.bn1(self.conv1(x))) 51 | out = self.bn2(self.conv2(out)) 52 | out += self.shortcut(x) 53 | out = F.relu(out) 54 | return out 55 | 56 | 57 | class CifarResNet(nn.Module): 58 | def __init__(self, block, num_blocks, num_classes=10): 59 | super(CifarResNet, self).__init__() 60 | self.in_planes = 16 61 | 62 | self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False) 63 | self.bn1 = nn.BatchNorm2d(16) 64 | self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1) 65 | self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2) 66 | self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) 67 | self.linear = nn.Linear(64, num_classes) 68 | 69 | # self.apply(_weights_init) 70 | # Initialize weights 71 | for m in self.modules(): 72 | if isinstance(m, nn.Conv2d): 73 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 74 | m.weight.data.normal_(0, math.sqrt(2. / n)) 75 | if m.bias: 76 | m.bias.data.zero_() 77 | 78 | def _make_layer(self, block, planes, num_blocks, stride): 79 | strides = [stride] + [1]*(num_blocks-1) 80 | layers = [] 81 | for stride in strides: 82 | layers.append(block(self.in_planes, planes, stride)) 83 | self.in_planes = planes * block.expansion 84 | 85 | return nn.Sequential(*layers) 86 | 87 | def forward(self, x): 88 | out = F.relu(self.bn1(self.conv1(x))) 89 | out = self.layer1(out) 90 | out = self.layer2(out) 91 | out = self.layer3(out) 92 | out = F.avg_pool2d(out, out.size()[3]) 93 | out = out.view(out.size(0), -1) 94 | out = self.linear(out) 95 | return out 96 | 97 | 98 | 99 | class CifarResNet_mid(nn.Module): 100 | def __init__(self, block, num_blocks, num_classes=10): 101 | super(CifarResNet_mid, self).__init__() 102 | self.in_planes = 16 103 | 104 | self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False) 105 | self.bn1 = nn.BatchNorm2d(16) 106 | self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1) 107 | self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2) 108 | self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) 109 | self.linear = nn.Linear(64, num_classes) 110 | 111 | # self.apply(_weights_init) 112 | # Initialize weights 113 | for m in self.modules(): 114 | if isinstance(m, nn.Conv2d): 115 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 116 | m.weight.data.normal_(0, math.sqrt(2. / n)) 117 | if m.bias: 118 | m.bias.data.zero_() 119 | 120 | def _make_layer(self, block, planes, num_blocks, stride): 121 | strides = [stride] + [1]*(num_blocks-1) 122 | layers = [] 123 | for stride in strides: 124 | layers.append(block(self.in_planes, planes, stride)) 125 | self.in_planes = planes * block.expansion 126 | 127 | return nn.Sequential(*layers) 128 | 129 | def forward(self, x): 130 | out = F.relu(self.bn1(self.conv1(x))) 131 | out = self.layer1(out) 132 | out = self.layer2(out) 133 | out = self.layer3(out) 134 | out = F.avg_pool2d(out, out.size()[3]) 135 | out = out.view(out.size(0), -1) 136 | return out 137 | 138 | 139 | def resnet20(num_classes=10): 140 | model = CifarResNet(BasicBlock, [3, 3, 3], num_classes) 141 | return model 142 | 143 | def resnet20_mid(num_classes=10): 144 | model = CifarResNet_mid(BasicBlock, [3, 3, 3], num_classes) 145 | return model 146 | 147 | def resnet32(num_classes=10): 148 | model = CifarResNet(BasicBlock, [5, 5, 5], num_classes) 149 | return model 150 | 151 | 152 | def resnet44(num_classes=10): 153 | model = CifarResNet(BasicBlock, [7, 7, 7], num_classes) 154 | return model 155 | 156 | 157 | def resnet56(num_classes=10): 158 | model = CifarResNet(BasicBlock, [9, 9, 9], num_classes) 159 | return model 160 | 161 | 162 | def resnet110(num_classes=10): 163 | model = CifarResNet(BasicBlock, [18, 18, 18], num_classes) 164 | return model 165 | 166 | -------------------------------------------------------------------------------- /models/quantization.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | from torchvision import models 5 | import torch.nn.functional as F 6 | 7 | 8 | class _quantize_func(torch.autograd.Function): 9 | @staticmethod 10 | def forward(ctx, input, step_size, half_lvls): 11 | # ctx is a context object that can be used to stash information 12 | # for backward computation 13 | ctx.step_size = step_size 14 | ctx.half_lvls = half_lvls 15 | 16 | # 'hardtanh' likes 'torch.clamp' function. However, 'step' is calculated by weight.abs().max(), 17 | # so this 'hardtanh' can be omitted. 18 | output = F.hardtanh(input, 19 | min_val=-ctx.half_lvls * ctx.step_size.item(), 20 | max_val=ctx.half_lvls * ctx.step_size.item()) 21 | 22 | output = torch.round(output / ctx.step_size) 23 | return output 24 | 25 | @staticmethod 26 | def backward(ctx, grad_output): 27 | grad_input = grad_output.clone() / ctx.step_size 28 | 29 | return grad_input, None, None 30 | 31 | 32 | quantize = _quantize_func.apply 33 | 34 | 35 | class quan_Conv2d(nn.Conv2d): 36 | def __init__(self, 37 | in_channels, 38 | out_channels, 39 | kernel_size, 40 | stride=1, 41 | padding=0, 42 | dilation=1, 43 | groups=1, 44 | bias=True, 45 | n_bits=8): 46 | super(quan_Conv2d, self).__init__(in_channels, 47 | out_channels, 48 | kernel_size, 49 | stride=stride, 50 | padding=padding, 51 | dilation=dilation, 52 | groups=groups, 53 | bias=bias) 54 | self.N_bits = n_bits 55 | self.full_lvls = 2**self.N_bits 56 | self.half_lvls = (self.full_lvls - 2) / 2 57 | # Initialize the step size 58 | self.step_size = nn.Parameter(torch.Tensor([1]), requires_grad=True) 59 | self.__reset_stepsize__() 60 | # flag to enable the inference with quantized weight or self.weight 61 | self.inf_with_weight = False # disabled by default 62 | 63 | # create a vector to identify the weight to each bit 64 | self.b_w = nn.Parameter(2**torch.arange(start=self.N_bits - 1, 65 | end=-1, 66 | step=-1).unsqueeze(-1).float(), 67 | requires_grad=False) 68 | 69 | self.b_w[0] = -self.b_w[0] # in-place change MSB to negative 70 | 71 | def forward(self, input): 72 | if self.inf_with_weight: 73 | return F.conv2d(input, self.weight * self.step_size, self.bias, 74 | self.stride, self.padding, self.dilation, 75 | self.groups) 76 | else: 77 | self.__reset_stepsize__() 78 | weight_quan = quantize(self.weight, self.step_size, 79 | self.half_lvls) * self.step_size 80 | return F.conv2d(input, weight_quan, self.bias, self.stride, 81 | self.padding, self.dilation, self.groups) 82 | 83 | def __reset_stepsize__(self): 84 | with torch.no_grad(): 85 | self.step_size.data = self.weight.abs().max() / self.half_lvls 86 | 87 | def __reset_weight__(self): 88 | ''' 89 | This function will reconstruct the weight stored in self.weight. 90 | Replacing the orginal floating-point with the quantized fix-point 91 | weight representation. 92 | ''' 93 | # replace the weight with the quantized version 94 | with torch.no_grad(): 95 | self.weight.data = quantize(self.weight, self.step_size, 96 | self.half_lvls) 97 | # enable the flag, thus now computation does not invovle weight quantization 98 | self.inf_with_weight = True 99 | 100 | 101 | class quan_Linear(nn.Linear): 102 | def __init__(self, in_features, out_features, bias=True, n_bits=8): 103 | super(quan_Linear, self).__init__(in_features, out_features, bias=bias) 104 | 105 | self.N_bits = n_bits 106 | self.full_lvls = 2**self.N_bits 107 | self.half_lvls = (self.full_lvls - 2) / 2 108 | # Initialize the step size 109 | self.step_size = nn.Parameter(torch.Tensor([1]), requires_grad=True) 110 | self.__reset_stepsize__() 111 | # flag to enable the inference with quantized weight or self.weight 112 | self.inf_with_weight = False # disabled by default 113 | 114 | # create a vector to identify the weight to each bit 115 | self.b_w = nn.Parameter(2**torch.arange(start=self.N_bits - 1, 116 | end=-1, 117 | step=-1).unsqueeze(-1).float(), 118 | requires_grad=False) 119 | 120 | self.b_w[0] = -self.b_w[0] #in-place reverse 121 | 122 | def forward(self, input): 123 | if self.inf_with_weight: 124 | return F.linear(input, self.weight * self.step_size, self.bias) 125 | else: 126 | self.__reset_stepsize__() 127 | weight_quan = quantize(self.weight, self.step_size, 128 | self.half_lvls) * self.step_size 129 | return F.linear(input, weight_quan, self.bias) 130 | 131 | def __reset_stepsize__(self): 132 | with torch.no_grad(): 133 | self.step_size.data = self.weight.abs().max() / self.half_lvls 134 | 135 | def __reset_weight__(self): 136 | ''' 137 | This function will reconstruct the weight stored in self.weight. 138 | Replacing the orginal floating-point with the quantized fix-point 139 | weight representation. 140 | ''' 141 | # replace the weight with the quantized version 142 | with torch.no_grad(): 143 | self.weight.data = quantize(self.weight, self.step_size, 144 | self.half_lvls) 145 | # enable the flag, thus now computation does not invovle weight quantization 146 | self.inf_with_weight = True -------------------------------------------------------------------------------- /models/quan_resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import init 5 | import math 6 | from models.quantization import * 7 | 8 | 9 | __all__ = ['CifarResNet', 'resnet20_quan', 'resnet32_quan', 'resnet44_quan', 'resnet56_quan', 'resnet110_quan'] 10 | 11 | def _weights_init(m): 12 | classname = m.__class__.__name__ 13 | print(classname) 14 | if isinstance(m, quan_Linear) or isinstance(m, quan_Conv2d): 15 | init.kaiming_normal(m.weight) 16 | 17 | class LambdaLayer(nn.Module): 18 | def __init__(self, lambd): 19 | super(LambdaLayer, self).__init__() 20 | self.lambd = lambd 21 | 22 | def forward(self, x): 23 | return self.lambd(x) 24 | 25 | 26 | class BasicBlock(nn.Module): 27 | expansion = 1 28 | 29 | def __init__(self, in_planes, planes, stride=1, option='A', n_bits=8): 30 | super(BasicBlock, self).__init__() 31 | self.conv1 = quan_Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False, n_bits=n_bits) 32 | self.bn1 = nn.BatchNorm2d(planes) 33 | self.conv2 = quan_Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False, n_bits=n_bits) 34 | self.bn2 = nn.BatchNorm2d(planes) 35 | 36 | self.shortcut = nn.Sequential() 37 | if stride != 1 or in_planes != planes: 38 | if option == 'A': 39 | """ 40 | For CIFAR10 ResNet paper uses option A. 41 | """ 42 | self.shortcut = LambdaLayer(lambda x: 43 | F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), "constant", 0)) 44 | elif option == 'B': 45 | self.shortcut = nn.Sequential( 46 | quan_Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False, n_bits=n_bits), 47 | nn.BatchNorm2d(self.expansion * planes) 48 | ) 49 | 50 | def forward(self, x): 51 | out = F.relu(self.bn1(self.conv1(x))) 52 | out = self.bn2(self.conv2(out)) 53 | out += self.shortcut(x) 54 | out = F.relu(out) 55 | return out 56 | 57 | 58 | class CifarResNet(nn.Module): 59 | def __init__(self, block, num_blocks, num_classes=10, n_bits=8): 60 | super(CifarResNet, self).__init__() 61 | self.in_planes = 16 62 | self.n_bits = n_bits 63 | 64 | self.conv1 = quan_Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False, n_bits=self.n_bits) 65 | self.bn1 = nn.BatchNorm2d(16) 66 | self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1) 67 | self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2) 68 | self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) 69 | self.linear = quan_Linear(64, num_classes, n_bits=self.n_bits) 70 | 71 | # self.apply(_weights_init) 72 | # Initialize weights 73 | for m in self.modules(): 74 | if isinstance(m, quan_Conv2d): 75 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 76 | m.weight.data.normal_(0, math.sqrt(2. / n)) 77 | if m.bias: 78 | m.bias.data.zero_() 79 | 80 | def _make_layer(self, block, planes, num_blocks, stride): 81 | strides = [stride] + [1]*(num_blocks-1) 82 | layers = [] 83 | for stride in strides: 84 | layers.append(block(self.in_planes, planes, stride, n_bits=self.n_bits)) 85 | self.in_planes = planes * block.expansion 86 | 87 | return nn.Sequential(*layers) 88 | 89 | def forward(self, x): 90 | out = F.relu(self.bn1(self.conv1(x))) 91 | out = self.layer1(out) 92 | out = self.layer2(out) 93 | out = self.layer3(out) 94 | out = F.avg_pool2d(out, out.size()[3]) 95 | out = out.view(out.size(0), -1) 96 | out = self.linear(out) 97 | return out 98 | 99 | 100 | class CifarResNet_mid(nn.Module): 101 | def __init__(self, block, num_blocks, num_classes=10, n_bits=8): 102 | super(CifarResNet_mid, self).__init__() 103 | self.in_planes = 16 104 | self.n_bits = n_bits 105 | 106 | self.conv1 = quan_Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False, n_bits=self.n_bits) 107 | self.bn1 = nn.BatchNorm2d(16) 108 | self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1) 109 | self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2) 110 | self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2) 111 | self.linear = quan_Linear(64, num_classes, n_bits=self.n_bits) 112 | 113 | # self.apply(_weights_init) 114 | # Initialize weights 115 | for m in self.modules(): 116 | if isinstance(m, quan_Conv2d): 117 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 118 | m.weight.data.normal_(0, math.sqrt(2. / n)) 119 | if m.bias: 120 | m.bias.data.zero_() 121 | 122 | def _make_layer(self, block, planes, num_blocks, stride): 123 | strides = [stride] + [1] * (num_blocks - 1) 124 | layers = [] 125 | for stride in strides: 126 | layers.append(block(self.in_planes, planes, stride, n_bits=self.n_bits)) 127 | self.in_planes = planes * block.expansion 128 | 129 | return nn.Sequential(*layers) 130 | 131 | def forward(self, x): 132 | out = F.relu(self.bn1(self.conv1(x))) 133 | out = self.layer1(out) 134 | out = self.layer2(out) 135 | out = self.layer3(out) 136 | out = F.avg_pool2d(out, out.size()[3]) 137 | out = out.view(out.size(0), -1) 138 | return out 139 | 140 | 141 | def resnet20_quan(num_classes=10, n_bits=8): 142 | model = CifarResNet(BasicBlock, [3, 3, 3], num_classes, n_bits) 143 | return model 144 | 145 | def resnet20_quan_mid(num_classes=10, n_bits=8): 146 | model = CifarResNet_mid(BasicBlock, [3, 3, 3], num_classes, n_bits) 147 | return model 148 | 149 | def resnet32_quan(num_classes=10, n_bits=8): 150 | model = CifarResNet(BasicBlock, [5, 5, 5], num_classes, n_bits) 151 | return model 152 | 153 | 154 | def resnet44_quan(num_classes=10, n_bits=8): 155 | model = CifarResNet(BasicBlock, [7, 7, 7], num_classes, n_bits) 156 | return model 157 | 158 | 159 | def resnet56_quan(num_classes=10, n_bits=8): 160 | model = CifarResNet(BasicBlock, [9, 9, 9], num_classes, n_bits) 161 | return model 162 | 163 | 164 | def resnet110_quan(num_classes=10, n_bits=8): 165 | model = CifarResNet(BasicBlock, [18, 18, 18], num_classes, n_bits) 166 | return model 167 | -------------------------------------------------------------------------------- /opt/process.py: -------------------------------------------------------------------------------- 1 | import os 2 | import copy 3 | 4 | import numpy as np 5 | 6 | import torch 7 | 8 | from .lf import loss_func_b_hat, loss_func_b_rel 9 | from .utils import assemble_b, project_box, project_shifted_Lp_ball 10 | from .utils import get_binary_v, test_asr, cal_acc 11 | 12 | 13 | def attack_v2(auglag_st, attack_info, all_data, labels_cuda, aux_idx, clean_output, 14 | lambdas, args, log_handle=None): 15 | 16 | source_class, target_class, attack_idx = attack_info 17 | all_idx = np.append(aux_idx, attack_idx) 18 | print(lambdas, attack_idx, target_class, file=log_handle) 19 | 20 | # set hyperparams 21 | inn_lr_b_hat = args.inn_lr 22 | inn_lr_b_rel = args.inn_lr 23 | projection_lp = args.projection_lp 24 | rho_fact = args.rho_fact 25 | n_aux = args.n_aux 26 | margin = args.margin 27 | 28 | rho1 = args.initial_rho1 29 | rho2 = args.initial_rho2 30 | rho3 = args.initial_rho3 31 | rho4 = args.initial_rho4 32 | max_rho1 = args.max_rho1 33 | max_rho2 = args.max_rho2 34 | max_rho3 = args.max_rho3 35 | max_rho4 = args.max_rho4 36 | lam1, lam2, lam3, lam4, lam5 = lambdas 37 | 38 | # do copy 39 | auglag_a = copy.deepcopy(auglag_st) 40 | auglag_b = copy.deepcopy(auglag_st) 41 | auglag_a_t = copy.deepcopy(auglag_st) 42 | auglag_b_t = copy.deepcopy(auglag_st) 43 | 44 | # save some original states 45 | w_ori = auglag_st.w_twos.data.view(-1).clone().detach() 46 | 47 | input_var = torch.autograd.Variable(all_data[all_idx], volatile=True) 48 | target_var = torch.autograd.Variable(labels_cuda[all_idx].long(), volatile=True) 49 | target_cf_b_hat = clean_output[attack_idx][[i for i in range(len(clean_output[-1])) if i != source_class]].max() + margin 50 | target_cf_b_rel = clean_output[attack_idx][source_class] 51 | 52 | # prepare intermediate variables and residual variables 53 | b_a = assemble_b(auglag_a, source_class, target_class) 54 | y1 = b_a 55 | y2 = y1 56 | 57 | b_b = assemble_b(auglag_b, source_class, target_class) 58 | y3 = b_b 59 | y4 = y3 60 | 61 | z1 = np.zeros_like(y1) 62 | z2 = np.zeros_like(y2) 63 | z3 = np.zeros_like(y3) 64 | z4 = np.zeros_like(y4) 65 | 66 | counter_ab = 0 67 | asr_a, asr_b = False, False 68 | min_n_flip_ab = np.iinfo(np.int16).max 69 | asr_calculater = lambda x:test_asr(x, all_data, attack_idx, target_class) 70 | acc_calculater = lambda x:cal_acc(x, all_data, labels_cuda, attack_idx, aux_idx, n_aux) 71 | 72 | for ext_iter in range(args.ext_max_iters): 73 | 74 | # update b_hat 75 | for inn_iter in range(args.inn_max_iters): 76 | output = auglag_b(input_var) 77 | loss_b_hat, loss_items_b_hat = \ 78 | loss_func_b_hat(output, target_var, target_cf_b_hat, source_class, target_class, b_a, auglag_b.w_twos, lam3, lam4, lam5, y3, y4, z3, z4, rho3, rho4) 79 | 80 | loss_b_hat.backward(retain_graph=True) 81 | auglag_b.w_twos.data[target_class] = auglag_b.w_twos.data[target_class] - inn_lr_b_hat * auglag_b.w_twos.grad.data[target_class] 82 | auglag_b.w_twos.data[source_class] = auglag_b.w_twos.data[source_class] - inn_lr_b_hat * auglag_b.w_twos.grad.data[source_class] 83 | auglag_b.w_twos.grad.zero_() 84 | 85 | b_b = assemble_b(auglag_b, source_class, target_class) 86 | 87 | # update intermediate variables y3, y4 88 | y3 = project_box(b_b + z3 / rho3) 89 | y4 = project_shifted_Lp_ball(b_b + z4 / rho4, projection_lp) 90 | 91 | temp3 = np.linalg.norm(b_b - y3) / max(np.linalg.norm(b_b), 2.2204e-16) 92 | temp4 = np.linalg.norm(b_b - y4) / max(np.linalg.norm(b_b), 2.2204e-16) 93 | 94 | # do log 95 | if ext_iter % args.log_interval == 0: 96 | # print('[B_hat] iter: %d, counter: %d, stop_threshold: %.6f, loss: %.4f' % (ext_iter, counter_ab, max(temp3, temp4), loss_b_hat.item()), loss_items_b_hat) 97 | auglag_a_t, dif_flag_a = get_binary_v(auglag_a, auglag_a_t) 98 | auglag_b_t, dif_flag_b = get_binary_v(auglag_b, auglag_b_t) 99 | if dif_flag_a: asr_a = asr_calculater(auglag_a_t) 100 | if dif_flag_b: asr_b = asr_calculater(auglag_b_t) 101 | if (dif_flag_a or dif_flag_b or 0 == ext_iter) and (asr_a == False and asr_b == True): 102 | n_bit_b = torch.norm(auglag_a_t.w_twos.data.view(-1) - auglag_b_t.w_twos.data.view(-1), p=0).item() 103 | counter_ab = 0 104 | if n_bit_b < min_n_flip_ab: 105 | min_n_flip_ab = n_bit_b 106 | temp_a = copy.deepcopy(auglag_a_t) 107 | temp_b = copy.deepcopy(auglag_b_t) 108 | else: counter_ab += args.log_interval # roughly estimate 109 | 110 | # update b_rel 111 | for inn_iter in range(args.inn_max_iters): 112 | output = auglag_a(input_var) 113 | loss_b_rel, loss_items_b_rel = \ 114 | loss_func_b_rel(output, target_var, target_cf_b_rel, source_class, target_class, b_b, auglag_a.w_twos, lam3, lam1, lam2, y1, y2, z1, z2, rho1, rho2) 115 | loss_b_rel.backward(retain_graph=True) 116 | auglag_a.w_twos.data[target_class] = auglag_a.w_twos.data[target_class] - inn_lr_b_rel * auglag_a.w_twos.grad.data[target_class] 117 | auglag_a.w_twos.data[source_class] = auglag_a.w_twos.data[source_class] - inn_lr_b_rel * auglag_a.w_twos.grad.data[source_class] 118 | auglag_a.w_twos.grad.zero_() 119 | b_a = assemble_b(auglag_a, source_class, target_class) 120 | 121 | # update y1, y2 122 | y1 = project_box(b_a + z1 / rho1) 123 | y2 = project_shifted_Lp_ball(b_a + z2 / rho2, projection_lp) 124 | 125 | temp1 = np.linalg.norm(b_a - y1) / max(np.linalg.norm(b_a), 2.2204e-16) 126 | temp2 = np.linalg.norm(b_a - y2) / max(np.linalg.norm(b_a), 2.2204e-16) 127 | 128 | # do log 129 | # if ext_iter % args.log_interval == 0: 130 | # print('[B_Rel] iter: %d, counter: %d, stop_threshold: %.6f, loss: %.4f' % (ext_iter, counter_ab, max(temp1, temp2), loss_b_rel.item()), loss_items_b_rel) 131 | 132 | # update residual variables 133 | z1 = z1 + rho1 * (b_a - y1) 134 | z2 = z2 + rho2 * (b_a - y2) 135 | z3 = z3 + rho3 * (b_b - y3) 136 | z4 = z4 + rho4 * (b_b - y4) 137 | 138 | # update rho 139 | if ext_iter % args.rho_refresh_int == 0: 140 | rho1 = min(rho_fact * rho1, max_rho1) 141 | rho2 = min(rho_fact * rho2, max_rho2) 142 | rho3 = min(rho_fact * rho3, max_rho3) 143 | rho4 = min(rho_fact * rho4, max_rho4) 144 | 145 | # judge if early stop 146 | # case 1: failure to explorem, NaN occurs 147 | if True in np.isnan(b_a) or True in np.isnan(b_b): 148 | break 149 | # case 2: lp-box ADMM converges(y1->b_a, y2->b_a or y3->b_b, y4->b_b) 150 | if max(temp1, temp2) <= args.stop_threshold and max(temp3, temp4) <= args.stop_threshold and ext_iter > 100: 151 | print('[Break Point1] END iter: %d, stop_threshold: %.6f, loss: %.4f' % (ext_iter, max(temp3, temp4), loss_b_hat.item())) 152 | print('[Break Point2] END iter: %d, stop_threshold: %.6f, loss: %.4f' % (ext_iter, max(temp1, temp2), loss_b_rel.item())) 153 | break 154 | # case 3: no improvement for b_b 155 | if ext_iter >= args.ext_min_iters and counter_ab >= args.counter_tolerance: 156 | print("No improvement! Early Stop") 157 | break 158 | if min_n_flip_ab == np.iinfo(np.int16).max: 159 | res = (-1, 0, 100, 0, 100) 160 | M_a = None 161 | else: 162 | n_bit_a = torch.norm(temp_a.w_twos.data.view(-1) - w_ori, p=0).item() 163 | acc_a = acc_calculater(temp_a) 164 | acc_b = acc_calculater(temp_b) 165 | res = (1, acc_a, n_bit_a, acc_b, min_n_flip_ab) 166 | 167 | M_a = temp_a 168 | print(res) 169 | return res, ext_iter, M_a -------------------------------------------------------------------------------- /models/quan_resnet_cifar.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import init 5 | import math 6 | from models.quantization import * 7 | 8 | class FakeReLU(torch.autograd.Function): 9 | @staticmethod 10 | def forward(ctx, input): 11 | return input.clamp(min=0) 12 | 13 | @staticmethod 14 | def backward(ctx, grad_output): 15 | return grad_output 16 | 17 | class SequentialWithArgs(torch.nn.Sequential): 18 | def forward(self, input, *args, **kwargs): 19 | vs = list(self._modules.values()) 20 | l = len(vs) 21 | for i in range(l): 22 | if i == l-1: 23 | input = vs[i](input, *args, **kwargs) 24 | else: 25 | input = vs[i](input) 26 | return input 27 | 28 | class LambdaLayer(nn.Module): 29 | def __init__(self, lambd): 30 | super(LambdaLayer, self).__init__() 31 | self.lambd = lambd 32 | 33 | def forward(self, x): 34 | return self.lambd(x) 35 | 36 | class BasicBlock(nn.Module): 37 | expansion = 1 38 | 39 | def __init__(self, in_planes, planes, stride=1, n_bits=8): 40 | super(BasicBlock, self).__init__() 41 | self.conv1 = quan_Conv2d( 42 | in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False, n_bits=n_bits) 43 | self.bn1 = nn.BatchNorm2d(planes) 44 | self.conv2 = quan_Conv2d(planes, planes, kernel_size=3, 45 | stride=1, padding=1, bias=False, n_bits=n_bits) 46 | self.bn2 = nn.BatchNorm2d(planes) 47 | 48 | self.shortcut = nn.Sequential() 49 | if stride != 1 or in_planes != self.expansion*planes: 50 | self.shortcut = nn.Sequential( 51 | quan_Conv2d(in_planes, self.expansion*planes, 52 | kernel_size=1, stride=stride, bias=False, n_bits=n_bits), 53 | nn.BatchNorm2d(self.expansion*planes) 54 | ) 55 | 56 | def forward(self, x, fake_relu=False): 57 | out = F.relu(self.bn1(self.conv1(x))) 58 | out = self.bn2(self.conv2(out)) 59 | out += self.shortcut(x) 60 | if fake_relu: 61 | return FakeReLU.apply(out) 62 | return F.relu(out) 63 | 64 | 65 | 66 | class Bottleneck(nn.Module): 67 | expansion = 4 68 | 69 | def __init__(self, in_planes, planes, stride=1, n_bits=8): 70 | super(Bottleneck, self).__init__() 71 | self.conv1 = quan_Conv2d(in_planes, planes, kernel_size=1, bias=False, n_bits=n_bits) 72 | self.bn1 = nn.BatchNorm2d(planes) 73 | self.conv2 = quan_Conv2d(planes, planes, kernel_size=3, 74 | stride=stride, padding=1, bias=False, n_bits=n_bits) 75 | self.bn2 = nn.BatchNorm2d(planes) 76 | self.conv3 = quan_Conv2d(planes, self.expansion * 77 | planes, kernel_size=1, bias=False, n_bits=n_bits) 78 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 79 | 80 | self.shortcut = nn.Sequential() 81 | if stride != 1 or in_planes != self.expansion*planes: 82 | self.shortcut = nn.Sequential( 83 | quan_Conv2d(in_planes, self.expansion*planes, 84 | kernel_size=1, stride=stride, bias=False, n_bits=n_bits), 85 | nn.BatchNorm2d(self.expansion*planes) 86 | ) 87 | 88 | def forward(self, x, fake_relu=False): 89 | out = F.relu(self.bn1(self.conv1(x))) 90 | out = F.relu(self.bn2(self.conv2(out))) 91 | out = self.bn3(self.conv3(out)) 92 | out += self.shortcut(x) 93 | if fake_relu: 94 | return FakeReLU.apply(out) 95 | return F.relu(out) 96 | 97 | 98 | class ResNet(nn.Module): 99 | def __init__(self, block, num_blocks, num_classes=10, n_bits=8): 100 | super(ResNet, self).__init__() 101 | self.in_planes = 64 102 | self.n_bits = n_bits 103 | 104 | self.conv1 = quan_Conv2d(3, 64, kernel_size=3, 105 | stride=1, padding=1, bias=False, n_bits=self.n_bits) 106 | self.bn1 = nn.BatchNorm2d(64) 107 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 108 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 109 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 110 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 111 | self.linear = quan_Linear(512*block.expansion, num_classes, n_bits=self.n_bits) 112 | 113 | def _make_layer(self, block, planes, num_blocks, stride): 114 | strides = [stride] + [1]*(num_blocks-1) 115 | layers = [] 116 | for stride in strides: 117 | layers.append(block(self.in_planes, planes, stride, n_bits=self.n_bits)) 118 | self.in_planes = planes * block.expansion 119 | return SequentialWithArgs(*layers) 120 | 121 | def forward(self, x, with_latent=False, fake_relu=False): 122 | out = F.relu(self.bn1(self.conv1(x))) 123 | out = self.layer1(out) 124 | out = self.layer2(out) 125 | out = self.layer3(out) 126 | out = self.layer4(out, fake_relu=fake_relu) 127 | out = F.avg_pool2d(out, 4) 128 | pre_out = out.view(out.size(0), -1) 129 | final = self.linear(pre_out) 130 | if with_latent: 131 | return final, pre_out 132 | return final 133 | 134 | class ResNet_mid(nn.Module): 135 | def __init__(self, block, num_blocks, num_classes=10, n_bits=8): 136 | super(ResNet_mid, self).__init__() 137 | self.in_planes = 64 138 | self.n_bits = n_bits 139 | 140 | self.conv1 = quan_Conv2d(3, 64, kernel_size=3, 141 | stride=1, padding=1, bias=False, n_bits=self.n_bits) 142 | self.bn1 = nn.BatchNorm2d(64) 143 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 144 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 145 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 146 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 147 | self.linear = quan_Linear(512*block.expansion, num_classes, n_bits=self.n_bits) 148 | 149 | def _make_layer(self, block, planes, num_blocks, stride): 150 | strides = [stride] + [1]*(num_blocks-1) 151 | layers = [] 152 | for stride in strides: 153 | layers.append(block(self.in_planes, planes, stride, n_bits=self.n_bits)) 154 | self.in_planes = planes * block.expansion 155 | return SequentialWithArgs(*layers) 156 | 157 | def forward(self, x): 158 | out = F.relu(self.bn1(self.conv1(x))) 159 | out = self.layer1(out) 160 | out = self.layer2(out) 161 | out = self.layer3(out) 162 | out = self.layer4(out) 163 | out = F.avg_pool2d(out, 4) 164 | out = out.view(out.size(0), -1) 165 | return out 166 | 167 | 168 | def ResNet18_quan(num_classes=10, n_bits=8): 169 | return ResNet(BasicBlock, [2, 2, 2, 2], num_classes, n_bits) 170 | 171 | def ResNet18_quan_mid(num_classes=10, n_bits=8): 172 | return ResNet_mid(BasicBlock, [2, 2, 2, 2], num_classes, n_bits) 173 | 174 | def ResNet34_quan(num_classes=10, n_bits=8): 175 | return ResNet(BasicBlock, [3, 4, 6, 3], num_classes, n_bits) 176 | 177 | def ResNet34_quan_mid(num_classes=10, n_bits=8): 178 | return ResNet_mid(BasicBlock, [3, 4, 6, 3], num_classes, n_bits) 179 | 180 | def ResNet50_quan(num_classes=10, n_bits=8): 181 | return ResNet(Bottleneck, [3, 4, 6, 3], num_classes, n_bits) 182 | 183 | 184 | def ResNet101_quan(num_classes=10, n_bits=8): 185 | return ResNet(Bottleneck, [3, 4, 23, 3], num_classes, n_bits) 186 | 187 | 188 | def ResNet152_quan(num_classes=10, n_bits=8): 189 | return ResNet(Bottleneck, [3, 8, 36, 3], num_classes, n_bits) 190 | 191 | 192 | def test(): 193 | net = ResNet18_quan() 194 | y = net(torch.randn(1, 3, 32, 32)) 195 | print(y.size()) 196 | 197 | # test() 198 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | warnings.filterwarnings("ignore") 3 | 4 | import argparse 5 | from tqdm import tqdm 6 | import os 7 | import time 8 | import copy 9 | import random 10 | 11 | import torch 12 | import torch.utils.data 13 | import torchvision.transforms as transforms 14 | import torchvision.datasets as datasets 15 | from models import quan_resnet_cifar, quan_resnet_imagenet, quan_vgg_cifar, quan_vgg_imagenet, resnet_imagenet, vgg_imagenet 16 | from models.quantization import * 17 | import numpy as np 18 | import config 19 | from models.AugLag import augLag_Linear 20 | 21 | show_init_info = True 22 | 23 | def initialize_env(seed: int = 42): 24 | print(f"pid {os.getpid()}") 25 | set_seed(seed) 26 | gpu_select() 27 | 28 | 29 | def set_seed(seed: int = 42) -> None: 30 | np.random.seed(seed) 31 | random.seed(seed) 32 | torch.manual_seed(seed) 33 | torch.cuda.manual_seed(seed) 34 | # When running on the CuDNN backend, two further options must be set 35 | torch.backends.cudnn.deterministic = True 36 | torch.backends.cudnn.benchmark = False 37 | # Set a fixed value for the hash seed 38 | os.environ["PYTHONHASHSEED"] = str(seed) 39 | # print(f"Random seed set as {seed}") 40 | 41 | def gpu_select(): 42 | exp = config.no_use_gpu_id 43 | import pynvml 44 | pynvml.nvmlInit() 45 | 46 | deviceCount = pynvml.nvmlDeviceGetCount() 47 | selected_index = 0 48 | min_used_ratio = 1 49 | for idx in range(deviceCount): 50 | if idx in exp or str(idx) in exp:continue 51 | handle = pynvml.nvmlDeviceGetHandleByIndex(idx) 52 | info = pynvml.nvmlDeviceGetMemoryInfo(handle) 53 | used_ratio = info.used / info.total 54 | if used_ratio < min_used_ratio: 55 | min_used_ratio = used_ratio 56 | selected_index = idx 57 | 58 | os.environ["CUDA_VISIBLE_DEVICES"] = str(selected_index) 59 | 60 | 61 | def load_state_dict(net, checkpoint): 62 | global show_init_info 63 | if show_init_info: 64 | if 'acc' in checkpoint.keys(): 65 | print(f"Float-point model Accuracy: {checkpoint['acc']}") 66 | show_init_info = False 67 | state_tmp = net.state_dict() 68 | 69 | b_ws = {} 70 | for key in state_tmp.keys(): 71 | if 'b_w' in key: 72 | b_ws[key] = state_tmp[key] 73 | 74 | if 'state_dict' in checkpoint.keys(): 75 | state_tmp.update(checkpoint['state_dict']) 76 | elif 'net' in checkpoint.keys(): 77 | state_tmp.update(checkpoint['net']) 78 | else: 79 | state_tmp.update(checkpoint) 80 | state_tmp.update(b_ws) 81 | 82 | net.load_state_dict(state_tmp) 83 | return net 84 | 85 | def load_model_cifar10(arch, bit_length, ck_path=''): 86 | model_path = config.model_root 87 | arch = arch + "_quan_mid" 88 | 89 | if 'ResNet' in arch: 90 | model = torch.nn.DataParallel(quan_resnet_cifar.__dict__[arch](10, bit_length)) 91 | elif 'vgg' in arch: 92 | model = torch.nn.DataParallel(quan_vgg_cifar.__dict__[arch](bit_length)) 93 | if isinstance(model, torch.nn.DataParallel): 94 | model = model.module 95 | if ck_path == '': 96 | ck_path = os.path.join(model_path, "model.th") 97 | ck_dict = torch.load(ck_path) 98 | model = load_state_dict(model, ck_dict) 99 | model.cuda() 100 | 101 | for m in model.modules(): 102 | if isinstance(m, quan_Linear): 103 | m.__reset_stepsize__() 104 | m.__reset_weight__() 105 | weight = m.weight.data.detach().cpu().numpy() 106 | bias = m.bias.data.detach().cpu().numpy() 107 | # step_size = np.array([m.step_size.detach().cpu().numpy()])[0] 108 | step_size = np.float32(m.step_size.detach().cpu().numpy()) 109 | return weight, bias, step_size 110 | 111 | def load_model_imagenet(arch, bit_length, ck_path=''): 112 | arch = arch + "_quan_mid" 113 | if 'ResNet' in arch: 114 | model = torch.nn.DataParallel(quan_resnet_imagenet.__dict__[arch](num_classes=1000, n_bits=bit_length)) 115 | elif 'vgg' in arch: 116 | model = torch.nn.DataParallel(quan_vgg_imagenet.__dict__[arch](bit_length)) 117 | if isinstance(model, torch.nn.DataParallel): 118 | model = model.module 119 | ck_dict = torch.load(ck_path) 120 | model = load_state_dict(model, ck_dict) 121 | model.cuda() 122 | 123 | for m in model.modules(): 124 | if isinstance(m, quan_Linear): 125 | m.__reset_stepsize__() 126 | m.__reset_weight__() 127 | weight = m.weight.data.detach().cpu().numpy() 128 | bias = m.bias.data.detach().cpu().numpy() 129 | # step_size = np.array([m.step_size.detach().cpu().numpy()])[0] 130 | step_size = np.float32(m.step_size.detach().cpu().numpy()) 131 | return weight, bias, step_size 132 | 133 | 134 | 135 | def load_model(dc, arch, bit_length, ck_path): 136 | if dc == 'cifar10': 137 | return load_model_cifar10(arch, bit_length, ck_path) 138 | elif dc == 'imagenet': 139 | return load_model_imagenet(arch, bit_length, ck_path) 140 | 141 | def load_data_cifar10(arch, bit_length, ck_path): 142 | mid_dim = { 143 | 'ResNet18':1 * 512, 144 | 'vgg16':512, 145 | 'vgg19':512, 146 | 'ResNet34':1 * 512, 147 | 'ResNet50':4 * 512 148 | }[arch] 149 | arch = arch + "_quan_mid" 150 | if 'ResNet' in arch: 151 | model = torch.nn.DataParallel(quan_resnet_cifar.__dict__[arch](10, bit_length)) 152 | elif 'vgg' in arch: 153 | model = torch.nn.DataParallel(quan_vgg_cifar.__dict__[arch](bit_length)) 154 | 155 | if isinstance(model, torch.nn.DataParallel): 156 | model = model.module 157 | ck_dict = torch.load(ck_path) 158 | model = load_state_dict(model, ck_dict) 159 | model.cuda() 160 | 161 | normalize = transforms.Normalize(mean=[0.4914, 0.4822, 0.4465], 162 | std=[0.2023, 0.1994, 0.2010]) 163 | 164 | val_set = datasets.CIFAR10(root=config.cifar_root, train=False, transform=transforms.Compose([ 165 | transforms.ToTensor(), 166 | normalize, 167 | ]), download=True) 168 | 169 | val_loader = torch.utils.data.DataLoader( 170 | dataset=val_set, 171 | batch_size=256, shuffle=False, pin_memory=True) 172 | 173 | mid_out = np.zeros([10000, mid_dim]) 174 | labels = np.zeros([10000]) 175 | start = 0 176 | model.eval() 177 | for i, (input, target) in tqdm(enumerate(val_loader)): 178 | input_var = torch.autograd.Variable(input, volatile=True).cuda() 179 | 180 | # compute output before FC layer. 181 | output = model(input_var) 182 | mid_out[start: start + 256] = output.detach().cpu().numpy() 183 | 184 | labels[start: start + 256] = target.numpy() 185 | start += 256 186 | 187 | mid_out = torch.tensor(mid_out).float().cuda() 188 | labels = torch.tensor(labels).float() 189 | 190 | return mid_out, labels 191 | 192 | 193 | def load_data(dc, arch, bit_length, ck_path): 194 | if dc == 'cifar10': 195 | return load_data_cifar10(arch, bit_length, ck_path) 196 | 197 | def load_clean_output(bit_length, weight, bias, step_size, all_data, args): 198 | tmp_path = os.path.join(config.intermediate_results, f"{args.ck_path.split('/')[-1]}_{bit_length}_clean_output.pth") 199 | if not os.path.exists(tmp_path): 200 | auglag_st = augLag_Linear(bit_length, weight, bias, step_size, init=True).cuda() 201 | clean_output = auglag_st(all_data).detach().cpu().numpy() 202 | torch.save(clean_output, tmp_path) 203 | else: 204 | clean_output = torch.load(tmp_path) 205 | return clean_output 206 | 207 | def load_auglag_st(bit_length, weight, bias, step_size, args): 208 | tmp_path = os.path.join(config.intermediate_results, f"{args.ck_path.split('/')[-1]}_{bit_length}_auglag_st.pth") 209 | if not os.path.exists(tmp_path): 210 | auglag_st = augLag_Linear(bit_length, weight, bias, step_size, init=True).cuda() 211 | torch.save(auglag_st, tmp_path) 212 | else: 213 | auglag_st = torch.load(tmp_path) 214 | return auglag_st 215 | -------------------------------------------------------------------------------- /models/quan_resnet_imagenet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import torch 4 | import torch.nn as nn 5 | import torchvision.models 6 | 7 | import math 8 | from models.quantization import * 9 | 10 | 11 | __all__ = [ 12 | 'ResNet', 'resnet18_quan_mid', 'resnet34_quan_mid', 'resnet18_quan', 'resnet34_quan', 'resnet50', 'resnet101', 13 | 'resnet152' 14 | ] 15 | # you need to download the models to ~/.torch/models 16 | # model_urls = { 17 | # 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 18 | # 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 19 | # 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 20 | # 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 21 | # 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 22 | # } 23 | models_dir = os.path.expanduser('~/.torch/models') 24 | model_name = { 25 | 'resnet18': 'resnet18-f37072fd.pth', 26 | 'resnet34': 'resnet34-b627a593.pth', 27 | 'resnet50': 'resnet50-19c8e357.pth', 28 | 'resnet101': 'resnet101-5d3b4d8f.pth', 29 | 'resnet152': 'resnet152-b121ed2d.pth', 30 | } 31 | 32 | 33 | def conv3x3(in_planes, out_planes, stride=1, n_bits=8): 34 | """3x3 convolution with padding""" 35 | return quan_Conv2d(in_planes, 36 | out_planes, 37 | kernel_size=3, 38 | stride=stride, 39 | padding=1, 40 | bias=False, 41 | n_bits=n_bits) 42 | 43 | def conv1x1(in_planes, out_planes, stride=1, n_bits=8): 44 | """1x1 convolution""" 45 | return quan_Conv2d(in_planes, 46 | out_planes, 47 | kernel_size=1, 48 | stride=stride, 49 | bias=False, 50 | n_bits=n_bits) 51 | 52 | class BasicBlock(nn.Module): 53 | expansion = 1 54 | 55 | def __init__(self, inplanes, planes, stride=1, downsample=None, n_bits=8): 56 | super(BasicBlock, self).__init__() 57 | self.conv1 = conv3x3(inplanes, planes, stride, n_bits) 58 | self.bn1 = nn.BatchNorm2d(planes) 59 | self.relu = nn.ReLU(inplace=True) 60 | self.conv2 = conv3x3(planes, planes, n_bits=n_bits) 61 | self.bn2 = nn.BatchNorm2d(planes) 62 | self.downsample = downsample 63 | self.stride = stride 64 | 65 | def forward(self, x): 66 | residual = x 67 | 68 | out = self.conv1(x) 69 | out = self.bn1(out) 70 | out = self.relu(out) 71 | 72 | out = self.conv2(out) 73 | out = self.bn2(out) 74 | 75 | if self.downsample is not None: 76 | residual = self.downsample(x) 77 | 78 | out += residual 79 | out = self.relu(out) 80 | 81 | return out 82 | 83 | 84 | class Bottleneck(nn.Module): 85 | expansion = 4 86 | 87 | def __init__(self, inplanes, planes, stride=1, downsample=None, n_bits=8): 88 | super(Bottleneck, self).__init__() 89 | self.conv1 = quan_Conv2d(inplanes, planes, kernel_size=1, bias=False, n_bits=n_bits) 90 | self.bn1 = nn.BatchNorm2d(planes) 91 | self.conv2 = quan_Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False, n_bits=n_bits) 92 | self.bn2 = nn.BatchNorm2d(planes) 93 | self.conv3 = quan_Conv2d(planes, planes * 4, kernel_size=1, bias=False, n_bits=n_bits) 94 | self.bn3 = nn.BatchNorm2d(planes * 4) 95 | self.relu = nn.ReLU(inplace=True) 96 | self.downsample = downsample 97 | self.stride = stride 98 | 99 | def forward(self, x): 100 | residual = x 101 | 102 | out = self.conv1(x) 103 | out = self.bn1(out) 104 | out = self.relu(out) 105 | 106 | out = self.conv2(out) 107 | out = self.bn2(out) 108 | out = self.relu(out) 109 | 110 | out = self.conv3(out) 111 | out = self.bn3(out) 112 | 113 | if self.downsample is not None: 114 | residual = self.downsample(x) 115 | 116 | out += residual 117 | out = self.relu(out) 118 | 119 | return out 120 | 121 | 122 | class ResNet_mid(nn.Module): 123 | 124 | def __init__(self, block, layers, num_classes=1000, n_bits=8): 125 | super(ResNet_mid, self).__init__() 126 | self.inplanes = 64 127 | self.n_bits = n_bits 128 | 129 | self.conv1 = quan_Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False, n_bits=n_bits) 130 | self.bn1 = nn.BatchNorm2d(64) 131 | self.relu = nn.ReLU(inplace=True) 132 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 133 | self.layer1 = self._make_layer(block, 64, layers[0]) 134 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 135 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 136 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 137 | self.avgpool = nn.AvgPool2d(7, stride=1) 138 | self.fc = quan_Linear(512 * block.expansion, num_classes, n_bits=n_bits) 139 | 140 | for m in self.modules(): 141 | if isinstance(m, nn.Conv2d): 142 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 143 | m.weight.data.normal_(0, math.sqrt(2. / n)) 144 | elif isinstance(m, nn.BatchNorm2d): 145 | m.weight.data.fill_(1) 146 | m.bias.data.zero_() 147 | 148 | def _make_layer(self, block, planes, blocks, stride=1): 149 | downsample = None 150 | if stride != 1 or self.inplanes != planes * block.expansion: 151 | downsample = nn.Sequential( 152 | quan_Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, 153 | stride=stride, bias=False, n_bits=self.n_bits), 154 | nn.BatchNorm2d(planes * block.expansion), 155 | ) 156 | 157 | layers = [] 158 | layers.append(block(self.inplanes, planes, stride, downsample, self.n_bits)) 159 | self.inplanes = planes * block.expansion 160 | for i in range(1, blocks): 161 | layers.append(block(self.inplanes, planes)) 162 | 163 | return nn.Sequential(*layers) 164 | 165 | def forward(self, x): 166 | x = self.conv1(x) 167 | x = self.bn1(x) 168 | x = self.relu(x) 169 | x = self.maxpool(x) 170 | 171 | x = self.layer1(x) 172 | x = self.layer2(x) 173 | x = self.layer3(x) 174 | x = self.layer4(x) 175 | 176 | x = self.avgpool(x) 177 | x = x.view(x.size(0), -1) 178 | 179 | return x 180 | 181 | class ResNet(nn.Module): 182 | 183 | def __init__(self, block, layers, num_classes=1000, n_bits=8): 184 | super(ResNet, self).__init__() 185 | self.inplanes = 64 186 | self.n_bits = n_bits 187 | 188 | self.conv1 = quan_Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False, n_bits=n_bits) 189 | self.bn1 = nn.BatchNorm2d(64) 190 | self.relu = nn.ReLU(inplace=True) 191 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 192 | self.layer1 = self._make_layer(block, 64, layers[0]) 193 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 194 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 195 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 196 | self.avgpool = nn.AvgPool2d(7, stride=1) 197 | self.fc = quan_Linear(512 * block.expansion, num_classes, n_bits=n_bits) 198 | 199 | for m in self.modules(): 200 | if isinstance(m, nn.Conv2d): 201 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 202 | m.weight.data.normal_(0, math.sqrt(2. / n)) 203 | elif isinstance(m, nn.BatchNorm2d): 204 | m.weight.data.fill_(1) 205 | m.bias.data.zero_() 206 | 207 | def _make_layer(self, block, planes, blocks, stride=1): 208 | downsample = None 209 | if stride != 1 or self.inplanes != planes * block.expansion: 210 | downsample = nn.Sequential( 211 | quan_Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, 212 | stride=stride, bias=False, n_bits=self.n_bits), 213 | nn.BatchNorm2d(planes * block.expansion), 214 | ) 215 | 216 | layers = [] 217 | layers.append(block(self.inplanes, planes, stride, downsample, self.n_bits)) 218 | self.inplanes = planes * block.expansion 219 | for i in range(1, blocks): 220 | layers.append(block(self.inplanes, planes)) 221 | 222 | return nn.Sequential(*layers) 223 | 224 | def forward(self, x): 225 | x = self.conv1(x) 226 | x = self.bn1(x) 227 | x = self.relu(x) 228 | x = self.maxpool(x) 229 | 230 | x = self.layer1(x) 231 | x = self.layer2(x) 232 | x = self.layer3(x) 233 | x = self.layer4(x) 234 | 235 | x = self.avgpool(x) 236 | x = x.view(x.size(0), -1) 237 | x = self.fc(x) 238 | 239 | return x 240 | 241 | 242 | def ResNet18_quan(**kwargs): 243 | """Constructs a ResNet-18 model. 244 | 245 | Args: 246 | pretrained (bool): If True, returns a model pre-trained on ImageNet 247 | """ 248 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) 249 | 250 | return model 251 | 252 | def ResNet18_quan_mid( **kwargs): 253 | """Constructs a ResNet-18 model. 254 | 255 | Args: 256 | pretrained (bool): If True, returns a model pre-trained on ImageNet 257 | """ 258 | model = ResNet_mid(BasicBlock, [2, 2, 2, 2], **kwargs) 259 | 260 | return model 261 | 262 | 263 | 264 | def ResNet34_quan(**kwargs): 265 | """Constructs a ResNet-34 model. 266 | 267 | Args: 268 | pretrained (bool): If True, returns a model pre-trained on ImageNet 269 | """ 270 | model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) 271 | 272 | return model 273 | 274 | def ResNet34_quan_mid(**kwargs): 275 | """Constructs a ResNet-34 model. 276 | 277 | Args: 278 | pretrained (bool): If True, returns a model pre-trained on ImageNet 279 | """ 280 | model = ResNet_mid(BasicBlock, [3, 4, 6, 3], **kwargs) 281 | return model 282 | 283 | 284 | def ResNet50(**kwargs): 285 | """Constructs a ResNet-50 model. 286 | 287 | Args: 288 | pretrained (bool): If True, returns a model pre-trained on ImageNet 289 | """ 290 | model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) 291 | return model 292 | 293 | def ResNet50_quan_mid(**kwargs): 294 | """Constructs a ResNet-50 model. 295 | 296 | Args: 297 | pretrained (bool): If True, returns a model pre-trained on ImageNet 298 | """ 299 | model = ResNet_mid(Bottleneck, [3, 4, 6, 3], **kwargs) 300 | return model 301 | 302 | 303 | def resnet101(pretrained=False, **kwargs): 304 | """Constructs a ResNet-101 model. 305 | 306 | Args: 307 | pretrained (bool): If True, returns a model pre-trained on ImageNet 308 | """ 309 | model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) 310 | if pretrained: 311 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet101']))) 312 | return model 313 | 314 | 315 | def resnet152(pretrained=False, **kwargs): 316 | """Constructs a ResNet-152 model. 317 | 318 | Args: 319 | pretrained (bool): If True, returns a model pre-trained on ImageNet 320 | """ 321 | model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) 322 | if pretrained: 323 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet152']))) 324 | return model -------------------------------------------------------------------------------- /models/resnet_imagenet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import torch 4 | import torch.nn as nn 5 | import torchvision.models 6 | 7 | 8 | __all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152'] 9 | 10 | # you need to download the models to ~/.torch/models 11 | # model_urls = { 12 | # 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 13 | # 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 14 | # 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 15 | # 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 16 | # 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 17 | # } 18 | models_dir = os.path.expanduser('~/.torch/models') 19 | model_name = { 20 | 'resnet18': 'resnet18-5c106cde.pth', 21 | 'resnet34': 'resnet34-333f7ec4.pth', 22 | 'resnet50': 'resnet50-19c8e357.pth', 23 | 'resnet101': 'resnet101-5d3b4d8f.pth', 24 | 'resnet152': 'resnet152-b121ed2d.pth', 25 | } 26 | 27 | class FakeReLU(torch.autograd.Function): 28 | @staticmethod 29 | def forward(ctx, input): 30 | return input.clamp(min=0) 31 | 32 | @staticmethod 33 | def backward(ctx, grad_output): 34 | return grad_output 35 | 36 | class SequentialWithArgs(torch.nn.Sequential): 37 | def forward(self, input, *args, **kwargs): 38 | vs = list(self._modules.values()) 39 | l = len(vs) 40 | for i in range(l): 41 | if i == l-1: 42 | input = vs[i](input, *args, **kwargs) 43 | else: 44 | input = vs[i](input) 45 | return input 46 | 47 | 48 | def conv3x3(in_planes, out_planes, stride=1): 49 | """3x3 convolution with padding""" 50 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) 51 | 52 | 53 | class BasicBlock(nn.Module): 54 | expansion = 1 55 | 56 | def __init__(self, inplanes, planes, stride=1, downsample=None): 57 | super(BasicBlock, self).__init__() 58 | self.conv1 = conv3x3(inplanes, planes, stride) 59 | self.bn1 = nn.BatchNorm2d(planes) 60 | self.relu = nn.ReLU(inplace=True) 61 | self.conv2 = conv3x3(planes, planes) 62 | self.bn2 = nn.BatchNorm2d(planes) 63 | self.downsample = downsample 64 | self.stride = stride 65 | 66 | def forward(self, x, fake_relu=False): 67 | residual = x 68 | 69 | out = self.conv1(x) 70 | out = self.bn1(out) 71 | out = self.relu(out) 72 | 73 | out = self.conv2(out) 74 | out = self.bn2(out) 75 | 76 | if self.downsample is not None: 77 | residual = self.downsample(x) 78 | 79 | out += residual 80 | if fake_relu: 81 | return FakeReLU.apply(out) 82 | out = self.relu(out) 83 | return out 84 | 85 | 86 | class Bottleneck(nn.Module): 87 | expansion = 4 88 | 89 | def __init__(self, inplanes, planes, stride=1, downsample=None): 90 | super(Bottleneck, self).__init__() 91 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 92 | self.bn1 = nn.BatchNorm2d(planes) 93 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 94 | self.bn2 = nn.BatchNorm2d(planes) 95 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 96 | self.bn3 = nn.BatchNorm2d(planes * 4) 97 | self.relu = nn.ReLU(inplace=True) 98 | self.downsample = downsample 99 | self.stride = stride 100 | 101 | def forward(self, x, fake_relu=False): 102 | residual = x 103 | 104 | out = self.conv1(x) 105 | out = self.bn1(out) 106 | out = self.relu(out) 107 | 108 | out = self.conv2(out) 109 | out = self.bn2(out) 110 | out = self.relu(out) 111 | 112 | out = self.conv3(out) 113 | out = self.bn3(out) 114 | 115 | if self.downsample is not None: 116 | residual = self.downsample(x) 117 | 118 | out += residual 119 | if fake_relu: 120 | return FakeReLU.apply(out) 121 | out = self.relu(out) 122 | return out 123 | 124 | 125 | class ResNet(nn.Module): 126 | 127 | def __init__(self, block, layers, num_classes=1000): 128 | super(ResNet, self).__init__() 129 | self.inplanes = 64 130 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) 131 | self.bn1 = nn.BatchNorm2d(64) 132 | self.relu = nn.ReLU(inplace=True) 133 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 134 | self.layer1 = self._make_layer(block, 64, layers[0]) 135 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 136 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 137 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 138 | self.avgpool = nn.AvgPool2d(7, stride=1) 139 | self.fc = nn.Linear(512 * block.expansion, num_classes) 140 | 141 | for m in self.modules(): 142 | if isinstance(m, nn.Conv2d): 143 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 144 | m.weight.data.normal_(0, math.sqrt(2. / n)) 145 | elif isinstance(m, nn.BatchNorm2d): 146 | m.weight.data.fill_(1) 147 | m.bias.data.zero_() 148 | 149 | def _make_layer(self, block, planes, blocks, stride=1): 150 | downsample = None 151 | if stride != 1 or self.inplanes != planes * block.expansion: 152 | downsample = nn.Sequential( 153 | nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, 154 | stride=stride, bias=False), 155 | nn.BatchNorm2d(planes * block.expansion), 156 | ) 157 | 158 | layers = [] 159 | layers.append(block(self.inplanes, planes, stride, downsample)) 160 | self.inplanes = planes * block.expansion 161 | for i in range(1, blocks): 162 | layers.append(block(self.inplanes, planes)) 163 | 164 | return SequentialWithArgs(*layers) 165 | 166 | def forward(self, x, with_latent=False, fake_relu=False): 167 | x = self.conv1(x) 168 | x = self.bn1(x) 169 | x = self.relu(x) 170 | x = self.maxpool(x) 171 | 172 | x = self.layer1(x) 173 | x = self.layer2(x) 174 | x = self.layer3(x) 175 | x = self.layer4(x, fake_relu=fake_relu) 176 | 177 | x = self.avgpool(x) 178 | x = x.view(x.size(0), -1) 179 | final = self.fc(x) 180 | if with_latent: 181 | return final, x 182 | return final 183 | 184 | 185 | class ResNet_mid(nn.Module): 186 | 187 | def __init__(self, block, layers, num_classes=1000): 188 | super(ResNet_mid, self).__init__() 189 | self.inplanes = 64 190 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) 191 | self.bn1 = nn.BatchNorm2d(64) 192 | self.relu = nn.ReLU(inplace=True) 193 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 194 | self.layer1 = self._make_layer(block, 64, layers[0]) 195 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 196 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 197 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 198 | self.avgpool = nn.AvgPool2d(7, stride=1) 199 | self.fc = nn.Linear(512 * block.expansion, num_classes) 200 | 201 | for m in self.modules(): 202 | if isinstance(m, nn.Conv2d): 203 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 204 | m.weight.data.normal_(0, math.sqrt(2. / n)) 205 | elif isinstance(m, nn.BatchNorm2d): 206 | m.weight.data.fill_(1) 207 | m.bias.data.zero_() 208 | 209 | def _make_layer(self, block, planes, blocks, stride=1): 210 | downsample = None 211 | if stride != 1 or self.inplanes != planes * block.expansion: 212 | downsample = nn.Sequential( 213 | nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, 214 | stride=stride, bias=False), 215 | nn.BatchNorm2d(planes * block.expansion), 216 | ) 217 | 218 | layers = [] 219 | layers.append(block(self.inplanes, planes, stride, downsample)) 220 | self.inplanes = planes * block.expansion 221 | for i in range(1, blocks): 222 | layers.append(block(self.inplanes, planes)) 223 | 224 | return nn.Sequential(*layers) 225 | 226 | def forward(self, x): 227 | x = self.conv1(x) 228 | x = self.bn1(x) 229 | x = self.relu(x) 230 | x = self.maxpool(x) 231 | 232 | x = self.layer1(x) 233 | x = self.layer2(x) 234 | x = self.layer3(x) 235 | x = self.layer4(x) 236 | 237 | x = self.avgpool(x) 238 | x = x.view(x.size(0), -1) 239 | 240 | return x 241 | 242 | 243 | def resnet18(pretrained=False, **kwargs): 244 | """Constructs a ResNet-18 model. 245 | 246 | Args: 247 | pretrained (bool): If True, returns a model pre-trained on ImageNet 248 | """ 249 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) 250 | if pretrained: 251 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet18']))) 252 | return model 253 | def resnet18_mid(**kwargs): 254 | """Constructs a ResNet-34 model. 255 | 256 | Args: 257 | pretrained (bool): If True, returns a model pre-trained on ImageNet 258 | """ 259 | model = ResNet_mid(BasicBlock, [2, 2, 2, 2], **kwargs) 260 | return model 261 | 262 | def resnet34(pretrained=False, **kwargs): 263 | """Constructs a ResNet-34 model. 264 | 265 | Args: 266 | pretrained (bool): If True, returns a model pre-trained on ImageNet 267 | """ 268 | model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) 269 | if pretrained: 270 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet34']))) 271 | return model 272 | 273 | def resnet34_mid(**kwargs): 274 | """Constructs a ResNet-34 model. 275 | 276 | Args: 277 | pretrained (bool): If True, returns a model pre-trained on ImageNet 278 | """ 279 | model = ResNet_mid(BasicBlock, [3, 4, 6, 3], **kwargs) 280 | return model 281 | 282 | def resnet50(pretrained=False, **kwargs): 283 | """Constructs a ResNet-50 model. 284 | 285 | Args: 286 | pretrained (bool): If True, returns a model pre-trained on ImageNet 287 | """ 288 | model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) 289 | if pretrained: 290 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet50']))) 291 | return model 292 | 293 | 294 | def resnet101(pretrained=False, **kwargs): 295 | """Constructs a ResNet-101 model. 296 | 297 | Args: 298 | pretrained (bool): If True, returns a model pre-trained on ImageNet 299 | """ 300 | model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) 301 | if pretrained: 302 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet101']))) 303 | return model 304 | 305 | 306 | def resnet152(pretrained=False, **kwargs): 307 | """Constructs a ResNet-152 model. 308 | 309 | Args: 310 | pretrained (bool): If True, returns a model pre-trained on ImageNet 311 | """ 312 | model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) 313 | if pretrained: 314 | model.load_state_dict(torch.load(os.path.join(models_dir, model_name['resnet152']))) 315 | return model 316 | 317 | ResNet34 = resnet34 -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /atk_insts/cifar10_atks.txt: -------------------------------------------------------------------------------- 1 | 0 9335 2 | 0 9330 3 | 0 7823 4 | 0 1598 5 | 0 3520 6 | 0 4381 7 | 0 1899 8 | 0 8298 9 | 0 3830 10 | 0 6389 11 | 0 7897 12 | 0 9327 13 | 0 117 14 | 0 6551 15 | 0 3762 16 | 0 2691 17 | 0 8137 18 | 0 9486 19 | 0 9532 20 | 0 4083 21 | 0 569 22 | 0 5194 23 | 0 4023 24 | 0 3330 25 | 0 2483 26 | 0 5360 27 | 0 5887 28 | 0 9133 29 | 0 5674 30 | 0 4414 31 | 0 8897 32 | 0 7331 33 | 0 2057 34 | 0 9050 35 | 0 8553 36 | 0 717 37 | 0 5433 38 | 0 9316 39 | 0 7396 40 | 0 8936 41 | 0 9066 42 | 0 432 43 | 0 5662 44 | 0 431 45 | 0 6094 46 | 0 3868 47 | 0 4164 48 | 0 9073 49 | 0 8526 50 | 0 2263 51 | 0 1970 52 | 0 3826 53 | 0 1283 54 | 0 3959 55 | 0 6084 56 | 0 1612 57 | 0 6727 58 | 0 688 59 | 0 2934 60 | 0 2515 61 | 0 9475 62 | 0 3224 63 | 0 8652 64 | 0 4615 65 | 0 1380 66 | 0 6556 67 | 0 1610 68 | 0 1174 69 | 0 4089 70 | 0 4146 71 | 0 5588 72 | 0 9941 73 | 0 8096 74 | 0 4483 75 | 0 8125 76 | 0 8463 77 | 0 897 78 | 0 4 79 | 0 1074 80 | 0 3370 81 | 0 5276 82 | 0 1635 83 | 0 5945 84 | 0 6286 85 | 0 6335 86 | 0 1124 87 | 0 4509 88 | 0 7452 89 | 0 487 90 | 0 2310 91 | 0 9033 92 | 0 6558 93 | 0 949 94 | 0 8786 95 | 0 1309 96 | 0 5062 97 | 0 6354 98 | 0 7932 99 | 0 3616 100 | 0 5340 101 | 1 2958 102 | 1 7679 103 | 1 7377 104 | 1 2766 105 | 1 3823 106 | 1 7760 107 | 1 7444 108 | 1 2954 109 | 1 6785 110 | 1 371 111 | 1 6663 112 | 1 1348 113 | 1 5247 114 | 1 2180 115 | 1 9976 116 | 1 8189 117 | 1 2869 118 | 1 5576 119 | 1 8589 120 | 1 4534 121 | 1 4132 122 | 1 4254 123 | 1 7096 124 | 1 2296 125 | 1 899 126 | 1 1102 127 | 1 5328 128 | 1 3422 129 | 1 6240 130 | 1 1324 131 | 1 9459 132 | 1 4854 133 | 1 228 134 | 1 5989 135 | 1 1512 136 | 1 3018 137 | 1 696 138 | 1 1366 139 | 1 7306 140 | 1 9024 141 | 1 2074 142 | 1 6107 143 | 1 9550 144 | 1 7751 145 | 1 8231 146 | 1 6512 147 | 1 8340 148 | 1 1030 149 | 1 7870 150 | 1 2821 151 | 1 22 152 | 1 3016 153 | 1 7972 154 | 1 200 155 | 1 4277 156 | 1 4984 157 | 1 5106 158 | 1 6696 159 | 1 2259 160 | 1 5578 161 | 1 3287 162 | 1 5213 163 | 1 3179 164 | 1 9383 165 | 1 2064 166 | 1 7542 167 | 1 9798 168 | 1 3439 169 | 1 8746 170 | 1 341 171 | 1 4265 172 | 1 14 173 | 1 2726 174 | 1 16 175 | 1 3590 176 | 1 9051 177 | 1 3919 178 | 1 4130 179 | 1 5877 180 | 1 4617 181 | 1 1955 182 | 1 5153 183 | 1 425 184 | 1 8947 185 | 1 2158 186 | 1 5522 187 | 1 6160 188 | 1 5408 189 | 1 6974 190 | 1 1720 191 | 1 956 192 | 1 8691 193 | 1 4510 194 | 1 8054 195 | 1 8675 196 | 1 2389 197 | 1 1296 198 | 1 6771 199 | 1 1315 200 | 1 7432 201 | 2 8339 202 | 2 389 203 | 2 8980 204 | 2 7160 205 | 2 2922 206 | 2 4169 207 | 2 376 208 | 2 827 209 | 2 778 210 | 2 9720 211 | 2 7804 212 | 2 9329 213 | 2 6827 214 | 2 1066 215 | 2 4897 216 | 2 8762 217 | 2 7332 218 | 2 5218 219 | 2 4627 220 | 2 1358 221 | 2 768 222 | 2 434 223 | 2 6531 224 | 2 3107 225 | 2 1160 226 | 2 6927 227 | 2 8061 228 | 2 3191 229 | 2 4989 230 | 2 9400 231 | 2 6514 232 | 2 1338 233 | 2 7568 234 | 2 9727 235 | 2 4825 236 | 2 523 237 | 2 9696 238 | 2 1668 239 | 2 5366 240 | 2 7453 241 | 2 1054 242 | 2 2858 243 | 2 4210 244 | 2 8313 245 | 2 5731 246 | 2 6944 247 | 2 2119 248 | 2 363 249 | 2 2843 250 | 2 4527 251 | 2 2026 252 | 2 3023 253 | 2 7509 254 | 2 3460 255 | 2 4981 256 | 2 1750 257 | 2 8259 258 | 2 2503 259 | 2 1036 260 | 2 5639 261 | 2 700 262 | 2 8381 263 | 2 7485 264 | 2 7770 265 | 2 3669 266 | 2 7550 267 | 2 1085 268 | 2 381 269 | 2 6235 270 | 2 6306 271 | 2 7626 272 | 2 9284 273 | 2 3982 274 | 2 6434 275 | 2 7978 276 | 2 8978 277 | 2 4763 278 | 2 5381 279 | 2 3317 280 | 2 7226 281 | 2 7388 282 | 2 4225 283 | 2 9841 284 | 2 7531 285 | 2 3569 286 | 2 7537 287 | 2 4555 288 | 2 6278 289 | 2 9079 290 | 2 4406 291 | 2 6274 292 | 2 6909 293 | 2 5483 294 | 2 632 295 | 2 3445 296 | 2 2322 297 | 2 6304 298 | 2 1931 299 | 2 5272 300 | 2 5361 301 | 3 70 302 | 3 8750 303 | 3 9105 304 | 3 3459 305 | 3 51 306 | 3 8968 307 | 3 7098 308 | 3 9341 309 | 3 3142 310 | 3 1209 311 | 3 6572 312 | 3 8387 313 | 3 3693 314 | 3 5321 315 | 3 9099 316 | 3 6539 317 | 3 8850 318 | 3 9735 319 | 3 914 320 | 3 2396 321 | 3 645 322 | 3 6115 323 | 3 1747 324 | 3 1533 325 | 3 5146 326 | 3 6899 327 | 3 1356 328 | 3 7899 329 | 3 1800 330 | 3 1226 331 | 3 9799 332 | 3 2052 333 | 3 2354 334 | 3 3737 335 | 3 7398 336 | 3 477 337 | 3 653 338 | 3 6902 339 | 3 4720 340 | 3 8787 341 | 3 1774 342 | 3 5245 343 | 3 6589 344 | 3 4088 345 | 3 2224 346 | 3 7501 347 | 3 9147 348 | 3 1057 349 | 3 7609 350 | 3 8796 351 | 3 1990 352 | 3 4415 353 | 3 43 354 | 3 4315 355 | 3 4634 356 | 3 9918 357 | 3 8784 358 | 3 1884 359 | 3 3574 360 | 3 2313 361 | 3 6048 362 | 3 8161 363 | 3 4397 364 | 3 5680 365 | 3 1387 366 | 3 4866 367 | 3 2826 368 | 3 979 369 | 3 1211 370 | 3 9035 371 | 3 1365 372 | 3 4433 373 | 3 5012 374 | 3 9625 375 | 3 7428 376 | 3 2432 377 | 3 9189 378 | 3 2451 379 | 3 9688 380 | 3 6353 381 | 3 4187 382 | 3 4669 383 | 3 107 384 | 3 5172 385 | 3 6122 386 | 3 6796 387 | 3 8801 388 | 3 6782 389 | 3 589 390 | 3 8823 391 | 3 997 392 | 3 6866 393 | 3 9117 394 | 3 2820 395 | 3 3844 396 | 3 6227 397 | 3 1017 398 | 3 7292 399 | 3 6192 400 | 3 2879 401 | 4 7712 402 | 4 8081 403 | 4 3373 404 | 4 9174 405 | 4 9106 406 | 4 4713 407 | 4 8312 408 | 4 8225 409 | 4 7164 410 | 4 274 411 | 4 7723 412 | 4 9473 413 | 4 3995 414 | 4 8492 415 | 4 1010 416 | 4 5298 417 | 4 5420 418 | 4 4789 419 | 4 3974 420 | 4 39 421 | 4 9267 422 | 4 7562 423 | 4 7454 424 | 4 2305 425 | 4 6173 426 | 4 3067 427 | 4 5653 428 | 4 5198 429 | 4 524 430 | 4 5156 431 | 4 7641 432 | 4 7995 433 | 4 1319 434 | 4 9938 435 | 4 8065 436 | 4 9347 437 | 4 2566 438 | 4 4672 439 | 4 6411 440 | 4 6046 441 | 4 9514 442 | 4 7533 443 | 4 8896 444 | 4 8971 445 | 4 4613 446 | 4 6405 447 | 4 8009 448 | 4 7690 449 | 4 8986 450 | 4 3388 451 | 4 9826 452 | 4 5716 453 | 4 4477 454 | 4 5277 455 | 4 1413 456 | 4 6571 457 | 4 4601 458 | 4 9926 459 | 4 2084 460 | 4 2042 461 | 4 8246 462 | 4 2210 463 | 4 8773 464 | 4 9719 465 | 4 2333 466 | 4 2444 467 | 4 5180 468 | 4 4286 469 | 4 7015 470 | 4 4026 471 | 4 2521 472 | 4 5802 473 | 4 4550 474 | 4 5517 475 | 4 2507 476 | 4 1711 477 | 4 8221 478 | 4 7945 479 | 4 9758 480 | 4 4859 481 | 4 1697 482 | 4 9779 483 | 4 132 484 | 4 7271 485 | 4 6570 486 | 4 8674 487 | 4 5376 488 | 4 8958 489 | 4 4797 490 | 4 2338 491 | 4 5112 492 | 4 6668 493 | 4 3 494 | 4 1909 495 | 4 5554 496 | 4 5648 497 | 4 3670 498 | 4 2349 499 | 4 5769 500 | 4 513 501 | 5 9702 502 | 5 2773 503 | 5 1261 504 | 5 1468 505 | 5 402 506 | 5 7677 507 | 5 7358 508 | 5 5544 509 | 5 3692 510 | 5 2476 511 | 5 2674 512 | 5 8000 513 | 5 2661 514 | 5 5464 515 | 5 6357 516 | 5 742 517 | 5 2574 518 | 5 2148 519 | 5 3070 520 | 5 9615 521 | 5 8155 522 | 5 4635 523 | 5 1815 524 | 5 1679 525 | 5 2241 526 | 5 4460 527 | 5 7189 528 | 5 559 529 | 5 6431 530 | 5 2431 531 | 5 9307 532 | 5 2363 533 | 5 8062 534 | 5 8645 535 | 5 1521 536 | 5 7373 537 | 5 5678 538 | 5 4215 539 | 5 5579 540 | 5 130 541 | 5 1785 542 | 5 7221 543 | 5 5889 544 | 5 3134 545 | 5 7410 546 | 5 1240 547 | 5 730 548 | 5 968 549 | 5 8376 550 | 5 5917 551 | 5 7815 552 | 5 5395 553 | 5 6284 554 | 5 3886 555 | 5 6102 556 | 5 2166 557 | 5 308 558 | 5 9724 559 | 5 6865 560 | 5 326 561 | 5 8583 562 | 5 1790 563 | 5 981 564 | 5 3792 565 | 5 7475 566 | 5 9395 567 | 5 8099 568 | 5 284 569 | 5 6630 570 | 5 6901 571 | 5 1947 572 | 5 2982 573 | 5 4104 574 | 5 2906 575 | 5 3601 576 | 5 1178 577 | 5 6334 578 | 5 7734 579 | 5 1027 580 | 5 2787 581 | 5 9919 582 | 5 9592 583 | 5 8754 584 | 5 2121 585 | 5 2169 586 | 5 9356 587 | 5 4441 588 | 5 8738 589 | 5 135 590 | 5 1953 591 | 5 1180 592 | 5 1250 593 | 5 74 594 | 5 6844 595 | 5 6886 596 | 5 3188 597 | 5 2547 598 | 5 818 599 | 5 715 600 | 5 9648 601 | 6 932 602 | 6 83 603 | 6 2292 604 | 6 7391 605 | 6 7921 606 | 6 5852 607 | 6 6207 608 | 6 8322 609 | 6 1141 610 | 6 5604 611 | 6 8123 612 | 6 8852 613 | 6 4849 614 | 6 6297 615 | 6 844 616 | 6 1525 617 | 6 1300 618 | 6 2205 619 | 6 386 620 | 6 412 621 | 6 1376 622 | 6 5930 623 | 6 8482 624 | 6 3071 625 | 6 2606 626 | 6 733 627 | 6 7187 628 | 6 3985 629 | 6 1783 630 | 6 5450 631 | 6 4757 632 | 6 9306 633 | 6 211 634 | 6 6071 635 | 6 7584 636 | 6 312 637 | 6 4994 638 | 6 6545 639 | 6 2388 640 | 6 6027 641 | 6 4782 642 | 6 3367 643 | 6 2080 644 | 6 7837 645 | 6 1859 646 | 6 3145 647 | 6 2077 648 | 6 2975 649 | 6 3034 650 | 6 9669 651 | 6 6600 652 | 6 452 653 | 6 8153 654 | 6 6655 655 | 6 6142 656 | 6 898 657 | 6 7300 658 | 6 7026 659 | 6 2391 660 | 6 1372 661 | 6 2724 662 | 6 9063 663 | 6 6708 664 | 6 3829 665 | 6 5617 666 | 6 7254 667 | 6 1437 668 | 6 4418 669 | 6 3042 670 | 6 7644 671 | 6 7020 672 | 6 9309 673 | 6 2985 674 | 6 6079 675 | 6 8445 676 | 6 1018 677 | 6 7106 678 | 6 5330 679 | 6 205 680 | 6 3565 681 | 6 6647 682 | 6 4945 683 | 6 8774 684 | 6 25 685 | 6 7198 686 | 6 8088 687 | 6 7048 688 | 6 6340 689 | 6 1222 690 | 6 5526 691 | 6 3041 692 | 6 3788 693 | 6 7825 694 | 6 281 695 | 6 3476 696 | 6 6680 697 | 6 9717 698 | 6 4015 699 | 6 5130 700 | 6 9825 701 | 7 5079 702 | 7 8456 703 | 7 978 704 | 7 4147 705 | 7 6712 706 | 7 3322 707 | 7 8727 708 | 7 3171 709 | 7 2125 710 | 7 4408 711 | 7 1938 712 | 7 235 713 | 7 8921 714 | 7 9950 715 | 7 7181 716 | 7 7286 717 | 7 1693 718 | 7 4170 719 | 7 5831 720 | 7 2317 721 | 7 801 722 | 7 3588 723 | 7 7163 724 | 7 9219 725 | 7 6068 726 | 7 3647 727 | 7 2636 728 | 7 3421 729 | 7 5231 730 | 7 1483 731 | 7 5089 732 | 7 4985 733 | 7 8243 734 | 7 9090 735 | 7 6582 736 | 7 8331 737 | 7 7884 738 | 7 9328 739 | 7 9992 740 | 7 8890 741 | 7 4123 742 | 7 1912 743 | 7 3038 744 | 7 2997 745 | 7 6090 746 | 7 4522 747 | 7 9378 748 | 7 9489 749 | 7 5020 750 | 7 2257 751 | 7 5239 752 | 7 5059 753 | 7 5911 754 | 7 7424 755 | 7 5480 756 | 7 4145 757 | 7 4430 758 | 7 9705 759 | 7 8794 760 | 7 9834 761 | 7 8888 762 | 7 9940 763 | 7 1271 764 | 7 2307 765 | 7 9722 766 | 7 8647 767 | 7 2709 768 | 7 9068 769 | 7 1538 770 | 7 9483 771 | 7 7376 772 | 7 2868 773 | 7 8636 774 | 7 1733 775 | 7 912 776 | 7 3178 777 | 7 5954 778 | 7 8454 779 | 7 5391 780 | 7 8453 781 | 7 9029 782 | 7 3392 783 | 7 6562 784 | 7 3775 785 | 7 753 786 | 7 6081 787 | 7 9590 788 | 7 9539 789 | 7 1114 790 | 7 1076 791 | 7 1700 792 | 7 6398 793 | 7 2448 794 | 7 5091 795 | 7 4266 796 | 7 6023 797 | 7 1672 798 | 7 2282 799 | 7 9158 800 | 7 4814 801 | 8 7821 802 | 8 1325 803 | 8 4099 804 | 8 2728 805 | 8 6703 806 | 8 6897 807 | 8 5582 808 | 8 2015 809 | 8 4523 810 | 8 5996 811 | 8 9966 812 | 8 399 813 | 8 5175 814 | 8 5753 815 | 8 8594 816 | 8 4819 817 | 8 5909 818 | 8 2370 819 | 8 1537 820 | 8 6247 821 | 8 1016 822 | 8 3862 823 | 8 2878 824 | 8 1751 825 | 8 6617 826 | 8 7389 827 | 8 8173 828 | 8 5324 829 | 8 6670 830 | 8 2214 831 | 8 4930 832 | 8 9584 833 | 8 7649 834 | 8 170 835 | 8 8924 836 | 8 6671 837 | 8 109 838 | 8 4212 839 | 8 7448 840 | 8 6370 841 | 8 3635 842 | 8 2514 843 | 8 286 844 | 8 7153 845 | 8 6584 846 | 8 6951 847 | 8 2970 848 | 8 7211 849 | 8 9815 850 | 8 5819 851 | 8 3733 852 | 8 2071 853 | 8 9665 854 | 8 4362 855 | 8 1524 856 | 8 7077 857 | 8 2642 858 | 8 2902 859 | 8 4175 860 | 8 1083 861 | 8 9470 862 | 8 1831 863 | 8 8093 864 | 8 9299 865 | 8 9760 866 | 8 4194 867 | 8 7767 868 | 8 5048 869 | 8 4342 870 | 8 57 871 | 8 7958 872 | 8 1914 873 | 8 5468 874 | 8 2382 875 | 8 512 876 | 8 329 877 | 8 6627 878 | 8 9191 879 | 8 6916 880 | 8 6140 881 | 8 7500 882 | 8 7212 883 | 8 1435 884 | 8 8504 885 | 8 2335 886 | 8 8429 887 | 8 6688 888 | 8 8473 889 | 8 4465 890 | 8 1958 891 | 8 352 892 | 8 8916 893 | 8 4569 894 | 8 7791 895 | 8 8835 896 | 8 5320 897 | 8 8433 898 | 8 718 899 | 8 7394 900 | 8 4383 901 | 9 606 902 | 9 8341 903 | 9 295 904 | 9 9142 905 | 9 9268 906 | 9 2012 907 | 9 8325 908 | 9 2800 909 | 9 7735 910 | 9 5184 911 | 9 544 912 | 9 9869 913 | 9 3338 914 | 9 2861 915 | 9 4591 916 | 9 6567 917 | 9 2375 918 | 9 1290 919 | 9 1230 920 | 9 8303 921 | 9 7128 922 | 9 9576 923 | 9 1829 924 | 9 3011 925 | 9 3153 926 | 9 955 927 | 9 4553 928 | 9 2424 929 | 9 5204 930 | 9 7741 931 | 9 4993 932 | 9 9488 933 | 9 7846 934 | 9 1592 935 | 9 1200 936 | 9 2563 937 | 9 7561 938 | 9 1333 939 | 9 3631 940 | 9 5752 941 | 9 7148 942 | 9 6000 943 | 9 9885 944 | 9 8324 945 | 9 9733 946 | 9 2504 947 | 9 5502 948 | 9 9042 949 | 9 3798 950 | 9 2187 951 | 9 7857 952 | 9 5179 953 | 9 950 954 | 9 5612 955 | 9 9263 956 | 9 2733 957 | 9 3767 958 | 9 6830 959 | 9 5211 960 | 9 4503 961 | 9 1053 962 | 9 7278 963 | 9 1355 964 | 9 3586 965 | 9 9694 966 | 9 4454 967 | 9 1978 968 | 9 4204 969 | 9 5071 970 | 9 6836 971 | 9 766 972 | 9 6360 973 | 9 6878 974 | 9 3412 975 | 9 6609 976 | 9 1849 977 | 9 842 978 | 9 5919 979 | 9 5983 980 | 9 9964 981 | 9 71 982 | 9 5418 983 | 9 6451 984 | 9 7878 985 | 9 6254 986 | 9 5942 987 | 9 8154 988 | 9 9650 989 | 9 8802 990 | 9 8538 991 | 9 8578 992 | 9 1908 993 | 9 5271 994 | 9 8047 995 | 9 7395 996 | 9 4375 997 | 9 9413 998 | 9 2927 999 | 9 8900 1000 | 9 6481 1001 | --------------------------------------------------------------------------------