├── README.md ├── Inversion ├── .gitignore ├── models │ ├── __init__.py │ ├── lenet.py │ ├── LICENSE │ ├── decoder.py │ ├── vgg.py │ ├── generator.py │ ├── mobilenet.py │ ├── spectral.py │ ├── mnist_generator.py │ ├── cifar10_generator.py │ ├── mobilenetv2.py │ ├── googlenet.py │ ├── resnext.py │ ├── dpn.py │ ├── densenet.py │ ├── shufflenet.py │ ├── wideresnet.py │ ├── efficientnet.py │ ├── senet.py │ ├── resnet.py │ ├── pnasnet.py │ └── shufflenetv2.py ├── generator.py ├── normalizer.py ├── unnormalizer.py ├── loss.py ├── rotator.py ├── classifier.py └── utils.py ├── Training ├── .gitignore ├── default.yaml ├── models │ ├── __init__.py │ ├── lenet.py │ ├── vgg.py │ ├── mobilenet.py │ ├── mobilenetv2.py │ ├── googlenet.py │ ├── resnext.py │ ├── dpn.py │ ├── densenet.py │ ├── shufflenet.py │ ├── wideresnet.py │ ├── efficientnet.py │ ├── senet.py │ ├── resnet.py │ └── pnasnet.py ├── LICENSE ├── cross_test.py ├── traintest.py ├── main.py └── utils.py └── Unlearning ├── .gitignore ├── default.yaml ├── models ├── __init__.py ├── lenet.py ├── decoder.py ├── vgg.py ├── generator.py ├── mobilenet.py ├── spectral.py ├── cifar10_generator.py ├── mnist_generator.py ├── mobilenetv2.py ├── googlenet.py ├── resnext.py ├── dpn.py ├── densenet.py ├── shufflenet.py ├── wideresnet.py ├── efficientnet.py ├── senet.py ├── resnet.py └── pnasnet.py ├── LICENSE ├── cross_test.py ├── main.py └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | # Few-shot-Unlearning -------------------------------------------------------------------------------- /Inversion/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | models/__pycache__/ 3 | -------------------------------------------------------------------------------- /Training/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | models/__pycache__/ 3 | -------------------------------------------------------------------------------- /Unlearning/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | models/__pycache__/ 3 | -------------------------------------------------------------------------------- /Training/default.yaml: -------------------------------------------------------------------------------- 1 | lr: 0.01 2 | epochs: 20 3 | dataset: "MNIST" 4 | task: "NINE" 5 | oracle: False 6 | -------------------------------------------------------------------------------- /Unlearning/default.yaml: -------------------------------------------------------------------------------- 1 | lr: 0.001 2 | epochs: 10 3 | dataset: "MNIST" 4 | task: "SEVEN" 5 | percentage: 0.03 6 | n: 500 7 | oracle: False 8 | normalization: "BatchNorm" 9 | generator_num: 1 10 | pretrained: False 11 | eraselabel: "Noisy" 12 | neutralization: False 13 | neutralization_epochs: 10 14 | -------------------------------------------------------------------------------- /Training/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .vgg import * 2 | from .dpn import * 3 | from .lenet import * 4 | from .senet import * 5 | from .pnasnet import * 6 | from .densenet import * 7 | from .googlenet import * 8 | from .shufflenet import * 9 | from .shufflenetv2 import * 10 | from .resnet import * 11 | from .resnext import * 12 | from .preact_resnet import * 13 | from .mobilenet import * 14 | from .mobilenetv2 import * 15 | from .efficientnet import * 16 | from .wideresnet import * 17 | -------------------------------------------------------------------------------- /Inversion/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .vgg import * 2 | from .dpn import * 3 | from .lenet import * 4 | from .senet import * 5 | from .pnasnet import * 6 | from .densenet import * 7 | from .googlenet import * 8 | from .shufflenet import * 9 | from .shufflenetv2 import * 10 | from .resnet import * 11 | from .resnext import * 12 | from .preact_resnet import * 13 | from .mobilenet import * 14 | from .mobilenetv2 import * 15 | from .efficientnet import * 16 | from .wideresnet import * 17 | from .decoder import * 18 | from .mnist_generator import * 19 | from .cifar10_generator import * 20 | -------------------------------------------------------------------------------- /Unlearning/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .vgg import * 2 | from .dpn import * 3 | from .lenet import * 4 | from .senet import * 5 | from .pnasnet import * 6 | from .densenet import * 7 | from .googlenet import * 8 | from .shufflenet import * 9 | from .shufflenetv2 import * 10 | from .resnet import * 11 | from .resnext import * 12 | from .preact_resnet import * 13 | from .mobilenet import * 14 | from .mobilenetv2 import * 15 | from .efficientnet import * 16 | from .wideresnet import * 17 | from .decoder import * 18 | from .mnist_generator import * 19 | from .cifar10_generator import * 20 | -------------------------------------------------------------------------------- /Training/models/lenet.py: -------------------------------------------------------------------------------- 1 | '''LeNet in PyTorch.''' 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class LeNet(nn.Module): 6 | def __init__(self): 7 | super(LeNet, self).__init__() 8 | self.bn1 = nn.BatchNorm2d(1) 9 | self.conv1 = nn.Conv2d(1, 6, 5) 10 | self.bn2 = nn.BatchNorm2d(6) 11 | self.conv2 = nn.Conv2d(6, 16, 5) 12 | self.fc1 = nn.Linear(16*5*5, 120) 13 | self.fc2 = nn.Linear(120, 84) 14 | self.fc3 = nn.Linear(84, 10) 15 | 16 | def forward(self, x): 17 | out = F.relu(self.conv1(self.bn1(x))) 18 | out = F.max_pool2d(out, 2) 19 | out = F.relu(self.conv2(self.bn2(out))) 20 | out = F.max_pool2d(out, 2) 21 | out = out.view(out.size(0), -1) 22 | self.embeddings = out 23 | out = F.relu(self.fc1(out)) 24 | out = F.relu(self.fc2(out)) 25 | out = self.fc3(out) 26 | return out 27 | -------------------------------------------------------------------------------- /Inversion/models/lenet.py: -------------------------------------------------------------------------------- 1 | '''LeNet in PyTorch.''' 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class LeNet(nn.Module): 6 | def __init__(self): 7 | super(LeNet, self).__init__() 8 | self.bn1 = nn.BatchNorm2d(1) 9 | self.conv1 = nn.Conv2d(1, 6, 5) 10 | self.bn2 = nn.BatchNorm2d(6) 11 | self.conv2 = nn.Conv2d(6, 16, 5) 12 | self.fc1 = nn.Linear(16*5*5, 120) 13 | self.fc2 = nn.Linear(120, 84) 14 | self.fc3 = nn.Linear(84, 10) 15 | 16 | def forward(self, x): 17 | out = F.relu(self.conv1(self.bn1(x))) 18 | out = F.max_pool2d(out, 2) 19 | out = F.relu(self.conv2(self.bn2(out))) 20 | out = F.max_pool2d(out, 2) 21 | out = out.view(out.size(0), -1) 22 | self.embeddings = out 23 | out = F.relu(self.fc1(out)) 24 | out = F.relu(self.fc2(out)) 25 | out = self.fc3(out) 26 | return out 27 | -------------------------------------------------------------------------------- /Unlearning/models/lenet.py: -------------------------------------------------------------------------------- 1 | '''LeNet in PyTorch.''' 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class LeNet(nn.Module): 6 | def __init__(self): 7 | super(LeNet, self).__init__() 8 | self.bn1 = nn.BatchNorm2d(1) 9 | self.conv1 = nn.Conv2d(1, 6, 5) 10 | self.bn2 = nn.BatchNorm2d(6) 11 | self.conv2 = nn.Conv2d(6, 16, 5) 12 | self.fc1 = nn.Linear(16*5*5, 120) 13 | self.fc2 = nn.Linear(120, 84) 14 | self.fc3 = nn.Linear(84, 10) 15 | 16 | def forward(self, x): 17 | out = F.relu(self.conv1(self.bn1(x))) 18 | out = F.max_pool2d(out, 2) 19 | out = F.relu(self.conv2(self.bn2(out))) 20 | out = F.max_pool2d(out, 2) 21 | out = out.view(out.size(0), -1) 22 | self.embeddings = out 23 | out = F.relu(self.fc1(out)) 24 | out = F.relu(self.fc2(out)) 25 | out = self.fc3(out) 26 | return out 27 | -------------------------------------------------------------------------------- /Inversion/generator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | import models 5 | 6 | class generator(nn.Module): 7 | def __init__(self, dataset, conditional=False, class_num=10): 8 | super(generator, self).__init__() 9 | 10 | self.dataset = dataset 11 | 12 | if dataset == "MNIST": model = models.mnist_generator.Generator 13 | elif dataset == "CIFAR10": model = models.cifar10_generator.Generator 14 | elif dataset == "CIFAR20": model = models.cifar20_generator.Generator 15 | elif dataset == "FMNIST": model = models.fmnist_generator.Generator 16 | 17 | self.model = model(conditional=conditional, class_num=class_num) 18 | 19 | print(f"{dataset} generator is ready") 20 | 21 | 22 | def forward(self, x, y=None): 23 | return self.model(x, y) 24 | 25 | def prepare_generator(dataset, conditional=False, class_num=10): 26 | return generator(dataset, conditional, class_num=class_num) 27 | 28 | -------------------------------------------------------------------------------- /Training/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 liukuang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Unlearning/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 liukuang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Inversion/models/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 liukuang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Inversion/models/decoder.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | 5 | import numpy as np 6 | 7 | 8 | class normalizer(nn.Module): 9 | def __init__(self): 10 | super(normalizer, self).__init__() 11 | 12 | def forward(self, x): 13 | x = torch.sigmoid(x) 14 | x = x * 2 15 | x = x - 1 16 | return x 17 | 18 | 19 | class Decoder(nn.Module): 20 | def __init__(self): 21 | super(Decoder, self).__init__() 22 | 23 | self.img_shape = 100 24 | 25 | self.L1 = nn.Linear(10, 512) 26 | self.L2 = nn.Linear(512,512) 27 | self.L3 = nn.Linear(512,100) 28 | 29 | self.model = nn.Sequential( 30 | self.L1, 31 | nn.LeakyReLU(0.2, inplace=True), 32 | self.L2, 33 | nn.BatchNorm1d(512), 34 | nn.LeakyReLU(0.2, inplace=True), 35 | self.L3, 36 | #nn.Tanh(), 37 | ) 38 | 39 | 40 | def forward(self, z): 41 | img_flat = self.model(z) 42 | img = img_flat.view(img_flat.shape[0], img_flat.shape[1])#, 1, 1) 43 | #img = img_flat.view(img_flat.shape[0], *self.img_shape) 44 | return img 45 | 46 | 47 | -------------------------------------------------------------------------------- /Unlearning/models/decoder.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | import torch 4 | 5 | import numpy as np 6 | 7 | 8 | class normalizer(nn.Module): 9 | def __init__(self): 10 | super(normalizer, self).__init__() 11 | 12 | def forward(self, x): 13 | x = torch.sigmoid(x) 14 | x = x * 2 15 | x = x - 1 16 | return x 17 | 18 | 19 | class Decoder(nn.Module): 20 | def __init__(self): 21 | super(Decoder, self).__init__() 22 | 23 | self.img_shape = 100 24 | 25 | self.L1 = nn.Linear(10, 512) 26 | self.L2 = nn.Linear(512,512) 27 | self.L3 = nn.Linear(512,100) 28 | 29 | self.model = nn.Sequential( 30 | self.L1, 31 | nn.LeakyReLU(0.2, inplace=True), 32 | self.L2, 33 | nn.BatchNorm1d(512), 34 | nn.LeakyReLU(0.2, inplace=True), 35 | self.L3, 36 | #nn.Tanh(), 37 | ) 38 | 39 | 40 | def forward(self, z): 41 | img_flat = self.model(z) 42 | img = img_flat.view(img_flat.shape[0], img_flat.shape[1])#, 1, 1) 43 | #img = img_flat.view(img_flat.shape[0], *self.img_shape) 44 | return img 45 | 46 | 47 | -------------------------------------------------------------------------------- /Inversion/normalizer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torchvision.transforms as transforms 5 | 6 | class normalizer(nn.Module): 7 | def __init__(self, dataset): 8 | super(normalizer, self).__init__() 9 | 10 | if dataset == "CIFAR10": 11 | self.net = transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) 12 | 13 | elif dataset == "CIFAR20": 14 | self.net = transforms.Normalize((0.5071, 0.4868, 0.4408), (0.2675, 0.2565, 0.2761)) 15 | 16 | elif dataset == "MNIST": 17 | self.net = transforms.Normalize((0.1307,), (0.3081,)) 18 | 19 | elif dataset == "SVHN": 20 | self.net = transforms.Normalize((0.5,), (0.5,)) 21 | 22 | elif dataset == "FMNIST": 23 | self.net = transforms.Normalize((0.2849,), (0.3516,)) 24 | 25 | elif dataset == "MFMNIST": 26 | self.net = transforms.Normalize((0.2849,), (0.3516,)) 27 | 28 | else: 29 | print("nothing is specified for normalizer!") 30 | exit() 31 | 32 | print(f"{dataset} is used for normalizer") 33 | 34 | def forward(self, x): 35 | return self.net(x) 36 | -------------------------------------------------------------------------------- /Inversion/unnormalizer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torchvision.transforms as transforms 5 | 6 | class unnormalizer(nn.Module): 7 | def __init__(self, dataset): 8 | super(unnormalizer, self).__init__() 9 | 10 | 11 | if dataset == "CIFAR10": 12 | self.net = transforms.Normalize((-0.4914/0.2023, -0.4822/0.1994, -0.4465/0.2010), (1/0.2023, 1/0.1994, 1/0.2010)) 13 | 14 | elif dataset == "CIFAR20": 15 | self.net = transforms.Normalize((-0.5071/0.2675, -0.4868/0.2565, -0.4408/0.2761), (1/0.2675, 1/0.2565, 1/0.2761)) 16 | 17 | elif dataset == "MNIST": 18 | self.net = transforms.Normalize((-0.1307/0.3081,), (1/0.3081,)) 19 | 20 | elif dataset == "SVHN": 21 | self.net = transforms.Normalize((-0.5/0.5,), (1/0.5,)) 22 | 23 | elif dataset == "FMNIST": 24 | self.net = transforms.Normalize((-0.2849/0.3516,), (1/0.3516,)) 25 | 26 | elif dataset == "MFMNIST": 27 | self.net = transforms.Normalize((-0.2849/0.3516,), (1/0.3516,)) 28 | 29 | else: 30 | print("nothing is specified for unnormalizer!") 31 | exit() 32 | 33 | print(f"{dataset} is used for unnormalizer") 34 | 35 | 36 | def forward(self, x): 37 | return self.net(x) 38 | -------------------------------------------------------------------------------- /Training/cross_test.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | myidx = numpy.array( 4 | [36, 5 | 41, 6 | 64, 7 | 75, 8 | 97, 9 | 114, 10 | 133, 11 | 144, 12 | 243, 13 | 282, 14 | 301, 15 | 307, 16 | 328, 17 | 468, 18 | 510, 19 | 522, 20 | 550, 21 | 551, 22 | 554, 23 | 605, 24 | 617, 25 | 684, 26 | 703, 27 | 810, 28 | 832, 29 | 841, 30 | 898, 31 | 930, 32 | 950, 33 | 960, 34 | 987, 35 | 1012, 36 | 1039, 37 | 1091, 38 | 1119, 39 | 1143, 40 | 1156, 41 | 1194, 42 | 1226, 43 | 1276, 44 | 1283, 45 | 1363, 46 | 1477, 47 | 1500, 48 | 1543, 49 | 1581, 50 | 1718, 51 | 1750, 52 | 1754, 53 | 1792, 54 | 1903, 55 | 1933, 56 | 1941, 57 | 2016, 58 | 2063, 59 | 2095, 60 | 2285, 61 | 2325, 62 | 2334, 63 | 2507, 64 | 2513, 65 | 2539, 66 | 2547, 67 | 2607, 68 | 2622, 69 | 2628, 70 | 2665, 71 | 2666, 72 | 2730, 73 | 2820, 74 | 2823, 75 | 2841, 76 | 2961, 77 | 3061, 78 | 3122, 79 | 3136, 80 | 3166, 81 | 3184, 82 | 3202, 83 | 3208, 84 | 3262, 85 | 3294, 86 | 3316, 87 | 3329, 88 | 3333, 89 | 3349, 90 | 3352, 91 | 3361, 92 | 3373, 93 | 3573, 94 | 3604, 95 | 3647, 96 | 3656, 97 | 3713, 98 | 3720, 99 | 3751, 100 | 3767, 101 | 3838, 102 | 3848, 103 | 3936, 104 | 4007, 105 | 4107, 106 | 4148, 107 | 4215, 108 | 4225, 109 | 4228, 110 | 4238, 111 | 4423, 112 | 4433, 113 | 4498, 114 | 4540, 115 | 4566, 116 | 4654, 117 | 4658, 118 | 4660, 119 | 4690, 120 | 4736, 121 | 4742, 122 | 4855, 123 | 4966, 124 | 5835, 125 | 5845, 126 | 5854, 127 | 5858, 128 | 5861, 129 | 5866, 130 | 5887, 131 | 5903, 132 | 5906, 133 | 5914, 134 | 9009, 135 | 9015, 136 | 9019, 137 | 9024, 138 | 9031, 139 | 9036, 140 | 9044,]) 141 | -------------------------------------------------------------------------------- /Inversion/models/vgg.py: -------------------------------------------------------------------------------- 1 | '''VGG11/13/16/19 in Pytorch.''' 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | cfg = { 7 | 'VGG11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 8 | 'VGG13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 9 | 'VGG16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 10 | 'VGG19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 11 | } 12 | 13 | 14 | class VGG(nn.Module): 15 | def __init__(self, vgg_name): 16 | super(VGG, self).__init__() 17 | self.features = self._make_layers(cfg[vgg_name]) 18 | self.classifier = nn.Linear(512, 10) 19 | 20 | def forward(self, x): 21 | out = self.features(x) 22 | out = out.view(out.size(0), -1) 23 | out = self.classifier(out) 24 | return out 25 | 26 | def _make_layers(self, cfg): 27 | layers = [] 28 | in_channels = 3 29 | for x in cfg: 30 | if x == 'M': 31 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 32 | else: 33 | layers += [nn.Conv2d(in_channels, x, kernel_size=3, padding=1), 34 | nn.BatchNorm2d(x), 35 | nn.ReLU(inplace=True)] 36 | in_channels = x 37 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 38 | return nn.Sequential(*layers) 39 | 40 | 41 | def test(): 42 | net = VGG('VGG11') 43 | x = torch.randn(2,3,32,32) 44 | y = net(x) 45 | print(y.size()) 46 | 47 | # test() 48 | -------------------------------------------------------------------------------- /Training/models/vgg.py: -------------------------------------------------------------------------------- 1 | '''VGG11/13/16/19 in Pytorch.''' 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | cfg = { 7 | 'VGG11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 8 | 'VGG13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 9 | 'VGG16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 10 | 'VGG19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 11 | } 12 | 13 | 14 | class VGG(nn.Module): 15 | def __init__(self, vgg_name): 16 | super(VGG, self).__init__() 17 | self.features = self._make_layers(cfg[vgg_name]) 18 | self.classifier = nn.Linear(512, 10) 19 | 20 | def forward(self, x): 21 | out = self.features(x) 22 | out = out.view(out.size(0), -1) 23 | out = self.classifier(out) 24 | return out 25 | 26 | def _make_layers(self, cfg): 27 | layers = [] 28 | in_channels = 3 29 | for x in cfg: 30 | if x == 'M': 31 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 32 | else: 33 | layers += [nn.Conv2d(in_channels, x, kernel_size=3, padding=1), 34 | nn.BatchNorm2d(x), 35 | nn.ReLU(inplace=True)] 36 | in_channels = x 37 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 38 | return nn.Sequential(*layers) 39 | 40 | 41 | def test(): 42 | net = VGG('VGG11') 43 | x = torch.randn(2,3,32,32) 44 | y = net(x) 45 | print(y.size()) 46 | 47 | # test() 48 | -------------------------------------------------------------------------------- /Unlearning/models/vgg.py: -------------------------------------------------------------------------------- 1 | '''VGG11/13/16/19 in Pytorch.''' 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | cfg = { 7 | 'VGG11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 8 | 'VGG13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 9 | 'VGG16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 10 | 'VGG19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 11 | } 12 | 13 | 14 | class VGG(nn.Module): 15 | def __init__(self, vgg_name): 16 | super(VGG, self).__init__() 17 | self.features = self._make_layers(cfg[vgg_name]) 18 | self.classifier = nn.Linear(512, 10) 19 | 20 | def forward(self, x): 21 | out = self.features(x) 22 | out = out.view(out.size(0), -1) 23 | out = self.classifier(out) 24 | return out 25 | 26 | def _make_layers(self, cfg): 27 | layers = [] 28 | in_channels = 3 29 | for x in cfg: 30 | if x == 'M': 31 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 32 | else: 33 | layers += [nn.Conv2d(in_channels, x, kernel_size=3, padding=1), 34 | nn.BatchNorm2d(x), 35 | nn.ReLU(inplace=True)] 36 | in_channels = x 37 | layers += [nn.AvgPool2d(kernel_size=1, stride=1)] 38 | return nn.Sequential(*layers) 39 | 40 | 41 | def test(): 42 | net = VGG('VGG11') 43 | x = torch.randn(2,3,32,32) 44 | y = net(x) 45 | print(y.size()) 46 | 47 | # test() 48 | -------------------------------------------------------------------------------- /Unlearning/cross_test.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | myidx = numpy.array( 4 | [36, 5 | 41, 6 | 64, 7 | 75, 8 | 97, 9 | 114, 10 | 133, 11 | 144, 12 | 243, 13 | 282, 14 | 301, 15 | 307, 16 | 328, 17 | 468, 18 | 510, 19 | 522, 20 | 550, 21 | 551, 22 | 554, 23 | 605, 24 | 617, 25 | 684, 26 | 703, 27 | 810, 28 | 832, 29 | 841, 30 | 898, 31 | 930, 32 | 950, 33 | 960, 34 | 987, 35 | 1012, 36 | 1039, 37 | 1091, 38 | 1119, 39 | 1143, 40 | 1156, 41 | 1194, 42 | 1226, 43 | 1276, 44 | 1283, 45 | 1363, 46 | 1477, 47 | 1500, 48 | 1543, 49 | 1581, 50 | 1718, 51 | 1750, 52 | 1754, 53 | 1792, 54 | 1903, 55 | 1933, 56 | 1941, 57 | 2016, 58 | 2063, 59 | 2095, 60 | 2285, 61 | 2325, 62 | 2334, 63 | 2507, 64 | 2513, 65 | 2539, 66 | 2547, 67 | 2607, 68 | 2622, 69 | 2628, 70 | 2665, 71 | 2666, 72 | 2730, 73 | 2820, 74 | 2823, 75 | 2841, 76 | 2961, 77 | 3061, 78 | 3122, 79 | 3136, 80 | 3166, 81 | 3184, 82 | 3202, 83 | 3208, 84 | 3262, 85 | 3294, 86 | 3316, 87 | 3329, 88 | 3333, 89 | 3349, 90 | 3352, 91 | 3361, 92 | 3373, 93 | 3573, 94 | 3604, 95 | 3647, 96 | 3656, 97 | 3713, 98 | 3720, 99 | 3751, 100 | 3767, 101 | 3838, 102 | 3848, 103 | 3936, 104 | 4007, 105 | 4107, 106 | 4148, 107 | 4215, 108 | 4225, 109 | 4228, 110 | 4238, 111 | 4423, 112 | 4433, 113 | 4498, 114 | 4540, 115 | 4566, 116 | 4654, 117 | 4658, 118 | 4660, 119 | 4690, 120 | 4736, 121 | 4742, 122 | 4855, 123 | 4966, 124 | 5835, 125 | 5845, 126 | 5854, 127 | 5858, 128 | 5861, 129 | 5866, 130 | 5887, 131 | 5903, 132 | 5906, 133 | 5914, 134 | 9009, 135 | 9015, 136 | 9019, 137 | 9024, 138 | 9031, 139 | 9036, 140 | 9044,]) 141 | 142 | if __name__ == "__main__": 143 | print(myidx.shape) 144 | -------------------------------------------------------------------------------- /Inversion/models/generator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | def weights_init(w): 7 | """ 8 | Initializes the weights of the layer, w. 9 | """ 10 | classname = w.__class__.__name__ 11 | if classname.find('conv') != -1: 12 | nn.init.normal_(w.weight.data, 0.0, 0.02) 13 | elif classname.find('bn') != -1: 14 | nn.init.normal_(w.weight.data, 1.0, 0.02) 15 | nn.init.constant_(w.bias.data, 0) 16 | 17 | # Define the Generator Network 18 | class Generator(nn.Module): 19 | def __init__(self, in_num, hidden_num, output_num): 20 | super().__init__() 21 | 22 | # Input is the latent vector Z. 23 | self.tconv1 = nn.ConvTranspose2d(in_num, hidden_num * 8, 24 | kernel_size=4, stride=1, padding=0, bias=False) 25 | self.bn1 = nn.BatchNorm2d(hidden_num * 8) 26 | 27 | # Input Dimension: (ngf*8) x 4 x 4 28 | self.tconv2 = nn.ConvTranspose2d(hidden_num*8, hidden_num*4, 29 | 4, 2, 1, bias=False) 30 | self.bn2 = nn.BatchNorm2d(hidden_num*4) 31 | 32 | # Input Dimension: (ngf*4) x 8 x 8 33 | self.tconv3 = nn.ConvTranspose2d(hidden_num*4, hidden_num*2, 34 | 4, 2, 1, bias=False) 35 | self.bn3 = nn.BatchNorm2d(hidden_num*2) 36 | 37 | # Input Dimension: (ngf*2) x 16 x 16 38 | self.tconv4 = nn.ConvTranspose2d(hidden_num*2, hidden_num, 39 | 4, 2, 1, bias=False) 40 | self.bn4 = nn.BatchNorm2d(hidden_num) 41 | 42 | # Input Dimension: (ngf) * 32 * 32 43 | self.tconv5 = nn.ConvTranspose2d(hidden_num, output_num, 44 | 4, 2, 1, bias=False) 45 | #Output Dimension: (nc) x 64 x 64 46 | 47 | def forward(self, x): 48 | x = F.relu(self.bn1(self.tconv1(x))) 49 | x = F.relu(self.bn2(self.tconv2(x))) 50 | x = F.relu(self.bn3(self.tconv3(x))) 51 | x = F.relu(self.bn4(self.tconv4(x))) 52 | 53 | x = F.tanh(self.tconv5(x)) 54 | 55 | return x 56 | -------------------------------------------------------------------------------- /Unlearning/models/generator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | def weights_init(w): 7 | """ 8 | Initializes the weights of the layer, w. 9 | """ 10 | classname = w.__class__.__name__ 11 | if classname.find('conv') != -1: 12 | nn.init.normal_(w.weight.data, 0.0, 0.02) 13 | elif classname.find('bn') != -1: 14 | nn.init.normal_(w.weight.data, 1.0, 0.02) 15 | nn.init.constant_(w.bias.data, 0) 16 | 17 | # Define the Generator Network 18 | class Generator(nn.Module): 19 | def __init__(self, in_num, hidden_num, output_num): 20 | super().__init__() 21 | 22 | # Input is the latent vector Z. 23 | self.tconv1 = nn.ConvTranspose2d(in_num, hidden_num * 8, 24 | kernel_size=4, stride=1, padding=0, bias=False) 25 | self.bn1 = nn.BatchNorm2d(hidden_num * 8) 26 | 27 | # Input Dimension: (ngf*8) x 4 x 4 28 | self.tconv2 = nn.ConvTranspose2d(hidden_num*8, hidden_num*4, 29 | 4, 2, 1, bias=False) 30 | self.bn2 = nn.BatchNorm2d(hidden_num*4) 31 | 32 | # Input Dimension: (ngf*4) x 8 x 8 33 | self.tconv3 = nn.ConvTranspose2d(hidden_num*4, hidden_num*2, 34 | 4, 2, 1, bias=False) 35 | self.bn3 = nn.BatchNorm2d(hidden_num*2) 36 | 37 | # Input Dimension: (ngf*2) x 16 x 16 38 | self.tconv4 = nn.ConvTranspose2d(hidden_num*2, hidden_num, 39 | 4, 2, 1, bias=False) 40 | self.bn4 = nn.BatchNorm2d(hidden_num) 41 | 42 | # Input Dimension: (ngf) * 32 * 32 43 | self.tconv5 = nn.ConvTranspose2d(hidden_num, output_num, 44 | 4, 2, 1, bias=False) 45 | #Output Dimension: (nc) x 64 x 64 46 | 47 | def forward(self, x): 48 | x = F.relu(self.bn1(self.tconv1(x))) 49 | x = F.relu(self.bn2(self.tconv2(x))) 50 | x = F.relu(self.bn3(self.tconv3(x))) 51 | x = F.relu(self.bn4(self.tconv4(x))) 52 | 53 | x = F.tanh(self.tconv5(x)) 54 | 55 | return x 56 | -------------------------------------------------------------------------------- /Inversion/loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | from utils import * 5 | 6 | 7 | def class_diversity_loss(z, embeddings): 8 | result = 0 9 | 10 | if len(embeddings.shape) > 2: 11 | mylayer = embeddings.view((embeddings.size(0), -1)) 12 | else: 13 | mylayer = embeddings 14 | tensor1 = mylayer.expand((z.shape[0], z.shape[0], mylayer.size(1))) 15 | tensor2 = mylayer.unsqueeze(dim=1) 16 | layer_dist = torch.abs(tensor1 - tensor2).mean(dim=(2,)) 17 | 18 | tensor1 = z.expand((z.shape[0], z.shape[0], z.size(1))) 19 | tensor2 = z.unsqueeze(dim=1) 20 | 21 | noise_dist = torch.pow(tensor1 - tensor2, 2) 22 | noise_dist = noise_dist.mean(dim=2) 23 | 24 | return torch.exp(torch.mean(-noise_dist * layer_dist)) 25 | 26 | def Diversity_loss(z, embeddings, z_class): 27 | result = 0 28 | 29 | for i in range(11): 30 | idx = np.where(z_class.detach().cpu().numpy() == i)[0] 31 | 32 | result += class_diversity_loss(z[idx], embeddings[idx]) 33 | 34 | return result 35 | 36 | 37 | def BN_loss(rescale, meanvar_layers, meanvar_origin): 38 | if len(meanvar_origin) == 0: 39 | return 0 40 | 41 | result = 0 42 | 43 | for idx, (my, pr) in enumerate(zip(meanvar_layers, meanvar_origin)): 44 | result += rescale[idx] * (torch.norm(pr[0] - my[0], 2) + torch.norm(pr[1] - my[1], 2)) 45 | 46 | 47 | return result 48 | 49 | 50 | def Augmentation_loss(output, output_aug): 51 | result = 0 52 | 53 | for augmented in output_aug: 54 | result += torch.norm(sm(output) - sm(augmented), 2) 55 | 56 | result /= output.shape[0] 57 | 58 | return result 59 | 60 | def Totalvariation_loss(x): 61 | return ( 62 | torch.norm(x[:, :, :, :-1] - x[:, :, :, 1:], 2) + 63 | torch.norm(x[:, :, :-1, :] - x[:, :, 1:, :], 2) 64 | ) / x.shape[0] 65 | #/(x.shape[1] * x.shape[2] * x.shape[3]) 66 | 67 | 68 | def L2norm(rescale, D_e, D_g): 69 | result = 0 70 | 71 | for idx, (d_e, d_g) in enumerate(zip(D_e, D_g)): 72 | result += rescale[idx] * (torch.norm(d_e[0] - d_g[0], 2))# + torch.norm(d_e[1] - d_g[1], 2)) 73 | 74 | return result 75 | -------------------------------------------------------------------------------- /Inversion/rotator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import torchvision.transforms as transforms 5 | 6 | class rotator(nn.Module): 7 | def __init__(self, degree, dataset): 8 | super(rotator, self).__init__() 9 | 10 | self.dataset = dataset 11 | 12 | if dataset == "CIFAR10": 13 | self.net = transforms.Compose([ 14 | transforms.RandomCrop(32, padding=4), 15 | transforms.RandomHorizontalFlip(), 16 | transforms.RandomRotation(degree), 17 | ]) 18 | 19 | elif dataset == "CIFAR20": 20 | self.net = transforms.Compose([ 21 | transforms.RandomCrop(32, padding=4), 22 | transforms.RandomHorizontalFlip(), 23 | transforms.RandomRotation(degree), 24 | ]) 25 | 26 | elif dataset == "SVHN": 27 | self.net = transforms.Compose([ 28 | transforms.RandomCrop(32, padding=4), 29 | transforms.RandomRotation(degree), 30 | ]) 31 | 32 | elif dataset == "MNIST": 33 | self.net = transforms.Compose([ 34 | transforms.RandomCrop(32, padding=4), 35 | transforms.RandomRotation(degree), 36 | ]) 37 | 38 | elif dataset == "FMNIST": 39 | self.net = transforms.Compose([ 40 | transforms.RandomCrop(32, padding=4), 41 | transforms.RandomRotation(degree), 42 | transforms.RandomHorizontalFlip(p=0.5), 43 | ]) 44 | 45 | elif dataset == "MFMNIST": 46 | self.net = transforms.Compose([ 47 | transforms.RandomCrop(32, padding=4), 48 | transforms.RandomRotation(degree), 49 | transforms.RandomHorizontalFlip(p=0.5), 50 | ]) 51 | 52 | else: 53 | print("nothing is specified for rotator!") 54 | exit() 55 | 56 | 57 | print(f"{dataset} is used for rotator") 58 | 59 | 60 | def forward(self, x): 61 | return self.net(x) 62 | -------------------------------------------------------------------------------- /Inversion/models/mobilenet.py: -------------------------------------------------------------------------------- 1 | '''MobileNet in PyTorch. 2 | 3 | See the paper "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications" 4 | for more details. 5 | ''' 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Block(nn.Module): 12 | '''Depthwise conv + Pointwise conv''' 13 | def __init__(self, in_planes, out_planes, stride=1): 14 | super(Block, self).__init__() 15 | self.conv1 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=in_planes, bias=False) 16 | self.bn1 = nn.BatchNorm2d(in_planes) 17 | self.conv2 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 18 | self.bn2 = nn.BatchNorm2d(out_planes) 19 | 20 | def forward(self, x): 21 | out = F.relu(self.bn1(self.conv1(x))) 22 | out = F.relu(self.bn2(self.conv2(out))) 23 | return out 24 | 25 | 26 | class MobileNet(nn.Module): 27 | # (128,2) means conv planes=128, conv stride=2, by default conv stride=1 28 | cfg = [64, (128,2), 128, (256,2), 256, (512,2), 512, 512, 512, 512, 512, (1024,2), 1024] 29 | 30 | def __init__(self, num_classes=10): 31 | super(MobileNet, self).__init__() 32 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False) 33 | self.bn1 = nn.BatchNorm2d(32) 34 | self.layers = self._make_layers(in_planes=32) 35 | self.linear = nn.Linear(1024, num_classes) 36 | 37 | def _make_layers(self, in_planes): 38 | layers = [] 39 | for x in self.cfg: 40 | out_planes = x if isinstance(x, int) else x[0] 41 | stride = 1 if isinstance(x, int) else x[1] 42 | layers.append(Block(in_planes, out_planes, stride)) 43 | in_planes = out_planes 44 | return nn.Sequential(*layers) 45 | 46 | def forward(self, x): 47 | out = F.relu(self.bn1(self.conv1(x))) 48 | out = self.layers(out) 49 | out = F.avg_pool2d(out, 2) 50 | out = out.view(out.size(0), -1) 51 | out = self.linear(out) 52 | return out 53 | 54 | 55 | def test(): 56 | net = MobileNet() 57 | x = torch.randn(1,3,32,32) 58 | y = net(x) 59 | print(y.size()) 60 | 61 | # test() 62 | -------------------------------------------------------------------------------- /Training/models/mobilenet.py: -------------------------------------------------------------------------------- 1 | '''MobileNet in PyTorch. 2 | 3 | See the paper "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications" 4 | for more details. 5 | ''' 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Block(nn.Module): 12 | '''Depthwise conv + Pointwise conv''' 13 | def __init__(self, in_planes, out_planes, stride=1): 14 | super(Block, self).__init__() 15 | self.conv1 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=in_planes, bias=False) 16 | self.bn1 = nn.BatchNorm2d(in_planes) 17 | self.conv2 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 18 | self.bn2 = nn.BatchNorm2d(out_planes) 19 | 20 | def forward(self, x): 21 | out = F.relu(self.bn1(self.conv1(x))) 22 | out = F.relu(self.bn2(self.conv2(out))) 23 | return out 24 | 25 | 26 | class MobileNet(nn.Module): 27 | # (128,2) means conv planes=128, conv stride=2, by default conv stride=1 28 | cfg = [64, (128,2), 128, (256,2), 256, (512,2), 512, 512, 512, 512, 512, (1024,2), 1024] 29 | 30 | def __init__(self, num_classes=10): 31 | super(MobileNet, self).__init__() 32 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False) 33 | self.bn1 = nn.BatchNorm2d(32) 34 | self.layers = self._make_layers(in_planes=32) 35 | self.linear = nn.Linear(1024, num_classes) 36 | 37 | def _make_layers(self, in_planes): 38 | layers = [] 39 | for x in self.cfg: 40 | out_planes = x if isinstance(x, int) else x[0] 41 | stride = 1 if isinstance(x, int) else x[1] 42 | layers.append(Block(in_planes, out_planes, stride)) 43 | in_planes = out_planes 44 | return nn.Sequential(*layers) 45 | 46 | def forward(self, x): 47 | out = F.relu(self.bn1(self.conv1(x))) 48 | out = self.layers(out) 49 | out = F.avg_pool2d(out, 2) 50 | out = out.view(out.size(0), -1) 51 | out = self.linear(out) 52 | return out 53 | 54 | 55 | def test(): 56 | net = MobileNet() 57 | x = torch.randn(1,3,32,32) 58 | y = net(x) 59 | print(y.size()) 60 | 61 | # test() 62 | -------------------------------------------------------------------------------- /Unlearning/models/mobilenet.py: -------------------------------------------------------------------------------- 1 | '''MobileNet in PyTorch. 2 | 3 | See the paper "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications" 4 | for more details. 5 | ''' 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Block(nn.Module): 12 | '''Depthwise conv + Pointwise conv''' 13 | def __init__(self, in_planes, out_planes, stride=1): 14 | super(Block, self).__init__() 15 | self.conv1 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=in_planes, bias=False) 16 | self.bn1 = nn.BatchNorm2d(in_planes) 17 | self.conv2 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 18 | self.bn2 = nn.BatchNorm2d(out_planes) 19 | 20 | def forward(self, x): 21 | out = F.relu(self.bn1(self.conv1(x))) 22 | out = F.relu(self.bn2(self.conv2(out))) 23 | return out 24 | 25 | 26 | class MobileNet(nn.Module): 27 | # (128,2) means conv planes=128, conv stride=2, by default conv stride=1 28 | cfg = [64, (128,2), 128, (256,2), 256, (512,2), 512, 512, 512, 512, 512, (1024,2), 1024] 29 | 30 | def __init__(self, num_classes=10): 31 | super(MobileNet, self).__init__() 32 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False) 33 | self.bn1 = nn.BatchNorm2d(32) 34 | self.layers = self._make_layers(in_planes=32) 35 | self.linear = nn.Linear(1024, num_classes) 36 | 37 | def _make_layers(self, in_planes): 38 | layers = [] 39 | for x in self.cfg: 40 | out_planes = x if isinstance(x, int) else x[0] 41 | stride = 1 if isinstance(x, int) else x[1] 42 | layers.append(Block(in_planes, out_planes, stride)) 43 | in_planes = out_planes 44 | return nn.Sequential(*layers) 45 | 46 | def forward(self, x): 47 | out = F.relu(self.bn1(self.conv1(x))) 48 | out = self.layers(out) 49 | out = F.avg_pool2d(out, 2) 50 | out = out.view(out.size(0), -1) 51 | out = self.linear(out) 52 | return out 53 | 54 | 55 | def test(): 56 | net = MobileNet() 57 | x = torch.randn(1,3,32,32) 58 | y = net(x) 59 | print(y.size()) 60 | 61 | # test() 62 | -------------------------------------------------------------------------------- /Inversion/classifier.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from models import * 5 | 6 | 7 | class classifier(nn.Module): 8 | def __init__(self, dataset, normalization, path=None): 9 | super(classifier, self).__init__() 10 | 11 | self.dataset = dataset 12 | self.path = path 13 | 14 | if dataset == "MNIST": 15 | num_channels = 1 16 | else: 17 | num_channels = 3 18 | 19 | if dataset != "CIFAR20": 20 | self.model = PreActResNet18(num_channels=num_channels, normalization=normalization) 21 | else: 22 | self.model = PreActResNet34(num_classes=20, num_channels=num_channels, normalization=normalization) 23 | 24 | if path is not None: 25 | self.model.load_state_dict(torch.load(path)) 26 | print(f"pre-trained {dataset} classifier is loaded") 27 | else: 28 | print("classifier model is not loaded . . .") 29 | 30 | def forward(self, x): 31 | return self.model(x) 32 | 33 | 34 | def prepare_classifier(dataset, task, path=None, normalization="BatchNorm"): 35 | 36 | if dataset == "MNIST": 37 | default_paths = [] 38 | default_paths.append(f"./pretrained/TRUE/resnet18_MNIST_NINE_False_None.pyt") 39 | default_paths.append(f"./pretrained/TRUE/resnet18_MNIST_CROSS7_False_None.pyt") 40 | if task == "NINE" or task == "SEVEN" or task == "EIGHT": default_path = default_paths[0] 41 | elif task == "CROSS7": default_path = default_paths[1] 42 | else: 43 | print("WRONG TASK! given task is ", task) 44 | exit() 45 | 46 | elif dataset == "CIFAR10": 47 | default_path = "./pretrained/resnet18_CIFAR10_NINE_False_None.pyt" 48 | default_path = f"./pretrained/resnet18_lr_{normalization}_CIFAR10_NINE_False_None.pyt" 49 | default_path = "./pretrained/resnet18_BN_BatchNorm_CIFAR10_NINE_False_None.pyt" 50 | 51 | elif dataset == "CIFAR20": 52 | default_path = "./pretrained/resnet18_CIFAR20_NINE_False_None.pyt" 53 | default_path = "./pretrained/resnet34_BN_BatchNorm_CIFAR20_NINE_False_None.pyt" 54 | 55 | print(f"classifier {default_path} is loading") 56 | 57 | return classifier(dataset=dataset, path=default_path, normalization=normalization) 58 | -------------------------------------------------------------------------------- /Inversion/models/spectral.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim.optimizer import Optimizer, required 3 | 4 | from torch.autograd import Variable 5 | import torch.nn.functional as F 6 | from torch import nn 7 | from torch import Tensor 8 | from torch.nn import Parameter 9 | 10 | def l2normalize(v, eps=1e-12): 11 | return v / (v.norm() + eps) 12 | 13 | 14 | class SpectralNorm(nn.Module): 15 | def __init__(self, module, name='weight', power_iterations=1): 16 | super(SpectralNorm, self).__init__() 17 | self.module = module 18 | self.name = name 19 | self.power_iterations = power_iterations 20 | if not self._made_params(): 21 | self._make_params() 22 | 23 | def _update_u_v(self): 24 | u = getattr(self.module, self.name + "_u") 25 | v = getattr(self.module, self.name + "_v") 26 | w = getattr(self.module, self.name + "_bar") 27 | 28 | height = w.data.shape[0] 29 | for _ in range(self.power_iterations): 30 | v.data = l2normalize(torch.mv(torch.t(w.view(height,-1).data), u.data)) 31 | u.data = l2normalize(torch.mv(w.view(height,-1).data, v.data)) 32 | 33 | # sigma = torch.dot(u.data, torch.mv(w.view(height,-1).data, v.data)) 34 | sigma = u.dot(w.view(height, -1).mv(v)) 35 | setattr(self.module, self.name, w / sigma.expand_as(w)) 36 | 37 | def _made_params(self): 38 | try: 39 | u = getattr(self.module, self.name + "_u") 40 | v = getattr(self.module, self.name + "_v") 41 | w = getattr(self.module, self.name + "_bar") 42 | return True 43 | except AttributeError: 44 | return False 45 | 46 | 47 | def _make_params(self): 48 | w = getattr(self.module, self.name) 49 | 50 | height = w.data.shape[0] 51 | width = w.view(height, -1).data.shape[1] 52 | 53 | u = Parameter(w.data.new(height).normal_(0, 1), requires_grad=False) 54 | v = Parameter(w.data.new(width).normal_(0, 1), requires_grad=False) 55 | u.data = l2normalize(u.data) 56 | v.data = l2normalize(v.data) 57 | w_bar = Parameter(w.data) 58 | 59 | del self.module._parameters[self.name] 60 | 61 | self.module.register_parameter(self.name + "_u", u) 62 | self.module.register_parameter(self.name + "_v", v) 63 | self.module.register_parameter(self.name + "_bar", w_bar) 64 | 65 | 66 | def forward(self, *args): 67 | self._update_u_v() 68 | return self.module.forward(*args) 69 | 70 | -------------------------------------------------------------------------------- /Unlearning/models/spectral.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim.optimizer import Optimizer, required 3 | 4 | from torch.autograd import Variable 5 | import torch.nn.functional as F 6 | from torch import nn 7 | from torch import Tensor 8 | from torch.nn import Parameter 9 | 10 | def l2normalize(v, eps=1e-12): 11 | return v / (v.norm() + eps) 12 | 13 | 14 | class SpectralNorm(nn.Module): 15 | def __init__(self, module, name='weight', power_iterations=1): 16 | super(SpectralNorm, self).__init__() 17 | self.module = module 18 | self.name = name 19 | self.power_iterations = power_iterations 20 | if not self._made_params(): 21 | self._make_params() 22 | 23 | def _update_u_v(self): 24 | u = getattr(self.module, self.name + "_u") 25 | v = getattr(self.module, self.name + "_v") 26 | w = getattr(self.module, self.name + "_bar") 27 | 28 | height = w.data.shape[0] 29 | for _ in range(self.power_iterations): 30 | v.data = l2normalize(torch.mv(torch.t(w.view(height,-1).data), u.data)) 31 | u.data = l2normalize(torch.mv(w.view(height,-1).data, v.data)) 32 | 33 | # sigma = torch.dot(u.data, torch.mv(w.view(height,-1).data, v.data)) 34 | sigma = u.dot(w.view(height, -1).mv(v)) 35 | setattr(self.module, self.name, w / sigma.expand_as(w)) 36 | 37 | def _made_params(self): 38 | try: 39 | u = getattr(self.module, self.name + "_u") 40 | v = getattr(self.module, self.name + "_v") 41 | w = getattr(self.module, self.name + "_bar") 42 | return True 43 | except AttributeError: 44 | return False 45 | 46 | 47 | def _make_params(self): 48 | w = getattr(self.module, self.name) 49 | 50 | height = w.data.shape[0] 51 | width = w.view(height, -1).data.shape[1] 52 | 53 | u = Parameter(w.data.new(height).normal_(0, 1), requires_grad=False) 54 | v = Parameter(w.data.new(width).normal_(0, 1), requires_grad=False) 55 | u.data = l2normalize(u.data) 56 | v.data = l2normalize(v.data) 57 | w_bar = Parameter(w.data) 58 | 59 | del self.module._parameters[self.name] 60 | 61 | self.module.register_parameter(self.name + "_u", u) 62 | self.module.register_parameter(self.name + "_v", v) 63 | self.module.register_parameter(self.name + "_bar", w_bar) 64 | 65 | 66 | def forward(self, *args): 67 | self._update_u_v() 68 | return self.module.forward(*args) 69 | 70 | -------------------------------------------------------------------------------- /Training/traintest.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import cross_test 3 | from utils import * 4 | import numpy as np 5 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 6 | cross_idx = cross_test.myidx 7 | 8 | def test(net, dataloader, criterion, task): 9 | net.eval() 10 | test_loss = 0 11 | correct = 0 12 | total = 0 13 | 14 | result_cross = torch.zeros((10, 10), dtype=int) 15 | result = torch.zeros((10, 10), dtype=int) 16 | 17 | predictions = [[] for i in range(10)] 18 | 19 | with torch.no_grad(): 20 | for batch_idx, (inputs, targets) in enumerate(dataloader): 21 | inputs, targets = inputs.to(device), targets.to(device) 22 | outputs = net(inputs) 23 | loss = criterion(outputs, targets) 24 | 25 | test_loss += loss.item() 26 | 27 | predicted = outputs.argmax(dim=1) 28 | 29 | 30 | for i in range(len(inputs)): 31 | predictions[targets[i].item()].append(torch.softmax(outputs[i], dim=0).detach().cpu().tolist()) 32 | 33 | total += targets.size(0) 34 | correct += predicted.eq(targets).sum().item() 35 | 36 | for idx, thing in enumerate(predicted): 37 | if (task == "CROSS7" or task == "SEVEN") and (batch_idx * inputs.shape[0] + idx in cross_idx):#baby_idx: 38 | result_cross[targets[idx]][predicted[idx]] += 1 39 | result[targets[idx]][predicted[idx]] += 1 40 | 41 | progress_bar(batch_idx, len(dataloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' % (test_loss/(batch_idx+1), 100.*correct/total, correct, total)) 42 | 43 | 44 | torch.set_printoptions(linewidth=400) 45 | 46 | print(result) 47 | 48 | if task == "CROSS7" or task == "SEVEN": 49 | print(result_cross[7]) 50 | 51 | 52 | def train(net, dataloader, criterion, optimizer): 53 | 54 | net.train() 55 | train_loss = 0 56 | correct = 0 57 | total = 0 58 | 59 | predictions = [[] for i in range(10)] 60 | 61 | for batch_idx, (inputs, targets) in enumerate(dataloader): 62 | 63 | inputs, targets = inputs.to(device), targets.to(device) 64 | 65 | optimizer.zero_grad() 66 | outputs = net(inputs) 67 | 68 | loss = criterion(outputs, targets) 69 | loss.backward() 70 | optimizer.step() 71 | 72 | train_loss += loss.item() 73 | predicted = outputs.argmax(dim=1) 74 | 75 | if len(targets.size()) > 1: 76 | targets = targets.argmax(dim=1) 77 | 78 | total += targets.size(0) 79 | correct += predicted.eq(targets).sum().item() 80 | 81 | progress_bar(batch_idx, len(dataloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' % (train_loss/(batch_idx+1), 100.*correct/total, correct, total)) 82 | 83 | -------------------------------------------------------------------------------- /Unlearning/main.py: -------------------------------------------------------------------------------- 1 | import torch.optim as optim 2 | 3 | import platform 4 | from sacred import Experiment 5 | from sacred.observers import FileStorageObserver 6 | from sacred import SETTINGS 7 | 8 | from sklearn.svm import SVC 9 | 10 | import argparse 11 | import sys 12 | import os 13 | 14 | from traintest import * 15 | from prepare import * 16 | from utils import * 17 | import models 18 | 19 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 20 | 21 | @ex.automain 22 | def main(lr, epochs, dataset, task, percentage, n, oracle, normalization, eraselabel, neutralization, neutralization_epochs, 23 | _run): 24 | 25 | print(f"my pid: {os.getpid()}") 26 | 27 | print(f"Dataset {dataset} is chosen") 28 | print(f"Percentage {percentage} is chosen") 29 | print(f"Geneated {n} samples") 30 | 31 | print('==> Building model..') 32 | 33 | net = prepare_classifier(dataset, task, normalization=normalization) 34 | 35 | net.eval() 36 | 37 | fe = feature_extractor() 38 | 39 | net.to(device) 40 | 41 | train_criterion = softXEnt 42 | test_criterion = nn.CrossEntropyLoss() 43 | optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=2e-5) 44 | 45 | tasks = [task] 46 | for task in tasks: 47 | print(f"Task {task} is chosen") 48 | 49 | print('==> Generating data..') 50 | 51 | #trainloader, testloader, validrloader, valideloader = prepare_dataloader(net, fe, dataset, task=task, percentage=percentage, n=n, oracle=oracle, eraselabel = eraselabel) 52 | trainloader, testloader, De, validrloader, valideloader, neutralizeloader = prepare_dataloader(net, fe, dataset, task=task, percentage=percentage, n=n, oracle=oracle, eraselabel = eraselabel) 53 | 54 | print('==> Membership_attack..') 55 | 56 | get_membership_attack_model(net, validrloader, valideloader, testloader) 57 | 58 | print("Finished Membership attack!") 59 | print('==> Start Training!') 60 | 61 | 62 | test(net, testloader, test_criterion, task) 63 | 64 | if neutralization: 65 | for epoch in range(neutralization_epochs): 66 | print('\nEpoch: %d' % epoch, "Neutralization") 67 | train(net, neutralizeloader, train_criterion, optimizer) 68 | test(net, testloader, test_criterion, task) 69 | evaluation(net, De) 70 | 71 | for epoch in range(epochs): 72 | print('\nEpoch: %d' % epoch, f"{eraselabel}") 73 | #check_output_KL(trainloader_false, net_gold, net) 74 | train(net, trainloader, train_criterion, optimizer) 75 | test(net, testloader, test_criterion, task) 76 | evaluation(net, De) 77 | 78 | get_membership_attack_model(net, validrloader, valideloader, testloader) 79 | 80 | torch.save(net.state_dict(), f"./ASR_models/{dataset}_{task}_{percentage}_{eraselabel}_{os.getpid()}.pyt") 81 | -------------------------------------------------------------------------------- /Training/main.py: -------------------------------------------------------------------------------- 1 | '''Train CIFAR10 with PyTorch.''' 2 | import torch 3 | import torch.optim as optim 4 | 5 | import os 6 | import numpy as np 7 | import platform 8 | 9 | from sacred import Experiment 10 | from sacred.observers import FileStorageObserver 11 | from sacred import SETTINGS 12 | 13 | from models import * 14 | #from traintest import * 15 | import traintest 16 | from utils import * 17 | from prepare import * 18 | 19 | SETTINGS.CAPTURE_MODE = 'no' 20 | 21 | ex = Experiment('Train Classifier') 22 | ex.add_config('./default.yaml') 23 | 24 | 25 | def softXEnt(inputs, targets): 26 | if len(targets.shape) < 2: 27 | num_classes = inputs.shape[1] 28 | targets = torch.nn.functional.one_hot(targets, num_classes=num_classes).float().to(device) 29 | 30 | logprobs = torch.nn.functional.log_softmax (inputs, dim = 1) 31 | 32 | ent = targets * logprobs 33 | 34 | ent = torch.clip(ent, max=10) 35 | 36 | return -ent.sum() / inputs.shape[0] 37 | 38 | @ex.automain 39 | def main(seed, lr, epochs, dataset, task, oracle, 40 | _run): 41 | 42 | normalization = "BatchNorm" 43 | save_dir = f"./save/resnet18_{dataset}_{task}_{oracle}_3" 44 | print(f"pid: {os.getpid()}") 45 | print(f"dataset: {dataset}") 46 | print(f"task: {task}") 47 | print(f"normalization: {normalization}") 48 | print(f"oracle: {oracle}") 49 | print(f"will be saved at: {save_dir}") 50 | 51 | trainloader, testloader = prepare_dataloader(dataset=dataset, task=task, oracle=oracle) 52 | 53 | if dataset == "MNIST": 54 | print(f"main: train with MNIST!!") 55 | net = PreActResNet18(num_classes=10, num_channels=1 if dataset=="MNIST" else 3, normalization=normalization) 56 | elif dataset == "CIFAR10": 57 | print("main: train with CIFAR10!") 58 | net = PreActResNet18(num_classes=10, num_channels=1 if dataset=="MNIST" else 3, normalization=normalization) 59 | elif dataset == "CIFAR100": 60 | print("main: train with CIFAR100!") 61 | net = PreActResNet34(num_classes=100, num_channels=3, normalization=normalization) 62 | else: 63 | print(f"main: train with {dataset}!!") 64 | net = PreActResNet34(num_classes=20, num_channels=3, normalization=normalization) 65 | 66 | net = net.to(device) 67 | 68 | train_criterion = softXEnt 69 | test_criterion = nn.CrossEntropyLoss() 70 | 71 | optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=5e-4) 72 | scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[20,30], gamma=0.1) 73 | 74 | print("Start Training!") 75 | 76 | for epoch in range(epochs): 77 | print('\nEpoch: %d' % epoch) 78 | 79 | #test (net, testloader, test_criterion, task) 80 | traintest.train(net, trainloader, train_criterion, optimizer) 81 | traintest.test (net, testloader, test_criterion, task) 82 | #exit() 83 | scheduler.step() 84 | 85 | save_model(net, _run, save_dir) 86 | -------------------------------------------------------------------------------- /Unlearning/models/cifar10_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2018 Erik Linder-Norén 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import torch.nn as nn 26 | import torch.nn.functional as F 27 | import torch 28 | 29 | import torchvision.transforms as transforms 30 | 31 | """ 32 | Code is taken from 33 | https://github.com/eriklindernoren/PyTorch-GAN 34 | """ 35 | 36 | class Generator(nn.Module): 37 | def __init__(self, conditional, class_num): 38 | super(Generator, self).__init__() 39 | 40 | self.init_size = 32 // 4 41 | ngf=64 42 | self.ngf = ngf 43 | 44 | if conditional is False: 45 | self.l1 = nn.Sequential(nn.Linear(100, ngf * 8 * self.init_size ** 2)) 46 | 47 | else: 48 | print("Conditional Generator is selected") 49 | self.ll = nn.Sequential(nn.Linear(100 + class_num, ngf * 8 * self.init_size ** 2)) 50 | 51 | self.conv_blocks = nn.Sequential( 52 | nn.BatchNorm2d(ngf * 8), 53 | nn.Upsample(scale_factor=2), 54 | nn.Conv2d(ngf * 8, ngf * 4, 3, stride=1, padding=1), 55 | nn.BatchNorm2d(ngf * 4, 0.8), 56 | nn.LeakyReLU(0.2, inplace=True), 57 | nn.Upsample(scale_factor=2), 58 | nn.Conv2d(ngf * 4, ngf * 2, 3, stride=1, padding=1), 59 | nn.BatchNorm2d(ngf * 2, 0.8), 60 | nn.LeakyReLU(0.2, inplace=True), 61 | nn.Conv2d(ngf * 2, 3, 3, stride=1, padding=1), 62 | nn.Sigmoid(), 63 | transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 64 | ) 65 | 66 | def forward(self, z, y): 67 | if y is None: return self.forward1(z) 68 | else: return self.forward2(z, y) 69 | 70 | def forward1(self, z): 71 | out = self.l1(z) 72 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 73 | img = self.conv_blocks(out) 74 | return img 75 | 76 | def forward2(self, z, y): 77 | zy = torch.concat([z, y], dim=1) 78 | 79 | out = self.ll(zy) 80 | 81 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 82 | img = self.conv_blocks(out) 83 | return img 84 | -------------------------------------------------------------------------------- /Inversion/models/mnist_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2018 Erik Linder-Norén 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import torch.nn as nn 26 | import torch.nn.functional as F 27 | import torch 28 | 29 | import torchvision.transforms as transforms 30 | 31 | """ 32 | Code is taken from 33 | https://github.com/eriklindernoren/PyTorch-GAN 34 | """ 35 | 36 | class Generator(nn.Module): 37 | def __init__(self, conditional=False, class_num=10): 38 | super(Generator, self).__init__() 39 | 40 | self.init_size = 32 // 4 41 | ngf=16 42 | self.ngf = ngf 43 | self.class_num = class_num 44 | 45 | if conditional is False: 46 | self.l1 = nn.Sequential(nn.Linear(100, ngf * 8 * self.init_size ** 2)) 47 | 48 | else: 49 | print("Conditional Generator is selected") 50 | self.ll = nn.Sequential(nn.Linear(100 + class_num, ngf * 8 * self.init_size ** 2)) 51 | 52 | self.conv_blocks = nn.Sequential( 53 | nn.BatchNorm2d(ngf * 8), 54 | nn.Upsample(scale_factor=2), 55 | nn.Conv2d(ngf * 8, ngf * 4, 3, stride=1, padding=1), 56 | nn.BatchNorm2d(ngf * 4, 0.8), 57 | nn.LeakyReLU(0.2, inplace=True), 58 | nn.Upsample(scale_factor=2), 59 | nn.Conv2d(ngf * 4, ngf * 2, 3, stride=1, padding=1), 60 | nn.BatchNorm2d(ngf * 2, 0.8), 61 | nn.LeakyReLU(0.2, inplace=True), 62 | nn.Conv2d(ngf * 2, 1, 3, stride=1, padding=1), 63 | nn.Sigmoid(), 64 | transforms.Normalize((0.1307,), (0.3081,)), 65 | ) 66 | 67 | def forward(self, z, y): 68 | if y is None: return self.forward1(z) 69 | else: return self.forward2(z, y) 70 | 71 | def forward1(self, z): 72 | out = self.l1(z) 73 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 74 | img = self.conv_blocks(out) 75 | return img 76 | 77 | def forward2(self, z, y): 78 | zy = torch.concat([z, y], dim=1) 79 | 80 | out = self.ll(zy) 81 | 82 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 83 | img = self.conv_blocks(out) 84 | return img 85 | -------------------------------------------------------------------------------- /Unlearning/models/mnist_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2018 Erik Linder-Norén 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import torch.nn as nn 26 | import torch.nn.functional as F 27 | import torch 28 | 29 | import torchvision.transforms as transforms 30 | 31 | """ 32 | Code is taken from 33 | https://github.com/eriklindernoren/PyTorch-GAN 34 | """ 35 | 36 | class Generator(nn.Module): 37 | def __init__(self, conditional=False, class_num=10): 38 | super(Generator, self).__init__() 39 | 40 | self.init_size = 32 // 4 41 | ngf=16 42 | self.ngf = ngf 43 | self.class_num = class_num 44 | 45 | if conditional is False: 46 | self.l1 = nn.Sequential(nn.Linear(100, ngf * 8 * self.init_size ** 2)) 47 | 48 | else: 49 | print("Conditional Generator is selected") 50 | self.ll = nn.Sequential(nn.Linear(100 + class_num, ngf * 8 * self.init_size ** 2)) 51 | 52 | self.conv_blocks = nn.Sequential( 53 | nn.BatchNorm2d(ngf * 8), 54 | nn.Upsample(scale_factor=2), 55 | nn.Conv2d(ngf * 8, ngf * 4, 3, stride=1, padding=1), 56 | nn.BatchNorm2d(ngf * 4, 0.8), 57 | nn.LeakyReLU(0.2, inplace=True), 58 | nn.Upsample(scale_factor=2), 59 | nn.Conv2d(ngf * 4, ngf * 2, 3, stride=1, padding=1), 60 | nn.BatchNorm2d(ngf * 2, 0.8), 61 | nn.LeakyReLU(0.2, inplace=True), 62 | nn.Conv2d(ngf * 2, 1, 3, stride=1, padding=1), 63 | nn.Sigmoid(), 64 | transforms.Normalize((0.1307,), (0.3081,)), 65 | ) 66 | 67 | def forward(self, z, y): 68 | if y is None: return self.forward1(z) 69 | else: return self.forward2(z, y) 70 | 71 | def forward1(self, z): 72 | out = self.l1(z) 73 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 74 | img = self.conv_blocks(out) 75 | return img 76 | 77 | def forward2(self, z, y): 78 | zy = torch.concat([z, y], dim=1) 79 | 80 | out = self.ll(zy) 81 | 82 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 83 | img = self.conv_blocks(out) 84 | return img 85 | -------------------------------------------------------------------------------- /Inversion/models/cifar10_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | MIT License 3 | 4 | Copyright (c) 2018 Erik Linder-Norén 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | """ 24 | 25 | import torch.nn as nn 26 | import torch.nn.functional as F 27 | import torch 28 | 29 | import torchvision.transforms as transforms 30 | 31 | """ 32 | Code is taken from 33 | https://github.com/eriklindernoren/PyTorch-GAN 34 | """ 35 | 36 | class Generator(nn.Module): 37 | def __init__(self, conditional, class_num): 38 | super(Generator, self).__init__() 39 | 40 | self.init_size = 32 // 4 41 | ngf=64 42 | self.ngf = ngf 43 | self.class_num=class_num 44 | 45 | if conditional is False: 46 | self.l1 = nn.Sequential(nn.Linear(100, ngf * 8 * self.init_size ** 2)) 47 | 48 | else: 49 | print("Conditional Generator is selected") 50 | self.ll = nn.Sequential(nn.Linear(100 + class_num, ngf * 8 * self.init_size ** 2)) 51 | 52 | self.conv_blocks = nn.Sequential( 53 | nn.BatchNorm2d(ngf * 8), 54 | nn.Upsample(scale_factor=2), 55 | nn.Conv2d(ngf * 8, ngf * 4, 3, stride=1, padding=1), 56 | nn.BatchNorm2d(ngf * 4, 0.8), 57 | nn.LeakyReLU(0.2, inplace=True), 58 | nn.Upsample(scale_factor=2), 59 | nn.Conv2d(ngf * 4, ngf * 2, 3, stride=1, padding=1), 60 | nn.BatchNorm2d(ngf * 2, 0.8), 61 | nn.LeakyReLU(0.2, inplace=True), 62 | nn.Conv2d(ngf * 2, 3, 3, stride=1, padding=1), 63 | nn.Sigmoid(), 64 | transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), 65 | ) 66 | 67 | def forward(self, z, y): 68 | if y is None: return self.forward1(z) 69 | else: return self.forward2(z, y) 70 | 71 | def forward1(self, z): 72 | out = self.l1(z) 73 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 74 | img = self.conv_blocks(out) 75 | return img 76 | 77 | def forward2(self, z, y): 78 | zy = torch.concat([z, y], dim=1) 79 | 80 | out = self.ll(zy) 81 | 82 | out = out.view(out.shape[0], self.ngf * 8, self.init_size, self.init_size) 83 | img = self.conv_blocks(out) 84 | return img 85 | -------------------------------------------------------------------------------- /Training/models/mobilenetv2.py: -------------------------------------------------------------------------------- 1 | '''MobileNetV2 in PyTorch. 2 | 3 | See the paper "Inverted Residuals and Linear Bottlenecks: 4 | Mobile Networks for Classification, Detection and Segmentation" for more details. 5 | ''' 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Block(nn.Module): 12 | '''expand + depthwise + pointwise''' 13 | def __init__(self, in_planes, out_planes, expansion, stride): 14 | super(Block, self).__init__() 15 | self.stride = stride 16 | 17 | planes = expansion * in_planes 18 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False) 19 | self.bn1 = nn.BatchNorm2d(planes) 20 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes, bias=False) 21 | self.bn2 = nn.BatchNorm2d(planes) 22 | self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 23 | self.bn3 = nn.BatchNorm2d(out_planes) 24 | 25 | self.shortcut = nn.Sequential() 26 | if stride == 1 and in_planes != out_planes: 27 | self.shortcut = nn.Sequential( 28 | nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False), 29 | nn.BatchNorm2d(out_planes), 30 | ) 31 | 32 | def forward(self, x): 33 | out = F.relu(self.bn1(self.conv1(x))) 34 | out = F.relu(self.bn2(self.conv2(out))) 35 | out = self.bn3(self.conv3(out)) 36 | out = out + self.shortcut(x) if self.stride==1 else out 37 | return out 38 | 39 | 40 | class MobileNetV2(nn.Module): 41 | # (expansion, out_planes, num_blocks, stride) 42 | cfg = [(1, 16, 1, 1), 43 | (6, 24, 2, 1), # NOTE: change stride 2 -> 1 for CIFAR10 44 | (6, 32, 3, 2), 45 | (6, 64, 4, 2), 46 | (6, 96, 3, 1), 47 | (6, 160, 3, 2), 48 | (6, 320, 1, 1)] 49 | 50 | def __init__(self, num_classes=10): 51 | super(MobileNetV2, self).__init__() 52 | # NOTE: change conv1 stride 2 -> 1 for CIFAR10 53 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False) 54 | self.bn1 = nn.BatchNorm2d(32) 55 | self.layers = self._make_layers(in_planes=32) 56 | self.conv2 = nn.Conv2d(320, 1280, kernel_size=1, stride=1, padding=0, bias=False) 57 | self.bn2 = nn.BatchNorm2d(1280) 58 | self.linear = nn.Linear(1280, num_classes) 59 | 60 | def _make_layers(self, in_planes): 61 | layers = [] 62 | for expansion, out_planes, num_blocks, stride in self.cfg: 63 | strides = [stride] + [1]*(num_blocks-1) 64 | for stride in strides: 65 | layers.append(Block(in_planes, out_planes, expansion, stride)) 66 | in_planes = out_planes 67 | return nn.Sequential(*layers) 68 | 69 | def forward(self, x): 70 | out = F.relu(self.bn1(self.conv1(x))) 71 | out = self.layers(out) 72 | out = F.relu(self.bn2(self.conv2(out))) 73 | # NOTE: change pooling kernel_size 7 -> 4 for CIFAR10 74 | out = F.avg_pool2d(out, 4) 75 | out = out.view(out.size(0), -1) 76 | out = self.linear(out) 77 | return out 78 | 79 | 80 | def test(): 81 | net = MobileNetV2() 82 | x = torch.randn(2,3,32,32) 83 | y = net(x) 84 | print(y.size()) 85 | 86 | # test() 87 | -------------------------------------------------------------------------------- /Inversion/models/mobilenetv2.py: -------------------------------------------------------------------------------- 1 | '''MobileNetV2 in PyTorch. 2 | 3 | See the paper "Inverted Residuals and Linear Bottlenecks: 4 | Mobile Networks for Classification, Detection and Segmentation" for more details. 5 | ''' 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Block(nn.Module): 12 | '''expand + depthwise + pointwise''' 13 | def __init__(self, in_planes, out_planes, expansion, stride): 14 | super(Block, self).__init__() 15 | self.stride = stride 16 | 17 | planes = expansion * in_planes 18 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False) 19 | self.bn1 = nn.BatchNorm2d(planes) 20 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes, bias=False) 21 | self.bn2 = nn.BatchNorm2d(planes) 22 | self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 23 | self.bn3 = nn.BatchNorm2d(out_planes) 24 | 25 | self.shortcut = nn.Sequential() 26 | if stride == 1 and in_planes != out_planes: 27 | self.shortcut = nn.Sequential( 28 | nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False), 29 | nn.BatchNorm2d(out_planes), 30 | ) 31 | 32 | def forward(self, x): 33 | out = F.relu(self.bn1(self.conv1(x))) 34 | out = F.relu(self.bn2(self.conv2(out))) 35 | out = self.bn3(self.conv3(out)) 36 | out = out + self.shortcut(x) if self.stride==1 else out 37 | return out 38 | 39 | 40 | class MobileNetV2(nn.Module): 41 | # (expansion, out_planes, num_blocks, stride) 42 | cfg = [(1, 16, 1, 1), 43 | (6, 24, 2, 1), # NOTE: change stride 2 -> 1 for CIFAR10 44 | (6, 32, 3, 2), 45 | (6, 64, 4, 2), 46 | (6, 96, 3, 1), 47 | (6, 160, 3, 2), 48 | (6, 320, 1, 1)] 49 | 50 | def __init__(self, num_classes=10): 51 | super(MobileNetV2, self).__init__() 52 | # NOTE: change conv1 stride 2 -> 1 for CIFAR10 53 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False) 54 | self.bn1 = nn.BatchNorm2d(32) 55 | self.layers = self._make_layers(in_planes=32) 56 | self.conv2 = nn.Conv2d(320, 1280, kernel_size=1, stride=1, padding=0, bias=False) 57 | self.bn2 = nn.BatchNorm2d(1280) 58 | self.linear = nn.Linear(1280, num_classes) 59 | 60 | def _make_layers(self, in_planes): 61 | layers = [] 62 | for expansion, out_planes, num_blocks, stride in self.cfg: 63 | strides = [stride] + [1]*(num_blocks-1) 64 | for stride in strides: 65 | layers.append(Block(in_planes, out_planes, expansion, stride)) 66 | in_planes = out_planes 67 | return nn.Sequential(*layers) 68 | 69 | def forward(self, x): 70 | out = F.relu(self.bn1(self.conv1(x))) 71 | out = self.layers(out) 72 | out = F.relu(self.bn2(self.conv2(out))) 73 | # NOTE: change pooling kernel_size 7 -> 4 for CIFAR10 74 | out = F.avg_pool2d(out, 4) 75 | out = out.view(out.size(0), -1) 76 | out = self.linear(out) 77 | return out 78 | 79 | 80 | def test(): 81 | net = MobileNetV2() 82 | x = torch.randn(2,3,32,32) 83 | y = net(x) 84 | print(y.size()) 85 | 86 | # test() 87 | -------------------------------------------------------------------------------- /Unlearning/models/mobilenetv2.py: -------------------------------------------------------------------------------- 1 | '''MobileNetV2 in PyTorch. 2 | 3 | See the paper "Inverted Residuals and Linear Bottlenecks: 4 | Mobile Networks for Classification, Detection and Segmentation" for more details. 5 | ''' 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | 11 | class Block(nn.Module): 12 | '''expand + depthwise + pointwise''' 13 | def __init__(self, in_planes, out_planes, expansion, stride): 14 | super(Block, self).__init__() 15 | self.stride = stride 16 | 17 | planes = expansion * in_planes 18 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False) 19 | self.bn1 = nn.BatchNorm2d(planes) 20 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes, bias=False) 21 | self.bn2 = nn.BatchNorm2d(planes) 22 | self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 23 | self.bn3 = nn.BatchNorm2d(out_planes) 24 | 25 | self.shortcut = nn.Sequential() 26 | if stride == 1 and in_planes != out_planes: 27 | self.shortcut = nn.Sequential( 28 | nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False), 29 | nn.BatchNorm2d(out_planes), 30 | ) 31 | 32 | def forward(self, x): 33 | out = F.relu(self.bn1(self.conv1(x))) 34 | out = F.relu(self.bn2(self.conv2(out))) 35 | out = self.bn3(self.conv3(out)) 36 | out = out + self.shortcut(x) if self.stride==1 else out 37 | return out 38 | 39 | 40 | class MobileNetV2(nn.Module): 41 | # (expansion, out_planes, num_blocks, stride) 42 | cfg = [(1, 16, 1, 1), 43 | (6, 24, 2, 1), # NOTE: change stride 2 -> 1 for CIFAR10 44 | (6, 32, 3, 2), 45 | (6, 64, 4, 2), 46 | (6, 96, 3, 1), 47 | (6, 160, 3, 2), 48 | (6, 320, 1, 1)] 49 | 50 | def __init__(self, num_classes=10): 51 | super(MobileNetV2, self).__init__() 52 | # NOTE: change conv1 stride 2 -> 1 for CIFAR10 53 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False) 54 | self.bn1 = nn.BatchNorm2d(32) 55 | self.layers = self._make_layers(in_planes=32) 56 | self.conv2 = nn.Conv2d(320, 1280, kernel_size=1, stride=1, padding=0, bias=False) 57 | self.bn2 = nn.BatchNorm2d(1280) 58 | self.linear = nn.Linear(1280, num_classes) 59 | 60 | def _make_layers(self, in_planes): 61 | layers = [] 62 | for expansion, out_planes, num_blocks, stride in self.cfg: 63 | strides = [stride] + [1]*(num_blocks-1) 64 | for stride in strides: 65 | layers.append(Block(in_planes, out_planes, expansion, stride)) 66 | in_planes = out_planes 67 | return nn.Sequential(*layers) 68 | 69 | def forward(self, x): 70 | out = F.relu(self.bn1(self.conv1(x))) 71 | out = self.layers(out) 72 | out = F.relu(self.bn2(self.conv2(out))) 73 | # NOTE: change pooling kernel_size 7 -> 4 for CIFAR10 74 | out = F.avg_pool2d(out, 4) 75 | out = out.view(out.size(0), -1) 76 | out = self.linear(out) 77 | return out 78 | 79 | 80 | def test(): 81 | net = MobileNetV2() 82 | x = torch.randn(2,3,32,32) 83 | y = net(x) 84 | print(y.size()) 85 | 86 | # test() 87 | -------------------------------------------------------------------------------- /Inversion/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchvision.utils import save_image 3 | from torchvision import transforms 4 | from torchvision import datasets as dsets 5 | 6 | from torch.nn import functional as F 7 | from torch.autograd import Variable 8 | import torch.nn as nn 9 | import numpy as np 10 | 11 | import cross_train 12 | cross_idx = cross_train.myidx 13 | 14 | device = 'cuda' 15 | 16 | 17 | def softXEnt(inputs, targets): 18 | if len(targets.shape) < 2: 19 | num_classes = inputs.shape[1] 20 | targets = torch.nn.functional.one_hot(targets, num_classes=num_classes).float().to(device) 21 | 22 | logprobs = torch.nn.functional.log_softmax (inputs, dim = 1) 23 | 24 | #if logprobs.min() < -100: 25 | #print(logprobs) 26 | #print("logprobs too low!") 27 | #_=input() 28 | 29 | return -(targets * logprobs).sum() / inputs.shape[0] 30 | 31 | def print_outputs(classified, classified1, classified2, classified3, classified4): 32 | np.set_printoptions(precision=4, linewidth=400, suppress=True) 33 | print(torch.softmax(classified[0:20], dim=1).detach().to('cpu').numpy().transpose()) 34 | print(torch.softmax(classified1[0:20], dim=1).detach().to('cpu').numpy().transpose()) 35 | print(torch.softmax(classified2[0:20], dim=1).detach().to('cpu').numpy().transpose()) 36 | print(torch.softmax(classified3[0:20], dim=1).detach().to('cpu').numpy().transpose()) 37 | print(torch.softmax(classified4[0:20], dim=1).detach().to('cpu').numpy().transpose()) 38 | 39 | 40 | 41 | def save_fixed_image(generator, unnormalizer, fixed_z, fixed_class, dir_name, epoch): 42 | 43 | fixed_image = generator(fixed_z, fixed_class) 44 | 45 | nrow = int(fixed_image.shape[0] ** 0.5) 46 | 47 | save_image(unnormalizer(fixed_image), f"{dir_name}/{epoch}.png", nrow=nrow, normalize=False) 48 | 49 | 50 | def sm(a): 51 | return torch.softmax(a, dim=1) 52 | 53 | 54 | 55 | def calculate_grad_norm(model): 56 | total_norm = 0 57 | 58 | parameters = [p for p in model.parameters() if p.grad is not None and p.requires_grad] 59 | for p in parameters: 60 | #mything = Variable(p.grad.data, requires_grad=True) 61 | param_norm = torch.norm(p.grad.detach().data, 1) 62 | total_norm = total_norm + param_norm 63 | 64 | return total_norm 65 | 66 | def calculate_grad_norm_losses(model, optimizer, losses): 67 | grad_norms = [] 68 | 69 | for idx, loss in enumerate(losses): 70 | if type(loss) == int or type(loss) == float: 71 | grad_norms.append(0) 72 | continue 73 | 74 | optimizer.zero_grad() 75 | 76 | loss.backward(retain_graph=True) 77 | 78 | grad_norms.append(calculate_grad_norm(model)) 79 | 80 | parameters = [p for p in model.parameters() if p.grad is not None and p.requires_grad] 81 | for p in parameters: 82 | p.grad = None 83 | 84 | optimizer.zero_grad() 85 | 86 | return grad_norms 87 | 88 | def check_gradient_of_losses(generator, optimizer, my_losses): 89 | loss_names = ["entropy", "entropy2","augmentation", "bn", "div", "tv", "bn2"] 90 | 91 | grad_norms = calculate_grad_norm_losses(generator, optimizer, my_losses) 92 | for idx, thing in enumerate(grad_norms): 93 | if grad_norms[idx] == 0: continue 94 | 95 | grad_norms[idx] = grad_norms[idx].detach().item() 96 | print("========Grad norms========") 97 | print(loss_names) 98 | print(["{%0.3f}" % (i) for i in grad_norms]) 99 | 100 | optimizer.zero_grad() 101 | 102 | -------------------------------------------------------------------------------- /Training/models/googlenet.py: -------------------------------------------------------------------------------- 1 | '''GoogLeNet with PyTorch.''' 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class Inception(nn.Module): 8 | def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes): 9 | super(Inception, self).__init__() 10 | # 1x1 conv branch 11 | self.b1 = nn.Sequential( 12 | nn.Conv2d(in_planes, n1x1, kernel_size=1), 13 | nn.BatchNorm2d(n1x1), 14 | nn.ReLU(True), 15 | ) 16 | 17 | # 1x1 conv -> 3x3 conv branch 18 | self.b2 = nn.Sequential( 19 | nn.Conv2d(in_planes, n3x3red, kernel_size=1), 20 | nn.BatchNorm2d(n3x3red), 21 | nn.ReLU(True), 22 | nn.Conv2d(n3x3red, n3x3, kernel_size=3, padding=1), 23 | nn.BatchNorm2d(n3x3), 24 | nn.ReLU(True), 25 | ) 26 | 27 | # 1x1 conv -> 5x5 conv branch 28 | self.b3 = nn.Sequential( 29 | nn.Conv2d(in_planes, n5x5red, kernel_size=1), 30 | nn.BatchNorm2d(n5x5red), 31 | nn.ReLU(True), 32 | nn.Conv2d(n5x5red, n5x5, kernel_size=3, padding=1), 33 | nn.BatchNorm2d(n5x5), 34 | nn.ReLU(True), 35 | nn.Conv2d(n5x5, n5x5, kernel_size=3, padding=1), 36 | nn.BatchNorm2d(n5x5), 37 | nn.ReLU(True), 38 | ) 39 | 40 | # 3x3 pool -> 1x1 conv branch 41 | self.b4 = nn.Sequential( 42 | nn.MaxPool2d(3, stride=1, padding=1), 43 | nn.Conv2d(in_planes, pool_planes, kernel_size=1), 44 | nn.BatchNorm2d(pool_planes), 45 | nn.ReLU(True), 46 | ) 47 | 48 | def forward(self, x): 49 | y1 = self.b1(x) 50 | y2 = self.b2(x) 51 | y3 = self.b3(x) 52 | y4 = self.b4(x) 53 | return torch.cat([y1,y2,y3,y4], 1) 54 | 55 | 56 | class GoogLeNet(nn.Module): 57 | def __init__(self): 58 | super(GoogLeNet, self).__init__() 59 | self.pre_layers = nn.Sequential( 60 | nn.Conv2d(3, 192, kernel_size=3, padding=1), 61 | nn.BatchNorm2d(192), 62 | nn.ReLU(True), 63 | ) 64 | 65 | self.a3 = Inception(192, 64, 96, 128, 16, 32, 32) 66 | self.b3 = Inception(256, 128, 128, 192, 32, 96, 64) 67 | 68 | self.maxpool = nn.MaxPool2d(3, stride=2, padding=1) 69 | 70 | self.a4 = Inception(480, 192, 96, 208, 16, 48, 64) 71 | self.b4 = Inception(512, 160, 112, 224, 24, 64, 64) 72 | self.c4 = Inception(512, 128, 128, 256, 24, 64, 64) 73 | self.d4 = Inception(512, 112, 144, 288, 32, 64, 64) 74 | self.e4 = Inception(528, 256, 160, 320, 32, 128, 128) 75 | 76 | self.a5 = Inception(832, 256, 160, 320, 32, 128, 128) 77 | self.b5 = Inception(832, 384, 192, 384, 48, 128, 128) 78 | 79 | self.avgpool = nn.AvgPool2d(8, stride=1) 80 | self.linear = nn.Linear(1024, 10) 81 | 82 | def forward(self, x): 83 | out = self.pre_layers(x) 84 | out = self.a3(out) 85 | out = self.b3(out) 86 | out = self.maxpool(out) 87 | out = self.a4(out) 88 | out = self.b4(out) 89 | out = self.c4(out) 90 | out = self.d4(out) 91 | out = self.e4(out) 92 | out = self.maxpool(out) 93 | out = self.a5(out) 94 | out = self.b5(out) 95 | out = self.avgpool(out) 96 | out = out.view(out.size(0), -1) 97 | out = self.linear(out) 98 | return out 99 | 100 | 101 | def test(): 102 | net = GoogLeNet() 103 | x = torch.randn(1,3,32,32) 104 | y = net(x) 105 | print(y.size()) 106 | 107 | # test() 108 | -------------------------------------------------------------------------------- /Inversion/models/googlenet.py: -------------------------------------------------------------------------------- 1 | '''GoogLeNet with PyTorch.''' 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class Inception(nn.Module): 8 | def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes): 9 | super(Inception, self).__init__() 10 | # 1x1 conv branch 11 | self.b1 = nn.Sequential( 12 | nn.Conv2d(in_planes, n1x1, kernel_size=1), 13 | nn.BatchNorm2d(n1x1), 14 | nn.ReLU(True), 15 | ) 16 | 17 | # 1x1 conv -> 3x3 conv branch 18 | self.b2 = nn.Sequential( 19 | nn.Conv2d(in_planes, n3x3red, kernel_size=1), 20 | nn.BatchNorm2d(n3x3red), 21 | nn.ReLU(True), 22 | nn.Conv2d(n3x3red, n3x3, kernel_size=3, padding=1), 23 | nn.BatchNorm2d(n3x3), 24 | nn.ReLU(True), 25 | ) 26 | 27 | # 1x1 conv -> 5x5 conv branch 28 | self.b3 = nn.Sequential( 29 | nn.Conv2d(in_planes, n5x5red, kernel_size=1), 30 | nn.BatchNorm2d(n5x5red), 31 | nn.ReLU(True), 32 | nn.Conv2d(n5x5red, n5x5, kernel_size=3, padding=1), 33 | nn.BatchNorm2d(n5x5), 34 | nn.ReLU(True), 35 | nn.Conv2d(n5x5, n5x5, kernel_size=3, padding=1), 36 | nn.BatchNorm2d(n5x5), 37 | nn.ReLU(True), 38 | ) 39 | 40 | # 3x3 pool -> 1x1 conv branch 41 | self.b4 = nn.Sequential( 42 | nn.MaxPool2d(3, stride=1, padding=1), 43 | nn.Conv2d(in_planes, pool_planes, kernel_size=1), 44 | nn.BatchNorm2d(pool_planes), 45 | nn.ReLU(True), 46 | ) 47 | 48 | def forward(self, x): 49 | y1 = self.b1(x) 50 | y2 = self.b2(x) 51 | y3 = self.b3(x) 52 | y4 = self.b4(x) 53 | return torch.cat([y1,y2,y3,y4], 1) 54 | 55 | 56 | class GoogLeNet(nn.Module): 57 | def __init__(self): 58 | super(GoogLeNet, self).__init__() 59 | self.pre_layers = nn.Sequential( 60 | nn.Conv2d(3, 192, kernel_size=3, padding=1), 61 | nn.BatchNorm2d(192), 62 | nn.ReLU(True), 63 | ) 64 | 65 | self.a3 = Inception(192, 64, 96, 128, 16, 32, 32) 66 | self.b3 = Inception(256, 128, 128, 192, 32, 96, 64) 67 | 68 | self.maxpool = nn.MaxPool2d(3, stride=2, padding=1) 69 | 70 | self.a4 = Inception(480, 192, 96, 208, 16, 48, 64) 71 | self.b4 = Inception(512, 160, 112, 224, 24, 64, 64) 72 | self.c4 = Inception(512, 128, 128, 256, 24, 64, 64) 73 | self.d4 = Inception(512, 112, 144, 288, 32, 64, 64) 74 | self.e4 = Inception(528, 256, 160, 320, 32, 128, 128) 75 | 76 | self.a5 = Inception(832, 256, 160, 320, 32, 128, 128) 77 | self.b5 = Inception(832, 384, 192, 384, 48, 128, 128) 78 | 79 | self.avgpool = nn.AvgPool2d(8, stride=1) 80 | self.linear = nn.Linear(1024, 10) 81 | 82 | def forward(self, x): 83 | out = self.pre_layers(x) 84 | out = self.a3(out) 85 | out = self.b3(out) 86 | out = self.maxpool(out) 87 | out = self.a4(out) 88 | out = self.b4(out) 89 | out = self.c4(out) 90 | out = self.d4(out) 91 | out = self.e4(out) 92 | out = self.maxpool(out) 93 | out = self.a5(out) 94 | out = self.b5(out) 95 | out = self.avgpool(out) 96 | out = out.view(out.size(0), -1) 97 | out = self.linear(out) 98 | return out 99 | 100 | 101 | def test(): 102 | net = GoogLeNet() 103 | x = torch.randn(1,3,32,32) 104 | y = net(x) 105 | print(y.size()) 106 | 107 | # test() 108 | -------------------------------------------------------------------------------- /Unlearning/models/googlenet.py: -------------------------------------------------------------------------------- 1 | '''GoogLeNet with PyTorch.''' 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class Inception(nn.Module): 8 | def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes): 9 | super(Inception, self).__init__() 10 | # 1x1 conv branch 11 | self.b1 = nn.Sequential( 12 | nn.Conv2d(in_planes, n1x1, kernel_size=1), 13 | nn.BatchNorm2d(n1x1), 14 | nn.ReLU(True), 15 | ) 16 | 17 | # 1x1 conv -> 3x3 conv branch 18 | self.b2 = nn.Sequential( 19 | nn.Conv2d(in_planes, n3x3red, kernel_size=1), 20 | nn.BatchNorm2d(n3x3red), 21 | nn.ReLU(True), 22 | nn.Conv2d(n3x3red, n3x3, kernel_size=3, padding=1), 23 | nn.BatchNorm2d(n3x3), 24 | nn.ReLU(True), 25 | ) 26 | 27 | # 1x1 conv -> 5x5 conv branch 28 | self.b3 = nn.Sequential( 29 | nn.Conv2d(in_planes, n5x5red, kernel_size=1), 30 | nn.BatchNorm2d(n5x5red), 31 | nn.ReLU(True), 32 | nn.Conv2d(n5x5red, n5x5, kernel_size=3, padding=1), 33 | nn.BatchNorm2d(n5x5), 34 | nn.ReLU(True), 35 | nn.Conv2d(n5x5, n5x5, kernel_size=3, padding=1), 36 | nn.BatchNorm2d(n5x5), 37 | nn.ReLU(True), 38 | ) 39 | 40 | # 3x3 pool -> 1x1 conv branch 41 | self.b4 = nn.Sequential( 42 | nn.MaxPool2d(3, stride=1, padding=1), 43 | nn.Conv2d(in_planes, pool_planes, kernel_size=1), 44 | nn.BatchNorm2d(pool_planes), 45 | nn.ReLU(True), 46 | ) 47 | 48 | def forward(self, x): 49 | y1 = self.b1(x) 50 | y2 = self.b2(x) 51 | y3 = self.b3(x) 52 | y4 = self.b4(x) 53 | return torch.cat([y1,y2,y3,y4], 1) 54 | 55 | 56 | class GoogLeNet(nn.Module): 57 | def __init__(self): 58 | super(GoogLeNet, self).__init__() 59 | self.pre_layers = nn.Sequential( 60 | nn.Conv2d(3, 192, kernel_size=3, padding=1), 61 | nn.BatchNorm2d(192), 62 | nn.ReLU(True), 63 | ) 64 | 65 | self.a3 = Inception(192, 64, 96, 128, 16, 32, 32) 66 | self.b3 = Inception(256, 128, 128, 192, 32, 96, 64) 67 | 68 | self.maxpool = nn.MaxPool2d(3, stride=2, padding=1) 69 | 70 | self.a4 = Inception(480, 192, 96, 208, 16, 48, 64) 71 | self.b4 = Inception(512, 160, 112, 224, 24, 64, 64) 72 | self.c4 = Inception(512, 128, 128, 256, 24, 64, 64) 73 | self.d4 = Inception(512, 112, 144, 288, 32, 64, 64) 74 | self.e4 = Inception(528, 256, 160, 320, 32, 128, 128) 75 | 76 | self.a5 = Inception(832, 256, 160, 320, 32, 128, 128) 77 | self.b5 = Inception(832, 384, 192, 384, 48, 128, 128) 78 | 79 | self.avgpool = nn.AvgPool2d(8, stride=1) 80 | self.linear = nn.Linear(1024, 10) 81 | 82 | def forward(self, x): 83 | out = self.pre_layers(x) 84 | out = self.a3(out) 85 | out = self.b3(out) 86 | out = self.maxpool(out) 87 | out = self.a4(out) 88 | out = self.b4(out) 89 | out = self.c4(out) 90 | out = self.d4(out) 91 | out = self.e4(out) 92 | out = self.maxpool(out) 93 | out = self.a5(out) 94 | out = self.b5(out) 95 | out = self.avgpool(out) 96 | out = out.view(out.size(0), -1) 97 | out = self.linear(out) 98 | return out 99 | 100 | 101 | def test(): 102 | net = GoogLeNet() 103 | x = torch.randn(1,3,32,32) 104 | y = net(x) 105 | print(y.size()) 106 | 107 | # test() 108 | -------------------------------------------------------------------------------- /Inversion/models/resnext.py: -------------------------------------------------------------------------------- 1 | '''ResNeXt in PyTorch. 2 | 3 | See the paper "Aggregated Residual Transformations for Deep Neural Networks" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class Block(nn.Module): 11 | '''Grouped convolution block.''' 12 | expansion = 2 13 | 14 | def __init__(self, in_planes, cardinality=32, bottleneck_width=4, stride=1): 15 | super(Block, self).__init__() 16 | group_width = cardinality * bottleneck_width 17 | self.conv1 = nn.Conv2d(in_planes, group_width, kernel_size=1, bias=False) 18 | self.bn1 = nn.BatchNorm2d(group_width) 19 | self.conv2 = nn.Conv2d(group_width, group_width, kernel_size=3, stride=stride, padding=1, groups=cardinality, bias=False) 20 | self.bn2 = nn.BatchNorm2d(group_width) 21 | self.conv3 = nn.Conv2d(group_width, self.expansion*group_width, kernel_size=1, bias=False) 22 | self.bn3 = nn.BatchNorm2d(self.expansion*group_width) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*group_width: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*group_width, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*group_width) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = F.relu(self.bn2(self.conv2(out))) 34 | out = self.bn3(self.conv3(out)) 35 | out += self.shortcut(x) 36 | out = F.relu(out) 37 | return out 38 | 39 | 40 | class ResNeXt(nn.Module): 41 | def __init__(self, num_blocks, cardinality, bottleneck_width, num_classes=10): 42 | super(ResNeXt, self).__init__() 43 | self.cardinality = cardinality 44 | self.bottleneck_width = bottleneck_width 45 | self.in_planes = 64 46 | 47 | self.conv1 = nn.Conv2d(3, 64, kernel_size=1, bias=False) 48 | self.bn1 = nn.BatchNorm2d(64) 49 | self.layer1 = self._make_layer(num_blocks[0], 1) 50 | self.layer2 = self._make_layer(num_blocks[1], 2) 51 | self.layer3 = self._make_layer(num_blocks[2], 2) 52 | # self.layer4 = self._make_layer(num_blocks[3], 2) 53 | self.linear = nn.Linear(cardinality*bottleneck_width*8, num_classes) 54 | 55 | def _make_layer(self, num_blocks, stride): 56 | strides = [stride] + [1]*(num_blocks-1) 57 | layers = [] 58 | for stride in strides: 59 | layers.append(Block(self.in_planes, self.cardinality, self.bottleneck_width, stride)) 60 | self.in_planes = Block.expansion * self.cardinality * self.bottleneck_width 61 | # Increase bottleneck_width by 2 after each stage. 62 | self.bottleneck_width *= 2 63 | return nn.Sequential(*layers) 64 | 65 | def forward(self, x): 66 | out = F.relu(self.bn1(self.conv1(x))) 67 | out = self.layer1(out) 68 | out = self.layer2(out) 69 | out = self.layer3(out) 70 | # out = self.layer4(out) 71 | out = F.avg_pool2d(out, 8) 72 | out = out.view(out.size(0), -1) 73 | out = self.linear(out) 74 | return out 75 | 76 | 77 | def ResNeXt29_2x64d(): 78 | return ResNeXt(num_blocks=[3,3,3], cardinality=2, bottleneck_width=64) 79 | 80 | def ResNeXt29_4x64d(): 81 | return ResNeXt(num_blocks=[3,3,3], cardinality=4, bottleneck_width=64) 82 | 83 | def ResNeXt29_8x64d(): 84 | return ResNeXt(num_blocks=[3,3,3], cardinality=8, bottleneck_width=64) 85 | 86 | def ResNeXt29_32x4d(): 87 | return ResNeXt(num_blocks=[3,3,3], cardinality=32, bottleneck_width=4) 88 | 89 | def test_resnext(): 90 | net = ResNeXt29_2x64d() 91 | x = torch.randn(1,3,32,32) 92 | y = net(x) 93 | print(y.size()) 94 | 95 | # test_resnext() 96 | -------------------------------------------------------------------------------- /Training/models/resnext.py: -------------------------------------------------------------------------------- 1 | '''ResNeXt in PyTorch. 2 | 3 | See the paper "Aggregated Residual Transformations for Deep Neural Networks" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class Block(nn.Module): 11 | '''Grouped convolution block.''' 12 | expansion = 2 13 | 14 | def __init__(self, in_planes, cardinality=32, bottleneck_width=4, stride=1): 15 | super(Block, self).__init__() 16 | group_width = cardinality * bottleneck_width 17 | self.conv1 = nn.Conv2d(in_planes, group_width, kernel_size=1, bias=False) 18 | self.bn1 = nn.BatchNorm2d(group_width) 19 | self.conv2 = nn.Conv2d(group_width, group_width, kernel_size=3, stride=stride, padding=1, groups=cardinality, bias=False) 20 | self.bn2 = nn.BatchNorm2d(group_width) 21 | self.conv3 = nn.Conv2d(group_width, self.expansion*group_width, kernel_size=1, bias=False) 22 | self.bn3 = nn.BatchNorm2d(self.expansion*group_width) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*group_width: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*group_width, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*group_width) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = F.relu(self.bn2(self.conv2(out))) 34 | out = self.bn3(self.conv3(out)) 35 | out += self.shortcut(x) 36 | out = F.relu(out) 37 | return out 38 | 39 | 40 | class ResNeXt(nn.Module): 41 | def __init__(self, num_blocks, cardinality, bottleneck_width, num_classes=10): 42 | super(ResNeXt, self).__init__() 43 | self.cardinality = cardinality 44 | self.bottleneck_width = bottleneck_width 45 | self.in_planes = 64 46 | 47 | self.conv1 = nn.Conv2d(3, 64, kernel_size=1, bias=False) 48 | self.bn1 = nn.BatchNorm2d(64) 49 | self.layer1 = self._make_layer(num_blocks[0], 1) 50 | self.layer2 = self._make_layer(num_blocks[1], 2) 51 | self.layer3 = self._make_layer(num_blocks[2], 2) 52 | # self.layer4 = self._make_layer(num_blocks[3], 2) 53 | self.linear = nn.Linear(cardinality*bottleneck_width*8, num_classes) 54 | 55 | def _make_layer(self, num_blocks, stride): 56 | strides = [stride] + [1]*(num_blocks-1) 57 | layers = [] 58 | for stride in strides: 59 | layers.append(Block(self.in_planes, self.cardinality, self.bottleneck_width, stride)) 60 | self.in_planes = Block.expansion * self.cardinality * self.bottleneck_width 61 | # Increase bottleneck_width by 2 after each stage. 62 | self.bottleneck_width *= 2 63 | return nn.Sequential(*layers) 64 | 65 | def forward(self, x): 66 | out = F.relu(self.bn1(self.conv1(x))) 67 | out = self.layer1(out) 68 | out = self.layer2(out) 69 | out = self.layer3(out) 70 | # out = self.layer4(out) 71 | out = F.avg_pool2d(out, 8) 72 | out = out.view(out.size(0), -1) 73 | out = self.linear(out) 74 | return out 75 | 76 | 77 | def ResNeXt29_2x64d(): 78 | return ResNeXt(num_blocks=[3,3,3], cardinality=2, bottleneck_width=64) 79 | 80 | def ResNeXt29_4x64d(): 81 | return ResNeXt(num_blocks=[3,3,3], cardinality=4, bottleneck_width=64) 82 | 83 | def ResNeXt29_8x64d(): 84 | return ResNeXt(num_blocks=[3,3,3], cardinality=8, bottleneck_width=64) 85 | 86 | def ResNeXt29_32x4d(): 87 | return ResNeXt(num_blocks=[3,3,3], cardinality=32, bottleneck_width=4) 88 | 89 | def test_resnext(): 90 | net = ResNeXt29_2x64d() 91 | x = torch.randn(1,3,32,32) 92 | y = net(x) 93 | print(y.size()) 94 | 95 | # test_resnext() 96 | -------------------------------------------------------------------------------- /Unlearning/models/resnext.py: -------------------------------------------------------------------------------- 1 | '''ResNeXt in PyTorch. 2 | 3 | See the paper "Aggregated Residual Transformations for Deep Neural Networks" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class Block(nn.Module): 11 | '''Grouped convolution block.''' 12 | expansion = 2 13 | 14 | def __init__(self, in_planes, cardinality=32, bottleneck_width=4, stride=1): 15 | super(Block, self).__init__() 16 | group_width = cardinality * bottleneck_width 17 | self.conv1 = nn.Conv2d(in_planes, group_width, kernel_size=1, bias=False) 18 | self.bn1 = nn.BatchNorm2d(group_width) 19 | self.conv2 = nn.Conv2d(group_width, group_width, kernel_size=3, stride=stride, padding=1, groups=cardinality, bias=False) 20 | self.bn2 = nn.BatchNorm2d(group_width) 21 | self.conv3 = nn.Conv2d(group_width, self.expansion*group_width, kernel_size=1, bias=False) 22 | self.bn3 = nn.BatchNorm2d(self.expansion*group_width) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*group_width: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*group_width, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*group_width) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = F.relu(self.bn2(self.conv2(out))) 34 | out = self.bn3(self.conv3(out)) 35 | out += self.shortcut(x) 36 | out = F.relu(out) 37 | return out 38 | 39 | 40 | class ResNeXt(nn.Module): 41 | def __init__(self, num_blocks, cardinality, bottleneck_width, num_classes=10): 42 | super(ResNeXt, self).__init__() 43 | self.cardinality = cardinality 44 | self.bottleneck_width = bottleneck_width 45 | self.in_planes = 64 46 | 47 | self.conv1 = nn.Conv2d(3, 64, kernel_size=1, bias=False) 48 | self.bn1 = nn.BatchNorm2d(64) 49 | self.layer1 = self._make_layer(num_blocks[0], 1) 50 | self.layer2 = self._make_layer(num_blocks[1], 2) 51 | self.layer3 = self._make_layer(num_blocks[2], 2) 52 | # self.layer4 = self._make_layer(num_blocks[3], 2) 53 | self.linear = nn.Linear(cardinality*bottleneck_width*8, num_classes) 54 | 55 | def _make_layer(self, num_blocks, stride): 56 | strides = [stride] + [1]*(num_blocks-1) 57 | layers = [] 58 | for stride in strides: 59 | layers.append(Block(self.in_planes, self.cardinality, self.bottleneck_width, stride)) 60 | self.in_planes = Block.expansion * self.cardinality * self.bottleneck_width 61 | # Increase bottleneck_width by 2 after each stage. 62 | self.bottleneck_width *= 2 63 | return nn.Sequential(*layers) 64 | 65 | def forward(self, x): 66 | out = F.relu(self.bn1(self.conv1(x))) 67 | out = self.layer1(out) 68 | out = self.layer2(out) 69 | out = self.layer3(out) 70 | # out = self.layer4(out) 71 | out = F.avg_pool2d(out, 8) 72 | out = out.view(out.size(0), -1) 73 | out = self.linear(out) 74 | return out 75 | 76 | 77 | def ResNeXt29_2x64d(): 78 | return ResNeXt(num_blocks=[3,3,3], cardinality=2, bottleneck_width=64) 79 | 80 | def ResNeXt29_4x64d(): 81 | return ResNeXt(num_blocks=[3,3,3], cardinality=4, bottleneck_width=64) 82 | 83 | def ResNeXt29_8x64d(): 84 | return ResNeXt(num_blocks=[3,3,3], cardinality=8, bottleneck_width=64) 85 | 86 | def ResNeXt29_32x4d(): 87 | return ResNeXt(num_blocks=[3,3,3], cardinality=32, bottleneck_width=4) 88 | 89 | def test_resnext(): 90 | net = ResNeXt29_2x64d() 91 | x = torch.randn(1,3,32,32) 92 | y = net(x) 93 | print(y.size()) 94 | 95 | # test_resnext() 96 | -------------------------------------------------------------------------------- /Inversion/models/dpn.py: -------------------------------------------------------------------------------- 1 | '''Dual Path Networks in PyTorch.''' 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class Bottleneck(nn.Module): 8 | def __init__(self, last_planes, in_planes, out_planes, dense_depth, stride, first_layer): 9 | super(Bottleneck, self).__init__() 10 | self.out_planes = out_planes 11 | self.dense_depth = dense_depth 12 | 13 | self.conv1 = nn.Conv2d(last_planes, in_planes, kernel_size=1, bias=False) 14 | self.bn1 = nn.BatchNorm2d(in_planes) 15 | self.conv2 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=32, bias=False) 16 | self.bn2 = nn.BatchNorm2d(in_planes) 17 | self.conv3 = nn.Conv2d(in_planes, out_planes+dense_depth, kernel_size=1, bias=False) 18 | self.bn3 = nn.BatchNorm2d(out_planes+dense_depth) 19 | 20 | self.shortcut = nn.Sequential() 21 | if first_layer: 22 | self.shortcut = nn.Sequential( 23 | nn.Conv2d(last_planes, out_planes+dense_depth, kernel_size=1, stride=stride, bias=False), 24 | nn.BatchNorm2d(out_planes+dense_depth) 25 | ) 26 | 27 | def forward(self, x): 28 | out = F.relu(self.bn1(self.conv1(x))) 29 | out = F.relu(self.bn2(self.conv2(out))) 30 | out = self.bn3(self.conv3(out)) 31 | x = self.shortcut(x) 32 | d = self.out_planes 33 | out = torch.cat([x[:,:d,:,:]+out[:,:d,:,:], x[:,d:,:,:], out[:,d:,:,:]], 1) 34 | out = F.relu(out) 35 | return out 36 | 37 | 38 | class DPN(nn.Module): 39 | def __init__(self, cfg): 40 | super(DPN, self).__init__() 41 | in_planes, out_planes = cfg['in_planes'], cfg['out_planes'] 42 | num_blocks, dense_depth = cfg['num_blocks'], cfg['dense_depth'] 43 | 44 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(64) 46 | self.last_planes = 64 47 | self.layer1 = self._make_layer(in_planes[0], out_planes[0], num_blocks[0], dense_depth[0], stride=1) 48 | self.layer2 = self._make_layer(in_planes[1], out_planes[1], num_blocks[1], dense_depth[1], stride=2) 49 | self.layer3 = self._make_layer(in_planes[2], out_planes[2], num_blocks[2], dense_depth[2], stride=2) 50 | self.layer4 = self._make_layer(in_planes[3], out_planes[3], num_blocks[3], dense_depth[3], stride=2) 51 | self.linear = nn.Linear(out_planes[3]+(num_blocks[3]+1)*dense_depth[3], 10) 52 | 53 | def _make_layer(self, in_planes, out_planes, num_blocks, dense_depth, stride): 54 | strides = [stride] + [1]*(num_blocks-1) 55 | layers = [] 56 | for i,stride in enumerate(strides): 57 | layers.append(Bottleneck(self.last_planes, in_planes, out_planes, dense_depth, stride, i==0)) 58 | self.last_planes = out_planes + (i+2) * dense_depth 59 | return nn.Sequential(*layers) 60 | 61 | def forward(self, x): 62 | out = F.relu(self.bn1(self.conv1(x))) 63 | out = self.layer1(out) 64 | out = self.layer2(out) 65 | out = self.layer3(out) 66 | out = self.layer4(out) 67 | out = F.avg_pool2d(out, 4) 68 | out = out.view(out.size(0), -1) 69 | out = self.linear(out) 70 | return out 71 | 72 | 73 | def DPN26(): 74 | cfg = { 75 | 'in_planes': (96,192,384,768), 76 | 'out_planes': (256,512,1024,2048), 77 | 'num_blocks': (2,2,2,2), 78 | 'dense_depth': (16,32,24,128) 79 | } 80 | return DPN(cfg) 81 | 82 | def DPN92(): 83 | cfg = { 84 | 'in_planes': (96,192,384,768), 85 | 'out_planes': (256,512,1024,2048), 86 | 'num_blocks': (3,4,20,3), 87 | 'dense_depth': (16,32,24,128) 88 | } 89 | return DPN(cfg) 90 | 91 | 92 | def test(): 93 | net = DPN92() 94 | x = torch.randn(1,3,32,32) 95 | y = net(x) 96 | print(y) 97 | 98 | # test() 99 | -------------------------------------------------------------------------------- /Training/models/dpn.py: -------------------------------------------------------------------------------- 1 | '''Dual Path Networks in PyTorch.''' 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class Bottleneck(nn.Module): 8 | def __init__(self, last_planes, in_planes, out_planes, dense_depth, stride, first_layer): 9 | super(Bottleneck, self).__init__() 10 | self.out_planes = out_planes 11 | self.dense_depth = dense_depth 12 | 13 | self.conv1 = nn.Conv2d(last_planes, in_planes, kernel_size=1, bias=False) 14 | self.bn1 = nn.BatchNorm2d(in_planes) 15 | self.conv2 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=32, bias=False) 16 | self.bn2 = nn.BatchNorm2d(in_planes) 17 | self.conv3 = nn.Conv2d(in_planes, out_planes+dense_depth, kernel_size=1, bias=False) 18 | self.bn3 = nn.BatchNorm2d(out_planes+dense_depth) 19 | 20 | self.shortcut = nn.Sequential() 21 | if first_layer: 22 | self.shortcut = nn.Sequential( 23 | nn.Conv2d(last_planes, out_planes+dense_depth, kernel_size=1, stride=stride, bias=False), 24 | nn.BatchNorm2d(out_planes+dense_depth) 25 | ) 26 | 27 | def forward(self, x): 28 | out = F.relu(self.bn1(self.conv1(x))) 29 | out = F.relu(self.bn2(self.conv2(out))) 30 | out = self.bn3(self.conv3(out)) 31 | x = self.shortcut(x) 32 | d = self.out_planes 33 | out = torch.cat([x[:,:d,:,:]+out[:,:d,:,:], x[:,d:,:,:], out[:,d:,:,:]], 1) 34 | out = F.relu(out) 35 | return out 36 | 37 | 38 | class DPN(nn.Module): 39 | def __init__(self, cfg): 40 | super(DPN, self).__init__() 41 | in_planes, out_planes = cfg['in_planes'], cfg['out_planes'] 42 | num_blocks, dense_depth = cfg['num_blocks'], cfg['dense_depth'] 43 | 44 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(64) 46 | self.last_planes = 64 47 | self.layer1 = self._make_layer(in_planes[0], out_planes[0], num_blocks[0], dense_depth[0], stride=1) 48 | self.layer2 = self._make_layer(in_planes[1], out_planes[1], num_blocks[1], dense_depth[1], stride=2) 49 | self.layer3 = self._make_layer(in_planes[2], out_planes[2], num_blocks[2], dense_depth[2], stride=2) 50 | self.layer4 = self._make_layer(in_planes[3], out_planes[3], num_blocks[3], dense_depth[3], stride=2) 51 | self.linear = nn.Linear(out_planes[3]+(num_blocks[3]+1)*dense_depth[3], 10) 52 | 53 | def _make_layer(self, in_planes, out_planes, num_blocks, dense_depth, stride): 54 | strides = [stride] + [1]*(num_blocks-1) 55 | layers = [] 56 | for i,stride in enumerate(strides): 57 | layers.append(Bottleneck(self.last_planes, in_planes, out_planes, dense_depth, stride, i==0)) 58 | self.last_planes = out_planes + (i+2) * dense_depth 59 | return nn.Sequential(*layers) 60 | 61 | def forward(self, x): 62 | out = F.relu(self.bn1(self.conv1(x))) 63 | out = self.layer1(out) 64 | out = self.layer2(out) 65 | out = self.layer3(out) 66 | out = self.layer4(out) 67 | out = F.avg_pool2d(out, 4) 68 | out = out.view(out.size(0), -1) 69 | out = self.linear(out) 70 | return out 71 | 72 | 73 | def DPN26(): 74 | cfg = { 75 | 'in_planes': (96,192,384,768), 76 | 'out_planes': (256,512,1024,2048), 77 | 'num_blocks': (2,2,2,2), 78 | 'dense_depth': (16,32,24,128) 79 | } 80 | return DPN(cfg) 81 | 82 | def DPN92(): 83 | cfg = { 84 | 'in_planes': (96,192,384,768), 85 | 'out_planes': (256,512,1024,2048), 86 | 'num_blocks': (3,4,20,3), 87 | 'dense_depth': (16,32,24,128) 88 | } 89 | return DPN(cfg) 90 | 91 | 92 | def test(): 93 | net = DPN92() 94 | x = torch.randn(1,3,32,32) 95 | y = net(x) 96 | print(y) 97 | 98 | # test() 99 | -------------------------------------------------------------------------------- /Unlearning/models/dpn.py: -------------------------------------------------------------------------------- 1 | '''Dual Path Networks in PyTorch.''' 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class Bottleneck(nn.Module): 8 | def __init__(self, last_planes, in_planes, out_planes, dense_depth, stride, first_layer): 9 | super(Bottleneck, self).__init__() 10 | self.out_planes = out_planes 11 | self.dense_depth = dense_depth 12 | 13 | self.conv1 = nn.Conv2d(last_planes, in_planes, kernel_size=1, bias=False) 14 | self.bn1 = nn.BatchNorm2d(in_planes) 15 | self.conv2 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=32, bias=False) 16 | self.bn2 = nn.BatchNorm2d(in_planes) 17 | self.conv3 = nn.Conv2d(in_planes, out_planes+dense_depth, kernel_size=1, bias=False) 18 | self.bn3 = nn.BatchNorm2d(out_planes+dense_depth) 19 | 20 | self.shortcut = nn.Sequential() 21 | if first_layer: 22 | self.shortcut = nn.Sequential( 23 | nn.Conv2d(last_planes, out_planes+dense_depth, kernel_size=1, stride=stride, bias=False), 24 | nn.BatchNorm2d(out_planes+dense_depth) 25 | ) 26 | 27 | def forward(self, x): 28 | out = F.relu(self.bn1(self.conv1(x))) 29 | out = F.relu(self.bn2(self.conv2(out))) 30 | out = self.bn3(self.conv3(out)) 31 | x = self.shortcut(x) 32 | d = self.out_planes 33 | out = torch.cat([x[:,:d,:,:]+out[:,:d,:,:], x[:,d:,:,:], out[:,d:,:,:]], 1) 34 | out = F.relu(out) 35 | return out 36 | 37 | 38 | class DPN(nn.Module): 39 | def __init__(self, cfg): 40 | super(DPN, self).__init__() 41 | in_planes, out_planes = cfg['in_planes'], cfg['out_planes'] 42 | num_blocks, dense_depth = cfg['num_blocks'], cfg['dense_depth'] 43 | 44 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(64) 46 | self.last_planes = 64 47 | self.layer1 = self._make_layer(in_planes[0], out_planes[0], num_blocks[0], dense_depth[0], stride=1) 48 | self.layer2 = self._make_layer(in_planes[1], out_planes[1], num_blocks[1], dense_depth[1], stride=2) 49 | self.layer3 = self._make_layer(in_planes[2], out_planes[2], num_blocks[2], dense_depth[2], stride=2) 50 | self.layer4 = self._make_layer(in_planes[3], out_planes[3], num_blocks[3], dense_depth[3], stride=2) 51 | self.linear = nn.Linear(out_planes[3]+(num_blocks[3]+1)*dense_depth[3], 10) 52 | 53 | def _make_layer(self, in_planes, out_planes, num_blocks, dense_depth, stride): 54 | strides = [stride] + [1]*(num_blocks-1) 55 | layers = [] 56 | for i,stride in enumerate(strides): 57 | layers.append(Bottleneck(self.last_planes, in_planes, out_planes, dense_depth, stride, i==0)) 58 | self.last_planes = out_planes + (i+2) * dense_depth 59 | return nn.Sequential(*layers) 60 | 61 | def forward(self, x): 62 | out = F.relu(self.bn1(self.conv1(x))) 63 | out = self.layer1(out) 64 | out = self.layer2(out) 65 | out = self.layer3(out) 66 | out = self.layer4(out) 67 | out = F.avg_pool2d(out, 4) 68 | out = out.view(out.size(0), -1) 69 | out = self.linear(out) 70 | return out 71 | 72 | 73 | def DPN26(): 74 | cfg = { 75 | 'in_planes': (96,192,384,768), 76 | 'out_planes': (256,512,1024,2048), 77 | 'num_blocks': (2,2,2,2), 78 | 'dense_depth': (16,32,24,128) 79 | } 80 | return DPN(cfg) 81 | 82 | def DPN92(): 83 | cfg = { 84 | 'in_planes': (96,192,384,768), 85 | 'out_planes': (256,512,1024,2048), 86 | 'num_blocks': (3,4,20,3), 87 | 'dense_depth': (16,32,24,128) 88 | } 89 | return DPN(cfg) 90 | 91 | 92 | def test(): 93 | net = DPN92() 94 | x = torch.randn(1,3,32,32) 95 | y = net(x) 96 | print(y) 97 | 98 | # test() 99 | -------------------------------------------------------------------------------- /Training/models/densenet.py: -------------------------------------------------------------------------------- 1 | '''DenseNet in PyTorch.''' 2 | import math 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | 9 | class Bottleneck(nn.Module): 10 | def __init__(self, in_planes, growth_rate): 11 | super(Bottleneck, self).__init__() 12 | self.bn1 = nn.BatchNorm2d(in_planes) 13 | self.conv1 = nn.Conv2d(in_planes, 4*growth_rate, kernel_size=1, bias=False) 14 | self.bn2 = nn.BatchNorm2d(4*growth_rate) 15 | self.conv2 = nn.Conv2d(4*growth_rate, growth_rate, kernel_size=3, padding=1, bias=False) 16 | 17 | def forward(self, x): 18 | out = self.conv1(F.relu(self.bn1(x))) 19 | out = self.conv2(F.relu(self.bn2(out))) 20 | out = torch.cat([out,x], 1) 21 | return out 22 | 23 | 24 | class Transition(nn.Module): 25 | def __init__(self, in_planes, out_planes): 26 | super(Transition, self).__init__() 27 | self.bn = nn.BatchNorm2d(in_planes) 28 | self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, bias=False) 29 | 30 | def forward(self, x): 31 | out = self.conv(F.relu(self.bn(x))) 32 | out = F.avg_pool2d(out, 2) 33 | return out 34 | 35 | 36 | class DenseNet(nn.Module): 37 | def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_classes=10): 38 | super(DenseNet, self).__init__() 39 | self.growth_rate = growth_rate 40 | 41 | num_planes = 2*growth_rate 42 | self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, padding=1, bias=False) 43 | 44 | self.dense1 = self._make_dense_layers(block, num_planes, nblocks[0]) 45 | num_planes += nblocks[0]*growth_rate 46 | out_planes = int(math.floor(num_planes*reduction)) 47 | self.trans1 = Transition(num_planes, out_planes) 48 | num_planes = out_planes 49 | 50 | self.dense2 = self._make_dense_layers(block, num_planes, nblocks[1]) 51 | num_planes += nblocks[1]*growth_rate 52 | out_planes = int(math.floor(num_planes*reduction)) 53 | self.trans2 = Transition(num_planes, out_planes) 54 | num_planes = out_planes 55 | 56 | self.dense3 = self._make_dense_layers(block, num_planes, nblocks[2]) 57 | num_planes += nblocks[2]*growth_rate 58 | out_planes = int(math.floor(num_planes*reduction)) 59 | self.trans3 = Transition(num_planes, out_planes) 60 | num_planes = out_planes 61 | 62 | self.dense4 = self._make_dense_layers(block, num_planes, nblocks[3]) 63 | num_planes += nblocks[3]*growth_rate 64 | 65 | self.bn = nn.BatchNorm2d(num_planes) 66 | self.linear = nn.Linear(num_planes, num_classes) 67 | 68 | def _make_dense_layers(self, block, in_planes, nblock): 69 | layers = [] 70 | for i in range(nblock): 71 | layers.append(block(in_planes, self.growth_rate)) 72 | in_planes += self.growth_rate 73 | return nn.Sequential(*layers) 74 | 75 | def forward(self, x): 76 | out = self.conv1(x) 77 | out = self.trans1(self.dense1(out)) 78 | out = self.trans2(self.dense2(out)) 79 | out = self.trans3(self.dense3(out)) 80 | out = self.dense4(out) 81 | out = F.avg_pool2d(F.relu(self.bn(out)), 4) 82 | out = out.view(out.size(0), -1) 83 | out = self.linear(out) 84 | return out 85 | 86 | def DenseNet121(): 87 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=32) 88 | 89 | def DenseNet169(): 90 | return DenseNet(Bottleneck, [6,12,32,32], growth_rate=32) 91 | 92 | def DenseNet201(): 93 | return DenseNet(Bottleneck, [6,12,48,32], growth_rate=32) 94 | 95 | def DenseNet161(): 96 | return DenseNet(Bottleneck, [6,12,36,24], growth_rate=48) 97 | 98 | def densenet_cifar(): 99 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=12) 100 | 101 | def test(): 102 | net = densenet_cifar() 103 | x = torch.randn(1,3,32,32) 104 | y = net(x) 105 | print(y) 106 | 107 | # test() 108 | -------------------------------------------------------------------------------- /Inversion/models/densenet.py: -------------------------------------------------------------------------------- 1 | '''DenseNet in PyTorch.''' 2 | import math 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | 9 | class Bottleneck(nn.Module): 10 | def __init__(self, in_planes, growth_rate): 11 | super(Bottleneck, self).__init__() 12 | self.bn1 = nn.BatchNorm2d(in_planes) 13 | self.conv1 = nn.Conv2d(in_planes, 4*growth_rate, kernel_size=1, bias=False) 14 | self.bn2 = nn.BatchNorm2d(4*growth_rate) 15 | self.conv2 = nn.Conv2d(4*growth_rate, growth_rate, kernel_size=3, padding=1, bias=False) 16 | 17 | def forward(self, x): 18 | out = self.conv1(F.relu(self.bn1(x))) 19 | out = self.conv2(F.relu(self.bn2(out))) 20 | out = torch.cat([out,x], 1) 21 | return out 22 | 23 | 24 | class Transition(nn.Module): 25 | def __init__(self, in_planes, out_planes): 26 | super(Transition, self).__init__() 27 | self.bn = nn.BatchNorm2d(in_planes) 28 | self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, bias=False) 29 | 30 | def forward(self, x): 31 | out = self.conv(F.relu(self.bn(x))) 32 | out = F.avg_pool2d(out, 2) 33 | return out 34 | 35 | 36 | class DenseNet(nn.Module): 37 | def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_classes=10): 38 | super(DenseNet, self).__init__() 39 | self.growth_rate = growth_rate 40 | 41 | num_planes = 2*growth_rate 42 | self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, padding=1, bias=False) 43 | 44 | self.dense1 = self._make_dense_layers(block, num_planes, nblocks[0]) 45 | num_planes += nblocks[0]*growth_rate 46 | out_planes = int(math.floor(num_planes*reduction)) 47 | self.trans1 = Transition(num_planes, out_planes) 48 | num_planes = out_planes 49 | 50 | self.dense2 = self._make_dense_layers(block, num_planes, nblocks[1]) 51 | num_planes += nblocks[1]*growth_rate 52 | out_planes = int(math.floor(num_planes*reduction)) 53 | self.trans2 = Transition(num_planes, out_planes) 54 | num_planes = out_planes 55 | 56 | self.dense3 = self._make_dense_layers(block, num_planes, nblocks[2]) 57 | num_planes += nblocks[2]*growth_rate 58 | out_planes = int(math.floor(num_planes*reduction)) 59 | self.trans3 = Transition(num_planes, out_planes) 60 | num_planes = out_planes 61 | 62 | self.dense4 = self._make_dense_layers(block, num_planes, nblocks[3]) 63 | num_planes += nblocks[3]*growth_rate 64 | 65 | self.bn = nn.BatchNorm2d(num_planes) 66 | self.linear = nn.Linear(num_planes, num_classes) 67 | 68 | def _make_dense_layers(self, block, in_planes, nblock): 69 | layers = [] 70 | for i in range(nblock): 71 | layers.append(block(in_planes, self.growth_rate)) 72 | in_planes += self.growth_rate 73 | return nn.Sequential(*layers) 74 | 75 | def forward(self, x): 76 | out = self.conv1(x) 77 | out = self.trans1(self.dense1(out)) 78 | out = self.trans2(self.dense2(out)) 79 | out = self.trans3(self.dense3(out)) 80 | out = self.dense4(out) 81 | out = F.avg_pool2d(F.relu(self.bn(out)), 4) 82 | out = out.view(out.size(0), -1) 83 | out = self.linear(out) 84 | return out 85 | 86 | def DenseNet121(): 87 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=32) 88 | 89 | def DenseNet169(): 90 | return DenseNet(Bottleneck, [6,12,32,32], growth_rate=32) 91 | 92 | def DenseNet201(): 93 | return DenseNet(Bottleneck, [6,12,48,32], growth_rate=32) 94 | 95 | def DenseNet161(): 96 | return DenseNet(Bottleneck, [6,12,36,24], growth_rate=48) 97 | 98 | def densenet_cifar(): 99 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=12) 100 | 101 | def test(): 102 | net = densenet_cifar() 103 | x = torch.randn(1,3,32,32) 104 | y = net(x) 105 | print(y) 106 | 107 | # test() 108 | -------------------------------------------------------------------------------- /Unlearning/models/densenet.py: -------------------------------------------------------------------------------- 1 | '''DenseNet in PyTorch.''' 2 | import math 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | 9 | class Bottleneck(nn.Module): 10 | def __init__(self, in_planes, growth_rate): 11 | super(Bottleneck, self).__init__() 12 | self.bn1 = nn.BatchNorm2d(in_planes) 13 | self.conv1 = nn.Conv2d(in_planes, 4*growth_rate, kernel_size=1, bias=False) 14 | self.bn2 = nn.BatchNorm2d(4*growth_rate) 15 | self.conv2 = nn.Conv2d(4*growth_rate, growth_rate, kernel_size=3, padding=1, bias=False) 16 | 17 | def forward(self, x): 18 | out = self.conv1(F.relu(self.bn1(x))) 19 | out = self.conv2(F.relu(self.bn2(out))) 20 | out = torch.cat([out,x], 1) 21 | return out 22 | 23 | 24 | class Transition(nn.Module): 25 | def __init__(self, in_planes, out_planes): 26 | super(Transition, self).__init__() 27 | self.bn = nn.BatchNorm2d(in_planes) 28 | self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, bias=False) 29 | 30 | def forward(self, x): 31 | out = self.conv(F.relu(self.bn(x))) 32 | out = F.avg_pool2d(out, 2) 33 | return out 34 | 35 | 36 | class DenseNet(nn.Module): 37 | def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_classes=10): 38 | super(DenseNet, self).__init__() 39 | self.growth_rate = growth_rate 40 | 41 | num_planes = 2*growth_rate 42 | self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, padding=1, bias=False) 43 | 44 | self.dense1 = self._make_dense_layers(block, num_planes, nblocks[0]) 45 | num_planes += nblocks[0]*growth_rate 46 | out_planes = int(math.floor(num_planes*reduction)) 47 | self.trans1 = Transition(num_planes, out_planes) 48 | num_planes = out_planes 49 | 50 | self.dense2 = self._make_dense_layers(block, num_planes, nblocks[1]) 51 | num_planes += nblocks[1]*growth_rate 52 | out_planes = int(math.floor(num_planes*reduction)) 53 | self.trans2 = Transition(num_planes, out_planes) 54 | num_planes = out_planes 55 | 56 | self.dense3 = self._make_dense_layers(block, num_planes, nblocks[2]) 57 | num_planes += nblocks[2]*growth_rate 58 | out_planes = int(math.floor(num_planes*reduction)) 59 | self.trans3 = Transition(num_planes, out_planes) 60 | num_planes = out_planes 61 | 62 | self.dense4 = self._make_dense_layers(block, num_planes, nblocks[3]) 63 | num_planes += nblocks[3]*growth_rate 64 | 65 | self.bn = nn.BatchNorm2d(num_planes) 66 | self.linear = nn.Linear(num_planes, num_classes) 67 | 68 | def _make_dense_layers(self, block, in_planes, nblock): 69 | layers = [] 70 | for i in range(nblock): 71 | layers.append(block(in_planes, self.growth_rate)) 72 | in_planes += self.growth_rate 73 | return nn.Sequential(*layers) 74 | 75 | def forward(self, x): 76 | out = self.conv1(x) 77 | out = self.trans1(self.dense1(out)) 78 | out = self.trans2(self.dense2(out)) 79 | out = self.trans3(self.dense3(out)) 80 | out = self.dense4(out) 81 | out = F.avg_pool2d(F.relu(self.bn(out)), 4) 82 | out = out.view(out.size(0), -1) 83 | out = self.linear(out) 84 | return out 85 | 86 | def DenseNet121(): 87 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=32) 88 | 89 | def DenseNet169(): 90 | return DenseNet(Bottleneck, [6,12,32,32], growth_rate=32) 91 | 92 | def DenseNet201(): 93 | return DenseNet(Bottleneck, [6,12,48,32], growth_rate=32) 94 | 95 | def DenseNet161(): 96 | return DenseNet(Bottleneck, [6,12,36,24], growth_rate=48) 97 | 98 | def densenet_cifar(): 99 | return DenseNet(Bottleneck, [6,12,24,16], growth_rate=12) 100 | 101 | def test(): 102 | net = densenet_cifar() 103 | x = torch.randn(1,3,32,32) 104 | y = net(x) 105 | print(y) 106 | 107 | # test() 108 | -------------------------------------------------------------------------------- /Inversion/models/shufflenet.py: -------------------------------------------------------------------------------- 1 | '''ShuffleNet in PyTorch. 2 | 3 | See the paper "ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class ShuffleBlock(nn.Module): 11 | def __init__(self, groups): 12 | super(ShuffleBlock, self).__init__() 13 | self.groups = groups 14 | 15 | def forward(self, x): 16 | '''Channel shuffle: [N,C,H,W] -> [N,g,C/g,H,W] -> [N,C/g,g,H,w] -> [N,C,H,W]''' 17 | N,C,H,W = x.size() 18 | g = self.groups 19 | return x.view(N,g,C//g,H,W).permute(0,2,1,3,4).reshape(N,C,H,W) 20 | 21 | 22 | class Bottleneck(nn.Module): 23 | def __init__(self, in_planes, out_planes, stride, groups): 24 | super(Bottleneck, self).__init__() 25 | self.stride = stride 26 | 27 | mid_planes = out_planes/4 28 | g = 1 if in_planes==24 else groups 29 | self.conv1 = nn.Conv2d(in_planes, mid_planes, kernel_size=1, groups=g, bias=False) 30 | self.bn1 = nn.BatchNorm2d(mid_planes) 31 | self.shuffle1 = ShuffleBlock(groups=g) 32 | self.conv2 = nn.Conv2d(mid_planes, mid_planes, kernel_size=3, stride=stride, padding=1, groups=mid_planes, bias=False) 33 | self.bn2 = nn.BatchNorm2d(mid_planes) 34 | self.conv3 = nn.Conv2d(mid_planes, out_planes, kernel_size=1, groups=groups, bias=False) 35 | self.bn3 = nn.BatchNorm2d(out_planes) 36 | 37 | self.shortcut = nn.Sequential() 38 | if stride == 2: 39 | self.shortcut = nn.Sequential(nn.AvgPool2d(3, stride=2, padding=1)) 40 | 41 | def forward(self, x): 42 | out = F.relu(self.bn1(self.conv1(x))) 43 | out = self.shuffle1(out) 44 | out = F.relu(self.bn2(self.conv2(out))) 45 | out = self.bn3(self.conv3(out)) 46 | res = self.shortcut(x) 47 | out = F.relu(torch.cat([out,res], 1)) if self.stride==2 else F.relu(out+res) 48 | return out 49 | 50 | 51 | class ShuffleNet(nn.Module): 52 | def __init__(self, cfg): 53 | super(ShuffleNet, self).__init__() 54 | out_planes = cfg['out_planes'] 55 | num_blocks = cfg['num_blocks'] 56 | groups = cfg['groups'] 57 | 58 | self.conv1 = nn.Conv2d(3, 24, kernel_size=1, bias=False) 59 | self.bn1 = nn.BatchNorm2d(24) 60 | self.in_planes = 24 61 | self.layer1 = self._make_layer(out_planes[0], num_blocks[0], groups) 62 | self.layer2 = self._make_layer(out_planes[1], num_blocks[1], groups) 63 | self.layer3 = self._make_layer(out_planes[2], num_blocks[2], groups) 64 | self.linear = nn.Linear(out_planes[2], 10) 65 | 66 | def _make_layer(self, out_planes, num_blocks, groups): 67 | layers = [] 68 | for i in range(num_blocks): 69 | stride = 2 if i == 0 else 1 70 | cat_planes = self.in_planes if i == 0 else 0 71 | layers.append(Bottleneck(self.in_planes, out_planes-cat_planes, stride=stride, groups=groups)) 72 | self.in_planes = out_planes 73 | return nn.Sequential(*layers) 74 | 75 | def forward(self, x): 76 | out = F.relu(self.bn1(self.conv1(x))) 77 | out = self.layer1(out) 78 | out = self.layer2(out) 79 | out = self.layer3(out) 80 | out = F.avg_pool2d(out, 4) 81 | out = out.view(out.size(0), -1) 82 | out = self.linear(out) 83 | return out 84 | 85 | 86 | def ShuffleNetG2(): 87 | cfg = { 88 | 'out_planes': [200,400,800], 89 | 'num_blocks': [4,8,4], 90 | 'groups': 2 91 | } 92 | return ShuffleNet(cfg) 93 | 94 | def ShuffleNetG3(): 95 | cfg = { 96 | 'out_planes': [240,480,960], 97 | 'num_blocks': [4,8,4], 98 | 'groups': 3 99 | } 100 | return ShuffleNet(cfg) 101 | 102 | 103 | def test(): 104 | net = ShuffleNetG2() 105 | x = torch.randn(1,3,32,32) 106 | y = net(x) 107 | print(y) 108 | 109 | # test() 110 | -------------------------------------------------------------------------------- /Training/models/shufflenet.py: -------------------------------------------------------------------------------- 1 | '''ShuffleNet in PyTorch. 2 | 3 | See the paper "ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class ShuffleBlock(nn.Module): 11 | def __init__(self, groups): 12 | super(ShuffleBlock, self).__init__() 13 | self.groups = groups 14 | 15 | def forward(self, x): 16 | '''Channel shuffle: [N,C,H,W] -> [N,g,C/g,H,W] -> [N,C/g,g,H,w] -> [N,C,H,W]''' 17 | N,C,H,W = x.size() 18 | g = self.groups 19 | return x.view(N,g,C//g,H,W).permute(0,2,1,3,4).reshape(N,C,H,W) 20 | 21 | 22 | class Bottleneck(nn.Module): 23 | def __init__(self, in_planes, out_planes, stride, groups): 24 | super(Bottleneck, self).__init__() 25 | self.stride = stride 26 | 27 | mid_planes = out_planes/4 28 | g = 1 if in_planes==24 else groups 29 | self.conv1 = nn.Conv2d(in_planes, mid_planes, kernel_size=1, groups=g, bias=False) 30 | self.bn1 = nn.BatchNorm2d(mid_planes) 31 | self.shuffle1 = ShuffleBlock(groups=g) 32 | self.conv2 = nn.Conv2d(mid_planes, mid_planes, kernel_size=3, stride=stride, padding=1, groups=mid_planes, bias=False) 33 | self.bn2 = nn.BatchNorm2d(mid_planes) 34 | self.conv3 = nn.Conv2d(mid_planes, out_planes, kernel_size=1, groups=groups, bias=False) 35 | self.bn3 = nn.BatchNorm2d(out_planes) 36 | 37 | self.shortcut = nn.Sequential() 38 | if stride == 2: 39 | self.shortcut = nn.Sequential(nn.AvgPool2d(3, stride=2, padding=1)) 40 | 41 | def forward(self, x): 42 | out = F.relu(self.bn1(self.conv1(x))) 43 | out = self.shuffle1(out) 44 | out = F.relu(self.bn2(self.conv2(out))) 45 | out = self.bn3(self.conv3(out)) 46 | res = self.shortcut(x) 47 | out = F.relu(torch.cat([out,res], 1)) if self.stride==2 else F.relu(out+res) 48 | return out 49 | 50 | 51 | class ShuffleNet(nn.Module): 52 | def __init__(self, cfg): 53 | super(ShuffleNet, self).__init__() 54 | out_planes = cfg['out_planes'] 55 | num_blocks = cfg['num_blocks'] 56 | groups = cfg['groups'] 57 | 58 | self.conv1 = nn.Conv2d(3, 24, kernel_size=1, bias=False) 59 | self.bn1 = nn.BatchNorm2d(24) 60 | self.in_planes = 24 61 | self.layer1 = self._make_layer(out_planes[0], num_blocks[0], groups) 62 | self.layer2 = self._make_layer(out_planes[1], num_blocks[1], groups) 63 | self.layer3 = self._make_layer(out_planes[2], num_blocks[2], groups) 64 | self.linear = nn.Linear(out_planes[2], 10) 65 | 66 | def _make_layer(self, out_planes, num_blocks, groups): 67 | layers = [] 68 | for i in range(num_blocks): 69 | stride = 2 if i == 0 else 1 70 | cat_planes = self.in_planes if i == 0 else 0 71 | layers.append(Bottleneck(self.in_planes, out_planes-cat_planes, stride=stride, groups=groups)) 72 | self.in_planes = out_planes 73 | return nn.Sequential(*layers) 74 | 75 | def forward(self, x): 76 | out = F.relu(self.bn1(self.conv1(x))) 77 | out = self.layer1(out) 78 | out = self.layer2(out) 79 | out = self.layer3(out) 80 | out = F.avg_pool2d(out, 4) 81 | out = out.view(out.size(0), -1) 82 | out = self.linear(out) 83 | return out 84 | 85 | 86 | def ShuffleNetG2(): 87 | cfg = { 88 | 'out_planes': [200,400,800], 89 | 'num_blocks': [4,8,4], 90 | 'groups': 2 91 | } 92 | return ShuffleNet(cfg) 93 | 94 | def ShuffleNetG3(): 95 | cfg = { 96 | 'out_planes': [240,480,960], 97 | 'num_blocks': [4,8,4], 98 | 'groups': 3 99 | } 100 | return ShuffleNet(cfg) 101 | 102 | 103 | def test(): 104 | net = ShuffleNetG2() 105 | x = torch.randn(1,3,32,32) 106 | y = net(x) 107 | print(y) 108 | 109 | # test() 110 | -------------------------------------------------------------------------------- /Unlearning/models/shufflenet.py: -------------------------------------------------------------------------------- 1 | '''ShuffleNet in PyTorch. 2 | 3 | See the paper "ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class ShuffleBlock(nn.Module): 11 | def __init__(self, groups): 12 | super(ShuffleBlock, self).__init__() 13 | self.groups = groups 14 | 15 | def forward(self, x): 16 | '''Channel shuffle: [N,C,H,W] -> [N,g,C/g,H,W] -> [N,C/g,g,H,w] -> [N,C,H,W]''' 17 | N,C,H,W = x.size() 18 | g = self.groups 19 | return x.view(N,g,C//g,H,W).permute(0,2,1,3,4).reshape(N,C,H,W) 20 | 21 | 22 | class Bottleneck(nn.Module): 23 | def __init__(self, in_planes, out_planes, stride, groups): 24 | super(Bottleneck, self).__init__() 25 | self.stride = stride 26 | 27 | mid_planes = out_planes/4 28 | g = 1 if in_planes==24 else groups 29 | self.conv1 = nn.Conv2d(in_planes, mid_planes, kernel_size=1, groups=g, bias=False) 30 | self.bn1 = nn.BatchNorm2d(mid_planes) 31 | self.shuffle1 = ShuffleBlock(groups=g) 32 | self.conv2 = nn.Conv2d(mid_planes, mid_planes, kernel_size=3, stride=stride, padding=1, groups=mid_planes, bias=False) 33 | self.bn2 = nn.BatchNorm2d(mid_planes) 34 | self.conv3 = nn.Conv2d(mid_planes, out_planes, kernel_size=1, groups=groups, bias=False) 35 | self.bn3 = nn.BatchNorm2d(out_planes) 36 | 37 | self.shortcut = nn.Sequential() 38 | if stride == 2: 39 | self.shortcut = nn.Sequential(nn.AvgPool2d(3, stride=2, padding=1)) 40 | 41 | def forward(self, x): 42 | out = F.relu(self.bn1(self.conv1(x))) 43 | out = self.shuffle1(out) 44 | out = F.relu(self.bn2(self.conv2(out))) 45 | out = self.bn3(self.conv3(out)) 46 | res = self.shortcut(x) 47 | out = F.relu(torch.cat([out,res], 1)) if self.stride==2 else F.relu(out+res) 48 | return out 49 | 50 | 51 | class ShuffleNet(nn.Module): 52 | def __init__(self, cfg): 53 | super(ShuffleNet, self).__init__() 54 | out_planes = cfg['out_planes'] 55 | num_blocks = cfg['num_blocks'] 56 | groups = cfg['groups'] 57 | 58 | self.conv1 = nn.Conv2d(3, 24, kernel_size=1, bias=False) 59 | self.bn1 = nn.BatchNorm2d(24) 60 | self.in_planes = 24 61 | self.layer1 = self._make_layer(out_planes[0], num_blocks[0], groups) 62 | self.layer2 = self._make_layer(out_planes[1], num_blocks[1], groups) 63 | self.layer3 = self._make_layer(out_planes[2], num_blocks[2], groups) 64 | self.linear = nn.Linear(out_planes[2], 10) 65 | 66 | def _make_layer(self, out_planes, num_blocks, groups): 67 | layers = [] 68 | for i in range(num_blocks): 69 | stride = 2 if i == 0 else 1 70 | cat_planes = self.in_planes if i == 0 else 0 71 | layers.append(Bottleneck(self.in_planes, out_planes-cat_planes, stride=stride, groups=groups)) 72 | self.in_planes = out_planes 73 | return nn.Sequential(*layers) 74 | 75 | def forward(self, x): 76 | out = F.relu(self.bn1(self.conv1(x))) 77 | out = self.layer1(out) 78 | out = self.layer2(out) 79 | out = self.layer3(out) 80 | out = F.avg_pool2d(out, 4) 81 | out = out.view(out.size(0), -1) 82 | out = self.linear(out) 83 | return out 84 | 85 | 86 | def ShuffleNetG2(): 87 | cfg = { 88 | 'out_planes': [200,400,800], 89 | 'num_blocks': [4,8,4], 90 | 'groups': 2 91 | } 92 | return ShuffleNet(cfg) 93 | 94 | def ShuffleNetG3(): 95 | cfg = { 96 | 'out_planes': [240,480,960], 97 | 'num_blocks': [4,8,4], 98 | 'groups': 3 99 | } 100 | return ShuffleNet(cfg) 101 | 102 | 103 | def test(): 104 | net = ShuffleNetG2() 105 | x = torch.randn(1,3,32,32) 106 | y = net(x) 107 | print(y) 108 | 109 | # test() 110 | -------------------------------------------------------------------------------- /Inversion/models/wideresnet.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class BasicBlock(nn.Module): 8 | def __init__(self, in_planes, out_planes, stride, dropRate=0.0): 9 | super(BasicBlock, self).__init__() 10 | self.bn1 = nn.BatchNorm2d(in_planes) 11 | self.relu1 = nn.ReLU(inplace=True) 12 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 13 | padding=1, bias=False) 14 | self.bn2 = nn.BatchNorm2d(out_planes) 15 | self.relu2 = nn.ReLU(inplace=True) 16 | self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, 17 | padding=1, bias=False) 18 | self.droprate = dropRate 19 | self.equalInOut = (in_planes == out_planes) 20 | self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, 21 | padding=0, bias=False) or None 22 | 23 | def forward(self, x): 24 | if not self.equalInOut: 25 | x = self.relu1(self.bn1(x)) 26 | else: 27 | out = self.relu1(self.bn1(x)) 28 | out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) 29 | if self.droprate > 0: 30 | out = F.dropout(out, p=self.droprate, training=self.training) 31 | out = self.conv2(out) 32 | return torch.add(x if self.equalInOut else self.convShortcut(x), out) 33 | 34 | 35 | class NetworkBlock(nn.Module): 36 | def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): 37 | super(NetworkBlock, self).__init__() 38 | self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) 39 | 40 | def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): 41 | layers = [] 42 | for i in range(int(nb_layers)): 43 | layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) 44 | return nn.Sequential(*layers) 45 | 46 | def forward(self, x): 47 | return self.layer(x) 48 | 49 | 50 | class WideResNet(nn.Module): 51 | def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0): 52 | super(WideResNet, self).__init__() 53 | nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] 54 | assert ((depth - 4) % 6 == 0) 55 | n = (depth - 4) / 6 56 | block = BasicBlock 57 | # 1st conv before any network block 58 | self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, 59 | padding=1, bias=False) 60 | # 1st block 61 | self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) 62 | # 2nd block 63 | self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) 64 | # 3rd block 65 | self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) 66 | # global average pooling and classifier 67 | self.bn1 = nn.BatchNorm2d(nChannels[3]) 68 | self.relu = nn.ReLU(inplace=True) 69 | self.fc = nn.Linear(nChannels[3], num_classes) 70 | self.nChannels = nChannels[3] 71 | 72 | for m in self.modules(): 73 | if isinstance(m, nn.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | m.weight.data.normal_(0, math.sqrt(2. / n)) 76 | elif isinstance(m, nn.BatchNorm2d): 77 | m.weight.data.fill_(1) 78 | m.bias.data.zero_() 79 | elif isinstance(m, nn.Linear): 80 | m.bias.data.zero_() 81 | 82 | def forward(self, x): 83 | out = self.conv1(x) 84 | out = self.block1(out) 85 | out = self.block2(out) 86 | out = self.block3(out) 87 | out = self.relu(self.bn1(out)) 88 | out = F.avg_pool2d(out, 8) 89 | out = out.view(-1, self.nChannels) 90 | self.embeddings = out 91 | return self.fc(out) 92 | 93 | -------------------------------------------------------------------------------- /Training/models/wideresnet.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class BasicBlock(nn.Module): 8 | def __init__(self, in_planes, out_planes, stride, dropRate=0.0): 9 | super(BasicBlock, self).__init__() 10 | self.bn1 = nn.BatchNorm2d(in_planes) 11 | self.relu1 = nn.ReLU(inplace=True) 12 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 13 | padding=1, bias=False) 14 | self.bn2 = nn.BatchNorm2d(out_planes) 15 | self.relu2 = nn.ReLU(inplace=True) 16 | self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, 17 | padding=1, bias=False) 18 | self.droprate = dropRate 19 | self.equalInOut = (in_planes == out_planes) 20 | self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, 21 | padding=0, bias=False) or None 22 | 23 | def forward(self, x): 24 | if not self.equalInOut: 25 | x = self.relu1(self.bn1(x)) 26 | else: 27 | out = self.relu1(self.bn1(x)) 28 | out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) 29 | if self.droprate > 0: 30 | out = F.dropout(out, p=self.droprate, training=self.training) 31 | out = self.conv2(out) 32 | return torch.add(x if self.equalInOut else self.convShortcut(x), out) 33 | 34 | 35 | class NetworkBlock(nn.Module): 36 | def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): 37 | super(NetworkBlock, self).__init__() 38 | self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) 39 | 40 | def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): 41 | layers = [] 42 | for i in range(int(nb_layers)): 43 | layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) 44 | return nn.Sequential(*layers) 45 | 46 | def forward(self, x): 47 | return self.layer(x) 48 | 49 | 50 | class WideResNet(nn.Module): 51 | def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0): 52 | super(WideResNet, self).__init__() 53 | nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] 54 | assert ((depth - 4) % 6 == 0) 55 | n = (depth - 4) / 6 56 | block = BasicBlock 57 | # 1st conv before any network block 58 | self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, 59 | padding=1, bias=False) 60 | # 1st block 61 | self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) 62 | # 2nd block 63 | self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) 64 | # 3rd block 65 | self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) 66 | # global average pooling and classifier 67 | self.bn1 = nn.BatchNorm2d(nChannels[3]) 68 | self.relu = nn.ReLU(inplace=True) 69 | self.fc = nn.Linear(nChannels[3], num_classes) 70 | self.nChannels = nChannels[3] 71 | 72 | for m in self.modules(): 73 | if isinstance(m, nn.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | m.weight.data.normal_(0, math.sqrt(2. / n)) 76 | elif isinstance(m, nn.BatchNorm2d): 77 | m.weight.data.fill_(1) 78 | m.bias.data.zero_() 79 | elif isinstance(m, nn.Linear): 80 | m.bias.data.zero_() 81 | 82 | def forward(self, x): 83 | out = self.conv1(x) 84 | out = self.block1(out) 85 | out = self.block2(out) 86 | out = self.block3(out) 87 | out = self.relu(self.bn1(out)) 88 | out = F.avg_pool2d(out, 8) 89 | out = out.view(-1, self.nChannels) 90 | self.embeddings = out 91 | return self.fc(out) 92 | 93 | -------------------------------------------------------------------------------- /Unlearning/models/wideresnet.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class BasicBlock(nn.Module): 8 | def __init__(self, in_planes, out_planes, stride, dropRate=0.0): 9 | super(BasicBlock, self).__init__() 10 | self.bn1 = nn.BatchNorm2d(in_planes) 11 | self.relu1 = nn.ReLU(inplace=True) 12 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 13 | padding=1, bias=False) 14 | self.bn2 = nn.BatchNorm2d(out_planes) 15 | self.relu2 = nn.ReLU(inplace=True) 16 | self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, 17 | padding=1, bias=False) 18 | self.droprate = dropRate 19 | self.equalInOut = (in_planes == out_planes) 20 | self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, 21 | padding=0, bias=False) or None 22 | 23 | def forward(self, x): 24 | if not self.equalInOut: 25 | x = self.relu1(self.bn1(x)) 26 | else: 27 | out = self.relu1(self.bn1(x)) 28 | out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) 29 | if self.droprate > 0: 30 | out = F.dropout(out, p=self.droprate, training=self.training) 31 | out = self.conv2(out) 32 | return torch.add(x if self.equalInOut else self.convShortcut(x), out) 33 | 34 | 35 | class NetworkBlock(nn.Module): 36 | def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): 37 | super(NetworkBlock, self).__init__() 38 | self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) 39 | 40 | def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): 41 | layers = [] 42 | for i in range(int(nb_layers)): 43 | layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) 44 | return nn.Sequential(*layers) 45 | 46 | def forward(self, x): 47 | return self.layer(x) 48 | 49 | 50 | class WideResNet(nn.Module): 51 | def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0): 52 | super(WideResNet, self).__init__() 53 | nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] 54 | assert ((depth - 4) % 6 == 0) 55 | n = (depth - 4) / 6 56 | block = BasicBlock 57 | # 1st conv before any network block 58 | self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, 59 | padding=1, bias=False) 60 | # 1st block 61 | self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) 62 | # 2nd block 63 | self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) 64 | # 3rd block 65 | self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) 66 | # global average pooling and classifier 67 | self.bn1 = nn.BatchNorm2d(nChannels[3]) 68 | self.relu = nn.ReLU(inplace=True) 69 | self.fc = nn.Linear(nChannels[3], num_classes) 70 | self.nChannels = nChannels[3] 71 | 72 | for m in self.modules(): 73 | if isinstance(m, nn.Conv2d): 74 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 75 | m.weight.data.normal_(0, math.sqrt(2. / n)) 76 | elif isinstance(m, nn.BatchNorm2d): 77 | m.weight.data.fill_(1) 78 | m.bias.data.zero_() 79 | elif isinstance(m, nn.Linear): 80 | m.bias.data.zero_() 81 | 82 | def forward(self, x): 83 | out = self.conv1(x) 84 | out = self.block1(out) 85 | out = self.block2(out) 86 | out = self.block3(out) 87 | out = self.relu(self.bn1(out)) 88 | out = F.avg_pool2d(out, 8) 89 | out = out.view(-1, self.nChannels) 90 | self.embeddings = out 91 | return self.fc(out) 92 | 93 | -------------------------------------------------------------------------------- /Inversion/models/efficientnet.py: -------------------------------------------------------------------------------- 1 | '''EfficientNet in PyTorch. 2 | 3 | Paper: "EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks". 4 | 5 | Reference: https://github.com/keras-team/keras-applications/blob/master/keras_applications/efficientnet.py 6 | ''' 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | 11 | 12 | def swish(x): 13 | return x * x.sigmoid() 14 | 15 | 16 | class Block(nn.Module): 17 | '''expansion + depthwise + pointwise + squeeze-excitation''' 18 | 19 | def __init__(self, in_planes, out_planes, kernel_size, stride, expand_ratio=1, se_ratio=0., drop_rate=0.): 20 | super(Block, self).__init__() 21 | self.stride = stride 22 | self.drop_rate = drop_rate 23 | 24 | # Expansion 25 | planes = expand_ratio * in_planes 26 | self.conv1 = nn.Conv2d( 27 | in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False) 28 | self.bn1 = nn.BatchNorm2d(planes) 29 | 30 | # Depthwise conv 31 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=kernel_size, 32 | stride=stride, padding=(kernel_size-1)//2, groups=planes, bias=False) 33 | self.bn2 = nn.BatchNorm2d(planes) 34 | 35 | # SE layers 36 | se_planes = max(1, int(planes * se_ratio)) 37 | self.se1 = nn.Conv2d(planes, se_planes, kernel_size=1) 38 | self.se2 = nn.Conv2d(se_planes, planes, kernel_size=1) 39 | 40 | # Output 41 | self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 42 | self.bn3 = nn.BatchNorm2d(out_planes) 43 | 44 | self.shortcut = nn.Sequential() 45 | if stride == 1 and in_planes != out_planes: 46 | self.shortcut = nn.Sequential( 47 | nn.Conv2d(in_planes, out_planes, kernel_size=1, 48 | stride=1, padding=0, bias=False), 49 | nn.BatchNorm2d(out_planes), 50 | ) 51 | 52 | 53 | def forward(self, x): 54 | out = swish(self.bn1(self.conv1(x))) 55 | out = swish(self.bn2(self.conv2(out))) 56 | # Squeeze-Excitation 57 | w = F.avg_pool2d(out, out.size(2)) 58 | w = swish(self.se1(w)) 59 | w = self.se2(w).sigmoid() 60 | out = out * w 61 | # Output 62 | out = self.bn3(self.conv3(out)) 63 | if self.drop_rate > 0: 64 | out = F.dropout2d(out, self.drop_rate) 65 | shortcut = self.shortcut(x) if self.stride == 1 else out 66 | out = out + shortcut 67 | return out 68 | 69 | 70 | class EfficientNet(nn.Module): 71 | def __init__(self, cfg, num_classes=10): 72 | super(EfficientNet, self).__init__() 73 | self.cfg = cfg 74 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, 75 | stride=1, padding=1, bias=False) 76 | self.bn1 = nn.BatchNorm2d(32) 77 | self.layers = self._make_layers(in_planes=32) 78 | self.linear = nn.Linear(cfg[-1][1], num_classes) 79 | 80 | def _make_layers(self, in_planes): 81 | layers = [] 82 | for expansion, out_planes, num_blocks, kernel_size, stride in self.cfg: 83 | strides = [stride] + [1]*(num_blocks-1) 84 | for stride in strides: 85 | layers.append(Block(in_planes, out_planes, kernel_size, stride, expansion, se_ratio=0.25, drop_rate=0)) 86 | in_planes = out_planes 87 | return nn.Sequential(*layers) 88 | 89 | def forward(self, x): 90 | out = swish(self.bn1(self.conv1(x))) 91 | out = self.layers(out) 92 | out = F.adaptive_avg_pool2d(out, 1) 93 | out = out.view(out.size(0), -1) 94 | out = self.linear(out) 95 | return out 96 | 97 | 98 | def EfficientNetB0(): 99 | # (expansion, out_planes, num_blocks, kernel_size, stride) 100 | cfg = [(1, 16, 1, 3, 1), 101 | (6, 24, 2, 3, 2), 102 | (6, 40, 2, 5, 2), 103 | (6, 80, 3, 3, 2), 104 | (6, 112, 3, 5, 1), 105 | (6, 192, 4, 5, 2), 106 | (6, 320, 1, 3, 1)] 107 | return EfficientNet(cfg) 108 | 109 | 110 | def test(): 111 | net = EfficientNetB0() 112 | x = torch.randn(2, 3, 32, 32) 113 | y = net(x) 114 | print(y.shape) 115 | 116 | 117 | if __name__ == '__main__': 118 | test() 119 | -------------------------------------------------------------------------------- /Training/models/efficientnet.py: -------------------------------------------------------------------------------- 1 | '''EfficientNet in PyTorch. 2 | 3 | Paper: "EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks". 4 | 5 | Reference: https://github.com/keras-team/keras-applications/blob/master/keras_applications/efficientnet.py 6 | ''' 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | 11 | 12 | def swish(x): 13 | return x * x.sigmoid() 14 | 15 | 16 | class Block(nn.Module): 17 | '''expansion + depthwise + pointwise + squeeze-excitation''' 18 | 19 | def __init__(self, in_planes, out_planes, kernel_size, stride, expand_ratio=1, se_ratio=0., drop_rate=0.): 20 | super(Block, self).__init__() 21 | self.stride = stride 22 | self.drop_rate = drop_rate 23 | 24 | # Expansion 25 | planes = expand_ratio * in_planes 26 | self.conv1 = nn.Conv2d( 27 | in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False) 28 | self.bn1 = nn.BatchNorm2d(planes) 29 | 30 | # Depthwise conv 31 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=kernel_size, 32 | stride=stride, padding=(kernel_size-1)//2, groups=planes, bias=False) 33 | self.bn2 = nn.BatchNorm2d(planes) 34 | 35 | # SE layers 36 | se_planes = max(1, int(planes * se_ratio)) 37 | self.se1 = nn.Conv2d(planes, se_planes, kernel_size=1) 38 | self.se2 = nn.Conv2d(se_planes, planes, kernel_size=1) 39 | 40 | # Output 41 | self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 42 | self.bn3 = nn.BatchNorm2d(out_planes) 43 | 44 | self.shortcut = nn.Sequential() 45 | if stride == 1 and in_planes != out_planes: 46 | self.shortcut = nn.Sequential( 47 | nn.Conv2d(in_planes, out_planes, kernel_size=1, 48 | stride=1, padding=0, bias=False), 49 | nn.BatchNorm2d(out_planes), 50 | ) 51 | 52 | 53 | def forward(self, x): 54 | out = swish(self.bn1(self.conv1(x))) 55 | out = swish(self.bn2(self.conv2(out))) 56 | # Squeeze-Excitation 57 | w = F.avg_pool2d(out, out.size(2)) 58 | w = swish(self.se1(w)) 59 | w = self.se2(w).sigmoid() 60 | out = out * w 61 | # Output 62 | out = self.bn3(self.conv3(out)) 63 | if self.drop_rate > 0: 64 | out = F.dropout2d(out, self.drop_rate) 65 | shortcut = self.shortcut(x) if self.stride == 1 else out 66 | out = out + shortcut 67 | return out 68 | 69 | 70 | class EfficientNet(nn.Module): 71 | def __init__(self, cfg, num_classes=10): 72 | super(EfficientNet, self).__init__() 73 | self.cfg = cfg 74 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, 75 | stride=1, padding=1, bias=False) 76 | self.bn1 = nn.BatchNorm2d(32) 77 | self.layers = self._make_layers(in_planes=32) 78 | self.linear = nn.Linear(cfg[-1][1], num_classes) 79 | 80 | def _make_layers(self, in_planes): 81 | layers = [] 82 | for expansion, out_planes, num_blocks, kernel_size, stride in self.cfg: 83 | strides = [stride] + [1]*(num_blocks-1) 84 | for stride in strides: 85 | layers.append(Block(in_planes, out_planes, kernel_size, stride, expansion, se_ratio=0.25, drop_rate=0)) 86 | in_planes = out_planes 87 | return nn.Sequential(*layers) 88 | 89 | def forward(self, x): 90 | out = swish(self.bn1(self.conv1(x))) 91 | out = self.layers(out) 92 | out = F.adaptive_avg_pool2d(out, 1) 93 | out = out.view(out.size(0), -1) 94 | out = self.linear(out) 95 | return out 96 | 97 | 98 | def EfficientNetB0(): 99 | # (expansion, out_planes, num_blocks, kernel_size, stride) 100 | cfg = [(1, 16, 1, 3, 1), 101 | (6, 24, 2, 3, 2), 102 | (6, 40, 2, 5, 2), 103 | (6, 80, 3, 3, 2), 104 | (6, 112, 3, 5, 1), 105 | (6, 192, 4, 5, 2), 106 | (6, 320, 1, 3, 1)] 107 | return EfficientNet(cfg) 108 | 109 | 110 | def test(): 111 | net = EfficientNetB0() 112 | x = torch.randn(2, 3, 32, 32) 113 | y = net(x) 114 | print(y.shape) 115 | 116 | 117 | if __name__ == '__main__': 118 | test() 119 | -------------------------------------------------------------------------------- /Unlearning/models/efficientnet.py: -------------------------------------------------------------------------------- 1 | '''EfficientNet in PyTorch. 2 | 3 | Paper: "EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks". 4 | 5 | Reference: https://github.com/keras-team/keras-applications/blob/master/keras_applications/efficientnet.py 6 | ''' 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | 11 | 12 | def swish(x): 13 | return x * x.sigmoid() 14 | 15 | 16 | class Block(nn.Module): 17 | '''expansion + depthwise + pointwise + squeeze-excitation''' 18 | 19 | def __init__(self, in_planes, out_planes, kernel_size, stride, expand_ratio=1, se_ratio=0., drop_rate=0.): 20 | super(Block, self).__init__() 21 | self.stride = stride 22 | self.drop_rate = drop_rate 23 | 24 | # Expansion 25 | planes = expand_ratio * in_planes 26 | self.conv1 = nn.Conv2d( 27 | in_planes, planes, kernel_size=1, stride=1, padding=0, bias=False) 28 | self.bn1 = nn.BatchNorm2d(planes) 29 | 30 | # Depthwise conv 31 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=kernel_size, 32 | stride=stride, padding=(kernel_size-1)//2, groups=planes, bias=False) 33 | self.bn2 = nn.BatchNorm2d(planes) 34 | 35 | # SE layers 36 | se_planes = max(1, int(planes * se_ratio)) 37 | self.se1 = nn.Conv2d(planes, se_planes, kernel_size=1) 38 | self.se2 = nn.Conv2d(se_planes, planes, kernel_size=1) 39 | 40 | # Output 41 | self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 42 | self.bn3 = nn.BatchNorm2d(out_planes) 43 | 44 | self.shortcut = nn.Sequential() 45 | if stride == 1 and in_planes != out_planes: 46 | self.shortcut = nn.Sequential( 47 | nn.Conv2d(in_planes, out_planes, kernel_size=1, 48 | stride=1, padding=0, bias=False), 49 | nn.BatchNorm2d(out_planes), 50 | ) 51 | 52 | 53 | def forward(self, x): 54 | out = swish(self.bn1(self.conv1(x))) 55 | out = swish(self.bn2(self.conv2(out))) 56 | # Squeeze-Excitation 57 | w = F.avg_pool2d(out, out.size(2)) 58 | w = swish(self.se1(w)) 59 | w = self.se2(w).sigmoid() 60 | out = out * w 61 | # Output 62 | out = self.bn3(self.conv3(out)) 63 | if self.drop_rate > 0: 64 | out = F.dropout2d(out, self.drop_rate) 65 | shortcut = self.shortcut(x) if self.stride == 1 else out 66 | out = out + shortcut 67 | return out 68 | 69 | 70 | class EfficientNet(nn.Module): 71 | def __init__(self, cfg, num_classes=10): 72 | super(EfficientNet, self).__init__() 73 | self.cfg = cfg 74 | self.conv1 = nn.Conv2d(3, 32, kernel_size=3, 75 | stride=1, padding=1, bias=False) 76 | self.bn1 = nn.BatchNorm2d(32) 77 | self.layers = self._make_layers(in_planes=32) 78 | self.linear = nn.Linear(cfg[-1][1], num_classes) 79 | 80 | def _make_layers(self, in_planes): 81 | layers = [] 82 | for expansion, out_planes, num_blocks, kernel_size, stride in self.cfg: 83 | strides = [stride] + [1]*(num_blocks-1) 84 | for stride in strides: 85 | layers.append(Block(in_planes, out_planes, kernel_size, stride, expansion, se_ratio=0.25, drop_rate=0)) 86 | in_planes = out_planes 87 | return nn.Sequential(*layers) 88 | 89 | def forward(self, x): 90 | out = swish(self.bn1(self.conv1(x))) 91 | out = self.layers(out) 92 | out = F.adaptive_avg_pool2d(out, 1) 93 | out = out.view(out.size(0), -1) 94 | out = self.linear(out) 95 | return out 96 | 97 | 98 | def EfficientNetB0(): 99 | # (expansion, out_planes, num_blocks, kernel_size, stride) 100 | cfg = [(1, 16, 1, 3, 1), 101 | (6, 24, 2, 3, 2), 102 | (6, 40, 2, 5, 2), 103 | (6, 80, 3, 3, 2), 104 | (6, 112, 3, 5, 1), 105 | (6, 192, 4, 5, 2), 106 | (6, 320, 1, 3, 1)] 107 | return EfficientNet(cfg) 108 | 109 | 110 | def test(): 111 | net = EfficientNetB0() 112 | x = torch.randn(2, 3, 32, 32) 113 | y = net(x) 114 | print(y.shape) 115 | 116 | 117 | if __name__ == '__main__': 118 | test() 119 | -------------------------------------------------------------------------------- /Inversion/models/senet.py: -------------------------------------------------------------------------------- 1 | '''SENet in PyTorch. 2 | 3 | SENet is the winner of ImageNet-2017. The paper is not released yet. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class BasicBlock(nn.Module): 11 | def __init__(self, in_planes, planes, stride=1): 12 | super(BasicBlock, self).__init__() 13 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 14 | self.bn1 = nn.BatchNorm2d(planes) 15 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 16 | self.bn2 = nn.BatchNorm2d(planes) 17 | 18 | self.shortcut = nn.Sequential() 19 | if stride != 1 or in_planes != planes: 20 | self.shortcut = nn.Sequential( 21 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False), 22 | nn.BatchNorm2d(planes) 23 | ) 24 | 25 | # SE layers 26 | self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) # Use nn.Conv2d instead of nn.Linear 27 | self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1) 28 | 29 | def forward(self, x): 30 | out = F.relu(self.bn1(self.conv1(x))) 31 | out = self.bn2(self.conv2(out)) 32 | 33 | # Squeeze 34 | w = F.avg_pool2d(out, out.size(2)) 35 | w = F.relu(self.fc1(w)) 36 | w = F.sigmoid(self.fc2(w)) 37 | # Excitation 38 | out = out * w # New broadcasting feature from v0.2! 39 | 40 | out += self.shortcut(x) 41 | out = F.relu(out) 42 | return out 43 | 44 | 45 | class PreActBlock(nn.Module): 46 | def __init__(self, in_planes, planes, stride=1): 47 | super(PreActBlock, self).__init__() 48 | self.bn1 = nn.BatchNorm2d(in_planes) 49 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 50 | self.bn2 = nn.BatchNorm2d(planes) 51 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 52 | 53 | if stride != 1 or in_planes != planes: 54 | self.shortcut = nn.Sequential( 55 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False) 56 | ) 57 | 58 | # SE layers 59 | self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) 60 | self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1) 61 | 62 | def forward(self, x): 63 | out = F.relu(self.bn1(x)) 64 | shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x 65 | out = self.conv1(out) 66 | out = self.conv2(F.relu(self.bn2(out))) 67 | 68 | # Squeeze 69 | w = F.avg_pool2d(out, out.size(2)) 70 | w = F.relu(self.fc1(w)) 71 | w = F.sigmoid(self.fc2(w)) 72 | # Excitation 73 | out = out * w 74 | 75 | out += shortcut 76 | return out 77 | 78 | 79 | class SENet(nn.Module): 80 | def __init__(self, block, num_blocks, num_classes=10): 81 | super(SENet, self).__init__() 82 | self.in_planes = 64 83 | 84 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 85 | self.bn1 = nn.BatchNorm2d(64) 86 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 87 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 88 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 89 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 90 | self.linear = nn.Linear(512, num_classes) 91 | 92 | def _make_layer(self, block, planes, num_blocks, stride): 93 | strides = [stride] + [1]*(num_blocks-1) 94 | layers = [] 95 | for stride in strides: 96 | layers.append(block(self.in_planes, planes, stride)) 97 | self.in_planes = planes 98 | return nn.Sequential(*layers) 99 | 100 | def forward(self, x): 101 | out = F.relu(self.bn1(self.conv1(x))) 102 | out = self.layer1(out) 103 | out = self.layer2(out) 104 | out = self.layer3(out) 105 | out = self.layer4(out) 106 | out = F.avg_pool2d(out, 4) 107 | out = out.view(out.size(0), -1) 108 | out = self.linear(out) 109 | return out 110 | 111 | 112 | def SENet18(): 113 | return SENet(PreActBlock, [2,2,2,2]) 114 | 115 | 116 | def test(): 117 | net = SENet18() 118 | y = net(torch.randn(1,3,32,32)) 119 | print(y.size()) 120 | 121 | # test() 122 | -------------------------------------------------------------------------------- /Training/models/senet.py: -------------------------------------------------------------------------------- 1 | '''SENet in PyTorch. 2 | 3 | SENet is the winner of ImageNet-2017. The paper is not released yet. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class BasicBlock(nn.Module): 11 | def __init__(self, in_planes, planes, stride=1): 12 | super(BasicBlock, self).__init__() 13 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 14 | self.bn1 = nn.BatchNorm2d(planes) 15 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 16 | self.bn2 = nn.BatchNorm2d(planes) 17 | 18 | self.shortcut = nn.Sequential() 19 | if stride != 1 or in_planes != planes: 20 | self.shortcut = nn.Sequential( 21 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False), 22 | nn.BatchNorm2d(planes) 23 | ) 24 | 25 | # SE layers 26 | self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) # Use nn.Conv2d instead of nn.Linear 27 | self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1) 28 | 29 | def forward(self, x): 30 | out = F.relu(self.bn1(self.conv1(x))) 31 | out = self.bn2(self.conv2(out)) 32 | 33 | # Squeeze 34 | w = F.avg_pool2d(out, out.size(2)) 35 | w = F.relu(self.fc1(w)) 36 | w = F.sigmoid(self.fc2(w)) 37 | # Excitation 38 | out = out * w # New broadcasting feature from v0.2! 39 | 40 | out += self.shortcut(x) 41 | out = F.relu(out) 42 | return out 43 | 44 | 45 | class PreActBlock(nn.Module): 46 | def __init__(self, in_planes, planes, stride=1): 47 | super(PreActBlock, self).__init__() 48 | self.bn1 = nn.BatchNorm2d(in_planes) 49 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 50 | self.bn2 = nn.BatchNorm2d(planes) 51 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 52 | 53 | if stride != 1 or in_planes != planes: 54 | self.shortcut = nn.Sequential( 55 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False) 56 | ) 57 | 58 | # SE layers 59 | self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) 60 | self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1) 61 | 62 | def forward(self, x): 63 | out = F.relu(self.bn1(x)) 64 | shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x 65 | out = self.conv1(out) 66 | out = self.conv2(F.relu(self.bn2(out))) 67 | 68 | # Squeeze 69 | w = F.avg_pool2d(out, out.size(2)) 70 | w = F.relu(self.fc1(w)) 71 | w = F.sigmoid(self.fc2(w)) 72 | # Excitation 73 | out = out * w 74 | 75 | out += shortcut 76 | return out 77 | 78 | 79 | class SENet(nn.Module): 80 | def __init__(self, block, num_blocks, num_classes=10): 81 | super(SENet, self).__init__() 82 | self.in_planes = 64 83 | 84 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 85 | self.bn1 = nn.BatchNorm2d(64) 86 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 87 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 88 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 89 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 90 | self.linear = nn.Linear(512, num_classes) 91 | 92 | def _make_layer(self, block, planes, num_blocks, stride): 93 | strides = [stride] + [1]*(num_blocks-1) 94 | layers = [] 95 | for stride in strides: 96 | layers.append(block(self.in_planes, planes, stride)) 97 | self.in_planes = planes 98 | return nn.Sequential(*layers) 99 | 100 | def forward(self, x): 101 | out = F.relu(self.bn1(self.conv1(x))) 102 | out = self.layer1(out) 103 | out = self.layer2(out) 104 | out = self.layer3(out) 105 | out = self.layer4(out) 106 | out = F.avg_pool2d(out, 4) 107 | out = out.view(out.size(0), -1) 108 | out = self.linear(out) 109 | return out 110 | 111 | 112 | def SENet18(): 113 | return SENet(PreActBlock, [2,2,2,2]) 114 | 115 | 116 | def test(): 117 | net = SENet18() 118 | y = net(torch.randn(1,3,32,32)) 119 | print(y.size()) 120 | 121 | # test() 122 | -------------------------------------------------------------------------------- /Unlearning/models/senet.py: -------------------------------------------------------------------------------- 1 | '''SENet in PyTorch. 2 | 3 | SENet is the winner of ImageNet-2017. The paper is not released yet. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class BasicBlock(nn.Module): 11 | def __init__(self, in_planes, planes, stride=1): 12 | super(BasicBlock, self).__init__() 13 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 14 | self.bn1 = nn.BatchNorm2d(planes) 15 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 16 | self.bn2 = nn.BatchNorm2d(planes) 17 | 18 | self.shortcut = nn.Sequential() 19 | if stride != 1 or in_planes != planes: 20 | self.shortcut = nn.Sequential( 21 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False), 22 | nn.BatchNorm2d(planes) 23 | ) 24 | 25 | # SE layers 26 | self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) # Use nn.Conv2d instead of nn.Linear 27 | self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1) 28 | 29 | def forward(self, x): 30 | out = F.relu(self.bn1(self.conv1(x))) 31 | out = self.bn2(self.conv2(out)) 32 | 33 | # Squeeze 34 | w = F.avg_pool2d(out, out.size(2)) 35 | w = F.relu(self.fc1(w)) 36 | w = F.sigmoid(self.fc2(w)) 37 | # Excitation 38 | out = out * w # New broadcasting feature from v0.2! 39 | 40 | out += self.shortcut(x) 41 | out = F.relu(out) 42 | return out 43 | 44 | 45 | class PreActBlock(nn.Module): 46 | def __init__(self, in_planes, planes, stride=1): 47 | super(PreActBlock, self).__init__() 48 | self.bn1 = nn.BatchNorm2d(in_planes) 49 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 50 | self.bn2 = nn.BatchNorm2d(planes) 51 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 52 | 53 | if stride != 1 or in_planes != planes: 54 | self.shortcut = nn.Sequential( 55 | nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False) 56 | ) 57 | 58 | # SE layers 59 | self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) 60 | self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1) 61 | 62 | def forward(self, x): 63 | out = F.relu(self.bn1(x)) 64 | shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x 65 | out = self.conv1(out) 66 | out = self.conv2(F.relu(self.bn2(out))) 67 | 68 | # Squeeze 69 | w = F.avg_pool2d(out, out.size(2)) 70 | w = F.relu(self.fc1(w)) 71 | w = F.sigmoid(self.fc2(w)) 72 | # Excitation 73 | out = out * w 74 | 75 | out += shortcut 76 | return out 77 | 78 | 79 | class SENet(nn.Module): 80 | def __init__(self, block, num_blocks, num_classes=10): 81 | super(SENet, self).__init__() 82 | self.in_planes = 64 83 | 84 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 85 | self.bn1 = nn.BatchNorm2d(64) 86 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 87 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 88 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 89 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 90 | self.linear = nn.Linear(512, num_classes) 91 | 92 | def _make_layer(self, block, planes, num_blocks, stride): 93 | strides = [stride] + [1]*(num_blocks-1) 94 | layers = [] 95 | for stride in strides: 96 | layers.append(block(self.in_planes, planes, stride)) 97 | self.in_planes = planes 98 | return nn.Sequential(*layers) 99 | 100 | def forward(self, x): 101 | out = F.relu(self.bn1(self.conv1(x))) 102 | out = self.layer1(out) 103 | out = self.layer2(out) 104 | out = self.layer3(out) 105 | out = self.layer4(out) 106 | out = F.avg_pool2d(out, 4) 107 | out = out.view(out.size(0), -1) 108 | out = self.linear(out) 109 | return out 110 | 111 | 112 | def SENet18(): 113 | return SENet(PreActBlock, [2,2,2,2]) 114 | 115 | 116 | def test(): 117 | net = SENet18() 118 | y = net(torch.randn(1,3,32,32)) 119 | print(y.size()) 120 | 121 | # test() 122 | -------------------------------------------------------------------------------- /Training/models/resnet.py: -------------------------------------------------------------------------------- 1 | '''ResNet in PyTorch. 2 | 3 | For Pre-activation ResNet, see 'preact_resnet.py'. 4 | 5 | Reference: 6 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 7 | Deep Residual Learning for Image Recognition. arXiv:1512.03385 8 | ''' 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | 13 | 14 | class BasicBlock(nn.Module): 15 | expansion = 1 16 | 17 | def __init__(self, in_planes, planes, stride=1): 18 | super(BasicBlock, self).__init__() 19 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 20 | self.bn1 = nn.BatchNorm2d(planes) 21 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 22 | self.bn2 = nn.BatchNorm2d(planes) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*planes: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*planes) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = self.bn2(self.conv2(out)) 34 | out += self.shortcut(x) 35 | out = F.relu(out) 36 | return out 37 | 38 | 39 | class Bottleneck(nn.Module): 40 | expansion = 4 41 | 42 | def __init__(self, in_planes, planes, stride=1): 43 | super(Bottleneck, self).__init__() 44 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(planes) 46 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 47 | self.bn2 = nn.BatchNorm2d(planes) 48 | self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) 49 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 50 | 51 | self.shortcut = nn.Sequential() 52 | if stride != 1 or in_planes != self.expansion*planes: 53 | self.shortcut = nn.Sequential( 54 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 55 | nn.BatchNorm2d(self.expansion*planes) 56 | ) 57 | 58 | def forward(self, x): 59 | out = F.relu(self.bn1(self.conv1(x))) 60 | out = F.relu(self.bn2(self.conv2(out))) 61 | out = self.bn3(self.conv3(out)) 62 | out += self.shortcut(x) 63 | out = F.relu(out) 64 | return out 65 | 66 | 67 | class ResNet(nn.Module): 68 | def __init__(self, block, num_blocks, num_classes=10): 69 | super(ResNet, self).__init__() 70 | self.in_planes = 64 71 | 72 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 73 | self.bn1 = nn.BatchNorm2d(64) 74 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 75 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 76 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 77 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 78 | self.linear = nn.Linear(512*block.expansion, num_classes) 79 | 80 | def _make_layer(self, block, planes, num_blocks, stride): 81 | strides = [stride] + [1]*(num_blocks-1) 82 | layers = [] 83 | for stride in strides: 84 | layers.append(block(self.in_planes, planes, stride)) 85 | self.in_planes = planes * block.expansion 86 | return nn.Sequential(*layers) 87 | 88 | def forward(self, x): 89 | out = F.relu(self.bn1(self.conv1(x))) 90 | out = self.layer1(out) 91 | out = self.layer2(out) 92 | out = self.layer3(out) 93 | out = self.layer4(out) 94 | out = F.avg_pool2d(out, 4) 95 | out = out.view(out.size(0), -1) 96 | self.embeddings = out 97 | out = self.linear(out) 98 | return out 99 | 100 | 101 | def ResNet18(): 102 | return ResNet(BasicBlock, [2,2,2,2]) 103 | 104 | def ResNet34(): 105 | return ResNet(BasicBlock, [3,4,6,3]) 106 | 107 | def ResNet50(num_classes=10): 108 | return ResNet(Bottleneck, [3,4,6,3], num_classes=num_classes) 109 | 110 | def ResNet101(): 111 | return ResNet(Bottleneck, [3,4,23,3]) 112 | 113 | def ResNet152(): 114 | return ResNet(Bottleneck, [3,8,36,3]) 115 | 116 | 117 | def test(): 118 | net = ResNet18() 119 | y = net(torch.randn(1,3,32,32)) 120 | print(y.size()) 121 | 122 | # test() 123 | -------------------------------------------------------------------------------- /Inversion/models/resnet.py: -------------------------------------------------------------------------------- 1 | '''ResNet in PyTorch. 2 | 3 | For Pre-activation ResNet, see 'preact_resnet.py'. 4 | 5 | Reference: 6 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 7 | Deep Residual Learning for Image Recognition. arXiv:1512.03385 8 | ''' 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | 13 | 14 | class BasicBlock(nn.Module): 15 | expansion = 1 16 | 17 | def __init__(self, in_planes, planes, stride=1): 18 | super(BasicBlock, self).__init__() 19 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 20 | self.bn1 = nn.BatchNorm2d(planes) 21 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 22 | self.bn2 = nn.BatchNorm2d(planes) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*planes: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*planes) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = self.bn2(self.conv2(out)) 34 | out += self.shortcut(x) 35 | out = F.relu(out) 36 | return out 37 | 38 | 39 | class Bottleneck(nn.Module): 40 | expansion = 4 41 | 42 | def __init__(self, in_planes, planes, stride=1): 43 | super(Bottleneck, self).__init__() 44 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(planes) 46 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 47 | self.bn2 = nn.BatchNorm2d(planes) 48 | self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) 49 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 50 | 51 | self.shortcut = nn.Sequential() 52 | if stride != 1 or in_planes != self.expansion*planes: 53 | self.shortcut = nn.Sequential( 54 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 55 | nn.BatchNorm2d(self.expansion*planes) 56 | ) 57 | 58 | def forward(self, x): 59 | out = F.relu(self.bn1(self.conv1(x))) 60 | out = F.relu(self.bn2(self.conv2(out))) 61 | out = self.bn3(self.conv3(out)) 62 | out += self.shortcut(x) 63 | out = F.relu(out) 64 | return out 65 | 66 | 67 | class ResNet(nn.Module): 68 | def __init__(self, block, num_blocks, num_classes=10): 69 | super(ResNet, self).__init__() 70 | self.in_planes = 64 71 | 72 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 73 | self.bn1 = nn.BatchNorm2d(64) 74 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 75 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 76 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 77 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 78 | self.linear = nn.Linear(512*block.expansion, num_classes) 79 | 80 | def _make_layer(self, block, planes, num_blocks, stride): 81 | strides = [stride] + [1]*(num_blocks-1) 82 | layers = [] 83 | for stride in strides: 84 | layers.append(block(self.in_planes, planes, stride)) 85 | self.in_planes = planes * block.expansion 86 | return nn.Sequential(*layers) 87 | 88 | def forward(self, x): 89 | out = F.relu(self.bn1(self.conv1(x))) 90 | out = self.layer1(out) 91 | out = self.layer2(out) 92 | out = self.layer3(out) 93 | out = self.layer4(out) 94 | out = F.avg_pool2d(out, 4) 95 | out = out.view(out.size(0), -1) 96 | self.embeddings = out 97 | out = self.linear(out) 98 | return out 99 | 100 | 101 | def ResNet18(): 102 | return ResNet(BasicBlock, [2,2,2,2]) 103 | 104 | def ResNet34(): 105 | return ResNet(BasicBlock, [3,4,6,3]) 106 | 107 | def ResNet50(num_classes=10): 108 | return ResNet(Bottleneck, [3,4,6,3], num_classes=num_classes) 109 | 110 | def ResNet101(): 111 | return ResNet(Bottleneck, [3,4,23,3]) 112 | 113 | def ResNet152(): 114 | return ResNet(Bottleneck, [3,8,36,3]) 115 | 116 | 117 | def test(): 118 | net = ResNet18() 119 | y = net(torch.randn(1,3,32,32)) 120 | print(y.size()) 121 | 122 | # test() 123 | -------------------------------------------------------------------------------- /Unlearning/models/resnet.py: -------------------------------------------------------------------------------- 1 | '''ResNet in PyTorch. 2 | 3 | For Pre-activation ResNet, see 'preact_resnet.py'. 4 | 5 | Reference: 6 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 7 | Deep Residual Learning for Image Recognition. arXiv:1512.03385 8 | ''' 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | 13 | 14 | class BasicBlock(nn.Module): 15 | expansion = 1 16 | 17 | def __init__(self, in_planes, planes, stride=1): 18 | super(BasicBlock, self).__init__() 19 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 20 | self.bn1 = nn.BatchNorm2d(planes) 21 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 22 | self.bn2 = nn.BatchNorm2d(planes) 23 | 24 | self.shortcut = nn.Sequential() 25 | if stride != 1 or in_planes != self.expansion*planes: 26 | self.shortcut = nn.Sequential( 27 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 28 | nn.BatchNorm2d(self.expansion*planes) 29 | ) 30 | 31 | def forward(self, x): 32 | out = F.relu(self.bn1(self.conv1(x))) 33 | out = self.bn2(self.conv2(out)) 34 | out += self.shortcut(x) 35 | out = F.relu(out) 36 | return out 37 | 38 | 39 | class Bottleneck(nn.Module): 40 | expansion = 4 41 | 42 | def __init__(self, in_planes, planes, stride=1): 43 | super(Bottleneck, self).__init__() 44 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) 45 | self.bn1 = nn.BatchNorm2d(planes) 46 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 47 | self.bn2 = nn.BatchNorm2d(planes) 48 | self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) 49 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 50 | 51 | self.shortcut = nn.Sequential() 52 | if stride != 1 or in_planes != self.expansion*planes: 53 | self.shortcut = nn.Sequential( 54 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 55 | nn.BatchNorm2d(self.expansion*planes) 56 | ) 57 | 58 | def forward(self, x): 59 | out = F.relu(self.bn1(self.conv1(x))) 60 | out = F.relu(self.bn2(self.conv2(out))) 61 | out = self.bn3(self.conv3(out)) 62 | out += self.shortcut(x) 63 | out = F.relu(out) 64 | return out 65 | 66 | 67 | class ResNet(nn.Module): 68 | def __init__(self, block, num_blocks, num_classes=10, num_channels=3): 69 | super(ResNet, self).__init__() 70 | self.in_planes = 64 71 | 72 | self.conv1 = nn.Conv2d(num_channels, 64, kernel_size=3, stride=1, padding=1, bias=False) 73 | self.bn1 = nn.BatchNorm2d(64) 74 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 75 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 76 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 77 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 78 | self.linear = nn.Linear(512*block.expansion, num_classes) 79 | 80 | def _make_layer(self, block, planes, num_blocks, stride): 81 | strides = [stride] + [1]*(num_blocks-1) 82 | layers = [] 83 | for stride in strides: 84 | layers.append(block(self.in_planes, planes, stride)) 85 | self.in_planes = planes * block.expansion 86 | return nn.Sequential(*layers) 87 | 88 | def forward(self, x): 89 | out = F.relu(self.bn1(self.conv1(x))) 90 | out = self.layer1(out) 91 | out = self.layer2(out) 92 | out = self.layer3(out) 93 | out = self.layer4(out) 94 | out = F.avg_pool2d(out, 4) 95 | out = out.view(out.size(0), -1) 96 | self.embeddings = out 97 | out = self.linear(out) 98 | return out 99 | 100 | 101 | def ResNet18(num_channels): 102 | return ResNet(BasicBlock, [2,2,2,2], num_channels=num_channels) 103 | 104 | def ResNet34(): 105 | return ResNet(BasicBlock, [3,4,6,3]) 106 | 107 | def ResNet50(num_classes=10): 108 | return ResNet(Bottleneck, [3,4,6,3], num_classes=num_classes) 109 | 110 | def ResNet101(): 111 | return ResNet(Bottleneck, [3,4,23,3]) 112 | 113 | def ResNet152(): 114 | return ResNet(Bottleneck, [3,8,36,3]) 115 | 116 | 117 | def test(): 118 | net = ResNet18() 119 | y = net(torch.randn(1,3,32,32)) 120 | print(y.size()) 121 | 122 | # test() 123 | -------------------------------------------------------------------------------- /Training/utils.py: -------------------------------------------------------------------------------- 1 | '''Some helper functions for PyTorch, including: 2 | - get_mean_and_std: calculate the mean and std value of dataset. 3 | - msr_init: net parameter initialization. 4 | - progress_bar: progress bar mimic xlua.progress. 5 | ''' 6 | import os 7 | import sys 8 | import time 9 | import math 10 | 11 | import torch.nn as nn 12 | import torch.nn.init as init 13 | import torch 14 | 15 | class CustomDataset(torch.utils.data.Dataset): 16 | def __init__(self, x, y, tf, targets_origin=None): 17 | self.data = x#.to('cuda') 18 | self.targets = list(y)#.to('cuda') 19 | self.tf = tf 20 | if len(self.data.shape) == 3: 21 | self.data = self.data.unsqueeze(dim=1) 22 | 23 | self.targets_origin=targets_origin 24 | 25 | print(x.min()) 26 | print(x.max()) 27 | 28 | def __len__(self): 29 | return len(self.data) 30 | 31 | def __getitem__(self, idx): 32 | x = self.tf(self.data[idx]) 33 | y = self.targets[idx] 34 | 35 | return x, y 36 | 37 | 38 | 39 | 40 | def get_mean_and_std(dataset): 41 | '''Compute the mean and std value of dataset.''' 42 | dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2) 43 | mean = torch.zeros(3) 44 | std = torch.zeros(3) 45 | print('==> Computing mean and std..') 46 | for inputs, targets in dataloader: 47 | for i in range(3): 48 | mean[i] += inputs[:,i,:,:].mean() 49 | std[i] += inputs[:,i,:,:].std() 50 | mean.div_(len(dataset)) 51 | std.div_(len(dataset)) 52 | return mean, std 53 | 54 | def init_params(net): 55 | '''Init layer parameters.''' 56 | for m in net.modules(): 57 | if isinstance(m, nn.Conv2d): 58 | init.kaiming_normal(m.weight, mode='fan_out') 59 | if m.bias: 60 | init.constant(m.bias, 0) 61 | elif isinstance(m, nn.BatchNorm2d): 62 | init.constant(m.weight, 1) 63 | init.constant(m.bias, 0) 64 | elif isinstance(m, nn.Linear): 65 | init.normal(m.weight, std=1e-3) 66 | if m.bias: 67 | init.constant(m.bias, 0) 68 | 69 | 70 | _, term_width = os.popen('stty size', 'r').read().split() 71 | term_width = int(term_width) 72 | 73 | TOTAL_BAR_LENGTH = 65. 74 | last_time = time.time() 75 | begin_time = last_time 76 | def progress_bar(current, total, msg=None): 77 | global last_time, begin_time 78 | if current == 0: 79 | begin_time = time.time() # Reset for new bar. 80 | 81 | cur_len = int(TOTAL_BAR_LENGTH*current/total) 82 | rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1 83 | 84 | sys.stdout.write(' [') 85 | for i in range(cur_len): 86 | sys.stdout.write('=') 87 | sys.stdout.write('>') 88 | for i in range(rest_len): 89 | sys.stdout.write('.') 90 | sys.stdout.write(']') 91 | 92 | cur_time = time.time() 93 | step_time = cur_time - last_time 94 | last_time = cur_time 95 | tot_time = cur_time - begin_time 96 | 97 | L = [] 98 | L.append(' Step: %s' % format_time(step_time)) 99 | L.append(' | Tot: %s' % format_time(tot_time)) 100 | if msg: 101 | L.append(' | ' + msg) 102 | 103 | msg = ''.join(L) 104 | sys.stdout.write(msg) 105 | for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3): 106 | sys.stdout.write(' ') 107 | 108 | # Go back to the center of the bar. 109 | for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2): 110 | sys.stdout.write('\b') 111 | sys.stdout.write(' %d/%d ' % (current+1, total)) 112 | 113 | if current < total-1: 114 | sys.stdout.write('\r') 115 | else: 116 | sys.stdout.write('\n') 117 | sys.stdout.flush() 118 | 119 | def format_time(seconds): 120 | days = int(seconds / 3600/24) 121 | seconds = seconds - days*3600*24 122 | hours = int(seconds / 3600) 123 | seconds = seconds - hours*3600 124 | minutes = int(seconds / 60) 125 | seconds = seconds - minutes*60 126 | secondsf = int(seconds) 127 | seconds = seconds - secondsf 128 | millis = int(seconds*1000) 129 | 130 | f = '' 131 | i = 1 132 | if days > 0: 133 | f += str(days) + 'D' 134 | i += 1 135 | if hours > 0 and i <= 2: 136 | f += str(hours) + 'h' 137 | i += 1 138 | if minutes > 0 and i <= 2: 139 | f += str(minutes) + 'm' 140 | i += 1 141 | if secondsf > 0 and i <= 2: 142 | f += str(secondsf) + 's' 143 | i += 1 144 | if millis > 0 and i <= 2: 145 | f += str(millis) + 'ms' 146 | i += 1 147 | if f == '': 148 | f = '0ms' 149 | return f 150 | 151 | -------------------------------------------------------------------------------- /Inversion/models/pnasnet.py: -------------------------------------------------------------------------------- 1 | '''PNASNet in PyTorch. 2 | 3 | Paper: Progressive Neural Architecture Search 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class SepConv(nn.Module): 11 | '''Separable Convolution.''' 12 | def __init__(self, in_planes, out_planes, kernel_size, stride): 13 | super(SepConv, self).__init__() 14 | self.conv1 = nn.Conv2d(in_planes, out_planes, 15 | kernel_size, stride, 16 | padding=(kernel_size-1)//2, 17 | bias=False, groups=in_planes) 18 | self.bn1 = nn.BatchNorm2d(out_planes) 19 | 20 | def forward(self, x): 21 | return self.bn1(self.conv1(x)) 22 | 23 | 24 | class CellA(nn.Module): 25 | def __init__(self, in_planes, out_planes, stride=1): 26 | super(CellA, self).__init__() 27 | self.stride = stride 28 | self.sep_conv1 = SepConv(in_planes, out_planes, kernel_size=7, stride=stride) 29 | if stride==2: 30 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 31 | self.bn1 = nn.BatchNorm2d(out_planes) 32 | 33 | def forward(self, x): 34 | y1 = self.sep_conv1(x) 35 | y2 = F.max_pool2d(x, kernel_size=3, stride=self.stride, padding=1) 36 | if self.stride==2: 37 | y2 = self.bn1(self.conv1(y2)) 38 | return F.relu(y1+y2) 39 | 40 | class CellB(nn.Module): 41 | def __init__(self, in_planes, out_planes, stride=1): 42 | super(CellB, self).__init__() 43 | self.stride = stride 44 | # Left branch 45 | self.sep_conv1 = SepConv(in_planes, out_planes, kernel_size=7, stride=stride) 46 | self.sep_conv2 = SepConv(in_planes, out_planes, kernel_size=3, stride=stride) 47 | # Right branch 48 | self.sep_conv3 = SepConv(in_planes, out_planes, kernel_size=5, stride=stride) 49 | if stride==2: 50 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 51 | self.bn1 = nn.BatchNorm2d(out_planes) 52 | # Reduce channels 53 | self.conv2 = nn.Conv2d(2*out_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 54 | self.bn2 = nn.BatchNorm2d(out_planes) 55 | 56 | def forward(self, x): 57 | # Left branch 58 | y1 = self.sep_conv1(x) 59 | y2 = self.sep_conv2(x) 60 | # Right branch 61 | y3 = F.max_pool2d(x, kernel_size=3, stride=self.stride, padding=1) 62 | if self.stride==2: 63 | y3 = self.bn1(self.conv1(y3)) 64 | y4 = self.sep_conv3(x) 65 | # Concat & reduce channels 66 | b1 = F.relu(y1+y2) 67 | b2 = F.relu(y3+y4) 68 | y = torch.cat([b1,b2], 1) 69 | return F.relu(self.bn2(self.conv2(y))) 70 | 71 | class PNASNet(nn.Module): 72 | def __init__(self, cell_type, num_cells, num_planes): 73 | super(PNASNet, self).__init__() 74 | self.in_planes = num_planes 75 | self.cell_type = cell_type 76 | 77 | self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, stride=1, padding=1, bias=False) 78 | self.bn1 = nn.BatchNorm2d(num_planes) 79 | 80 | self.layer1 = self._make_layer(num_planes, num_cells=6) 81 | self.layer2 = self._downsample(num_planes*2) 82 | self.layer3 = self._make_layer(num_planes*2, num_cells=6) 83 | self.layer4 = self._downsample(num_planes*4) 84 | self.layer5 = self._make_layer(num_planes*4, num_cells=6) 85 | 86 | self.linear = nn.Linear(num_planes*4, 10) 87 | 88 | def _make_layer(self, planes, num_cells): 89 | layers = [] 90 | for _ in range(num_cells): 91 | layers.append(self.cell_type(self.in_planes, planes, stride=1)) 92 | self.in_planes = planes 93 | return nn.Sequential(*layers) 94 | 95 | def _downsample(self, planes): 96 | layer = self.cell_type(self.in_planes, planes, stride=2) 97 | self.in_planes = planes 98 | return layer 99 | 100 | def forward(self, x): 101 | out = F.relu(self.bn1(self.conv1(x))) 102 | out = self.layer1(out) 103 | out = self.layer2(out) 104 | out = self.layer3(out) 105 | out = self.layer4(out) 106 | out = self.layer5(out) 107 | out = F.avg_pool2d(out, 8) 108 | out = self.linear(out.view(out.size(0), -1)) 109 | return out 110 | 111 | 112 | def PNASNetA(): 113 | return PNASNet(CellA, num_cells=6, num_planes=44) 114 | 115 | def PNASNetB(): 116 | return PNASNet(CellB, num_cells=6, num_planes=32) 117 | 118 | 119 | def test(): 120 | net = PNASNetB() 121 | x = torch.randn(1,3,32,32) 122 | y = net(x) 123 | print(y) 124 | 125 | # test() 126 | -------------------------------------------------------------------------------- /Training/models/pnasnet.py: -------------------------------------------------------------------------------- 1 | '''PNASNet in PyTorch. 2 | 3 | Paper: Progressive Neural Architecture Search 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class SepConv(nn.Module): 11 | '''Separable Convolution.''' 12 | def __init__(self, in_planes, out_planes, kernel_size, stride): 13 | super(SepConv, self).__init__() 14 | self.conv1 = nn.Conv2d(in_planes, out_planes, 15 | kernel_size, stride, 16 | padding=(kernel_size-1)//2, 17 | bias=False, groups=in_planes) 18 | self.bn1 = nn.BatchNorm2d(out_planes) 19 | 20 | def forward(self, x): 21 | return self.bn1(self.conv1(x)) 22 | 23 | 24 | class CellA(nn.Module): 25 | def __init__(self, in_planes, out_planes, stride=1): 26 | super(CellA, self).__init__() 27 | self.stride = stride 28 | self.sep_conv1 = SepConv(in_planes, out_planes, kernel_size=7, stride=stride) 29 | if stride==2: 30 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 31 | self.bn1 = nn.BatchNorm2d(out_planes) 32 | 33 | def forward(self, x): 34 | y1 = self.sep_conv1(x) 35 | y2 = F.max_pool2d(x, kernel_size=3, stride=self.stride, padding=1) 36 | if self.stride==2: 37 | y2 = self.bn1(self.conv1(y2)) 38 | return F.relu(y1+y2) 39 | 40 | class CellB(nn.Module): 41 | def __init__(self, in_planes, out_planes, stride=1): 42 | super(CellB, self).__init__() 43 | self.stride = stride 44 | # Left branch 45 | self.sep_conv1 = SepConv(in_planes, out_planes, kernel_size=7, stride=stride) 46 | self.sep_conv2 = SepConv(in_planes, out_planes, kernel_size=3, stride=stride) 47 | # Right branch 48 | self.sep_conv3 = SepConv(in_planes, out_planes, kernel_size=5, stride=stride) 49 | if stride==2: 50 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 51 | self.bn1 = nn.BatchNorm2d(out_planes) 52 | # Reduce channels 53 | self.conv2 = nn.Conv2d(2*out_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 54 | self.bn2 = nn.BatchNorm2d(out_planes) 55 | 56 | def forward(self, x): 57 | # Left branch 58 | y1 = self.sep_conv1(x) 59 | y2 = self.sep_conv2(x) 60 | # Right branch 61 | y3 = F.max_pool2d(x, kernel_size=3, stride=self.stride, padding=1) 62 | if self.stride==2: 63 | y3 = self.bn1(self.conv1(y3)) 64 | y4 = self.sep_conv3(x) 65 | # Concat & reduce channels 66 | b1 = F.relu(y1+y2) 67 | b2 = F.relu(y3+y4) 68 | y = torch.cat([b1,b2], 1) 69 | return F.relu(self.bn2(self.conv2(y))) 70 | 71 | class PNASNet(nn.Module): 72 | def __init__(self, cell_type, num_cells, num_planes): 73 | super(PNASNet, self).__init__() 74 | self.in_planes = num_planes 75 | self.cell_type = cell_type 76 | 77 | self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, stride=1, padding=1, bias=False) 78 | self.bn1 = nn.BatchNorm2d(num_planes) 79 | 80 | self.layer1 = self._make_layer(num_planes, num_cells=6) 81 | self.layer2 = self._downsample(num_planes*2) 82 | self.layer3 = self._make_layer(num_planes*2, num_cells=6) 83 | self.layer4 = self._downsample(num_planes*4) 84 | self.layer5 = self._make_layer(num_planes*4, num_cells=6) 85 | 86 | self.linear = nn.Linear(num_planes*4, 10) 87 | 88 | def _make_layer(self, planes, num_cells): 89 | layers = [] 90 | for _ in range(num_cells): 91 | layers.append(self.cell_type(self.in_planes, planes, stride=1)) 92 | self.in_planes = planes 93 | return nn.Sequential(*layers) 94 | 95 | def _downsample(self, planes): 96 | layer = self.cell_type(self.in_planes, planes, stride=2) 97 | self.in_planes = planes 98 | return layer 99 | 100 | def forward(self, x): 101 | out = F.relu(self.bn1(self.conv1(x))) 102 | out = self.layer1(out) 103 | out = self.layer2(out) 104 | out = self.layer3(out) 105 | out = self.layer4(out) 106 | out = self.layer5(out) 107 | out = F.avg_pool2d(out, 8) 108 | out = self.linear(out.view(out.size(0), -1)) 109 | return out 110 | 111 | 112 | def PNASNetA(): 113 | return PNASNet(CellA, num_cells=6, num_planes=44) 114 | 115 | def PNASNetB(): 116 | return PNASNet(CellB, num_cells=6, num_planes=32) 117 | 118 | 119 | def test(): 120 | net = PNASNetB() 121 | x = torch.randn(1,3,32,32) 122 | y = net(x) 123 | print(y) 124 | 125 | # test() 126 | -------------------------------------------------------------------------------- /Unlearning/models/pnasnet.py: -------------------------------------------------------------------------------- 1 | '''PNASNet in PyTorch. 2 | 3 | Paper: Progressive Neural Architecture Search 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class SepConv(nn.Module): 11 | '''Separable Convolution.''' 12 | def __init__(self, in_planes, out_planes, kernel_size, stride): 13 | super(SepConv, self).__init__() 14 | self.conv1 = nn.Conv2d(in_planes, out_planes, 15 | kernel_size, stride, 16 | padding=(kernel_size-1)//2, 17 | bias=False, groups=in_planes) 18 | self.bn1 = nn.BatchNorm2d(out_planes) 19 | 20 | def forward(self, x): 21 | return self.bn1(self.conv1(x)) 22 | 23 | 24 | class CellA(nn.Module): 25 | def __init__(self, in_planes, out_planes, stride=1): 26 | super(CellA, self).__init__() 27 | self.stride = stride 28 | self.sep_conv1 = SepConv(in_planes, out_planes, kernel_size=7, stride=stride) 29 | if stride==2: 30 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 31 | self.bn1 = nn.BatchNorm2d(out_planes) 32 | 33 | def forward(self, x): 34 | y1 = self.sep_conv1(x) 35 | y2 = F.max_pool2d(x, kernel_size=3, stride=self.stride, padding=1) 36 | if self.stride==2: 37 | y2 = self.bn1(self.conv1(y2)) 38 | return F.relu(y1+y2) 39 | 40 | class CellB(nn.Module): 41 | def __init__(self, in_planes, out_planes, stride=1): 42 | super(CellB, self).__init__() 43 | self.stride = stride 44 | # Left branch 45 | self.sep_conv1 = SepConv(in_planes, out_planes, kernel_size=7, stride=stride) 46 | self.sep_conv2 = SepConv(in_planes, out_planes, kernel_size=3, stride=stride) 47 | # Right branch 48 | self.sep_conv3 = SepConv(in_planes, out_planes, kernel_size=5, stride=stride) 49 | if stride==2: 50 | self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 51 | self.bn1 = nn.BatchNorm2d(out_planes) 52 | # Reduce channels 53 | self.conv2 = nn.Conv2d(2*out_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False) 54 | self.bn2 = nn.BatchNorm2d(out_planes) 55 | 56 | def forward(self, x): 57 | # Left branch 58 | y1 = self.sep_conv1(x) 59 | y2 = self.sep_conv2(x) 60 | # Right branch 61 | y3 = F.max_pool2d(x, kernel_size=3, stride=self.stride, padding=1) 62 | if self.stride==2: 63 | y3 = self.bn1(self.conv1(y3)) 64 | y4 = self.sep_conv3(x) 65 | # Concat & reduce channels 66 | b1 = F.relu(y1+y2) 67 | b2 = F.relu(y3+y4) 68 | y = torch.cat([b1,b2], 1) 69 | return F.relu(self.bn2(self.conv2(y))) 70 | 71 | class PNASNet(nn.Module): 72 | def __init__(self, cell_type, num_cells, num_planes): 73 | super(PNASNet, self).__init__() 74 | self.in_planes = num_planes 75 | self.cell_type = cell_type 76 | 77 | self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, stride=1, padding=1, bias=False) 78 | self.bn1 = nn.BatchNorm2d(num_planes) 79 | 80 | self.layer1 = self._make_layer(num_planes, num_cells=6) 81 | self.layer2 = self._downsample(num_planes*2) 82 | self.layer3 = self._make_layer(num_planes*2, num_cells=6) 83 | self.layer4 = self._downsample(num_planes*4) 84 | self.layer5 = self._make_layer(num_planes*4, num_cells=6) 85 | 86 | self.linear = nn.Linear(num_planes*4, 10) 87 | 88 | def _make_layer(self, planes, num_cells): 89 | layers = [] 90 | for _ in range(num_cells): 91 | layers.append(self.cell_type(self.in_planes, planes, stride=1)) 92 | self.in_planes = planes 93 | return nn.Sequential(*layers) 94 | 95 | def _downsample(self, planes): 96 | layer = self.cell_type(self.in_planes, planes, stride=2) 97 | self.in_planes = planes 98 | return layer 99 | 100 | def forward(self, x): 101 | out = F.relu(self.bn1(self.conv1(x))) 102 | out = self.layer1(out) 103 | out = self.layer2(out) 104 | out = self.layer3(out) 105 | out = self.layer4(out) 106 | out = self.layer5(out) 107 | out = F.avg_pool2d(out, 8) 108 | out = self.linear(out.view(out.size(0), -1)) 109 | return out 110 | 111 | 112 | def PNASNetA(): 113 | return PNASNet(CellA, num_cells=6, num_planes=44) 114 | 115 | def PNASNetB(): 116 | return PNASNet(CellB, num_cells=6, num_planes=32) 117 | 118 | 119 | def test(): 120 | net = PNASNetB() 121 | x = torch.randn(1,3,32,32) 122 | y = net(x) 123 | print(y) 124 | 125 | # test() 126 | -------------------------------------------------------------------------------- /Unlearning/utils.py: -------------------------------------------------------------------------------- 1 | '''Some helper functions for PyTorch, including: 2 | - get_mean_and_std: calculate the mean and std value of dataset. 3 | - msr_init: net parameter initialization. 4 | - progress_bar: progress bar mimic xlua.progress. 5 | ''' 6 | import os 7 | import sys 8 | import time 9 | import torch 10 | 11 | import torch.nn as nn 12 | import torch.nn.init as init 13 | from torch.utils.data import Dataset 14 | import torchvision 15 | import torchvision.transforms as transforms 16 | 17 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 18 | 19 | class CustomDataset(Dataset): 20 | def __init__(self, x, y, tf): 21 | self.data = x.detach().cpu() 22 | self.targets = y.detach().cpu() 23 | self.tf = tf 24 | if len(self.data.shape) == 3: 25 | self.data = self.data.unsqueeze(dim=1) 26 | 27 | def __len__(self): 28 | return len(self.data) 29 | 30 | def __getitem__(self, idx): 31 | x = self.tf(self.data[idx]) 32 | y = self.targets[idx] 33 | 34 | return x, y 35 | 36 | 37 | 38 | def softXEnt(inputs, targets): 39 | if len(targets.shape) < 2: 40 | num_classes = inputs.shape[1] 41 | targets = torch.nn.functional.one_hot(targets, num_classes=num_classes).float() 42 | 43 | logprobs = torch.nn.functional.log_softmax(inputs, dim = 1) 44 | 45 | ent = targets * logprobs 46 | 47 | ent = torch.where(ent<10, -ent, torch.tensor(10.).float().to(device)) 48 | 49 | return ent.sum() / inputs.shape[0] 50 | 51 | 52 | 53 | class Normalize(nn.Module) : 54 | def __init__(self, mean, std) : 55 | super(Normalize, self).__init__() 56 | self.register_buffer('mean', torch.Tensor(mean)) 57 | self.register_buffer('std', torch.Tensor(std)) 58 | 59 | def forward(self, inputs): 60 | mean = self.mean.reshape(1, 3, 1, 1) 61 | std = self.std.reshape(1, 3, 1, 1) 62 | return (inputs - mean) / std 63 | 64 | 65 | class feature_extractor(nn.Module): 66 | def __init__(self): 67 | super(feature_extractor, self).__init__() 68 | 69 | def forward(self, model, image): 70 | model.eval() 71 | 72 | model(image) 73 | embeddings = model.embeddings 74 | 75 | return embeddings 76 | 77 | 78 | def get_mean_and_std(dataset): 79 | '''Compute the mean and std value of dataset.''' 80 | dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2) 81 | mean = torch.zeros(3) 82 | std = torch.zeros(3) 83 | print('==> Computing mean and std..') 84 | for inputs, targets in dataloader: 85 | for i in range(3): 86 | mean[i] += inputs[:,i,:,:].mean() 87 | std[i] += inputs[:,i,:,:].std() 88 | mean.div_(len(dataset)) 89 | std.div_(len(dataset)) 90 | return mean, std 91 | 92 | def init_params(net): 93 | '''Init layer parameters.''' 94 | for m in net.modules(): 95 | if isinstance(m, nn.Conv2d): 96 | init.kaiming_normal(m.weight, mode='fan_out') 97 | if m.bias: 98 | init.constant(m.bias, 0) 99 | elif isinstance(m, nn.BatchNorm2d): 100 | init.constant(m.weight, 1) 101 | init.constant(m.bias, 0) 102 | elif isinstance(m, nn.Linear): 103 | init.normal(m.weight, std=1e-3) 104 | if m.bias: 105 | init.constant(m.bias, 0) 106 | 107 | 108 | _, term_width = os.popen('stty size', 'r').read().split() 109 | term_width = int(term_width) 110 | 111 | TOTAL_BAR_LENGTH = 65. 112 | last_time = time.time() 113 | begin_time = last_time 114 | def progress_bar(current, total, msg=None): 115 | global last_time, begin_time 116 | if current == 0: 117 | begin_time = time.time() # Reset for new bar. 118 | 119 | cur_len = int(TOTAL_BAR_LENGTH*current/total) 120 | rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1 121 | 122 | sys.stdout.write(' [') 123 | for i in range(cur_len): 124 | sys.stdout.write('=') 125 | sys.stdout.write('>') 126 | for i in range(rest_len): 127 | sys.stdout.write('.') 128 | sys.stdout.write(']') 129 | 130 | cur_time = time.time() 131 | step_time = cur_time - last_time 132 | last_time = cur_time 133 | tot_time = cur_time - begin_time 134 | 135 | L = [] 136 | L.append(' Step: %s' % format_time(step_time)) 137 | L.append(' | Tot: %s' % format_time(tot_time)) 138 | if msg: 139 | L.append(' | ' + msg) 140 | 141 | msg = ''.join(L) 142 | sys.stdout.write(msg) 143 | for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3): 144 | sys.stdout.write(' ') 145 | 146 | # Go back to the center of the bar. 147 | for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2): 148 | sys.stdout.write('\b') 149 | sys.stdout.write(' %d/%d ' % (current+1, total)) 150 | 151 | if current < total-1: 152 | sys.stdout.write('\r') 153 | else: 154 | sys.stdout.write('\n') 155 | sys.stdout.flush() 156 | 157 | def format_time(seconds): 158 | days = int(seconds / 3600/24) 159 | seconds = seconds - days*3600*24 160 | hours = int(seconds / 3600) 161 | seconds = seconds - hours*3600 162 | minutes = int(seconds / 60) 163 | seconds = seconds - minutes*60 164 | secondsf = int(seconds) 165 | seconds = seconds - secondsf 166 | millis = int(seconds*1000) 167 | 168 | f = '' 169 | i = 1 170 | if days > 0: 171 | f += str(days) + 'D' 172 | i += 1 173 | if hours > 0 and i <= 2: 174 | f += str(hours) + 'h' 175 | i += 1 176 | if minutes > 0 and i <= 2: 177 | f += str(minutes) + 'm' 178 | i += 1 179 | if secondsf > 0 and i <= 2: 180 | f += str(secondsf) + 's' 181 | i += 1 182 | if millis > 0 and i <= 2: 183 | f += str(millis) + 'ms' 184 | i += 1 185 | if f == '': 186 | f = '0ms' 187 | return f 188 | 189 | -------------------------------------------------------------------------------- /Inversion/models/shufflenetv2.py: -------------------------------------------------------------------------------- 1 | '''ShuffleNetV2 in PyTorch. 2 | 3 | See the paper "ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design" for more details. 4 | ''' 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class ShuffleBlock(nn.Module): 11 | def __init__(self, groups=2): 12 | super(ShuffleBlock, self).__init__() 13 | self.groups = groups 14 | 15 | def forward(self, x): 16 | '''Channel shuffle: [N,C,H,W] -> [N,g,C/g,H,W] -> [N,C/g,g,H,w] -> [N,C,H,W]''' 17 | N, C, H, W = x.size() 18 | g = self.groups 19 | return x.view(N, g, C//g, H, W).permute(0, 2, 1, 3, 4).reshape(N, C, H, W) 20 | 21 | 22 | class SplitBlock(nn.Module): 23 | def __init__(self, ratio): 24 | super(SplitBlock, self).__init__() 25 | self.ratio = ratio 26 | 27 | def forward(self, x): 28 | c = int(x.size(1) * self.ratio) 29 | return x[:, :c, :, :], x[:, c:, :, :] 30 | 31 | 32 | class BasicBlock(nn.Module): 33 | def __init__(self, in_channels, split_ratio=0.5): 34 | super(BasicBlock, self).__init__() 35 | self.split = SplitBlock(split_ratio) 36 | in_channels = int(in_channels * split_ratio) 37 | self.conv1 = nn.Conv2d(in_channels, in_channels, 38 | kernel_size=1, bias=False) 39 | self.bn1 = nn.BatchNorm2d(in_channels) 40 | self.conv2 = nn.Conv2d(in_channels, in_channels, 41 | kernel_size=3, stride=1, padding=1, groups=in_channels, bias=False) 42 | self.bn2 = nn.BatchNorm2d(in_channels) 43 | self.conv3 = nn.Conv2d(in_channels, in_channels, 44 | kernel_size=1, bias=False) 45 | self.bn3 = nn.BatchNorm2d(in_channels) 46 | self.shuffle = ShuffleBlock() 47 | 48 | def forward(self, x): 49 | x1, x2 = self.split(x) 50 | out = F.relu(self.bn1(self.conv1(x2))) 51 | out = self.bn2(self.conv2(out)) 52 | out = F.relu(self.bn3(self.conv3(out))) 53 | out = torch.cat([x1, out], 1) 54 | out = self.shuffle(out) 55 | return out 56 | 57 | 58 | class DownBlock(nn.Module): 59 | def __init__(self, in_channels, out_channels): 60 | super(DownBlock, self).__init__() 61 | mid_channels = out_channels // 2 62 | # left 63 | self.conv1 = nn.Conv2d(in_channels, in_channels, 64 | kernel_size=3, stride=2, padding=1, groups=in_channels, bias=False) 65 | self.bn1 = nn.BatchNorm2d(in_channels) 66 | self.conv2 = nn.Conv2d(in_channels, mid_channels, 67 | kernel_size=1, bias=False) 68 | self.bn2 = nn.BatchNorm2d(mid_channels) 69 | # right 70 | self.conv3 = nn.Conv2d(in_channels, mid_channels, 71 | kernel_size=1, bias=False) 72 | self.bn3 = nn.BatchNorm2d(mid_channels) 73 | self.conv4 = nn.Conv2d(mid_channels, mid_channels, 74 | kernel_size=3, stride=2, padding=1, groups=mid_channels, bias=False) 75 | self.bn4 = nn.BatchNorm2d(mid_channels) 76 | self.conv5 = nn.Conv2d(mid_channels, mid_channels, 77 | kernel_size=1, bias=False) 78 | self.bn5 = nn.BatchNorm2d(mid_channels) 79 | 80 | self.shuffle = ShuffleBlock() 81 | 82 | def forward(self, x): 83 | # left 84 | out1 = self.bn1(self.conv1(x)) 85 | out1 = F.relu(self.bn2(self.conv2(out1))) 86 | # right 87 | out2 = F.relu(self.bn3(self.conv3(x))) 88 | out2 = self.bn4(self.conv4(out2)) 89 | out2 = F.relu(self.bn5(self.conv5(out2))) 90 | # concat 91 | out = torch.cat([out1, out2], 1) 92 | out = self.shuffle(out) 93 | return out 94 | 95 | 96 | class ShuffleNetV2(nn.Module): 97 | def __init__(self, net_size): 98 | super(ShuffleNetV2, self).__init__() 99 | out_channels = configs[net_size]['out_channels'] 100 | num_blocks = configs[net_size]['num_blocks'] 101 | 102 | self.conv1 = nn.Conv2d(3, 24, kernel_size=3, 103 | stride=1, padding=1, bias=False) 104 | self.bn1 = nn.BatchNorm2d(24) 105 | self.in_channels = 24 106 | self.layer1 = self._make_layer(out_channels[0], num_blocks[0]) 107 | self.layer2 = self._make_layer(out_channels[1], num_blocks[1]) 108 | self.layer3 = self._make_layer(out_channels[2], num_blocks[2]) 109 | self.conv2 = nn.Conv2d(out_channels[2], out_channels[3], 110 | kernel_size=1, stride=1, padding=0, bias=False) 111 | self.bn2 = nn.BatchNorm2d(out_channels[3]) 112 | self.linear = nn.Linear(out_channels[3], 10) 113 | 114 | def _make_layer(self, out_channels, num_blocks): 115 | layers = [DownBlock(self.in_channels, out_channels)] 116 | for i in range(num_blocks): 117 | layers.append(BasicBlock(out_channels)) 118 | self.in_channels = out_channels 119 | return nn.Sequential(*layers) 120 | 121 | def forward(self, x): 122 | out = F.relu(self.bn1(self.conv1(x))) 123 | # out = F.max_pool2d(out, 3, stride=2, padding=1) 124 | out = self.layer1(out) 125 | out = self.layer2(out) 126 | out = self.layer3(out) 127 | out = F.relu(self.bn2(self.conv2(out))) 128 | out = F.avg_pool2d(out, 4) 129 | out = out.view(out.size(0), -1) 130 | out = self.linear(out) 131 | return out 132 | 133 | 134 | configs = { 135 | 0.5: { 136 | 'out_channels': (48, 96, 192, 1024), 137 | 'num_blocks': (3, 7, 3) 138 | }, 139 | 140 | 1: { 141 | 'out_channels': (116, 232, 464, 1024), 142 | 'num_blocks': (3, 7, 3) 143 | }, 144 | 1.5: { 145 | 'out_channels': (176, 352, 704, 1024), 146 | 'num_blocks': (3, 7, 3) 147 | }, 148 | 2: { 149 | 'out_channels': (224, 488, 976, 2048), 150 | 'num_blocks': (3, 7, 3) 151 | } 152 | } 153 | 154 | 155 | def test(): 156 | net = ShuffleNetV2(net_size=0.5) 157 | x = torch.randn(3, 3, 32, 32) 158 | y = net(x) 159 | print(y.shape) 160 | 161 | 162 | # test() 163 | --------------------------------------------------------------------------------