├── .gitignore ├── LICENSE ├── README.md ├── scripts ├── run_cross_entropy.sh ├── run_joint_confidence.sh └── test.sh └── src ├── calculate_log.py ├── data_loader.py ├── models ├── __init__.py ├── gan.py └── vgg.py ├── run_cross_entropy.py ├── run_joint_confidence.py └── test_detection.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Google App Engine generated folder 2 | appengine-generated/ 3 | *.pyc 4 | data/* 5 | results/* 6 | test/* 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 alinlab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Training Confidence-Calibrated Classifier for Detecting Out-of-Distribution Samples 2 | 3 | This project is for the paper "[Training Confidence-Calibrated Classifier for Detecting Out-of-Distribution Samples](https://arxiv.org/abs/1711.09325)". Some codes are from [odin-pytorch](https://github.com/ShiyuLiang/odin-pytorch). 4 | 5 | ## Preliminaries 6 | 7 | It is tested under Ubuntu Linux 16.04.1 and Python 2.7 environment, and requries Pytorch package to be installed: 8 | 9 | * [Pytorch](http://pytorch.org/): Only GPU version is available. 10 | 11 | ### Downloading Out-of-Distribtion Datasets 12 | We use download links of two out-of-distributin datasets from [odin-pytorch](https://github.com/ShiyuLiang/odin-pytorch): 13 | 14 | * **[Tiny-ImageNet (resize)](https://www.dropbox.com/s/kp3my3412u5k9rl/Imagenet_resize.tar.gz)** 15 | * **[LSUN (resize)](https://www.dropbox.com/s/moqh2wh8696c3yl/LSUN_resize.tar.gz)** 16 | 17 | ## Training scripts 18 | 19 | * [`run_cross_entropy.sh`](./scripts/run_cross_entropy.sh): train the models using standard cross entropy loss. 20 | * [`run_joint_confidence.sh`](./scripts/run_joint_confidence.sh): train the models using joint confidence loss. 21 | 22 | ## Test scripts 23 | 24 | * [`test.sh`](./scripts/test.sh) --dataset --out_dataset --pre_trained_net \ 25 | --dataset = name of in-distribution (svhn or cifar10) \ 26 | --out_dataset = name of out-of-distribution (svhn, cifar10, lsun or imagenet) \ 27 | --pre_trained_net = path to pre_trained_net 28 | -------------------------------------------------------------------------------- /scripts/run_cross_entropy.sh: -------------------------------------------------------------------------------- 1 | # baseline 2 | export save=./results/cross_entropy/${RANDOM}/ 3 | mkdir -p $save 4 | python ./src/run_cross_entropy.py --outf $save --dataroot ./data 2>&1 | tee $save/log.txt 5 | -------------------------------------------------------------------------------- /scripts/run_joint_confidence.sh: -------------------------------------------------------------------------------- 1 | # baseline 2 | export save=./results/joint_confidence_loss/${RANDOM}/ 3 | mkdir -p $save 4 | python ./src/run_joint_confidence.py --outf $save --dataroot ./data 2>&1 | tee $save/log.txt 5 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | # baseline 2 | export save=./test/${RANDOM}/ 3 | mkdir -p $save 4 | python ./src/test_detection.py --outf $save --dataset $1 --out_dataset $2 --pre_trained_net $3 --dataroot ./data 2>&1 | tee $save/log.txt 5 | -------------------------------------------------------------------------------- /src/calculate_log.py: -------------------------------------------------------------------------------- 1 | ## original code is from https://github.com/ShiyuLiang/odin-pytorch/blob/master/code/calMetric.py 2 | ## Modeified by Kimin Lee 3 | from __future__ import print_function 4 | import torch 5 | from torch.autograd import Variable 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import numpy as np 9 | import torch.optim as optim 10 | import torchvision 11 | import torchvision.transforms as transforms 12 | import numpy as np 13 | import time 14 | from scipy import misc 15 | 16 | def tpr95(dir_name): 17 | #calculate the falsepositive error when tpr is 95% 18 | cifar = np.loadtxt('%s/confidence_Base_In.txt'%dir_name, delimiter=',') 19 | other = np.loadtxt('%s/confidence_Base_Out.txt'%dir_name, delimiter=',') 20 | Y1 = other 21 | X1 = cifar 22 | end = np.max([np.max(X1), np.max(Y1)]) 23 | start = np.min([np.min(X1),np.min(Y1)]) 24 | gap = (end- start)/200000 # precision:200000 25 | 26 | total = 0.0 27 | fpr = 0.0 28 | for delta in np.arange(start, end, gap): 29 | tpr = np.sum(np.sum(X1 >= delta)) / np.float(len(X1)) 30 | error2 = np.sum(np.sum(Y1 > delta)) / np.float(len(Y1)) 31 | if tpr <= 0.96 and tpr >= 0.94: 32 | fpr += error2 33 | total += 1 34 | if total == 0: 35 | print('corner case') 36 | fprBase = 1 37 | else: 38 | fprBase = fpr/total 39 | 40 | return fprBase 41 | 42 | 43 | def auroc(dir_name): 44 | #calculate the AUROC 45 | f1 = open('%s/Update_Base_ROC_tpr.txt'%dir_name, 'w') 46 | f2 = open('%s/Update_Base_ROC_fpr.txt'%dir_name, 'w') 47 | 48 | cifar = np.loadtxt('%s/confidence_Base_In.txt'%dir_name, delimiter=',') 49 | other = np.loadtxt('%s/confidence_Base_Out.txt'%dir_name, delimiter=',') 50 | Y1 = other 51 | X1 = cifar 52 | end = np.max([np.max(X1), np.max(Y1)]) 53 | start = np.min([np.min(X1),np.min(Y1)]) 54 | gap = (end- start)/200000 55 | 56 | aurocBase = 0.0 57 | fprTemp = 1.0 58 | for delta in np.arange(start, end, gap): 59 | tpr = np.sum(np.sum(X1 >= delta)) / np.float(len(X1)) 60 | fpr = np.sum(np.sum(Y1 > delta)) / np.float(len(Y1)) 61 | f1.write("{}\n".format(tpr)) 62 | f2.write("{}\n".format(fpr)) 63 | aurocBase += (-fpr+fprTemp)*tpr 64 | fprTemp = fpr 65 | 66 | return aurocBase 67 | 68 | def auprIn(dir_name): 69 | #calculate the AUPR 70 | 71 | cifar = np.loadtxt('%s/confidence_Base_In.txt'%dir_name, delimiter=',') 72 | other = np.loadtxt('%s/confidence_Base_Out.txt'%dir_name, delimiter=',') 73 | precisionVec = [] 74 | recallVec = [] 75 | Y1 = other 76 | X1 = cifar 77 | end = np.max([np.max(X1), np.max(Y1)]) 78 | start = np.min([np.min(X1),np.min(Y1)]) 79 | gap = (end- start)/200000 80 | 81 | auprBase = 0.0 82 | recallTemp = 1.0 83 | for delta in np.arange(start, end, gap): 84 | tp = np.sum(np.sum(X1 >= delta)) / np.float(len(X1)) 85 | fp = np.sum(np.sum(Y1 >= delta)) / np.float(len(Y1)) 86 | if tp + fp == 0: continue 87 | precision = tp / (tp + fp) 88 | recall = tp 89 | precisionVec.append(precision) 90 | recallVec.append(recall) 91 | auprBase += (recallTemp-recall)*precision 92 | recallTemp = recall 93 | auprBase += recall * precision 94 | 95 | return auprBase 96 | 97 | def auprOut(dir_name): 98 | #calculate the AUPR 99 | cifar = np.loadtxt('%s/confidence_Base_In.txt'%dir_name, delimiter=',') 100 | other = np.loadtxt('%s/confidence_Base_Out.txt'%dir_name, delimiter=',') 101 | Y1 = other 102 | X1 = cifar 103 | end = np.max([np.max(X1), np.max(Y1)]) 104 | start = np.min([np.min(X1),np.min(Y1)]) 105 | gap = (end- start)/200000 106 | 107 | auprBase = 0.0 108 | recallTemp = 1.0 109 | for delta in np.arange(end, start, -gap): 110 | fp = np.sum(np.sum(X1 < delta)) / np.float(len(X1)) 111 | tp = np.sum(np.sum(Y1 < delta)) / np.float(len(Y1)) 112 | if tp + fp == 0: break 113 | precision = tp / (tp + fp) 114 | recall = tp 115 | auprBase += (recallTemp-recall)*precision 116 | recallTemp = recall 117 | auprBase += recall * precision 118 | 119 | return auprBase 120 | 121 | def detection(dir_name): 122 | #calculate the minimum detection error 123 | 124 | cifar = np.loadtxt('%s/confidence_Base_In.txt'%dir_name, delimiter=',') 125 | other = np.loadtxt('%s/confidence_Base_Out.txt'%dir_name, delimiter=',') 126 | Y1 = other 127 | X1 = cifar 128 | end = np.max([np.max(X1), np.max(Y1)]) 129 | start = np.min([np.min(X1),np.min(Y1)]) 130 | gap = (end- start)/200000 131 | 132 | errorBase = 1.0 133 | for delta in np.arange(start, end, gap): 134 | tpr = np.sum(np.sum(X1 < delta)) / np.float(len(X1)) 135 | error2 = np.sum(np.sum(Y1 > delta)) / np.float(len(Y1)) 136 | errorBase = np.minimum(errorBase, (tpr+error2)/2.0) 137 | 138 | return errorBase 139 | 140 | def metric(dir_name): 141 | print("{:>34}".format("Performance of Baseline detector")) 142 | fprBase = tpr95(dir_name) 143 | print("{:20}{:13.3f}%".format("TNR at TPR 95%:",(1-fprBase)*100)) 144 | aurocBase = auroc(dir_name) 145 | print("{:20}{:13.3f}%".format("AUROC:",aurocBase*100)) 146 | errorBase = detection(dir_name) 147 | print("{:20}{:13.3f}%".format("Detection acc:",(1-errorBase)*100)) 148 | auprinBase = auprIn(dir_name) 149 | print("{:20}{:13.3f}%".format("AUPR In:",auprinBase*100)) 150 | auproutBase = auprOut(dir_name) 151 | print("{:20}{:13.3f}%".format("AUPR Out:",auproutBase*100)) 152 | -------------------------------------------------------------------------------- /src/data_loader.py: -------------------------------------------------------------------------------- 1 | # original code is from https://github.com/aaron-xichen/pytorch-playground 2 | # modified by Kimin Lee 3 | import torch 4 | from torchvision import datasets, transforms 5 | from torch.utils.data import DataLoader 6 | import os 7 | import numpy.random as nr 8 | import numpy as np 9 | from torch.utils.data.sampler import SubsetRandomSampler 10 | 11 | def getSVHN(batch_size, img_size=32, data_root='/tmp/public_dataset/pytorch', train=True, val=True, **kwargs): 12 | data_root = os.path.expanduser(os.path.join(data_root, 'svhn-data')) 13 | num_workers = kwargs.setdefault('num_workers', 1) 14 | kwargs.pop('input_size', None) 15 | print("Building SVHN data loader with {} workers".format(num_workers)) 16 | 17 | def target_transform(target): 18 | new_target = target - 1 19 | if new_target == -1: 20 | new_target = 9 21 | return new_target 22 | 23 | ds = [] 24 | if train: 25 | train_loader = torch.utils.data.DataLoader( 26 | datasets.SVHN( 27 | root=data_root, split='train', download=True, 28 | transform=transforms.Compose([ 29 | transforms.Scale(img_size), 30 | transforms.ToTensor(), 31 | ]), 32 | target_transform=target_transform, 33 | ), 34 | batch_size=batch_size, shuffle=True, **kwargs) 35 | ds.append(train_loader) 36 | 37 | if val: 38 | test_loader = torch.utils.data.DataLoader( 39 | datasets.SVHN( 40 | root=data_root, split='test', download=True, 41 | transform=transforms.Compose([ 42 | transforms.Scale(img_size), 43 | transforms.ToTensor(), 44 | ]), 45 | target_transform=target_transform 46 | ), 47 | batch_size=batch_size, shuffle=False, **kwargs) 48 | ds.append(test_loader) 49 | ds = ds[0] if len(ds) == 1 else ds 50 | return ds 51 | 52 | def getCIFAR10(batch_size, img_size=32, data_root='/tmp/public_dataset/pytorch', train=True, val=True, **kwargs): 53 | data_root = os.path.expanduser(os.path.join(data_root, 'cifar10-data')) 54 | num_workers = kwargs.setdefault('num_workers', 1) 55 | kwargs.pop('input_size', None) 56 | print("Building CIFAR-10 data loader with {} workers".format(num_workers)) 57 | ds = [] 58 | if train: 59 | train_loader = torch.utils.data.DataLoader( 60 | datasets.CIFAR10( 61 | root=data_root, train=True, download=True, 62 | transform=transforms.Compose([ 63 | transforms.Scale(img_size), 64 | transforms.ToTensor(), 65 | ])), 66 | batch_size=batch_size, shuffle=True, **kwargs) 67 | ds.append(train_loader) 68 | if val: 69 | test_loader = torch.utils.data.DataLoader( 70 | datasets.CIFAR10( 71 | root=data_root, train=False, download=True, 72 | transform=transforms.Compose([ 73 | transforms.Scale(img_size), 74 | transforms.ToTensor(), 75 | ])), 76 | batch_size=batch_size, shuffle=False, **kwargs) 77 | ds.append(test_loader) 78 | ds = ds[0] if len(ds) == 1 else ds 79 | return ds 80 | 81 | 82 | def getTargetDataSet(data_type, batch_size, imageSize, dataroot): 83 | if data_type == 'cifar10': 84 | train_loader, test_loader = getCIFAR10(batch_size=batch_size, img_size=imageSize, data_root=dataroot, num_workers=1) 85 | elif data_type == 'svhn': 86 | train_loader, test_loader = getSVHN(batch_size=batch_size, img_size=imageSize, data_root=dataroot, num_workers=1) 87 | 88 | return train_loader, test_loader 89 | 90 | def getNonTargetDataSet(data_type, batch_size, imageSize, dataroot): 91 | if data_type == 'cifar10': 92 | _, test_loader = getCIFAR10(batch_size=batch_size, img_size=imageSize, data_root=dataroot, num_workers=1) 93 | elif data_type == 'svhn': 94 | _, test_loader = getSVHN(batch_size=batch_size, img_size=imageSize, data_root=dataroot, num_workers=1) 95 | elif data_type == 'imagenet': 96 | testsetout = datasets.ImageFolder(dataroot+"/Imagenet_resize", transform=transforms.Compose([transforms.Scale(imageSize),transforms.ToTensor()])) 97 | test_loader = torch.utils.data.DataLoader(testsetout, batch_size=batch_size, shuffle=False, num_workers=1) 98 | elif data_type == 'lsun': 99 | testsetout = datasets.ImageFolder(dataroot+"/LSUN_resize", transform=transforms.Compose([transforms.Scale(imageSize),transforms.ToTensor()])) 100 | test_loader = torch.utils.data.DataLoader(testsetout, batch_size=batch_size, shuffle=False, num_workers=1) 101 | 102 | return test_loader 103 | -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- 1 | ## Models from 2 | 3 | """The models subpackage contains definitions for the following model 4 | architectures: 5 | - `AlexNet`_ 6 | - `VGG`_ 7 | - `ResNet`_ 8 | - `SqueezeNet`_ 9 | - `DenseNet`_ 10 | You can construct a model with random weights by calling its constructor: 11 | .. code:: python 12 | import torchvision.models as models 13 | resnet18 = models.resnet18() 14 | alexnet = models.alexnet() 15 | squeezenet = models.squeezenet1_0() 16 | densenet = models.densenet_161() 17 | We provide pre-trained models for the ResNet variants and AlexNet, using the 18 | PyTorch :mod:`torch.utils.model_zoo`. These can constructed by passing 19 | ``pretrained=True``: 20 | .. code:: python 21 | import torchvision.models as models 22 | resnet18 = models.resnet18(pretrained=True) 23 | alexnet = models.alexnet(pretrained=True) 24 | All pre-trained models expect input images normalized in the same way, 25 | i.e. mini-batches of 3-channel RGB images of shape (3 x H x W), 26 | where H and W are expected to be atleast 224. 27 | The images have to be loaded in to a range of [0, 1] and then normalized 28 | using ``mean = [0.485, 0.456, 0.406]`` and ``std = [0.229, 0.224, 0.225]``. 29 | You can use the following transform to normalize:: 30 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 31 | std=[0.229, 0.224, 0.225]) 32 | An example of such normalization can be found in the imagenet example 33 | `here `_ 34 | ImageNet 1-crop error rates (224x224) 35 | ================================ ============= ============= 36 | Network Top-1 error Top-5 error 37 | ================================ ============= ============= 38 | ResNet-18 30.24 10.92 39 | ResNet-34 26.70 8.58 40 | ResNet-50 23.85 7.13 41 | ResNet-101 22.63 6.44 42 | ResNet-152 21.69 5.94 43 | Inception v3 22.55 6.44 44 | AlexNet 43.45 20.91 45 | VGG-11 30.98 11.37 46 | VGG-13 30.07 10.75 47 | VGG-16 28.41 9.62 48 | VGG-19 27.62 9.12 49 | VGG-11 with batch normalization 29.62 10.19 50 | VGG-13 with batch normalization 28.45 9.63 51 | VGG-16 with batch normalization 26.63 8.50 52 | VGG-19 with batch normalization 25.76 8.15 53 | SqueezeNet 1.0 41.90 19.58 54 | SqueezeNet 1.1 41.81 19.38 55 | Densenet-121 25.35 7.83 56 | Densenet-169 24.00 7.00 57 | Densenet-201 22.80 6.43 58 | Densenet-161 22.35 6.20 59 | ================================ ============= ============= 60 | .. _AlexNet: https://arxiv.org/abs/1404.5997 61 | .. _VGG: https://arxiv.org/abs/1409.1556 62 | .. _ResNet: https://arxiv.org/abs/1512.03385 63 | .. _SqueezeNet: https://arxiv.org/abs/1602.07360 64 | .. _DenseNet: https://arxiv.org/abs/1608.06993 65 | """ 66 | 67 | from .vgg import * 68 | from .gan import * 69 | -------------------------------------------------------------------------------- /src/models/gan.py: -------------------------------------------------------------------------------- 1 | ## reference code is https://github.com/pytorch/examples/blob/master/dcgan/main.py 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import os 7 | 8 | from models import * 9 | 10 | 11 | def weights_init(m): 12 | classname = m.__class__.__name__ 13 | if classname.find('Conv') != -1: 14 | m.weight.data.normal_(0.0, 0.02) 15 | elif classname.find('BatchNorm') != -1: 16 | m.weight.data.normal_(1.0, 0.02) 17 | m.bias.data.fill_(0) 18 | 19 | class _netD(nn.Module): 20 | def __init__(self, ngpu, nc, ndf): 21 | super(_netD, self).__init__() 22 | self.ngpu = ngpu 23 | self.main = nn.Sequential( 24 | # input size. (nc) x 32 x 32 25 | nn.Conv2d(nc, ndf * 2, 4, 2, 1, bias=False), 26 | nn.LeakyReLU(0.2, inplace=True), 27 | # state size. (ndf*2) x 16 x 16 28 | nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), 29 | nn.BatchNorm2d(ndf * 4), 30 | nn.LeakyReLU(0.2, inplace=True), 31 | # state size. (ndf*4) x 8 x 8 32 | nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), 33 | nn.BatchNorm2d(ndf * 8), 34 | nn.LeakyReLU(0.2, inplace=True), 35 | # state size. (ndf*8) x 4 x 4 36 | nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), 37 | nn.Sigmoid() 38 | ) 39 | 40 | def forward(self, input): 41 | if isinstance(input.data, torch.cuda.FloatTensor) and self.ngpu > 1: 42 | output = nn.parallel.data_parallel(self.main, input, range(self.ngpu)) 43 | else: 44 | output = self.main(input) 45 | 46 | return output.view(-1, 1) 47 | 48 | class _netG(nn.Module): 49 | def __init__(self, ngpu, nz, ngf, nc): 50 | super(_netG, self).__init__() 51 | self.ngpu = ngpu 52 | self.main = nn.Sequential( 53 | # input is Z, going into a convolution 54 | nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False), 55 | nn.BatchNorm2d(ngf * 8), 56 | nn.ReLU(True), 57 | # state size. (ngf*8) x 4 x 4 58 | nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), 59 | nn.BatchNorm2d(ngf * 4), 60 | nn.ReLU(True), 61 | # state size. (ngf*4) x 8 x 8 62 | nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), 63 | nn.BatchNorm2d(ngf * 2), 64 | nn.ReLU(True), 65 | # state size. (ngf*2) x 16 x 16 66 | nn.ConvTranspose2d(ngf * 2, nc, 4, 2, 1, bias=False), 67 | nn.Sigmoid() 68 | # state size. (nc) x 32 x 32 69 | ) 70 | 71 | def forward(self, input): 72 | if isinstance(input.data, torch.cuda.FloatTensor) and self.ngpu > 1: 73 | output = nn.parallel.data_parallel(self.main, input, range(self.ngpu)) 74 | else: 75 | output = self.main(input) 76 | return output 77 | 78 | def Generator(n_gpu, nz, ngf, nc): 79 | model = _netG(n_gpu, nz, ngf, nc) 80 | model.apply(weights_init) 81 | return model 82 | 83 | def Discriminator(n_gpu, nc, ndf): 84 | model = _netD(n_gpu, nc, ndf) 85 | model.apply(weights_init) 86 | return model 87 | 88 | -------------------------------------------------------------------------------- /src/models/vgg.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import math 3 | 4 | 5 | __all__ = [ 6 | 'VGG', 'vgg13', 7 | ] 8 | 9 | class VGG(nn.Module): 10 | 11 | def __init__(self, features, num_classes=10): 12 | super(VGG, self).__init__() 13 | self.features = features 14 | self.classifier = nn.Sequential( 15 | nn.Linear(512, 512), 16 | nn.ReLU(True), 17 | nn.Dropout(), 18 | nn.Linear(512, 512), 19 | nn.ReLU(True), 20 | nn.Dropout(), 21 | nn.Linear(512, num_classes), 22 | ) 23 | self._initialize_weights() 24 | 25 | def forward(self, x): 26 | x = self.features(x) 27 | x = x.view(x.size(0), -1) 28 | x = self.classifier(x) 29 | return x 30 | 31 | def _initialize_weights(self): 32 | for m in self.modules(): 33 | if isinstance(m, nn.Conv2d): 34 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 35 | m.weight.data.normal_(0, math.sqrt(2. / n)) 36 | if m.bias is not None: 37 | m.bias.data.zero_() 38 | elif isinstance(m, nn.BatchNorm2d): 39 | m.weight.data.fill_(1) 40 | m.bias.data.zero_() 41 | elif isinstance(m, nn.Linear): 42 | m.weight.data.normal_(0, 0.01) 43 | m.bias.data.zero_() 44 | 45 | def make_layers(cfg): 46 | layers = [] 47 | in_channels = 3 48 | for v in cfg: 49 | if v == 'M': 50 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 51 | else: 52 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 53 | layers += [conv2d, nn.ReLU(inplace=True)] 54 | in_channels = v 55 | return nn.Sequential(*layers) 56 | 57 | 58 | cfg = { 59 | 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 60 | 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 61 | 'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 62 | 'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 63 | } 64 | 65 | def vgg13(pretrained=False, **kwargs): 66 | """VGG 13-layer model (configuration "B") 67 | Args: 68 | pretrained (bool): If True, returns a model pre-trained on ImageNet 69 | """ 70 | model = VGG(make_layers(cfg['B']), **kwargs) 71 | return model 72 | -------------------------------------------------------------------------------- /src/run_cross_entropy.py: -------------------------------------------------------------------------------- 1 | ############################################## 2 | # This code is based on samples from pytorch # 3 | ############################################## 4 | # Writer: Kimin Lee 5 | 6 | from __future__ import print_function 7 | import argparse 8 | import torch 9 | import torch.nn as nn 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | import data_loader 13 | import numpy as np 14 | import torchvision.utils as vutils 15 | import models 16 | 17 | from torchvision import datasets, transforms 18 | from torch.autograd import Variable 19 | 20 | 21 | # Training settings 22 | parser = argparse.ArgumentParser(description='Training code - cross entropy') 23 | parser.add_argument('--batch-size', type=int, default=128, help='input batch size for training') 24 | parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train') 25 | parser.add_argument('--lr', type=float, default=0.0002, help='learning rate') 26 | parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') 27 | parser.add_argument('--seed', type=int, default=1, help='random seed') 28 | parser.add_argument('--log-interval', type=int, default=100, help='how many batches to wait before logging training status') 29 | parser.add_argument('--dataset', default='cifar10', help='cifar10 | svhn') 30 | parser.add_argument('--dataroot', required=True, help='path to dataset') 31 | parser.add_argument('--imageSize', type=int, default=32, help='the height / width of the input image to network') 32 | parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints') 33 | parser.add_argument('--wd', type=float, default=0.0, help='weight decay') 34 | parser.add_argument('--droprate', type=float, default=0.1, help='learning rate decay') 35 | parser.add_argument('--decreasing_lr', default='60', help='decreasing strategy') 36 | 37 | args = parser.parse_args() 38 | print(args) 39 | args.cuda = not args.no_cuda and torch.cuda.is_available() 40 | print("Random Seed: ", args.seed) 41 | torch.manual_seed(args.seed) 42 | 43 | if args.cuda: 44 | torch.cuda.manual_seed(args.seed) 45 | 46 | kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} 47 | 48 | print('load data: ',args.dataset) 49 | train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, args.imageSize, args.dataroot) 50 | 51 | print('Load model') 52 | model = models.vgg13() 53 | print(model) 54 | 55 | if args.cuda: 56 | model.cuda() 57 | 58 | print('Setup optimizer') 59 | optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) 60 | decreasing_lr = list(map(int, args.decreasing_lr.split(','))) 61 | 62 | def train(epoch): 63 | model.train() 64 | for batch_idx, (data, target) in enumerate(train_loader): 65 | if args.cuda: 66 | data, target = data.cuda(), target.cuda() 67 | data, target = Variable(data), Variable(target) 68 | optimizer.zero_grad() 69 | output = F.log_softmax(model(data)) 70 | loss = F.nll_loss(output, target) 71 | loss.backward() 72 | optimizer.step() 73 | if batch_idx % args.log_interval == 0: 74 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 75 | epoch, batch_idx * len(data), len(train_loader.dataset), 76 | 100. * batch_idx / len(train_loader), loss.data.item())) 77 | 78 | def test(epoch): 79 | model.eval() 80 | test_loss = 0 81 | correct = 0 82 | total = 0 83 | for data, target in test_loader: 84 | total += data.size(0) 85 | if args.cuda: 86 | data, target = data.cuda(), target.cuda() 87 | data, target = Variable(data, volatile=True), Variable(target) 88 | output = F.log_softmax(model(data)) 89 | test_loss += F.nll_loss(output, target).data.item() 90 | pred = output.data.max(1)[1] # get the index of the max log-probability 91 | correct += pred.eq(target.data).cpu().sum() 92 | 93 | test_loss = test_loss 94 | test_loss /= len(test_loader) # loss function already averages over batch size 95 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 96 | test_loss, correct, total, 97 | 100. * correct / total)) 98 | 99 | 100 | for epoch in range(1, args.epochs + 1): 101 | train(epoch) 102 | if epoch in decreasing_lr: 103 | optimizer.param_groups[0]['lr'] *= args.droprate 104 | test(epoch) 105 | if epoch % 50 == 0: 106 | torch.save(model.state_dict(), '%s/model_epoch_%d.pth' % (args.outf, epoch)) 107 | -------------------------------------------------------------------------------- /src/run_joint_confidence.py: -------------------------------------------------------------------------------- 1 | ############################################## 2 | # This code is based on samples from pytorch # 3 | ############################################## 4 | # Writer: Kimin Lee 5 | from __future__ import print_function 6 | import argparse 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | import data_loader 12 | import numpy as np 13 | import torchvision.utils as vutils 14 | import models 15 | 16 | from torchvision import datasets, transforms 17 | from torch.autograd import Variable 18 | 19 | 20 | # Training settings 21 | parser = argparse.ArgumentParser(description='Training code - joint confidence') 22 | parser.add_argument('--batch-size', type=int, default=128, help='input batch size for training') 23 | parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train') 24 | parser.add_argument('--lr', type=float, default=0.0002, help='learning rate') 25 | parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') 26 | parser.add_argument('--seed', type=int, default=1, help='random seed') 27 | parser.add_argument('--log-interval', type=int, default=100, help='how many batches to wait before logging training status') 28 | parser.add_argument('--dataset', default='svhn', help='cifar10 | svhn') 29 | parser.add_argument('--dataroot', required=True, help='path to dataset') 30 | parser.add_argument('--imageSize', type=int, default=32, help='the height / width of the input image to network') 31 | parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints') 32 | parser.add_argument('--wd', type=float, default=0.0, help='weight decay') 33 | parser.add_argument('--droprate', type=float, default=0.1, help='learning rate decay') 34 | parser.add_argument('--decreasing_lr', default='60', help='decreasing strategy') 35 | parser.add_argument('--num_classes', type=int, default=10, help='the # of classes') 36 | parser.add_argument('--beta', type=float, default=1, help='penalty parameter for KL term') 37 | 38 | args = parser.parse_args() 39 | 40 | if args.dataset == 'cifar10': 41 | args.beta = 0.1 42 | args.batch_size = 64 43 | 44 | print(args) 45 | args.cuda = not args.no_cuda and torch.cuda.is_available() 46 | print("Random Seed: ", args.seed) 47 | torch.manual_seed(args.seed) 48 | 49 | if args.cuda: 50 | torch.cuda.manual_seed(args.seed) 51 | 52 | kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} 53 | 54 | print('load data: ',args.dataset) 55 | train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, args.imageSize, args.dataroot) 56 | 57 | print('Load model') 58 | model = models.vgg13() 59 | print(model) 60 | 61 | print('load GAN') 62 | nz = 100 63 | netG = models.Generator(1, nz, 64, 3) # ngpu, nz, ngf, nc 64 | netD = models.Discriminator(1, 3, 64) # ngpu, nc, ndf 65 | # Initial setup for GAN 66 | real_label = 1 67 | fake_label = 0 68 | criterion = nn.BCELoss() 69 | fixed_noise = torch.FloatTensor(64, nz, 1, 1).normal_(0, 1) 70 | 71 | if args.cuda: 72 | model.cuda() 73 | netD.cuda() 74 | netG.cuda() 75 | criterion.cuda() 76 | fixed_noise = fixed_noise.cuda() 77 | fixed_noise = Variable(fixed_noise) 78 | 79 | print('Setup optimizer') 80 | optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) 81 | optimizerD = optim.Adam(netD.parameters(), lr=args.lr, betas=(0.5, 0.999)) 82 | optimizerG = optim.Adam(netG.parameters(), lr=args.lr, betas=(0.5, 0.999)) 83 | decreasing_lr = list(map(int, args.decreasing_lr.split(','))) 84 | 85 | def train(epoch): 86 | model.train() 87 | for batch_idx, (data, target) in enumerate(train_loader): 88 | 89 | gan_target = torch.FloatTensor(target.size()).fill_(0) 90 | uniform_dist = torch.Tensor(data.size(0), args.num_classes).fill_((1./args.num_classes)) 91 | 92 | if args.cuda: 93 | data, target = data.cuda(), target.cuda() 94 | gan_target, uniform_dist = gan_target.cuda(), uniform_dist.cuda() 95 | 96 | data, target, uniform_dist = Variable(data), Variable(target), Variable(uniform_dist) 97 | 98 | ########################### 99 | # (1) Update D network # 100 | ########################### 101 | # train with real 102 | gan_target.fill_(real_label) 103 | targetv = Variable(gan_target) 104 | optimizerD.zero_grad() 105 | output = netD(data) 106 | errD_real = criterion(output, targetv) 107 | errD_real.backward() 108 | D_x = output.data.mean() 109 | 110 | # train with fake 111 | noise = torch.FloatTensor(data.size(0), nz, 1, 1).normal_(0, 1).cuda() 112 | if args.cuda: 113 | noise = noise.cuda() 114 | noise = Variable(noise) 115 | fake = netG(noise) 116 | targetv = Variable(gan_target.fill_(fake_label)) 117 | output = netD(fake.detach()) 118 | errD_fake = criterion(output, targetv) 119 | errD_fake.backward() 120 | D_G_z1 = output.data.mean() 121 | errD = errD_real + errD_fake 122 | optimizerD.step() 123 | 124 | ########################### 125 | # (2) Update G network # 126 | ########################### 127 | optimizerG.zero_grad() 128 | # Original GAN loss 129 | targetv = Variable(gan_target.fill_(real_label)) 130 | output = netD(fake) 131 | errG = criterion(output, targetv) 132 | D_G_z2 = output.data.mean() 133 | 134 | # minimize the true distribution 135 | KL_fake_output = F.log_softmax(model(fake)) 136 | errG_KL = F.kl_div(KL_fake_output, uniform_dist)*args.num_classes 137 | generator_loss = errG + args.beta*errG_KL 138 | generator_loss.backward() 139 | optimizerG.step() 140 | 141 | ########################### 142 | # (3) Update classifier # 143 | ########################### 144 | # cross entropy loss 145 | optimizer.zero_grad() 146 | output = F.log_softmax(model(data)) 147 | loss = F.nll_loss(output, target) 148 | 149 | # KL divergence 150 | noise = torch.FloatTensor(data.size(0), nz, 1, 1).normal_(0, 1).cuda() 151 | if args.cuda: 152 | noise = noise.cuda() 153 | noise = Variable(noise) 154 | fake = netG(noise) 155 | KL_fake_output = F.log_softmax(model(fake)) 156 | KL_loss_fake = F.kl_div(KL_fake_output, uniform_dist)*args.num_classes 157 | total_loss = loss + args.beta*KL_loss_fake 158 | total_loss.backward() 159 | optimizer.step() 160 | 161 | if batch_idx % args.log_interval == 0: 162 | print('Classification Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}, KL fake Loss: {:.6f}'.format( 163 | epoch, batch_idx * len(data), len(train_loader.dataset), 164 | 100. * batch_idx / len(train_loader), loss.data.item(), KL_loss_fake.data.item())) 165 | fake = netG(fixed_noise) 166 | vutils.save_image(fake.data, '%s/gan_samples_epoch_%03d.png'%(args.outf, epoch), normalize=True) 167 | 168 | def test(epoch): 169 | model.eval() 170 | test_loss = 0 171 | correct = 0 172 | total = 0 173 | for data, target in test_loader: 174 | total += data.size(0) 175 | if args.cuda: 176 | data, target = data.cuda(), target.cuda() 177 | data, target = Variable(data, volatile=True), Variable(target) 178 | output = F.log_softmax(model(data)) 179 | test_loss += F.nll_loss(output, target).data.item() 180 | pred = output.data.max(1)[1] # get the index of the max log-probability 181 | correct += pred.eq(target.data).cpu().sum() 182 | 183 | test_loss = test_loss 184 | test_loss /= len(test_loader) # loss function already averages over batch size 185 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( 186 | test_loss, correct, total, 187 | 100. * correct / total)) 188 | 189 | 190 | for epoch in range(1, args.epochs + 1): 191 | train(epoch) 192 | test(epoch) 193 | if epoch in decreasing_lr: 194 | optimizerG.param_groups[0]['lr'] *= args.droprate 195 | optimizerD.param_groups[0]['lr'] *= args.droprate 196 | optimizer.param_groups[0]['lr'] *= args.droprate 197 | if epoch % 20 == 0: 198 | # do checkpointing 199 | torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (args.outf, epoch)) 200 | torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % (args.outf, epoch)) 201 | torch.save(model.state_dict(), '%s/model_epoch_%d.pth' % (args.outf, epoch)) 202 | -------------------------------------------------------------------------------- /src/test_detection.py: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # Measure the detection performance: reference code is https://github.com/ShiyuLiang/odin-pytorch # 3 | ################################################################################################### 4 | # Writer: Kimin Lee 5 | from __future__ import print_function 6 | import argparse 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | import data_loader 12 | import numpy as np 13 | import torchvision.utils as vutils 14 | import calculate_log as callog 15 | import models 16 | import math 17 | 18 | from torch.utils.serialization import load_lua 19 | from torchvision import datasets, transforms 20 | from torch.nn.parameter import Parameter 21 | from torch.autograd import Variable 22 | from numpy.linalg import inv 23 | 24 | # Training settings 25 | parser = argparse.ArgumentParser(description='Test code - measure the detection peformance') 26 | parser.add_argument('--batch-size', type=int, default=128, help='batch size') 27 | parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA') 28 | parser.add_argument('--seed', type=int, default=1,help='random seed') 29 | parser.add_argument('--dataset', required=True, help='target dataset: cifar10 | svhn') 30 | parser.add_argument('--dataroot', required=True, help='path to dataset') 31 | parser.add_argument('--imageSize', type=int, default=32, help='the height / width of the input image to network') 32 | parser.add_argument('--outf', default='/home/rack/KM/2017_Codes/overconfidence/test/log_entropy', help='folder to output images and model checkpoints') 33 | parser.add_argument('--out_dataset', required=True, help='out-of-dist dataset: cifar10 | svhn | imagenet | lsun') 34 | parser.add_argument('--num_classes', type=int, default=10, help='number of classes (default: 10)') 35 | parser.add_argument('--pre_trained_net', default='', help="path to pre trained_net") 36 | 37 | args = parser.parse_args() 38 | print(args) 39 | args.cuda = not args.no_cuda and torch.cuda.is_available() 40 | print("Random Seed: ", args.seed) 41 | torch.manual_seed(args.seed) 42 | 43 | if args.cuda: 44 | torch.cuda.manual_seed(args.seed) 45 | 46 | kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} 47 | 48 | print('Load model') 49 | model = models.vgg13() 50 | model.load_state_dict(torch.load(args.pre_trained_net)) 51 | print(model) 52 | 53 | print('load target data: ',args.dataset) 54 | _, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, args.imageSize, args.dataroot) 55 | 56 | print('load non target data: ',args.out_dataset) 57 | nt_test_loader = data_loader.getNonTargetDataSet(args.out_dataset, args.batch_size, args.imageSize, args.dataroot) 58 | 59 | if args.cuda: 60 | model.cuda() 61 | 62 | def generate_target(): 63 | model.eval() 64 | correct = 0 65 | total = 0 66 | f1 = open('%s/confidence_Base_In.txt'%args.outf, 'w') 67 | 68 | for data, target in test_loader: 69 | total += data.size(0) 70 | #vutils.save_image(data, '%s/target_samples.png'%args.outf, normalize=True) 71 | if args.cuda: 72 | data, target = data.cuda(), target.cuda() 73 | data, target = Variable(data, volatile=True), Variable(target) 74 | batch_output = model(data) 75 | 76 | # compute the accuracy 77 | pred = batch_output.data.max(1)[1] 78 | equal_flag = pred.eq(target.data).cpu() 79 | correct += equal_flag.sum() 80 | for i in range(data.size(0)): 81 | # confidence score: max_y p(y|x) 82 | output = batch_output[i].view(1,-1) 83 | soft_out = F.softmax(output) 84 | soft_out = torch.max(soft_out.data) 85 | f1.write("{}\n".format(soft_out)) 86 | 87 | print('\n Final Accuracy: {}/{} ({:.2f}%)\n'.format(correct, total, 100. * correct / total)) 88 | 89 | def generate_non_target(): 90 | model.eval() 91 | total = 0 92 | f2 = open('%s/confidence_Base_Out.txt'%args.outf, 'w') 93 | 94 | for data, target in nt_test_loader: 95 | total += data.size(0) 96 | if args.cuda: 97 | data, target = data.cuda(), target.cuda() 98 | data, target = Variable(data, volatile=True), Variable(target) 99 | batch_output = model(data) 100 | for i in range(data.size(0)): 101 | # confidence score: max_y p(y|x) 102 | output = batch_output[i].view(1,-1) 103 | soft_out = F.softmax(output) 104 | soft_out = torch.max(soft_out.data) 105 | f2.write("{}\n".format(soft_out)) 106 | 107 | print('generate log from in-distribution data') 108 | generate_target() 109 | print('generate log from out-of-distribution data') 110 | generate_non_target() 111 | print('calculate metrics') 112 | callog.metric(args.outf) 113 | --------------------------------------------------------------------------------