├── Auto-encoder-cuda ├── AdversarialAE.lua ├── AdversarialAE_alex.lua ├── DenoisingAE.lua ├── VariationalAE.lua └── deploy_adversarialAE_alex.lua ├── BinarizedNeurons.lua ├── Cluster Visualisation ├── README.md ├── advAE_cnn_mnist.lua ├── calc_imp_bits.lua ├── convert_dict_to_features.lua └── visualiseCluster.m ├── FGVC └── dataPrep.lua ├── Pretrainedencoder.py ├── Results ├── README.md ├── Reconstructions_mnist_cnn.png └── Training_mnist_advCnn.png ├── advAE_cnn_mnist.lua ├── mnist_deploy.lua ├── prepare_dict.lua ├── resnet.py ├── retrieval_from_dict.lua └── std_mean_ret.lua /Auto-encoder-cuda/AdversarialAE.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | require 'cutorch'; 7 | require 'cunn'; 8 | require 'cudnn'; 9 | 10 | local Gaussian,parent = torch.class('nn.Gaussian','nn.Module') 11 | 12 | function Gaussian:__init(mean, stdv) 13 | parent.__init(self) 14 | self.mean = mean or 0 15 | self.stdv = stdv or 1 16 | end 17 | 18 | function Gaussian:updateOutput(inp) 19 | self.output:resizeAs(inp) 20 | self.output:normal(self.mean,self.stdv) 21 | return self.output 22 | end 23 | 24 | function Gaussian:updateGradInput(inp, gradOut) 25 | self.gradInput:resizeAs(inp) 26 | self.gradInput:zero() 27 | return self.gradInput 28 | end 29 | 30 | zSize = 128 31 | --encoder 32 | encoder = nn.Sequential(); 33 | encoder:add(nn.SpatialConvolution(3,64,3,3,1,1,1,1)) --1 34 | encoder:add(nn.ReLU()) --2 35 | encoder:add(nn.SpatialConvolution(64,64,3,3,1,1,1,1)) --3 36 | encoder:add(nn.ReLU()) --4 37 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --5 38 | 39 | encoder:add(nn.SpatialConvolution(64,128,3,3,1,1,1,1)) --6 40 | encoder:add(nn.ReLU()) --7 41 | encoder:add(nn.SpatialConvolution(128,128,3,3,1,1,1,1)) --8 42 | encoder:add(nn.ReLU()) --9 43 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --10 44 | 45 | encoder:add(nn.SpatialConvolution(128,256,3,3,1,1,1,1)) --11 46 | encoder:add(nn.ReLU()) --12 47 | encoder:add(nn.SpatialConvolution(256,256,3,3,1,1,1,1)) --13 48 | encoder:add(nn.ReLU()) --14 49 | encoder:add(nn.SpatialConvolution(256,256,3,3,1,1,1,1)) --15 50 | encoder:add(nn.ReLU()) --16 51 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --17 52 | 53 | encoder:add(nn.SpatialConvolution(256,512,3,3,1,1,1,1)) --18 54 | encoder:add(nn.ReLU()) --19 55 | encoder:add(nn.SpatialConvolution(512,512,3,3,1,1,1,1)) --20 56 | encoder:add(nn.ReLU()) --21 57 | encoder:add(nn.SpatialConvolution(512,512,1,1,1,1)) --22 58 | encoder:add(nn.ReLU()) --23 59 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --24 60 | 61 | encoder:add(nn.View(-1,2048)) --25 62 | encoder:add(nn.Linear(2048,512)) --26 63 | encoder:add(nn.BatchNormalization(512)) --27 64 | encoder:add(nn.ReLU()) --28 65 | encoder:add(nn.Linear(512,zSize)) --29 66 | 67 | --decoder 68 | decoder = nn.Sequential() 69 | decoder:add(nn.Linear(zSize, 512)) --1 70 | decoder:add(nn.BatchNormalization(512)) --2 71 | decoder:add(nn.ReLU()) --3 72 | decoder:add(nn.Linear(512, 2048)) --4 73 | decoder:add(nn.BatchNormalization(2048)) --5 74 | decoder:add(nn.ReLU()) --6 75 | decoder:add(nn.View(512,2,2)) --7 76 | 77 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(24))) --8 78 | decoder:add(nn.SpatialConvolution(512,512,1,1,1,1)) --9 79 | decoder:add(nn.ReLU()) --10 80 | decoder:add(nn.SpatialConvolution(512,512,3,3,1,1,1,1)) --11 81 | decoder:add(nn.ReLU()) --12 82 | decoder:add(nn.SpatialConvolution(512,256,3,3,1,1,1,1)) --13 83 | decoder:add(nn.ReLU()) --14 84 | 85 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(17))) --15 86 | decoder:add(nn.SpatialConvolution(256,256,3,3,1,1,1,1)) --16 87 | decoder:add(nn.ReLU()) --17 88 | decoder:add(nn.SpatialConvolution(256,256,3,3,1,1,1,1)) --18 89 | decoder:add(nn.ReLU()) --19 90 | decoder:add(nn.SpatialConvolution(256,128,3,3,1,1,1,1)) --20 91 | decoder:add(nn.ReLU()) --21 92 | 93 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(10))) --22 94 | decoder:add(nn.SpatialConvolution(128,128,3,3,1,1,1,1)) --23 95 | decoder:add(nn.ReLU()) --24 96 | decoder:add(nn.SpatialConvolution(128,64,3,3,1,1,1,1)) --25 97 | decoder:add(nn.ReLU()) --26 98 | 99 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(5))) --27 100 | decoder:add(nn.SpatialConvolution(64,64,3,3,1,1,1,1)) --28 101 | decoder:add(nn.ReLU()) --29 102 | decoder:add(nn.SpatialConvolution(64,3,1,1,1,1)) --30 -- Used a 1x1 convolution instead of 3x3 -- Make change across run 103 | decoder:add(nn.Sigmoid()) --31 104 | 105 | 106 | --autoencoder 107 | autoencoder = nn.Sequential() 108 | autoencoder:add(encoder) 109 | autoencoder:add(decoder) 110 | 111 | autoencoder = autoencoder:cuda() 112 | print(autoencoder) 113 | 114 | --adversary network 115 | adversary = nn.Sequential() 116 | adversary:add(nn.Linear(zSize, 64)) 117 | adversary:add(nn.BatchNormalization(64)) 118 | adversary:add(nn.ReLU()) 119 | adversary:add(nn.Linear(64, 16)) 120 | adversary:add(nn.BatchNormalization(16)) 121 | adversary:add(nn.ReLU()) 122 | adversary:add(nn.Linear(16, 1)) 123 | adversary:add(nn.BatchNormalization(1)) 124 | adversary:add(nn.Sigmoid()) 125 | 126 | adversary = adversary:cuda() 127 | print(adversary) 128 | 129 | --load MNIST data 130 | --[[trainData = mnist.traindataset().data:double():div(255):cuda() 131 | trainlabels = mnist.traindataset().label:cuda() 132 | N = mnist.traindataset().size 133 | 134 | testData = mnist.testdataset().data:double():div(255):cuda() 135 | testlabels = mnist.testdataset().label:cuda() 136 | --]] 137 | trainset = torch.load('cifar10-train.t7') 138 | testset = torch.load('cifar10-test.t7') 139 | classes = {'airplane', 'automobile', 'bird', 'cat', 140 | 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'} 141 | 142 | trainData = trainset.data:double():div(255):cuda() 143 | --[[trainData = torch.ByteTensor(trainset.data:size()) 144 | print(#trainData) 145 | for i=1,trainData:size()[1] do 146 | trainData[i] = image.rgb2hsv(trainset.data[i]) 147 | end 148 | trainData = trainData:double():div(255):cuda() 149 | --]] 150 | trainlabel=trainset.label:cuda() 151 | N = trainData:size()[1] 152 | testData = testset.data 153 | testlabels = testset.label:cuda() 154 | 155 | 156 | local theta,gradTheta = autoencoder:getParameters() 157 | local thetaAdv,gradThetaAdv = adversary:getParameters() 158 | 159 | local criterion = nn.BCECriterion():cuda() 160 | 161 | local x 162 | 163 | batchSize = 500 164 | iterations = 25 165 | 166 | local feval = function(params) 167 | if theta~=params then 168 | theta:copy(params) 169 | end 170 | gradTheta:zero() 171 | gradThetaAdv:zero() 172 | --print(#x) 173 | local xHat = autoencoder:forward(x) 174 | local loss = criterion:forward(xHat,x) 175 | local gradLoss = criterion:backward(xHat,x) 176 | autoencoder:backward(x,gradLoss) 177 | 178 | local real = torch.Tensor(batchSize,zSize):bernoulli():typeAs(trainData) -- Real Samples 179 | local YReal = torch.ones(batchSize):typeAs(trainData) -- labels for real samples 180 | local YFake = torch.zeros(batchSize):typeAs(trainData) --labels for generated samples 181 | 182 | -- Train adversary to maximise log probability of real samples: max_D log(D(x)) 183 | local pred = adversary:forward(real) 184 | local realLoss = criterion:forward(pred,YReal) 185 | local gradRealLoss = criterion:backward(pred,YReal) 186 | adversary:backward(real,gradRealLoss) 187 | 188 | --Train adversary to minimise log probability of fake samples: max_D log(1 - D(G(x))) 189 | pred = adversary:forward(encoder.output) 190 | local fakeLoss = criterion:forward(pred,YFake) 191 | advLoss = realLoss + fakeLoss 192 | local gradFakeLoss = criterion:backward(pred,YFake) 193 | local gradFake = adversary:backward(encoder.output, gradFakeLoss) 194 | 195 | -- Train encoder (generator) to play a minimax game with the adversary (discriminator): min_G max_D log(1 - D(G(x))) 196 | local minimaxLoss = criterion:forward(pred,YReal) -- Technically use max_G max_D log(D(G(x))) for same fixed point, stronger initial gradients 197 | loss = loss + minimaxLoss 198 | local gradMinimaxLoss = criterion:backward(pred,YReal) 199 | local gradMinimax = adversary:updateGradInput(encoder.output, gradMinimaxLoss) 200 | encoder:backward(x,gradMinimax) 201 | 202 | return loss, gradTheta 203 | end 204 | 205 | local advFeval = function(params) 206 | if thetaAdv~=params then 207 | thetaAdv:copy(params) 208 | end 209 | return advLoss, gradThetaAdv 210 | end 211 | 212 | --Train 213 | print('Training Starting') 214 | local optimParams = {learningRate = 0.1} 215 | local advOptimParams = {learningRate = 0.1} 216 | local _,loss 217 | local losses, advLosses = {},{} 218 | for epoch=1,iterations do 219 | collectgarbage() 220 | print('Epoch '..epoch..'/'..iterations) 221 | for n=1,N, batchSize do 222 | collectgarbage() 223 | x = trainData:narrow(1,n,batchSize) 224 | _,loss = optim.adam(feval,theta,optimParams) 225 | losses[#losses + 1] = loss[1] 226 | _,loss = optim.adam(advFeval,thetaAdv,advOptimParams) 227 | advLosses[#advLosses + 1] = loss[1] 228 | end 229 | local plots={{'Adv AE', torch.linspace(1,#losses,#losses), torch.Tensor(losses), '-'}} 230 | plots[2]={'Adversary', torch.linspace(1,#advLosses,#advLosses), torch.Tensor(advLosses), '-'} 231 | gnuplot.pngfigure('AdvAE/Training_bernoulli_cifar_vggsrnn_hsv.png') 232 | gnuplot.plot(table.unpack(plots)) 233 | gnuplot.ylabel('Loss') 234 | gnuplot.xlabel('Batch #') 235 | gnuplot.plotflush() 236 | 237 | --permute training data 238 | trainData = trainData:index(1,torch.randperm(trainData:size(1)):long()) 239 | 240 | 241 | x = testData:narrow(1,1,50) 242 | --[[local x_hsv = torch.Tensor(x:size()):typeAs(x) 243 | for i=1,x:size()[1] do 244 | x_hsv[i] = image.rgb2hsv(x[i]) 245 | end 246 | --]] 247 | x_hsv = x--[[_hsv--]]:double():div(255):cuda() 248 | local xHat_hsv= autoencoder:forward(x_hsv) 249 | --[[xHat_hsv = xHat_hsv:mul(255):byte() 250 | for i=1,50 do 251 | print(i) 252 | print(xHat_hsv[i][1]:min(),xHat_hsv[i][1]:max()) 253 | print(xHat_hsv[i][2]:min(),xHat_hsv[i][2]:max()) 254 | print(xHat_hsv[i][3]:min(),xHat_hsv[i][3]:max()) 255 | end 256 | --]] 257 | --[[local xHat = torch.Tensor(xHat_hsv:size()):typeAs(xHat_hsv) 258 | for i=1,xHat_hsv:size()[1] do 259 | xHat[i] = image.hsv2rgb(xHat_hsv[i]) 260 | end 261 | --]] 262 | 263 | --print (#x) 264 | ---print(#xHat) 265 | --temp=torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2) 266 | --print (#temp) 267 | image.save('AdvAE/Reconstructions_bernoulli_cifar_vggsrnn_hsv_temp.png', torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat_hsv,2,50),2)) 268 | end 269 | 270 | print('Testing') 271 | x = testData:narrow(1,1,50) 272 | --[[local x_hsv = torch.Tensor(x:size()):typeAs(x) 273 | for i=1,x:size()[1] do 274 | x_hsv[i] = image.rgb2hsv(x[i]) 275 | end 276 | --]] 277 | x_hsv = x--[[_hsv--]]:double():div(255):cuda() 278 | local xHat_hsv= autoencoder:forward(x_hsv) 279 | --[[xHat_hsv = xHat_hsv:mul(255):byte() 280 | for i=1,50 do 281 | print(i) 282 | print(x_hsv[i][1]:min(),x_hsv[i][1]:min()) 283 | print(x_hsv[i][2]:min(),x_hsv[i][2]:min()) 284 | print(x_hsv[i][3]:min(),x_hsv[i][3]:min()) 285 | end 286 | --]] 287 | --[[local xHat = torch.Tensor(xHat_hsv:size()):typeAs(xHat_hsv) 288 | for i=1,xHat_hsv:size()[1] do 289 | xHat[i] = image.hsv2rgb(xHat_hsv[i]) 290 | end 291 | --]] 292 | 293 | --print (#x) 294 | ---print(#xHat) 295 | --temp=torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2) 296 | --print (#temp) 297 | image.save('AdvAE/Reconstructions_bernoulli_cifar_vggsrnn_hsv.png', torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat_hsv,2,50),2)) -------------------------------------------------------------------------------- /Auto-encoder-cuda/AdversarialAE_alex.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | require 'cutorch'; 7 | require 'cunn'; 8 | require 'cudnn'; 9 | torch.manualSeed(1) 10 | local Threshold,parent = torch.class('nn.Threshold','nn.Module') 11 | 12 | function Threshold:__init(th,v, ip) 13 | parent.__init(self) 14 | self.threshold = th or 0 15 | self.val = v or 0 16 | if (th and type(th) ~= 'number') or (v and type(v) ~= 'number') then 17 | error('nn.Threshold(threshold, value)') 18 | end 19 | self.inplace = ip or false 20 | if (ip and type(ip) ~= 'boolean') then 21 | error('in-place flag must be boolean') 22 | end 23 | self:validateParameters() 24 | end 25 | 26 | function Threshold:updateOutput(input) 27 | self:validateParameters() 28 | input.THNN.Threshold_updateOutput( 29 | input:cdata(), 30 | self.output:cdata(), 31 | self.threshold, 32 | self.val, 33 | self.inplace 34 | ) 35 | return self.output 36 | end 37 | 38 | function Threshold:updateGradInput(input, gradOutput) 39 | self:validateParameters() 40 | input.THNN.Threshold_updateGradInput( 41 | input:cdata(), 42 | gradOutput:cdata(), 43 | self.gradInput:cdata(), 44 | self.threshold, 45 | self.val, 46 | self.inplace 47 | ) 48 | return self.gradInput 49 | end 50 | 51 | function Threshold:validateParameters() 52 | self.inplace = self.inplace or false -- backwards compatibility pre inplace 53 | if self.inplace then 54 | if self.val > self.threshold then 55 | error('in-place processing requires value (' .. self.val .. 56 | ') not exceed threshold (' .. self.threshold .. ')') 57 | end 58 | end 59 | end 60 | 61 | zSize = 128 62 | --encoder 63 | encoder = nn.Sequential(); 64 | encoder:add(nn.SpatialConvolution(3,96,3,3,1,1,1,1)) --1 65 | encoder:add(nn.ReLU()) --2 66 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --3 67 | 68 | encoder:add(nn.SpatialConvolution(96,256,3,3,1,1,1,1)) --4 69 | encoder:add(nn.ReLU()) --5 70 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --6 71 | 72 | encoder:add(nn.SpatialConvolution(256,384,3,3,1,1,1,1)) --7 73 | encoder:add(nn.ReLU()) --8 74 | encoder:add(nn.SpatialConvolution(384,384,3,3,1,1,1,1)) --9 75 | encoder:add(nn.ReLU()) --10 76 | encoder:add(nn.SpatialConvolution(384,256,3,3,1,1,1,1)) --11 77 | encoder:add(nn.ReLU()) --12 78 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --13 79 | 80 | encoder:add(nn.View(-1,4096)) --14 81 | encoder:add(nn.Linear(4096,1024)) --15 82 | --encoder:add(nn.BatchNormalization(1024)) --16 83 | encoder:add(nn.ReLU()) --17 84 | encoder:add(nn.Linear(1024,512)) --18 85 | --encoder:add(nn.BatchNormalization(512)) --19 86 | encoder:add(nn.ReLU()) --20 87 | encoder:add(nn.Linear(512,zSize)) --21 88 | --encoder:add(nn.Sigmoid()) --22 89 | 90 | --decoder 91 | decoder = nn.Sequential() 92 | decoder:add(nn.Linear(zSize, 512)) --1 93 | --decoder:add(nn.BatchNormalization(512)) --2 94 | decoder:add(nn.ReLU()) --3 95 | decoder:add(nn.Linear(512, 1024)) --4 96 | --decoder:add(nn.BatchNormalization(1024)) --5 97 | decoder:add(nn.ReLU()) --6 98 | decoder:add(nn.Linear(1024, 4096)) --7 99 | --decoder:add(nn.BatchNormalization(4096)) --8 100 | decoder:add(nn.ReLU()) --9 101 | decoder:add(nn.View(256,4,4)) --10 102 | 103 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(13))) --11 104 | decoder:add(nn.SpatialConvolution(256,384,3,3,1,1,1,1)) --12 105 | decoder:add(nn.ReLU()) --13 106 | decoder:add(nn.SpatialConvolution(384,384,3,3,1,1,1,1)) --14 107 | decoder:add(nn.ReLU()) --15 108 | decoder:add(nn.SpatialConvolution(384,256,3,3,1,1,1,1)) --16 109 | decoder:add(nn.ReLU()) --17 110 | 111 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(6))) --18 112 | decoder:add(nn.SpatialConvolution(256,96,3,3,1,1,1,1)) --19 113 | decoder:add(nn.ReLU()) --20 114 | 115 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(3))) --21 116 | decoder:add(nn.SpatialConvolution(96,3,3,3,1,1,1,1)) --22 117 | decoder:add(nn.Sigmoid()) --23 118 | 119 | 120 | --autoencoder 121 | autoencoder = nn.Sequential() 122 | autoencoder:add(encoder) 123 | autoencoder:add(decoder) 124 | 125 | autoencoder = autoencoder:cuda() 126 | print(autoencoder) 127 | 128 | --adversary network 129 | adversary = nn.Sequential() 130 | adversary:add(nn.Linear(zSize, 64)) 131 | adversary:add(nn.BatchNormalization(64)) 132 | adversary:add(nn.ReLU()) 133 | adversary:add(nn.Linear(64, 16)) 134 | adversary:add(nn.BatchNormalization(16)) 135 | adversary:add(nn.ReLU()) 136 | adversary:add(nn.Linear(16, 1)) 137 | adversary:add(nn.BatchNormalization(1)) 138 | adversary:add(nn.Sigmoid()) 139 | 140 | adversary = adversary:cuda() 141 | print(adversary) 142 | 143 | --load MNIST data 144 | --[[trainData = mnist.traindataset().data:double():div(255):cuda() 145 | trainlabels = mnist.traindataset().label:cuda() 146 | N = mnist.traindataset().size 147 | 148 | testData = mnist.testdataset().data:double():div(255):cuda() 149 | testlabels = mnist.testdataset().label:cuda() 150 | --]] 151 | trainset = torch.load('cifar10-train.t7') 152 | testset = torch.load('cifar10-test.t7') 153 | classes = {'airplane', 'automobile', 'bird', 'cat', 154 | 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'} 155 | 156 | --trainData = trainset.data:double():div(255):cuda() 157 | trainData = torch.DoubleTensor(trainset.data:size()) 158 | print(#trainData) 159 | for i=1,trainData:size()[1] do 160 | trainData[i] = image.rgb2hsv(trainset.data[i]:double():div(255)) 161 | end 162 | 163 | trainData = trainData:cuda() 164 | 165 | trainlabel=trainset.label:cuda() 166 | N = trainData:size()[1] 167 | testData = testset.data 168 | testlabels = testset.label:cuda() 169 | 170 | 171 | local theta,gradTheta = autoencoder:getParameters() 172 | local thetaAdv,gradThetaAdv = adversary:getParameters() 173 | 174 | local criterion = nn.MSECriterion():cuda() 175 | 176 | local x 177 | 178 | batchSize = 512 179 | iterations = 250 180 | 181 | local feval = function(params) 182 | if theta~=params then 183 | theta:copy(params) 184 | end 185 | gradTheta:zero() 186 | gradThetaAdv:zero() 187 | --print("x") 188 | --print(x[1][1]) 189 | local xHat = autoencoder:forward(x) 190 | --print("xHat") 191 | --print(xHat[1][1]) 192 | local loss = criterion:forward(xHat,x) 193 | --print(loss) 194 | local gradLoss = criterion:backward(xHat,x) 195 | autoencoder:backward(x,gradLoss) 196 | 197 | local real = torch.Tensor(batchSize,zSize)--[[:bernoulli()--]]:normal(0,1):typeAs(trainData) -- Real Samples 198 | local YReal = torch.ones(batchSize):typeAs(trainData) -- labels for real samples 199 | local YFake = torch.zeros(batchSize):typeAs(trainData) --labels for generated samples 200 | 201 | -- Train adversary to maximise log probability of real samples: max_D log(D(x)) 202 | local pred = adversary:forward(real) 203 | local realLoss = criterion:forward(pred,YReal) 204 | local gradRealLoss = criterion:backward(pred,YReal) 205 | adversary:backward(real,gradRealLoss) 206 | 207 | --Train adversary to minimise log probability of fake samples: max_D log(1 - D(G(x))) 208 | pred = adversary:forward(encoder.output) 209 | local fakeLoss = criterion:forward(pred,YFake) 210 | advLoss = realLoss + fakeLoss 211 | local gradFakeLoss = criterion:backward(pred,YFake) 212 | local gradFake = adversary:backward(encoder.output, gradFakeLoss) 213 | 214 | -- Train encoder (generator) to play a minimax game with the adversary (discriminator): min_G max_D log(1 - D(G(x))) 215 | local minimaxLoss = criterion:forward(pred,YReal) -- Technically use max_G max_D log(D(G(x))) for same fixed point, stronger initial gradients 216 | loss = loss + minimaxLoss 217 | local gradMinimaxLoss = criterion:backward(pred,YReal) 218 | local gradMinimax = adversary:updateGradInput(encoder.output, gradMinimaxLoss) 219 | encoder:backward(x,gradMinimax) 220 | 221 | return loss, gradTheta 222 | end 223 | 224 | local advFeval = function(params) 225 | if thetaAdv~=params then 226 | thetaAdv:copy(params) 227 | end 228 | return advLoss, gradThetaAdv 229 | end 230 | 231 | --Train 232 | print('Training Starting') 233 | local optimParams = {learningRate = 0.002} 234 | local advOptimParams = {learningRate = 0.002} 235 | local _,loss 236 | local losses, advLosses = {},{} 237 | for epoch=1,iterations do 238 | collectgarbage() 239 | print('Epoch '..epoch..'/'..iterations) 240 | for n=1,N, batchSize do 241 | if n+batchSize-1>N then break end 242 | collectgarbage() 243 | x = trainData:narrow(1,n,batchSize) 244 | _,loss = optim.adam(feval,theta,optimParams) 245 | losses[#losses + 1] = loss[1] 246 | _,loss = optim.adam(advFeval,thetaAdv,advOptimParams) 247 | advLosses[#advLosses + 1] = loss[1] 248 | end 249 | local plots={{'Adv AE', torch.linspace(1,#losses,#losses), torch.Tensor(losses), '-'}} 250 | plots[2]={'Adversary', torch.linspace(1,#advLosses,#advLosses), torch.Tensor(advLosses), '-'} 251 | gnuplot.pngfigure('AdvAE/Training_normal_cifar_hsv_alex.png') 252 | gnuplot.plot(table.unpack(plots)) 253 | gnuplot.ylabel('Loss') 254 | gnuplot.xlabel('Batch #') 255 | gnuplot.plotflush() 256 | 257 | --permute training data 258 | trainData = trainData:index(1,torch.randperm(trainData:size(1)):long()) 259 | 260 | 261 | local x1 = trainData:narrow(1,1,50) 262 | local xHat= autoencoder:forward(x1) 263 | --print(x[1]) 264 | --print("xHat111") 265 | --print(xHat[1][3]) 266 | --[[xHat_hsv = xHat_hsv:mul(255):byte() 267 | for i=1,50 do 268 | print(i) 269 | print(xHat_hsv[i][1]:min(),xHat_hsv[i][1]:max()) 270 | print(xHat_hsv[i][2]:min(),xHat_hsv[i][2]:max()) 271 | print(xHat_hsv[i][3]:min(),xHat_hsv[i][3]:max()) 272 | end 273 | --]] 274 | --[[local xHat = torch.DoubleTensor(xHat_hsv:size()) 275 | for i=1,xHat_hsv:size()[1] do 276 | xHat[i] = image.hsv2rgb(xHat_hsv[i]:double()) 277 | end--]] 278 | 279 | --print (#x) 280 | ---print(#xHat) 281 | --temp=torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2) 282 | --print (#temp) 283 | image.save('AdvAE/Reconstructions_normal_cifar_hsv_alex_temp.png', torch.cat(image.toDisplayTensor(x1,2,50),image.toDisplayTensor(xHat,2,50),2)) 284 | end 285 | 286 | torch.save('AdvAE/autoencoder_model_normal.t7',autoencoder) 287 | torch.save('AdvAE/adversary_model_normal.t7',adversary) 288 | print('Testing') 289 | x = testData:narrow(1,1,50):double():div(255):cuda() 290 | local x_hsv = torch.Tensor(x:size()):typeAs(x) 291 | for i=1,x:size()[1] do 292 | x_hsv[i] = image.rgb2hsv(x[i]:double()) -- It showed an error in this line since the typecasting from CUDA to TorchDouble tensor was not written (DS / 7:20 PM @ 6 Jul) 293 | end 294 | x_hsv = x_hsv:cuda() 295 | local xHat_hsv= autoencoder:forward(x_hsv) 296 | --print(x[1]) 297 | --print("xHat111") 298 | --print(xHat[1][1]) 299 | --[[xHat_hsv = xHat_hsv:mul(255):byte() 300 | for i=1,50 do 301 | print(i) 302 | print(xHat_hsv[i][1]:min(),xHat_hsv[i][1]:max()) 303 | print(xHat_hsv[i][2]:min(),xHat_hsv[i][2]:max()) 304 | print(xHat_hsv[i][3]:min(),xHat_hsv[i][3]:max()) 305 | end 306 | --]] 307 | local xHat = torch.DoubleTensor(xHat_hsv:size()) 308 | for i=1,xHat_hsv:size()[1] do 309 | xHat[i] = image.hsv2rgb(xHat_hsv[i]:double()) 310 | end 311 | --print (#x) 312 | ---print(#xHat) 313 | --temp=torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2) 314 | --print (#temp) 315 | image.save('AdvAE/Reconstructions_normal_cifar_hsv_alex.png', torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2)) -------------------------------------------------------------------------------- /Auto-encoder-cuda/DenoisingAE.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'dpnn'; 3 | require 'image'; 4 | mnist = require 'mnist'; 5 | require 'optim'; 6 | require 'gnuplot'; 7 | require 'cudnn'; 8 | require 'cutorch'; 9 | require 'cunn'; 10 | 11 | --encoder 12 | encoder = nn.Sequential(); 13 | encoder:add(nn.View(-1,1,28,28)) 14 | encoder:add(nn.SpatialConvolution(1,32,3,3,1,1,1,1)) 15 | encoder:add(nn.ReLU()) 16 | encoder:add(nn.SpatialMaxPooling(2,2,2,2,1,1)) 17 | encoder:add(nn.SpatialConvolution(32,32,3,3,1,1,1,1)) 18 | encoder:add(nn.ReLU()) 19 | encoder:add(nn.SpatialMaxPooling(2,2,2,2)) 20 | 21 | --decoder 22 | decoder = nn.Sequential() 23 | decoder:add(nn.SpatialConvolution(32,32,3,3,1,1,1,1)) 24 | decoder:add(nn.ReLU()) 25 | decoder:add(nn.SpatialUpSamplingNearest(2)) 26 | decoder:add(nn.SpatialConvolution(32,32,3,3,1,1,1,1)) 27 | decoder:add(nn.ReLU()) 28 | decoder:add(nn.SpatialUpSamplingNearest(2)) 29 | decoder:add(nn.SpatialConvolution(32,1,3,3,1,1,1,1)) 30 | decoder:add(nn.Sigmoid()) 31 | decoder:add(nn.View(28,28)) 32 | 33 | --autoencoder 34 | autoencoder = nn.Sequential() 35 | noiser = nn.WhiteNoise(0,0.2) 36 | autoencoder:add(noiser) 37 | autoencoder:add(encoder) 38 | autoencoder:add(decoder) 39 | 40 | autoencoder = autoencoder:cuda() 41 | print(autoencoder) 42 | 43 | trainData = mnist.traindataset().data:double():div(255):cuda() 44 | trainlabels = mnist.traindataset().label:cuda() 45 | N = mnist.traindataset().size 46 | 47 | testData = mnist.testdataset().data:double():div(255):cuda() 48 | testlabels = mnist.testdataset().label:cuda() 49 | 50 | 51 | local theta,gradTheta = autoencoder:getParameters() 52 | 53 | local criterion = nn.BCECriterion():cuda() 54 | 55 | local x 56 | 57 | local feval = function(params) 58 | if theta~=params then 59 | theta:copy(params) 60 | end 61 | gradTheta:zero() 62 | --print(#x) 63 | local xHat = autoencoder:forward(x) 64 | local loss = criterion:forward(xHat,x) 65 | local gradLoss = criterion:backward(xHat,x) 66 | autoencoder:backward(x,gradLoss) 67 | --print(xHat[1]) 68 | return loss, gradTheta 69 | end 70 | 71 | --Train 72 | batchSize = 5000 73 | iterations = 1 74 | print('Training Starting') 75 | autoencoder:training() 76 | local optimParams = {learningRate = 0.1} 77 | local _,loss 78 | local losses = {} 79 | for epoch=1,iterations do 80 | collectgarbage() 81 | print('Epoch '..epoch..'/'..iterations) 82 | for n=1,N, batchSize do 83 | collectgarbage() 84 | x = trainData:narrow(1,n,batchSize) 85 | _,loss = optim.sgd(feval,theta,optimParams) 86 | losses[#losses + 1] = loss[1] 87 | end 88 | local plots={{'AE', torch.linspace(1,#losses,#losses), torch.Tensor(losses), '-'}} 89 | --[[gnuplot.pngfigure('DenoisingAE/Training_2.png') 90 | gnuplot.plot(table.unpack(plots)) 91 | gnuplot.ylabel('Loss') 92 | gnuplot.xlabel('Batch #') 93 | gnuplot.plotflush() 94 | --]] 95 | --permute training data 96 | trainData = trainData:index(1,torch.randperm(trainData:size(1)):long()) 97 | end 98 | 99 | print('Testing') 100 | --autoencoder:evaluate() 101 | x = testData:narrow(1,1,50) 102 | --print(x[1]) 103 | local xHat= autoencoder:forward(x) 104 | x1= noiser.output 105 | print(#encoder.output) 106 | --print(xHat[1]) 107 | --image.save('DenoisingAE/Reconstructions_2.png', torch.cat(image.toDisplayTensor(x1,2,50),image.toDisplayTensor(xHat,2,50),1)) 108 | -------------------------------------------------------------------------------- /Auto-encoder-cuda/VariationalAE.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | require 'cudnn'; 7 | require 'cunn'; 8 | require 'cutorch'; 9 | 10 | local Gaussian,parent = torch.class('nn.Gaussian','nn.Module') 11 | 12 | function Gaussian:__init(mean, stdv) 13 | parent.__init(self) 14 | self.mean = mean or 0 15 | self.stdv = stdv or 1 16 | end 17 | 18 | function Gaussian:updateOutput(inp) 19 | self.output:resizeAs(inp) 20 | self.output:normal(self.mean,self.stdv) 21 | return self.output 22 | end 23 | 24 | function Gaussian:updateGradInput(inp, gradOut) 25 | self.gradInput:resizeAs(inp) 26 | self.gradInput:zero() 27 | return self.gradInput 28 | end 29 | 30 | --encoder 31 | encoder = nn.Sequential(); 32 | encoder:add(nn.View(-1,1,28,28)) 33 | encoder:add(nn.SpatialConvolution(1,32,3,3,1,1,1,1)) 34 | encoder:add(nn.ReLU()) 35 | encoder:add(nn.SpatialMaxPooling(2,2,2,2,1,1)) 36 | encoder:add(nn.SpatialConvolution(32,32,3,3,1,1,1,1)) 37 | encoder:add(nn.ReLU()) 38 | encoder:add(nn.SpatialMaxPooling(2,2,2,2)) 39 | encoder:add(nn.View(-1,1568)) 40 | encoder:add(nn.Linear(1568,784)) 41 | encoder:add(nn.BatchNormalization(784)) 42 | encoder:add(nn.ReLU()) 43 | encoder:add(nn.Linear(784,128)) 44 | encoder:add(nn.BatchNormalization(128)) 45 | encoder:add(nn.ReLU()) 46 | encoder:add(nn.Linear(128,64)) 47 | encoder:add(nn.BatchNormalization(64)) 48 | encoder:add(nn.ReLU()) 49 | 50 | local zLayer = nn.ConcatTable() 51 | zLayer:add(nn.Linear(64,10)) 52 | zLayer:add(nn.Linear(64,10)) 53 | encoder:add(zLayer) 54 | 55 | local noiseModule = nn.Sequential() 56 | local noiseModuleInternal = nn.ConcatTable() 57 | local stdModule = nn.Sequential() 58 | stdModule:add(nn.MulConstant(0.5)) 59 | stdModule:add(nn.Exp()) 60 | noiseModuleInternal:add(stdModule) 61 | noiseModuleInternal:add(nn.Gaussian(0,1)) 62 | noiseModule:add(noiseModuleInternal) 63 | noiseModule:add(nn.CMulTable()) 64 | 65 | local sampler = nn.Sequential() 66 | local samplerInternal = nn.ParallelTable() 67 | samplerInternal:add(nn.Identity()) 68 | samplerInternal:add(noiseModule) 69 | sampler:add(samplerInternal) 70 | sampler:add(nn.CAddTable()) 71 | 72 | --decoder 73 | decoder = nn.Sequential() 74 | decoder:add(nn.Linear(10, 64)) 75 | decoder:add(nn.BatchNormalization(64)) 76 | decoder:add(nn.ReLU()) 77 | decoder:add(nn.Linear(64, 128)) 78 | decoder:add(nn.BatchNormalization(128)) 79 | decoder:add(nn.ReLU()) 80 | decoder:add(nn.Linear(128, 784)) 81 | decoder:add(nn.ReLU()) 82 | decoder:add(nn.Linear(784, 1568)) 83 | decoder:add(nn.ReLU()) 84 | decoder:add(nn.View(32,7, 7)) 85 | decoder:add(nn.SpatialConvolution(32,32,3,3,1,1,1,1)) 86 | decoder:add(nn.ReLU()) 87 | decoder:add(nn.SpatialUpSamplingNearest(2)) 88 | decoder:add(nn.SpatialConvolution(32,32,3,3,1,1,1,1)) 89 | decoder:add(nn.ReLU()) 90 | decoder:add(nn.SpatialUpSamplingNearest(2)) 91 | decoder:add(nn.SpatialConvolution(32,1,3,3,1,1,1,1)) 92 | decoder:add(nn.Sigmoid()) 93 | decoder:add(nn.View(28,28)) 94 | 95 | --autoencoder 96 | autoencoder = nn.Sequential() 97 | autoencoder:add(encoder) 98 | autoencoder:add(sampler) 99 | autoencoder:add(decoder) 100 | autoencoder=autoencoder:cuda() 101 | print(autoencoder) 102 | 103 | trainData = mnist.traindataset().data:double():div(255):cuda() 104 | trainlabels = mnist.traindataset().label:cuda() 105 | N = mnist.traindataset().size 106 | 107 | testData = mnist.testdataset().data:double():div(255):cuda() 108 | testlabels = mnist.testdataset().label:cuda() 109 | 110 | local theta,gradTheta = autoencoder:getParameters() 111 | 112 | local criterion = nn.BCECriterion():cuda() 113 | 114 | local x 115 | 116 | local feval = function(params) 117 | if theta~=params then 118 | theta:copy(params) 119 | end 120 | gradTheta:zero() 121 | --print(#x) 122 | local xHat = autoencoder:forward(x) 123 | local loss = criterion:forward(xHat,x) 124 | local gradLoss = criterion:backward(xHat,x) 125 | autoencoder:backward(x,gradLoss) 126 | 127 | -- Optimise Gaussian KL divergence between inference model and prior: DKL[q(z|x)||N(0, σI)] = log(σ2/σ1) + ((σ1^2 - σ2^2) + (μ1 - μ2)^2) / 2σ2^2 128 | local nElements = xHat:nElement() 129 | local mean, logVar_sq = table.unpack(encoder.output) 130 | local var_sq = torch.exp(logVar_sq) 131 | local KLLoss = 0.5 * torch.sum(torch.pow(mean, 2) + var_sq - logVar_sq - 1) 132 | KLLoss = KLLoss/nElements 133 | loss = loss + KLLoss 134 | local gradKLLoss = {mean / nElements, 0.5*(var_sq - 1) / nElements} 135 | encoder:backward(x,gradKLLoss) 136 | return loss, gradTheta 137 | end 138 | 139 | --Train 140 | batchSize = 4000 141 | iterations = 25 142 | print('Training Starting') 143 | local optimParams = {learningRate = 0.1} 144 | local _,loss 145 | local losses = {} 146 | for epoch=1,iterations do 147 | collectgarbage() 148 | print('Epoch '..epoch..'/'..iterations) 149 | for n=1,N, batchSize do 150 | collectgarbage() 151 | x = trainData:narrow(1,n,batchSize) 152 | _,loss = optim.sgd(feval,theta,optimParams) 153 | losses[#losses + 1] = loss[1] 154 | end 155 | local plots={{'AE', torch.linspace(1,#losses,#losses), torch.Tensor(losses), '-'}} 156 | gnuplot.pngfigure('VarAE/Training_2.png') 157 | gnuplot.plot(table.unpack(plots)) 158 | gnuplot.ylabel('Loss') 159 | gnuplot.xlabel('Batch #') 160 | gnuplot.plotflush() 161 | 162 | --permute training data 163 | trainData = trainData:index(1,torch.randperm(trainData:size(1)):long()) 164 | end 165 | 166 | print('Testing') 167 | autoencoder:evaluate() 168 | x = testData:narrow(1,1,10) 169 | local xHat= autoencoder:forward(x) 170 | 171 | image.save('VarAE/Reconstructions_2.png', torch.cat(image.toDisplayTensor(x,2,10),image.toDisplayTensor(xHat,2,10),1)) -------------------------------------------------------------------------------- /Auto-encoder-cuda/deploy_adversarialAE_alex.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | require 'optim'; 4 | require 'gnuplot'; 5 | require 'cutorch'; 6 | require 'cunn'; 7 | require 'cudnn'; 8 | 9 | --define function binarise 10 | function binarise(x,thresh,low,high) 11 | y = torch.Tensor(x:size()) 12 | for i=1,x:size(2) do 13 | if x[1][i]<=thresh then 14 | y[1][i]=low 15 | else 16 | y[1][i]=high 17 | end 18 | end 19 | return y 20 | end 21 | 22 | --define function to sort table 23 | function compare(a,b) 24 | return tostring(a[2])1000: 399 | break 400 | t_inputs,t_labels = t_data 401 | t_inputs = Variable(t_inputs).cuda() 402 | t_labels = t_labels.cuda() 403 | t_outputs = autoencoder(t_inputs) 404 | c_pred = classification(t_inputs) 405 | _, predicted = torch.max(c_pred.data, 1) 406 | #print predicted.type() , t_labels.type() 407 | total += t_labels.size(0) 408 | correct += (predicted == t_labels).sum() 409 | if (epoch + 1)%1 == 0: 410 | print("saving image") 411 | test_result_path = os.path.join('', 'batch_%d_%d'%((epoch+1)/1,t_i+1) + '.jpg') 412 | image_tensor = torch.cat((t_inputs.data[0:8], t_outputs.data[0:8]), 0) 413 | torchvision.utils.save_image(image_tensor, test_result_path) 414 | 415 | print('Accuracy of the network on the 8000 test images: %d %%' % (100 * correct / total)) 416 | 417 | print('Finished Training and Testing') 418 | 419 | 420 | """ 421 | classification_criterion = nn.NLLLoss() 422 | for i,data in enumerate(testloader): 423 | inputs,labels = data 424 | inputs,labels = Variable(inputs).cuda(), Variable(labels).cuda() 425 | test_model = Autoencoder().cuda() 426 | test_model.load_state_dict(torch.load('/home/siplab/Saket/AE_Classifier/Autoencoder/autoencoder-500.pkl',map_location=lambda storage, loc: storage.cuda(1))) 427 | outputs = test_model(inputs) 428 | test_result_path = os.path.join('/home/siplab/Saket/AE_Classifier/Test_results/', 'batch_%d'%(i+1) + '.jpg') 429 | image_tensor = torch.cat((inputs.data[0:8], outputs.data[0:8]), 0) 430 | torchvision.utils.save_image(image_tensor, test_result_path) 431 | """ 432 | -------------------------------------------------------------------------------- /Results/README.md: -------------------------------------------------------------------------------- 1 | # Result Readme 2 | This folder contains the training curves and reconstruction images. 3 | -------------------------------------------------------------------------------- /Results/Reconstructions_mnist_cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arnaghosh/Auto-Encoder/c0a2e8fd7eb2b4c609d2096ede2915a7dd61fba7/Results/Reconstructions_mnist_cnn.png -------------------------------------------------------------------------------- /Results/Training_mnist_advCnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arnaghosh/Auto-Encoder/c0a2e8fd7eb2b4c609d2096ede2915a7dd61fba7/Results/Training_mnist_advCnn.png -------------------------------------------------------------------------------- /advAE_cnn_mnist.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | --require 'cutorch'; 7 | --require 'cunn'; 8 | --require 'cudnn'; 9 | require './BinarizedNeurons' 10 | 11 | zSize = 14 12 | --encoder 13 | encoder = nn.Sequential(); 14 | encoder:add(nn.SpatialConvolution(1,6,5,5,1,1,2,2)) --1 15 | encoder:add(nn.ReLU()) --2 16 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --3 17 | -- size: 6X14X14 18 | 19 | encoder:add(nn.SpatialConvolution(6,16,5,5,1,1,2,2)) --4 20 | encoder:add(nn.ReLU()) --5 21 | encoder:add(nn.SpatialMaxPooling(2, 2, 2, 2)) --6 22 | -- size: 16X7X7 23 | 24 | encoder:add(nn.View(-1):setNumInputDims(3)) --7 25 | encoder:add(nn.Linear(784,120)) --8 26 | encoder:add(nn.ReLU()) --9 27 | encoder:add(nn.Linear(120,84)) --10 28 | encoder:add(nn.ReLU()) --11 29 | encoder:add(nn.Linear(84,zSize)) --12 30 | 31 | binariser = nn.Sequential(); 32 | binariser:add(BinarizedNeurons()) 33 | --encoder = torch.load('model_MNIST2.t7') 34 | --encoder:remove(13) 35 | --encoder:add(nn.SoftMax()) 36 | 37 | --decoder 38 | decoder = nn.Sequential() 39 | decoder:add(nn.Linear(zSize, 84)) --1 40 | decoder:add(nn.BatchNormalization(84)) --2 41 | decoder:add(nn.ReLU()) --3 42 | decoder:add(nn.Linear(84, 120)) --4 43 | decoder:add(nn.BatchNormalization(120)) --5 44 | decoder:add(nn.ReLU()) --6 45 | decoder:add(nn.Linear(120, 784)) --7 46 | decoder:add(nn.BatchNormalization(784)) --8 47 | decoder:add(nn.ReLU()) --9 48 | decoder:add(nn.View(16,7,7)) --10 49 | --size: 16X7X7 50 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(6))) --11 51 | decoder:add(nn.SpatialConvolution(16,6,5,5,1,1,2,2)) --12 52 | decoder:add(nn.ReLU()) --13 53 | --size: 6X14X14 54 | 55 | decoder:add(nn.SpatialMaxUnpooling(encoder:get(3))) --14 56 | decoder:add(nn.SpatialConvolution(6,1,5,5,1,1,2,2)) --15 57 | decoder:add(nn.Sigmoid()) --16 58 | --size: 1X28X28 59 | 60 | --autoencoder 61 | autoencoder = nn.Sequential() 62 | autoencoder:add(encoder) 63 | autoencoder:add(binariser) 64 | autoencoder:add(decoder) 65 | 66 | autoencoder = autoencoder--:cuda() 67 | print(autoencoder) 68 | 69 | --adversary network 70 | adversary = nn.Sequential() 71 | adversary:add(nn.Linear(zSize, 64)) 72 | --adversary:add(nn.BatchNormalization(64)) 73 | adversary:add(nn.ReLU()) 74 | adversary:add(nn.Linear(64, 16)) 75 | --adversary:add(nn.BatchNormalization(16)) 76 | adversary:add(nn.ReLU()) 77 | adversary:add(nn.Linear(16, 10)) 78 | --adversary:add(nn.BatchNormalization(1)) 79 | adversary:add(nn.LogSoftMax()) 80 | 81 | adversary = adversary--:cuda() 82 | print(adversary) 83 | 84 | --load MNIST data 85 | trainData = mnist.traindataset().data:double():div(255):reshape(60000,1,28,28)--:cuda() 86 | trainlabels = (mnist.traindataset().label+1)--:cuda() 87 | N = mnist.traindataset().size 88 | 89 | testData = mnist.testdataset().data:double():div(255):reshape(10000,1,28,28)--:cuda() 90 | testlabels = (mnist.testdataset().label+1)--:cuda() 91 | teSize = mnist.testdataset().size 92 | print(N,teSize) 93 | --[[ 94 | trainset = torch.load('cifar10-train.t7') 95 | testset = torch.load('cifar10-test.t7') 96 | classes = {'airplane', 'automobile', 'bird', 'cat', 97 | 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'} 98 | 99 | trainData = trainset.data:double():div(255):cuda() 100 | --]] 101 | --[[trainData = torch.ByteTensor(trainset.data:size()) 102 | print(#trainData) 103 | for i=1,trainData:size()[1] do 104 | trainData[i] = image.rgb2hsv(trainset.data[i]) 105 | end 106 | trainData = trainData:double():div(255):cuda() 107 | --]] 108 | --[[trainlabel=trainset.label:cuda() 109 | N = trainData:size()[1] 110 | testData = testset.data 111 | testlabels = testset.label:cuda() 112 | --]] 113 | 114 | local theta,gradTheta = autoencoder:getParameters() 115 | local thetaAdv,gradThetaAdv = adversary:getParameters() 116 | 117 | local criterion = nn.BCECriterion()--:cuda() 118 | local classification_criterion = nn.ClassNLLCriterion()--:cuda() 119 | 120 | local x,y 121 | 122 | batchSize = 3000 123 | iterations = 50 124 | 125 | local feval = function(params) 126 | if theta~=params then 127 | theta:copy(params) 128 | end 129 | gradTheta:zero() 130 | gradThetaAdv:zero() 131 | --print(#x) 132 | --[[local x1 = encoder:forward(x) 133 | local x2 = torch.sign(x1) 134 | local x2 = nn.Threshold(0,0):forward(x2) 135 | local xHat = decoder:forward(x2) 136 | 137 | --]] 138 | local xHat = autoencoder:forward(x) 139 | local loss = criterion:forward(xHat,x) 140 | local gradLoss = criterion:backward(xHat,x) 141 | --[[decoder:backward(x2,gradLoss) 142 | local gradEnc = decoder:updateGradInput(x2,gradLoss) 143 | encoder:backward(x,gradEnc) 144 | --]] 145 | autoencoder:backward(x,gradLoss) 146 | 147 | -- Train adversary to maximise log probability of real samples: max_D log(D(x)) 148 | local pred = adversary:forward(encoder.output) 149 | advLoss = classification_criterion:forward(pred,y) 150 | local gradAdvLoss = classification_criterion:backward(pred,y) 151 | adversary:backward(encoder.output,gradAdvLoss) 152 | local gradAdv = adversary:updateGradInput(encoder.output, gradAdvLoss) 153 | encoder:backward(x,gradAdv) 154 | -- Train encoder (generator) to play a minimax game with the adversary (discriminator): min_G max_D log(1 - D(G(x))) 155 | --[[local minimaxLoss = criterion:forward(pred,YReal) -- Technically use max_G max_D log(D(G(x))) for same fixed point, stronger initial gradients 156 | loss = loss + minimaxLoss 157 | local gradMinimaxLoss = criterion:backward(pred,YReal) 158 | local gradMinimax = adversary:updateGradInput(encoder.output, gradMinimaxLoss) 159 | encoder:backward(x,gradMinimax) 160 | --]] 161 | return loss, gradTheta 162 | end 163 | 164 | local advFeval = function(params) 165 | if thetaAdv~=params then 166 | thetaAdv:copy(params) 167 | end 168 | return advLoss, gradThetaAdv 169 | end 170 | 171 | --Train 172 | print('Training Starting') 173 | local optimParams = {learningRate = 0.001} 174 | local advOptimParams = {learningRate = 0.001} 175 | local _,loss 176 | local losses, advLosses = {},{} 177 | for epoch=1,iterations do 178 | autoencoder:training() 179 | collectgarbage() 180 | print('Epoch '..epoch..'/'..iterations) 181 | for n=1,N,batchSize do 182 | collectgarbage() 183 | x = trainData:narrow(1,n,batchSize)--:cuda() 184 | --print(x:size()) 185 | y = trainlabels:narrow(1,n,batchSize)--:cuda() 186 | _,loss = optim.adam(feval,theta,optimParams) 187 | losses[#losses + 1] = loss[1] 188 | _,loss = optim.adam(advFeval,thetaAdv,advOptimParams) 189 | advLosses[#advLosses + 1] = loss[1] 190 | end 191 | local plots={{'reconstruction', torch.linspace(1,#losses,#losses), torch.Tensor(losses), '-'}} 192 | plots[2]={'Adversary', torch.linspace(1,#advLosses,#advLosses), torch.Tensor(advLosses), '-'} 193 | totLoss = torch.Tensor(losses)+torch.Tensor(advLosses) 194 | plots[3]={'Recons+Adversary', torch.linspace(1,totLoss:size(1),totLoss:size(1)), totLoss, '-'} 195 | gnuplot.pngfigure('AdvAE/Training_mnist_advCnn.png') 196 | gnuplot.plot(table.unpack(plots)) 197 | gnuplot.ylabel('Loss') 198 | gnuplot.xlabel('Batch #') 199 | gnuplot.plotflush() 200 | 201 | --permute training data 202 | indices = torch.randperm(trainData:size(1)):long()--:cuda() 203 | trainData = trainData:index(1,indices)--:cuda() 204 | trainlabels = trainlabels:index(1,indices)--:cuda() 205 | 206 | autoencoder:evaluate() 207 | x = testData:narrow(1,1,50)--:cuda() 208 | --[[local x_hsv = torch.Tensor(x:size()):typeAs(x) 209 | for i=1,x:size()[1] do 210 | x_hsv[i] = image.rgb2hsv(x[i]) 211 | end 212 | --]] 213 | x_hsv = x; 214 | local xHat_hsv= autoencoder:forward(x_hsv) 215 | --[[xHat_hsv = xHat_hsv:mul(255):byte() 216 | for i=1,50 do 217 | print(i) 218 | print(xHat_hsv[i][1]:min(),xHat_hsv[i][1]:max()) 219 | print(xHat_hsv[i][2]:min(),xHat_hsv[i][2]:max()) 220 | print(xHat_hsv[i][3]:min(),xHat_hsv[i][3]:max()) 221 | end 222 | --]] 223 | --[[local xHat = torch.Tensor(xHat_hsv:size()):typeAs(xHat_hsv) 224 | for i=1,xHat_hsv:size()[1] do 225 | xHat[i] = image.hsv2rgb(xHat_hsv[i]) 226 | end 227 | --]] 228 | 229 | --print (#x) 230 | ---print(#xHat) 231 | --temp=torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2) 232 | --print (#temp) 233 | image.save('AdvAE/Reconstructions_mnist_temp.png', torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat_hsv,2,50),2)) 234 | end 235 | 236 | print('Testing') 237 | x = testData:narrow(1,1,50)--:cuda() 238 | --[[local x_hsv = torch.Tensor(x:size()):typeAs(x) 239 | for i=1,x:size()[1] do 240 | x_hsv[i] = image.rgb2hsv(x[i]) 241 | end 242 | --]] 243 | x_hsv = x 244 | local xHat_hsv= autoencoder:forward(x_hsv) 245 | --[[xHat_hsv = xHat_hsv:mul(255):byte() 246 | for i=1,50 do 247 | print(i) 248 | print(x_hsv[i][1]:min(),x_hsv[i][1]:min()) 249 | print(x_hsv[i][2]:min(),x_hsv[i][2]:min()) 250 | print(x_hsv[i][3]:min(),x_hsv[i][3]:min()) 251 | end 252 | --]] 253 | --[[local xHat = torch.Tensor(xHat_hsv:size()):typeAs(xHat_hsv) 254 | for i=1,xHat_hsv:size()[1] do 255 | xHat[i] = image.hsv2rgb(xHat_hsv[i]) 256 | end 257 | --]] 258 | 259 | --print (#x) 260 | ---print(#xHat) 261 | --temp=torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat,2,50),2) 262 | --print (#temp) 263 | image.save('AdvAE/Reconstructions_mnist_cnn.png', torch.cat(image.toDisplayTensor(x,2,50),image.toDisplayTensor(xHat_hsv,2,50),2)) 264 | torch.save('AdvAE/encoder_1.t7',encoder) 265 | torch.save('AdvAE/decoder_1.t7',decoder) 266 | -------------------------------------------------------------------------------- /mnist_deploy.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | require 'cutorch'; 7 | require 'cunn'; 8 | require 'cudnn'; 9 | require './BinarizedNeurons' 10 | 11 | classes ={0,1,2,3,4,5,6,7,8,9} 12 | zSize = 16 13 | --encoder 14 | encoder = torch.load('/home/siplab/AE/mnist/encoder_1.t7'); 15 | classifier=torch.load('/home/siplab/AE/mnist/classifier.t7'); 16 | binariser = nn.Sequential(); 17 | binariser:add(BinarizedNeurons()) 18 | 19 | autoencoder = nn.Sequential() 20 | autoencoder:add(encoder) 21 | autoencoder:add(binariser) 22 | autoencoder:add(classifier) 23 | autoencoder = autoencoder:cuda() 24 | print(autoencoder) 25 | 26 | --load MNIST data 27 | trainData = mnist.traindataset().data:double():div(255):reshape(60000,1,28,28):cuda() 28 | trainlabels = (mnist.traindataset().label+1):cuda() 29 | N = mnist.traindataset().size 30 | 31 | testData = mnist.testdataset().data:double():div(255):reshape(10000,1,28,28):cuda() 32 | testlabels = (mnist.testdataset().label+1):cuda() 33 | teSize = mnist.testdataset().size 34 | print(N,teSize) 35 | class_performance = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0} 36 | gt=torch.Tensor(10000,1):cuda() 37 | for i=1,10000 do 38 | gt[i]=testlabels[i] 39 | end 40 | count=0 41 | 42 | for n=1,10000 do 43 | local groundtruth = gt[n] 44 | x=autoencoder:forward(testData[n]) 45 | confidences, indices = torch.sort(x, true) 46 | print (indices[1],gt[n]) 47 | if torch.Tensor(1):fill(indices[1]):cuda():equal(groundtruth) then 48 | --print(1,indices[1]) 49 | count=count+1 50 | --print(2,groundtruth) 51 | class_performance[indices[1]] = class_performance[indices[1]] + 1 52 | end 53 | end 54 | end 55 | for i=1,#classes do 56 | print (class_performance[i]) 57 | end 58 | for i=1,#classes do 59 | print(classes[i], 100*class_performance[i]/1000 .. ' %') 60 | end 61 | -------------------------------------------------------------------------------- /prepare_dict.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | --require 'cutorch'; 7 | --require 'cunn'; 8 | --require 'cudnn'; 9 | require './BinarizedNeurons' 10 | 11 | zSize = 14 12 | --encoder 13 | encoder = torch.load('/media/arna/340fd3c9-2648-4333-9ec9-239babc34bb7/arna_data/AdvAE_data/encoder1.t7'); 14 | 15 | binariser = nn.Sequential(); 16 | binariser:add(BinarizedNeurons()) 17 | 18 | autoencoder = nn.Sequential() 19 | autoencoder:add(encoder) 20 | autoencoder:add(binariser) 21 | 22 | autoencoder = autoencoder--:cuda() 23 | print(autoencoder) 24 | 25 | --load MNIST data 26 | trainData = mnist.traindataset().data:double():div(255):reshape(60000,1,28,28)--:cuda() 27 | trainlabels = (mnist.traindataset().label+1)--:cuda() 28 | N = mnist.traindataset().size 29 | 30 | testData = mnist.testdataset().data:double():div(255):reshape(10000,1,28,28)--:cuda() 31 | testlabels = (mnist.testdataset().label+1)--:cuda() 32 | teSize = mnist.testdataset().size 33 | print(N,teSize) 34 | 35 | local dict = torch.Tensor(N,zSize); 36 | 37 | local x,y 38 | 39 | batchSize = 3000 40 | iterations = 50 41 | 42 | 43 | --Train 44 | print('Dictionary preparing') 45 | 46 | for n=1,N,batchSize do 47 | collectgarbage() 48 | x = trainData:narrow(1,n,batchSize)--:cuda() 49 | --print(x:size()) 50 | x1 = autoencoder:forward(x) 51 | dict[{{n,n+batchSize-1},{}}] = x1; 52 | end 53 | 54 | torch.save('AdvAE/mnist_dict.t7',dict) -------------------------------------------------------------------------------- /resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | import torchvision 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from torchvision import datasets, models,transforms 7 | import torch.optim as optim 8 | from torch.optim import lr_scheduler 9 | import numpy as np 10 | import os 11 | import matplotlib.pyplot as plt 12 | from torch.autograd import Function 13 | from collections import OrderedDict 14 | import torch.nn as nn 15 | import math 16 | zsize = 48 17 | batch_size = 11 18 | iterations = 500 19 | learningRate= 0.0001 20 | 21 | import torchvision.models as models 22 | #ResNEt##################################### 23 | def conv3x3(in_planes, out_planes, stride=1): 24 | """3x3 convolution with padding""" 25 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 26 | padding=1, bias=False) 27 | 28 | class BasicBlock(nn.Module): 29 | expansion = 1 30 | 31 | def __init__(self, inplanes, planes, stride=1, downsample=None): 32 | super(BasicBlock, self).__init__() 33 | self.conv1 = conv3x3(inplanes, planes, stride) 34 | self.bn1 = nn.BatchNorm2d(planes) 35 | self.relu = nn.ReLU(inplace=True) 36 | self.conv2 = conv3x3(planes, planes) 37 | self.bn2 = nn.BatchNorm2d(planes) 38 | self.downsample = downsample 39 | self.stride = stride 40 | 41 | def forward(self, x): 42 | residual = x 43 | 44 | out = self.conv1(x) 45 | out = self.bn1(out) 46 | out = self.relu(out) 47 | 48 | out = self.conv2(out) 49 | out = self.bn2(out) 50 | 51 | if self.downsample is not None: 52 | residual = self.downsample(x) 53 | 54 | out += residual 55 | out = self.relu(out) 56 | 57 | return out 58 | 59 | 60 | class Bottleneck(nn.Module): 61 | expansion = 4 62 | 63 | def __init__(self, inplanes, planes, stride=1, downsample=None): 64 | super(Bottleneck, self).__init__() 65 | self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) 66 | self.bn1 = nn.BatchNorm2d(planes) 67 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, 68 | padding=1, bias=False) 69 | self.bn2 = nn.BatchNorm2d(planes) 70 | self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) 71 | self.bn3 = nn.BatchNorm2d(planes * 4) 72 | self.relu = nn.ReLU(inplace=True) 73 | self.downsample = downsample 74 | self.stride = stride 75 | 76 | def forward(self, x): 77 | residual = x 78 | 79 | out = self.conv1(x) 80 | 81 | out = self.bn1(out) 82 | out = self.relu(out) 83 | 84 | out = self.conv2(out) 85 | out = self.bn2(out) 86 | out = self.relu(out) 87 | 88 | out = self.conv3(out) 89 | out = self.bn3(out) 90 | 91 | if self.downsample is not None: 92 | residual = self.downsample(x) 93 | 94 | out += residual 95 | out = self.relu(out) 96 | 97 | return out 98 | 99 | ############################################################### 100 | 101 | 102 | 103 | ############################################################### 104 | class Encoder(nn.Module): 105 | 106 | def __init__(self, block, layers, num_classes=23): 107 | self.inplanes = 64 108 | super (Encoder, self).__init__() 109 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 110 | bias=False) 111 | self.bn1 = nn.BatchNorm2d(64) 112 | self.relu = nn.ReLU(inplace=True) 113 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)#, return_indices = True) 114 | self.layer1 = self._make_layer(block, 64, layers[0]) 115 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 116 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 117 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 118 | self.avgpool = nn.AvgPool2d(7, stride=1) 119 | self.fc = nn.Linear(512 * block.expansion, 1000) 120 | #self.fc = nn.Linear(num_classes,16) 121 | for m in self.modules(): 122 | if isinstance(m, nn.Conv2d): 123 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 124 | m.weight.data.normal_(0, math.sqrt(2. / n)) 125 | elif isinstance(m, nn.BatchNorm2d): 126 | m.weight.data.fill_(1) 127 | m.bias.data.zero_() 128 | 129 | def _make_layer(self, block, planes, blocks, stride=1): 130 | downsample = None 131 | if stride != 1 or self.inplanes != planes * block.expansion: 132 | downsample = nn.Sequential( 133 | nn.Conv2d(self.inplanes, planes * block.expansion, 134 | kernel_size=1, stride=stride, bias=False), 135 | nn.BatchNorm2d(planes * block.expansion), 136 | ) 137 | 138 | layers = [] 139 | layers.append(block(self.inplanes, planes, stride, downsample)) 140 | self.inplanes = planes * block.expansion 141 | for i in range(1, blocks): 142 | layers.append(block(self.inplanes, planes)) 143 | 144 | return nn.Sequential(*layers) 145 | 146 | def forward(self, x): 147 | x = self.conv1(x) 148 | 149 | x = self.bn1(x) 150 | x = self.relu(x) 151 | 152 | x = self.maxpool(x) 153 | 154 | x = self.layer1(x) 155 | x = self.layer2(x) 156 | x = self.layer3(x) 157 | x = self.layer4(x) 158 | 159 | x = self.avgpool(x) 160 | x = x.view(x.size(0), -1) 161 | x = self.fc(x) 162 | 163 | return x 164 | encoder = Encoder(Bottleneck, [3, 4, 6, 3]) 165 | encoder.load_state_dict(torch.load('/home/deepkliv/Downloads/resnet50-19c8e357.pth'))#,map_location=lambda storage, loc: storage.cuda(1)),strict=False) 166 | #loaded_weights = torch.load('/home/siplab/Saket/resnet18-5c106cde.pth') 167 | #print encoder.layer1[1].conv1.weight.data[0][0] 168 | encoder.fc = nn.Linear(2048, 48) 169 | #for param in encoder.parameters(): 170 | # param.requires_grad = False 171 | encoder=encoder.cuda() 172 | y=torch.rand(1,3,224,224) 173 | x=torch.rand(1,128) 174 | x=Variable(x.cuda()) 175 | #print decoder(x) 176 | #y=Variable(y.cuda()) 177 | #print("\n") 178 | #encoder(y) 179 | #print encoder(y) 180 | ########################################################################## 181 | class Binary(Function): 182 | 183 | @staticmethod 184 | def forward(ctx, input): 185 | return F.relu(Variable(input.sign())).data 186 | 187 | @staticmethod 188 | def backward(ctx, grad_output): 189 | return grad_output 190 | 191 | binary = Binary() 192 | ########################################################################## 193 | class Decoder(nn.Module): 194 | def __init__(self): 195 | super(Decoder,self).__init__() 196 | self.dfc3 = nn.Linear(zsize, 4096) 197 | self.bn3 = nn.BatchNorm2d(4096) 198 | self.dfc2 = nn.Linear(4096, 4096) 199 | self.bn2 = nn.BatchNorm2d(4096) 200 | self.dfc1 = nn.Linear(4096,256 * 6 * 6) 201 | self.bn1 = nn.BatchNorm2d(256*6*6) 202 | self.upsample1=nn.Upsample(scale_factor=2) 203 | self.dconv5 = nn.ConvTranspose2d(256, 256, 3, padding = 0) 204 | self.dconv4 = nn.ConvTranspose2d(256, 384, 3, padding = 1) 205 | self.dconv3 = nn.ConvTranspose2d(384, 192, 3, padding = 1) 206 | self.dconv2 = nn.ConvTranspose2d(192, 64, 5, padding = 2) 207 | self.dconv1 = nn.ConvTranspose2d(64, 3, 12, stride = 4, padding = 4) 208 | 209 | def forward(self,x):#,i1,i2,i3): 210 | 211 | x = self.dfc3(x) 212 | #x = F.relu(x) 213 | x = F.relu(self.bn3(x)) 214 | 215 | x = self.dfc2(x) 216 | x = F.relu(self.bn2(x)) 217 | #x = F.relu(x) 218 | x = self.dfc1(x) 219 | x = F.relu(self.bn1(x)) 220 | #x = F.relu(x) 221 | #print(x.size()) 222 | x = x.view(batch_size,256,6,6) 223 | #print (x.size()) 224 | x=self.upsample1(x) 225 | #print x.size() 226 | x = self.dconv5(x) 227 | #print x.size() 228 | x = F.relu(x) 229 | #print x.size() 230 | x = F.relu(self.dconv4(x)) 231 | #print x.size() 232 | x = F.relu(self.dconv3(x)) 233 | #print x.size() 234 | x=self.upsample1(x) 235 | #print x.size() 236 | x = self.dconv2(x) 237 | #print x.size() 238 | x = F.relu(x) 239 | x=self.upsample1(x) 240 | #print x.size() 241 | x = self.dconv1(x) 242 | #print x.size() 243 | x = F.sigmoid(x) 244 | #print x 245 | return x 246 | decoder = Decoder() 247 | ########################################## 248 | class Autoencoder(nn.Module): 249 | def __init__(self): 250 | super(Autoencoder,self).__init__() 251 | self.encoder = encoder 252 | self.binary = Binary() 253 | self.decoder = Decoder() 254 | 255 | def forward(self,x): 256 | #x=Encoder(x) 257 | x = self.encoder(x) 258 | x = binary.apply(x) 259 | #print x 260 | #x,i2,i1 = self.binary(x) 261 | #x=Variable(x) 262 | x = self.decoder(x) 263 | return x 264 | 265 | #print Autoencoder() 266 | 267 | autoencoder = Autoencoder() 268 | #autoencoder = torch.nn.DataParallel(autoencoder, device_ids=[0, 1, 2]) 269 | class Classifier(nn.Module): 270 | def __init__(self): 271 | super(Classifier,self).__init__() 272 | self.L1 = nn.Linear(zsize,64) 273 | self.L2 = nn.Linear(64,32) 274 | self.L3 = nn.Linear(32,23) 275 | 276 | def forward(self,x): 277 | x = F.relu(self.L1(x)) 278 | x = F.relu(self.L2(x)) 279 | x = F.log_softmax(self.L3(x)) 280 | return x 281 | 282 | 283 | #print Classifier() 284 | classifier = Classifier() 285 | #classifier = torch.nn.DataParallel(classifier, device_ids=[0, 1, 2]) 286 | class Classification(nn.Module): 287 | def __init__(self): 288 | super(Classification,self).__init__() 289 | self.encoder = encoder 290 | self.binary = Binary() 291 | self.classifier = Classifier() 292 | 293 | def forward(self,x): 294 | x= self.encoder(x) 295 | x = binary.apply(x) 296 | #x= self.binary(x) 297 | x = self.classifier(x) 298 | return x 299 | 300 | #print Classification() 301 | classification = Classification() 302 | 303 | ########################## 304 | 305 | if torch.cuda.is_available(): 306 | autoencoder.cuda() 307 | classification.cuda() 308 | decoder.cuda() 309 | encoder.cuda() 310 | classifier.cuda() 311 | #data 312 | 313 | plt.ion() 314 | 315 | 316 | use_gpu = torch.cuda.is_available() 317 | if use_gpu: 318 | pinMem = True # Flag for pinning GPU memory 319 | print('GPU is available!') 320 | else: 321 | pinMem = False 322 | net = models.resnet18(pretrained=False) 323 | transform = transforms.Compose( 324 | [ 325 | transforms.Scale((224,224), interpolation=2), 326 | transforms.ToTensor(), 327 | #transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]), 328 | #transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5)) 329 | ]) 330 | trainset=torchvision.datasets.ImageFolder("/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/dataset/train", transform=transform, target_transform=None) 331 | trainloader = torch.utils.data.DataLoader(trainset, shuffle = True , batch_size = batch_size , num_workers = 2) 332 | testset=torchvision.datasets.ImageFolder("/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/dataset/test", transform=transform, target_transform=None) 333 | testloader = torch.utils.data.DataLoader(testset, shuffle = True , batch_size = batch_size , num_workers = 2) 334 | 335 | autoencoder_criterion = nn.MSELoss() 336 | classification_criterion = nn.NLLLoss() 337 | 338 | autoencoder_optimizer = optim.Adam(autoencoder.parameters(), lr = learningRate) 339 | classification_optimizer = optim.Adam(classification.parameters(), lr = learningRate) 340 | #encoder_optimizer = optim.Adam(Encoder.parameters(), lr = learningRate) 341 | list_a_loss = [] 342 | list_c_loss = [] 343 | 344 | #fig = plt.figure() 345 | for epoch in range(iterations): 346 | run_loss = 0 347 | run_c_loss = 0 348 | autoencoder.train(True) # For training 349 | classification.train(True) 350 | for i,data in enumerate(trainloader): 351 | #print i 352 | inputs, labels = data 353 | inputs, labels = Variable(inputs).cuda(), Variable(labels).cuda() 354 | 355 | 356 | autoencoder_optimizer.zero_grad() 357 | classification_optimizer.zero_grad() 358 | #print(inputs.size()) 359 | pred = autoencoder(inputs) 360 | #torchvision.utils.save_image(pred.data[0:8], os.path.join('/home/deepkliv/Saket/AE_Classifier/', 'batch_%d_%d'%((epoch+1)/1,i+1) + '.jpg')) 361 | a_loss = autoencoder_criterion(pred , inputs) 362 | a_loss.backward() 363 | autoencoder_optimizer.step() 364 | 365 | #print("efc3", autoencoder.encoder.fc3.bias.grad) 366 | 367 | class_pred = classification(inputs) 368 | 369 | c_loss = classification_criterion(class_pred , labels) 370 | 371 | #_,xxpred = torch.max(class_pred.data, 1) 372 | #print("class_pred") 373 | #print(xxpred.cpu().numpy()) 374 | c_loss.backward(retain_graph=True) 375 | classification_optimizer.step() 376 | #encoder_optimizer.step() 377 | 378 | run_loss += a_loss.data[0] 379 | run_c_loss += c_loss.data[0] 380 | #print i 381 | if (i +1) % 2 == 0: 382 | print('[%d, %5d] Autoencoder loss: %.3f Classification loss: %.3f' % (epoch + 1, i + 1 , run_loss/2 , run_c_loss/2)) 383 | #print('[%d,%5d] Classification loss: %.3f' % (epoch + 1, i + 1, run_c_loss/10)) 384 | run_c_loss = 0.0 385 | run_loss = 0.0 386 | 387 | 388 | decoder_path = os.path.join('/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/Decoder/', 'decoder-%d.pkl' %(epoch+1)) 389 | encoder_path = os.path.join('/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/Encoder/', 'encoder-%d.pkl' %(epoch+1)) 390 | autoencoder_path = os.path.join('/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/Autoencoder/', 'autoencoder-%d.pkl' %(epoch+1)) 391 | classifier_path = os.path.join('/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/Classifier/', 'classifier-%d.pkl' %(epoch+1)) 392 | classification_path = os.path.join('/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/Classification','classification-%d.pkl' %(epoch+1)) 393 | 394 | torch.save(decoder.state_dict(), decoder_path) 395 | torch.save(encoder.state_dict(), encoder_path) 396 | torch.save(autoencoder.state_dict(), autoencoder_path) 397 | torch.save(classifier.state_dict(), classifier_path) 398 | torch.save(classification.state_dict(), classification_path) 399 | 400 | if ( epoch+1 )% 1 == 0: 401 | list_a_loss.append(run_loss/5000) 402 | list_c_loss.append(run_c_loss/5000) 403 | 404 | #plt.plot(range(epoch+1),list_a_loss,'r--',label='autoencoder') 405 | #plt.plot(range(epoch+1),list_c_loss,'b--',label='classifier') 406 | #if epoch==0: 407 | #plt.legend(loc='upper left') 408 | #plt.xlabel('Epochs') 409 | #plt.ylabel('Loss') 410 | #fig.savefig('/home/deepkliv/Saket/loss_plot.png') 411 | correct = 0 412 | total = 0 413 | print('\n Testing ....') 414 | autoencoder.train(False) # For training 415 | classification.train(False) 416 | for t_i,t_data in enumerate(testloader): 417 | 418 | if t_i * batch_size >1000: 419 | break 420 | t_inputs,t_labels = t_data 421 | t_inputs = Variable(t_inputs).cuda() 422 | t_labels = t_labels.cuda() 423 | t_outputs = autoencoder(t_inputs) 424 | c_pred = classification(t_inputs) 425 | _, predicted = torch.max(c_pred.data, 1) 426 | #print predicted.type() , t_labels.type() 427 | total += t_labels.size(0) 428 | correct += (predicted == t_labels).sum() 429 | if (epoch + 1)%1 == 0: 430 | print("saving image") 431 | test_result_path = os.path.join('/home/deepkliv/Desktop/AE/ram/AE_classifier/fashion/Test_results/', 'batch_%d_%d'%((epoch+1)/1,t_i+1) + '.jpg') 432 | image_tensor = torch.cat((t_inputs.data[0:8], t_outputs.data[0:8]), 0) 433 | torchvision.utils.save_image(image_tensor, test_result_path) 434 | 435 | print('Accuracy of the network on the 8000 test images: %d %%' % (100 * correct / total)) 436 | 437 | print('Finished Training and Testing') 438 | 439 | 440 | """ 441 | classification_criterion = nn.NLLLoss() 442 | for i,data in enumerate(testloader): 443 | inputs,labels = data 444 | inputs,labels = Variable(inputs).cuda(), Variable(labels).cuda() 445 | test_model = Autoencoder().cuda() 446 | test_model.load_state_dict(torch.load('/home/deepkliv/Saket/AE_Classifier/Autoencoder/autoencoder-500.pkl',map_location=lambda storage, loc: storage.cuda(1))) 447 | outputs = test_model(inputs) 448 | test_result_path = os.path.join('/home/deepkliv/Saket/AE_Classifier/Test_results/', 'batch_%d'%(i+1) + '.jpg') 449 | image_tensor = torch.cat((inputs.data[0:8], outputs.data[0:8]), 0) 450 | torchvision.utils.save_image(image_tensor, test_result_path) 451 | """ 452 | 453 | -------------------------------------------------------------------------------- /retrieval_from_dict.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | require 'cutorch'; 7 | require 'cunn'; 8 | --require 'cudnn'; 9 | require './BinarizedNeurons' 10 | 11 | zSize = 14 12 | --encoder 13 | encoder = torch.load('/media/arna/340fd3c9-2648-4333-9ec9-239babc34bb7/arna_data/AdvAE_data/encoder1.t7'); 14 | 15 | binariser = nn.Sequential(); 16 | binariser:add(BinarizedNeurons()) 17 | 18 | autoencoder = nn.Sequential() 19 | autoencoder:add(encoder) 20 | autoencoder:add(binariser) 21 | 22 | autoencoder = autoencoder:cuda() 23 | print(autoencoder) 24 | 25 | --load MNIST data 26 | trainData = mnist.traindataset().data:double():div(255):reshape(60000,1,28,28) 27 | trainlabels = (mnist.traindataset().label+1) 28 | N = mnist.traindataset().size 29 | 30 | testData = mnist.testdataset().data:double():div(255):reshape(10000,1,28,28):cuda() 31 | testlabels = (mnist.testdataset().label+1):cuda() 32 | teSize = mnist.testdataset().size 33 | print(N,teSize) 34 | 35 | local dict = torch.load('AdvAE/mnist_dict.t7'); 36 | 37 | local x,y 38 | 39 | batchSize = 3000 40 | iterations = 50 41 | num_retrieval = 5; 42 | retrieval_vec = torch.Tensor(teSize,3) 43 | 44 | --Train 45 | print('Dictionary retrieving') 46 | x1 = autoencoder:forward(testData) 47 | --testData = nil; 48 | --autoencoder = nil; 49 | --collectgarbage() 50 | --dict = dict:cuda() 51 | dict_val = torch.Tensor(dict:size(1)) 52 | dict_val = torch.mm(x1:double(),dict:transpose(1,2)) 53 | print("mult done") 54 | for n=1,teSize do 55 | collectgarbage() 56 | if n%100==0 then print("yeah") end 57 | --x = testData[n]; 58 | --print(x:size()) 59 | --x1 = autoencoder:forward(x) 60 | retrieval_vec[n][1] = testlabels[n]; 61 | retrieval_vec[n][2] = 0; 62 | retrieval_vec[n][3] = 0; 63 | for i=1,dict:size(1) do 64 | --dict_val[i] = x1:dot(dict[i]:cuda()) 65 | if (dict_val[n][i]>=zSize-4) then --- Hamming Distance<=2 --> dot product val>=zSize-4 66 | retrieval_vec[n][2] = retrieval_vec[n][2]+1; 67 | if(trainlabels[i]==testlabels[n]) then 68 | retrieval_vec[n][3] = retrieval_vec[n][3]+1; 69 | end 70 | end 71 | end 72 | --_,index = torch.sort(dict_val,true) 73 | --retrieval_vec[{{n},{2,num_retrieval+1}}] = trainlabels:index(1,index[{{1,num_retrieval}}]:long()); 74 | end 75 | 76 | torch.save('AdvAE/mnist_retrieval.t7',retrieval_vec) 77 | -------------------------------------------------------------------------------- /std_mean_ret.lua: -------------------------------------------------------------------------------- 1 | require 'nn'; 2 | require 'image'; 3 | mnist = require 'mnist'; 4 | require 'optim'; 5 | require 'gnuplot'; 6 | require 'cutorch'; 7 | require 'cunn'; 8 | require 'cudnn'; 9 | require './BinarizedNeurons' 10 | local fashion_mnist = require 'fashion-mnist'; 11 | ret_vec=torch.load("/home/siplab/AE/AE/fashion_mnist_retrieval_cvpr.t7") 12 | mytable={} 13 | mytable2={} 14 | for j=1,10 do 15 | x=0 16 | mytable3={} 17 | count=1 18 | y=torch.zeros(1000) 19 | for i=1,10000 do 20 | 21 | if (ret_vec[i][1]==j) 22 | then temp=ret_vec[i][2] x=x+temp y[count] = temp count=count+1 23 | --print(ret_vec[i][1],ret_vec[i][2],ret_vec[i][3],y) 24 | end 25 | 26 | end 27 | 28 | table.insert(mytable,j,torch.std(y)) 29 | table.insert(mytable2,j,x/count) 30 | end 31 | print(mytable,mytable2) 32 | --------------------------------------------------------------------------------