├── imgs ├── orig_0.png ├── orig_10.png ├── orig_100.png ├── orig_110.png ├── orig_120.png ├── orig_130.png ├── orig_140.png ├── orig_150.png ├── orig_160.png ├── orig_170.png ├── orig_180.png ├── orig_190.png ├── orig_20.png ├── orig_200.png ├── orig_210.png ├── orig_30.png ├── orig_40.png ├── orig_50.png ├── orig_60.png ├── orig_70.png ├── orig_80.png ├── orig_90.png ├── reconstruction_0.png ├── reconstruction_10.png ├── reconstruction_100.png ├── reconstruction_110.png ├── reconstruction_120.png ├── reconstruction_130.png ├── reconstruction_140.png ├── reconstruction_150.png ├── reconstruction_160.png ├── reconstruction_170.png ├── reconstruction_180.png ├── reconstruction_190.png ├── reconstruction_20.png ├── reconstruction_200.png ├── reconstruction_210.png ├── reconstruction_30.png ├── reconstruction_40.png ├── reconstruction_50.png ├── reconstruction_60.png ├── reconstruction_70.png ├── reconstruction_80.png └── reconstruction_90.png ├── LICENSE ├── README.md ├── model.py └── run.py /imgs/orig_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_0.png -------------------------------------------------------------------------------- /imgs/orig_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_10.png -------------------------------------------------------------------------------- /imgs/orig_100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_100.png -------------------------------------------------------------------------------- /imgs/orig_110.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_110.png -------------------------------------------------------------------------------- /imgs/orig_120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_120.png -------------------------------------------------------------------------------- /imgs/orig_130.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_130.png -------------------------------------------------------------------------------- /imgs/orig_140.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_140.png -------------------------------------------------------------------------------- /imgs/orig_150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_150.png -------------------------------------------------------------------------------- /imgs/orig_160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_160.png -------------------------------------------------------------------------------- /imgs/orig_170.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_170.png -------------------------------------------------------------------------------- /imgs/orig_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_180.png -------------------------------------------------------------------------------- /imgs/orig_190.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_190.png -------------------------------------------------------------------------------- /imgs/orig_20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_20.png -------------------------------------------------------------------------------- /imgs/orig_200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_200.png -------------------------------------------------------------------------------- /imgs/orig_210.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_210.png -------------------------------------------------------------------------------- /imgs/orig_30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_30.png -------------------------------------------------------------------------------- /imgs/orig_40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_40.png -------------------------------------------------------------------------------- /imgs/orig_50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_50.png -------------------------------------------------------------------------------- /imgs/orig_60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_60.png -------------------------------------------------------------------------------- /imgs/orig_70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_70.png -------------------------------------------------------------------------------- /imgs/orig_80.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_80.png -------------------------------------------------------------------------------- /imgs/orig_90.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/orig_90.png -------------------------------------------------------------------------------- /imgs/reconstruction_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_0.png -------------------------------------------------------------------------------- /imgs/reconstruction_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_10.png -------------------------------------------------------------------------------- /imgs/reconstruction_100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_100.png -------------------------------------------------------------------------------- /imgs/reconstruction_110.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_110.png -------------------------------------------------------------------------------- /imgs/reconstruction_120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_120.png -------------------------------------------------------------------------------- /imgs/reconstruction_130.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_130.png -------------------------------------------------------------------------------- /imgs/reconstruction_140.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_140.png -------------------------------------------------------------------------------- /imgs/reconstruction_150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_150.png -------------------------------------------------------------------------------- /imgs/reconstruction_160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_160.png -------------------------------------------------------------------------------- /imgs/reconstruction_170.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_170.png -------------------------------------------------------------------------------- /imgs/reconstruction_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_180.png -------------------------------------------------------------------------------- /imgs/reconstruction_190.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_190.png -------------------------------------------------------------------------------- /imgs/reconstruction_20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_20.png -------------------------------------------------------------------------------- /imgs/reconstruction_200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_200.png -------------------------------------------------------------------------------- /imgs/reconstruction_210.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_210.png -------------------------------------------------------------------------------- /imgs/reconstruction_30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_30.png -------------------------------------------------------------------------------- /imgs/reconstruction_40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_40.png -------------------------------------------------------------------------------- /imgs/reconstruction_50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_50.png -------------------------------------------------------------------------------- /imgs/reconstruction_60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_60.png -------------------------------------------------------------------------------- /imgs/reconstruction_70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_70.png -------------------------------------------------------------------------------- /imgs/reconstruction_80.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_80.png -------------------------------------------------------------------------------- /imgs/reconstruction_90.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShayanPersonal/stacked-autoencoder-pytorch/HEAD/imgs/reconstruction_90.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Shayan Sadigh (https://github.com/ShayanPersonal/stacked-autoencoder-pytorch) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # stacked-autoencoder-pytorch 2 | Stacked denoising convolutional autoencoder written in Pytorch for some experiments. 3 | 4 | This model performs unsupervised reconstruction of the input using a setup similar to Hinton in https://www.cs.toronto.edu/~hinton/science.pdf. 5 | Instead of training layers one at a time, I allow them to train at the same time. Each is trained locally and no backpropagation is used. 6 | The autoencoder is denoising as in http://machinelearning.org/archive/icml2008/papers/592.pdf and convolutional. ReLU activation function is used. 7 | 8 | The quality of the feature vector is tested with a linear classifier reinitialized every 10 epochs. This is a toy model and you shouldn't expect good performance. You can try adding more layers and play around with adding more noise or regularization if better accuracy is desired. 9 | 10 | Setup: 11 | - Python 3 12 | - Pytorch 0.3.0 with torchvision 13 | 14 | Run "python run.py" to start training. 15 | 16 | Observations: 17 | - Although the loss doesn't propagate through layers the overall quality of the reconstruction improves anyways. 18 | - When using ReLU, the sparsity of the feature vector at the last layer increases from 50% at the start to 80%+ as you continue training even though sparsity isn't enforced. 19 | - Augmenting with rotation slows down training and decreases the quality of the feature vector. 20 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | import torch.nn.functional as F 5 | 6 | 7 | class CDAutoEncoder(nn.Module): 8 | r""" 9 | Convolutional denoising autoencoder layer for stacked autoencoders. 10 | This module is automatically trained when in model.training is True. 11 | 12 | Args: 13 | input_size: The number of features in the input 14 | output_size: The number of features to output 15 | stride: Stride of the convolutional layers. 16 | """ 17 | def __init__(self, input_size, output_size, stride): 18 | super(CDAutoEncoder, self).__init__() 19 | 20 | self.forward_pass = nn.Sequential( 21 | nn.Conv2d(input_size, output_size, kernel_size=2, stride=stride, padding=0), 22 | nn.ReLU(), 23 | ) 24 | self.backward_pass = nn.Sequential( 25 | nn.ConvTranspose2d(output_size, input_size, kernel_size=2, stride=2, padding=0), 26 | nn.ReLU(), 27 | ) 28 | 29 | self.criterion = nn.MSELoss() 30 | self.optimizer = torch.optim.SGD(self.parameters(), lr=0.1) 31 | 32 | def forward(self, x): 33 | # Train each autoencoder individually 34 | x = x.detach() 35 | # Add noise, but use the original lossless input as the target. 36 | x_noisy = x * (Variable(x.data.new(x.size()).normal_(0, 0.1)) > -.1).type_as(x) 37 | y = self.forward_pass(x_noisy) 38 | 39 | if self.training: 40 | x_reconstruct = self.backward_pass(y) 41 | loss = self.criterion(x_reconstruct, Variable(x.data, requires_grad=False)) 42 | self.optimizer.zero_grad() 43 | loss.backward() 44 | self.optimizer.step() 45 | 46 | return y.detach() 47 | 48 | def reconstruct(self, x): 49 | return self.backward_pass(x) 50 | 51 | 52 | class StackedAutoEncoder(nn.Module): 53 | r""" 54 | A stacked autoencoder made from the convolutional denoising autoencoders above. 55 | Each autoencoder is trained independently and at the same time. 56 | """ 57 | 58 | def __init__(self): 59 | super(StackedAutoEncoder, self).__init__() 60 | 61 | self.ae1 = CDAutoEncoder(3, 128, 2) 62 | self.ae2 = CDAutoEncoder(128, 256, 2) 63 | self.ae3 = CDAutoEncoder(256, 512, 2) 64 | 65 | def forward(self, x): 66 | a1 = self.ae1(x) 67 | a2 = self.ae2(a1) 68 | a3 = self.ae3(a2) 69 | 70 | if self.training: 71 | return a3 72 | 73 | else: 74 | return a3, self.reconstruct(a3) 75 | 76 | def reconstruct(self, x): 77 | a2_reconstruct = self.ae3.reconstruct(x) 78 | a1_reconstruct = self.ae2.reconstruct(a2_reconstruct) 79 | x_reconstruct = self.ae1.reconstruct(a1_reconstruct) 80 | return x_reconstruct 81 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | import torch 5 | import torchvision 6 | from torch import nn 7 | from torch.autograd import Variable 8 | from torch.utils.data import DataLoader 9 | from torchvision import transforms 10 | from torchvision.datasets import MNIST, CIFAR10 11 | from torchvision.utils import save_image 12 | 13 | from model import StackedAutoEncoder 14 | 15 | if not os.path.exists('./imgs'): 16 | os.mkdir('./imgs') 17 | 18 | def to_img(x): 19 | x = x.view(x.size(0), 3, 32, 32) 20 | return x 21 | 22 | num_epochs = 1000 23 | batch_size = 128 24 | 25 | img_transform = transforms.Compose([ 26 | #transforms.RandomRotation(360), 27 | transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0, hue=0), 28 | transforms.ToTensor(), 29 | ]) 30 | 31 | dataset = CIFAR10('../data/cifar10/', transform=img_transform) 32 | dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=8) 33 | 34 | model = StackedAutoEncoder().cuda() 35 | 36 | for epoch in range(num_epochs): 37 | if epoch % 10 == 0: 38 | # Test the quality of our features with a randomly initialzed linear classifier. 39 | classifier = nn.Linear(512 * 16, 10).cuda() 40 | criterion = nn.CrossEntropyLoss() 41 | optimizer = torch.optim.Adam(classifier.parameters(), lr=0.001) 42 | 43 | model.train() 44 | total_time = time.time() 45 | correct = 0 46 | for i, data in enumerate(dataloader): 47 | img, target = data 48 | target = Variable(target).cuda() 49 | img = Variable(img).cuda() 50 | features = model(img).detach() 51 | prediction = classifier(features.view(features.size(0), -1)) 52 | loss = criterion(prediction, target) 53 | 54 | optimizer.zero_grad() 55 | loss.backward() 56 | optimizer.step() 57 | pred = prediction.data.max(1, keepdim=True)[1] 58 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 59 | 60 | total_time = time.time() - total_time 61 | 62 | model.eval() 63 | img, _ = data 64 | img = Variable(img).cuda() 65 | features, x_reconstructed = model(img) 66 | reconstruction_loss = torch.mean((x_reconstructed.data - img.data)**2) 67 | 68 | if epoch % 10 == 0: 69 | print("Saving epoch {}".format(epoch)) 70 | orig = to_img(img.cpu().data) 71 | save_image(orig, './imgs/orig_{}.png'.format(epoch)) 72 | pic = to_img(x_reconstructed.cpu().data) 73 | save_image(pic, './imgs/reconstruction_{}.png'.format(epoch)) 74 | 75 | print("Epoch {} complete\tTime: {:.4f}s\t\tLoss: {:.4f}".format(epoch, total_time, reconstruction_loss)) 76 | print("Feature Statistics\tMean: {:.4f}\t\tMax: {:.4f}\t\tSparsity: {:.4f}%".format( 77 | torch.mean(features.data), torch.max(features.data), torch.sum(features.data == 0.0)*100 / features.data.numel()) 78 | ) 79 | print("Linear classifier performance: {}/{} = {:.2f}%".format(correct, len(dataloader)*batch_size, 100*float(correct) / (len(dataloader)*batch_size))) 80 | print("="*80) 81 | 82 | torch.save(model.state_dict(), './CDAE.pth') 83 | --------------------------------------------------------------------------------