├── .gitignore ├── README.md ├── assignment_1 ├── README.md ├── assignment_1.pdf ├── code │ ├── cifar10 │ │ └── get_cifar10.sh │ ├── cifar10_utils.py │ ├── convnet_pytorch.py │ ├── custom_batchnorm.py │ ├── gradient_check.py │ ├── mlp_numpy.py │ ├── mlp_pytorch.py │ ├── modules.py │ ├── train_convnet_pytorch.py │ ├── train_mlp_numpy.py │ ├── train_mlp_pytorch.py │ └── unittests.py └── environment.yml ├── assignment_2 ├── README.md ├── assignment_2.pdf ├── part1 │ ├── __init__.py │ ├── dataset.py │ ├── lstm.py │ ├── train.py │ └── vanilla_rnn.py └── part2 │ ├── __init__.py │ ├── assets │ ├── book_EN_democracy_in_the_US.txt │ ├── book_EN_grimms_fairy_tails.txt │ ├── book_NL_darwin_reis_om_de_wereld.txt │ ├── book_NL_tolstoy_anna_karenina.txt │ └── book_TeX_calculus_made_easy.tex │ ├── dataset.py │ ├── model.py │ └── train.py └── assignment_3 ├── assignment_3.pdf └── templates ├── a3_gan_template.py ├── a3_nf_template.py ├── a3_vae_template.py ├── datasets ├── __init__.py ├── bmnist.py └── mnist.py └── unittests.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | */.DS_Store 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning Course - Practical Assignments 2 | 3 | University of Amsterdam - Fall 2019 4 | -------------------------------------------------------------------------------- /assignment_1/README.md: -------------------------------------------------------------------------------- 1 | # Assignment 1 2 | 3 | You can find all necessary instructions in **assignment_1.pdf**. 4 | 5 | We provide to you simple unit tests that can check your implementation. However be aware that even if all tests are passed it still doesn't mean that your implementation is correct. You can find tests in **unittests.py**. 6 | 7 | 8 | 9 | We also provide a Conda environment you can use to install the necessary Python packages. 10 | In order to use it on SURFSARA, follow this instructions: 11 | 12 | - add the following lines in your ".bashrc": 13 | module load 2019 14 | module load CUDA/10.0.130 15 | module load Anaconda3/2018.12 16 | 17 | - logout and login again 18 | 19 | - copy the **environment.yml** file on SURFSARA (eg. in your home directory) 20 | 21 | - move to the folder containing the **environment.yml** file 22 | 23 | - run the following command once: 24 | conda env create -f environment.yml 25 | 26 | - add the following line at the beginning of your experiment script (.sh), before running your Python script: 27 | source activate dl 28 | 29 | for further information about Anaconda/Miniconda: 30 | 31 | https://docs.conda.io/projects/conda/en/latest/ 32 | https://docs.conda.io/en/latest/miniconda.html 33 | https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html 34 | -------------------------------------------------------------------------------- /assignment_1/assignment_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_1/assignment_1.pdf -------------------------------------------------------------------------------- /assignment_1/code/cifar10/get_cifar10.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Get CIFAR10 4 | wget http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz 5 | tar -xzvf cifar-10-python.tar.gz 6 | rm cifar-10-python.tar.gz 7 | -------------------------------------------------------------------------------- /assignment_1/code/cifar10_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements utility functions for downloading and reading CIFAR10 data. 3 | You don't need to change anything here. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | import numpy as np 10 | import os 11 | import pickle 12 | 13 | # Default paths for downloading CIFAR10 data 14 | CIFAR10_FOLDER = 'cifar10/cifar-10-batches-py' 15 | 16 | def load_cifar10_batch(batch_filename): 17 | """ 18 | Loads single batch of CIFAR10 data. 19 | Args: 20 | batch_filename: Filename of batch to get data from. 21 | Returns: 22 | X: CIFAR10 batch data in numpy array with shape (10000, 32, 32, 3). 23 | Y: CIFAR10 batch labels in numpy array with shape (10000, ). 24 | """ 25 | with open(batch_filename, 'rb') as f: 26 | batch = pickle.load(f, encoding='latin1') 27 | X = batch['data'] 28 | Y = batch['labels'] 29 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype(np.float32) 30 | Y = np.array(Y) 31 | return X, Y 32 | 33 | def load_cifar10(cifar10_folder): 34 | """ 35 | Loads CIFAR10 train and test splits. 36 | Args: 37 | cifar10_folder: Folder which contains downloaded CIFAR10 data. 38 | Returns: 39 | X_train: CIFAR10 train data in numpy array with shape (50000, 32, 32, 3). 40 | Y_train: CIFAR10 train labels in numpy array with shape (50000, ). 41 | X_test: CIFAR10 test data in numpy array with shape (10000, 32, 32, 3). 42 | Y_test: CIFAR10 test labels in numpy array with shape (10000, ). 43 | 44 | """ 45 | Xs = [] 46 | Ys = [] 47 | for b in range(1, 6): 48 | batch_filename = os.path.join(cifar10_folder, 'data_batch_' + str(b)) 49 | X, Y = load_cifar10_batch(batch_filename) 50 | Xs.append(X) 51 | Ys.append(Y) 52 | X_train = np.concatenate(Xs) 53 | Y_train = np.concatenate(Ys) 54 | X_test, Y_test = load_cifar10_batch(os.path.join(cifar10_folder, 'test_batch')) 55 | return X_train, Y_train, X_test, Y_test 56 | 57 | def get_cifar10_raw_data(data_dir): 58 | """ 59 | Gets raw CIFAR10 data from http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz. 60 | 61 | Args: 62 | data_dir: Data directory. 63 | Returns: 64 | X_train: CIFAR10 train data in numpy array with shape (50000, 32, 32, 3). 65 | Y_train: CIFAR10 train labels in numpy array with shape (50000, ). 66 | X_test: CIFAR10 test data in numpy array with shape (10000, 32, 32, 3). 67 | Y_test: CIFAR10 test labels in numpy array with shape (10000, ). 68 | """ 69 | 70 | X_train, Y_train, X_test, Y_test = load_cifar10(data_dir) 71 | 72 | return X_train, Y_train, X_test, Y_test 73 | 74 | def preprocess_cifar10_data(X_train_raw, Y_train_raw, X_test_raw, Y_test_raw): 75 | """ 76 | Preprocesses CIFAR10 data by substracting mean from all images. 77 | Args: 78 | X_train_raw: CIFAR10 raw train data in numpy array. 79 | Y_train_raw: CIFAR10 raw train labels in numpy array. 80 | X_test_raw: CIFAR10 raw test data in numpy array. 81 | Y_test_raw: CIFAR10 raw test labels in numpy array. 82 | num_val: Number of validation samples. 83 | Returns: 84 | X_train: CIFAR10 train data in numpy array. 85 | Y_train: CIFAR10 train labels in numpy array. 86 | X_test: CIFAR10 test data in numpy array. 87 | Y_test: CIFAR10 test labels in numpy array. 88 | """ 89 | X_train = X_train_raw.copy() 90 | Y_train = Y_train_raw.copy() 91 | X_test = X_test_raw.copy() 92 | Y_test = Y_test_raw.copy() 93 | 94 | # Substract the mean 95 | mean_image = np.mean(X_train, axis=0) 96 | X_train -= mean_image 97 | X_test -= mean_image 98 | 99 | # Transpose 100 | X_train = X_train.transpose(0, 3, 1, 2).astype(np.float32) 101 | X_test = X_test.transpose(0, 3, 1, 2).astype(np.float32) 102 | return X_train, Y_train, X_test, Y_test 103 | 104 | def dense_to_one_hot(labels_dense, num_classes): 105 | """ 106 | Convert class labels from scalars to one-hot vectors. 107 | Args: 108 | labels_dense: Dense labels. 109 | num_classes: Number of classes. 110 | Returns: 111 | labels_one_hot: One-hot encoding for labels. 112 | """ 113 | num_labels = labels_dense.shape[0] 114 | index_offset = np.arange(num_labels) * num_classes 115 | labels_one_hot = np.zeros((num_labels, num_classes)) 116 | labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 117 | return labels_one_hot 118 | 119 | class DataSet(object): 120 | """ 121 | Utility class to handle dataset structure. 122 | """ 123 | 124 | def __init__(self, images, labels): 125 | """ 126 | Builds dataset with images and labels. 127 | Args: 128 | images: Images data. 129 | labels: Labels data 130 | """ 131 | assert images.shape[0] == labels.shape[0], ( 132 | "images.shape: {0}, labels.shape: {1}".format(str(images.shape), str(labels.shape))) 133 | 134 | self._num_examples = images.shape[0] 135 | self._images = images 136 | self._labels = labels 137 | self._epochs_completed = 0 138 | self._index_in_epoch = 0 139 | 140 | @property 141 | def images(self): 142 | return self._images 143 | 144 | @property 145 | def labels(self): 146 | return self._labels 147 | 148 | @property 149 | def num_examples(self): 150 | return self._num_examples 151 | 152 | @property 153 | def epochs_completed(self): 154 | return self._epochs_completed 155 | 156 | def next_batch(self, batch_size): 157 | """ 158 | Return the next `batch_size` examples from this data set. 159 | Args: 160 | batch_size: Batch size. 161 | """ 162 | start = self._index_in_epoch 163 | self._index_in_epoch += batch_size 164 | if self._index_in_epoch > self._num_examples: 165 | self._epochs_completed += 1 166 | 167 | perm = np.arange(self._num_examples) 168 | np.random.shuffle(perm) 169 | self._images = self._images[perm] 170 | self._labels = self._labels[perm] 171 | 172 | start = 0 173 | self._index_in_epoch = batch_size 174 | assert batch_size <= self._num_examples 175 | 176 | end = self._index_in_epoch 177 | return self._images[start:end], self._labels[start:end] 178 | 179 | def read_data_sets(data_dir, one_hot = True, validation_size = 0): 180 | """ 181 | Returns the dataset readed from data_dir. 182 | Uses or not uses one-hot encoding for the labels. 183 | Subsamples validation set with specified size if necessary. 184 | Args: 185 | data_dir: Data directory. 186 | one_hot: Flag for one hot encoding. 187 | validation_size: Size of validation set 188 | Returns: 189 | Dictionary with Train, Validation, Test Datasets 190 | """ 191 | # Extract CIFAR10 data 192 | train_images_raw, train_labels_raw, test_images_raw, test_labels_raw = \ 193 | get_cifar10_raw_data(data_dir) 194 | train_images, train_labels, test_images, test_labels = \ 195 | preprocess_cifar10_data(train_images_raw, train_labels_raw, test_images_raw, test_labels_raw) 196 | 197 | # Apply one-hot encoding if specified 198 | if one_hot: 199 | num_classes = len(np.unique(train_labels)) 200 | train_labels = dense_to_one_hot(train_labels, num_classes) 201 | test_labels = dense_to_one_hot(test_labels, num_classes) 202 | 203 | # Subsample the validation set from the train set 204 | if not 0 <= validation_size <= len(train_images): 205 | raise ValueError("Validation size should be between 0 and {0}. Received: {1}.".format( 206 | len(train_images), validation_size)) 207 | 208 | validation_images = train_images[:validation_size] 209 | validation_labels = train_labels[:validation_size] 210 | train_images = train_images[validation_size:] 211 | train_labels = train_labels[validation_size:] 212 | 213 | # Create datasets 214 | train = DataSet(train_images, train_labels) 215 | validation = DataSet(validation_images, validation_labels) 216 | test = DataSet(test_images, test_labels) 217 | 218 | return {'train': train, 'validation': validation, 'test': test} 219 | 220 | def get_cifar10(data_dir = CIFAR10_FOLDER, one_hot = True, validation_size = 0): 221 | """ 222 | Prepares CIFAR10 dataset. 223 | Args: 224 | data_dir: Data directory. 225 | one_hot: Flag for one hot encoding. 226 | validation_size: Size of validation set 227 | Returns: 228 | Dictionary with Train, Validation, Test Datasets 229 | """ 230 | return read_data_sets(data_dir, one_hot, validation_size) 231 | -------------------------------------------------------------------------------- /assignment_1/code/convnet_pytorch.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements a Convolutional Neural Network in PyTorch. 3 | You should fill in code into indicated sections. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | class ConvNet(nn.Module): 10 | """ 11 | This class implements a Convolutional Neural Network in PyTorch. 12 | It handles the different layers and parameters of the model. 13 | Once initialized an ConvNet object can perform forward. 14 | """ 15 | 16 | def __init__(self, n_channels, n_classes): 17 | """ 18 | Initializes ConvNet object. 19 | 20 | Args: 21 | n_channels: number of input channels 22 | n_classes: number of classes of the classification problem 23 | 24 | 25 | TODO: 26 | Implement initialization of the network. 27 | """ 28 | 29 | ######################## 30 | # PUT YOUR CODE HERE # 31 | ####################### 32 | raise NotImplementedError 33 | ######################## 34 | # END OF YOUR CODE # 35 | ####################### 36 | 37 | def forward(self, x): 38 | """ 39 | Performs forward pass of the input. Here an input tensor x is transformed through 40 | several layer transformations. 41 | 42 | Args: 43 | x: input to the network 44 | Returns: 45 | out: outputs of the network 46 | 47 | TODO: 48 | Implement forward pass of the network. 49 | """ 50 | 51 | ######################## 52 | # PUT YOUR CODE HERE # 53 | ####################### 54 | raise NotImplementedError 55 | ######################## 56 | # END OF YOUR CODE # 57 | ####################### 58 | 59 | return out 60 | -------------------------------------------------------------------------------- /assignment_1/code/custom_batchnorm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | """ 6 | The modules/function here implement custom versions of batch normalization in PyTorch. 7 | In contrast to more advanced implementations no use of a running mean/variance is made. 8 | You should fill in code into indicated sections. 9 | """ 10 | 11 | ###################################################################################### 12 | # Code for Question 3.1 13 | ###################################################################################### 14 | 15 | class CustomBatchNormAutograd(nn.Module): 16 | """ 17 | This nn.module implements a custom version of the batch norm operation for MLPs. 18 | The operations called in self.forward track the history if the input tensors have the 19 | flag requires_grad set to True. The backward pass does not need to be implemented, it 20 | is dealt with by the automatic differentiation provided by PyTorch. 21 | """ 22 | 23 | def __init__(self, n_neurons, eps=1e-5): 24 | """ 25 | Initializes CustomBatchNormAutograd object. 26 | 27 | Args: 28 | n_neurons: int specifying the number of neurons 29 | eps: small float to be added to the variance for stability 30 | 31 | TODO: 32 | Save parameters for the number of neurons and eps. 33 | Initialize parameters gamma and beta via nn.Parameter 34 | """ 35 | super(CustomBatchNormAutograd, self).__init__() 36 | 37 | ######################## 38 | # PUT YOUR CODE HERE # 39 | ####################### 40 | 41 | ######################## 42 | # END OF YOUR CODE # 43 | ####################### 44 | 45 | def forward(self, input): 46 | """ 47 | Compute the batch normalization 48 | 49 | Args: 50 | input: input tensor of shape (n_batch, n_neurons) 51 | Returns: 52 | out: batch-normalized tensor 53 | 54 | TODO: 55 | Check for the correctness of the shape of the input tensor. 56 | Implement batch normalization forward pass as given in the assignment. 57 | For the case that you make use of torch.var be aware that the flag unbiased=False should be set. 58 | """ 59 | 60 | ######################## 61 | # PUT YOUR CODE HERE # 62 | ####################### 63 | 64 | ######################## 65 | # END OF YOUR CODE # 66 | ####################### 67 | 68 | return out 69 | 70 | 71 | 72 | ###################################################################################### 73 | # Code for Question 3.2 b) 74 | ###################################################################################### 75 | 76 | 77 | class CustomBatchNormManualFunction(torch.autograd.Function): 78 | """ 79 | This torch.autograd.Function implements a functional custom version of the batch norm operation for MLPs. 80 | Using torch.autograd.Function allows you to write a custom backward function. 81 | The function will be called from the nn.Module CustomBatchNormManualModule 82 | Inside forward the tensors are (automatically) not recorded for automatic differentiation since the backward 83 | pass is done via the backward method. 84 | The forward pass is not called directly but via the apply() method. This makes sure that the context objects 85 | are dealt with correctly. Example: 86 | my_bn_fct = CustomBatchNormManualFunction() 87 | normalized = fct.apply(input, gamma, beta, eps) 88 | """ 89 | 90 | @staticmethod 91 | def forward(ctx, input, gamma, beta, eps=1e-5): 92 | """ 93 | Compute the batch normalization 94 | 95 | Args: 96 | ctx: context object handling storing and retrival of tensors and constants and specifying 97 | whether tensors need gradients in backward pass 98 | input: input tensor of shape (n_batch, n_neurons) 99 | gamma: variance scaling tensor, applied per neuron, shpae (n_neurons) 100 | beta: mean bias tensor, applied per neuron, shpae (n_neurons) 101 | eps: small float added to the variance for stability 102 | Returns: 103 | out: batch-normalized tensor 104 | 105 | TODO: 106 | Implement the forward pass of batch normalization 107 | Store constant non-tensor objects via ctx.constant=myconstant 108 | Store tensors which you need in the backward pass via ctx.save_for_backward(tensor1, tensor2, ...) 109 | Intermediate results can be decided to be either recomputed in the backward pass or to be stored 110 | for the backward pass. Do not store tensors which are unnecessary for the backward pass to save memory! 111 | For the case that you make use of torch.var be aware that the flag unbiased=False should be set. 112 | """ 113 | 114 | ######################## 115 | # PUT YOUR CODE HERE # 116 | ####################### 117 | 118 | ######################## 119 | # END OF YOUR CODE # 120 | ####################### 121 | 122 | return out 123 | 124 | 125 | @staticmethod 126 | def backward(ctx, grad_output): 127 | """ 128 | Compute backward pass of the batch normalization. 129 | 130 | Args: 131 | ctx: context object handling storing and retrival of tensors and constants and specifying 132 | whether tensors need gradients in backward pass 133 | Returns: 134 | out: tuple containing gradients for all input arguments 135 | 136 | TODO: 137 | Retrieve saved tensors and constants via ctx.saved_tensors and ctx.constant 138 | Compute gradients for inputs where ctx.needs_input_grad[idx] is True. Set gradients for other 139 | inputs to None. This should be decided dynamically. 140 | """ 141 | 142 | ######################## 143 | # PUT YOUR CODE HERE # 144 | ####################### 145 | 146 | ######################## 147 | # END OF YOUR CODE # 148 | ####################### 149 | 150 | # return gradients of the three tensor inputs and None for the constant eps 151 | return grad_input, grad_gamma, grad_beta, None 152 | 153 | 154 | 155 | ###################################################################################### 156 | # Code for Question 3.2 c) 157 | ###################################################################################### 158 | 159 | class CustomBatchNormManualModule(nn.Module): 160 | """ 161 | This nn.module implements a custom version of the batch norm operation for MLPs. 162 | In self.forward the functional version CustomBatchNormManualFunction.forward is called. 163 | The automatic differentiation of PyTorch calls the backward method of this function in the backward pass. 164 | """ 165 | 166 | def __init__(self, n_neurons, eps=1e-5): 167 | """ 168 | Initializes CustomBatchNormManualModule object. 169 | 170 | Args: 171 | n_neurons: int specifying the number of neurons 172 | eps: small float to be added to the variance for stability 173 | 174 | TODO: 175 | Save parameters for the number of neurons and eps. 176 | Initialize parameters gamma and beta via nn.Parameter 177 | """ 178 | super(CustomBatchNormManualModule, self).__init__() 179 | 180 | ######################## 181 | # PUT YOUR CODE HERE # 182 | ####################### 183 | 184 | ######################## 185 | # END OF YOUR CODE # 186 | ####################### 187 | 188 | def forward(self, input): 189 | """ 190 | Compute the batch normalization via CustomBatchNormManualFunction 191 | 192 | Args: 193 | input: input tensor of shape (n_batch, n_neurons) 194 | Returns: 195 | out: batch-normalized tensor 196 | 197 | TODO: 198 | Check for the correctness of the shape of the input tensor. 199 | Instantiate a CustomBatchNormManualFunction. 200 | Call it via its .apply() method. 201 | """ 202 | 203 | ######################## 204 | # PUT YOUR CODE HERE # 205 | ####################### 206 | 207 | ######################## 208 | # END OF YOUR CODE # 209 | ####################### 210 | 211 | return out 212 | 213 | 214 | 215 | 216 | 217 | 218 | if __name__=='__main__': 219 | # create test batch 220 | n_batch = 128 221 | n_neurons = 4 222 | # create random tensor with variance 2 and mean 3 223 | x = 2*torch.randn(n_batch, n_neurons, requires_grad=True)+10 224 | print('Input data:\n\tmeans={}\n\tvars={}'.format(x.mean(dim=0).data, x.var(dim=0).data)) 225 | 226 | # test CustomBatchNormAutograd 227 | print('3.1) Test automatic differentation version') 228 | bn_auto = CustomBatchNormAutograd(n_neurons) 229 | y_auto = bn_auto(x) 230 | print('\tmeans={}\n\tvars={}'.format(y_auto.mean(dim=0).data, y_auto.var(dim=0).data)) 231 | 232 | # test CustomBatchNormManualFunction 233 | # this is recommended to be done in double precision 234 | print('3.2 b) Test functional version') 235 | input = x.double() 236 | gamma = torch.sqrt(10*torch.arange(n_neurons, dtype=torch.float64, requires_grad=True)) 237 | beta = 100*torch.arange(n_neurons, dtype=torch.float64, requires_grad=True) 238 | bn_manual_fct = CustomBatchNormManualFunction(n_neurons) 239 | y_manual_fct = bn_manual_fct.apply(input, gamma, beta) 240 | print('\tmeans={}\n\tvars={}'.format(y_manual_fct.mean(dim=0).data, y_manual_fct.var(dim=0).data)) 241 | # gradient check 242 | grad_correct = torch.autograd.gradcheck(bn_manual_fct.apply, (input,gamma,beta)) 243 | if grad_correct: 244 | print('\tgradient check successful') 245 | else: 246 | raise ValueError('gradient check failed') 247 | 248 | # test CustomBatchNormManualModule 249 | print('3.2 c) Test module of functional version') 250 | bn_manual_mod = CustomBatchNormManualModule(n_neurons) 251 | y_manual_mod = bn_manual_mod(x) 252 | print('\tmeans={}\n\tvars={}'.format(y_manual_mod.mean(dim=0).data, y_manual_mod.var(dim=0).data)) -------------------------------------------------------------------------------- /assignment_1/code/gradient_check.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from random import randrange 3 | 4 | def eval_numerical_gradient(f, x, verbose=True, h=0.00001): 5 | fx = f(x) 6 | grad = np.zeros_like(x) 7 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 8 | while not it.finished: 9 | 10 | ix = it.multi_index 11 | oldval = x[ix] 12 | x[ix] = oldval + h 13 | fxph = f(x) 14 | x[ix] = oldval - h 15 | fxmh = f(x) 16 | x[ix] = oldval 17 | 18 | grad[ix] = (fxph - fxmh) / (2 * h) 19 | if verbose: 20 | print(ix, grad[ix]) 21 | it.iternext() 22 | 23 | return grad 24 | 25 | def eval_numerical_gradient_array(f, x, df, h=1e-5): 26 | grad = np.zeros_like(x) 27 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 28 | while not it.finished: 29 | ix = it.multi_index 30 | 31 | oldval = x[ix] 32 | x[ix] = oldval + h 33 | pos = f(x).copy() 34 | x[ix] = oldval - h 35 | neg = f(x).copy() 36 | x[ix] = oldval 37 | 38 | grad[ix] = np.sum((pos - neg) * df) / (2 * h) 39 | it.iternext() 40 | return grad 41 | -------------------------------------------------------------------------------- /assignment_1/code/mlp_numpy.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements a multi-layer perceptron (MLP) in NumPy. 3 | You should fill in code into indicated sections. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | from modules import * 10 | 11 | class MLP(object): 12 | """ 13 | This class implements a Multi-layer Perceptron in NumPy. 14 | It handles the different layers and parameters of the model. 15 | Once initialized an MLP object can perform forward and backward. 16 | """ 17 | 18 | def __init__(self, n_inputs, n_hidden, n_classes, neg_slope): 19 | """ 20 | Initializes MLP object. 21 | 22 | Args: 23 | n_inputs: number of inputs. 24 | n_hidden: list of ints, specifies the number of units 25 | in each linear layer. If the list is empty, the MLP 26 | will not have any linear layers, and the model 27 | will simply perform a multinomial logistic regression. 28 | n_classes: number of classes of the classification problem. 29 | This number is required in order to specify the 30 | output dimensions of the MLP 31 | neg_slope: negative slope parameter for LeakyReLU 32 | 33 | TODO: 34 | Implement initialization of the network. 35 | """ 36 | 37 | ######################## 38 | # PUT YOUR CODE HERE # 39 | ####################### 40 | raise NotImplementedError 41 | ######################## 42 | # END OF YOUR CODE # 43 | ####################### 44 | 45 | def forward(self, x): 46 | """ 47 | Performs forward pass of the input. Here an input tensor x is transformed through 48 | several layer transformations. 49 | 50 | Args: 51 | x: input to the network 52 | Returns: 53 | out: outputs of the network 54 | 55 | TODO: 56 | Implement forward pass of the network. 57 | """ 58 | 59 | ######################## 60 | # PUT YOUR CODE HERE # 61 | ####################### 62 | raise NotImplementedError 63 | ######################## 64 | # END OF YOUR CODE # 65 | ####################### 66 | 67 | return out 68 | 69 | def backward(self, dout): 70 | """ 71 | Performs backward pass given the gradients of the loss. 72 | 73 | Args: 74 | dout: gradients of the loss 75 | 76 | TODO: 77 | Implement backward pass of the network. 78 | """ 79 | 80 | ######################## 81 | # PUT YOUR CODE HERE # 82 | ####################### 83 | raise NotImplementedError 84 | ######################## 85 | # END OF YOUR CODE # 86 | ####################### 87 | 88 | return 89 | -------------------------------------------------------------------------------- /assignment_1/code/mlp_pytorch.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements a multi-layer perceptron (MLP) in PyTorch. 3 | You should fill in code into indicated sections. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | class MLP(nn.Module): 10 | """ 11 | This class implements a Multi-layer Perceptron in PyTorch. 12 | It handles the different layers and parameters of the model. 13 | Once initialized an MLP object can perform forward. 14 | """ 15 | 16 | def __init__(self, n_inputs, n_hidden, n_classes, neg_slope): 17 | """ 18 | Initializes MLP object. 19 | 20 | Args: 21 | n_inputs: number of inputs. 22 | n_hidden: list of ints, specifies the number of units 23 | in each linear layer. If the list is empty, the MLP 24 | will not have any linear layers, and the model 25 | will simply perform a multinomial logistic regression. 26 | n_classes: number of classes of the classification problem. 27 | This number is required in order to specify the 28 | output dimensions of the MLP 29 | neg_slope: negative slope parameter for LeakyReLU 30 | 31 | TODO: 32 | Implement initialization of the network. 33 | """ 34 | 35 | ######################## 36 | # PUT YOUR CODE HERE # 37 | ####################### 38 | raise NotImplementedError 39 | ######################## 40 | # END OF YOUR CODE # 41 | ####################### 42 | 43 | def forward(self, x): 44 | """ 45 | Performs forward pass of the input. Here an input tensor x is transformed through 46 | several layer transformations. 47 | 48 | Args: 49 | x: input to the network 50 | Returns: 51 | out: outputs of the network 52 | 53 | TODO: 54 | Implement forward pass of the network. 55 | """ 56 | 57 | ######################## 58 | # PUT YOUR CODE HERE # 59 | ####################### 60 | raise NotImplementedError 61 | ######################## 62 | # END OF YOUR CODE # 63 | ####################### 64 | 65 | return out 66 | -------------------------------------------------------------------------------- /assignment_1/code/modules.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements various modules of the network. 3 | You should fill in code into indicated sections. 4 | """ 5 | import numpy as np 6 | 7 | class LinearModule(object): 8 | """ 9 | Linear module. Applies a linear transformation to the input data. 10 | """ 11 | def __init__(self, in_features, out_features): 12 | """ 13 | Initializes the parameters of the module. 14 | 15 | Args: 16 | in_features: size of each input sample 17 | out_features: size of each output sample 18 | 19 | TODO: 20 | Initialize weights self.params['weight'] using normal distribution with mean = 0 and 21 | std = 0.0001. Initialize biases self.params['bias'] with 0. 22 | 23 | Also, initialize gradients with zeros. 24 | """ 25 | 26 | ######################## 27 | # PUT YOUR CODE HERE # 28 | ####################### 29 | self.params = {'weight': None, 'bias': None} 30 | self.grads = {'weight': None, 'bias': None} 31 | raise NotImplementedError 32 | ######################## 33 | # END OF YOUR CODE # 34 | ####################### 35 | 36 | def forward(self, x): 37 | """ 38 | Forward pass. 39 | 40 | Args: 41 | x: input to the module 42 | Returns: 43 | out: output of the module 44 | 45 | TODO: 46 | Implement forward pass of the module. 47 | 48 | Hint: You can store intermediate variables inside the object. They can be used in backward pass computation. # 49 | """ 50 | 51 | ######################## 52 | # PUT YOUR CODE HERE # 53 | ####################### 54 | raise NotImplementedError 55 | ######################## 56 | # END OF YOUR CODE # 57 | ####################### 58 | 59 | return out 60 | 61 | def backward(self, dout): 62 | """ 63 | Backward pass. 64 | 65 | Args: 66 | dout: gradients of the previous module 67 | Returns: 68 | dx: gradients with respect to the input of the module 69 | 70 | TODO: 71 | Implement backward pass of the module. Store gradient of the loss with respect to 72 | layer parameters in self.grads['weight'] and self.grads['bias']. 73 | """ 74 | 75 | ######################## 76 | # PUT YOUR CODE HERE # 77 | ####################### 78 | raise NotImplementedError 79 | ######################## 80 | # END OF YOUR CODE # 81 | ####################### 82 | 83 | return dx 84 | 85 | class LeakyReLUModule(object): 86 | """ 87 | Leaky ReLU activation module. 88 | """ 89 | def __init__(self, neg_slope): 90 | """ 91 | Initializes the parameters of the module. 92 | 93 | Args: 94 | neg_slope: negative slope parameter. 95 | 96 | TODO: 97 | Initialize the module. 98 | """ 99 | 100 | ######################## 101 | # PUT YOUR CODE HERE # 102 | ####################### 103 | raise NotImplementedError 104 | ######################## 105 | # END OF YOUR CODE # 106 | ####################### 107 | 108 | def forward(self, x): 109 | """ 110 | Forward pass. 111 | 112 | Args: 113 | x: input to the module 114 | Returns: 115 | out: output of the module 116 | 117 | TODO: 118 | Implement forward pass of the module. 119 | 120 | Hint: You can store intermediate variables inside the object. They can be used in backward pass computation. # 121 | """ 122 | 123 | ######################## 124 | # PUT YOUR CODE HERE # 125 | ####################### 126 | raise NotImplementedError 127 | ######################## 128 | # END OF YOUR CODE # 129 | ####################### 130 | 131 | return out 132 | 133 | def backward(self, dout): 134 | """ 135 | Backward pass. 136 | 137 | Args: 138 | dout: gradients of the previous module 139 | Returns: 140 | dx: gradients with respect to the input of the module 141 | 142 | TODO: 143 | Implement backward pass of the module. 144 | """ 145 | 146 | ######################## 147 | # PUT YOUR CODE HERE # 148 | ####################### 149 | raise NotImplementedError 150 | ######################## 151 | # END OF YOUR CODE # 152 | ####################### 153 | 154 | return dx 155 | 156 | 157 | class SoftMaxModule(object): 158 | """ 159 | Softmax activation module. 160 | """ 161 | 162 | def forward(self, x): 163 | """ 164 | Forward pass. 165 | Args: 166 | x: input to the module 167 | Returns: 168 | out: output of the module 169 | 170 | TODO: 171 | Implement forward pass of the module. 172 | To stabilize computation you should use the so-called Max Trick - https://timvieira.github.io/blog/post/2014/02/11/exp-normalize-trick/ 173 | 174 | Hint: You can store intermediate variables inside the object. They can be used in backward pass computation. 175 | """ 176 | 177 | ######################## 178 | # PUT YOUR CODE HERE # 179 | ####################### 180 | raise NotImplementedError 181 | ######################## 182 | # END OF YOUR CODE # 183 | ####################### 184 | 185 | return out 186 | 187 | def backward(self, dout): 188 | """ 189 | Backward pass. 190 | Args: 191 | dout: gradients of the previous modul 192 | Returns: 193 | dx: gradients with respect to the input of the module 194 | 195 | TODO: 196 | Implement backward pass of the module. 197 | """ 198 | 199 | ######################## 200 | # PUT YOUR CODE HERE # 201 | ####################### 202 | raise NotImplementedError 203 | ####################### 204 | # END OF YOUR CODE # 205 | ####################### 206 | 207 | return dx 208 | 209 | class CrossEntropyModule(object): 210 | """ 211 | Cross entropy loss module. 212 | """ 213 | 214 | def forward(self, x, y): 215 | """ 216 | Forward pass. 217 | Args: 218 | x: input to the module 219 | y: labels of the input 220 | Returns: 221 | out: cross entropy loss 222 | 223 | TODO: 224 | Implement forward pass of the module. 225 | """ 226 | 227 | ######################## 228 | # PUT YOUR CODE HERE # 229 | ####################### 230 | raise NotImplementedError 231 | ######################## 232 | # END OF YOUR CODE # 233 | ####################### 234 | 235 | return out 236 | 237 | def backward(self, x, y): 238 | """ 239 | Backward pass. 240 | Args: 241 | x: input to the module 242 | y: labels of the input 243 | Returns: 244 | dx: gradient of the loss with the respect to the input x. 245 | 246 | TODO: 247 | Implement backward pass of the module. 248 | """ 249 | 250 | ######################## 251 | # PUT YOUR CODE HERE # 252 | ####################### 253 | raise NotImplementedError 254 | ######################## 255 | # END OF YOUR CODE # 256 | ####################### 257 | 258 | return dx -------------------------------------------------------------------------------- /assignment_1/code/train_convnet_pytorch.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements training and evaluation of a Convolutional Neural Network in PyTorch. 3 | You should fill in code into indicated sections. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | import argparse 10 | import numpy as np 11 | import os 12 | from convnet_pytorch import ConvNet 13 | import cifar10_utils 14 | 15 | # Default constants 16 | LEARNING_RATE_DEFAULT = 1e-4 17 | BATCH_SIZE_DEFAULT = 32 18 | MAX_STEPS_DEFAULT = 5000 19 | EVAL_FREQ_DEFAULT = 500 20 | OPTIMIZER_DEFAULT = 'ADAM' 21 | 22 | # Directory in which cifar data is saved 23 | DATA_DIR_DEFAULT = './cifar10/cifar-10-batches-py' 24 | 25 | FLAGS = None 26 | 27 | def accuracy(predictions, targets): 28 | """ 29 | Computes the prediction accuracy, i.e. the average of correct predictions 30 | of the network. 31 | 32 | Args: 33 | predictions: 2D float array of size [batch_size, n_classes] 34 | labels: 2D int array of size [batch_size, n_classes] 35 | with one-hot encoding. Ground truth labels for 36 | each sample in the batch 37 | Returns: 38 | accuracy: scalar float, the accuracy of predictions, 39 | i.e. the average correct predictions over the whole batch 40 | 41 | TODO: 42 | Implement accuracy computation. 43 | """ 44 | 45 | ######################## 46 | # PUT YOUR CODE HERE # 47 | ####################### 48 | raise NotImplementedError 49 | ######################## 50 | # END OF YOUR CODE # 51 | ####################### 52 | 53 | return accuracy 54 | 55 | def train(): 56 | """ 57 | Performs training and evaluation of ConvNet model. 58 | 59 | TODO: 60 | Implement training and evaluation of ConvNet model. Evaluate your model on the whole test set each eval_freq iterations. 61 | """ 62 | 63 | ### DO NOT CHANGE SEEDS! 64 | # Set the random seeds for reproducibility 65 | np.random.seed(42) 66 | 67 | ######################## 68 | # PUT YOUR CODE HERE # 69 | ####################### 70 | raise NotImplementedError 71 | ######################## 72 | # END OF YOUR CODE # 73 | ####################### 74 | 75 | def print_flags(): 76 | """ 77 | Prints all entries in FLAGS variable. 78 | """ 79 | for key, value in vars(FLAGS).items(): 80 | print(key + ' : ' + str(value)) 81 | 82 | def main(): 83 | """ 84 | Main function 85 | """ 86 | # Print all Flags to confirm parameter settings 87 | print_flags() 88 | 89 | if not os.path.exists(FLAGS.data_dir): 90 | os.makedirs(FLAGS.data_dir) 91 | 92 | # Run the training operation 93 | train() 94 | 95 | if __name__ == '__main__': 96 | # Command line arguments 97 | parser = argparse.ArgumentParser() 98 | parser.add_argument('--learning_rate', type = float, default = LEARNING_RATE_DEFAULT, 99 | help='Learning rate') 100 | parser.add_argument('--max_steps', type = int, default = MAX_STEPS_DEFAULT, 101 | help='Number of steps to run trainer.') 102 | parser.add_argument('--batch_size', type = int, default = BATCH_SIZE_DEFAULT, 103 | help='Batch size to run trainer.') 104 | parser.add_argument('--eval_freq', type=int, default=EVAL_FREQ_DEFAULT, 105 | help='Frequency of evaluation on the test set') 106 | parser.add_argument('--data_dir', type = str, default = DATA_DIR_DEFAULT, 107 | help='Directory for storing input data') 108 | FLAGS, unparsed = parser.parse_known_args() 109 | 110 | main() -------------------------------------------------------------------------------- /assignment_1/code/train_mlp_numpy.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements training and evaluation of a multi-layer perceptron in NumPy. 3 | You should fill in code into indicated sections. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | import argparse 10 | import numpy as np 11 | import os 12 | from mlp_numpy import MLP 13 | from modules import CrossEntropyModule 14 | import cifar10_utils 15 | 16 | # Default constants 17 | DNN_HIDDEN_UNITS_DEFAULT = '100' 18 | LEARNING_RATE_DEFAULT = 2e-3 19 | MAX_STEPS_DEFAULT = 1500 20 | BATCH_SIZE_DEFAULT = 200 21 | EVAL_FREQ_DEFAULT = 100 22 | NEG_SLOPE_DEFAULT = 0.02 23 | 24 | # Directory in which cifar data is saved 25 | DATA_DIR_DEFAULT = './cifar10/cifar-10-batches-py' 26 | 27 | FLAGS = None 28 | 29 | def accuracy(predictions, targets): 30 | """ 31 | Computes the prediction accuracy, i.e. the average of correct predictions 32 | of the network. 33 | 34 | Args: 35 | predictions: 2D float array of size [batch_size, n_classes] 36 | labels: 2D int array of size [batch_size, n_classes] 37 | with one-hot encoding. Ground truth labels for 38 | each sample in the batch 39 | Returns: 40 | accuracy: scalar float, the accuracy of predictions, 41 | i.e. the average correct predictions over the whole batch 42 | 43 | TODO: 44 | Implement accuracy computation. 45 | """ 46 | 47 | ######################## 48 | # PUT YOUR CODE HERE # 49 | ####################### 50 | raise NotImplementedError 51 | ######################## 52 | # END OF YOUR CODE # 53 | ####################### 54 | 55 | return accuracy 56 | 57 | def train(): 58 | """ 59 | Performs training and evaluation of MLP model. 60 | 61 | TODO: 62 | Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. 63 | """ 64 | 65 | ### DO NOT CHANGE SEEDS! 66 | # Set the random seeds for reproducibility 67 | np.random.seed(42) 68 | 69 | ## Prepare all functions 70 | # Get number of units in each hidden layer specified in the string such as 100,100 71 | if FLAGS.dnn_hidden_units: 72 | dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") 73 | dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units] 74 | else: 75 | dnn_hidden_units = [] 76 | 77 | # Get negative slope parameter for LeakyReLU 78 | neg_slope = FLAGS.neg_slope 79 | 80 | ######################## 81 | # PUT YOUR CODE HERE # 82 | ####################### 83 | raise NotImplementedError 84 | ######################## 85 | # END OF YOUR CODE # 86 | ####################### 87 | 88 | def print_flags(): 89 | """ 90 | Prints all entries in FLAGS variable. 91 | """ 92 | for key, value in vars(FLAGS).items(): 93 | print(key + ' : ' + str(value)) 94 | 95 | def main(): 96 | """ 97 | Main function 98 | """ 99 | # Print all Flags to confirm parameter settings 100 | print_flags() 101 | 102 | if not os.path.exists(FLAGS.data_dir): 103 | os.makedirs(FLAGS.data_dir) 104 | 105 | # Run the training operation 106 | train() 107 | 108 | if __name__ == '__main__': 109 | # Command line arguments 110 | parser = argparse.ArgumentParser() 111 | parser.add_argument('--dnn_hidden_units', type = str, default = DNN_HIDDEN_UNITS_DEFAULT, 112 | help='Comma separated list of number of units in each hidden layer') 113 | parser.add_argument('--learning_rate', type = float, default = LEARNING_RATE_DEFAULT, 114 | help='Learning rate') 115 | parser.add_argument('--max_steps', type = int, default = MAX_STEPS_DEFAULT, 116 | help='Number of steps to run trainer.') 117 | parser.add_argument('--batch_size', type = int, default = BATCH_SIZE_DEFAULT, 118 | help='Batch size to run trainer.') 119 | parser.add_argument('--eval_freq', type=int, default=EVAL_FREQ_DEFAULT, 120 | help='Frequency of evaluation on the test set') 121 | parser.add_argument('--data_dir', type = str, default = DATA_DIR_DEFAULT, 122 | help='Directory for storing input data') 123 | parser.add_argument('--neg_slope', type=float, default=NEG_SLOPE_DEFAULT, 124 | help='Negative slope parameter for LeakyReLU') 125 | FLAGS, unparsed = parser.parse_known_args() 126 | 127 | main() -------------------------------------------------------------------------------- /assignment_1/code/train_mlp_pytorch.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements training and evaluation of a multi-layer perceptron in PyTorch. 3 | You should fill in code into indicated sections. 4 | """ 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | import argparse 10 | import numpy as np 11 | import os 12 | from mlp_pytorch import MLP 13 | import cifar10_utils 14 | 15 | # Default constants 16 | DNN_HIDDEN_UNITS_DEFAULT = '100' 17 | LEARNING_RATE_DEFAULT = 2e-3 18 | MAX_STEPS_DEFAULT = 1500 19 | BATCH_SIZE_DEFAULT = 200 20 | EVAL_FREQ_DEFAULT = 100 21 | NEG_SLOPE_DEFAULT = 0.02 22 | 23 | # Directory in which cifar data is saved 24 | DATA_DIR_DEFAULT = './cifar10/cifar-10-batches-py' 25 | 26 | FLAGS = None 27 | 28 | def accuracy(predictions, targets): 29 | """ 30 | Computes the prediction accuracy, i.e. the average of correct predictions 31 | of the network. 32 | 33 | Args: 34 | predictions: 2D float array of size [batch_size, n_classes] 35 | labels: 2D int array of size [batch_size, n_classes] 36 | with one-hot encoding. Ground truth labels for 37 | each sample in the batch 38 | Returns: 39 | accuracy: scalar float, the accuracy of predictions, 40 | i.e. the average correct predictions over the whole batch 41 | 42 | TODO: 43 | Implement accuracy computation. 44 | """ 45 | 46 | ######################## 47 | # PUT YOUR CODE HERE # 48 | ####################### 49 | raise NotImplementedError 50 | ######################## 51 | # END OF YOUR CODE # 52 | ####################### 53 | 54 | return accuracy 55 | 56 | def train(): 57 | """ 58 | Performs training and evaluation of MLP model. 59 | 60 | TODO: 61 | Implement training and evaluation of MLP model. Evaluate your model on the whole test set each eval_freq iterations. 62 | """ 63 | 64 | ### DO NOT CHANGE SEEDS! 65 | # Set the random seeds for reproducibility 66 | np.random.seed(42) 67 | 68 | ## Prepare all functions 69 | # Get number of units in each hidden layer specified in the string such as 100,100 70 | if FLAGS.dnn_hidden_units: 71 | dnn_hidden_units = FLAGS.dnn_hidden_units.split(",") 72 | dnn_hidden_units = [int(dnn_hidden_unit_) for dnn_hidden_unit_ in dnn_hidden_units] 73 | else: 74 | dnn_hidden_units = [] 75 | 76 | # Get negative slope parameter for LeakyReLU 77 | neg_slope = FLAGS.neg_slope 78 | 79 | ######################## 80 | # PUT YOUR CODE HERE # 81 | ####################### 82 | raise NotImplementedError 83 | ######################## 84 | # END OF YOUR CODE # 85 | ####################### 86 | 87 | def print_flags(): 88 | """ 89 | Prints all entries in FLAGS variable. 90 | """ 91 | for key, value in vars(FLAGS).items(): 92 | print(key + ' : ' + str(value)) 93 | 94 | def main(): 95 | """ 96 | Main function 97 | """ 98 | # Print all Flags to confirm parameter settings 99 | print_flags() 100 | 101 | if not os.path.exists(FLAGS.data_dir): 102 | os.makedirs(FLAGS.data_dir) 103 | 104 | # Run the training operation 105 | train() 106 | 107 | if __name__ == '__main__': 108 | # Command line arguments 109 | parser = argparse.ArgumentParser() 110 | parser.add_argument('--dnn_hidden_units', type = str, default = DNN_HIDDEN_UNITS_DEFAULT, 111 | help='Comma separated list of number of units in each hidden layer') 112 | parser.add_argument('--learning_rate', type = float, default = LEARNING_RATE_DEFAULT, 113 | help='Learning rate') 114 | parser.add_argument('--max_steps', type = int, default = MAX_STEPS_DEFAULT, 115 | help='Number of steps to run trainer.') 116 | parser.add_argument('--batch_size', type = int, default = BATCH_SIZE_DEFAULT, 117 | help='Batch size to run trainer.') 118 | parser.add_argument('--eval_freq', type=int, default=EVAL_FREQ_DEFAULT, 119 | help='Frequency of evaluation on the test set') 120 | parser.add_argument('--data_dir', type = str, default = DATA_DIR_DEFAULT, 121 | help='Directory for storing input data') 122 | parser.add_argument('--neg_slope', type=float, default=NEG_SLOPE_DEFAULT, 123 | help='Negative slope parameter for LeakyReLU') 124 | FLAGS, unparsed = parser.parse_known_args() 125 | 126 | main() -------------------------------------------------------------------------------- /assignment_1/code/unittests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | 6 | from modules import LinearModule, LeakyReLUModule, SoftMaxModule, CrossEntropyModule 7 | from custom_batchnorm import CustomBatchNormAutograd, CustomBatchNormManualFunction, CustomBatchNormManualModule 8 | from gradient_check import eval_numerical_gradient, eval_numerical_gradient_array 9 | 10 | def rel_error(x, y): 11 | return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y)))) 12 | 13 | def dense_to_one_hot(labels_dense, num_classes): 14 | num_labels = labels_dense.shape[0] 15 | index_offset = np.arange(num_labels) * num_classes 16 | labels_one_hot = np.zeros((num_labels, num_classes)) 17 | labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 18 | return labels_one_hot 19 | 20 | class TestBatchNorm(unittest.TestCase): 21 | 22 | def test_autograd(self): 23 | np.random.seed(42) 24 | torch.manual_seed(42) 25 | for test_num in range(10): 26 | n_batch = int(np.random.choice(range(32, 128))) 27 | n_neurons = int(np.random.choice(range(1, 10))) 28 | x = 2 * torch.randn(n_batch, n_neurons, requires_grad=True) + 10 29 | bn_auto = CustomBatchNormAutograd(n_neurons) 30 | y_auto = bn_auto(x) 31 | self.assertLess(np.max(np.abs(y_auto.mean(dim=0).data.numpy())), 1e-5) 32 | self.assertLess(np.max(np.abs(y_auto.var(dim=0).data.numpy() - 1)), 1e-1) 33 | 34 | def test_manual_function(self): 35 | np.random.seed(42) 36 | torch.manual_seed(42) 37 | for test_num in range(5): 38 | n_batch = int(np.random.choice(range(64, 256))) 39 | n_neurons = int(np.random.choice(range(4, 10))) 40 | x = 2 * torch.randn(n_batch, n_neurons, requires_grad=True) + 10 41 | input = x.double() 42 | gamma = torch.sqrt(10 * torch.arange(n_neurons, dtype=torch.float64, requires_grad=True)) 43 | beta = 100 * torch.arange(n_neurons, dtype=torch.float64, requires_grad=True) 44 | bn_manual_fct = CustomBatchNormManualFunction(n_neurons) 45 | y_manual_fct = bn_manual_fct.apply(input, gamma, beta) 46 | grad_correct = torch.autograd.gradcheck(bn_manual_fct.apply, (input,gamma,beta)) 47 | self.assertLess(np.max(np.abs(y_manual_fct.mean(dim=0).data.numpy()) - np.arange(0, n_neurons * 100, 100)), 1e-5) 48 | self.assertLess(np.max(np.abs(y_manual_fct.var(dim=0).data.numpy() - np.arange(0, n_neurons * 10, 10))), 1) 49 | self.assertEqual(grad_correct, True) 50 | 51 | def test_manual_module(self): 52 | np.random.seed(42) 53 | torch.manual_seed(42) 54 | for test_num in range(10): 55 | n_batch = int(np.random.choice(range(32, 128))) 56 | n_neurons = int(np.random.choice(range(1, 10))) 57 | x = 2 * torch.randn(n_batch, n_neurons, requires_grad=True) + 10 58 | bn_manual_mod = CustomBatchNormManualModule(n_neurons) 59 | y_manual_mod = bn_manual_mod(x) 60 | self.assertLess(np.max(np.abs(y_manual_mod.mean(dim=0).data.numpy())), 1e-5) 61 | self.assertLess(np.max(np.abs(y_manual_mod.var(dim=0).data.numpy() - 1)), 1e-1) 62 | 63 | class TestLosses(unittest.TestCase): 64 | 65 | def test_crossentropy_loss(self): 66 | np.random.seed(42) 67 | rel_error_max = 1e-5 68 | 69 | for test_num in range(10): 70 | N = np.random.choice(range(1, 100)) 71 | C = np.random.choice(range(1, 10)) 72 | X = np.random.randn(N, C) 73 | y = np.random.randint(C, size=(N,)) 74 | y = dense_to_one_hot(y, C) 75 | X = np.exp(X - np.max(X, axis = 1, keepdims = True)) 76 | X /= np.sum(X, axis = 1, keepdims = True) 77 | 78 | loss = CrossEntropyModule().forward(X, y) 79 | grads = CrossEntropyModule().backward(X, y) 80 | 81 | f = lambda _: CrossEntropyModule().forward(X, y) 82 | grads_num = eval_numerical_gradient(f, X, verbose = False, h = 1e-5) 83 | self.assertLess(rel_error(grads_num, grads), rel_error_max) 84 | 85 | class TestLayers(unittest.TestCase): 86 | 87 | def test_linear_backward(self): 88 | np.random.seed(42) 89 | rel_error_max = 1e-5 90 | 91 | for test_num in range(10): 92 | N = np.random.choice(range(1, 20)) 93 | D = np.random.choice(range(1, 100)) 94 | C = np.random.choice(range(1, 10)) 95 | x = np.random.randn(N, D) 96 | dout = np.random.randn(N, C) 97 | 98 | layer = LinearModule(D, C) 99 | 100 | out = layer.forward(x) 101 | dx = layer.backward(dout) 102 | dw = layer.grads['weight'] 103 | dx_num = eval_numerical_gradient_array(lambda xx: layer.forward(xx), x, dout) 104 | dw_num = eval_numerical_gradient_array(lambda w: layer.forward(x), layer.params['weight'], dout) 105 | 106 | self.assertLess(rel_error(dx, dx_num), rel_error_max) 107 | self.assertLess(rel_error(dw, dw_num), rel_error_max) 108 | 109 | def test_relu_backward(self): 110 | np.random.seed(42) 111 | rel_error_max = 1e-6 112 | 113 | for test_num in range(10): 114 | N = np.random.choice(range(1, 20)) 115 | D = np.random.choice(range(1, 100)) 116 | x = np.random.randn(N, D) 117 | dout = np.random.randn(*x.shape) 118 | 119 | layer = LeakyReLUModule(0) 120 | 121 | out = layer.forward(x) 122 | dx = layer.backward(dout) 123 | dx_num = eval_numerical_gradient_array(lambda xx: layer.forward(xx), x, dout) 124 | 125 | self.assertLess(rel_error(dx, dx_num), rel_error_max) 126 | 127 | def test_softmax_backward(self): 128 | np.random.seed(42) 129 | rel_error_max = 1e-5 130 | 131 | for test_num in range(10): 132 | N = np.random.choice(range(1, 20)) 133 | D = np.random.choice(range(1, 100)) 134 | x = np.random.randn(N, D) 135 | dout = np.random.randn(*x.shape) 136 | 137 | layer = SoftMaxModule() 138 | 139 | out = layer.forward(x) 140 | dx = layer.backward(dout) 141 | dx_num = eval_numerical_gradient_array(lambda xx: layer.forward(xx), x, dout) 142 | 143 | self.assertLess(rel_error(dx, dx_num), rel_error_max) 144 | 145 | 146 | if __name__ == '__main__': 147 | suite = unittest.TestLoader().loadTestsFromTestCase(TestLosses) 148 | unittest.TextTestRunner(verbosity=2).run(suite) 149 | 150 | suite = unittest.TestLoader().loadTestsFromTestCase(TestLayers) 151 | unittest.TextTestRunner(verbosity=2).run(suite) 152 | 153 | suite = unittest.TestLoader().loadTestsFromTestCase(TestBatchNorm) 154 | unittest.TextTestRunner(verbosity=3).run(suite) 155 | -------------------------------------------------------------------------------- /assignment_1/environment.yml: -------------------------------------------------------------------------------- 1 | name: dl 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - _libgcc_mutex=0.1 8 | - blas=1.0 9 | - ca-certificates=2019.1.23 10 | - certifi=2019.3.9 11 | - cffi=1.12.2 12 | - cudatoolkit=10.0.130 13 | - cycler=0.10.0 14 | - dbus=1.13.6 15 | - expat=2.2.6 16 | - fontconfig=2.13.0 17 | - freetype=2.9.1 18 | - gettext=0.19.8.1 19 | - glib=2.56.2 20 | - gst-plugins-base=1.14.0 21 | - gstreamer=1.14.0 22 | - icu=58.2 23 | - intel-openmp=2019.3 24 | - jpeg=9b 25 | - kiwisolver=1.0.1 26 | - libblas=3.8.0 27 | - libcblas=3.8.0 28 | - libedit=3.1.20181209 29 | - libffi=3.2.1 30 | - libgcc-ng=8.2.0 31 | - libgfortran-ng=7.3.0 32 | - libiconv=1.15 33 | - liblapack=3.8.0 34 | - libpng=1.6.36 35 | - libstdcxx-ng=8.2.0 36 | - libtiff=4.0.10 37 | - libuuid=1.0.3 38 | - libxcb=1.13 39 | - libxml2=2.9.9 40 | - lz4=2.2.1 41 | - lz4-c=1.8.3 42 | - matplotlib=3.0.3 43 | - mkl=2019.3 44 | - mkl_fft=1.0.10 45 | - mkl_random=1.0.2 46 | - ncurses=6.1 47 | - ninja=1.9.0 48 | - numpy=1.16.2 49 | - numpy-base=1.16.2 50 | - olefile=0.46 51 | - openssl=1.1.1b 52 | - pandas=0.24.2 53 | - pcre=8.43 54 | - pillow=5.4.1 55 | - pip=19.0.3 56 | - pthread-stubs=0.4 57 | - pycparser=2.19 58 | - pyparsing=2.3.1 59 | - pyqt=5.9.2 60 | - python=3.7.3 61 | - python-dateutil=2.8.0 62 | - pytorch=1.2.0 63 | - pytz=2018.9 64 | - qt=5.9.7 65 | - readline=7.0 66 | - scipy=1.2.1 67 | - setuptools=40.8.0 68 | - sip=4.19.8 69 | - six=1.12.0 70 | - sqlite=3.27.2 71 | - tk=8.6.8 72 | - torchvision=0.2.2 73 | - tornado=6.0.2 74 | - wheel=0.33.1 75 | - xorg-libxau=1.0.9 76 | - xorg-libxdmcp=1.1.3 77 | - xz=5.2.4 78 | - zlib=1.2.11 79 | - zstd=1.3.7 80 | 81 | -------------------------------------------------------------------------------- /assignment_2/README.md: -------------------------------------------------------------------------------- 1 | # Assignment 2: Recurrent Neural Networks 2 | 3 | The second assignment will cover the topic of Recurrent Neural Networks, including backpropagation over time and LSTMs. All details can be found in the PDF document **assignment_2.pdf**. 4 | 5 | Unlike the first assignment, there are no unittests this time. We will use PyTorch and its autograd function throughout the assignment. 6 | 7 | To execute the code, the working directory is assumed to be the main directory containing `part1` and `part2`. During grading, we will run your code from this working directory. In case you also want to execute your code in the part-specific directories, you can add the following two lines in your training file: 8 | ```Python 9 | import sys 10 | sys.path.append("..") 11 | ``` 12 | 13 | ### Prerequisites 14 | 15 | You can use the same environment that you used for the first assignment. 16 | 17 | The first task can be mostly performed on your own computer (CPU), but especially for the second task, you will require a GPU to speed up your training. Hence, we suggest to run experiments on SURFSARA. 18 | 19 | ## Task 1. RNNs versus LSTMs 20 | 21 | For the first task, you will compare vanilla Recurrent Neural Networks (RNN) with Long-Short Term Networks (LSTM). You have to implement both network modules in the files `lstm.py` and `vanilla_rnn.py` from scratch (i.e. you are not allowed to use `nn.LSTM` or `nn.Linear`, but work with functionalities like `nn.Parameter`). The palindrome dataset generation is provided in `dataset.py` and can be used without any changes. 22 | 23 | The file `train.py` gives a initial structure for training your models. Make sure to integrate all (hyper-)parameters that are given for the `ArgumentParser`. Feel free to add more parameters if needed. 24 | 25 | ## Task 2. Text Generation 26 | 27 | In the second task, you will use the built-in LSTM function, `nn.LSTM`, to generate text. 28 | 29 | ### Training Data 30 | 31 | Make sure you download the books as plain text (.txt) file. Possible sources to get books from are: 32 | 33 | 1. Project Gutenberg, Dutch: https://www.gutenberg.org/browse/languages/nl 34 | 2. Project Gutenberg, English: https://www.gutenberg.org/browse/languages/en 35 | 36 | Feel free to use other languages and/or other sources. Remember to include the datasets/books you used for training in your submission. 37 | 38 | ### Bonus questions 39 | 40 | The assignment contains two bonus questions in which you can experiment further with your model. Note that they are not strictly necessary to get full points on this assignment (max. 100 points), and might require more effort compared to other questions for the same amount of points. 41 | 42 | ## Task 3. Graph Neural Networks 43 | 44 | In the third task, you will have to answer a pen-and-paper questions about Graph Neural Networks. No implementation will be needed. Sources for explaining GNNs can be found in the assignment. 45 | 46 | ## Report 47 | 48 | Similar to the first assignment, we expect you to write a small report on the study of recurrent neural networks, in which you answer all the questions in the assignment. Please, make your report to be self-contained without this README.md file. Include all results, used hyperparameters, etc. to reproduce your experiments. 49 | 50 | ### Deliverables 51 | 52 | Create zip archive with the following structure: 53 | 54 | ``` 55 | lastname_assignment_2.zip 56 | │ report_lastname.pdf 57 | │ part_1/ 58 | │ dataset.py 59 | │ lstm.py 60 | │ train.py 61 | │ vanilla_rnn.py 62 | | grads_over_time.py 63 | │ part_2/ 64 | │ dataset.py 65 | │ model.py 66 | │ train.py 67 | | assets/ 68 | ``` 69 | 70 | Replace `lastname` with your last name. In case you have created any other python files to obtain results in the report, include those in the corresponding directory as well. Remember to replace the data in the assets folder with the ones you have used (i.e. put the txt files of the books you used in this folder). Given example datasets which were not used do not have to be included in the submission. 71 | 72 | Finally, please submit your zip-file on Canvas. 73 | 74 | The deadline for the assignment is the **29 November, 23:59** (extended from 27 November). 75 | 76 | -------------------------------------------------------------------------------- /assignment_2/assignment_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_2/assignment_2.pdf -------------------------------------------------------------------------------- /assignment_2/part1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_2/part1/__init__.py -------------------------------------------------------------------------------- /assignment_2/part1/dataset.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # MIT License 3 | # 4 | # Copyright (c) 2019 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to conditions. 12 | # 13 | # Author: Deep Learning Course | Fall 2019 14 | # Date Created: 2019-09-06 15 | ################################################################################ 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import math 22 | import sys 23 | 24 | import numpy as np 25 | import torch.utils.data as data 26 | 27 | 28 | class PalindromeDataset(data.Dataset): 29 | 30 | def __init__(self, seq_length): 31 | self.seq_length = seq_length 32 | 33 | def __len__(self): 34 | # Number of possible palindroms can be very big: 35 | # (10**(seq_length/2) or (10**((seq_length+1)/2) 36 | # Therefore we return the maximum integer value 37 | return sys.maxsize 38 | 39 | def __getitem__(self, idx): 40 | # Keep last digit as target label. Note: one-hot encoding for inputs is 41 | # more suitable for training, but this also works. 42 | full_palindrome = self.generate_palindrome() 43 | # Split palindrome into inputs (N-1 digits) and target (1 digit) 44 | return full_palindrome[0:-1], int(full_palindrome[-1]) 45 | 46 | def generate_palindrome(self): 47 | # Generates a single, random palindrome number of 'length' digits. 48 | left = [np.random.randint(0, 10) for _ in range(math.ceil(self.seq_length/2))] 49 | left = np.asarray(left, dtype=np.float32) 50 | right = np.flip(left, 0) if self.seq_length % 2 == 0 else np.flip(left[:-1], 0) 51 | return np.concatenate((left, right)) -------------------------------------------------------------------------------- /assignment_2/part1/lstm.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # MIT License 3 | # 4 | # Copyright (c) 2019 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to conditions. 12 | # 13 | # Author: Deep Learning Course | Fall 2019 14 | # Date Created: 2019-09-06 15 | ################################################################################ 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import torch 22 | import torch.nn as nn 23 | 24 | ################################################################################ 25 | 26 | class LSTM(nn.Module): 27 | 28 | def __init__(self, seq_length, input_dim, num_hidden, num_classes, device='cpu'): 29 | super(LSTM, self).__init__() 30 | # Initialization here ... 31 | 32 | def forward(self, x): 33 | # Implementation here ... 34 | pass -------------------------------------------------------------------------------- /assignment_2/part1/train.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # MIT License 3 | # 4 | # Copyright (c) 2019 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to conditions. 12 | # 13 | # Author: Deep Learning Course | Fall 2019 14 | # Date Created: 2019-09-06 15 | ################################################################################ 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import argparse 22 | import time 23 | from datetime import datetime 24 | import numpy as np 25 | 26 | import torch 27 | from torch.utils.data import DataLoader 28 | 29 | from part1.dataset import PalindromeDataset 30 | from part1.vanilla_rnn import VanillaRNN 31 | from part1.lstm import LSTM 32 | 33 | # You may want to look into tensorboard for logging 34 | # from torch.utils.tensorboard import SummaryWriter 35 | 36 | ################################################################################ 37 | 38 | def train(config): 39 | 40 | assert config.model_type in ('RNN', 'LSTM') 41 | 42 | # Initialize the device which to run the model on 43 | device = torch.device(config.device) 44 | 45 | # Initialize the model that we are going to use 46 | model = None # fixme 47 | 48 | # Initialize the dataset and data loader (note the +1) 49 | dataset = PalindromeDataset(config.input_length+1) 50 | data_loader = DataLoader(dataset, config.batch_size, num_workers=1) 51 | 52 | # Setup the loss and optimizer 53 | criterion = None # fixme 54 | optimizer = None # fixme 55 | 56 | for step, (batch_inputs, batch_targets) in enumerate(data_loader): 57 | 58 | # Only for time measurement of step through network 59 | t1 = time.time() 60 | 61 | # Add more code here ... 62 | 63 | ############################################################################ 64 | # QUESTION: what happens here and why? 65 | ############################################################################ 66 | torch.nn.utils.clip_grad_norm(model.parameters(), max_norm=config.max_norm) 67 | ############################################################################ 68 | 69 | # Add more code here ... 70 | 71 | loss = np.inf # fixme 72 | accuracy = 0.0 # fixme 73 | 74 | # Just for time measurement 75 | t2 = time.time() 76 | examples_per_second = config.batch_size/float(t2-t1) 77 | 78 | if step % 10 == 0: 79 | 80 | print("[{}] Train Step {:04d}/{:04d}, Batch Size = {}, Examples/Sec = {:.2f}, " 81 | "Accuracy = {:.2f}, Loss = {:.3f}".format( 82 | datetime.now().strftime("%Y-%m-%d %H:%M"), step, 83 | config.train_steps, config.batch_size, examples_per_second, 84 | accuracy, loss 85 | )) 86 | 87 | if step == config.train_steps: 88 | # If you receive a PyTorch data-loader error, check this bug report: 89 | # https://github.com/pytorch/pytorch/pull/9655 90 | break 91 | 92 | print('Done training.') 93 | 94 | 95 | ################################################################################ 96 | ################################################################################ 97 | 98 | if __name__ == "__main__": 99 | 100 | # Parse training configuration 101 | parser = argparse.ArgumentParser() 102 | 103 | # Model params 104 | parser.add_argument('--model_type', type=str, default="RNN", help="Model type, should be 'RNN' or 'LSTM'") 105 | parser.add_argument('--input_length', type=int, default=10, help='Length of an input sequence') 106 | parser.add_argument('--input_dim', type=int, default=1, help='Dimensionality of input sequence') 107 | parser.add_argument('--num_classes', type=int, default=10, help='Dimensionality of output sequence') 108 | parser.add_argument('--num_hidden', type=int, default=128, help='Number of hidden units in the model') 109 | parser.add_argument('--batch_size', type=int, default=128, help='Number of examples to process in a batch') 110 | parser.add_argument('--learning_rate', type=float, default=0.001, help='Learning rate') 111 | parser.add_argument('--train_steps', type=int, default=10000, help='Number of training steps') 112 | parser.add_argument('--max_norm', type=float, default=10.0) 113 | parser.add_argument('--device', type=str, default="cuda:0", help="Training device 'cpu' or 'cuda:0'") 114 | 115 | config = parser.parse_args() 116 | 117 | # Train the model 118 | train(config) -------------------------------------------------------------------------------- /assignment_2/part1/vanilla_rnn.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # MIT License 3 | # 4 | # Copyright (c) 2019 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to conditions. 12 | # 13 | # Author: Deep Learning Course | Fall 2019 14 | # Date Created: 2019-09-06 15 | ################################################################################ 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import torch 22 | import torch.nn as nn 23 | 24 | ################################################################################ 25 | 26 | class VanillaRNN(nn.Module): 27 | 28 | def __init__(self, seq_length, input_dim, num_hidden, num_classes, device='cpu'): 29 | super(VanillaRNN, self).__init__() 30 | # Initialization here ... 31 | 32 | def forward(self, x): 33 | # Implementation here ... 34 | pass 35 | -------------------------------------------------------------------------------- /assignment_2/part2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_2/part2/__init__.py -------------------------------------------------------------------------------- /assignment_2/part2/assets/book_TeX_calculus_made_easy.tex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_2/part2/assets/book_TeX_calculus_made_easy.tex -------------------------------------------------------------------------------- /assignment_2/part2/dataset.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2019 Tom Runia 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to conditions. 11 | # 12 | # Author: Deep Learning Course | Fall 2019 13 | # Date Created: 2019-09-06 14 | ################################################################################ 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import os 21 | import numpy as np 22 | import torch.utils.data as data 23 | 24 | 25 | class TextDataset(data.Dataset): 26 | 27 | def __init__(self, filename, seq_length): 28 | assert os.path.splitext(filename)[1] == ".txt" 29 | self._seq_length = seq_length 30 | self._data = open(filename, 'r').read() 31 | self._chars = list(set(self._data)) 32 | self._data_size, self._vocab_size = len(self._data), len(self._chars) 33 | print("Initialize dataset with {} characters, {} unique.".format( 34 | self._data_size, self._vocab_size)) 35 | self._char_to_ix = { ch:i for i,ch in enumerate(self._chars) } 36 | self._ix_to_char = { i:ch for i,ch in enumerate(self._chars) } 37 | self._offset = 0 38 | 39 | def __getitem__(self, item): 40 | offset = np.random.randint(0, len(self._data)-self._seq_length-2) 41 | inputs = [self._char_to_ix[ch] for ch in self._data[offset:offset+self._seq_length]] 42 | targets = [self._char_to_ix[ch] for ch in self._data[offset+1:offset+self._seq_length+1]] 43 | return torch.LongTensor(inputs), torch.LongTensor(targets) 44 | 45 | def convert_to_string(self, char_ix): 46 | return ''.join(self._ix_to_char[ix] for ix in char_ix) 47 | 48 | def __len__(self): 49 | return self._data_size 50 | 51 | @property 52 | def vocab_size(self): 53 | return self._vocab_size -------------------------------------------------------------------------------- /assignment_2/part2/model.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2019 Tom Runia 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to conditions. 11 | # 12 | # Author: Deep Learning Course | Fall 2019 13 | # Date Created: 2019-09-06 14 | ################################################################################ 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import torch.nn as nn 21 | 22 | 23 | class TextGenerationModel(nn.Module): 24 | 25 | def __init__(self, batch_size, seq_length, vocabulary_size, 26 | lstm_num_hidden=256, lstm_num_layers=2, device='cuda:0'): 27 | 28 | super(TextGenerationModel, self).__init__() 29 | # Initialization here... 30 | 31 | def forward(self, x): 32 | # Implementation here... 33 | pass 34 | -------------------------------------------------------------------------------- /assignment_2/part2/train.py: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2019 Tom Runia 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to conditions. 11 | # 12 | # Author: Deep Learning Course | Fall 2019 13 | # Date Created: 2019-09-06 14 | ################################################################################ 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import os 21 | import time 22 | from datetime import datetime 23 | import argparse 24 | 25 | import numpy as np 26 | 27 | import torch 28 | import torch.optim as optim 29 | from torch.utils.data import DataLoader 30 | 31 | from part2.dataset import TextDataset 32 | from part2.model import TextGenerationModel 33 | 34 | ################################################################################ 35 | 36 | def train(config): 37 | 38 | # Initialize the device which to run the model on 39 | device = torch.device(config.device) 40 | 41 | # Initialize the model that we are going to use 42 | model = TextGenerationModel( ... ) # fixme 43 | 44 | # Initialize the dataset and data loader (note the +1) 45 | dataset = TextDataset( ... ) # fixme 46 | data_loader = DataLoader(dataset, config.batch_size, num_workers=1) 47 | 48 | # Setup the loss and optimizer 49 | criterion = None # fixme 50 | optimizer = None # fixme 51 | 52 | for step, (batch_inputs, batch_targets) in enumerate(data_loader): 53 | 54 | # Only for time measurement of step through network 55 | t1 = time.time() 56 | 57 | ####################################################### 58 | # Add more code here ... 59 | ####################################################### 60 | 61 | loss = np.inf # fixme 62 | accuracy = 0.0 # fixme 63 | 64 | # Just for time measurement 65 | t2 = time.time() 66 | examples_per_second = config.batch_size/float(t2-t1) 67 | 68 | if step % config.print_every == 0: 69 | 70 | print("[{}] Train Step {:04d}/{:04d}, Batch Size = {}, Examples/Sec = {:.2f}, " 71 | "Accuracy = {:.2f}, Loss = {:.3f}".format( 72 | datetime.now().strftime("%Y-%m-%d %H:%M"), step, 73 | config.train_steps, config.batch_size, examples_per_second, 74 | accuracy, loss 75 | )) 76 | 77 | if step == config.sample_every: 78 | # Generate some sentences by sampling from the model 79 | pass 80 | 81 | if step == config.train_steps: 82 | # If you receive a PyTorch data-loader error, check this bug report: 83 | # https://github.com/pytorch/pytorch/pull/9655 84 | break 85 | 86 | print('Done training.') 87 | 88 | 89 | ################################################################################ 90 | ################################################################################ 91 | 92 | if __name__ == "__main__": 93 | 94 | # Parse training configuration 95 | parser = argparse.ArgumentParser() 96 | 97 | # Model params 98 | parser.add_argument('--txt_file', type=str, required=True, help="Path to a .txt file to train on") 99 | parser.add_argument('--seq_length', type=int, default=30, help='Length of an input sequence') 100 | parser.add_argument('--lstm_num_hidden', type=int, default=128, help='Number of hidden units in the LSTM') 101 | parser.add_argument('--lstm_num_layers', type=int, default=2, help='Number of LSTM layers in the model') 102 | 103 | # Training params 104 | parser.add_argument('--batch_size', type=int, default=64, help='Number of examples to process in a batch') 105 | parser.add_argument('--learning_rate', type=float, default=2e-3, help='Learning rate') 106 | 107 | # It is not necessary to implement the following three params, but it may help training. 108 | parser.add_argument('--learning_rate_decay', type=float, default=0.96, help='Learning rate decay fraction') 109 | parser.add_argument('--learning_rate_step', type=int, default=5000, help='Learning rate step') 110 | parser.add_argument('--dropout_keep_prob', type=float, default=1.0, help='Dropout keep probability') 111 | 112 | parser.add_argument('--train_steps', type=int, default=1e6, help='Number of training steps') 113 | parser.add_argument('--max_norm', type=float, default=5.0, help='--') 114 | 115 | # Misc params 116 | parser.add_argument('--summary_path', type=str, default="./summaries/", help='Output path for summaries') 117 | parser.add_argument('--print_every', type=int, default=5, help='How often to print training progress') 118 | parser.add_argument('--sample_every', type=int, default=100, help='How often to sample from the model') 119 | 120 | config = parser.parse_args() 121 | 122 | # Train the model 123 | train(config) 124 | -------------------------------------------------------------------------------- /assignment_3/assignment_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_3/assignment_3.pdf -------------------------------------------------------------------------------- /assignment_3/templates/a3_gan_template.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torchvision.transforms as transforms 7 | from torchvision.utils import save_image 8 | from torchvision import datasets 9 | 10 | 11 | class Generator(nn.Module): 12 | def __init__(self): 13 | super(Generator, self).__init__() 14 | 15 | # Construct generator. You are free to experiment with your model, 16 | # but the following is a good start: 17 | # Linear args.latent_dim -> 128 18 | # LeakyReLU(0.2) 19 | # Linear 128 -> 256 20 | # Bnorm 21 | # LeakyReLU(0.2) 22 | # Linear 256 -> 512 23 | # Bnorm 24 | # LeakyReLU(0.2) 25 | # Linear 512 -> 1024 26 | # Bnorm 27 | # LeakyReLU(0.2) 28 | # Linear 1024 -> 768 29 | # Output non-linearity 30 | 31 | def forward(self, z): 32 | # Generate images from z 33 | pass 34 | 35 | 36 | class Discriminator(nn.Module): 37 | def __init__(self): 38 | super(Discriminator, self).__init__() 39 | 40 | # Construct distriminator. You are free to experiment with your model, 41 | # but the following is a good start: 42 | # Linear 784 -> 512 43 | # LeakyReLU(0.2) 44 | # Linear 512 -> 256 45 | # LeakyReLU(0.2) 46 | # Linear 256 -> 1 47 | # Output non-linearity 48 | 49 | def forward(self, img): 50 | # return discriminator score for img 51 | pass 52 | 53 | 54 | def train(dataloader, discriminator, generator, optimizer_G, optimizer_D): 55 | for epoch in range(args.n_epochs): 56 | for i, (imgs, _) in enumerate(dataloader): 57 | 58 | imgs.cuda() 59 | 60 | # Train Generator 61 | # --------------- 62 | 63 | # Train Discriminator 64 | # ------------------- 65 | optimizer_D.zero_grad() 66 | 67 | # Save Images 68 | # ----------- 69 | batches_done = epoch * len(dataloader) + i 70 | if batches_done % args.save_interval == 0: 71 | # You can use the function save_image(Tensor (shape Bx1x28x28), 72 | # filename, number of rows, normalize) to save the generated 73 | # images, e.g.: 74 | # save_image(gen_imgs[:25], 75 | # 'images/{}.png'.format(batches_done), 76 | # nrow=5, normalize=True) 77 | pass 78 | 79 | 80 | def main(): 81 | # Create output image directory 82 | os.makedirs('images', exist_ok=True) 83 | 84 | # load data 85 | dataloader = torch.utils.data.DataLoader( 86 | datasets.MNIST('./data/mnist', train=True, download=True, 87 | transform=transforms.Compose([ 88 | transforms.ToTensor(), 89 | transforms.Normalize((0.5, 0.5, 0.5), 90 | (0.5, 0.5, 0.5))])), 91 | batch_size=args.batch_size, shuffle=True) 92 | 93 | # Initialize models and optimizers 94 | generator = Generator() 95 | discriminator = Discriminator() 96 | optimizer_G = torch.optim.Adam(generator.parameters(), lr=args.lr) 97 | optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=args.lr) 98 | 99 | # Start training 100 | train(dataloader, discriminator, generator, optimizer_G, optimizer_D) 101 | 102 | # You can save your generator here to re-use it to generate images for your 103 | # report, e.g.: 104 | # torch.save(generator.state_dict(), "mnist_generator.pt") 105 | 106 | 107 | if __name__ == "__main__": 108 | parser = argparse.ArgumentParser() 109 | parser.add_argument('--n_epochs', type=int, default=200, 110 | help='number of epochs') 111 | parser.add_argument('--batch_size', type=int, default=64, 112 | help='batch size') 113 | parser.add_argument('--lr', type=float, default=0.0002, 114 | help='learning rate') 115 | parser.add_argument('--latent_dim', type=int, default=100, 116 | help='dimensionality of the latent space') 117 | parser.add_argument('--save_interval', type=int, default=500, 118 | help='save every SAVE_INTERVAL iterations') 119 | args = parser.parse_args() 120 | 121 | main() 122 | -------------------------------------------------------------------------------- /assignment_3/templates/a3_nf_template.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import torch 4 | import torch.nn as nn 5 | import matplotlib.pyplot as plt 6 | import torch.nn.functional as F 7 | import numpy as np 8 | from datasets.mnist import mnist 9 | import os 10 | from torchvision.utils import make_grid 11 | 12 | 13 | def log_prior(x): 14 | """ 15 | Compute the elementwise log probability of a standard Gaussian, i.e. 16 | N(x | mu=0, sigma=1). 17 | """ 18 | raise NotImplementedError 19 | return logp 20 | 21 | 22 | def sample_prior(size): 23 | """ 24 | Sample from a standard Gaussian. 25 | """ 26 | raise NotImplementedError 27 | 28 | if torch.cuda.is_available(): 29 | sample = sample.cuda() 30 | 31 | return sample 32 | 33 | 34 | def get_mask(): 35 | mask = np.zeros((28, 28), dtype='float32') 36 | for i in range(28): 37 | for j in range(28): 38 | if (i + j) % 2 == 0: 39 | mask[i, j] = 1 40 | 41 | mask = mask.reshape(1, 28*28) 42 | mask = torch.from_numpy(mask) 43 | 44 | return mask 45 | 46 | 47 | class Coupling(torch.nn.Module): 48 | def __init__(self, c_in, mask, n_hidden=1024): 49 | super().__init__() 50 | self.n_hidden = n_hidden 51 | 52 | # Assigns mask to self.mask and creates reference for pytorch. 53 | self.register_buffer('mask', mask) 54 | 55 | # Create shared architecture to generate both the translation and 56 | # scale variables. 57 | # Suggestion: Linear ReLU Linear ReLU Linear. 58 | self.nn = torch.nn.Sequential( 59 | None 60 | ) 61 | 62 | # The nn should be initialized such that the weights of the last layer 63 | # is zero, so that its initial transform is identity. 64 | self.nn[-1].weight.data.zero_() 65 | self.nn[-1].bias.data.zero_() 66 | 67 | def forward(self, z, ldj, reverse=False): 68 | # Implement the forward and inverse for an affine coupling layer. Split 69 | # the input using the mask in self.mask. Transform one part with 70 | # Make sure to account for the log Jacobian determinant (ldj). 71 | # For reference, check: Density estimation using RealNVP. 72 | 73 | # NOTE: For stability, it is advised to model the scale via: 74 | # log_scale = tanh(h), where h is the scale-output 75 | # from the NN. 76 | 77 | if not reverse: 78 | raise NotImplementedError 79 | else: 80 | raise NotImplementedError 81 | 82 | return z, ldj 83 | 84 | 85 | class Flow(nn.Module): 86 | def __init__(self, shape, n_flows=4): 87 | super().__init__() 88 | channels, = shape 89 | 90 | mask = get_mask() 91 | 92 | self.layers = torch.nn.ModuleList() 93 | 94 | for i in range(n_flows): 95 | self.layers.append(Coupling(c_in=channels, mask=mask)) 96 | self.layers.append(Coupling(c_in=channels, mask=1-mask)) 97 | 98 | self.z_shape = (channels,) 99 | 100 | def forward(self, z, logdet, reverse=False): 101 | if not reverse: 102 | for layer in self.layers: 103 | z, logdet = layer(z, logdet) 104 | else: 105 | for layer in reversed(self.layers): 106 | z, logdet = layer(z, logdet, reverse=True) 107 | 108 | return z, logdet 109 | 110 | 111 | class Model(nn.Module): 112 | def __init__(self, shape): 113 | super().__init__() 114 | self.flow = Flow(shape) 115 | 116 | def dequantize(self, z): 117 | return z + torch.rand_like(z) 118 | 119 | def logit_normalize(self, z, logdet, reverse=False): 120 | """ 121 | Inverse sigmoid normalization. 122 | """ 123 | alpha = 1e-5 124 | 125 | if not reverse: 126 | # Divide by 256 and update ldj. 127 | z = z / 256. 128 | logdet -= np.log(256) * np.prod(z.size()[1:]) 129 | 130 | # Logit normalize 131 | z = z*(1-alpha) + alpha*0.5 132 | logdet += torch.sum(-torch.log(z) - torch.log(1-z), dim=1) 133 | z = torch.log(z) - torch.log(1-z) 134 | 135 | else: 136 | # Inverse normalize 137 | z = torch.sigmoid(z) 138 | logdet += torch.sum(torch.log(z) + torch.log(1-z), dim=1) 139 | z = (z - alpha*0.5)/(1 - alpha) 140 | 141 | # Multiply by 256. 142 | logdet += np.log(256) * np.prod(z.size()[1:]) 143 | z = z * 256. 144 | 145 | return z, logdet 146 | 147 | def forward(self, input): 148 | """ 149 | Given input, encode the input to z space. Also keep track of ldj. 150 | """ 151 | z = input 152 | ldj = torch.zeros(z.size(0), device=z.device) 153 | 154 | z = self.dequantize(z) 155 | z, ldj = self.logit_normalize(z, ldj) 156 | 157 | z, ldj = self.flow(z, ldj) 158 | 159 | # Compute log_pz and log_px per example 160 | 161 | raise NotImplementedError 162 | 163 | return log_px 164 | 165 | def sample(self, n_samples): 166 | """ 167 | Sample n_samples from the model. Sample from prior and create ldj. 168 | Then invert the flow and invert the logit_normalize. 169 | """ 170 | z = sample_prior((n_samples,) + self.flow.z_shape) 171 | ldj = torch.zeros(z.size(0), device=z.device) 172 | 173 | raise NotImplementedError 174 | 175 | return z 176 | 177 | 178 | def epoch_iter(model, data, optimizer): 179 | """ 180 | Perform a single epoch for either the training or validation. 181 | use model.training to determine if in 'training mode' or not. 182 | 183 | Returns the average bpd ("bits per dimension" which is the negative 184 | log_2 likelihood per dimension) averaged over the complete epoch. 185 | """ 186 | 187 | avg_bpd = None 188 | 189 | return avg_bpd 190 | 191 | 192 | def run_epoch(model, data, optimizer): 193 | """ 194 | Run a train and validation epoch and return average bpd for each. 195 | """ 196 | traindata, valdata = data 197 | 198 | model.train() 199 | train_bpd = epoch_iter(model, traindata, optimizer) 200 | 201 | model.eval() 202 | val_bpd = epoch_iter(model, valdata, optimizer) 203 | 204 | return train_bpd, val_bpd 205 | 206 | 207 | def save_bpd_plot(train_curve, val_curve, filename): 208 | plt.figure(figsize=(12, 6)) 209 | plt.plot(train_curve, label='train bpd') 210 | plt.plot(val_curve, label='validation bpd') 211 | plt.legend() 212 | plt.xlabel('epochs') 213 | plt.ylabel('bpd') 214 | plt.tight_layout() 215 | plt.savefig(filename) 216 | 217 | 218 | def main(): 219 | data = mnist()[:2] # ignore test split 220 | 221 | model = Model(shape=[784]) 222 | 223 | if torch.cuda.is_available(): 224 | model = model.cuda() 225 | 226 | optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) 227 | 228 | os.makedirs('images_nfs', exist_ok=True) 229 | 230 | train_curve, val_curve = [], [] 231 | for epoch in range(ARGS.epochs): 232 | bpds = run_epoch(model, data, optimizer) 233 | train_bpd, val_bpd = bpds 234 | train_curve.append(train_bpd) 235 | val_curve.append(val_bpd) 236 | print("[Epoch {epoch}] train bpd: {train_bpd} val_bpd: {val_bpd}".format( 237 | epoch=epoch, train_bpd=train_bpd, val_bpd=val_bpd)) 238 | 239 | # -------------------------------------------------------------------- 240 | # Add functionality to plot samples from model during training. 241 | # You can use the make_grid functionality that is already imported. 242 | # Save grid to images_nfs/ 243 | # -------------------------------------------------------------------- 244 | 245 | save_bpd_plot(train_curve, val_curve, 'nfs_bpd.pdf') 246 | 247 | 248 | if __name__ == "__main__": 249 | parser = argparse.ArgumentParser() 250 | parser.add_argument('--epochs', default=40, type=int, 251 | help='max number of epochs') 252 | 253 | ARGS = parser.parse_args() 254 | 255 | main() 256 | -------------------------------------------------------------------------------- /assignment_3/templates/a3_vae_template.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import torch 4 | import torch.nn as nn 5 | import matplotlib.pyplot as plt 6 | from torchvision.utils import make_grid 7 | 8 | from datasets.bmnist import bmnist 9 | 10 | 11 | class Encoder(nn.Module): 12 | 13 | def __init__(self, hidden_dim=500, z_dim=20): 14 | super().__init__() 15 | 16 | def forward(self, input): 17 | """ 18 | Perform forward pass of encoder. 19 | 20 | Returns mean and std with shape [batch_size, z_dim]. Make sure 21 | that any constraints are enforced. 22 | """ 23 | mean, std = None, None 24 | raise NotImplementedError() 25 | 26 | return mean, std 27 | 28 | 29 | class Decoder(nn.Module): 30 | 31 | def __init__(self, hidden_dim=500, z_dim=20): 32 | super().__init__() 33 | 34 | def forward(self, input): 35 | """ 36 | Perform forward pass of encoder. 37 | 38 | Returns mean with shape [batch_size, 784]. 39 | """ 40 | mean = None 41 | raise NotImplementedError() 42 | 43 | return mean 44 | 45 | 46 | class VAE(nn.Module): 47 | 48 | def __init__(self, hidden_dim=500, z_dim=20): 49 | super().__init__() 50 | 51 | self.z_dim = z_dim 52 | self.encoder = Encoder(hidden_dim, z_dim) 53 | self.decoder = Decoder(hidden_dim, z_dim) 54 | 55 | def forward(self, input): 56 | """ 57 | Given input, perform an encoding and decoding step and return the 58 | negative average elbo for the given batch. 59 | """ 60 | average_negative_elbo = None 61 | raise NotImplementedError() 62 | return average_negative_elbo 63 | 64 | def sample(self, n_samples): 65 | """ 66 | Sample n_samples from the model. Return both the sampled images 67 | (from bernoulli) and the means for these bernoullis (as these are 68 | used to plot the data manifold). 69 | """ 70 | sampled_ims, im_means = None, None 71 | raise NotImplementedError() 72 | 73 | return sampled_ims, im_means 74 | 75 | 76 | def epoch_iter(model, data, optimizer): 77 | """ 78 | Perform a single epoch for either the training or validation. 79 | use model.training to determine if in 'training mode' or not. 80 | 81 | Returns the average elbo for the complete epoch. 82 | """ 83 | average_epoch_elbo = None 84 | raise NotImplementedError() 85 | 86 | return average_epoch_elbo 87 | 88 | 89 | def run_epoch(model, data, optimizer): 90 | """ 91 | Run a train and validation epoch and return average elbo for each. 92 | """ 93 | traindata, valdata = data 94 | 95 | model.train() 96 | train_elbo = epoch_iter(model, traindata, optimizer) 97 | 98 | model.eval() 99 | val_elbo = epoch_iter(model, valdata, optimizer) 100 | 101 | return train_elbo, val_elbo 102 | 103 | 104 | def save_elbo_plot(train_curve, val_curve, filename): 105 | plt.figure(figsize=(12, 6)) 106 | plt.plot(train_curve, label='train elbo') 107 | plt.plot(val_curve, label='validation elbo') 108 | plt.legend() 109 | plt.xlabel('epochs') 110 | plt.ylabel('ELBO') 111 | plt.tight_layout() 112 | plt.savefig(filename) 113 | 114 | 115 | def main(): 116 | data = bmnist()[:2] # ignore test split 117 | model = VAE(z_dim=ARGS.zdim) 118 | optimizer = torch.optim.Adam(model.parameters()) 119 | 120 | train_curve, val_curve = [], [] 121 | for epoch in range(ARGS.epochs): 122 | elbos = run_epoch(model, data, optimizer) 123 | train_elbo, val_elbo = elbos 124 | train_curve.append(train_elbo) 125 | val_curve.append(val_elbo) 126 | print(f"[Epoch {epoch}] train elbo: {train_elbo} val_elbo: {val_elbo}") 127 | 128 | # -------------------------------------------------------------------- 129 | # Add functionality to plot samples from model during training. 130 | # You can use the make_grid functioanlity that is already imported. 131 | # -------------------------------------------------------------------- 132 | 133 | # -------------------------------------------------------------------- 134 | # Add functionality to plot plot the learned data manifold after 135 | # if required (i.e., if zdim == 2). You can use the make_grid 136 | # functionality that is already imported. 137 | # -------------------------------------------------------------------- 138 | 139 | save_elbo_plot(train_curve, val_curve, 'elbo.pdf') 140 | 141 | 142 | if __name__ == "__main__": 143 | parser = argparse.ArgumentParser() 144 | parser.add_argument('--epochs', default=40, type=int, 145 | help='max number of epochs') 146 | parser.add_argument('--zdim', default=20, type=int, 147 | help='dimensionality of latent space') 148 | 149 | ARGS = parser.parse_args() 150 | 151 | main() 152 | -------------------------------------------------------------------------------- /assignment_3/templates/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uvadlc/DL_assignments_2019/b396da3ef01084e06eb8d6791f354c8afa8fafbd/assignment_3/templates/datasets/__init__.py -------------------------------------------------------------------------------- /assignment_3/templates/datasets/bmnist.py: -------------------------------------------------------------------------------- 1 | import os 2 | import errno 3 | 4 | import numpy as np 5 | import torch 6 | import torch.utils.data as data 7 | from torchvision.datasets.utils import download_url 8 | import torchvision.transforms as transforms 9 | from PIL import Image 10 | 11 | 12 | BMNIST_BASE_URL = "http://www.cs.toronto.edu/~larocheh/public/datasets/" \ 13 | "binarized_mnist/binarized_mnist_{}.amat" 14 | 15 | 16 | class BMNIST(data.Dataset): 17 | """ BINARY MNIST """ 18 | 19 | urls = [BMNIST_BASE_URL.format(split) for split in 20 | ['train', 'valid', 'test']] 21 | raw_folder = "raw" 22 | processed_folder = "processed" 23 | training_file = "train.pt" 24 | val_file = "val.pt" 25 | test_file = "test.pt" 26 | 27 | def __init__(self, root, split='train', transform=None, download=False): 28 | self.root = os.path.expanduser(root) 29 | self.transform = transform 30 | self.split = split 31 | 32 | if split not in ('train', 'val', 'test'): 33 | raise ValueError('split should be one of {train, val, test}') 34 | 35 | if download: 36 | self.download() 37 | 38 | if not self._check_exists(): 39 | raise RuntimeError('Dataset not found.' + 40 | ' You can use download=True to download it') 41 | 42 | data_file = {'train': self.training_file, 43 | 'val': self.val_file, 44 | 'test': self.test_file}[split] 45 | path = os.path.join(self.root, self.processed_folder, data_file) 46 | self.data = torch.load(path) 47 | 48 | def __getitem__(self, index): 49 | img = self.data[index] 50 | 51 | # doing this so that it is consistent with all other datasets 52 | # to return a PIL Image 53 | img = Image.fromarray(img.float().numpy()) 54 | 55 | if self.transform is not None: 56 | img = self.transform(img) 57 | 58 | return img 59 | 60 | def __len__(self): 61 | return len(self.data) 62 | 63 | def _check_exists(self): 64 | processed_folder = os.path.join(self.root, self.processed_folder) 65 | train_path = os.path.join(processed_folder, self.training_file) 66 | val_path = os.path.join(processed_folder, self.val_file) 67 | test_path = os.path.join(processed_folder, self.test_file) 68 | return os.path.exists(train_path) and os.path.exists(val_path) and \ 69 | os.path.exists(test_path) 70 | 71 | def _read_raw_image_file(self, path): 72 | 73 | raw_file = os.path.join(self.root, self.raw_folder, path) 74 | all_images = [] 75 | with open(raw_file) as f: 76 | for line in f: 77 | im = [int(x) for x in line.strip().split()] 78 | assert len(im) == 28 ** 2 79 | all_images.append(im) 80 | return torch.from_numpy(np.array(all_images)).view(-1, 28, 28) 81 | 82 | def download(self): 83 | """ 84 | Download the BMNIST data if it doesn't exist in 85 | processed_folder already. 86 | """ 87 | 88 | if self._check_exists(): 89 | return 90 | 91 | # Create folders 92 | try: 93 | os.makedirs(os.path.join(self.root, self.raw_folder)) 94 | os.makedirs(os.path.join(self.root, self.processed_folder)) 95 | except OSError as e: 96 | if e.errno == errno.EEXIST: 97 | pass 98 | else: 99 | raise 100 | 101 | for url in self.urls: 102 | filename = url.rpartition('/')[2] 103 | download_url(url, root=os.path.join(self.root, self.raw_folder), 104 | filename=filename, md5=None) 105 | 106 | # process and save as torch files 107 | print('Processing raw data..') 108 | 109 | training_set = self._read_raw_image_file('binarized_mnist_train.amat') 110 | val_set = self._read_raw_image_file('binarized_mnist_valid.amat') 111 | test_set = self._read_raw_image_file('binarized_mnist_test.amat') 112 | 113 | processed_dir = os.path.join(self.root, self.processed_folder) 114 | with open(os.path.join(processed_dir, self.training_file), 'wb') as f: 115 | torch.save(training_set, f) 116 | with open(os.path.join(processed_dir, self.val_file), 'wb') as f: 117 | torch.save(val_set, f) 118 | with open(os.path.join(processed_dir, self.test_file), 'wb') as f: 119 | torch.save(test_set, f) 120 | 121 | print('Completed data download.') 122 | 123 | def __repr__(self): 124 | fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' 125 | fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) 126 | fmt_str += ' Split: {}\n'.format(self.split) 127 | fmt_str += ' Root Location: {}\n'.format(self.root) 128 | tmp = ' Transforms (if any): ' 129 | tmp_ = self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)) 130 | fmt_str += '{0}{1}\n'.format(tmp, tmp_) 131 | return fmt_str 132 | 133 | 134 | def bmnist(root='./data/', batch_size=128, download=True): 135 | 136 | data_transforms = transforms.Compose([transforms.ToTensor()]) 137 | 138 | train_set = BMNIST(root, 'train', data_transforms, download) 139 | val_set = BMNIST(root, 'val', data_transforms, download) 140 | test_set = BMNIST(root, 'test', data_transforms, download) 141 | 142 | trainloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, 143 | shuffle=True, num_workers=4) 144 | valloader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, 145 | shuffle=False, num_workers=10) 146 | testloader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, 147 | shuffle=False, num_workers=10) 148 | 149 | return trainloader, valloader, testloader 150 | -------------------------------------------------------------------------------- /assignment_3/templates/datasets/mnist.py: -------------------------------------------------------------------------------- 1 | import os 2 | import errno 3 | 4 | import numpy as np 5 | import torch 6 | import torch.utils.data as data 7 | import torchvision.transforms as transforms 8 | import torchvision 9 | 10 | 11 | class toTensor: 12 | def __init__(self): 13 | pass 14 | 15 | def __call__(self, img): 16 | img = np.array(img) 17 | if len(img.shape) == 2: 18 | # Add channel dimension 19 | img = img[:, :, None] 20 | 21 | img = np.array(img).transpose(2, 0, 1) 22 | return torch.from_numpy(img) 23 | 24 | 25 | class Flatten: 26 | def __init__(self): 27 | pass 28 | 29 | def __call__(self, img): 30 | return img.view(-1) 31 | 32 | 33 | class toFloat: 34 | def __init__(self): 35 | pass 36 | 37 | def __call__(self, img): 38 | return img.float() 39 | 40 | 41 | def mnist(root='./data/', batch_size=128, download=True): 42 | train_transforms = transforms.Compose([ 43 | transforms.Pad(int(np.ceil(28 * 0.05)), padding_mode='edge'), 44 | transforms.RandomAffine(degrees=0, translate=(0.05, 0.05)), 45 | transforms.CenterCrop(28), 46 | toTensor(), 47 | toFloat(), 48 | Flatten() 49 | ]) 50 | data_transforms = transforms.Compose([ 51 | transforms.ToTensor(), 52 | toTensor(), 53 | toFloat(), 54 | Flatten() 55 | ]) 56 | 57 | dataset = torchvision.datasets.MNIST( 58 | root, train=True, transform=train_transforms, target_transform=None, 59 | download=True) 60 | test_set = torchvision.datasets.MNIST( 61 | root, train=False, transform=data_transforms, target_transform=None, 62 | download=True) 63 | 64 | train_dataset = data.dataset.Subset(dataset, np.arange(40000)) 65 | val_dataset = data.dataset.Subset(dataset, np.arange(40000, 50000)) 66 | 67 | trainloader = data.DataLoader( 68 | train_dataset, batch_size=batch_size, shuffle=True, num_workers=4) 69 | valloader = data.DataLoader( 70 | val_dataset, batch_size=batch_size, shuffle=False, num_workers=4) 71 | testloader = data.DataLoader( 72 | test_set, batch_size=batch_size, shuffle=False, num_workers=4) 73 | 74 | return trainloader, valloader, testloader 75 | -------------------------------------------------------------------------------- /assignment_3/templates/unittests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy as np 3 | import torch 4 | 5 | from a3_nf_template import Coupling, Flow, get_mask 6 | 7 | 8 | def mean_error(x, y): 9 | return np.mean(np.abs(x - y)) 10 | 11 | 12 | def mean_rel_error(x, y): 13 | return np.mean(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y)))) 14 | 15 | 16 | def f_layer(layer, x, logdet): 17 | with torch.no_grad(): 18 | z, logdet = layer(x, logdet, reverse=False) 19 | recon, logdet = layer(z, logdet, reverse=True) 20 | 21 | x, recon, logdet = x.numpy(), recon.numpy(), logdet.numpy() 22 | 23 | return x, recon, logdet 24 | 25 | 26 | class TestLayers(unittest.TestCase): 27 | 28 | def test_flow(self): 29 | np.random.seed(42) 30 | error_max = 1e-5 31 | 32 | for test_num in range(10): 33 | N = np.random.choice(range(1, 20)) 34 | C = 784 35 | x = torch.randn(N, C) 36 | logdet = torch.zeros(N) 37 | 38 | layer = Flow([C], n_flows=2) 39 | 40 | x, recon, logdet = f_layer(layer, x, logdet) 41 | 42 | self.assertLess(mean_rel_error(x, recon), error_max) 43 | self.assertLess(mean_error(logdet, np.zeros(N)), error_max) 44 | 45 | def test_coupling(self): 46 | np.random.seed(42) 47 | error_max = 1e-5 48 | 49 | for test_num in range(10): 50 | N = np.random.choice(range(1, 20)) 51 | C = 784 52 | x = torch.randn(N, C) 53 | logdet = torch.zeros(N) 54 | 55 | layer = Coupling(c_in=C, mask=get_mask()) 56 | 57 | x, recon, logdet = f_layer(layer, x, logdet) 58 | 59 | self.assertLess(mean_rel_error(x, recon), error_max) 60 | self.assertLess(mean_error(logdet, np.zeros(N)), error_max) 61 | 62 | 63 | if __name__ == '__main__': 64 | suite = unittest.TestLoader().loadTestsFromTestCase(TestLayers) 65 | unittest.TextTestRunner(verbosity=2).run(suite) 66 | --------------------------------------------------------------------------------