├── .gitattributes ├── .gitignore ├── README.md ├── img ├── correct-classification.png ├── incorrect-classification.png ├── incorrect-classsification.png ├── legend.png ├── lenet5-model.png └── logo.png ├── requirements.txt ├── setup.py └── src ├── demo_notebooks └── demo-cnnumpy-fast.ipynb ├── fast ├── data │ ├── plot.png │ ├── t10k-images-idx3-ubyte.gz │ ├── t10k-labels-idx1-ubyte.gz │ ├── train-images-idx3-ubyte.gz │ └── train-labels-idx1-ubyte.gz ├── layers.py ├── model.py ├── predict.py ├── save_weights │ ├── demo_weights.pkl │ └── final_weights.pkl ├── train.py ├── unit_test │ └── unit_test.py └── utils.py └── slow ├── data ├── t10k-images-idx3-ubyte.gz ├── t10k-labels-idx1-ubyte.gz ├── train-images-idx3-ubyte.gz └── train-labels-idx1-ubyte.gz ├── layers.py ├── model.py ├── predict.py ├── save_weights └── final_weights.pkl ├── train.py ├── unit_test └── unit_test.py └── utils.py /.gitattributes: -------------------------------------------------------------------------------- 1 | * linguist-vendored 2 | *.py linguist-vendored=false 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | CNNumpy.egg-info 2 | myenv/ 3 | __pycache__/ 4 | .ipynb_checkpoints/ 5 | .vscode/ 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Introduction 4 | 5 | - **CNNumpy** is a Convolutional Neural Network written in pure Numpy (educational purpose only). 6 | - There are 2 implementation versions: 7 | - Slow: The naive version with nested for loops. 8 | - Fast: The im2col/col2im version. 9 | - The [slow implementation][slow-implementation] takes around **4 hours for 1 epoch** where the [fast implementation][fast-implementation] takes only **6 min for 1 epoch**. 10 | - For your information, with the same architecture using **Pytorch**, it will take around **1 min for 1 epoch**. 11 | - **For more details, here are my blog posts explaining in depth what's going on under the hood for each implementation ([slow][slow-blog] and [fast][fast-blog]).** 12 | - In the [`demo-cnnumpy-fast.ipynb`][demo-notebook] notebook, the im2col/col2im implementation can achieve an accuracy up to **97.2% in 1 epoch (~6 min)**. Here are some results: 13 | 14 | 15 | 16 | 17 | 18 | ## Installation 19 | 20 | - Create a virtual environment in the root folder using [virtualenv][virtualenv] and activate it. 21 | 22 | ```bash 23 | # On Linux terminal, using virtualenv. 24 | virtualenv myenv 25 | # Activate it. 26 | source myenv/bin/activate 27 | ``` 28 | 29 | - Install **requirements.txt**. 30 | 31 | ```bash 32 | pip install -r requirements.txt 33 | # Tidy up the root folder. 34 | python3 setup.py clean 35 | ``` 36 | 37 | ## Usage of demo notebooks 38 | 39 | To play with the `demo-notebooks/` files, you need to make sure jupyter notebook can select your virtual environnment as a kernel. 40 | 41 | - Follow **"Installation"** instructions first and make sure your virtual environment is still activated. 42 | - Run the following line in the terminal. 43 | ```bash 44 | python -m ipykernel install --user--name=myenv 45 | ``` 46 | - Run the notebook file **only from `demo_notebooks/`** and then select **Kernel > Switch Kernel > myenv**. You are now ready to go ! 47 | 48 | 51 | [slow-implementation]: https://github.com/3outeille/CNNumpy/tree/master/src/slow 52 | [fast-implementation]: https://github.com/3outeille/CNNumpy/tree/master/src/fast 53 | [slow-blog]: https://hackmd.io/@machine-learning/blog-post-cnnumpy-slow 54 | [fast-blog]: https://hackmd.io/@machine-learning/blog-post-cnnumpy-fast 55 | [demo-notebook]: https://github.com/3outeille/CNNumpy/blob/master/src/demo_notebooks/demo-cnnumpy-fast.ipynb 56 | [virtualenv]: https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/ 57 | -------------------------------------------------------------------------------- /img/correct-classification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/img/correct-classification.png -------------------------------------------------------------------------------- /img/incorrect-classification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/img/incorrect-classification.png -------------------------------------------------------------------------------- /img/incorrect-classsification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/img/incorrect-classsification.png -------------------------------------------------------------------------------- /img/legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/img/legend.png -------------------------------------------------------------------------------- /img/lenet5-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/img/lenet5-model.png -------------------------------------------------------------------------------- /img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/img/logo.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # install CNNumpy as top-level editable package. 2 | -e . 3 | numpy==1.18.5 4 | torch==1.5.0 5 | scikit-learn==0.23.1 6 | scipy==1.4.1 7 | sklearn==0.0 8 | matplotlib==3.2.2 9 | scikit-image==0.17.2 10 | tqdm==4.47.0 11 | ipykernel==5.3.2 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from glob import glob 2 | from os.path import abspath, basename, dirname, join, normpath, relpath 3 | from shutil import rmtree 4 | from setuptools import setup, find_packages 5 | from setuptools import Command 6 | 7 | here = normpath(abspath(dirname(__file__))) 8 | class CleanCommand(Command): 9 | """ 10 | Custom clean command to tidy up the root folder. 11 | """ 12 | CLEAN_FILES = './build ./dist ./*.pyc ./*.tgz ./*.egg-info ./__pycache__'.split(' ') 13 | 14 | user_options = [] 15 | 16 | def initialize_options(self): 17 | pass 18 | def finalize_options(self): 19 | pass 20 | def run(self): 21 | global here 22 | 23 | for path_spec in self.CLEAN_FILES: 24 | # Make paths absolute and relative to this path 25 | abs_paths = glob(normpath(join(here, path_spec))) 26 | for path in [str(p) for p in abs_paths]: 27 | if not path.startswith(here): 28 | # Die if path in CLEAN_FILES is absolute + outside this directory 29 | raise ValueError("%s is not a path inside %s" % (path, here)) 30 | print('removing %s' % relpath(path)) 31 | rmtree(path) 32 | 33 | setup( 34 | cmdclass={'clean': CleanCommand}, 35 | name='CNNumpy', 36 | version='1.0', 37 | packages=find_packages() 38 | ) 39 | -------------------------------------------------------------------------------- /src/fast/data/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/data/plot.png -------------------------------------------------------------------------------- /src/fast/data/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/data/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /src/fast/data/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/data/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /src/fast/data/train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/data/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /src/fast/data/train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/data/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /src/fast/layers.py: -------------------------------------------------------------------------------- 1 | from src.fast.utils import get_indices, im2col, col2im 2 | import numpy as np 3 | 4 | class Conv(): 5 | 6 | def __init__(self, nb_filters, filter_size, nb_channels, stride=1, padding=0): 7 | self.n_F = nb_filters 8 | self.f = filter_size 9 | self.n_C = nb_channels 10 | self.s = stride 11 | self.p = padding 12 | 13 | # Xavier-Glorot initialization - used for sigmoid, tanh. 14 | self.W = {'val': np.random.randn(self.n_F, self.n_C, self.f, self.f) * np.sqrt(1. / (self.f)), 15 | 'grad': np.zeros((self.n_F, self.n_C, self.f, self.f))} 16 | self.b = {'val': np.random.randn(self.n_F) * np.sqrt(1. / self.n_F), 'grad': np.zeros((self.n_F))} 17 | 18 | self.cache = None 19 | 20 | def forward(self, X): 21 | """ 22 | Performs a forward convolution. 23 | 24 | Parameters: 25 | - X : Last conv layer of shape (m, n_C_prev, n_H_prev, n_W_prev). 26 | Returns: 27 | - out: previous layer convolved. 28 | """ 29 | m, n_C_prev, n_H_prev, n_W_prev = X.shape 30 | 31 | n_C = self.n_F 32 | n_H = int((n_H_prev + 2 * self.p - self.f)/ self.s) + 1 33 | n_W = int((n_W_prev + 2 * self.p - self.f)/ self.s) + 1 34 | 35 | X_col = im2col(X, self.f, self.f, self.s, self.p) 36 | w_col = self.W['val'].reshape((self.n_F, -1)) 37 | b_col = self.b['val'].reshape(-1, 1) 38 | # Perform matrix multiplication. 39 | out = w_col @ X_col + b_col 40 | # Reshape back matrix to image. 41 | out = np.array(np.hsplit(out, m)).reshape((m, n_C, n_H, n_W)) 42 | self.cache = X, X_col, w_col 43 | return out 44 | 45 | def backward(self, dout): 46 | """ 47 | Distributes error from previous layer to convolutional layer and 48 | compute error for the current convolutional layer. 49 | 50 | Parameters: 51 | - dout: error from previous layer. 52 | 53 | Returns: 54 | - dX: error of the current convolutional layer. 55 | - self.W['grad']: weights gradient. 56 | - self.b['grad']: bias gradient. 57 | """ 58 | X, X_col, w_col = self.cache 59 | m, _, _, _ = X.shape 60 | # Compute bias gradient. 61 | self.b['grad'] = np.sum(dout, axis=(0,2,3)) 62 | # Reshape dout properly. 63 | dout = dout.reshape(dout.shape[0] * dout.shape[1], dout.shape[2] * dout.shape[3]) 64 | dout = np.array(np.vsplit(dout, m)) 65 | dout = np.concatenate(dout, axis=-1) 66 | # Perform matrix multiplication between reshaped dout and w_col to get dX_col. 67 | dX_col = w_col.T @ dout 68 | # Perform matrix multiplication between reshaped dout and X_col to get dW_col. 69 | dw_col = dout @ X_col.T 70 | # Reshape back to image (col2im). 71 | dX = col2im(dX_col, X.shape, self.f, self.f, self.s, self.p) 72 | # Reshape dw_col into dw. 73 | self.W['grad'] = dw_col.reshape((dw_col.shape[0], self.n_C, self.f, self.f)) 74 | 75 | return dX, self.W['grad'], self.b['grad'] 76 | 77 | class AvgPool(): 78 | 79 | def __init__(self, filter_size, stride=1, padding=0): 80 | self.f = filter_size 81 | self.s = stride 82 | self.p = padding 83 | self.cache = None 84 | 85 | def forward(self, X): 86 | """ 87 | Apply average pooling. 88 | 89 | Parameters: 90 | - X: Output of activation function. 91 | 92 | Returns: 93 | - A_pool: X after average pooling layer. 94 | """ 95 | self.cache = X 96 | 97 | m, n_C_prev, n_H_prev, n_W_prev = X.shape 98 | n_C = n_C_prev 99 | n_H = int((n_H_prev + 2 * self.p - self.f)/ self.s) + 1 100 | n_W = int((n_W_prev + 2 * self.p - self.f)/ self.s) + 1 101 | 102 | X_col = im2col(X, self.f, self.f, self.s, self.p) 103 | X_col = X_col.reshape(n_C, X_col.shape[0]//n_C, -1) 104 | A_pool = np.mean(X_col, axis=1) 105 | # Reshape A_pool properly. 106 | A_pool = np.array(np.hsplit(A_pool, m)) 107 | A_pool = A_pool.reshape(m, n_C, n_H, n_W) 108 | 109 | return A_pool 110 | 111 | def backward(self, dout): 112 | """ 113 | Distributes error through pooling layer. 114 | 115 | Parameters: 116 | - dout: Previous layer with the error. 117 | 118 | Returns: 119 | - dX: Conv layer updated with error. 120 | """ 121 | X = self.cache 122 | m, n_C_prev, n_H_prev, n_W_prev = X.shape 123 | 124 | n_C = n_C_prev 125 | n_H = int((n_H_prev + 2 * self.p - self.f)/ self.s) + 1 126 | n_W = int((n_W_prev + 2 * self.p - self.f)/ self.s) + 1 127 | 128 | dout_flatten = dout.reshape(n_C, -1) / (self.f * self.f) 129 | dX_col = np.repeat(dout_flatten, self.f*self.f, axis=0) 130 | dX = col2im(dX_col, X.shape, self.f, self.f, self.s, self.p) 131 | # Reshape dX properly. 132 | dX = dX.reshape(m, -1) 133 | dX = np.array(np.hsplit(dX, n_C_prev)) 134 | dX = dX.reshape(m, n_C_prev, n_H_prev, n_W_prev) 135 | return dX 136 | 137 | class Fc(): 138 | 139 | def __init__(self, row, column): 140 | self.row = row 141 | self.col = column 142 | 143 | # Xavier-Glorot initialization - used for sigmoid, tanh. 144 | self.W = {'val': np.random.randn(self.row, self.col) * np.sqrt(1./self.col), 'grad': 0} 145 | self.b = {'val': np.random.randn(1, self.row) * np.sqrt(1./self.row), 'grad': 0} 146 | 147 | self.cache = None 148 | 149 | def forward(self, fc): 150 | """ 151 | Performs a forward propagation between 2 fully connected layers. 152 | 153 | Parameters: 154 | - fc: fully connected layer. 155 | 156 | Returns: 157 | - A_fc: new fully connected layer. 158 | """ 159 | self.cache = fc 160 | A_fc = np.dot(fc, self.W['val'].T) + self.b['val'] 161 | return A_fc 162 | 163 | def backward(self, deltaL): 164 | """ 165 | Returns the error of the current layer and compute gradients. 166 | 167 | Parameters: 168 | - deltaL: error at last layer. 169 | 170 | Returns: 171 | - new_deltaL: error at current layer. 172 | - self.W['grad']: weights gradient. 173 | - self.b['grad']: bias gradient. 174 | """ 175 | fc = self.cache 176 | m = fc.shape[0] 177 | 178 | #Compute gradient. 179 | self.W['grad'] = (1/m) * np.dot(deltaL.T, fc) 180 | self.b['grad'] = (1/m) * np.sum(deltaL, axis = 0) 181 | 182 | #Compute error. 183 | new_deltaL = np.dot(deltaL, self.W['val']) 184 | #We still need to multiply new_deltaL by the derivative of the activation 185 | #function which is done in TanH.backward(). 186 | return new_deltaL, self.W['grad'], self.b['grad'] 187 | 188 | class SGD(): 189 | 190 | def __init__(self, lr, params): 191 | self.lr = lr 192 | self.params = params 193 | 194 | def update_params(self, grads): 195 | for key in self.params: 196 | self.params[key] = self.params[key] - self.lr * grads['d' + key] 197 | return self.params 198 | 199 | class AdamGD(): 200 | 201 | def __init__(self, lr, beta1, beta2, epsilon, params): 202 | self.lr = lr 203 | self.beta1 = beta1 204 | self.beta2 = beta2 205 | self.epsilon = epsilon 206 | self.params = params 207 | 208 | self.momentum = {} 209 | self.rmsprop = {} 210 | 211 | for key in self.params: 212 | self.momentum['vd' + key] = np.zeros(self.params[key].shape) 213 | self.rmsprop['sd' + key] = np.zeros(self.params[key].shape) 214 | 215 | def update_params(self, grads): 216 | 217 | for key in self.params: 218 | # Momentum update. 219 | self.momentum['vd' + key] = (self.beta1 * self.momentum['vd' + key]) + (1 - self.beta1) * grads['d' + key] 220 | # RMSprop update. 221 | self.rmsprop['sd' + key] = (self.beta2 * self.rmsprop['sd' + key]) + (1 - self.beta2) * (grads['d' + key]**2) 222 | # Update parameters. 223 | self.params[key] = self.params[key] - (self.lr * self.momentum['vd' + key]) / (np.sqrt(self.rmsprop['sd' + key]) + self.epsilon) 224 | 225 | return self.params 226 | 227 | class TanH(): 228 | 229 | def __init__(self, alpha = 1.7159): 230 | self.alpha = alpha 231 | self.cache = None 232 | 233 | def forward(self, X): 234 | """ 235 | Apply tanh function to X. 236 | 237 | Parameters: 238 | - X: input tensor. 239 | """ 240 | self.cache = X 241 | return self.alpha * np.tanh(X) 242 | 243 | def backward(self, new_deltaL): 244 | """ 245 | Finishes computation of error by multiplying new_deltaL by the 246 | derivative of tanH. 247 | 248 | Parameters: 249 | - new_deltaL: error previously computed. 250 | """ 251 | X = self.cache 252 | return new_deltaL * (1 - np.tanh(X)**2) 253 | 254 | class Softmax(): 255 | 256 | def __init__(self): 257 | pass 258 | 259 | def forward(self, X): 260 | """ 261 | Compute softmax values for each sets of scores in X. 262 | 263 | Parameters: 264 | - X: input vector. 265 | """ 266 | e_x = np.exp(X - np.max(X)) 267 | return e_x / np.sum(e_x, axis=1)[:, np.newaxis] 268 | 269 | def backward(self, y_pred, y): 270 | return y_pred - y 271 | 272 | 273 | class CrossEntropyLoss(): 274 | 275 | def __init__(self): 276 | pass 277 | 278 | def get(self, y_pred, y): 279 | """ 280 | Return the negative log likelihood and the error at the last layer. 281 | 282 | Parameters: 283 | - y_pred: model predictions. 284 | - y: ground truth labels. 285 | """ 286 | loss = -np.sum(y * np.log(y_pred)) 287 | return loss -------------------------------------------------------------------------------- /src/fast/model.py: -------------------------------------------------------------------------------- 1 | from src.fast.layers import * 2 | from src.fast.utils import * 3 | import numpy as np 4 | 5 | class LeNet5(): 6 | 7 | def __init__(self): 8 | self.conv1 = Conv(nb_filters = 6, filter_size = 5, nb_channels = 1) 9 | self.tanh1 = TanH() 10 | self.pool1 = AvgPool(filter_size = 2, stride = 2) 11 | self.conv2 = Conv(nb_filters = 16, filter_size = 5, nb_channels = 6) 12 | self.tanh2 = TanH() 13 | self.pool2 = AvgPool(filter_size = 2, stride = 2) 14 | self.pool2_shape = None 15 | self.fc1 = Fc(row = 120, column = 5*5*16) 16 | self.tanh3 = TanH() 17 | self.fc2 = Fc(row = 84, column = 120) 18 | self.tanh4 = TanH() 19 | self.fc3 = Fc(row = 10 , column = 84) 20 | self.softmax = Softmax() 21 | 22 | self.layers = [self.conv1, self.conv2, self.fc1, self.fc2, self.fc3] 23 | 24 | 25 | def forward(self, X): 26 | conv1 = self.conv1.forward(X) #(6x28x28) 27 | act1 = self.tanh1.forward(conv1) 28 | pool1 = self.pool1.forward(act1) #(6x14x14) 29 | 30 | conv2 = self.conv2.forward(pool1) #(16x10x10) 31 | act2 = self.tanh2.forward(conv2) 32 | pool2 = self.pool2.forward(act2) #(16x5x5) 33 | 34 | self.pool2_shape = pool2.shape #Need it in backpropagation. 35 | pool2_flatten = pool2.reshape(self.pool2_shape[0], -1) #(1x400) 36 | 37 | fc1 = self.fc1.forward(pool2_flatten) #(1x120) 38 | act3 = self.tanh3.forward(fc1) 39 | 40 | fc2 = self.fc2.forward(act3) #(1x84) 41 | act4 = self.tanh4.forward(fc2) 42 | 43 | fc3 = self.fc3.forward(act4) #(1x10) 44 | 45 | y_pred = self.softmax.forward(fc3) 46 | 47 | return y_pred 48 | 49 | def backward(self, y_pred, y): 50 | deltaL = self.softmax.backward(y_pred, y) 51 | #Compute gradient for weight/bias between fc3 and fc2. 52 | deltaL, dW5, db5, = self.fc3.backward(deltaL) 53 | #Compute error at fc2 layer. 54 | deltaL = self.tanh4.backward(deltaL) #(1x84) 55 | 56 | #Compute gradient for weight/bias between fc2 and fc1. 57 | deltaL, dW4, db4 = self.fc2.backward(deltaL) 58 | #Compute error at fc1 layer. 59 | deltaL = self.tanh3.backward(deltaL) #(1x120) 60 | 61 | #Compute gradient for weight/bias between fc1 and pool2 and compute 62 | #error too (don't need to backpropagate through tanh here). 63 | deltaL, dW3, db3 = self.fc1.backward(deltaL) #(1x400) 64 | deltaL = deltaL.reshape(self.pool2_shape) #(16x5x5) 65 | 66 | #Distribute error through pool2 to conv2. 67 | deltaL = self.pool2.backward(deltaL) #(16x10x10) 68 | #Distribute error through tanh. 69 | deltaL = self.tanh2.backward(deltaL) 70 | 71 | #Compute gradient for weight/bias at conv2 layer and backpropagate 72 | #error to conv1 layer. 73 | deltaL, dW2, db2 = self.conv2.backward(deltaL) #(6x14x14) 74 | 75 | #Distribute error through pool1 by creating a temporary pooling layer 76 | #of conv1 shape. 77 | deltaL = self.pool1.backward(deltaL) #(6x28x28) 78 | #Distribute error through tanh. 79 | deltaL = self.tanh1.backward(deltaL) 80 | 81 | #Compute gradient for weight/bias at conv1 layer and backpropagate 82 | #error at conv1 layer. 83 | deltaL, dW1, db1 = self.conv1.backward(deltaL) #(1x32x32) 84 | 85 | grads = { 86 | 'dW1': dW1, 'db1': db1, 87 | 'dW2': dW2, 'db2': db2, 88 | 'dW3': dW3, 'db3': db3, 89 | 'dW4': dW4, 'db4': db4, 90 | 'dW5': dW5, 'db5': db5 91 | } 92 | 93 | return grads 94 | 95 | def get_params(self): 96 | params = {} 97 | for i, layer in enumerate(self.layers): 98 | params['W' + str(i+1)] = layer.W['val'] 99 | params['b' + str(i+1)] = layer.b['val'] 100 | 101 | return params 102 | 103 | def set_params(self, params): 104 | for i, layer in enumerate(self.layers): 105 | layer.W['val'] = params['W'+ str(i+1)] 106 | layer.b['val'] = params['b' + str(i+1)] 107 | -------------------------------------------------------------------------------- /src/fast/predict.py: -------------------------------------------------------------------------------- 1 | from src.fast.utils import * 2 | from src.fast.layers import * 3 | from src.fast.model import LeNet5 4 | import numpy as np 5 | import pickle 6 | import matplotlib.pyplot as plt 7 | from tqdm import trange 8 | 9 | filename = [ 10 | ["training_images","train-images-idx3-ubyte.gz"], 11 | ["test_images","t10k-images-idx3-ubyte.gz"], 12 | ["training_labels","train-labels-idx1-ubyte.gz"], 13 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 14 | ] 15 | 16 | def test(isNotebook=False): 17 | print("\n--------------------EXTRACTION-------------------\n") 18 | X, y, X_test, y_test = load(filename) 19 | X = (X - np.mean(X)) / np.std(X) 20 | X_test = (X_test - np.mean(X_test)) / np.std(X_test) 21 | 22 | print("\n------------------PREPROCESSING------------------\n") 23 | X_test = resize_dataset(X_test) 24 | print("Resize dataset: OK") 25 | y_test = one_hot_encoding(y_test) 26 | print("One-Hot-Encoding: OK") 27 | 28 | print("\n--------------LOAD PRETRAINED MODEL--------------\n") 29 | cost = CrossEntropyLoss() 30 | model = LeNet5() 31 | model = load_params_from_file(model, isNotebook=isNotebook) 32 | print("Load pretrained model: OK\n") 33 | 34 | print("--------------------EVALUATION-------------------\n") 35 | 36 | BATCH_SIZE = 100 37 | 38 | nb_test_examples = len(X_test) 39 | test_loss = 0 40 | test_acc = 0 41 | 42 | pbar = trange(nb_test_examples // BATCH_SIZE) 43 | test_loader = dataloader(X_test, y_test, BATCH_SIZE) 44 | 45 | for i, (X_batch, y_batch) in zip(pbar, test_loader): 46 | 47 | y_pred = model.forward(X_batch) 48 | loss = cost.get(y_pred, y_batch) 49 | 50 | test_loss += loss * BATCH_SIZE 51 | test_acc += sum((np.argmax(y_batch, axis=1) == np.argmax(y_pred, axis=1))) 52 | 53 | pbar.set_description("Evaluation") 54 | 55 | test_loss /= nb_test_examples 56 | test_acc /= nb_test_examples 57 | 58 | info_test = "test-loss: {:0.6f} | test-acc: {:0.3f}" 59 | print(info_test.format(test_loss, test_acc)) 60 | 61 | # Display correct examples. 62 | images, labels = X_test[:1000, ...], y_test[:1000, ...] 63 | y_pred = model.forward(images) 64 | predicted, labels= np.argmax(y_pred, axis=1), np.argmax(labels, axis=1) 65 | print('\nSome correct classification:') 66 | plot_example(images, labels, predicted) 67 | 68 | # Display wrong examples. 69 | print('\nSome incorrect classification:') 70 | plot_example_errors(images, labels, predicted) 71 | 72 | # Uncomment to launch testing. 73 | # test() -------------------------------------------------------------------------------- /src/fast/save_weights/demo_weights.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/save_weights/demo_weights.pkl -------------------------------------------------------------------------------- /src/fast/save_weights/final_weights.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/fast/save_weights/final_weights.pkl -------------------------------------------------------------------------------- /src/fast/train.py: -------------------------------------------------------------------------------- 1 | from src.fast.utils import * 2 | from src.fast.layers import * 3 | from src.fast.model import LeNet5 4 | import numpy as np 5 | import pickle 6 | import matplotlib.pyplot as plt 7 | from tqdm import trange 8 | from timeit import default_timer as timer 9 | 10 | filename = [ 11 | ["training_images","train-images-idx3-ubyte.gz"], 12 | ["test_images","t10k-images-idx3-ubyte.gz"], 13 | ["training_labels","train-labels-idx1-ubyte.gz"], 14 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 15 | ] 16 | 17 | def train(): 18 | print("\n----------------EXTRACTION---------------\n") 19 | X, y, X_test, y_test = load(filename) 20 | 21 | print("\n--------------PREPROCESSING--------------\n") 22 | X = resize_dataset(X) 23 | print("Resize dataset: OK") 24 | X = (X - np.mean(X)) / np.std(X) 25 | print("Normalize dataset: OK") 26 | y = one_hot_encoding(y) 27 | print("One-Hot-Encoding: OK") 28 | X_train, y_train, X_val, y_val = train_val_split(X, y) 29 | print("Train and Validation set split: OK\n") 30 | 31 | model = LeNet5() 32 | cost = CrossEntropyLoss() 33 | lr = 0.001 34 | 35 | optimizer = AdamGD(lr = lr, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, params = model.get_params()) 36 | train_costs, val_costs = [], [] 37 | 38 | print("----------------TRAINING-----------------\n") 39 | 40 | NB_EPOCH = 1 41 | BATCH_SIZE = 100 42 | 43 | print("EPOCHS: {}".format(NB_EPOCH)) 44 | print("BATCH_SIZE: {}".format(BATCH_SIZE)) 45 | print("LR: {}".format(lr)) 46 | print() 47 | 48 | nb_train_examples = len(X_train) 49 | nb_val_examples = len(X_val) 50 | 51 | best_val_loss = float('inf') 52 | 53 | 54 | for epoch in range(NB_EPOCH): 55 | 56 | #------------------------------------------------------------------------------- 57 | # 58 | # TRAINING PART 59 | # 60 | #------------------------------------------------------------------------------- 61 | 62 | train_loss = 0 63 | train_acc = 0 64 | 65 | pbar = trange(nb_train_examples // BATCH_SIZE) 66 | train_loader = dataloader(X_train, y_train, BATCH_SIZE) 67 | 68 | start = timer() 69 | 70 | for i, (X_batch, y_batch) in zip(pbar, train_loader): 71 | 72 | y_pred = model.forward(X_batch) 73 | loss = cost.get(y_pred, y_batch) 74 | 75 | grads = model.backward(y_pred, y_batch) 76 | params = optimizer.update_params(grads) 77 | model.set_params(params) 78 | 79 | train_loss += loss * BATCH_SIZE 80 | train_acc += sum((np.argmax(y_batch, axis=1) == np.argmax(y_pred, axis=1))) 81 | 82 | pbar.set_description("[Train] Epoch {}".format(epoch+1)) 83 | 84 | end = timer() 85 | 86 | train_loss /= nb_train_examples 87 | train_costs.append(train_loss) 88 | train_acc /= nb_train_examples 89 | 90 | info_train = "train-loss: {:0.6f} | train-acc: {:0.3f}" 91 | print(info_train.format(train_loss, train_acc)) 92 | print() 93 | print(f"Elapsed time for epoch {epoch+1}: {(end-start)/60} min.", end="\n") 94 | 95 | #------------------------------------------------------------------------------- 96 | # 97 | # VALIDATION PART 98 | # 99 | #------------------------------------------------------------------------------- 100 | val_loss = 0 101 | val_acc = 0 102 | 103 | pbar = trange(nb_val_examples // BATCH_SIZE) 104 | val_loader = dataloader(X_val, y_val, BATCH_SIZE) 105 | 106 | for i, (X_batch, y_batch) in zip(pbar, val_loader): 107 | 108 | y_pred = model.forward(X_batch) 109 | loss = cost.get(y_pred, y_batch) 110 | 111 | grads = model.backward(y_pred, y_batch) 112 | params = optimizer.update_params(grads) 113 | model.set_params(params) 114 | 115 | val_loss += loss * BATCH_SIZE 116 | val_acc += sum((np.argmax(y_batch, axis=1) == np.argmax(y_pred, axis=1))) 117 | 118 | pbar.set_description("[Val] Epoch {}".format(epoch+1)) 119 | 120 | val_loss /= nb_val_examples 121 | val_costs.append(val_loss) 122 | val_acc /= nb_val_examples 123 | 124 | info_val = "val-loss: {:0.6f} | val-acc: {:0.3f}" 125 | print(info_val.format(val_loss, val_acc)) 126 | 127 | if best_val_loss > val_loss: 128 | print("Validation loss decreased from {:0.6f} to {:0.6f}. Model saved".format(best_val_loss, val_loss)) 129 | save_params_to_file(model) 130 | best_val_loss = val_loss 131 | 132 | print() 133 | 134 | pbar.close() 135 | 136 | # fig = plt.figure(figsize=(10,10)) 137 | # fig.add_subplot(2, 1, 1) 138 | 139 | # plt.plot(train_costs) 140 | # plt.title("Training loss") 141 | # plt.xlabel('Epochs') 142 | # plt.ylabel('Loss') 143 | 144 | # fig.add_subplot(2, 1, 2) 145 | # plt.plot(val_costs) 146 | # plt.title("Validation loss") 147 | # plt.xlabel('Epochs') 148 | # plt.ylabel('Loss') 149 | 150 | # plt.show() 151 | 152 | # Uncomment to launch training. 153 | # train() 154 | -------------------------------------------------------------------------------- /src/fast/unit_test/unit_test.py: -------------------------------------------------------------------------------- 1 | from src.fast.layers import Conv, AvgPool 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | np.random.seed(42) 8 | torch.set_printoptions(precision=8) 9 | 10 | test_registry = {} 11 | def register_test(func): 12 | test_registry[func.__name__] = func 13 | return func 14 | 15 | @register_test 16 | def test_conv(): 17 | n, c, h, w = 20, 10, 5, 5 # Image. 18 | nf, cf, hf, wf = 2, 10, 3, 3 # Filters. 19 | x = np.random.randn(n, c, h, w) 20 | x = x.astype('float64') 21 | W = np.random.randn(nf, cf, hf, wf) 22 | W = W.astype('float64') 23 | b = np.random.randn(nf) 24 | b = b.astype('float64') 25 | deltaL = np.random.rand(n, nf, h-hf+1, h-hf+1) # Assuming stride=1/padding=0. 26 | 27 | # CNNumpy. 28 | inputs_cnn = x 29 | weights_cnn, biases_cnn = W, b 30 | conv_cnn = Conv(nb_filters=nf, filter_size=hf, nb_channels=cf, stride=1, padding=0) 31 | conv_cnn.W['val'], conv_cnn.W['grad'] = weights_cnn, np.zeros_like(weights_cnn) 32 | conv_cnn.b['val'], conv_cnn.b['grad'] = biases_cnn, np.zeros_like(biases_cnn) 33 | 34 | out_cnn = conv_cnn.forward(inputs_cnn) # Forward. 35 | _, conv_cnn.W['grad'], conv_cnn.b['grad'] = conv_cnn.backward(deltaL) # Backward. 36 | 37 | # Pytorch 38 | inputs_pt = torch.Tensor(x).double() 39 | conv_pt = nn.Conv2d(c, nf, hf, stride=1, padding=0, bias=True) 40 | conv_pt.weight = nn.Parameter(torch.DoubleTensor(W)) 41 | conv_pt.bias = nn.Parameter(torch.DoubleTensor(b)) 42 | out_pt = conv_pt(inputs_pt) # Forward. 43 | out_pt.backward(torch.Tensor(deltaL)) # Backward. 44 | 45 | # Check if inputs are equals. 46 | assert np.allclose(inputs_cnn, inputs_pt.numpy(), atol=1e-8) # 1e-8 tolerance. 47 | # Check if weights are equals. 48 | assert np.allclose(weights_cnn, conv_pt.weight.data.numpy(), atol=1e-8) # 1e-8 tolerance. 49 | # Check if biases are equals. 50 | assert np.allclose(biases_cnn, conv_pt.bias.data.numpy(), atol=1e-8) # 1e-8 tolerance. 51 | # Check if conv forward outputs are equals. 52 | assert np.allclose(out_cnn, out_pt.data.numpy(), atol=1e-5) # 1e-5 tolerance. 53 | # Check if conv backward outputs are equals. 54 | assert np.allclose(conv_cnn.W['grad'], conv_pt.weight.grad.numpy(), atol=1e-5) # 1e-5 tolerance. 55 | assert np.allclose(conv_cnn.b['grad'], conv_pt.bias.grad.numpy(), atol=1e-5) # 1e-5 tolerance. 56 | 57 | @register_test 58 | def test_avgpool(): 59 | n, c, h, w = 10, 4, 5, 5 # Image. 60 | kernel_size, stride = 2, 1 61 | x = np.random.randn(n, c, h, w) 62 | x = x.astype('float64') 63 | H = int((h - kernel_size)/ stride) + 1 64 | W = int((w - kernel_size)/ stride) + 1 65 | deltaL = np.random.rand(n, c, H, W) 66 | 67 | # CNNumpy. 68 | inputs_cnn = x 69 | avg_cnn = AvgPool(kernel_size, stride=stride) 70 | out_cnn = avg_cnn.forward(inputs_cnn) # Forward. 71 | inputs_cnn_grad = avg_cnn.backward(deltaL) # Backward. 72 | 73 | # Pytorch. 74 | inputs_pt = torch.Tensor(x).double() 75 | inputs_pt.requires_grad = True 76 | avg_pt = nn.AvgPool2d(kernel_size, stride = stride) 77 | out_pt = avg_pt(inputs_pt) # Forward. 78 | out_pt.backward(torch.Tensor(deltaL)) # Backward. 79 | 80 | # Check if inputs are equals. 81 | assert np.allclose(inputs_cnn, inputs_pt.data.numpy(), atol=1e-8) # 1e-8 tolerance. 82 | # Check if conv forward outputs are equals. 83 | assert np.allclose(out_cnn, out_pt.data.numpy(), atol=1e-8) # 1e-8 tolerance. 84 | # Check if conv backward outputs are equals. 85 | assert np.allclose(inputs_cnn_grad, inputs_pt.grad.numpy(), atol=1e-7) # 1e-7 tolerance. 86 | 87 | for name, test in test_registry.items(): 88 | print(f'Running {name}:', end=" ") 89 | test() 90 | print("OK") 91 | -------------------------------------------------------------------------------- /src/fast/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import gzip 3 | import math 4 | import pickle 5 | import numpy as np 6 | import urllib.request 7 | from PIL import Image 8 | from src.fast.data import * 9 | from skimage import transform 10 | import matplotlib.pyplot as plt 11 | import concurrent.futures as cf 12 | 13 | def download_mnist(filename): 14 | """ 15 | Downloads dataset from filename. 16 | 17 | Parameters: 18 | - filename: [ 19 | ["training_images","train-images-idx3-ubyte.gz"], 20 | ["test_images","t10k-images-idx3-ubyte.gz"], 21 | ["training_labels","train-labels-idx1-ubyte.gz"], 22 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 23 | ] 24 | """ 25 | # Make data/ accessible from every folders. 26 | terminal_path = ['src/fast/data/', 'fast/data/', '../fast/data/', 'data/', '../data'] 27 | dirPath = None 28 | for path in terminal_path: 29 | if os.path.isdir(path): 30 | dirPath = path 31 | if dirPath == None: 32 | raise FileNotFoundError("extract_mnist(): Impossible to find data/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 33 | 34 | base_url = "http://yann.lecun.com/exdb/mnist/" 35 | for elt in filename: 36 | print("Downloading " + elt[1] + " in data/ ...") 37 | urllib.request.urlretrieve(base_url + elt[1], dirPath + elt[1]) 38 | print("Download complete.") 39 | 40 | def extract_mnist(filename): 41 | """ 42 | Extracts dataset from filename. 43 | 44 | Parameters: 45 | - filename: [ 46 | ["training_images","train-images-idx3-ubyte.gz"], 47 | ["test_images","t10k-images-idx3-ubyte.gz"], 48 | ["training_labels","train-labels-idx1-ubyte.gz"], 49 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 50 | ] 51 | """ 52 | # Make data/ accessible from every folders. 53 | terminal_path = ['src/fast/data/', 'fast/data/', '../fast/data/', 'data/', '../data'] 54 | dirPath = None 55 | for path in terminal_path: 56 | if os.path.isdir(path): 57 | dirPath = path 58 | if dirPath == None: 59 | raise FileNotFoundError("extract_mnist(): Impossible to find data/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 60 | 61 | mnist = {} 62 | for elt in filename[:2]: 63 | print('Extracting data/' + elt[0] + '...') 64 | with gzip.open(dirPath + elt[1]) as f: 65 | #According to the doc on MNIST website, offset for image starts at 16. 66 | mnist[elt[0]] = np.frombuffer(f.read(), dtype=np.uint8, offset=16).reshape(-1, 1, 28, 28) 67 | 68 | for elt in filename[2:]: 69 | print('Extracting data/' + elt[0] + '...') 70 | with gzip.open(dirPath + elt[1]) as f: 71 | #According to the doc on MNIST website, offset for label starts at 8. 72 | mnist[elt[0]] = np.frombuffer(f.read(), dtype=np.uint8, offset=8) 73 | 74 | print('Files extraction: OK') 75 | 76 | return mnist 77 | 78 | def load(filename): 79 | """ 80 | Loads dataset to variables. 81 | 82 | Parameters: 83 | - filename: [ 84 | ["training_images","train-images-idx3-ubyte.gz"], 85 | ["test_images","t10k-images-idx3-ubyte.gz"], 86 | ["training_labels","train-labels-idx1-ubyte.gz"], 87 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 88 | ] 89 | """ 90 | # Make data/ accessible from every folders. 91 | terminal_path = ['src/fast/data/', 'fast/data/', '../fast/data/', 'data/', '../data'] 92 | dirPath = None 93 | for path in terminal_path: 94 | if os.path.isdir(path): 95 | dirPath = path 96 | if dirPath == None: 97 | raise FileNotFoundError("extract_mnist(): Impossible to find data/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 98 | 99 | L = [elt[1] for elt in filename] 100 | count = 0 101 | 102 | #Check if the 4 .gz files exist. 103 | for elt in L: 104 | if os.path.isfile(dirPath + elt): 105 | count += 1 106 | 107 | #If the 4 .gz are not in data/, we download and extract them. 108 | if count != 4: 109 | download_mnist(filename) 110 | mnist = extract_mnist(filename) 111 | else: #We just extract them. 112 | mnist = extract_mnist(filename) 113 | 114 | print('Loading dataset: OK') 115 | return mnist["training_images"], mnist["training_labels"], mnist["test_images"], mnist["test_labels"] 116 | 117 | def resize_dataset(dataset): 118 | """ 119 | Resizes dataset of MNIST images to (32, 32). 120 | 121 | Parameters: 122 | -dataset: a numpy array of size [?, 1, 28, 28]. 123 | """ 124 | args = [dataset[i:i+1000] for i in range(0, len(dataset), 1000)] 125 | 126 | def f(chunk): 127 | return transform.resize(chunk, (chunk.shape[0], 1, 32, 32)) 128 | 129 | with cf.ThreadPoolExecutor() as executor: 130 | res = executor.map(f, args) 131 | 132 | res = np.array([*res]) 133 | res = res.reshape(-1, 1, 32, 32) 134 | return res 135 | 136 | 137 | def dataloader(X, y, BATCH_SIZE): 138 | """ 139 | Returns a data generator. 140 | 141 | Parameters: 142 | - X: dataset examples. 143 | - y: ground truth labels. 144 | """ 145 | n = len(X) 146 | for t in range(0, n, BATCH_SIZE): 147 | yield X[t:t+BATCH_SIZE, ...], y[t:t+BATCH_SIZE, ...] 148 | 149 | def one_hot_encoding(y): 150 | """ 151 | Performs one-hot-encoding on y. 152 | 153 | Parameters: 154 | - y: ground truth labels. 155 | """ 156 | N = y.shape[0] 157 | Z = np.zeros((N, 10)) 158 | Z[np.arange(N), y] = 1 159 | return Z 160 | 161 | def train_val_split(X, y, val=50000): 162 | """ 163 | Splits X and y into training and validation set. 164 | 165 | Parameters: 166 | - X: dataset examples. 167 | - y: ground truth labels. 168 | """ 169 | X_train, X_val = X[:val, :], X[val:, :] 170 | y_train, y_val = y[:val, :], y[val:, :] 171 | 172 | return X_train, y_train, X_val, y_val 173 | 174 | def save_params_to_file(model): 175 | """ 176 | Saves model parameters to a file. 177 | 178 | Parameters: 179 | -model: a CNN architecture. 180 | """ 181 | # Make save_weights/ accessible from every folders. 182 | terminal_path = ["src/fast/save_weights/", "fast/save_weights/", '../fast/save_weights/', "save_weights/", "../save_weights/"] 183 | dirPath = None 184 | for path in terminal_path: 185 | if os.path.isdir(path): 186 | dirPath = path 187 | if dirPath == None: 188 | raise FileNotFoundError("save_params_to_file(): Impossible to find save_weights/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 189 | 190 | weights = model.get_params() 191 | if dirPath == '../fast/save_weights/': # We run the code from demo notebook. 192 | with open(dirPath + "demo_weights.pkl","wb") as f: 193 | pickle.dump(weights, f) 194 | else: 195 | with open(dirPath + "final_weights.pkl","wb") as f: 196 | pickle.dump(weights, f) 197 | 198 | def load_params_from_file(model, isNotebook=False): 199 | """ 200 | Loads model parameters from a file. 201 | 202 | Parameters: 203 | -model: a CNN architecture. 204 | """ 205 | if isNotebook: # We run from demo-notebooks/ 206 | pickle_in = open("../fast/save_weights/demo_weights.pkl", 'rb') 207 | params = pickle.load(pickle_in) 208 | model.set_params(params) 209 | else: 210 | # Make final_weights.pkl file accessible from every folders. 211 | terminal_path = ["src/fast/save_weights/final_weights.pkl", "fast/save_weights/final_weights.pkl", 212 | "save_weights/final_weights.pkl", "../save_weights/final_weights.pkl"] 213 | 214 | filePath = None 215 | for path in terminal_path: 216 | if os.path.isfile(path): 217 | filePath = path 218 | if filePath == None: 219 | raise FileNotFoundError('load_params_from_file(): Cannot find final_weights.pkl from your current folder. You need to manually add it to terminal_path list and the run the function again.') 220 | 221 | pickle_in = open(filePath, 'rb') 222 | params = pickle.load(pickle_in) 223 | model.set_params(params) 224 | return model 225 | 226 | def prettyPrint3D(M): 227 | """ 228 | Displays a 3D matrix in a pretty way. 229 | 230 | Parameters: 231 | -M: Matrix of shape (m, n_H, n_W, n_C) with m, the number 3D matrices. 232 | """ 233 | m, n_C, n_H, n_W = M.shape 234 | 235 | for i in range(m): 236 | 237 | for c in range(n_C): 238 | print('Image {}, channel {}'.format(i + 1, c + 1), end='\n\n') 239 | 240 | for h in range(n_H): 241 | print("/", end="") 242 | 243 | for j in range(n_W): 244 | 245 | print(M[i, c, h, j], end = ",") 246 | 247 | print("/", end='\n\n') 248 | 249 | print('-------------------', end='\n\n') 250 | 251 | def plot_example(X, y, y_pred=None): 252 | """ 253 | Plots 9 examples and their associate labels. 254 | 255 | Parameters: 256 | -X: Training examples. 257 | -y: true labels. 258 | -y_pred: predicted labels. 259 | """ 260 | # Create figure with 3 x 3 sub-plots. 261 | fig, axes = plt.subplots(3, 3) 262 | fig.subplots_adjust(hspace=0.3, wspace=0.3) 263 | 264 | X, y = X[:9, 0, ...], y[:9] 265 | 266 | for i, ax in enumerate(axes.flat): 267 | # Plot image. 268 | ax.imshow(X[i]) 269 | 270 | # Show true and predicted classes. 271 | if y_pred is None: 272 | xlabel = "True: {0}".format(y[i]) 273 | else: 274 | xlabel = "True: {0}, Pred: {1}".format(y[i], y_pred[i]) 275 | 276 | # Show the classes as the label on the x-axis. 277 | ax.set_xlabel(xlabel) 278 | 279 | # Remove ticks from the plot. 280 | ax.set_xticks([]) 281 | ax.set_yticks([]) 282 | 283 | # Ensure the plot is shown correctly with multiple plots in a single Notebook cell. 284 | plt.show() 285 | 286 | def plot_example_errors(X, y, y_pred): 287 | """ 288 | Plots 9 example errors and their associate true/predicted labels. 289 | 290 | Parameters: 291 | -X: Training examples. 292 | -y: true labels. 293 | -y_pred: predicted labels. 294 | 295 | """ 296 | incorrect = (y != y_pred) 297 | 298 | X = X[incorrect] 299 | y = y[incorrect] 300 | y_pred = y_pred[incorrect] 301 | 302 | # Plot the first 9 images. 303 | plot_example(X, y, y_pred) 304 | 305 | def get_indices(X_shape, HF, WF, stride, pad): 306 | """ 307 | Returns index matrices in order to transform our input image into a matrix. 308 | 309 | Parameters: 310 | -X_shape: Input image shape. 311 | -HF: filter height. 312 | -WF: filter width. 313 | -stride: stride value. 314 | -pad: padding value. 315 | 316 | Returns: 317 | -i: matrix of index i. 318 | -j: matrix of index j. 319 | -d: matrix of index d. 320 | (Use to mark delimitation for each channel 321 | during multi-dimensional arrays indexing). 322 | """ 323 | # get input size 324 | m, n_C, n_H, n_W = X_shape 325 | 326 | # get output size 327 | out_h = int((n_H + 2 * pad - HF) / stride) + 1 328 | out_w = int((n_W + 2 * pad - WF) / stride) + 1 329 | 330 | # ----Compute matrix of index i---- 331 | 332 | # Level 1 vector. 333 | level1 = np.repeat(np.arange(HF), WF) 334 | # Duplicate for the other channels. 335 | level1 = np.tile(level1, n_C) 336 | # Create a vector with an increase by 1 at each level. 337 | everyLevels = stride * np.repeat(np.arange(out_h), out_w) 338 | # Create matrix of index i at every levels for each channel. 339 | i = level1.reshape(-1, 1) + everyLevels.reshape(1, -1) 340 | 341 | # ----Compute matrix of index j---- 342 | 343 | # Slide 1 vector. 344 | slide1 = np.tile(np.arange(WF), HF) 345 | # Duplicate for the other channels. 346 | slide1 = np.tile(slide1, n_C) 347 | # Create a vector with an increase by 1 at each slide. 348 | everySlides = stride * np.tile(np.arange(out_w), out_h) 349 | # Create matrix of index j at every slides for each channel. 350 | j = slide1.reshape(-1, 1) + everySlides.reshape(1, -1) 351 | 352 | # ----Compute matrix of index d---- 353 | 354 | # This is to mark delimitation for each channel 355 | # during multi-dimensional arrays indexing. 356 | d = np.repeat(np.arange(n_C), HF * WF).reshape(-1, 1) 357 | 358 | return i, j, d 359 | 360 | def im2col(X, HF, WF, stride, pad): 361 | """ 362 | Transforms our input image into a matrix. 363 | 364 | Parameters: 365 | - X: input image. 366 | - HF: filter height. 367 | - WF: filter width. 368 | - stride: stride value. 369 | - pad: padding value. 370 | 371 | Returns: 372 | -cols: output matrix. 373 | """ 374 | # Padding 375 | X_padded = np.pad(X, ((0,0), (0,0), (pad, pad), (pad, pad)), mode='constant') 376 | i, j, d = get_indices(X.shape, HF, WF, stride, pad) 377 | # Multi-dimensional arrays indexing. 378 | cols = X_padded[:, d, i, j] 379 | cols = np.concatenate(cols, axis=-1) 380 | return cols 381 | 382 | def col2im(dX_col, X_shape, HF, WF, stride, pad): 383 | """ 384 | Transform our matrix back to the input image. 385 | 386 | Parameters: 387 | - dX_col: matrix with error. 388 | - X_shape: input image shape. 389 | - HF: filter height. 390 | - WF: filter width. 391 | - stride: stride value. 392 | - pad: padding value. 393 | 394 | Returns: 395 | -x_padded: input image with error. 396 | """ 397 | # Get input size 398 | N, D, H, W = X_shape 399 | # Add padding if needed. 400 | H_padded, W_padded = H + 2 * pad, W + 2 * pad 401 | X_padded = np.zeros((N, D, H_padded, W_padded)) 402 | 403 | # Index matrices, necessary to transform our input image into a matrix. 404 | i, j, d = get_indices(X_shape, HF, WF, stride, pad) 405 | # Retrieve batch dimension by spliting dX_col N times: (X, Y) => (N, X, Y) 406 | dX_col_reshaped = np.array(np.hsplit(dX_col, N)) 407 | # Reshape our matrix back to image. 408 | # slice(None) is used to produce the [::] effect which means "for every elements". 409 | np.add.at(X_padded, (slice(None), d, i, j), dX_col_reshaped) 410 | # Remove padding from new image if needed. 411 | if pad == 0: 412 | return X_padded 413 | elif type(pad) is int: 414 | return X_padded[pad:-pad, pad:-pad, :, :] 415 | -------------------------------------------------------------------------------- /src/slow/data/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/slow/data/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /src/slow/data/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/slow/data/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /src/slow/data/train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/slow/data/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /src/slow/data/train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/slow/data/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /src/slow/layers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | 4 | class Conv(): 5 | """ 6 | Convolutional layer. 7 | """ 8 | 9 | def __init__(self, nb_filters, filter_size, nb_channels, stride=1, padding=0): 10 | self.n_F = nb_filters 11 | self.f = filter_size 12 | self.n_C = nb_channels 13 | self.s = stride 14 | self.p = padding 15 | 16 | 17 | #Xavier initialization. 18 | bound = 1 / math.sqrt(self.f * self.f) 19 | self.W = {'val': np.random.uniform(-bound, bound, size=(self.n_F, self.n_C, self.f, self.f)), 20 | 'grad': np.zeros((self.n_F, self.n_C, self.f, self.f))} 21 | 22 | self.b = {'val': np.random.uniform(-bound, bound, size=(self.n_F)), 23 | 'grad': np.zeros((self.n_F))} 24 | 25 | self.cache = None 26 | 27 | def forward(self, X): 28 | """ 29 | Performs a forward convolution. 30 | 31 | Parameters: 32 | - X : Last conv layer of shape (m, n_C_prev, n_H_prev, n_W_prev). 33 | Returns: 34 | - out: output of convolution. 35 | """ 36 | self.cache = X 37 | m, n_C_prev, n_H_prev, n_W_prev = X.shape 38 | 39 | n_C = self.n_F 40 | n_H = int((n_H_prev + 2 * self.p - self.f)/ self.s) + 1 41 | n_W = int((n_W_prev + 2 * self.p - self.f)/ self.s) + 1 42 | 43 | out = np.zeros((m, n_C, n_H, n_W)) 44 | 45 | for i in range(m): #For each image. 46 | 47 | for c in range(n_C): #For each channel. 48 | 49 | for h in range(n_H): #Slide the filter vertically. 50 | h_start = h * self.s 51 | h_end = h_start + self.f 52 | 53 | for w in range(n_W): #Slide the filter horizontally. 54 | w_start = w * self.s 55 | w_end = w_start + self.f 56 | 57 | out[i, c, h, w] = np.sum(X[i, :, h_start:h_end, w_start:w_end] 58 | * self.W['val'][c, ...]) + self.b['val'][c] 59 | return out 60 | 61 | def backward(self, dout): 62 | """ 63 | Distributes error from previous layer to convolutional layer and 64 | compute error for the current convolutional layer. 65 | 66 | Parameters: 67 | - dout: error from previous layer. 68 | 69 | Returns: 70 | - dX: error of the current convolutional layer. 71 | - self.W['grad']: weights gradient. 72 | - self.b['grad']: bias gradient. 73 | """ 74 | X = self.cache 75 | 76 | m, n_C, n_H, n_W = X.shape 77 | m, n_C_dout, n_H_dout, n_W_dout = dout.shape 78 | 79 | dX = np.zeros(X.shape) 80 | 81 | #Compute dW. 82 | for i in range(m): #For each examples. 83 | 84 | for c in range(n_C_dout): 85 | 86 | for h in range(n_H_dout): 87 | h_start = h * self.s 88 | h_end = h_start + self.f 89 | 90 | for w in range(n_W_dout): 91 | w_start = w * self.s 92 | w_end = w_start + self.f 93 | 94 | self.W['grad'][c, ...] += dout[i, c, h, w] * X[i, :, h_start:h_end, w_start:w_end] 95 | dX[i, :, h_start:h_end, w_start:w_end] += dout[i, c, h, w] * self.W['val'][c, ...] 96 | #Compute db. 97 | for c in range(self.n_F): 98 | self.b['grad'][c, ...] = np.sum(dout[:, c, ...]) 99 | 100 | return dX, self.W['grad'], self.b['grad'] 101 | 102 | class AvgPool(): 103 | 104 | def __init__(self, filter_size, stride): 105 | self.f = filter_size 106 | self.s = stride 107 | self.cache = None 108 | 109 | def forward(self, X): 110 | """ 111 | Apply average pooling. 112 | 113 | Parameters: 114 | - X: Output of activation function. 115 | 116 | Returns: 117 | - A_pool: X after average pooling layer. 118 | """ 119 | m, n_C_prev, n_H_prev, n_W_prev = X.shape 120 | 121 | n_C = n_C_prev 122 | n_H = int((n_H_prev - self.f)/ self.s) + 1 123 | n_W = int((n_W_prev - self.f)/ self.s) + 1 124 | 125 | A_pool = np.zeros((m, n_C, n_H, n_W)) 126 | 127 | for i in range(m): 128 | 129 | for c in range(n_C): 130 | 131 | for h in range(n_H): 132 | h_start = h * self.s 133 | h_end = h_start + self.f 134 | 135 | for w in range(n_W): 136 | w_start = w * self.s 137 | w_end = w_start + self.f 138 | 139 | A_pool[i, c, h, w] = np.mean(X[i, c, h_start:h_end, w_start:w_end]) 140 | 141 | self.cache = X 142 | 143 | return A_pool 144 | 145 | def backward(self, dout): 146 | """ 147 | Distributes error through pooling layer. 148 | 149 | Parameters: 150 | - dout: Previous layer with the error. 151 | 152 | Returns: 153 | - dX: Conv layer updated with error. 154 | """ 155 | X = self.cache 156 | m, n_C, n_H, n_W = dout.shape 157 | dX = np.zeros(X.shape) 158 | 159 | for i in range(m): 160 | 161 | for c in range(n_C): 162 | 163 | for h in range(n_H): 164 | h_start = h * self.s 165 | h_end = h_start + self.f 166 | 167 | for w in range(n_W): 168 | w_start = w * self.s 169 | w_end = w_start + self.f 170 | 171 | average = dout[i, c, h, w] / (self.f * self.f) 172 | filter_average = np.full((self.f, self.f), average) 173 | dX[i, c, h_start:h_end, w_start:w_end] += filter_average 174 | 175 | return dX 176 | 177 | class Fc(): 178 | 179 | def __init__(self, row, column): 180 | self.row = row 181 | self.col = column 182 | 183 | #Initialize Weight/bias. 184 | bound = 1 / np.sqrt(self.row) 185 | self.W = {'val': np.random.uniform(low=-bound, high=bound, size=(self.row, self.col)), 'grad': 0} 186 | self.b = {'val': np.random.uniform(low=-bound, high=bound, size=(1, self.row)), 'grad': 0} 187 | 188 | self.cache = None 189 | 190 | def forward(self, fc): 191 | """ 192 | Performs a forward propagation between 2 fully connected layers. 193 | 194 | Parameters: 195 | - fc: fully connected layer. 196 | 197 | Returns: 198 | - A_fc: new fully connected layer. 199 | """ 200 | self.cache = fc 201 | A_fc = np.dot(fc, self.W['val'].T) + self.b['val'] 202 | return A_fc 203 | 204 | def backward(self, deltaL): 205 | """ 206 | Returns the error of the current layer and compute gradients. 207 | 208 | Parameters: 209 | - deltaL: error at last layer. 210 | 211 | Returns: 212 | - new_deltaL: error at current layer. 213 | - self.W['grad']: weights gradient. 214 | - self.b['grad']: bias gradient. 215 | """ 216 | fc = self.cache 217 | m = fc.shape[0] 218 | 219 | #Compute gradient. 220 | 221 | self.W['grad'] = (1/m) * np.dot(deltaL.T, fc) 222 | self.b['grad'] = (1/m) * np.sum(deltaL, axis = 0) 223 | 224 | #Compute error. 225 | new_deltaL = np.dot(deltaL, self.W['val']) 226 | #We still need to multiply new_deltaL by the derivative of the activation 227 | #function which is done in TanH.backward(). 228 | 229 | return new_deltaL, self.W['grad'], self.b['grad'] 230 | 231 | class SGD(): 232 | 233 | def __init__(self, lr, params): 234 | self.lr = lr 235 | self.params = params 236 | 237 | def update_params(self, grads): 238 | #print('Update Params') 239 | for key in self.params: 240 | self.params[key] += - self.lr * grads['d' + key] 241 | 242 | return self.params 243 | 244 | class AdamGD(): 245 | 246 | def __init__(self, lr, beta1, beta2, epsilon, params): 247 | self.lr = lr 248 | self.beta1 = beta1 249 | self.beta2 = beta2 250 | self.epsilon = epsilon 251 | self.params = params 252 | 253 | self.momentum = {} 254 | self.rmsprop = {} 255 | 256 | for key in self.params: 257 | self.momentum['vd' + key] = np.zeros(self.params[key].shape) 258 | self.rmsprop['sd' + key] = np.zeros(self.params[key].shape) 259 | 260 | def update_params(self, grads): 261 | 262 | for key in self.params: 263 | # Momentum update. 264 | self.momentum['vd' + key] = (self.beta1 * self.momentum['vd' + key]) + (1 - self.beta1) * grads['d' + key] 265 | # RMSprop update. 266 | self.rmsprop['sd' + key] = (self.beta2 * self.rmsprop['sd' + key]) + (1 - self.beta2) * grads['d' + key]**2 267 | # Update parameters. 268 | self.params[key] += -self.lr * self.momentum['vd' + key] / (np.sqrt(self.rmsprop['sd' + key]) + self.epsilon) 269 | 270 | return self.params 271 | 272 | class TanH(): 273 | 274 | def __init__(self, alpha = 1.7159): 275 | self.alpha = alpha 276 | self.cache = None 277 | 278 | def forward(self, X): 279 | """ 280 | Apply tanh function to X. 281 | 282 | Parameters: 283 | - X: input tensor. 284 | """ 285 | self.cache = X 286 | return self.alpha * np.tanh(X) 287 | 288 | def backward(self, new_deltaL): 289 | """ 290 | Finishes computation of error by multiplying new_deltaL by the 291 | derivative of tanH. 292 | 293 | Parameters: 294 | - new_deltaL: error previously computed. 295 | """ 296 | X = self.cache 297 | return new_deltaL * (1 - np.tanh(X)**2) 298 | 299 | class Softmax(): 300 | 301 | def __init__(self): 302 | pass 303 | 304 | def forward(self, X): 305 | """ 306 | Compute softmax values for each sets of scores in X. 307 | 308 | Parameters: 309 | - X: input vector. 310 | """ 311 | return np.exp(X) / np.sum(np.exp(X), axis=1)[:, np.newaxis] 312 | 313 | class CrossEntropyLoss(): 314 | 315 | def __init__(self): 316 | pass 317 | 318 | def get(self, y_pred, y): 319 | """ 320 | Return the negative log likelihood and the error at the last layer. 321 | 322 | Parameters: 323 | - y_pred: model predictions. 324 | - y: ground truth labels. 325 | """ 326 | batch_size = y_pred.shape[1] 327 | deltaL = y_pred - y 328 | loss = -np.sum(y * np.log(y_pred)) / batch_size 329 | return loss, deltaL 330 | 331 | -------------------------------------------------------------------------------- /src/slow/model.py: -------------------------------------------------------------------------------- 1 | from src.slow.layers import * 2 | from src.slow.utils import * 3 | import numpy as np 4 | 5 | class LeNet5(): 6 | 7 | def __init__(self): 8 | self.conv1 = Conv(nb_filters = 6, filter_size = 5, nb_channels = 1) 9 | self.tanh1 = TanH() 10 | self.pool1 = AvgPool(filter_size = 2, stride = 2) 11 | self.conv2 = Conv(nb_filters = 16, filter_size = 5, nb_channels = 6) 12 | self.tanh2 = TanH() 13 | self.pool2 = AvgPool(filter_size = 2, stride = 2) 14 | self.pool2_shape = None 15 | self.fc1 = Fc(row = 120, column = 5*5*16) 16 | self.tanh3 = TanH() 17 | self.fc2 = Fc(row = 84, column = 120) 18 | self.tanh4 = TanH() 19 | self.fc3 = Fc(row = 10 , column = 84) 20 | self.softmax = Softmax() 21 | 22 | self.layers = [self.conv1, self.conv2, self.fc1, self.fc2, self.fc3] 23 | 24 | 25 | def forward(self, X): 26 | conv1 = self.conv1.forward(X) #(6x28x28) 27 | act1 = self.tanh1.forward(conv1) 28 | pool1 = self.pool1.forward(act1) #(6x14x14) 29 | 30 | conv2 = self.conv2.forward(pool1) #(16x10x10) 31 | act2 = self.tanh2.forward(conv2) 32 | pool2 = self.pool2.forward(act2) #(16x5x5) 33 | 34 | self.pool2_shape = pool2.shape #Need it in backpropagation. 35 | pool2_flatten = pool2.reshape(self.pool2_shape[0], -1) #(1x400) 36 | 37 | fc1 = self.fc1.forward(pool2_flatten) #(1x120) 38 | act3 = self.tanh3.forward(fc1) 39 | 40 | fc2 = self.fc2.forward(act3) #(1x84) 41 | act4 = self.tanh4.forward(fc2) 42 | 43 | fc3 = self.fc3.forward(act4) #(1x10) 44 | 45 | y_pred = self.softmax.forward(fc3) 46 | 47 | return y_pred 48 | 49 | def backward(self, deltaL): 50 | #Compute gradient for weight/bias between fc3 and fc2. 51 | deltaL, dW5, db5, = self.fc3.backward(deltaL) 52 | #Compute error at fc2 layer. 53 | deltaL = self.tanh4.backward(deltaL) #(1x84) 54 | 55 | #Compute gradient for weight/bias between fc2 and fc1. 56 | deltaL, dW4, db4 = self.fc2.backward(deltaL) 57 | #Compute error at fc1 layer. 58 | deltaL = self.tanh3.backward(deltaL) #(1x120) 59 | 60 | #Compute gradient for weight/bias between fc1 and pool2 and compute 61 | #error too (don't need to backpropagate through tanh here). 62 | deltaL, dW3, db3 = self.fc1.backward(deltaL) #(1x400) 63 | deltaL = deltaL.reshape(self.pool2_shape) #(16x5x5) 64 | 65 | #Distribute error through pool2 to conv2. 66 | deltaL = self.pool2.backward(deltaL) #(16x10x10) 67 | #Distribute error through tanh. 68 | deltaL = self.tanh2.backward(deltaL) 69 | 70 | #Compute gradient for weight/bias at conv2 layer and backpropagate 71 | #error to conv1 layer. 72 | deltaL, dW2, db2 = self.conv2.backward(deltaL) #(6x14x14) 73 | 74 | #Distribute error through pool1 by creating a temporary pooling layer 75 | #of conv1 shape. 76 | deltaL = self.pool1.backward(deltaL) #(6x28x28) 77 | #Distribute error through tanh. 78 | deltaL = self.tanh1.backward(deltaL) 79 | 80 | #Compute gradient for weight/bias at conv1 layer and backpropagate 81 | #error at conv1 layer. 82 | deltaL, dW1, db1 = self.conv1.backward(deltaL) #(1x32x32) 83 | 84 | grads = { 85 | 'dW1': dW1, 'db1': db1, 86 | 'dW2': dW2, 'db2': db2, 87 | 'dW3': dW3, 'db3': db3, 88 | 'dW4': dW4, 'db4': db4, 89 | 'dW5': dW5, 'db5': db5 90 | } 91 | 92 | return grads 93 | 94 | 95 | def get_params(self): 96 | params = {} 97 | for i, layer in enumerate(self.layers): 98 | params['W' + str(i+1)] = layer.W['val'] 99 | params['b' + str(i+1)] = layer.b['val'] 100 | 101 | return params 102 | 103 | def set_params(self, params): 104 | for i, layer in enumerate(self.layers): 105 | layer.W['val'] = params['W'+ str(i+1)] 106 | layer.b['val'] = params['b' + str(i+1)] -------------------------------------------------------------------------------- /src/slow/predict.py: -------------------------------------------------------------------------------- 1 | from src.slow.layers import * 2 | from src.slow.utils import * 3 | from src.slow.model import * 4 | import numpy as np 5 | import pickle 6 | import matplotlib.pyplot as plt 7 | from tqdm import trange 8 | 9 | filename = [ 10 | ["training_images","train-images-idx3-ubyte.gz"], 11 | ["test_images","t10k-images-idx3-ubyte.gz"], 12 | ["training_labels","train-labels-idx1-ubyte.gz"], 13 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 14 | ] 15 | 16 | def test(): 17 | print("\n--------------------EXTRACTION-------------------\n") 18 | X, y, X_test, y_test = load(filename) 19 | X, X_test = X/float(255), X_test/float(255) 20 | X -= np.mean(X) 21 | X_test -= np.mean(X_test) 22 | 23 | print("\n------------------PREPROCESSING------------------\n") 24 | X_test = resize_dataset(X_test) 25 | print("Resize dataset: OK") 26 | y_test = one_hot_encoding(y_test) 27 | print("One-Hot-Encoding: OK") 28 | 29 | print("\n--------------LOAD PRETRAINED MODEL--------------\n") 30 | cost = CrossEntropyLoss() 31 | model = LeNet5() 32 | model = load_params_from_file(model) 33 | print("Load pretrained model: OK\n") 34 | 35 | print("--------------------EVALUATION-------------------\n") 36 | 37 | BATCH_SIZE = 100 38 | 39 | nb_test_examples = len(X_test) 40 | test_loss = 0 41 | test_acc = 0 42 | 43 | pbar = trange(nb_test_examples // BATCH_SIZE) 44 | test_loader = dataloader(X_test, y_test, BATCH_SIZE) 45 | 46 | for i, (X_batch, y_batch) in zip(pbar, test_loader): 47 | 48 | y_pred = model.forward(X_batch) 49 | loss, deltaL = cost.get(y_pred, y_batch) 50 | 51 | test_loss += loss * BATCH_SIZE 52 | test_acc += sum((np.argmax(y_batch, axis=1) == np.argmax(y_pred, axis=1))) 53 | 54 | pbar.set_description("Evaluation") 55 | 56 | test_loss /= nb_test_examples 57 | test_acc /= nb_test_examples 58 | 59 | info_test = "test-loss: {:0.6f} | test-acc: {:0.3f}" 60 | print(info_test.format(test_loss, test_acc)) 61 | 62 | test() -------------------------------------------------------------------------------- /src/slow/save_weights/final_weights.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/3outeille/CNNumpy/9c78b400a02ad9e030736964c11cb86d9ae03922/src/slow/save_weights/final_weights.pkl -------------------------------------------------------------------------------- /src/slow/train.py: -------------------------------------------------------------------------------- 1 | from src.slow.layers import * 2 | from src.slow.utils import * 3 | from src.slow.model import * 4 | import numpy as np 5 | import pickle 6 | import matplotlib.pyplot as plt 7 | from tqdm import trange 8 | 9 | filename = [ 10 | ["training_images","train-images-idx3-ubyte.gz"], 11 | ["test_images","t10k-images-idx3-ubyte.gz"], 12 | ["training_labels","train-labels-idx1-ubyte.gz"], 13 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 14 | ] 15 | 16 | def train(): 17 | print("\n----------------EXTRACTION---------------\n") 18 | X, y, X_test, y_test = load(filename) 19 | X, X_test = X/float(255), X_test/float(255) 20 | X -= np.mean(X) 21 | X_test -= np.mean(X_test) 22 | 23 | print("\n--------------PREPROCESSING--------------\n") 24 | X = resize_dataset(X) 25 | print("Resize dataset: OK") 26 | y = one_hot_encoding(y) 27 | print("One-Hot-Encoding: OK") 28 | X_train, y_train, X_val, y_val = train_val_split(X, y) 29 | print("Train and Validation set split: OK\n") 30 | 31 | model = LeNet5() 32 | cost = CrossEntropyLoss() 33 | 34 | params = model.get_params() 35 | 36 | optimizer = AdamGD(lr = 0.001, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, params = model.get_params()) 37 | train_costs, val_costs = [], [] 38 | 39 | print("----------------TRAINING-----------------\n") 40 | 41 | NB_EPOCH = 1 42 | BATCH_SIZE = 100 43 | 44 | print("EPOCHS: {}".format(NB_EPOCH)) 45 | print("BATCH_SIZE: {}".format(BATCH_SIZE)) 46 | print() 47 | 48 | nb_train_examples = len(X_train) 49 | nb_val_examples = len(X_val) 50 | 51 | best_val_loss = float('inf') 52 | 53 | 54 | for epoch in range(NB_EPOCH): 55 | 56 | #------------------------------------------------------------------------------- 57 | # 58 | # TRAINING PART 59 | # 60 | #------------------------------------------------------------------------------- 61 | 62 | train_loss = 0 63 | train_acc = 0 64 | 65 | pbar = trange(nb_train_examples // BATCH_SIZE) 66 | train_loader = dataloader(X_train, y_train, BATCH_SIZE) 67 | 68 | for i, (X_batch, y_batch) in zip(pbar, train_loader): 69 | 70 | y_pred = model.forward(X_batch) 71 | loss = cost.get(y_pred, y_batch) 72 | 73 | grads = model.backward(y_pred, y_batch) 74 | params = optimizer.update_params(grads) 75 | model.set_params(params) 76 | 77 | train_loss += loss * BATCH_SIZE 78 | train_acc += sum((np.argmax(y_batch, axis=1) == np.argmax(y_pred, axis=1))) 79 | 80 | pbar.set_description("[Train] Epoch {}".format(epoch+1)) 81 | 82 | train_loss /= nb_train_examples 83 | train_costs.append(train_loss) 84 | train_acc /= nb_train_examples 85 | 86 | info_train = "train-loss: {:0.6f} | train-acc: {:0.3f}" 87 | print(info_train.format(train_loss, train_acc)) 88 | 89 | #------------------------------------------------------------------------------- 90 | # 91 | # VALIDATION PART 92 | # 93 | #------------------------------------------------------------------------------- 94 | val_loss = 0 95 | val_acc = 0 96 | 97 | pbar = trange(nb_val_examples // BATCH_SIZE) 98 | val_loader = dataloader(X_val, y_val, BATCH_SIZE) 99 | 100 | for i, (X_batch, y_batch) in zip(pbar, val_loader): 101 | 102 | y_pred = model.forward(X_batch) 103 | loss, deltaL = cost.get(y_pred, y_batch) 104 | 105 | grads = model.backward(deltaL) 106 | params = optimizer.update_params(grads) 107 | model.set_params(params) 108 | 109 | val_loss += loss * BATCH_SIZE 110 | val_acc += sum((np.argmax(y_batch, axis=1) == np.argmax(y_pred, axis=1))) 111 | 112 | pbar.set_description("[Val] Epoch {}".format(epoch+1)) 113 | 114 | val_loss /= nb_val_examples 115 | val_costs.append(val_loss) 116 | val_acc /= nb_val_examples 117 | 118 | info_val = "val-loss: {:0.6f} | val-acc: {:0.3f}" 119 | print(info_val.format(val_loss, val_acc)) 120 | 121 | if best_val_loss > val_loss: 122 | print("Validation loss decreased from {:0.6f} to {:0.6f}. Model saved".format(best_val_loss, val_loss)) 123 | save_params_to_file(model) 124 | best_val_loss = val_loss 125 | 126 | print() 127 | 128 | pbar.close() 129 | 130 | train() 131 | -------------------------------------------------------------------------------- /src/slow/unit_test/unit_test.py: -------------------------------------------------------------------------------- 1 | from src.slow.layers import Conv, AvgPool 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | np.random.seed(42) 8 | torch.set_printoptions(precision=8) 9 | 10 | test_registry = {} 11 | def register_test(func): 12 | test_registry[func.__name__] = func 13 | return func 14 | 15 | @register_test 16 | def test_conv(): 17 | n, c, h, w = 2, 1, 3, 3 # Image. 18 | nf, cf, hf, wf = 2, 1, 2, 2 # Filters. 19 | x = np.random.randn(n, c, h, w) 20 | x = x.astype('float64') 21 | W = np.random.randn(nf, cf, hf, wf) 22 | W = W.astype('float64') 23 | b = np.random.randn(nf) 24 | b = b.astype('float64') 25 | deltaL = np.random.rand(n, nf, h-hf+1, h-hf+1) # Assuming stride=1/padding=0. 26 | 27 | # CNNumpy. 28 | inputs_cnn = x 29 | weights_cnn, biases_cnn = W, b 30 | conv_cnn = Conv(nb_filters=nf, filter_size=hf, nb_channels=cf, stride=1, padding=0) 31 | conv_cnn.W['val'], conv_cnn.W['grad'] = weights_cnn, np.zeros_like(weights_cnn) 32 | conv_cnn.b['val'], conv_cnn.b['grad'] = biases_cnn, np.zeros_like(biases_cnn) 33 | 34 | out_cnn = conv_cnn.forward(inputs_cnn) # Forward. 35 | _, conv_cnn.W['grad'], conv_cnn.b['grad'] = conv_cnn.backward(deltaL) # Backward. 36 | 37 | # Pytorch 38 | inputs_pt = torch.Tensor(x).double() 39 | conv_pt = nn.Conv2d(c, nf, hf, stride=1, padding=0, bias=True) 40 | conv_pt.weight = nn.Parameter(torch.DoubleTensor(W)) 41 | conv_pt.bias = nn.Parameter(torch.DoubleTensor(b)) 42 | out_pt = conv_pt(inputs_pt) # Forward. 43 | out_pt.backward(torch.Tensor(deltaL)) # Backward. 44 | 45 | # Check if inputs are equals. 46 | assert np.allclose(inputs_cnn, inputs_pt.numpy(), atol=1e-8) # 1e-8 tolerance. 47 | # Check if weights are equals. 48 | assert np.allclose(weights_cnn, conv_pt.weight.data.numpy(), atol=1e-8) # 1e-8 tolerance. 49 | # Check if biases are equals. 50 | assert np.allclose(biases_cnn, conv_pt.bias.data.numpy(), atol=1e-8) # 1e-8 tolerance. 51 | # Check if conv forward outputs are equals. 52 | assert np.allclose(out_cnn, out_pt.data.numpy(), atol=1e-8) # 1e-8 tolerance. 53 | # Check if conv backward outputs are equals. 54 | assert np.allclose(conv_cnn.W['grad'], conv_pt.weight.grad.numpy(), atol=1e-7) # 1e-7 tolerance. 55 | assert np.allclose(conv_cnn.b['grad'], conv_pt.bias.grad.numpy(), atol=1e-7) # 1e-7 tolerance. 56 | 57 | @register_test 58 | def test_avgpool(): 59 | n, c, h, w = 10, 6, 4, 4 # Image. 60 | kernel_size, stride = 2, 2 61 | x = np.random.randn(n, c, h, w) 62 | x = x.astype('float64') 63 | H = int((h - kernel_size)/ stride) + 1 64 | W = int((w - kernel_size)/ stride) + 1 65 | deltaL = np.random.rand(n, c, H, W) 66 | 67 | # CNNumpy. 68 | inputs_cnn = x 69 | avg_cnn = AvgPool(kernel_size, stride=stride) 70 | out_cnn = avg_cnn.forward(inputs_cnn) # Forward. 71 | inputs_cnn_grad = avg_cnn.backward(deltaL) # Backward. 72 | 73 | # Pytorch. 74 | inputs_pt = torch.Tensor(x).double() 75 | inputs_pt.requires_grad = True 76 | avg_pt = nn.AvgPool2d(kernel_size, stride = stride) 77 | out_pt = avg_pt(inputs_pt) # Forward. 78 | out_pt.backward(torch.Tensor(deltaL)) # Backward. 79 | 80 | # Check if inputs are equals. 81 | assert np.allclose(inputs_cnn, inputs_pt.data.numpy(), atol=1e-8) # 1e-8 tolerance. 82 | # Check if conv forward outputs are equals. 83 | assert np.allclose(out_cnn, out_pt.data.numpy(), atol=1e-8) # 1e-8 tolerance. 84 | # Check if conv backward outputs are equals. 85 | assert np.allclose(inputs_cnn_grad, inputs_pt.grad.numpy(), atol=1e-7) # 1e-7 tolerance. 86 | 87 | for name, test in test_registry.items(): 88 | print(f'Running {name}:', end=" ") 89 | test() 90 | print("OK") -------------------------------------------------------------------------------- /src/slow/utils.py: -------------------------------------------------------------------------------- 1 | from src.slow.data import * 2 | import concurrent.futures as cf 3 | import urllib.request 4 | import gzip 5 | import os 6 | from skimage import transform 7 | from PIL import Image 8 | import numpy as np 9 | import math 10 | import pickle 11 | 12 | def download_mnist(filename): 13 | """ 14 | Downloads dataset from filename. 15 | 16 | Parameters: 17 | - filename: [ 18 | ["training_images","train-images-idx3-ubyte.gz"], 19 | ["test_images","t10k-images-idx3-ubyte.gz"], 20 | ["training_labels","train-labels-idx1-ubyte.gz"], 21 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 22 | ] 23 | """ 24 | # Make data/ accessible from every folders. 25 | terminal_path = ['src/slow/data/', 'slow/data/', 'data/', '../data'] 26 | dirPath = None 27 | for path in terminal_path: 28 | if os.path.isdir(path): 29 | dirPath = path 30 | if dirPath == None: 31 | raise FileNotFoundError("extract_mnist(): Impossible to find data/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 32 | 33 | base_url = "http://yann.lecun.com/exdb/mnist/" 34 | for elt in filename: 35 | print("Downloading " + elt[1] + " in data/ ...") 36 | urllib.request.urlretrieve(base_url + elt[1], dirPath + elt[1]) 37 | print("Download complete.") 38 | 39 | def extract_mnist(filename): 40 | """ 41 | Extracts dataset from filename. 42 | 43 | Parameters: 44 | - filename: [ 45 | ["training_images","train-images-idx3-ubyte.gz"], 46 | ["test_images","t10k-images-idx3-ubyte.gz"], 47 | ["training_labels","train-labels-idx1-ubyte.gz"], 48 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 49 | ] 50 | """ 51 | # Make data/ accessible from every folders. 52 | terminal_path = ['src/slow/data/', 'slow/data/', 'data/', '../data'] 53 | dirPath = None 54 | for path in terminal_path: 55 | if os.path.isdir(path): 56 | dirPath = path 57 | if dirPath == None: 58 | raise FileNotFoundError("extract_mnist(): Impossible to find data/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 59 | 60 | mnist = {} 61 | for elt in filename[:2]: 62 | print('Extracting data/' + elt[0] + '...') 63 | with gzip.open(dirPath + elt[1]) as f: 64 | #According to the doc on MNIST website, offset for image starts at 16. 65 | mnist[elt[0]] = np.frombuffer(f.read(), dtype=np.uint8, offset=16).reshape(-1, 1, 28, 28) 66 | 67 | for elt in filename[2:]: 68 | print('Extracting data/' + elt[0] + '...') 69 | with gzip.open(dirPath + elt[1]) as f: 70 | #According to the doc on MNIST website, offset for label starts at 8. 71 | mnist[elt[0]] = np.frombuffer(f.read(), dtype=np.uint8, offset=8) 72 | 73 | print('Files extraction: OK') 74 | 75 | return mnist 76 | 77 | def load(filename): 78 | """ 79 | Loads dataset to variables. 80 | 81 | Parameters: 82 | - filename: [ 83 | ["training_images","train-images-idx3-ubyte.gz"], 84 | ["test_images","t10k-images-idx3-ubyte.gz"], 85 | ["training_labels","train-labels-idx1-ubyte.gz"], 86 | ["test_labels","t10k-labels-idx1-ubyte.gz"] 87 | ] 88 | """ 89 | # Make data/ accessible from every folders. 90 | terminal_path = ['src/slow/data/', 'slow/data/', 'data/', '../data'] 91 | dirPath = None 92 | for path in terminal_path: 93 | if os.path.isdir(path): 94 | dirPath = path 95 | if dirPath == None: 96 | raise FileNotFoundError("extract_mnist(): Impossible to find data/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 97 | 98 | L = [elt[1] for elt in filename] 99 | count = 0 100 | 101 | #Check if the 4 .gz files exist. 102 | for elt in L: 103 | if os.path.isfile(dirPath + elt): 104 | count += 1 105 | 106 | #If the 4 .gz are not in data/, we download and extract them. 107 | if count != 4: 108 | download_mnist(filename) 109 | mnist = extract_mnist(filename) 110 | else: #We just extract them. 111 | mnist = extract_mnist(filename) 112 | 113 | print('Loading dataset: OK') 114 | return mnist["training_images"], mnist["training_labels"], mnist["test_images"], mnist["test_labels"] 115 | 116 | def resize_dataset(dataset): 117 | """ 118 | Resizes dataset of MNIST images to (32, 32). 119 | 120 | Parameters: 121 | -dataset: a numpy array of size [?, 1, 28, 28]. 122 | """ 123 | args = [dataset[i:i+1000] for i in range(0, len(dataset), 1000)] 124 | 125 | def f(chunk): 126 | return transform.resize(chunk, (chunk.shape[0], 1, 32, 32)) 127 | 128 | with cf.ThreadPoolExecutor() as executor: 129 | res = executor.map(f, args) 130 | 131 | res = np.array([*res]) 132 | res = res.reshape(-1, 1, 32, 32) 133 | return res 134 | 135 | 136 | def dataloader(X, y, BATCH_SIZE): 137 | """ 138 | Returns a data generator. 139 | 140 | Parameters: 141 | - X: dataset examples. 142 | - y: ground truth labels. 143 | """ 144 | n = len(X) 145 | for t in range(0, n, BATCH_SIZE): 146 | yield X[t:t+BATCH_SIZE, ...], y[t:t+BATCH_SIZE, ...] 147 | 148 | def one_hot_encoding(y): 149 | """ 150 | Performs one-hot-encoding on y. 151 | 152 | Parameters: 153 | - y: ground truth labels. 154 | """ 155 | N = y.shape[0] 156 | Z = np.zeros((N, 10)) 157 | Z[np.arange(N), y] = 1 158 | return Z 159 | 160 | def train_val_split(X, y, val=50000): 161 | """ 162 | Splits X and y into training and validation set. 163 | 164 | Parameters: 165 | - X: dataset examples. 166 | - y: ground truth labels. 167 | """ 168 | X_train, X_val = X[:val, :], X[val:, :] 169 | y_train, y_val = y[:val, :], y[val:, :] 170 | 171 | return X_train, y_train, X_val, y_val 172 | 173 | def save_params_to_file(model): 174 | """ 175 | Saves model parameters to a file. 176 | 177 | Parameters: 178 | -model: a CNN architecture. 179 | """ 180 | # Make save_weights/ accessible from every folders. 181 | terminal_path = ["src/slow/save_weights/", "slow/save_weights/", "save_weights/", "../save_weights/"] 182 | dirPath = None 183 | for path in terminal_path: 184 | if os.path.isdir(path): 185 | dirPath = path 186 | if dirPath == None: 187 | raise FileNotFoundError("save_params_to_file(): Impossible to find save_weights/ from current folder. You need to manually add the path to it in the \'terminal_path\' list and the run the function again.") 188 | 189 | weights = model.get_params() 190 | with open(dirPath + "final_weights.pkl","wb") as f: 191 | pickle.dump(weights, f) 192 | 193 | def load_params_from_file(model): 194 | """ 195 | Loads model parameters from a file. 196 | 197 | Parameters: 198 | -model: a CNN architecture. 199 | """ 200 | # Make final_weights.pkl file accessible from every folders. 201 | terminal_path = ["src/slow/save_weights/final_weights.pkl", "slow/save_weights/final_weights.pkl", 202 | "save_weights/final_weights.pkl", "../save_weights/final_weights.pkl"] 203 | 204 | filePath = None 205 | for path in terminal_path: 206 | if os.path.isfile(path): 207 | filePath = path 208 | if filePath == None: 209 | raise FileNotFoundError('load_params_from_file(): Cannot find final_weights.pkl from your current folder. You need to manually add it to terminal_path list and the run the function again.') 210 | 211 | pickle_in = open(filePath, 'rb') 212 | params = pickle.load(pickle_in) 213 | model.set_params(params) 214 | return model 215 | 216 | def prettyPrint3D(M): 217 | """ 218 | Displays a 3D matrix in a pretty way. 219 | 220 | Parameters: 221 | -M: Matrix of shape (m, n_H, n_W, n_C) with m, the number 3D matrices. 222 | """ 223 | m, n_C, n_H, n_W = M.shape 224 | 225 | for i in range(m): 226 | 227 | for c in range(n_C): 228 | print('Image {}, channel {}'.format(i + 1, c + 1), end='\n\n') 229 | 230 | for h in range(n_H): 231 | print("/", end="") 232 | 233 | for j in range(n_W): 234 | 235 | print(M[i, c, h, j], end = ",") 236 | 237 | print("/", end='\n\n') 238 | 239 | print('-------------------', end='\n\n') 240 | --------------------------------------------------------------------------------