├── .DS_Store ├── CNN ├── __pycache__ │ ├── backward.cpython-36.pyc │ ├── forward.cpython-36.pyc │ ├── network.cpython-36.pyc │ └── utils.cpython-36.pyc ├── backward.py ├── forward.py ├── network.py └── utils.py ├── README.md ├── images ├── test_accuracy.png └── training_progress.png ├── measure_performance.py ├── params.pkl ├── requirements.txt ├── t10k-images-idx3-ubyte.gz ├── t10k-labels-idx1-ubyte.gz ├── train-images-idx3-ubyte.gz ├── train-labels-idx1-ubyte.gz └── train_cnn.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/.DS_Store -------------------------------------------------------------------------------- /CNN/__pycache__/backward.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/CNN/__pycache__/backward.cpython-36.pyc -------------------------------------------------------------------------------- /CNN/__pycache__/forward.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/CNN/__pycache__/forward.cpython-36.pyc -------------------------------------------------------------------------------- /CNN/__pycache__/network.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/CNN/__pycache__/network.cpython-36.pyc -------------------------------------------------------------------------------- /CNN/__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/CNN/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /CNN/backward.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: backpropagation operations for a convolutional neural network 3 | 4 | Author: Alejandro Escontrela 5 | Version: 1.0 6 | Date: June 12th, 2018 7 | ''' 8 | 9 | import numpy as np 10 | 11 | from CNN.utils import * 12 | 13 | ##################################################### 14 | ############### Backward Operations ################# 15 | ##################################################### 16 | 17 | def convolutionBackward(dconv_prev, conv_in, filt, s): 18 | ''' 19 | Backpropagation through a convolutional layer. 20 | ''' 21 | (n_f, n_c, f, _) = filt.shape 22 | (_, orig_dim, _) = conv_in.shape 23 | ## initialize derivatives 24 | dout = np.zeros(conv_in.shape) 25 | dfilt = np.zeros(filt.shape) 26 | dbias = np.zeros((n_f,1)) 27 | for curr_f in range(n_f): 28 | # loop through all filters 29 | curr_y = out_y = 0 30 | while curr_y + f <= orig_dim: 31 | curr_x = out_x = 0 32 | while curr_x + f <= orig_dim: 33 | # loss gradient of filter (used to update the filter) 34 | dfilt[curr_f] += dconv_prev[curr_f, out_y, out_x] * conv_in[:, curr_y:curr_y+f, curr_x:curr_x+f] 35 | # loss gradient of the input to the convolution operation (conv1 in the case of this network) 36 | dout[:, curr_y:curr_y+f, curr_x:curr_x+f] += dconv_prev[curr_f, out_y, out_x] * filt[curr_f] 37 | curr_x += s 38 | out_x += 1 39 | curr_y += s 40 | out_y += 1 41 | # loss gradient of the bias 42 | dbias[curr_f] = np.sum(dconv_prev[curr_f]) 43 | 44 | return dout, dfilt, dbias 45 | 46 | 47 | 48 | def maxpoolBackward(dpool, orig, f, s): 49 | ''' 50 | Backpropagation through a maxpooling layer. The gradients are passed through the indices of greatest value in the original maxpooling during the forward step. 51 | ''' 52 | (n_c, orig_dim, _) = orig.shape 53 | 54 | dout = np.zeros(orig.shape) 55 | 56 | for curr_c in range(n_c): 57 | curr_y = out_y = 0 58 | while curr_y + f <= orig_dim: 59 | curr_x = out_x = 0 60 | while curr_x + f <= orig_dim: 61 | # obtain index of largest value in input for current window 62 | (a, b) = nanargmax(orig[curr_c, curr_y:curr_y+f, curr_x:curr_x+f]) 63 | dout[curr_c, curr_y+a, curr_x+b] = dpool[curr_c, out_y, out_x] 64 | 65 | curr_x += s 66 | out_x += 1 67 | curr_y += s 68 | out_y += 1 69 | 70 | return dout 71 | -------------------------------------------------------------------------------- /CNN/forward.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: forward operations for a convolutional neural network 3 | 4 | Author: Alejandro Escontrela 5 | Version: 1.0 6 | Date: June 12th, 2018 7 | ''' 8 | import numpy as np 9 | 10 | 11 | ##################################################### 12 | ################ Forward Operations ################# 13 | ##################################################### 14 | 15 | 16 | def convolution(image, filt, bias, s=1): 17 | ''' 18 | Confolves `filt` over `image` using stride `s` 19 | ''' 20 | (n_f, n_c_f, f, _) = filt.shape # filter dimensions 21 | n_c, in_dim, _ = image.shape # image dimensions 22 | 23 | out_dim = int((in_dim - f)/s)+1 # calculate output dimensions 24 | 25 | assert n_c == n_c_f, "Dimensions of filter must match dimensions of input image" 26 | 27 | out = np.zeros((n_f,out_dim,out_dim)) 28 | 29 | # convolve the filter over every part of the image, adding the bias at each step. 30 | for curr_f in range(n_f): 31 | curr_y = out_y = 0 32 | while curr_y + f <= in_dim: 33 | curr_x = out_x = 0 34 | while curr_x + f <= in_dim: 35 | out[curr_f, out_y, out_x] = np.sum(filt[curr_f] * image[:,curr_y:curr_y+f, curr_x:curr_x+f]) + bias[curr_f] 36 | curr_x += s 37 | out_x += 1 38 | curr_y += s 39 | out_y += 1 40 | 41 | return out 42 | 43 | def maxpool(image, f=2, s=2): 44 | ''' 45 | Downsample `image` using kernel size `f` and stride `s` 46 | ''' 47 | n_c, h_prev, w_prev = image.shape 48 | 49 | h = int((h_prev - f)/s)+1 50 | w = int((w_prev - f)/s)+1 51 | 52 | downsampled = np.zeros((n_c, h, w)) 53 | for i in range(n_c): 54 | # slide maxpool window over each part of the image and assign the max value at each step to the output 55 | curr_y = out_y = 0 56 | while curr_y + f <= h_prev: 57 | curr_x = out_x = 0 58 | while curr_x + f <= w_prev: 59 | downsampled[i, out_y, out_x] = np.max(image[i, curr_y:curr_y+f, curr_x:curr_x+f]) 60 | curr_x += s 61 | out_x += 1 62 | curr_y += s 63 | out_y += 1 64 | return downsampled 65 | 66 | def softmax(X): 67 | out = np.exp(X) 68 | return out/np.sum(out) 69 | 70 | def categoricalCrossEntropy(probs, label): 71 | return -np.sum(label * np.log(probs)) 72 | 73 | -------------------------------------------------------------------------------- /CNN/network.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: methods to set up and train the network's parameters. 3 | 4 | Author: Alejandro Escontrela 5 | Version: V.1. 6 | Date: June 12th, 2018 7 | ''' 8 | from CNN.forward import * 9 | from CNN.backward import * 10 | from CNN.utils import * 11 | 12 | import numpy as np 13 | import pickle 14 | from tqdm import tqdm 15 | ##################################################### 16 | ############### Building The Network ################ 17 | ##################################################### 18 | 19 | def conv(image, label, params, conv_s, pool_f, pool_s): 20 | 21 | [f1, f2, w3, w4, b1, b2, b3, b4] = params 22 | 23 | ################################################ 24 | ############## Forward Operation ############### 25 | ################################################ 26 | conv1 = convolution(image, f1, b1, conv_s) # convolution operation 27 | conv1[conv1<=0] = 0 # pass through ReLU non-linearity 28 | 29 | conv2 = convolution(conv1, f2, b2, conv_s) # second convolution operation 30 | conv2[conv2<=0] = 0 # pass through ReLU non-linearity 31 | 32 | pooled = maxpool(conv2, pool_f, pool_s) # maxpooling operation 33 | 34 | (nf2, dim2, _) = pooled.shape 35 | fc = pooled.reshape((nf2 * dim2 * dim2, 1)) # flatten pooled layer 36 | 37 | z = w3.dot(fc) + b3 # first dense layer 38 | z[z<=0] = 0 # pass through ReLU non-linearity 39 | 40 | out = w4.dot(z) + b4 # second dense layer 41 | 42 | probs = softmax(out) # predict class probabilities with the softmax activation function 43 | 44 | ################################################ 45 | #################### Loss ###################### 46 | ################################################ 47 | 48 | loss = categoricalCrossEntropy(probs, label) # categorical cross-entropy loss 49 | 50 | ################################################ 51 | ############# Backward Operation ############### 52 | ################################################ 53 | dout = probs - label # derivative of loss w.r.t. final dense layer output 54 | dw4 = dout.dot(z.T) # loss gradient of final dense layer weights 55 | db4 = np.sum(dout, axis = 1).reshape(b4.shape) # loss gradient of final dense layer biases 56 | 57 | dz = w4.T.dot(dout) # loss gradient of first dense layer outputs 58 | dz[z<=0] = 0 # backpropagate through ReLU 59 | dw3 = dz.dot(fc.T) 60 | db3 = np.sum(dz, axis = 1).reshape(b3.shape) 61 | 62 | dfc = w3.T.dot(dz) # loss gradients of fully-connected layer (pooling layer) 63 | dpool = dfc.reshape(pooled.shape) # reshape fully connected into dimensions of pooling layer 64 | 65 | dconv2 = maxpoolBackward(dpool, conv2, pool_f, pool_s) # backprop through the max-pooling layer(only neurons with highest activation in window get updated) 66 | dconv2[conv2<=0] = 0 # backpropagate through ReLU 67 | 68 | dconv1, df2, db2 = convolutionBackward(dconv2, conv1, f2, conv_s) # backpropagate previous gradient through second convolutional layer. 69 | dconv1[conv1<=0] = 0 # backpropagate through ReLU 70 | 71 | dimage, df1, db1 = convolutionBackward(dconv1, image, f1, conv_s) # backpropagate previous gradient through first convolutional layer. 72 | 73 | grads = [df1, df2, dw3, dw4, db1, db2, db3, db4] 74 | 75 | return grads, loss 76 | 77 | ##################################################### 78 | ################### Optimization #################### 79 | ##################################################### 80 | 81 | def adamGD(batch, num_classes, lr, dim, n_c, beta1, beta2, params, cost): 82 | ''' 83 | update the parameters through Adam gradient descnet. 84 | ''' 85 | [f1, f2, w3, w4, b1, b2, b3, b4] = params 86 | 87 | X = batch[:,0:-1] # get batch inputs 88 | X = X.reshape(len(batch), n_c, dim, dim) 89 | Y = batch[:,-1] # get batch labels 90 | 91 | cost_ = 0 92 | batch_size = len(batch) 93 | 94 | # initialize gradients and momentum,RMS params 95 | df1 = np.zeros(f1.shape) 96 | df2 = np.zeros(f2.shape) 97 | dw3 = np.zeros(w3.shape) 98 | dw4 = np.zeros(w4.shape) 99 | db1 = np.zeros(b1.shape) 100 | db2 = np.zeros(b2.shape) 101 | db3 = np.zeros(b3.shape) 102 | db4 = np.zeros(b4.shape) 103 | 104 | v1 = np.zeros(f1.shape) 105 | v2 = np.zeros(f2.shape) 106 | v3 = np.zeros(w3.shape) 107 | v4 = np.zeros(w4.shape) 108 | bv1 = np.zeros(b1.shape) 109 | bv2 = np.zeros(b2.shape) 110 | bv3 = np.zeros(b3.shape) 111 | bv4 = np.zeros(b4.shape) 112 | 113 | s1 = np.zeros(f1.shape) 114 | s2 = np.zeros(f2.shape) 115 | s3 = np.zeros(w3.shape) 116 | s4 = np.zeros(w4.shape) 117 | bs1 = np.zeros(b1.shape) 118 | bs2 = np.zeros(b2.shape) 119 | bs3 = np.zeros(b3.shape) 120 | bs4 = np.zeros(b4.shape) 121 | 122 | for i in range(batch_size): 123 | 124 | x = X[i] 125 | y = np.eye(num_classes)[int(Y[i])].reshape(num_classes, 1) # convert label to one-hot 126 | 127 | # Collect Gradients for training example 128 | grads, loss = conv(x, y, params, 1, 2, 2) 129 | [df1_, df2_, dw3_, dw4_, db1_, db2_, db3_, db4_] = grads 130 | 131 | df1+=df1_ 132 | db1+=db1_ 133 | df2+=df2_ 134 | db2+=db2_ 135 | dw3+=dw3_ 136 | db3+=db3_ 137 | dw4+=dw4_ 138 | db4+=db4_ 139 | 140 | cost_+= loss 141 | 142 | # Parameter Update 143 | 144 | v1 = beta1*v1 + (1-beta1)*df1/batch_size # momentum update 145 | s1 = beta2*s1 + (1-beta2)*(df1/batch_size)**2 # RMSProp update 146 | f1 -= lr * v1/np.sqrt(s1+1e-7) # combine momentum and RMSProp to perform update with Adam 147 | 148 | bv1 = beta1*bv1 + (1-beta1)*db1/batch_size 149 | bs1 = beta2*bs1 + (1-beta2)*(db1/batch_size)**2 150 | b1 -= lr * bv1/np.sqrt(bs1+1e-7) 151 | 152 | v2 = beta1*v2 + (1-beta1)*df2/batch_size 153 | s2 = beta2*s2 + (1-beta2)*(df2/batch_size)**2 154 | f2 -= lr * v2/np.sqrt(s2+1e-7) 155 | 156 | bv2 = beta1*bv2 + (1-beta1) * db2/batch_size 157 | bs2 = beta2*bs2 + (1-beta2)*(db2/batch_size)**2 158 | b2 -= lr * bv2/np.sqrt(bs2+1e-7) 159 | 160 | v3 = beta1*v3 + (1-beta1) * dw3/batch_size 161 | s3 = beta2*s3 + (1-beta2)*(dw3/batch_size)**2 162 | w3 -= lr * v3/np.sqrt(s3+1e-7) 163 | 164 | bv3 = beta1*bv3 + (1-beta1) * db3/batch_size 165 | bs3 = beta2*bs3 + (1-beta2)*(db3/batch_size)**2 166 | b3 -= lr * bv3/np.sqrt(bs3+1e-7) 167 | 168 | v4 = beta1*v4 + (1-beta1) * dw4/batch_size 169 | s4 = beta2*s4 + (1-beta2)*(dw4/batch_size)**2 170 | w4 -= lr * v4 / np.sqrt(s4+1e-7) 171 | 172 | bv4 = beta1*bv4 + (1-beta1)*db4/batch_size 173 | bs4 = beta2*bs4 + (1-beta2)*(db4/batch_size)**2 174 | b4 -= lr * bv4 / np.sqrt(bs4+1e-7) 175 | 176 | 177 | cost_ = cost_/batch_size 178 | cost.append(cost_) 179 | 180 | params = [f1, f2, w3, w4, b1, b2, b3, b4] 181 | 182 | return params, cost 183 | 184 | ##################################################### 185 | ##################### Training ###################### 186 | ##################################################### 187 | 188 | def train(num_classes = 10, lr = 0.01, beta1 = 0.95, beta2 = 0.99, img_dim = 28, img_depth = 1, f = 5, num_filt1 = 8, num_filt2 = 8, batch_size = 32, num_epochs = 2, save_path = 'params.pkl'): 189 | 190 | # training data 191 | m =50000 192 | X = extract_data('train-images-idx3-ubyte.gz', m, img_dim) 193 | y_dash = extract_labels('train-labels-idx1-ubyte.gz', m).reshape(m,1) 194 | X-= int(np.mean(X)) 195 | X/= int(np.std(X)) 196 | train_data = np.hstack((X,y_dash)) 197 | 198 | np.random.shuffle(train_data) 199 | 200 | ## Initializing all the parameters 201 | f1, f2, w3, w4 = (num_filt1 ,img_depth,f,f), (num_filt2 ,num_filt1,f,f), (128,800), (10, 128) 202 | f1 = initializeFilter(f1) 203 | f2 = initializeFilter(f2) 204 | w3 = initializeWeight(w3) 205 | w4 = initializeWeight(w4) 206 | 207 | b1 = np.zeros((f1.shape[0],1)) 208 | b2 = np.zeros((f2.shape[0],1)) 209 | b3 = np.zeros((w3.shape[0],1)) 210 | b4 = np.zeros((w4.shape[0],1)) 211 | 212 | params = [f1, f2, w3, w4, b1, b2, b3, b4] 213 | 214 | cost = [] 215 | 216 | print("LR:"+str(lr)+", Batch Size:"+str(batch_size)) 217 | 218 | for epoch in range(num_epochs): 219 | np.random.shuffle(train_data) 220 | batches = [train_data[k:k + batch_size] for k in range(0, train_data.shape[0], batch_size)] 221 | 222 | t = tqdm(batches) 223 | for x,batch in enumerate(t): 224 | params, cost = adamGD(batch, num_classes, lr, img_dim, img_depth, beta1, beta2, params, cost) 225 | t.set_description("Cost: %.2f" % (cost[-1])) 226 | 227 | to_save = [params, cost] 228 | 229 | with open(save_path, 'wb') as file: 230 | pickle.dump(to_save, file) 231 | 232 | return cost 233 | -------------------------------------------------------------------------------- /CNN/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: Utility methods for a Convolutional Neural Network 3 | 4 | Author: Alejandro Escontrela 5 | Version: V.1. 6 | Date: June 12th, 2018 7 | ''' 8 | from CNN.forward import * 9 | import numpy as np 10 | import gzip 11 | 12 | ##################################################### 13 | ################## Utility Methods ################## 14 | ##################################################### 15 | 16 | def extract_data(filename, num_images, IMAGE_WIDTH): 17 | ''' 18 | Extract images by reading the file bytestream. Reshape the read values into a 3D matrix of dimensions [m, h, w], where m 19 | is the number of training examples. 20 | ''' 21 | print('Extracting', filename) 22 | with gzip.open(filename) as bytestream: 23 | bytestream.read(16) 24 | buf = bytestream.read(IMAGE_WIDTH * IMAGE_WIDTH * num_images) 25 | data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32) 26 | data = data.reshape(num_images, IMAGE_WIDTH*IMAGE_WIDTH) 27 | return data 28 | 29 | def extract_labels(filename, num_images): 30 | ''' 31 | Extract label into vector of integer values of dimensions [m, 1], where m is the number of images. 32 | ''' 33 | print('Extracting', filename) 34 | with gzip.open(filename) as bytestream: 35 | bytestream.read(8) 36 | buf = bytestream.read(1 * num_images) 37 | labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64) 38 | return labels 39 | 40 | def initializeFilter(size, scale = 1.0): 41 | stddev = scale/np.sqrt(np.prod(size)) 42 | return np.random.normal(loc = 0, scale = stddev, size = size) 43 | 44 | def initializeWeight(size): 45 | return np.random.standard_normal(size=size) * 0.01 46 | 47 | def nanargmax(arr): 48 | idx = np.nanargmax(arr) 49 | idxs = np.unravel_index(idx, arr.shape) 50 | return idxs 51 | 52 | def predict(image, f1, f2, w3, w4, b1, b2, b3, b4, conv_s = 1, pool_f = 2, pool_s = 2): 53 | ''' 54 | Make predictions with trained filters/weights. 55 | ''' 56 | conv1 = convolution(image, f1, b1, conv_s) # convolution operation 57 | conv1[conv1<=0] = 0 #relu activation 58 | 59 | conv2 = convolution(conv1, f2, b2, conv_s) # second convolution operation 60 | conv2[conv2<=0] = 0 # pass through ReLU non-linearity 61 | 62 | pooled = maxpool(conv2, pool_f, pool_s) # maxpooling operation 63 | (nf2, dim2, _) = pooled.shape 64 | fc = pooled.reshape((nf2 * dim2 * dim2, 1)) # flatten pooled layer 65 | 66 | z = w3.dot(fc) + b3 # first dense layer 67 | z[z<=0] = 0 # pass through ReLU non-linearity 68 | 69 | out = w4.dot(z) + b4 # second dense layer 70 | probs = softmax(out) # predict class probabilities with the softmax activation function 71 | 72 | return np.argmax(probs), np.max(probs) 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Numpy-CNN 2 | A numpy-only implementation of a Convolutional Neural Network, from the ground up. 3 | 4 | *Written by Alejandro Escontrela for [this article](https://towardsdatascience.com/convolutional-neural-networks-from-the-ground-up-c67bb41454e1) on CNNs.* 5 | 6 | ## Purpose 7 | 8 | To gain a quality understanding of convolutional neural networks and what makes them peform so well, I constructed one from scratch with NumPy. This CNN is in no way intended to replace popular DL frameworks such as Tensorflow or Torch, *it is instead meant to serve as an instructional tool.* 9 | 10 | ## Training the network 11 | 12 | To train the network on your machine, first install all necessary dependencies using: 13 | 14 | 15 | `$ pip install -r requirements.txt` 16 | 17 | Afterwards, you can train the network using the following command: 18 | 19 | `$ python3 train_cnn.py '.pkl'` 20 | 21 | Replace `` with whatever file name you would like. The terminal should display the following progress bar to indicate the training progress, as well as the cost for the current training batch: 22 |

23 | portfolio_view 24 |

25 | 26 | 27 | After the CNN has finished training, a .pkl file containing the network's parameters is saved to the directory where the script was run. 28 | 29 | The network takes about 5 hours to train on my macbook pro. I included the trained params in the GitHub repo under the name `params.pkl` . To use the pretrained params when measuring the network's performance, replace `` with params.pkl. 30 | 31 | ## Measuring Performance 32 | To measure the network's accuracy, run the following command in the terminal: 33 | 34 | `$ python3 measure_performance.py '.pkl'` 35 | 36 | This command will use the trained parameters to run predictions on all 10,000 digits in the test dataset. After all predictions are made, a value displaying the network's accuracy will appear in the command prompt: 37 | 38 |

39 | portfolio_view 40 |

41 | -------------------------------------------------------------------------------- /images/test_accuracy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/images/test_accuracy.png -------------------------------------------------------------------------------- /images/training_progress.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/images/training_progress.png -------------------------------------------------------------------------------- /measure_performance.py: -------------------------------------------------------------------------------- 1 | from CNN.utils import * 2 | 3 | from tqdm import tqdm 4 | import argparse 5 | import matplotlib.pyplot as plt 6 | import pickle 7 | 8 | parser = argparse.ArgumentParser(description='Predict the network accuracy.') 9 | parser.add_argument('parameters', metavar = 'parameters', help='name of file parameters were saved in. These parameters will be used to measure the accuracy.') 10 | 11 | if __name__ == '__main__': 12 | args = parser.parse_args() 13 | save_path = args.parameters 14 | 15 | params, cost = pickle.load(open(save_path, 'rb')) 16 | [f1, f2, w3, w4, b1, b2, b3, b4] = params 17 | 18 | # Get test data 19 | m =10000 20 | X = extract_data('t10k-images-idx3-ubyte.gz', m, 28) 21 | y_dash = extract_labels('t10k-labels-idx1-ubyte.gz', m).reshape(m,1) 22 | # Normalize the data 23 | X-= int(np.mean(X)) # subtract mean 24 | X/= int(np.std(X)) # divide by standard deviation 25 | test_data = np.hstack((X,y_dash)) 26 | 27 | X = test_data[:,0:-1] 28 | X = X.reshape(len(test_data), 1, 28, 28) 29 | y = test_data[:,-1] 30 | 31 | corr = 0 32 | digit_count = [0 for i in range(10)] 33 | digit_correct = [0 for i in range(10)] 34 | 35 | print() 36 | print("Computing accuracy over test set:") 37 | 38 | t = tqdm(range(len(X)), leave=True) 39 | 40 | for i in t: 41 | x = X[i] 42 | pred, prob = predict(x, f1, f2, w3, w4, b1, b2, b3, b4) 43 | digit_count[int(y[i])]+=1 44 | if pred==y[i]: 45 | corr+=1 46 | digit_correct[pred]+=1 47 | 48 | t.set_description("Acc:%0.2f%%" % (float(corr/(i+1))*100)) 49 | 50 | print("Overall Accuracy: %.2f" % (float(corr/len(test_data)*100))) 51 | x = np.arange(10) 52 | digit_recall = [x/y for x,y in zip(digit_correct, digit_count)] 53 | plt.xlabel('Digits') 54 | plt.ylabel('Recall') 55 | plt.title("Recall on Test Set") 56 | plt.bar(x,digit_recall) 57 | plt.show() -------------------------------------------------------------------------------- /params.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/params.pkl -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | tqdm 3 | matplotlib -------------------------------------------------------------------------------- /t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alescontrela/Numpy-CNN/be15d0139327a690813343a6e15d651fd043fa89/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /train_cnn.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: Script to train the network and measure its performance on the test set. 3 | 4 | Author: Alejandro Escontrela 5 | Version: V.1. 6 | Date: June 12th, 2018 7 | ''' 8 | from CNN.network import * 9 | from CNN.utils import * 10 | 11 | from tqdm import tqdm 12 | import argparse 13 | import matplotlib.pyplot as plt 14 | import pickle 15 | 16 | parser = argparse.ArgumentParser(description='Train a convolutional neural network.') 17 | parser.add_argument('save_path', metavar = 'Save Path', help='name of file to save parameters in.') 18 | 19 | if __name__ == '__main__': 20 | 21 | args = parser.parse_args() 22 | save_path = args.save_path 23 | 24 | cost = train(save_path = save_path) 25 | 26 | params, cost = pickle.load(open(save_path, 'rb')) 27 | [f1, f2, w3, w4, b1, b2, b3, b4] = params 28 | 29 | # Plot cost 30 | plt.plot(cost, 'r') 31 | plt.xlabel('# Iterations') 32 | plt.ylabel('Cost') 33 | plt.legend('Loss', loc='upper right') 34 | plt.show() 35 | 36 | # Get test data 37 | m =10000 38 | X = extract_data('t10k-images-idx3-ubyte.gz', m, 28) 39 | y_dash = extract_labels('t10k-labels-idx1-ubyte.gz', m).reshape(m,1) 40 | # Normalize the data 41 | X-= int(np.mean(X)) # subtract mean 42 | X/= int(np.std(X)) # divide by standard deviation 43 | test_data = np.hstack((X,y_dash)) 44 | 45 | X = test_data[:,0:-1] 46 | X = X.reshape(len(test_data), 1, 28, 28) 47 | y = test_data[:,-1] 48 | 49 | corr = 0 50 | digit_count = [0 for i in range(10)] 51 | digit_correct = [0 for i in range(10)] 52 | 53 | print() 54 | print("Computing accuracy over test set:") 55 | 56 | t = tqdm(range(len(X)), leave=True) 57 | 58 | for i in t: 59 | x = X[i] 60 | pred, prob = predict(x, f1, f2, w3, w4, b1, b2, b3, b4) 61 | digit_count[int(y[i])]+=1 62 | if pred==y[i]: 63 | corr+=1 64 | digit_correct[pred]+=1 65 | 66 | t.set_description("Acc:%0.2f%%" % (float(corr/(i+1))*100)) 67 | 68 | print("Overall Accuracy: %.2f" % (float(corr/len(test_data)*100))) 69 | x = np.arange(10) 70 | digit_recall = [x/y for x,y in zip(digit_correct, digit_count)] 71 | plt.xlabel('Digits') 72 | plt.ylabel('Recall') 73 | plt.title("Recall on Test Set") 74 | plt.bar(x,digit_recall) 75 | plt.show() --------------------------------------------------------------------------------