├── .DS_Store ├── partOne.pyc ├── partSix.pyc ├── partTwo.pyc ├── partFive.pyc ├── partFour.pyc ├── images └── simpleNetwork.png ├── partOne.py ├── README.md ├── partTwo.py ├── partFour.py ├── partFive.py ├── partSix.py ├── Part 5 Numerical Gradient Checking.ipynb ├── .ipynb_checkpoints ├── Part 5 Numerical Gradient Checking-checkpoint.ipynb ├── Part 2 Forward Propagation-checkpoint.ipynb └── Part 3 Gradient Descent-checkpoint.ipynb ├── Part 2 Forward Propagation.ipynb ├── Part 3 Gradient Descent.ipynb └── Part 4 Backpropagation.ipynb /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/.DS_Store -------------------------------------------------------------------------------- /partOne.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/partOne.pyc -------------------------------------------------------------------------------- /partSix.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/partSix.pyc -------------------------------------------------------------------------------- /partTwo.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/partTwo.pyc -------------------------------------------------------------------------------- /partFive.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/partFive.pyc -------------------------------------------------------------------------------- /partFour.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/partFour.pyc -------------------------------------------------------------------------------- /images/simpleNetwork.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Todo/Neural-Networks-Demystified/master/images/simpleNetwork.png -------------------------------------------------------------------------------- /partOne.py: -------------------------------------------------------------------------------- 1 | # Neural Networks Demystified 2 | # Part 1: Data + Architecture 3 | # 4 | # Supporting code for short YouTube series on artificial neural networks. 5 | # 6 | # Stephen Welch 7 | # @stephencwelch 8 | 9 | import numpy as np 10 | 11 | # X = (hours sleeping, hours studying), y = Score on test 12 | X = np.array(([3,5], [5,1], [10,2]), dtype=float) 13 | y = np.array(([75], [82], [93]), dtype=float) 14 | 15 | # Normalize 16 | X = X/np.amax(X, axis=0) 17 | y = y/100 #Max test score is 100 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #Neural Networks Demystified 2 | 3 | Supporting iPython notebooks for the YouTube Series Neural Networks Demystified. I've included formulas, code, and the text of the movies in the iPython notebooks, in addition to raw code in python scripts. 4 | 5 | iPython notebooks can be downloaded and run locally, or viewed using nbviewer: http://nbviewer.ipython.org/. 6 | 7 | ##Using the iPython notebook 8 | The iPython/Jupyter notebook is an incredible tool, but can be a little tricky to setup. I recommend the [anaconda] (https://store.continuum.io/cshop/anaconda/) distribution of python. I've written and tested this code with the the anaconda build of python 2 running on OSX. You will likely get a few warinings about contour plotting - if anyone has a fix for this, feel free to submit a pull request. 9 | 10 | -------------------------------------------------------------------------------- /partTwo.py: -------------------------------------------------------------------------------- 1 | # Neural Networks Demystified 2 | # Part 2: Forward Propagation 3 | # 4 | # Supporting code for short YouTube series on artificial neural networks. 5 | # 6 | # Stephen Welch 7 | # @stephencwelch 8 | 9 | 10 | ## ----------------------- Part 1 ---------------------------- ## 11 | import numpy as np 12 | 13 | # X = (hours sleeping, hours studying), y = Score on test 14 | X = np.array(([3,5], [5,1], [10,2]), dtype=float) 15 | y = np.array(([75], [82], [93]), dtype=float) 16 | 17 | # Normalize 18 | X = X/np.amax(X, axis=0) 19 | y = y/100 #Max test score is 100 20 | 21 | ## ----------------------- Part 2 ---------------------------- ## 22 | 23 | class Neural_Network(object): 24 | def __init__(self): 25 | #Define Hyperparameters 26 | self.inputLayerSize = 2 27 | self.outputLayerSize = 1 28 | self.hiddenLayerSize = 3 29 | 30 | #Weights (parameters) 31 | self.W1 = np.random.randn(self.inputLayerSize, self.hiddenLayerSize) 32 | self.W2 = np.random.randn(self.hiddenLayerSize, self.outputLayerSize) 33 | 34 | def forward(self, X): 35 | #Propagate inputs though network 36 | self.z2 = np.dot(X, self.W1) 37 | self.a2 = self.sigmoid(self.z2) 38 | self.z3 = np.dot(self.a2, self.W2) 39 | yHat = self.sigmoid(self.z3) 40 | return yHat 41 | 42 | def sigmoid(self, z): 43 | #Apply sigmoid activation function to scalar, vector, or matrix 44 | return 1/(1+np.exp(-z)) -------------------------------------------------------------------------------- /partFour.py: -------------------------------------------------------------------------------- 1 | # Neural Networks Demystified 2 | # Part 4: Backpropagation 3 | # 4 | # Supporting code for short YouTube series on artificial neural networks. 5 | # 6 | # Stephen Welch 7 | # @stephencwelch 8 | 9 | 10 | ## ----------------------- Part 1 ---------------------------- ## 11 | import numpy as np 12 | 13 | # X = (hours sleeping, hours studying), y = Score on test 14 | X = np.array(([3,5], [5,1], [10,2]), dtype=float) 15 | y = np.array(([75], [82], [93]), dtype=float) 16 | 17 | # Normalize 18 | X = X/np.amax(X, axis=0) 19 | y = y/100 #Max test score is 100 20 | 21 | ## ----------------------- Part 4 ---------------------------- ## 22 | 23 | # Whole Class with additions: 24 | class Neural_Network(object): 25 | def __init__(self): 26 | #Define Hyperparameters 27 | self.inputLayerSize = 2 28 | self.outputLayerSize = 1 29 | self.hiddenLayerSize = 3 30 | 31 | #Weights (parameters) 32 | self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize) 33 | self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize) 34 | 35 | def forward(self, X): 36 | #Propagate inputs through network 37 | self.z2 = np.dot(X, self.W1) 38 | self.a2 = self.sigmoid(self.z2) 39 | self.z3 = np.dot(self.a2, self.W2) 40 | yHat = self.sigmoid(self.z3) 41 | return yHat 42 | 43 | def sigmoid(self, z): 44 | #Apply sigmoid activation function to scalar, vector, or matrix 45 | return 1/(1+np.exp(-z)) 46 | 47 | def sigmoidPrime(self,z): 48 | #Gradient of sigmoid 49 | return np.exp(-z)/((1+np.exp(-z))**2) 50 | 51 | def costFunction(self, X, y): 52 | #Compute cost for given X,y, use weights already stored in class. 53 | self.yHat = self.forward(X) 54 | J = 0.5*sum((y-self.yHat)**2) 55 | return J 56 | 57 | def costFunctionPrime(self, X, y): 58 | #Compute derivative with respect to W and W2 for a given X and y: 59 | self.yHat = self.forward(X) 60 | 61 | delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3)) 62 | dJdW2 = np.dot(self.a2.T, delta3) 63 | 64 | delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2) 65 | dJdW1 = np.dot(X.T, delta2) 66 | 67 | return dJdW1, dJdW2 68 | -------------------------------------------------------------------------------- /partFive.py: -------------------------------------------------------------------------------- 1 | # Neural Networks Demystified 2 | # Part 5: Numerical Gradient Checking 3 | # 4 | # Supporting code for short YouTube series on artificial neural networks. 5 | # 6 | # Stephen Welch 7 | # @stephencwelch 8 | 9 | 10 | ## ----------------------- Part 1 ---------------------------- ## 11 | import numpy as np 12 | 13 | # X = (hours sleeping, hours studying), y = Score on test 14 | X = np.array(([3,5], [5,1], [10,2]), dtype=float) 15 | y = np.array(([75], [82], [93]), dtype=float) 16 | 17 | # Normalize 18 | X = X/np.amax(X, axis=0) 19 | y = y/100 #Max test score is 100 20 | 21 | ## ----------------------- Part 5 ---------------------------- ## 22 | 23 | class Neural_Network(object): 24 | def __init__(self): 25 | #Define Hyperparameters 26 | self.inputLayerSize = 2 27 | self.outputLayerSize = 1 28 | self.hiddenLayerSize = 3 29 | 30 | #Weights (parameters) 31 | self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize) 32 | self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize) 33 | 34 | def forward(self, X): 35 | #Propogate inputs though network 36 | self.z2 = np.dot(X, self.W1) 37 | self.a2 = self.sigmoid(self.z2) 38 | self.z3 = np.dot(self.a2, self.W2) 39 | yHat = self.sigmoid(self.z3) 40 | return yHat 41 | 42 | def sigmoid(self, z): 43 | #Apply sigmoid activation function to scalar, vector, or matrix 44 | return 1/(1+np.exp(-z)) 45 | 46 | def sigmoidPrime(self,z): 47 | #Gradient of sigmoid 48 | return np.exp(-z)/((1+np.exp(-z))**2) 49 | 50 | def costFunction(self, X, y): 51 | #Compute cost for given X,y, use weights already stored in class. 52 | self.yHat = self.forward(X) 53 | J = 0.5*sum((y-self.yHat)**2) 54 | return J 55 | 56 | def costFunctionPrime(self, X, y): 57 | #Compute derivative with respect to W and W2 for a given X and y: 58 | self.yHat = self.forward(X) 59 | 60 | delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3)) 61 | dJdW2 = np.dot(self.a2.T, delta3) 62 | 63 | delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2) 64 | dJdW1 = np.dot(X.T, delta2) 65 | 66 | return dJdW1, dJdW2 67 | 68 | #Helper Functions for interacting with other classes: 69 | def getParams(self): 70 | #Get W1 and W2 unrolled into vector: 71 | params = np.concatenate((self.W1.ravel(), self.W2.ravel())) 72 | return params 73 | 74 | def setParams(self, params): 75 | #Set W1 and W2 using single paramater vector. 76 | W1_start = 0 77 | W1_end = self.hiddenLayerSize * self.inputLayerSize 78 | self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize , self.hiddenLayerSize)) 79 | W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize 80 | self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize)) 81 | 82 | def computeGradients(self, X, y): 83 | dJdW1, dJdW2 = self.costFunctionPrime(X, y) 84 | return np.concatenate((dJdW1.ravel(), dJdW2.ravel())) 85 | 86 | def computeNumericalGradient(N, X, y): 87 | paramsInitial = N.getParams() 88 | numgrad = np.zeros(paramsInitial.shape) 89 | perturb = np.zeros(paramsInitial.shape) 90 | e = 1e-4 91 | 92 | for p in range(len(paramsInitial)): 93 | #Set perturbation vector 94 | perturb[p] = e 95 | N.setParams(paramsInitial + perturb) 96 | loss2 = N.costFunction(X, y) 97 | 98 | N.setParams(paramsInitial - perturb) 99 | loss1 = N.costFunction(X, y) 100 | 101 | #Compute Numerical Gradient 102 | numgrad[p] = (loss2 - loss1) / (2*e) 103 | 104 | #Return the value we changed to zero: 105 | perturb[p] = 0 106 | 107 | #Return Params to original value: 108 | N.setParams(paramsInitial) 109 | 110 | return numgrad 111 | 112 | -------------------------------------------------------------------------------- /partSix.py: -------------------------------------------------------------------------------- 1 | # Neural Networks Demystified 2 | # Part 6: Training 3 | # 4 | # Supporting code for short YouTube series on artificial neural networks. 5 | # 6 | # Stephen Welch 7 | # @stephencwelch 8 | 9 | 10 | ## ----------------------- Part 1 ---------------------------- ## 11 | import numpy as np 12 | 13 | # X = (hours sleeping, hours studying), y = Score on test 14 | X = np.array(([3,5], [5,1], [10,2]), dtype=float) 15 | y = np.array(([75], [82], [93]), dtype=float) 16 | 17 | # Normalize 18 | X = X/np.amax(X, axis=0) 19 | y = y/100 #Max test score is 100 20 | 21 | ## ----------------------- Part 5 ---------------------------- ## 22 | 23 | class Neural_Network(object): 24 | def __init__(self): 25 | #Define Hyperparameters 26 | self.inputLayerSize = 2 27 | self.outputLayerSize = 1 28 | self.hiddenLayerSize = 3 29 | 30 | #Weights (parameters) 31 | self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize) 32 | self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize) 33 | 34 | def forward(self, X): 35 | #Propogate inputs though network 36 | self.z2 = np.dot(X, self.W1) 37 | self.a2 = self.sigmoid(self.z2) 38 | self.z3 = np.dot(self.a2, self.W2) 39 | yHat = self.sigmoid(self.z3) 40 | return yHat 41 | 42 | def sigmoid(self, z): 43 | #Apply sigmoid activation function to scalar, vector, or matrix 44 | return 1/(1+np.exp(-z)) 45 | 46 | def sigmoidPrime(self,z): 47 | #Gradient of sigmoid 48 | return np.exp(-z)/((1+np.exp(-z))**2) 49 | 50 | def costFunction(self, X, y): 51 | #Compute cost for given X,y, use weights already stored in class. 52 | self.yHat = self.forward(X) 53 | J = 0.5*sum((y-self.yHat)**2) 54 | return J 55 | 56 | def costFunctionPrime(self, X, y): 57 | #Compute derivative with respect to W and W2 for a given X and y: 58 | self.yHat = self.forward(X) 59 | 60 | delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3)) 61 | dJdW2 = np.dot(self.a2.T, delta3) 62 | 63 | delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2) 64 | dJdW1 = np.dot(X.T, delta2) 65 | 66 | return dJdW1, dJdW2 67 | 68 | #Helper Functions for interacting with other classes: 69 | def getParams(self): 70 | #Get W1 and W2 unrolled into vector: 71 | params = np.concatenate((self.W1.ravel(), self.W2.ravel())) 72 | return params 73 | 74 | def setParams(self, params): 75 | #Set W1 and W2 using single paramater vector. 76 | W1_start = 0 77 | W1_end = self.hiddenLayerSize * self.inputLayerSize 78 | self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize , self.hiddenLayerSize)) 79 | W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize 80 | self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize)) 81 | 82 | def computeGradients(self, X, y): 83 | dJdW1, dJdW2 = self.costFunctionPrime(X, y) 84 | return np.concatenate((dJdW1.ravel(), dJdW2.ravel())) 85 | 86 | def computeNumericalGradient(N, X, y): 87 | paramsInitial = N.getParams() 88 | numgrad = np.zeros(paramsInitial.shape) 89 | perturb = np.zeros(paramsInitial.shape) 90 | e = 1e-4 91 | 92 | for p in range(len(paramsInitial)): 93 | #Set perturbation vector 94 | perturb[p] = e 95 | N.setParams(paramsInitial + perturb) 96 | loss2 = N.costFunction(X, y) 97 | 98 | N.setParams(paramsInitial - perturb) 99 | loss1 = N.costFunction(X, y) 100 | 101 | #Compute Numerical Gradient 102 | numgrad[p] = (loss2 - loss1) / (2*e) 103 | 104 | #Return the value we changed to zero: 105 | perturb[p] = 0 106 | 107 | #Return Params to original value: 108 | N.setParams(paramsInitial) 109 | 110 | return numgrad 111 | 112 | ## ----------------------- Part 6 ---------------------------- ## 113 | from scipy import optimize 114 | 115 | 116 | class trainer(object): 117 | def __init__(self, N): 118 | #Make Local reference to network: 119 | self.N = N 120 | 121 | def callbackF(self, params): 122 | self.N.setParams(params) 123 | self.J.append(self.N.costFunction(self.X, self.y)) 124 | 125 | def costFunctionWrapper(self, params, X, y): 126 | self.N.setParams(params) 127 | cost = self.N.costFunction(X, y) 128 | grad = self.N.computeGradients(X,y) 129 | return cost, grad 130 | 131 | def train(self, X, y): 132 | #Make an internal variable for the callback function: 133 | self.X = X 134 | self.y = y 135 | 136 | #Make empty list to store costs: 137 | self.J = [] 138 | 139 | params0 = self.N.getParams() 140 | 141 | options = {'maxiter': 200, 'disp' : True} 142 | _res = optimize.minimize(self.costFunctionWrapper, params0, jac=True, method='BFGS', \ 143 | args=(X, y), options=options, callback=self.callbackF) 144 | 145 | self.N.setParams(_res.x) 146 | self.optimizationResults = _res 147 | 148 | -------------------------------------------------------------------------------- /Part 5 Numerical Gradient Checking.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 5: Numerical Gradient Checking

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 12, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 12, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('pHMzNW8Agq4')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "Last time, we did a bunch of calculus to find the rate of change of our cost, J, with respect to our parameters, W. Although each calculus step was pretty straight forward, it’s still easy to make mistakes. What’s worse, is that our network doesn’t have a good way to tell us that it’s broken – code with incorrectly implemented gradients may appear to be functioning just fine." 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "This is the most nefarious kind of error when building complex systems. Big, in-your-face errors suck initially, but it’s clear that you must fix this error for your work to succeed. More subtle errors can be more troublesome because they hide in your code and steal hours of your time, slowly degrading performance, while you wonder what the problem is. " 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "A good solution here is to test the gradient computation part of our code, just as developer would unit test new portions of their code. We’ll combine a simple understanding of the derivative with some mild cleverness to perform numerical gradient checking. If our code passes this test, we can be quite confident that we have computed and coded up our gradients correctly. " 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "To get started, let’s quickly review derivatives. Derivates tell us the slope, or how steep a function is. Once you’re familiar with calculus, it’s easy to take for granted the inner workings of the derivative - we just accept that the derivative of x^2 is 2x by the power rule. However, depending on how mean your calculus teacher was, you may have spent months not being taught the power rule, and instead required to compute derivatives using the definition. Taking derivatives this way is a bit tedious, but still important - it provides us a deeper understanding of what a derivative is, and it’s going to help us solve our current problem. " 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "The definition of the derivative is really a glorified slope formula. The numerator gives us the change in y values, while the denominator is convenient way to express the change in x values. By including the limit, we are applying the slope formula across an infinitely small region – it’s like zooming in on our function, until it becomes linear. \n", 81 | "\n" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "The definition tells us to zoom in until our x distance is infinitely small, but computers can’t really handle infinitely small numbers, especially when they’re in the bottom parts of fractions - if we try to plug in something too small, we will quickly lose precision. The good news here is that if we plug in something reasonable small, we can still get surprisingly good numerical estimates of the derivative. " 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "We’ll modify our approach slightly by picking a point in the middle of the interval we would like to test, and call the distance we move in each direction epsilon. " 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "Let’s test our method with a simple function, x squared. We’ll choose a reasonable small value for epsilon, and compute the slope of x^2 at a given point by finding the function value just above and just below our test point. We can then compare our result to our symbolic derivative 2x, at the test point. If the numbers match, we’re in business!" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 1, 108 | "metadata": { 109 | "collapsed": false 110 | }, 111 | "outputs": [ 112 | { 113 | "name": "stdout", 114 | "output_type": "stream", 115 | "text": [ 116 | "Populating the interactive namespace from numpy and matplotlib\n" 117 | ] 118 | } 119 | ], 120 | "source": [ 121 | "%pylab inline\n", 122 | "#Import Code from previous videos:\n", 123 | "from partFour import *" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 2, 129 | "metadata": { 130 | "collapsed": false 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "def f(x):\n", 135 | " return x**2" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 3, 141 | "metadata": { 142 | "collapsed": false 143 | }, 144 | "outputs": [], 145 | "source": [ 146 | "epsilon = 1e-4\n", 147 | "x = 1.5" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 4, 153 | "metadata": { 154 | "collapsed": false 155 | }, 156 | "outputs": [], 157 | "source": [ 158 | "numericalGradient = (f(x+epsilon)- f(x-epsilon))/(2*epsilon)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 5, 164 | "metadata": { 165 | "collapsed": false 166 | }, 167 | "outputs": [ 168 | { 169 | "data": { 170 | "text/plain": [ 171 | "(2.9999999999996696, 3.0)" 172 | ] 173 | }, 174 | "execution_count": 5, 175 | "metadata": {}, 176 | "output_type": "execute_result" 177 | } 178 | ], 179 | "source": [ 180 | "numericalGradient, 2*x" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "Add helper functions to our neural network class: " 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 6, 193 | "metadata": { 194 | "collapsed": false 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "class Neural_Network(object):\n", 199 | " def __init__(self): \n", 200 | " #Define Hyperparameters\n", 201 | " self.inputLayerSize = 2\n", 202 | " self.outputLayerSize = 1\n", 203 | " self.hiddenLayerSize = 3\n", 204 | " \n", 205 | " #Weights (parameters)\n", 206 | " self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)\n", 207 | " self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)\n", 208 | " \n", 209 | " def forward(self, X):\n", 210 | " #Propogate inputs though network\n", 211 | " self.z2 = np.dot(X, self.W1)\n", 212 | " self.a2 = self.sigmoid(self.z2)\n", 213 | " self.z3 = np.dot(self.a2, self.W2)\n", 214 | " yHat = self.sigmoid(self.z3) \n", 215 | " return yHat\n", 216 | " \n", 217 | " def sigmoid(self, z):\n", 218 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 219 | " return 1/(1+np.exp(-z))\n", 220 | " \n", 221 | " def sigmoidPrime(self,z):\n", 222 | " #Gradient of sigmoid\n", 223 | " return np.exp(-z)/((1+np.exp(-z))**2)\n", 224 | " \n", 225 | " def costFunction(self, X, y):\n", 226 | " #Compute cost for given X,y, use weights already stored in class.\n", 227 | " self.yHat = self.forward(X)\n", 228 | " J = 0.5*sum((y-self.yHat)**2)\n", 229 | " return J\n", 230 | " \n", 231 | " def costFunctionPrime(self, X, y):\n", 232 | " #Compute derivative with respect to W and W2 for a given X and y:\n", 233 | " self.yHat = self.forward(X)\n", 234 | " \n", 235 | " delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n", 236 | " dJdW2 = np.dot(self.a2.T, delta3)\n", 237 | " \n", 238 | " delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)\n", 239 | " dJdW1 = np.dot(X.T, delta2) \n", 240 | " \n", 241 | " return dJdW1, dJdW2\n", 242 | " \n", 243 | " #Helper Functions for interacting with other classes:\n", 244 | " def getParams(self):\n", 245 | " #Get W1 and W2 unrolled into vector:\n", 246 | " params = np.concatenate((self.W1.ravel(), self.W2.ravel()))\n", 247 | " return params\n", 248 | " \n", 249 | " def setParams(self, params):\n", 250 | " #Set W1 and W2 using single paramater vector.\n", 251 | " W1_start = 0\n", 252 | " W1_end = self.hiddenLayerSize * self.inputLayerSize\n", 253 | " self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize , self.hiddenLayerSize))\n", 254 | " W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize\n", 255 | " self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize))\n", 256 | " \n", 257 | " def computeGradients(self, X, y):\n", 258 | " dJdW1, dJdW2 = self.costFunctionPrime(X, y)\n", 259 | " return np.concatenate((dJdW1.ravel(), dJdW2.ravel()))\n", 260 | " " 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": {}, 266 | "source": [ 267 | "We can use the same approach to numerically evaluate the gradient of our neural network. It’s a little more complicated this time, since we have 9 gradient values, and we’re interested in the gradient of our cost function. We’ll make things simpler by testing one gradient at a time. We’ll “perturb” each weight - adding epsilon to the current value and computing the cost function, subtracting epsilon from the current value and computing the cost function, and then computing the slope between these two values. " 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 7, 273 | "metadata": { 274 | "collapsed": false 275 | }, 276 | "outputs": [], 277 | "source": [ 278 | "def computeNumericalGradient(N, X, y):\n", 279 | " paramsInitial = N.getParams()\n", 280 | " numgrad = np.zeros(paramsInitial.shape)\n", 281 | " perturb = np.zeros(paramsInitial.shape)\n", 282 | " e = 1e-4\n", 283 | "\n", 284 | " for p in range(len(paramsInitial)):\n", 285 | " #Set perturbation vector\n", 286 | " perturb[p] = e\n", 287 | " N.setParams(paramsInitial + perturb)\n", 288 | " loss2 = N.costFunction(X, y)\n", 289 | " \n", 290 | " N.setParams(paramsInitial - perturb)\n", 291 | " loss1 = N.costFunction(X, y)\n", 292 | "\n", 293 | " #Compute Numerical Gradient\n", 294 | " numgrad[p] = (loss2 - loss1) / (2*e)\n", 295 | "\n", 296 | " #Return the value we changed to zero:\n", 297 | " perturb[p] = 0\n", 298 | " \n", 299 | " #Return Params to original value:\n", 300 | " N.setParams(paramsInitial)\n", 301 | "\n", 302 | " return numgrad " 303 | ] 304 | }, 305 | { 306 | "cell_type": "markdown", 307 | "metadata": {}, 308 | "source": [ 309 | "We’ll repeat this process across all our weights, and when we’re done we’ll have a numerical gradient vector, with the same number of values as we have weights. It’s this vector we would like to compare to our official gradient calculation. We see that our vectors appear very similar, which is a good sign, but we need to quantify just how similar they are. " 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": 8, 315 | "metadata": { 316 | "collapsed": false 317 | }, 318 | "outputs": [], 319 | "source": [ 320 | "NN = Neural_Network()" 321 | ] 322 | }, 323 | { 324 | "cell_type": "code", 325 | "execution_count": 9, 326 | "metadata": { 327 | "collapsed": false 328 | }, 329 | "outputs": [ 330 | { 331 | "data": { 332 | "text/plain": [ 333 | "array([ 0.04889799, -0.06551702, 0.02013916, 0.04615717, -0.05295928,\n", 334 | " 0.01661075, -0.25285186, -0.13721988, -0.10629924])" 335 | ] 336 | }, 337 | "execution_count": 9, 338 | "metadata": {}, 339 | "output_type": "execute_result" 340 | } 341 | ], 342 | "source": [ 343 | "numgrad = computeNumericalGradient(NN, X, y)\n", 344 | "numgrad" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 10, 350 | "metadata": { 351 | "collapsed": false 352 | }, 353 | "outputs": [ 354 | { 355 | "data": { 356 | "text/plain": [ 357 | "array([ 0.04889799, -0.06551702, 0.02013916, 0.04615717, -0.05295928,\n", 358 | " 0.01661075, -0.25285186, -0.13721988, -0.10629924])" 359 | ] 360 | }, 361 | "execution_count": 10, 362 | "metadata": {}, 363 | "output_type": "execute_result" 364 | } 365 | ], 366 | "source": [ 367 | "grad = NN.computeGradients(X,y)\n", 368 | "grad" 369 | ] 370 | }, 371 | { 372 | "cell_type": "markdown", 373 | "metadata": {}, 374 | "source": [ 375 | "A nice way to do this is to divide the norm of the difference by the norm of the sum of the vectors we would like to compare. Typical results should be on the order of 10^-8 or less if you’ve computed your gradient correctly. " 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "execution_count": 11, 381 | "metadata": { 382 | "collapsed": false 383 | }, 384 | "outputs": [ 385 | { 386 | "data": { 387 | "text/plain": [ 388 | "2.0134439421141014e-10" 389 | ] 390 | }, 391 | "execution_count": 11, 392 | "metadata": {}, 393 | "output_type": "execute_result" 394 | } 395 | ], 396 | "source": [ 397 | "norm(grad-numgrad)/norm(grad+numgrad)" 398 | ] 399 | }, 400 | { 401 | "cell_type": "markdown", 402 | "metadata": {}, 403 | "source": [ 404 | "And that’s it, we can now check our computations and eliminate gradient errors before they become a problem. Next time we’ll train our Neural Network. " 405 | ] 406 | } 407 | ], 408 | "metadata": { 409 | "kernelspec": { 410 | "display_name": "Python 2", 411 | "language": "python", 412 | "name": "python2" 413 | }, 414 | "language_info": { 415 | "codemirror_mode": { 416 | "name": "ipython", 417 | "version": 2 418 | }, 419 | "file_extension": ".py", 420 | "mimetype": "text/x-python", 421 | "name": "python", 422 | "nbconvert_exporter": "python", 423 | "pygments_lexer": "ipython2", 424 | "version": "2.7.10" 425 | } 426 | }, 427 | "nbformat": 4, 428 | "nbformat_minor": 0 429 | } 430 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Part 5 Numerical Gradient Checking-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 5: Numerical Gradient Checking

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 12, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 12, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('pHMzNW8Agq4')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "Last time, we did a bunch of calculus to find the rate of change of our cost, J, with respect to our parameters, W. Although each calculus step was pretty straight forward, it’s still easy to make mistakes. What’s worse, is that our network doesn’t have a good way to tell us that it’s broken – code with incorrectly implemented gradients may appear to be functioning just fine." 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "This is the most nefarious kind of error when building complex systems. Big, in-your-face errors suck initially, but it’s clear that you must fix this error for your work to succeed. More subtle errors can be more troublesome because they hide in your code and steal hours of your time, slowly degrading performance, while you wonder what the problem is. " 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "A good solution here is to test the gradient computation part of our code, just as developer would unit test new portions of their code. We’ll combine a simple understanding of the derivative with some mild cleverness to perform numerical gradient checking. If our code passes this test, we can be quite confident that we have computed and coded up our gradients correctly. " 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "To get started, let’s quickly review derivatives. Derivates tell us the slope, or how steep a function is. Once you’re familiar with calculus, it’s easy to take for granted the inner workings of the derivative - we just accept that the derivative of x^2 is 2x by the power rule. However, depending on how mean your calculus teacher was, you may have spent months not being taught the power rule, and instead required to compute derivatives using the definition. Taking derivatives this way is a bit tedious, but still important - it provides us a deeper understanding of what a derivative is, and it’s going to help us solve our current problem. " 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "The definition of the derivative is really a glorified slope formula. The numerator gives us the change in y values, while the denominator is convenient way to express the change in x values. By including the limit, we are applying the slope formula across an infinitely small region – it’s like zooming in on our function, until it becomes linear. \n", 81 | "\n" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "The definition tells us to zoom in until our x distance is infinitely small, but computers can’t really handle infinitely small numbers, especially when they’re in the bottom parts of fractions - if we try to plug in something too small, we will quickly lose precision. The good news here is that if we plug in something reasonable small, we can still get surprisingly good numerical estimates of the derivative. " 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "We’ll modify our approach slightly by picking a point in the middle of the interval we would like to test, and call the distance we move in each direction epsilon. " 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "Let’s test our method with a simple function, x squared. We’ll choose a reasonable small value for epsilon, and compute the slope of x^2 at a given point by finding the function value just above and just below our test point. We can then compare our result to our symbolic derivative 2x, at the test point. If the numbers match, we’re in business!" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 1, 108 | "metadata": { 109 | "collapsed": false 110 | }, 111 | "outputs": [ 112 | { 113 | "name": "stdout", 114 | "output_type": "stream", 115 | "text": [ 116 | "Populating the interactive namespace from numpy and matplotlib\n" 117 | ] 118 | } 119 | ], 120 | "source": [ 121 | "%pylab inline\n", 122 | "#Import Code from previous videos:\n", 123 | "from partFour import *" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 2, 129 | "metadata": { 130 | "collapsed": false 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "def f(x):\n", 135 | " return x**2" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 3, 141 | "metadata": { 142 | "collapsed": false 143 | }, 144 | "outputs": [], 145 | "source": [ 146 | "epsilon = 1e-4\n", 147 | "x = 1.5" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 4, 153 | "metadata": { 154 | "collapsed": false 155 | }, 156 | "outputs": [], 157 | "source": [ 158 | "numericalGradient = (f(x+epsilon)- f(x-epsilon))/(2*epsilon)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 5, 164 | "metadata": { 165 | "collapsed": false 166 | }, 167 | "outputs": [ 168 | { 169 | "data": { 170 | "text/plain": [ 171 | "(2.9999999999996696, 3.0)" 172 | ] 173 | }, 174 | "execution_count": 5, 175 | "metadata": {}, 176 | "output_type": "execute_result" 177 | } 178 | ], 179 | "source": [ 180 | "numericalGradient, 2*x" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "Add helper functions to our neural network class: " 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 6, 193 | "metadata": { 194 | "collapsed": false 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "class Neural_Network(object):\n", 199 | " def __init__(self): \n", 200 | " #Define Hyperparameters\n", 201 | " self.inputLayerSize = 2\n", 202 | " self.outputLayerSize = 1\n", 203 | " self.hiddenLayerSize = 3\n", 204 | " \n", 205 | " #Weights (parameters)\n", 206 | " self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)\n", 207 | " self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)\n", 208 | " \n", 209 | " def forward(self, X):\n", 210 | " #Propogate inputs though network\n", 211 | " self.z2 = np.dot(X, self.W1)\n", 212 | " self.a2 = self.sigmoid(self.z2)\n", 213 | " self.z3 = np.dot(self.a2, self.W2)\n", 214 | " yHat = self.sigmoid(self.z3) \n", 215 | " return yHat\n", 216 | " \n", 217 | " def sigmoid(self, z):\n", 218 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 219 | " return 1/(1+np.exp(-z))\n", 220 | " \n", 221 | " def sigmoidPrime(self,z):\n", 222 | " #Gradient of sigmoid\n", 223 | " return np.exp(-z)/((1+np.exp(-z))**2)\n", 224 | " \n", 225 | " def costFunction(self, X, y):\n", 226 | " #Compute cost for given X,y, use weights already stored in class.\n", 227 | " self.yHat = self.forward(X)\n", 228 | " J = 0.5*sum((y-self.yHat)**2)\n", 229 | " return J\n", 230 | " \n", 231 | " def costFunctionPrime(self, X, y):\n", 232 | " #Compute derivative with respect to W and W2 for a given X and y:\n", 233 | " self.yHat = self.forward(X)\n", 234 | " \n", 235 | " delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n", 236 | " dJdW2 = np.dot(self.a2.T, delta3)\n", 237 | " \n", 238 | " delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)\n", 239 | " dJdW1 = np.dot(X.T, delta2) \n", 240 | " \n", 241 | " return dJdW1, dJdW2\n", 242 | " \n", 243 | " #Helper Functions for interacting with other classes:\n", 244 | " def getParams(self):\n", 245 | " #Get W1 and W2 unrolled into vector:\n", 246 | " params = np.concatenate((self.W1.ravel(), self.W2.ravel()))\n", 247 | " return params\n", 248 | " \n", 249 | " def setParams(self, params):\n", 250 | " #Set W1 and W2 using single paramater vector.\n", 251 | " W1_start = 0\n", 252 | " W1_end = self.hiddenLayerSize * self.inputLayerSize\n", 253 | " self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize , self.hiddenLayerSize))\n", 254 | " W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize\n", 255 | " self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize))\n", 256 | " \n", 257 | " def computeGradients(self, X, y):\n", 258 | " dJdW1, dJdW2 = self.costFunctionPrime(X, y)\n", 259 | " return np.concatenate((dJdW1.ravel(), dJdW2.ravel()))\n", 260 | " " 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": {}, 266 | "source": [ 267 | "We can use the same approach to numerically evaluate the gradient of our neural network. It’s a little more complicated this time, since we have 9 gradient values, and we’re interested in the gradient of our cost function. We’ll make things simpler by testing one gradient at a time. We’ll “perturb” each weight - adding epsilon to the current value and computing the cost function, subtracting epsilon from the current value and computing the cost function, and then computing the slope between these two values. " 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 7, 273 | "metadata": { 274 | "collapsed": false 275 | }, 276 | "outputs": [], 277 | "source": [ 278 | "def computeNumericalGradient(N, X, y):\n", 279 | " paramsInitial = N.getParams()\n", 280 | " numgrad = np.zeros(paramsInitial.shape)\n", 281 | " perturb = np.zeros(paramsInitial.shape)\n", 282 | " e = 1e-4\n", 283 | "\n", 284 | " for p in range(len(paramsInitial)):\n", 285 | " #Set perturbation vector\n", 286 | " perturb[p] = e\n", 287 | " N.setParams(paramsInitial + perturb)\n", 288 | " loss2 = N.costFunction(X, y)\n", 289 | " \n", 290 | " N.setParams(paramsInitial - perturb)\n", 291 | " loss1 = N.costFunction(X, y)\n", 292 | "\n", 293 | " #Compute Numerical Gradient\n", 294 | " numgrad[p] = (loss2 - loss1) / (2*e)\n", 295 | "\n", 296 | " #Return the value we changed to zero:\n", 297 | " perturb[p] = 0\n", 298 | " \n", 299 | " #Return Params to original value:\n", 300 | " N.setParams(paramsInitial)\n", 301 | "\n", 302 | " return numgrad " 303 | ] 304 | }, 305 | { 306 | "cell_type": "markdown", 307 | "metadata": {}, 308 | "source": [ 309 | "We’ll repeat this process across all our weights, and when we’re done we’ll have a numerical gradient vector, with the same number of values as we have weights. It’s this vector we would like to compare to our official gradient calculation. We see that our vectors appear very similar, which is a good sign, but we need to quantify just how similar they are. " 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": 8, 315 | "metadata": { 316 | "collapsed": false 317 | }, 318 | "outputs": [], 319 | "source": [ 320 | "NN = Neural_Network()" 321 | ] 322 | }, 323 | { 324 | "cell_type": "code", 325 | "execution_count": 9, 326 | "metadata": { 327 | "collapsed": false 328 | }, 329 | "outputs": [ 330 | { 331 | "data": { 332 | "text/plain": [ 333 | "array([ 0.04889799, -0.06551702, 0.02013916, 0.04615717, -0.05295928,\n", 334 | " 0.01661075, -0.25285186, -0.13721988, -0.10629924])" 335 | ] 336 | }, 337 | "execution_count": 9, 338 | "metadata": {}, 339 | "output_type": "execute_result" 340 | } 341 | ], 342 | "source": [ 343 | "numgrad = computeNumericalGradient(NN, X, y)\n", 344 | "numgrad" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 10, 350 | "metadata": { 351 | "collapsed": false 352 | }, 353 | "outputs": [ 354 | { 355 | "data": { 356 | "text/plain": [ 357 | "array([ 0.04889799, -0.06551702, 0.02013916, 0.04615717, -0.05295928,\n", 358 | " 0.01661075, -0.25285186, -0.13721988, -0.10629924])" 359 | ] 360 | }, 361 | "execution_count": 10, 362 | "metadata": {}, 363 | "output_type": "execute_result" 364 | } 365 | ], 366 | "source": [ 367 | "grad = NN.computeGradients(X,y)\n", 368 | "grad" 369 | ] 370 | }, 371 | { 372 | "cell_type": "markdown", 373 | "metadata": {}, 374 | "source": [ 375 | "A nice way to do this is to divide the norm of the difference by the norm of the sum of the vectors we would like to compare. Typical results should be on the order of 10^-8 or less if you’ve computed your gradient correctly. " 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "execution_count": 11, 381 | "metadata": { 382 | "collapsed": false 383 | }, 384 | "outputs": [ 385 | { 386 | "data": { 387 | "text/plain": [ 388 | "2.0134439421141014e-10" 389 | ] 390 | }, 391 | "execution_count": 11, 392 | "metadata": {}, 393 | "output_type": "execute_result" 394 | } 395 | ], 396 | "source": [ 397 | "norm(grad-numgrad)/norm(grad+numgrad)" 398 | ] 399 | }, 400 | { 401 | "cell_type": "markdown", 402 | "metadata": {}, 403 | "source": [ 404 | "And that’s it, we can now check our computations and eliminate gradient errors before they become a problem. Next time we’ll train our Neural Network. " 405 | ] 406 | } 407 | ], 408 | "metadata": { 409 | "kernelspec": { 410 | "display_name": "Python 2", 411 | "language": "python", 412 | "name": "python2" 413 | }, 414 | "language_info": { 415 | "codemirror_mode": { 416 | "name": "ipython", 417 | "version": 2 418 | }, 419 | "file_extension": ".py", 420 | "mimetype": "text/x-python", 421 | "name": "python", 422 | "nbconvert_exporter": "python", 423 | "pygments_lexer": "ipython2", 424 | "version": "2.7.10" 425 | } 426 | }, 427 | "nbformat": 4, 428 | "nbformat_minor": 0 429 | } 430 | -------------------------------------------------------------------------------- /Part 2 Forward Propagation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 2: Forward Propagation

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 1, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('UJwK6jAStmg')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "

Variables

\n", 53 | "\n", 54 | "|Code Symbol | Math Symbol | Definition | Dimensions\n", 55 | "| :-: | :-: | :-: | :-: |\n", 56 | "|X|$$X$$|Input Data, each row in an example| (numExamples, inputLayerSize)|\n", 57 | "|y |$$y$$|target data|(numExamples, outputLayerSize)|\n", 58 | "|W1 | $$W^{(1)}$$ | Layer 1 weights | (inputLayerSize, hiddenLayerSize) |\n", 59 | "|W2 | $$W^{(2)}$$ | Layer 2 weights | (hiddenLayerSize, outputLayerSize) |\n", 60 | "|z2 | $$z^{(2)}$$ | Layer 2 activation | (numExamples, hiddenLayerSize) |\n", 61 | "|a2 | $$a^{(2)}$$ | Layer 2 activity | (numExamples, hiddenLayerSize) |\n", 62 | "|z3 | $$z^{(3)}$$ | Layer 3 activation | (numExamples, outputLayerSize) |" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "Last time, we setup our neural network on paper. This time, we’ll implement it in the programming language python. We’ll build our network as a python class and our init method will take care of instantiating important constants and variables. We’ll make these values accessible to the whole class by placing a self dot in front of each variable name." 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "Our network has 2 inputs, 3 hidden units, and 1 output. These are examples of hyperparameters. Hyperparameters are constants that establish the structure and behavior of a neural network, but are not updated as we train the network. Our learning algorithm is not capable of, for example, deciding that it needs another hidden unit, this is something that WE must decide on before training. What a neural network does learn are parameters, specifically the weights on the synapses." 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "We’ll take care of moving data through our network in a method called forward. Rather than pass inputs through the network one at a time, we’re going to use matrices to pass through multiple inputs at once. Doing this allows for big computational speedups, especially when using tools like MATLAB or Numpy. Our input data matrix, X, is of dimension 3 by 2, because we have 3, 2-dimensional examples. Our corresponding output data, y, is of dimension 3 by 1." 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 1, 89 | "metadata": { 90 | "collapsed": false 91 | }, 92 | "outputs": [ 93 | { 94 | "name": "stdout", 95 | "output_type": "stream", 96 | "text": [ 97 | "Populating the interactive namespace from numpy and matplotlib\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "#Import code from last time\n", 103 | "%pylab inline\n", 104 | "from partOne import *" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 2, 110 | "metadata": { 111 | "collapsed": false 112 | }, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "(3, 2) (3, 1)\n" 119 | ] 120 | } 121 | ], 122 | "source": [ 123 | "print X.shape, y.shape" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": { 130 | "collapsed": false 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "class Neural_Network(object):\n", 135 | " def __init__(self): \n", 136 | " #Define Hyperparameters\n", 137 | " self.inputLayerSize = 2\n", 138 | " self.outputLayerSize = 1\n", 139 | " self.hiddenLayerSize = 3\n", 140 | " \n", 141 | " def forward(self, X):\n", 142 | " #Propagate inputs though network" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": {}, 148 | "source": [ 149 | "Each input value, or element in matrix X, needs to be multiplied by a corresponding weight and then added together with all the other results for each neuron. This is a complex operation, but if we take the three outputs we're looking for as a single row of a matrix, and place all our individual weights into a matrix of weights, we can create the exact behavior we need by multiplying our input data matrix by our weight matrix. Using matrix multiplication allows us to pass multiple inputs through at once by simply adding rows to the matrix X. From here on out, we'll refer to these matrics as X, W one, and z two, where z two the activity of our second layer. Notice that each entry in z is a sum of weighted inputs to each hidden neuron. Z is of size 3 by 3, one row for each example, and one column for each hidden unit. " 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "We now have our first official formula, $z^{(2)} = XW^{(1)}$. Matrix notation is really nice here, becuase it allows us to express the complex underlying process in a single line!" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "$$\n", 164 | "z^{(2)} = XW^{(1)} \\tag{1}\\\\\n", 165 | "$$\n" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "Now that we have the activities for our second layer, z two, we need to apply the activation function. We'll independently apply the function to each entry in matrix z using a python method for this called sigmoid, because we’re using a sigmoid as our activation function. Using numpy is really nice here, because we can pass in a scalar, vector, or matrix, Numpy will apply the activation function element-wise, and return a result of the same dimension as it was given." 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 3, 178 | "metadata": { 179 | "collapsed": false 180 | }, 181 | "outputs": [], 182 | "source": [ 183 | "def sigmoid(z):\n", 184 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 185 | " return 1/(1+np.exp(-z))" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 4, 191 | "metadata": { 192 | "collapsed": false 193 | }, 194 | "outputs": [ 195 | { 196 | "data": { 197 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEACAYAAACuzv3DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFdW19/HvshlUBNtIxBtEMYAGHNIYxDGxNRqQSMyo\n8IoJiUaNEyZGAY2KQ9RcvYpTDElUTDRqrmJUrvPVjkaviuKEAoKAgjMqKkSgm17vH/tgN20Ph+46\np07t8/s8z3m6qk9xei1LFtWr9t5l7o6IiGTXBmkHICIiHaNCLiKScSrkIiIZp0IuIpJxKuQiIhmn\nQi4iknFtFnIzu9bM3jGzF1s55nIzm2dmz5vZ4GRDFBGR1uRzRX4dMLylN81sBNDf3QcARwFXJxSb\niIjkoc1C7u6PAh+2csh3gOtzxz4JVJpZr2TCExGRtiTRI+8NLG60vwTYKoHPFRGRPCR1s9Oa7Gve\nv4hIkXRK4DPeAPo02t8q9711mJmKu4hIO7h704vldSRRyO8EjgduNrPdgWXu/k4LwSTw40rTpEmT\nmDRpUtphFEzM+cWcG5RffvX18O678MYbsGTJul/feCO89957sHQp1NW1/+duuCF06xZeG2/c8mvt\n+xttFP5Mly7QtWv42ni7pa8DB7Zaw4E8CrmZ3QTsA/Q0s8XAWUBnAHef4u53m9kIM5sPrAB+2v7/\nNNm1aNGitEMoqJjzizk3iDO/FStgwQJ49VWYPn0R774L8+eH/cWLobY2v8/p3h169oQvfrHhtfnm\nUFkJm276+VePHg3bnTsXNsf10WYhd/fReRxzfDLhiIg0WLkS5syBl16CWbMaXk3/bXrmmXX3N98c\neveGrbZa92vv3tCrVyjYPXuGK+QYJNFaEWDs2LFph1BQMecXc26Qnfxqa0ORfuopmDEjfH35ZViz\n5vPHdu4M224L/fpB165j2WefsN2vH/TtG1oZ5cSK1bc2M4+5Ry4i62fFCnj8cXj4YfjnP2HmzHAF\n3tgGG8CAAbDjjuu++veHTmVyGWpmRbnZKUBNTQ3V1dVph1EwMecXc25QOvnV1cETT8C990JNTbji\nbtrLHjAAdt0Vhg4NX6uq2r66LpX80qRCLiIFs2wZ3Hcf3HUX3HMPfPBBw3sbbABDhkB1dXjtsQd8\n4QtpRZptaq2ISKI++QTuuANuugnuv3/dIX79+sFBB8H++8Pee4fRIdI6tVZEpCjq6uDuu+Gvf4Xp\n0xt63RtsEK62DzoovLbbDqztYdGynrQeeUJqamrSDqGgYs4v5tygsPm99hqceSZssw0cfDDcemso\n4l//Olx1Fbz1VriZefLJsP32hSnisZ+/fOiKXETWi3u4Wflf/xWuwtd2TAcMgCOPhNGjoU+fVj9C\nEqYeuYjkpa4uXHFffHHDBJyuXeEHP4CjjoJvfENtk0JQj1xEOqyuLvS+zzmnYUblF78IJ5wAv/hF\nmCEp6VKPPCGx9+lizi/m3KD9+dXXh5EngwbBz34Wivh228GUKaE3fsYZpVHEYz9/+dAVuYh8zkMP\nwUknwYu5J/X27w+TJsGoUVBRkWpo0gz1yEXkMwsXwq9/DdOmhf2ttw6jUn7849Ja7a+cqEcuInlZ\ntQrOPx9+97uw3a0bnHYa/OpX8awQGDP1yBMSe58u5vxizg3azu/xx8OaJuecE4r4mDEwd24o5Fko\n4rGfv3yokIuUqeXL4cQTw1T5OXPChJ1HHw0jVHr3Tjs6WR/qkYuUoWeeCRN35s0LNy/Hjw+jULJw\nBV5u1CMXkXXU18Mll4S2SW1tWNv7r38NrRXJLrVWEhJ7ny7m/GLODRryW7oUDjwQTjklFPHjjw9r\ngme9iMd+/vKhK3KRMjBzJnzve/D66+F5ltddByNHph2VJEU9cpHI3XhjWMxq5crw5J3bbgsPI5Zs\nyKdHrtaKSKTq60MbZcyYUMSPOAIeeURFPEYq5AmJvU8Xc34x5rZyJRx6aFipsKKihquvhj/9KaxW\nGJsYz9/6Uo9cJDIffBAe8vCvf0GPHnDWWXDMMWlHJYWkHrlIRF57DYYPDxN8ttoqPPhhp53Sjko6\nQuPIRcrIq6/CfvuFkSk77RSKuPrh5UE98oTE3qeLOb8Ycps7Nzyh5/XXYY891r2pGUN+rYk9v3yo\nkItk3EsvwT77wJtvhmJ+331QWZl2VFJM6pGLZNjs2aF4L10K++8Pd9wBG2+cdlSSJI0jF4nYwoWh\neC9dCsOGwV13qYiXKxXyhMTep4s5vyzm9tZboYivbadMm9byyoVZzG99xJ5fPlTIRTLm/ffhgANg\nwQIYMkRX4qIeuUimfPppGGL4xBPh6fb//GdpPMleCkc9cpGI1NfD4YeHIr711nD//SriEqiQJyT2\nPl3M+WUltwkTwsqFPXqEyT75Po4tK/m1V+z55UOFXCQDpkyBiy6CTp1CMd9hh7QjklLSZo/czIYD\nk4EK4M/u/rsm728K3AD0IUz5v9jdpzbzOeqRi7TDgw+G9VPWrIFrroGf/SztiKSY8umRt1rIzawC\nmAvsD7wBzABGu/vsRsecBnR394lm1jN3fC93r2vyWSrkIutp0SL42tfCioYTJsAFF6QdkRRbEjc7\nhwLz3X2Ru9cCNwMHNzmmHuiR2+4BvN+0iJeD2Pt0MedXqrn9+9/h8WwffAAjRsB557Xvc0o1v6TE\nnl8+2irkvYHFjfaX5L7X2JXAIDN7E3geGJdceCLlyR2OPhqeew769YMbboCKirSjklLV1jK2+fRC\nhgMz3X1fM+sHPGBmX3X3T5oeOHbsWPr27QtAZWUlVVVVVFdXAw3/qmZ1f+33SiUe5Zf/fnV1dUnF\nA3DiiTXccAN061bNP/4Bzz8fV36xn7+O7NfU1DB16lSAz+plW9rqke8OTHL34bn9iUB94xueZjYd\nuMDdH8vt/y8w3t2fbvJZ6pGL5OHJJ2HvvaGuDv7+d/jRj9KOSNKURI/8aWCAmfU1sy7AocCdTY55\nnXAzFDPrBWwPLGhfyNm19l/UWMWcXynltmwZjBoVivhJJyVTxEspv0KIPb98tNpacfc6MzseuI8w\n/PAad59tZkfn3p8CnAtMNbMXAANOdfcPChy3SHTc4aijGkaqXHhh2hFJVmitFZESMWVKeEhy9+4w\ncyb07592RFIKOjyOPOFgVMhFWvDiizB0KKxcCX/7G4wenXZEUiq0aFYRxd6nizm/tHNbtQoOOywU\n8SOOSL6Ip51focWeXz5UyEVSNmlSuCLv3x8uuyztaCSL1FoRSdHjj8PXvx62H30U9twz3Xik9Ki1\nIlLCVqyAn/wkrDN+yikq4tJ+KuQJib1PF3N+aeV26qkwfz7stBOcfXbhfk7M5w7izy8fKuQiKXjw\nQfj976FzZ/jrX6Fr17QjkixTj1ykyFasCFfhCxeGFQ1PPz3tiKSUqUcuUoLOOisU8a9+NbRXRDpK\nhTwhsffpYs6vmLk98wxceilssAH8+c+htVJoMZ87iD+/fKiQixRJbS0ceWQYpTJuHAwZknZEEgv1\nyEWK5D//E8aPh759YdYs6NYt7YgkC7TWikiJePVV2HHHMA3/3nth2LC0I5Ks0M3OIoq9TxdzfoXO\nzR2OPTYU8cMOK34Rj/ncQfz55UOFXKTAbr8d7r8fKivhkkvSjkZipNaKSAH9+98wcCC8/jpceSUc\nd1zaEUnWqLUikrILLwxFvKoqPDRCpBBUyBMSe58u5vwKldurr4aRKhCuxisqCvJj2hTzuYP488uH\nCrlIgZx0UnhoxI9/DHvtlXY0EjP1yEUKYPp0GDkSevSAuXNhyy3TjkiySj1ykRSsXBlmbkJYnlZF\nXApNhTwhsffpYs4v6dwuvxwWLIAddiiNUSoxnzuIP798qJCLJOjdd+G3vw3bl15anEWxRNQjF0nQ\nscfC1VfDgQfC3XenHY3EQGutiBTRyy/DzjuH7RdegEGD0o1H4qCbnUUUe58u5vySyu2UU2DNGjjq\nqNIq4jGfO4g/v3yokIsk4P77Qyule3eYNCntaKTcqLUi0kFr1sDgwfDii2FK/vjxaUckMVFrRaQI\nrrsuFPFttmkYPy5STCrkCYm9Txdzfh3JbcUKOOOMsH3hhbDhhsnElKSYzx3En18+VMhFOuCyy+Dt\nt2HXXeHQQ9OORsqVeuQi7fT++/DlL8PHH8P//i/st1/aEUmM1CMXKaALLwxF/IADVMQlXSrkCYm9\nTxdzfu3JbckSuOKKsH3BBcnGk7SYzx3En18+VMhF2uHss8Na44ccAl/7WtrRSLlrs0duZsOByUAF\n8Gd3/10zx1QDlwKdgaXuXt3MMeqRSxTmzAkrG5qFafnbbZd2RBKzfHrkndr4gArgSmB/4A1ghpnd\n6e6zGx1TCVwFDHP3JWbWs+Ohi5Su3/wG6uvDVHwVcSkFbbVWhgLz3X2Ru9cCNwMHNznm/wG3ufsS\nAHdfmnyYpS/2Pl3M+a1PbjNmwG23hfHiZ55ZuJiSFPO5g/jzy0dbhbw3sLjR/pLc9xobAHzBzB42\ns6fN7PAkAxQpFe4wYULYHjcOejf9myCSklZ75Gb2A2C4u/88tz8G2M3dT2h0zJXALsA3gY2B/wO+\n7e7zmnyWeuSSaQ88AN/6FlRWhicAbbZZ2hFJOehwj5zQF+/TaL8P4aq8scWEG5yfAp+a2SPAV4F5\nTY5j7Nix9O3bF4DKykqqqqqorq4GGn490r72S3H/4YdrcuuoVHPqqfD886UVn/bj2a+pqWHq1KkA\nn9XLNrl7iy9CoX8V6At0AZ4DBjY55ivAg4RRLRsDLwKDmvksj9nDDz+cdggFFXN++eR2993u4N6z\np/snnxQ+piTFfO7c488vVztbrdWtXpG7e52ZHQ/clyvU17j7bDM7Ovf+FHefY2b3Ai8A9cCf3P3l\n/P4ZESl97g03NsePh002STcekaa01opIG6ZPh5EjYYstQm+8W7e0I5JyorVWRDrIHc46K2xPmKAi\nLqVJhTwha29WxCrm/FrL7c47YeZM2HJLOOaY4sWUpJjPHcSfXz5UyEVa4N7w/M0JE2CjjVINR6RF\n6pGLtOD22+H734cvfQnmz1chl3SoRy7STvX1Db3xiRNVxKW0qZAnJPY+Xcz5NZfbtGnhgcq9e8OR\nRxY/piTFfO4g/vzyoUIu0kR9fUNv/PTTS/OByiKNqUcu0sQtt8CoUdCnD8ybB127ph2RlDP1yEXW\n05o14ek/ENYdVxGXLFAhT0jsfbqY82uc2y23wOzZsM02MHZsaiElKuZzB/Hnlw8VcpGcNWvgnHPC\n9m9+A126pBuPSL7UIxfJueEGOPxw2HZbmDsXOndOOyIR9chF8lZX19AbP+MMFXHJFhXyhMTep4s5\nv5qaGm68Mcze7NcvXJXHJOZzB/Hnlw8Vcil7dXVw7rlh+8wzoVNbz80SKTHqkUvZu/ZaOOIIGDAA\nXn5ZhVxKi3rkIm2orYXzzgvbZ52lIi7ZpEKekNj7dLHmd/31sHBhDV/5SpjNGaNYz91aseeXDxVy\nKVurV6/bG6+oSDcekfZSj1zK1pQp4ak/gwbBCy+okEtpyqdHrkIuZWnVqnBzc/HiMC3/kEPSjkik\nebrZWUSx9+liy++aa0IR33FH6NmzJu1wCiq2c9dU7PnlQ4Vcys7KlXD++WF70iTYQH8LJOPUWpGy\nc8UVcOKJsPPO8OyzKuRS2tQjF2ni00/DNPy33gqPc/ve99KOSKR16pEXUex9uljymzIlFPHBg+G7\n3w3fiyW3lii/+KmQS9lYsQIuuCBsn3MOWKvXOCLZodaKlI2LL4ZTToFdd4Unn1Qhl2xQj1wkZ/ny\n8MCIpUvhnntg+PC0IxLJj3rkRRR7ny7r+V15ZSjiu+8Ow4at+17Wc2uL8oufCrlE7+OP4aKLwrZ6\n4xIjtVYkeuedFx7ftvfe8MgjKuSSLeqRS9n76CPo2xeWLYOHHoJ99007IpH1ox55EcXep8tqfpMn\nhyJeXd1yEc9qbvlSfvFTIZdoffghXHJJ2D777HRjESkktVYkWmecEfrj++8PDzyQdjQi7ZNIa8XM\nhpvZHDObZ2bjWzluVzOrM7PvtydYkSS9/35oq4CuxiV+rRZyM6sArgSGA4OA0WY2sIXjfgfcC5Tl\nmIDY+3RZy++ii8IkoGHDYM89Wz82a7mtL+UXv7auyIcC8919kbvXAjcDBzdz3AnArcB7Cccnst7e\nfhsuvzxsn3NOurGIFEOrPXIz+yEwzN1/ntsfA+zm7ic0OqY3cAOwH3AtcJe7T2vms9Qjl6I4/ni4\n6qqwuuHtt6cdjUjHJNEjz6fyTgYm5Kq0UaatFSkNCxfCH/8YJv2ce27a0YgUR6c23n8D6NNovw+w\npMkxXwNutjBdridwoJnVuvudTT9s7Nix9O3bF4DKykqqqqqorq4GGvpcWd2fPHlyVPlkNb9rr62m\nthYOOKCGpUsB2v7zjXusacdfiH3ll639mpoapk6dCvBZvWxLW62VTsBc4JvAm8BTwGh3n93C8ddR\npq2Vmpqaz05KjLKQ36xZ4fFtnTrB3LlhtcN8ZCG3jlB+2ZbIFH0zO5DQPqkArnH3C8zsaAB3n9Lk\n2LIt5JK+734X7rgDjjsurHYoEgOttSJl48knwxK1G20ECxbAllumHZFIMrTWShE17tPFqNTzO+20\n8HXcuPUv4qWeW0cpv/ipkEvmPfhgWNmwshJOPTXtaESKT60VyTR32G03mDEDzj8fJk5MOyKRZKlH\nLtG75RYYNSq0U+bPh27d0o5IJFnqkRdR7H26Usxv1SqYMCFsn3NO+4t4KeaWJOUXPxVyyayrroJF\ni2DQIPjpT9OORiQ9aq1IJn3wAfTvHx4eMX06fPvbaUckUhhqrUi0fvvbUMT32w9GjEg7GpF0qZAn\nJPY+XSnlt2BBw8zNiy4KC2R1RCnlVgjKL34q5JI5p50Gq1fD4YfDLrukHY1I+tQjl0xZOxW/a1d4\n5RXYeuu0IxIpLPXIJSrucPLJYfuXv1QRF1lLhTwhsffpSiG/m26Cxx6DLbZoGD+ehFLIrZCUX/xU\nyCUTli+HU04J2xdcAJtumm48IqVEPXLJhNNPD2upDBkS+uQb6BJEyoTWWpEovPpqmL25ejU8/jjs\nsUfaEYkUj252FlHsfbo08zv55IbhhoUo4jp32RZ7fvlQIZeS9sAD4fFt3brBhRemHY1IaVJrRUrW\n6tVQVQWzZ4ciPn582hGJFJ9aK5JpF18civiAAXDSSWlHI1K6VMgTEnufrtj5LVgA554btn//+zCT\ns1B07rIt9vzyoUIuJccdjjsOVq6Eww6D/fdPOyKR0qYeuZSc//5vOOSQ8DDlOXOgV6+0IxJJj3rk\nkjkffwzjxoXtCy9UERfJhwp5QmLv0xUrv9/8Bt56K6xw+POfF+VH6txlXOz55UOFXErGY4+FB0ZU\nVMAf/qBp+CL5Uo9cSsKnn4Yx46+8EtZVOe+8tCMSKQ3qkUtmnHlmKOI77ABnnJF2NCLZokKekNj7\ndIXM78kn4ZJLQivluusKO2a8OTp32RZ7fvlQIZdUrVwJP/0p1NfDr38Nu+6adkQi2aMeuaRq4sQw\nzHD77eG552DDDdOOSKS0aD1yKWmPPALV1WAGjz4Ke+6ZdkQipUc3O4so9j5d0vktWxbWF3cPV+Vp\nFnGdu2yLPb98qJBLKo47Dl5/PfTEzzor7WhEsk2tFSm6G2+EMWPCwyKefTYsUysizVNrRUrOokVw\n7LFhe/JkFXGRJORVyM1suJnNMbN5Zva557SY2WFm9ryZvWBmj5nZzsmHWtpi79Mlkd+qVfCjH4WF\nsb73PTjiiI7HlQSdu2yLPb98tFnIzawCuBIYDgwCRpvZwCaHLQC+4e47A+cCf0w6UMm+X/0Knn4a\n+vaFa64Jo1VEpOPa7JGb2R7AWe4+PLc/AcDdm30UrpltBrzo7ls1+b565GXs5pth9Gjo0iUsjjVk\nSNoRiWRDUj3y3sDiRvtLct9ryRHA3Xl8rpSJ2bPhyCPD9uTJKuIiSeuUxzF5X0ab2b7Az4C9mnt/\n7Nix9O3bF4DKykqqqqqorq4GGvpcWd2fPHlyVPkkld8uu1Tzwx/CihU17LcfHHNMaeTTeL9xj7UU\n4lF+5Z1fTU0NU6dOBfisXrbJ3Vt9AbsD9zbanwiMb+a4nYH5QP8WPsdj9vDDD6cdQkG1J7+6OveR\nI93BfeBA908+ST6uJOjcZVvs+eVqZ6t1Op8eeSdgLvBN4E3gKWC0u89udMzWwEPAGHd/ooXP8bZ+\nlsRl7Toqm20GTz0F/funHZFI9uTTI2+zteLudWZ2PHAfUAFc4+6zzezo3PtTgDOBzYCrLQxFqHX3\noR1NQLLrxhtDEa+ogFtvVREXKaS8xpG7+z3uvr2793f3C3Lfm5Ir4rj7ke6+ubsPzr3Krog37tPF\naH3ye+qphjHil10G++1XmJiSonOXbbHnlw/N7JREzZsHBx0UJv8cfXTDLE4RKRyttSKJeecd2GMP\nWLgQhg2Du+6Czp3Tjkok27TWihTNJ5/AiBGhiA8ZEvriKuIixaFCnpDY+3St5bdqFfzgBzBzJvTr\nB//zP7DJJsWLraPK+dzFIPb88qFCLh1SWwuHHgoPPABbbAH33Re+ikjxqEcu7VZbG9ZPue02qKyE\nhx6CwYPTjkokLuqRS8HU1YVHtd12G2y6abgiVxEXSYcKeUJi79M1zq+2Fn7yE7jlFujePbRTsrwQ\nVjmduxjFnl8+8lk0S+Qzn34KhxwC06eHG5r33gu77ZZ2VCLlTT1yydtHH8F3vgOPPAJf+ALccw8M\nLbs5vCLFlchaKyIA774LBx4Yhhh+6UuhJz5oUNpRiQioR56YmPt0s2bBzjvXfDZO/LHH4iriMZ87\nUH7lQIVcWnXvvbDnnmH6/dCh8K9/hWduikjpUI9cmuUOV1wBv/wl1NeHG5xTp8JGG6UdmUh50Thy\naZfly2HMGBg3LhTxM8+Em25SERcpVSrkCYmlT/fyy6GF8re/QbduoYCffTY88khN2qEVTCznriXK\nL34q5AKEVsr118Ouu4an3g8aBDNmwKhRaUcmIm1Rj1x4773wEIjbbw/7Y8bAH/4QrshFJF3qkUub\n7rgDdtwxFPHu3eG66+Avf1ERF8kSFfKEZK1P9+abYfnZ7343TPbZd1948UUYOxasmX/7s5bf+og5\nN1B+5UCFvMysWQNXXgkDB8Lf/w4bbwyXXgoPPgjbbJN2dCLSHuqRl5GaGjj55DDNHmDkyDBWXAVc\npHSpRy5AGFI4cmRon8ycCVttFXrid9yhIi4SAxXyhJRin27BAjjySNhpp4ZlZ889F+bMCb3x5nrh\nLSnF/JISc26g/MqBVj+M0CuvwPnnww03hJ54RQUccwxMmgS9eqUdnYgkTT3ySLjD44/DZZeFx6/V\n14cCPmYMnHYabLdd2hGKSHtoPfIysHJleOTa5Zc33MTs3BmOOAImTIAvfznd+ESk8NQjT0gx+3Tu\noWifeGK4cTl2bNjv2RNOPx0WLoQ//jHZIh5zHzLm3ED5lQNdkWfI4sVh7Pf114fJO2sNHhyK+qhR\nsOGG6cUnIulQj7zEzZ8fet7TpsFTTzV8v2dPOOywcDVeVZVaeCJSYOqRZ9Cnn4an8Nx/f3g6z6xZ\nDe9tvHF4buaYMTBiBHTpkl6cIlI61CNPSHv7dKtXwxNPwMUXw/Dh4en03/pW2J81C3r0CFfe06aF\nVQpvvTWMAS92EY+5DxlzbqD8yoGuyIvsvfdCi+Sxx8KV94wZYeRJY1VVoZgPGwZ77QVdu6YTq4hk\ng3rkBVJfH0aPPPssPPdceD37bFh1sKmvfCUU7OpqOOAATdoRkQbqkRfBRx+FmZRz54bXnDnh67x5\nn7/ShjBNfvDg8GT6vfYKXzffvPhxi0g82izkZjYcmAxUAH929981c8zlwIHAv4Gx7v5s0oGmYfVq\neOedcBX92mvh9frr624vW7b26Bqgep0//x//EdokVVWheFdVQb9+sEEG70zU1NRQXV2ddhgFEXNu\noPzKQauF3MwqgCuB/YE3gBlmdqe7z250zAigv7sPMLPdgKuB3QsYc7vU18PHH4fC++GH4bVsGXzw\nQSjWzb0+/LDtz91oIxgwAMyeY+TIarbfHrbfPkyJ33TTwudVLM8991y0f1lizg2UXzlo64p8KDDf\n3RcBmNnNwMHA7EbHfAe4HsDdnzSzSjPr5e7vrG8wa9bAqlXhtXp1w/aqVaFNsXw5rFjR8Gptf/ny\ndQv2Rx+FYr4+Kipgiy3ClfU224TX1luvu92zZ1hFcNKkZUyatL4ZZ8eyhl89ohNzbqD8ykFbhbw3\nsLjR/hJgtzyO2Qr4XCEfMmTd4ty0YK9Z044M1kP37lBZCZttFl5rt3v1av61+ebZbIOISHlpq5Dn\nO8yk6R3VZv/cM8+08SEWhtp16RK+Nn5tuGF4IPAmm4Sva1+N95tuNy7am24KnQp4a3fRokWF+/AS\nEHN+MecGyq8ctDr80Mx2Bya5+/Dc/kSgvvENTzP7A1Dj7jfn9ucA+zRtrZhZ+Yw9FBFJUEeHHz4N\nDDCzvsCbwKHA6CbH3AkcD9ycK/zLmuuPtxWIiIi0T6uF3N3rzOx44D7C8MNr3H22mR2de3+Ku99t\nZiPMbD6wAvhpwaMWEZHPFG1mp4iIFEZRx2SY2QlmNtvMZpnZ5yYWxcDMTjazejP7QtqxJMnMLsqd\nu+fNbJqZRTFK3syGm9kcM5tnZuPTjidJZtbHzB42s5dyf+dOTDumpJlZhZk9a2Z3pR1L0nJDuW/N\n/b17Ode6blbRCrmZ7UsYc76zu+8IXFysn10sZtYHOAB4Le1YCuB+YAd3/yrwCjAx5Xg6rNGEt+HA\nIGC0mQ1MN6pE1QK/dPcdCJP0jossP4BxwMvkP8IuSy4D7nb3gcDOrDt/Zx3FvCL/BXCBu9cCuPt7\nRfzZxXIJcGraQRSCuz/g7munVD1JmCuQdZ9NeMv9f7l2wlsU3P1td38ut72cUAi+lG5UyTGzrYAR\nwJ/5/BDoTMv9xvt1d78Wwv1Kd/+opeOLWcgHAN8wsyfMrMbMhhTxZxecmR0MLHH3F9KOpQh+Btyd\ndhAJaG4EOyqhAAAB+klEQVQyW++UYimo3MizwYR/hGNxKXAKsJ5ztjNhW+A9M7vOzGaa2Z/MbOOW\nDk50ioyZPQBs2cxbp+d+1mbuvruZ7Qr8HcjUM97byG8i8K3GhxclqAS1kt9p7n5X7pjTgdXu/rei\nBlcYMf46/jlmtglwKzAud2WeeWZ2EPCuuz9rZtVpx1MAnYBdgOPdfYaZTQYmAGe2dHBi3P2Alt4z\ns18A03LHzcjdENzc3d9PMoZCaik/M9uR8C/o82YGoe3wjJkNdfd3ixhih7R2/gDMbCzhV9lvFiWg\nwnsD6NNovw/hqjwaZtYZuA24wd3/kXY8CdoT+E5u0b4NgR5m9hd3/3HKcSVlCeE3/Bm5/VsJhbxZ\nxWyt/APYD8DMtgO6ZKmIt8bdZ7l7L3ff1t23JZyEXbJUxNuSW874FOBgd29mpfVM+mzCm5l1IUx4\nuzPlmBJj4ariGuBld5+cdjxJcvfT3L1P7u/bKOChiIo47v42sDhXKyGsQPtSS8cX88ES1wLXmtmL\nwGogmv/ozYjxV/YrgC7AA7nfOv7P3Y9NN6SOaWnCW8phJWkvYAzwgpmtfUbARHe/N8WYCiXGv3Mn\nADfmLjJepZXJlpoQJCKScVqkVUQk41TIRUQyToVcRCTjVMhFRDJOhVxEJONUyEVEMk6FXEQk41TI\nRUQy7v8DtTV/D9zJEKcAAAAASUVORK5CYII=\n", 198 | "text/plain": [ 199 | "" 200 | ] 201 | }, 202 | "metadata": {}, 203 | "output_type": "display_data" 204 | } 205 | ], 206 | "source": [ 207 | "testInput = np.arange(-6,6,0.01)\n", 208 | "plot(testInput, sigmoid(testInput), linewidth= 2)\n", 209 | "grid(1)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 5, 215 | "metadata": { 216 | "collapsed": false 217 | }, 218 | "outputs": [ 219 | { 220 | "data": { 221 | "text/plain": [ 222 | "0.7310585786300049" 223 | ] 224 | }, 225 | "execution_count": 5, 226 | "metadata": {}, 227 | "output_type": "execute_result" 228 | } 229 | ], 230 | "source": [ 231 | "sigmoid(1)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 6, 237 | "metadata": { 238 | "collapsed": false 239 | }, 240 | "outputs": [ 241 | { 242 | "data": { 243 | "text/plain": [ 244 | "array([ 0.26894142, 0.5 , 0.73105858])" 245 | ] 246 | }, 247 | "execution_count": 6, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "sigmoid(np.array([-1,0,1]))" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 7, 259 | "metadata": { 260 | "collapsed": false 261 | }, 262 | "outputs": [ 263 | { 264 | "data": { 265 | "text/plain": [ 266 | "array([[ 0.12719082, 0.49382416, 0.3385655 ],\n", 267 | " [ 0.79648358, 0.46650512, 0.61030083],\n", 268 | " [ 0.46101266, 0.34537011, 0.48941003]])" 269 | ] 270 | }, 271 | "execution_count": 7, 272 | "metadata": {}, 273 | "output_type": "execute_result" 274 | } 275 | ], 276 | "source": [ 277 | "sigmoid(np.random.randn(3,3))" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": {}, 283 | "source": [ 284 | " We now have our second formula for forward propagation, using f to denote our activation function, we can write that a two, our second layer activity, is equal to f of z two. a two will be a matrix of the same size as z two, 3 by 3." 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "$$\n", 292 | "a^{(2)} = f(z^{(2)}) \\tag{2}\\\\\n", 293 | "$$" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "To finish forward propagation we need to propagate a two all the way to the output, yhat. We've already done the heavy lifting in the previous layer, so all we have to do now is multiply a two by our senond layer weights W2 and apply one more activation funcion. W2 will be of size 3x1, one weight for each synapse. Multiplying a2, a 3 by 3, by W2, a 3 by 1 results in a 3 by 1 matrix z three, the activity or our third layer. z3 has three activity values, one for each example. Last but not least, we'll apply our activation function to z three yielding our official estimate of your test score, yHat. " 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "$$\n", 308 | "z^{(3)} = a^{(2)}W^{(2)} \\tag{3}\\\\\n", 309 | "$$\n", 310 | "$$\n", 311 | "\\hat{y} = f(z^{(3)}) \\tag{4}\\\\\n", 312 | "$$" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "We need to implement our forward propagation formulas in python. First we'll initialize our weight matrices in our init method. For starting values, we'll use random numbers. " 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "We'll implement forward propagation in our forward method, using numpy's built in dot method for matrix multiplication and our own sigmoid method. " 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 8, 332 | "metadata": { 333 | "collapsed": false 334 | }, 335 | "outputs": [], 336 | "source": [ 337 | "class Neural_Network(object):\n", 338 | " def __init__(self): \n", 339 | " #Define Hyperparameters\n", 340 | " self.inputLayerSize = 2\n", 341 | " self.outputLayerSize = 1\n", 342 | " self.hiddenLayerSize = 3\n", 343 | " \n", 344 | " #Weights (parameters)\n", 345 | " self.W1 = np.random.randn(self.inputLayerSize, self.hiddenLayerSize)\n", 346 | " self.W2 = np.random.randn(self.hiddenLayerSize, self.outputLayerSize)\n", 347 | " \n", 348 | " def forward(self, X):\n", 349 | " #Propagate inputs though network\n", 350 | " self.z2 = np.dot(X, self.W1)\n", 351 | " self.a2 = self.sigmoid(self.z2)\n", 352 | " self.z3 = np.dot(self.a2, self.W2)\n", 353 | " yHat = self.sigmoid(self.z3) \n", 354 | " return yHat\n", 355 | " \n", 356 | " def sigmoid(self, z):\n", 357 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 358 | " return 1/(1+np.exp(-z))" 359 | ] 360 | }, 361 | { 362 | "cell_type": "markdown", 363 | "metadata": {}, 364 | "source": [ 365 | "And there you have it, a python class capable of estimating your test score given how many hours you sleep and how many hours you study. We can pass in our input data and get real outputs. Now, you may be noticing that our estimates are quite terrible. That's because we have not yet trained our network, that's what we'll work on next time." 366 | ] 367 | } 368 | ], 369 | "metadata": { 370 | "kernelspec": { 371 | "display_name": "Python 2", 372 | "language": "python", 373 | "name": "python2" 374 | }, 375 | "language_info": { 376 | "codemirror_mode": { 377 | "name": "ipython", 378 | "version": 2 379 | }, 380 | "file_extension": ".py", 381 | "mimetype": "text/x-python", 382 | "name": "python", 383 | "nbconvert_exporter": "python", 384 | "pygments_lexer": "ipython2", 385 | "version": "2.7.10" 386 | } 387 | }, 388 | "nbformat": 4, 389 | "nbformat_minor": 0 390 | } 391 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Part 2 Forward Propagation-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 2: Forward Propagation

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 1, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('UJwK6jAStmg')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "

Variables

\n", 53 | "\n", 54 | "|Code Symbol | Math Symbol | Definition | Dimensions\n", 55 | "| :-: | :-: | :-: | :-: |\n", 56 | "|X|$$X$$|Input Data, each row in an example| (numExamples, inputLayerSize)|\n", 57 | "|y |$$y$$|target data|(numExamples, outputLayerSize)|\n", 58 | "|W1 | $$W^{(1)}$$ | Layer 1 weights | (inputLayerSize, hiddenLayerSize) |\n", 59 | "|W2 | $$W^{(2)}$$ | Layer 2 weights | (hiddenLayerSize, outputLayerSize) |\n", 60 | "|z2 | $$z^{(2)}$$ | Layer 2 activation | (numExamples, hiddenLayerSize) |\n", 61 | "|a2 | $$a^{(2)}$$ | Layer 2 activity | (numExamples, hiddenLayerSize) |\n", 62 | "|z3 | $$z^{(3)}$$ | Layer 3 activation | (numExamples, outputLayerSize) |" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "Last time, we setup our neural network on paper. This time, we’ll implement it in the programming language python. We’ll build our network as a python class and our init method will take care of instantiating important constants and variables. We’ll make these values accessible to the whole class by placing a self dot in front of each variable name." 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "Our network has 2 inputs, 3 hidden units, and 1 output. These are examples of hyperparameters. Hyperparameters are constants that establish the structure and behavior of a neural network, but are not updated as we train the network. Our learning algorithm is not capable of, for example, deciding that it needs another hidden unit, this is something that WE must decide on before training. What a neural network does learn are parameters, specifically the weights on the synapses." 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "We’ll take care of moving data through our network in a method called forward. Rather than pass inputs through the network one at a time, we’re going to use matrices to pass through multiple inputs at once. Doing this allows for big computational speedups, especially when using tools like MATLAB or Numpy. Our input data matrix, X, is of dimension 3 by 2, because we have 3, 2-dimensional examples. Our corresponding output data, y, is of dimension 3 by 1." 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 1, 89 | "metadata": { 90 | "collapsed": false 91 | }, 92 | "outputs": [ 93 | { 94 | "name": "stdout", 95 | "output_type": "stream", 96 | "text": [ 97 | "Populating the interactive namespace from numpy and matplotlib\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "#Import code from last time\n", 103 | "%pylab inline\n", 104 | "from partOne import *" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 2, 110 | "metadata": { 111 | "collapsed": false 112 | }, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "(3, 2) (3, 1)\n" 119 | ] 120 | } 121 | ], 122 | "source": [ 123 | "print X.shape, y.shape" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": { 130 | "collapsed": false 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "class Neural_Network(object):\n", 135 | " def __init__(self): \n", 136 | " #Define Hyperparameters\n", 137 | " self.inputLayerSize = 2\n", 138 | " self.outputLayerSize = 1\n", 139 | " self.hiddenLayerSize = 3\n", 140 | " \n", 141 | " def forward(self, X):\n", 142 | " #Propagate inputs though network" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": {}, 148 | "source": [ 149 | "Each input value, or element in matrix X, needs to be multiplied by a corresponding weight and then added together with all the other results for each neuron. This is a complex operation, but if we take the three outputs we're looking for as a single row of a matrix, and place all our individual weights into a matrix of weights, we can create the exact behavior we need by multiplying our input data matrix by our weight matrix. Using matrix multiplication allows us to pass mulitple inputs through at once by simply adding rows to the matrix X. From here on out, we'll refer to these matrics as X, W one, and z two, where z two the activity of our second layer. Notice that each entry in z is a sum of weighted inputs to each hidden neuron. Z is of size 3 by 3, one row for each example, and one column for each hidden unit. " 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "We now have our first official formula, $z^{(2)} = XW^{(1)}$. Matrix notation is realy nice here, becuase it allows us to express the complex underlying process in a single line!" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "$$\n", 164 | "z^{(2)} = XW^{(1)} \\tag{1}\\\\\n", 165 | "$$\n" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "Now that we have the activities for our second layer, z two, we need to apply the activation function. We'll independently apply the function to each entry in matrix z using a python method for this called sigmoid, because we’re using a sigmoid as our activation function. Using numpy is really nice here, because we can pass in a scalar, vector, or matrix, Numpy will apply the activation function element-wise, and return a result of the same dimension as it was given." 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 3, 178 | "metadata": { 179 | "collapsed": false 180 | }, 181 | "outputs": [], 182 | "source": [ 183 | "def sigmoid(z):\n", 184 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 185 | " return 1/(1+np.exp(-z))" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 4, 191 | "metadata": { 192 | "collapsed": false 193 | }, 194 | "outputs": [ 195 | { 196 | "data": { 197 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEACAYAAACuzv3DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFdW19/HvshlUBNtIxBtEMYAGHNIYxDGxNRqQSMyo\n8IoJiUaNEyZGAY2KQ9RcvYpTDElUTDRqrmJUrvPVjkaviuKEAoKAgjMqKkSgm17vH/tgN20Ph+46\np07t8/s8z3m6qk9xei1LFtWr9t5l7o6IiGTXBmkHICIiHaNCLiKScSrkIiIZp0IuIpJxKuQiIhmn\nQi4iknFtFnIzu9bM3jGzF1s55nIzm2dmz5vZ4GRDFBGR1uRzRX4dMLylN81sBNDf3QcARwFXJxSb\niIjkoc1C7u6PAh+2csh3gOtzxz4JVJpZr2TCExGRtiTRI+8NLG60vwTYKoHPFRGRPCR1s9Oa7Gve\nv4hIkXRK4DPeAPo02t8q9711mJmKu4hIO7h704vldSRRyO8EjgduNrPdgWXu/k4LwSTw40rTpEmT\nmDRpUtphFEzM+cWcG5RffvX18O678MYbsGTJul/feCO89957sHQp1NW1/+duuCF06xZeG2/c8mvt\n+xttFP5Mly7QtWv42ni7pa8DB7Zaw4E8CrmZ3QTsA/Q0s8XAWUBnAHef4u53m9kIM5sPrAB+2v7/\nNNm1aNGitEMoqJjzizk3iDO/FStgwQJ49VWYPn0R774L8+eH/cWLobY2v8/p3h169oQvfrHhtfnm\nUFkJm276+VePHg3bnTsXNsf10WYhd/fReRxzfDLhiIg0WLkS5syBl16CWbMaXk3/bXrmmXX3N98c\neveGrbZa92vv3tCrVyjYPXuGK+QYJNFaEWDs2LFph1BQMecXc26Qnfxqa0ORfuopmDEjfH35ZViz\n5vPHdu4M224L/fpB165j2WefsN2vH/TtG1oZ5cSK1bc2M4+5Ry4i62fFCnj8cXj4YfjnP2HmzHAF\n3tgGG8CAAbDjjuu++veHTmVyGWpmRbnZKUBNTQ3V1dVph1EwMecXc25QOvnV1cETT8C990JNTbji\nbtrLHjAAdt0Vhg4NX6uq2r66LpX80qRCLiIFs2wZ3Hcf3HUX3HMPfPBBw3sbbABDhkB1dXjtsQd8\n4QtpRZptaq2ISKI++QTuuANuugnuv3/dIX79+sFBB8H++8Pee4fRIdI6tVZEpCjq6uDuu+Gvf4Xp\n0xt63RtsEK62DzoovLbbDqztYdGynrQeeUJqamrSDqGgYs4v5tygsPm99hqceSZssw0cfDDcemso\n4l//Olx1Fbz1VriZefLJsP32hSnisZ+/fOiKXETWi3u4Wflf/xWuwtd2TAcMgCOPhNGjoU+fVj9C\nEqYeuYjkpa4uXHFffHHDBJyuXeEHP4CjjoJvfENtk0JQj1xEOqyuLvS+zzmnYUblF78IJ5wAv/hF\nmCEp6VKPPCGx9+lizi/m3KD9+dXXh5EngwbBz34Wivh228GUKaE3fsYZpVHEYz9/+dAVuYh8zkMP\nwUknwYu5J/X27w+TJsGoUVBRkWpo0gz1yEXkMwsXwq9/DdOmhf2ttw6jUn7849Ja7a+cqEcuInlZ\ntQrOPx9+97uw3a0bnHYa/OpX8awQGDP1yBMSe58u5vxizg3azu/xx8OaJuecE4r4mDEwd24o5Fko\n4rGfv3yokIuUqeXL4cQTw1T5OXPChJ1HHw0jVHr3Tjs6WR/qkYuUoWeeCRN35s0LNy/Hjw+jULJw\nBV5u1CMXkXXU18Mll4S2SW1tWNv7r38NrRXJLrVWEhJ7ny7m/GLODRryW7oUDjwQTjklFPHjjw9r\ngme9iMd+/vKhK3KRMjBzJnzve/D66+F5ltddByNHph2VJEU9cpHI3XhjWMxq5crw5J3bbgsPI5Zs\nyKdHrtaKSKTq60MbZcyYUMSPOAIeeURFPEYq5AmJvU8Xc34x5rZyJRx6aFipsKKihquvhj/9KaxW\nGJsYz9/6Uo9cJDIffBAe8vCvf0GPHnDWWXDMMWlHJYWkHrlIRF57DYYPDxN8ttoqPPhhp53Sjko6\nQuPIRcrIq6/CfvuFkSk77RSKuPrh5UE98oTE3qeLOb8Ycps7Nzyh5/XXYY891r2pGUN+rYk9v3yo\nkItk3EsvwT77wJtvhmJ+331QWZl2VFJM6pGLZNjs2aF4L10K++8Pd9wBG2+cdlSSJI0jF4nYwoWh\neC9dCsOGwV13qYiXKxXyhMTep4s5vyzm9tZboYivbadMm9byyoVZzG99xJ5fPlTIRTLm/ffhgANg\nwQIYMkRX4qIeuUimfPppGGL4xBPh6fb//GdpPMleCkc9cpGI1NfD4YeHIr711nD//SriEqiQJyT2\nPl3M+WUltwkTwsqFPXqEyT75Po4tK/m1V+z55UOFXCQDpkyBiy6CTp1CMd9hh7QjklLSZo/czIYD\nk4EK4M/u/rsm728K3AD0IUz5v9jdpzbzOeqRi7TDgw+G9VPWrIFrroGf/SztiKSY8umRt1rIzawC\nmAvsD7wBzABGu/vsRsecBnR394lm1jN3fC93r2vyWSrkIutp0SL42tfCioYTJsAFF6QdkRRbEjc7\nhwLz3X2Ru9cCNwMHNzmmHuiR2+4BvN+0iJeD2Pt0MedXqrn9+9/h8WwffAAjRsB557Xvc0o1v6TE\nnl8+2irkvYHFjfaX5L7X2JXAIDN7E3geGJdceCLlyR2OPhqeew769YMbboCKirSjklLV1jK2+fRC\nhgMz3X1fM+sHPGBmX3X3T5oeOHbsWPr27QtAZWUlVVVVVFdXAw3/qmZ1f+33SiUe5Zf/fnV1dUnF\nA3DiiTXccAN061bNP/4Bzz8fV36xn7+O7NfU1DB16lSAz+plW9rqke8OTHL34bn9iUB94xueZjYd\nuMDdH8vt/y8w3t2fbvJZ6pGL5OHJJ2HvvaGuDv7+d/jRj9KOSNKURI/8aWCAmfU1sy7AocCdTY55\nnXAzFDPrBWwPLGhfyNm19l/UWMWcXynltmwZjBoVivhJJyVTxEspv0KIPb98tNpacfc6MzseuI8w\n/PAad59tZkfn3p8CnAtMNbMXAANOdfcPChy3SHTc4aijGkaqXHhh2hFJVmitFZESMWVKeEhy9+4w\ncyb07592RFIKOjyOPOFgVMhFWvDiizB0KKxcCX/7G4wenXZEUiq0aFYRxd6nizm/tHNbtQoOOywU\n8SOOSL6Ip51focWeXz5UyEVSNmlSuCLv3x8uuyztaCSL1FoRSdHjj8PXvx62H30U9twz3Xik9Ki1\nIlLCVqyAn/wkrDN+yikq4tJ+KuQJib1PF3N+aeV26qkwfz7stBOcfXbhfk7M5w7izy8fKuQiKXjw\nQfj976FzZ/jrX6Fr17QjkixTj1ykyFasCFfhCxeGFQ1PPz3tiKSUqUcuUoLOOisU8a9+NbRXRDpK\nhTwhsffpYs6vmLk98wxceilssAH8+c+htVJoMZ87iD+/fKiQixRJbS0ceWQYpTJuHAwZknZEEgv1\nyEWK5D//E8aPh759YdYs6NYt7YgkC7TWikiJePVV2HHHMA3/3nth2LC0I5Ks0M3OIoq9TxdzfoXO\nzR2OPTYU8cMOK34Rj/ncQfz55UOFXKTAbr8d7r8fKivhkkvSjkZipNaKSAH9+98wcCC8/jpceSUc\nd1zaEUnWqLUikrILLwxFvKoqPDRCpBBUyBMSe58u5vwKldurr4aRKhCuxisqCvJj2hTzuYP488uH\nCrlIgZx0UnhoxI9/DHvtlXY0EjP1yEUKYPp0GDkSevSAuXNhyy3TjkiySj1ykRSsXBlmbkJYnlZF\nXApNhTwhsffpYs4v6dwuvxwWLIAddiiNUSoxnzuIP798qJCLJOjdd+G3vw3bl15anEWxRNQjF0nQ\nscfC1VfDgQfC3XenHY3EQGutiBTRyy/DzjuH7RdegEGD0o1H4qCbnUUUe58u5vySyu2UU2DNGjjq\nqNIq4jGfO4g/v3yokIsk4P77Qyule3eYNCntaKTcqLUi0kFr1sDgwfDii2FK/vjxaUckMVFrRaQI\nrrsuFPFttmkYPy5STCrkCYm9Txdzfh3JbcUKOOOMsH3hhbDhhsnElKSYzx3En18+VMhFOuCyy+Dt\nt2HXXeHQQ9OORsqVeuQi7fT++/DlL8PHH8P//i/st1/aEUmM1CMXKaALLwxF/IADVMQlXSrkCYm9\nTxdzfu3JbckSuOKKsH3BBcnGk7SYzx3En18+VMhF2uHss8Na44ccAl/7WtrRSLlrs0duZsOByUAF\n8Gd3/10zx1QDlwKdgaXuXt3MMeqRSxTmzAkrG5qFafnbbZd2RBKzfHrkndr4gArgSmB/4A1ghpnd\n6e6zGx1TCVwFDHP3JWbWs+Ohi5Su3/wG6uvDVHwVcSkFbbVWhgLz3X2Ru9cCNwMHNznm/wG3ufsS\nAHdfmnyYpS/2Pl3M+a1PbjNmwG23hfHiZ55ZuJiSFPO5g/jzy0dbhbw3sLjR/pLc9xobAHzBzB42\ns6fN7PAkAxQpFe4wYULYHjcOejf9myCSklZ75Gb2A2C4u/88tz8G2M3dT2h0zJXALsA3gY2B/wO+\n7e7zmnyWeuSSaQ88AN/6FlRWhicAbbZZ2hFJOehwj5zQF+/TaL8P4aq8scWEG5yfAp+a2SPAV4F5\nTY5j7Nix9O3bF4DKykqqqqqorq4GGn490r72S3H/4YdrcuuoVHPqqfD886UVn/bj2a+pqWHq1KkA\nn9XLNrl7iy9CoX8V6At0AZ4DBjY55ivAg4RRLRsDLwKDmvksj9nDDz+cdggFFXN++eR2993u4N6z\np/snnxQ+piTFfO7c488vVztbrdWtXpG7e52ZHQ/clyvU17j7bDM7Ovf+FHefY2b3Ai8A9cCf3P3l\n/P4ZESl97g03NsePh002STcekaa01opIG6ZPh5EjYYstQm+8W7e0I5JyorVWRDrIHc46K2xPmKAi\nLqVJhTwha29WxCrm/FrL7c47YeZM2HJLOOaY4sWUpJjPHcSfXz5UyEVa4N7w/M0JE2CjjVINR6RF\n6pGLtOD22+H734cvfQnmz1chl3SoRy7STvX1Db3xiRNVxKW0qZAnJPY+Xcz5NZfbtGnhgcq9e8OR\nRxY/piTFfO4g/vzyoUIu0kR9fUNv/PTTS/OByiKNqUcu0sQtt8CoUdCnD8ybB127ph2RlDP1yEXW\n05o14ek/ENYdVxGXLFAhT0jsfbqY82uc2y23wOzZsM02MHZsaiElKuZzB/Hnlw8VcpGcNWvgnHPC\n9m9+A126pBuPSL7UIxfJueEGOPxw2HZbmDsXOndOOyIR9chF8lZX19AbP+MMFXHJFhXyhMTep4s5\nv5qaGm68Mcze7NcvXJXHJOZzB/Hnlw8Vcil7dXVw7rlh+8wzoVNbz80SKTHqkUvZu/ZaOOIIGDAA\nXn5ZhVxKi3rkIm2orYXzzgvbZ52lIi7ZpEKekNj7dLHmd/31sHBhDV/5SpjNGaNYz91aseeXDxVy\nKVurV6/bG6+oSDcekfZSj1zK1pQp4ak/gwbBCy+okEtpyqdHrkIuZWnVqnBzc/HiMC3/kEPSjkik\nebrZWUSx9+liy++aa0IR33FH6NmzJu1wCiq2c9dU7PnlQ4Vcys7KlXD++WF70iTYQH8LJOPUWpGy\nc8UVcOKJsPPO8OyzKuRS2tQjF2ni00/DNPy33gqPc/ve99KOSKR16pEXUex9uljymzIlFPHBg+G7\n3w3fiyW3lii/+KmQS9lYsQIuuCBsn3MOWKvXOCLZodaKlI2LL4ZTToFdd4Unn1Qhl2xQj1wkZ/ny\n8MCIpUvhnntg+PC0IxLJj3rkRRR7ny7r+V15ZSjiu+8Ow4at+17Wc2uL8oufCrlE7+OP4aKLwrZ6\n4xIjtVYkeuedFx7ftvfe8MgjKuSSLeqRS9n76CPo2xeWLYOHHoJ99007IpH1ox55EcXep8tqfpMn\nhyJeXd1yEc9qbvlSfvFTIZdoffghXHJJ2D777HRjESkktVYkWmecEfrj++8PDzyQdjQi7ZNIa8XM\nhpvZHDObZ2bjWzluVzOrM7PvtydYkSS9/35oq4CuxiV+rRZyM6sArgSGA4OA0WY2sIXjfgfcC5Tl\nmIDY+3RZy++ii8IkoGHDYM89Wz82a7mtL+UXv7auyIcC8919kbvXAjcDBzdz3AnArcB7Cccnst7e\nfhsuvzxsn3NOurGIFEOrPXIz+yEwzN1/ntsfA+zm7ic0OqY3cAOwH3AtcJe7T2vms9Qjl6I4/ni4\n6qqwuuHtt6cdjUjHJNEjz6fyTgYm5Kq0UaatFSkNCxfCH/8YJv2ce27a0YgUR6c23n8D6NNovw+w\npMkxXwNutjBdridwoJnVuvudTT9s7Nix9O3bF4DKykqqqqqorq4GGvpcWd2fPHlyVPlkNb9rr62m\nthYOOKCGpUsB2v7zjXusacdfiH3ll639mpoapk6dCvBZvWxLW62VTsBc4JvAm8BTwGh3n93C8ddR\npq2Vmpqaz05KjLKQ36xZ4fFtnTrB3LlhtcN8ZCG3jlB+2ZbIFH0zO5DQPqkArnH3C8zsaAB3n9Lk\n2LIt5JK+734X7rgDjjsurHYoEgOttSJl48knwxK1G20ECxbAllumHZFIMrTWShE17tPFqNTzO+20\n8HXcuPUv4qWeW0cpv/ipkEvmPfhgWNmwshJOPTXtaESKT60VyTR32G03mDEDzj8fJk5MOyKRZKlH\nLtG75RYYNSq0U+bPh27d0o5IJFnqkRdR7H26Usxv1SqYMCFsn3NO+4t4KeaWJOUXPxVyyayrroJF\ni2DQIPjpT9OORiQ9aq1IJn3wAfTvHx4eMX06fPvbaUckUhhqrUi0fvvbUMT32w9GjEg7GpF0qZAn\nJPY+XSnlt2BBw8zNiy4KC2R1RCnlVgjKL34q5JI5p50Gq1fD4YfDLrukHY1I+tQjl0xZOxW/a1d4\n5RXYeuu0IxIpLPXIJSrucPLJYfuXv1QRF1lLhTwhsffpSiG/m26Cxx6DLbZoGD+ehFLIrZCUX/xU\nyCUTli+HU04J2xdcAJtumm48IqVEPXLJhNNPD2upDBkS+uQb6BJEyoTWWpEovPpqmL25ejU8/jjs\nsUfaEYkUj252FlHsfbo08zv55IbhhoUo4jp32RZ7fvlQIZeS9sAD4fFt3brBhRemHY1IaVJrRUrW\n6tVQVQWzZ4ciPn582hGJFJ9aK5JpF18civiAAXDSSWlHI1K6VMgTEnufrtj5LVgA554btn//+zCT\ns1B07rIt9vzyoUIuJccdjjsOVq6Eww6D/fdPOyKR0qYeuZSc//5vOOSQ8DDlOXOgV6+0IxJJj3rk\nkjkffwzjxoXtCy9UERfJhwp5QmLv0xUrv9/8Bt56K6xw+POfF+VH6txlXOz55UOFXErGY4+FB0ZU\nVMAf/qBp+CL5Uo9cSsKnn4Yx46+8EtZVOe+8tCMSKQ3qkUtmnHlmKOI77ABnnJF2NCLZokKekNj7\ndIXM78kn4ZJLQivluusKO2a8OTp32RZ7fvlQIZdUrVwJP/0p1NfDr38Nu+6adkQi2aMeuaRq4sQw\nzHD77eG552DDDdOOSKS0aD1yKWmPPALV1WAGjz4Ke+6ZdkQipUc3O4so9j5d0vktWxbWF3cPV+Vp\nFnGdu2yLPb98qJBLKo47Dl5/PfTEzzor7WhEsk2tFSm6G2+EMWPCwyKefTYsUysizVNrRUrOokVw\n7LFhe/JkFXGRJORVyM1suJnNMbN5Zva557SY2WFm9ryZvWBmj5nZzsmHWtpi79Mlkd+qVfCjH4WF\nsb73PTjiiI7HlQSdu2yLPb98tFnIzawCuBIYDgwCRpvZwCaHLQC+4e47A+cCf0w6UMm+X/0Knn4a\n+vaFa64Jo1VEpOPa7JGb2R7AWe4+PLc/AcDdm30UrpltBrzo7ls1+b565GXs5pth9Gjo0iUsjjVk\nSNoRiWRDUj3y3sDiRvtLct9ryRHA3Xl8rpSJ2bPhyCPD9uTJKuIiSeuUxzF5X0ab2b7Az4C9mnt/\n7Nix9O3bF4DKykqqqqqorq4GGvpcWd2fPHlyVPkkld8uu1Tzwx/CihU17LcfHHNMaeTTeL9xj7UU\n4lF+5Z1fTU0NU6dOBfisXrbJ3Vt9AbsD9zbanwiMb+a4nYH5QP8WPsdj9vDDD6cdQkG1J7+6OveR\nI93BfeBA908+ST6uJOjcZVvs+eVqZ6t1Op8eeSdgLvBN4E3gKWC0u89udMzWwEPAGHd/ooXP8bZ+\nlsRl7Toqm20GTz0F/funHZFI9uTTI2+zteLudWZ2PHAfUAFc4+6zzezo3PtTgDOBzYCrLQxFqHX3\noR1NQLLrxhtDEa+ogFtvVREXKaS8xpG7+z3uvr2793f3C3Lfm5Ir4rj7ke6+ubsPzr3Krog37tPF\naH3ye+qphjHil10G++1XmJiSonOXbbHnlw/N7JREzZsHBx0UJv8cfXTDLE4RKRyttSKJeecd2GMP\nWLgQhg2Du+6Czp3Tjkok27TWihTNJ5/AiBGhiA8ZEvriKuIixaFCnpDY+3St5bdqFfzgBzBzJvTr\nB//zP7DJJsWLraPK+dzFIPb88qFCLh1SWwuHHgoPPABbbAH33Re+ikjxqEcu7VZbG9ZPue02qKyE\nhx6CwYPTjkokLuqRS8HU1YVHtd12G2y6abgiVxEXSYcKeUJi79M1zq+2Fn7yE7jlFujePbRTsrwQ\nVjmduxjFnl8+8lk0S+Qzn34KhxwC06eHG5r33gu77ZZ2VCLlTT1yydtHH8F3vgOPPAJf+ALccw8M\nLbs5vCLFlchaKyIA774LBx4Yhhh+6UuhJz5oUNpRiQioR56YmPt0s2bBzjvXfDZO/LHH4iriMZ87\nUH7lQIVcWnXvvbDnnmH6/dCh8K9/hWduikjpUI9cmuUOV1wBv/wl1NeHG5xTp8JGG6UdmUh50Thy\naZfly2HMGBg3LhTxM8+Em25SERcpVSrkCYmlT/fyy6GF8re/QbduoYCffTY88khN2qEVTCznriXK\nL34q5AKEVsr118Ouu4an3g8aBDNmwKhRaUcmIm1Rj1x4773wEIjbbw/7Y8bAH/4QrshFJF3qkUub\n7rgDdtwxFPHu3eG66+Avf1ERF8kSFfKEZK1P9+abYfnZ7343TPbZd1948UUYOxasmX/7s5bf+og5\nN1B+5UCFvMysWQNXXgkDB8Lf/w4bbwyXXgoPPgjbbJN2dCLSHuqRl5GaGjj55DDNHmDkyDBWXAVc\npHSpRy5AGFI4cmRon8ycCVttFXrid9yhIi4SAxXyhJRin27BAjjySNhpp4ZlZ889F+bMCb3x5nrh\nLSnF/JISc26g/MqBVj+M0CuvwPnnww03hJ54RQUccwxMmgS9eqUdnYgkTT3ySLjD44/DZZeFx6/V\n14cCPmYMnHYabLdd2hGKSHtoPfIysHJleOTa5Zc33MTs3BmOOAImTIAvfznd+ESk8NQjT0gx+3Tu\noWifeGK4cTl2bNjv2RNOPx0WLoQ//jHZIh5zHzLm3ED5lQNdkWfI4sVh7Pf114fJO2sNHhyK+qhR\nsOGG6cUnIulQj7zEzZ8fet7TpsFTTzV8v2dPOOywcDVeVZVaeCJSYOqRZ9Cnn4an8Nx/f3g6z6xZ\nDe9tvHF4buaYMTBiBHTpkl6cIlI61CNPSHv7dKtXwxNPwMUXw/Dh4en03/pW2J81C3r0CFfe06aF\nVQpvvTWMAS92EY+5DxlzbqD8yoGuyIvsvfdCi+Sxx8KV94wZYeRJY1VVoZgPGwZ77QVdu6YTq4hk\ng3rkBVJfH0aPPPssPPdceD37bFh1sKmvfCUU7OpqOOAATdoRkQbqkRfBRx+FmZRz54bXnDnh67x5\nn7/ShjBNfvDg8GT6vfYKXzffvPhxi0g82izkZjYcmAxUAH929981c8zlwIHAv4Gx7v5s0oGmYfVq\neOedcBX92mvh9frr624vW7b26Bqgep0//x//EdokVVWheFdVQb9+sEEG70zU1NRQXV2ddhgFEXNu\noPzKQauF3MwqgCuB/YE3gBlmdqe7z250zAigv7sPMLPdgKuB3QsYc7vU18PHH4fC++GH4bVsGXzw\nQSjWzb0+/LDtz91oIxgwAMyeY+TIarbfHrbfPkyJ33TTwudVLM8991y0f1lizg2UXzlo64p8KDDf\n3RcBmNnNwMHA7EbHfAe4HsDdnzSzSjPr5e7vrG8wa9bAqlXhtXp1w/aqVaFNsXw5rFjR8Gptf/ny\ndQv2Rx+FYr4+Kipgiy3ClfU224TX1luvu92zZ1hFcNKkZUyatL4ZZ8eyhl89ohNzbqD8ykFbhbw3\nsLjR/hJgtzyO2Qr4XCEfMmTd4ty0YK9Z044M1kP37lBZCZttFl5rt3v1av61+ebZbIOISHlpq5Dn\nO8yk6R3VZv/cM8+08SEWhtp16RK+Nn5tuGF4IPAmm4Sva1+N95tuNy7am24KnQp4a3fRokWF+/AS\nEHN+MecGyq8ctDr80Mx2Bya5+/Dc/kSgvvENTzP7A1Dj7jfn9ucA+zRtrZhZ+Yw9FBFJUEeHHz4N\nDDCzvsCbwKHA6CbH3AkcD9ycK/zLmuuPtxWIiIi0T6uF3N3rzOx44D7C8MNr3H22mR2de3+Ku99t\nZiPMbD6wAvhpwaMWEZHPFG1mp4iIFEZRx2SY2QlmNtvMZpnZ5yYWxcDMTjazejP7QtqxJMnMLsqd\nu+fNbJqZRTFK3syGm9kcM5tnZuPTjidJZtbHzB42s5dyf+dOTDumpJlZhZk9a2Z3pR1L0nJDuW/N\n/b17Ode6blbRCrmZ7UsYc76zu+8IXFysn10sZtYHOAB4Le1YCuB+YAd3/yrwCjAx5Xg6rNGEt+HA\nIGC0mQ1MN6pE1QK/dPcdCJP0jossP4BxwMvkP8IuSy4D7nb3gcDOrDt/Zx3FvCL/BXCBu9cCuPt7\nRfzZxXIJcGraQRSCuz/g7munVD1JmCuQdZ9NeMv9f7l2wlsU3P1td38ut72cUAi+lG5UyTGzrYAR\nwJ/5/BDoTMv9xvt1d78Wwv1Kd/+opeOLWcgHAN8wsyfMrMbMhhTxZxecmR0MLHH3F9KOpQh+Btyd\ndhAJaG4EOyqhAAAB+klEQVQyW++UYimo3MizwYR/hGNxKXAKsJ5ztjNhW+A9M7vOzGaa2Z/MbOOW\nDk50ioyZPQBs2cxbp+d+1mbuvruZ7Qr8HcjUM97byG8i8K3GhxclqAS1kt9p7n5X7pjTgdXu/rei\nBlcYMf46/jlmtglwKzAud2WeeWZ2EPCuuz9rZtVpx1MAnYBdgOPdfYaZTQYmAGe2dHBi3P2Alt4z\ns18A03LHzcjdENzc3d9PMoZCaik/M9uR8C/o82YGoe3wjJkNdfd3ixhih7R2/gDMbCzhV9lvFiWg\nwnsD6NNovw/hqjwaZtYZuA24wd3/kXY8CdoT+E5u0b4NgR5m9hd3/3HKcSVlCeE3/Bm5/VsJhbxZ\nxWyt/APYD8DMtgO6ZKmIt8bdZ7l7L3ff1t23JZyEXbJUxNuSW874FOBgd29mpfVM+mzCm5l1IUx4\nuzPlmBJj4ariGuBld5+cdjxJcvfT3L1P7u/bKOChiIo47v42sDhXKyGsQPtSS8cX88ES1wLXmtmL\nwGogmv/ozYjxV/YrgC7AA7nfOv7P3Y9NN6SOaWnCW8phJWkvYAzwgpmtfUbARHe/N8WYCiXGv3Mn\nADfmLjJepZXJlpoQJCKScVqkVUQk41TIRUQyToVcRCTjVMhFRDJOhVxEJONUyEVEMk6FXEQk41TI\nRUQy7v8DtTV/D9zJEKcAAAAASUVORK5CYII=\n", 198 | "text/plain": [ 199 | "" 200 | ] 201 | }, 202 | "metadata": {}, 203 | "output_type": "display_data" 204 | } 205 | ], 206 | "source": [ 207 | "testInput = np.arange(-6,6,0.01)\n", 208 | "plot(testInput, sigmoid(testInput), linewidth= 2)\n", 209 | "grid(1)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 5, 215 | "metadata": { 216 | "collapsed": false 217 | }, 218 | "outputs": [ 219 | { 220 | "data": { 221 | "text/plain": [ 222 | "0.7310585786300049" 223 | ] 224 | }, 225 | "execution_count": 5, 226 | "metadata": {}, 227 | "output_type": "execute_result" 228 | } 229 | ], 230 | "source": [ 231 | "sigmoid(1)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 6, 237 | "metadata": { 238 | "collapsed": false 239 | }, 240 | "outputs": [ 241 | { 242 | "data": { 243 | "text/plain": [ 244 | "array([ 0.26894142, 0.5 , 0.73105858])" 245 | ] 246 | }, 247 | "execution_count": 6, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "sigmoid(np.array([-1,0,1]))" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 7, 259 | "metadata": { 260 | "collapsed": false 261 | }, 262 | "outputs": [ 263 | { 264 | "data": { 265 | "text/plain": [ 266 | "array([[ 0.12719082, 0.49382416, 0.3385655 ],\n", 267 | " [ 0.79648358, 0.46650512, 0.61030083],\n", 268 | " [ 0.46101266, 0.34537011, 0.48941003]])" 269 | ] 270 | }, 271 | "execution_count": 7, 272 | "metadata": {}, 273 | "output_type": "execute_result" 274 | } 275 | ], 276 | "source": [ 277 | "sigmoid(np.random.randn(3,3))" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": {}, 283 | "source": [ 284 | " We now have our second formula for forward propogation, using f to denote our activation function, we can write that a two, our second layer activity, is equal to f of z two. a two will be a matrix of the same size as z two, 3 by 3." 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "$$\n", 292 | "a^{(2)} = f(z^{(2)}) \\tag{2}\\\\\n", 293 | "$$" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "To finish forward propogation we need to propogate a two all the way to the output, yhat. We've already done the heavy lifting in the previous layer, so all we have to do now is multiply a two by our senond layer wieghts W2 and apply one more activation funcion. W2 will be of size 3x1, one weight for each synapse. Multiplying a2, a 3 by 3, by W2, a 3 by 1 results in a 3 by 1 matrix z three, the activity or our third layer. z3 has three activity values, one for each example. Last but not least, we'll apply our activation function to z three yeilding our official estimate of your test score, y Hat. " 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "$$\n", 308 | "z^{(3)} = a^{(2)}W^{(2)} \\tag{3}\\\\\n", 309 | "$$\n", 310 | "$$\n", 311 | "\\hat{y} = f(z^{(3)}) \\tag{4}\\\\\n", 312 | "$$" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "We need to implement our forward propogation formulas in python. First we'll initialize our weight matrices in our init method. For starting values, we'll use random numbers. " 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "We'll implement forward propogation in our forward method, using numpy's built in dot method for matrix multiplication and our own sigmoid method. " 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 8, 332 | "metadata": { 333 | "collapsed": false 334 | }, 335 | "outputs": [], 336 | "source": [ 337 | "class Neural_Network(object):\n", 338 | " def __init__(self): \n", 339 | " #Define Hyperparameters\n", 340 | " self.inputLayerSize = 2\n", 341 | " self.outputLayerSize = 1\n", 342 | " self.hiddenLayerSize = 3\n", 343 | " \n", 344 | " #Weights (parameters)\n", 345 | " self.W1 = np.random.randn(self.inputLayerSize, self.hiddenLayerSize)\n", 346 | " self.W2 = np.random.randn(self.hiddenLayerSize, self.outputLayerSize)\n", 347 | " \n", 348 | " def forward(self, X):\n", 349 | " #Propagate inputs though network\n", 350 | " self.z2 = np.dot(X, self.W1)\n", 351 | " self.a2 = self.sigmoid(self.z2)\n", 352 | " self.z3 = np.dot(self.a2, self.W2)\n", 353 | " yHat = self.sigmoid(self.z3) \n", 354 | " return yHat\n", 355 | " \n", 356 | " def sigmoid(self, z):\n", 357 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 358 | " return 1/(1+np.exp(-z))" 359 | ] 360 | }, 361 | { 362 | "cell_type": "markdown", 363 | "metadata": {}, 364 | "source": [ 365 | "And there you have it, a python class capable of estimating your test score given how many hours you sleep and how many hours you study. We can pass in our input data and get real outputs. Now, you may be noticing that our estimates are quite terrible. That's because we have not yet trained our network, that's what we'll work on next time." 366 | ] 367 | } 368 | ], 369 | "metadata": { 370 | "kernelspec": { 371 | "display_name": "Python 2", 372 | "language": "python", 373 | "name": "python2" 374 | }, 375 | "language_info": { 376 | "codemirror_mode": { 377 | "name": "ipython", 378 | "version": 2 379 | }, 380 | "file_extension": ".py", 381 | "mimetype": "text/x-python", 382 | "name": "python", 383 | "nbconvert_exporter": "python", 384 | "pygments_lexer": "ipython2", 385 | "version": "2.7.10" 386 | } 387 | }, 388 | "nbformat": 4, 389 | "nbformat_minor": 0 390 | } 391 | -------------------------------------------------------------------------------- /Part 3 Gradient Descent.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 3: Gradient Descent

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 1, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('5u0jaA3qAGk')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "

Variables

\n", 53 | "\n", 54 | "|Code Symbol | Math Symbol | Definition | Dimensions\n", 55 | "| :-: | :-: | :-: | :-: |\n", 56 | "|X|$$X$$|Input Data, each row in an example| (numExamples, inputLayerSize)|\n", 57 | "|y |$$y$$|target data|(numExamples, outputLayerSize)|\n", 58 | "|W1 | $$W^{(1)}$$ | Layer 1 weights | (inputLayerSize, hiddenLayerSize) |\n", 59 | "|W2 | $$W^{(2)}$$ | Layer 2 weights | (hiddenLayerSize, outputLayerSize) |\n", 60 | "|z2 | $$z^{(2)}$$ | Layer 2 activation | (numExamples, hiddenLayerSize) |\n", 61 | "|a2 | $$a^{(2)}$$ | Layer 2 activity | (numExamples, hiddenLayerSize) |\n", 62 | "|z3 | $$z^{(3)}$$ | Layer 3 activation | (numExamples, outputLayerSize) |\n", 63 | "|J | $$J$$ | Cost | (1, outputLayerSize) |" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "Last time we built a neural network in python that made really bad predictions of your score on a test based on how many hours you slept and how many hours you studied the night before. This time we'll focus on the theory of making those predictions better." 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "We can initialize the network we built last time and pass in our normalized data, X, using our forward method, and have a look at our estimate of y, yHat." 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 1, 83 | "metadata": { 84 | "collapsed": false 85 | }, 86 | "outputs": [ 87 | { 88 | "name": "stdout", 89 | "output_type": "stream", 90 | "text": [ 91 | "Populating the interactive namespace from numpy and matplotlib\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "%pylab inline\n", 97 | "\n", 98 | "#Import code from last time:\n", 99 | "from partTwo import *" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 2, 105 | "metadata": { 106 | "collapsed": false 107 | }, 108 | "outputs": [], 109 | "source": [ 110 | "NN = Neural_Network()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 3, 116 | "metadata": { 117 | "collapsed": false 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "yHat = NN.forward(X)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 4, 127 | "metadata": { 128 | "collapsed": false 129 | }, 130 | "outputs": [ 131 | { 132 | "data": { 133 | "text/plain": [ 134 | "array([[ 0.55850487],\n", 135 | " [ 0.55993739],\n", 136 | " [ 0.56372181]])" 137 | ] 138 | }, 139 | "execution_count": 4, 140 | "metadata": {}, 141 | "output_type": "execute_result" 142 | } 143 | ], 144 | "source": [ 145 | "yHat" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 5, 151 | "metadata": { 152 | "collapsed": false 153 | }, 154 | "outputs": [ 155 | { 156 | "data": { 157 | "text/plain": [ 158 | "array([[ 0.75],\n", 159 | " [ 0.82],\n", 160 | " [ 0.93]])" 161 | ] 162 | }, 163 | "execution_count": 5, 164 | "metadata": {}, 165 | "output_type": "execute_result" 166 | } 167 | ], 168 | "source": [ 169 | "y" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 6, 175 | "metadata": { 176 | "collapsed": false 177 | }, 178 | "outputs": [ 179 | { 180 | "data": { 181 | "text/plain": [ 182 | "" 183 | ] 184 | }, 185 | "execution_count": 6, 186 | "metadata": {}, 187 | "output_type": "execute_result" 188 | }, 189 | { 190 | "data": { 191 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEACAYAAABI5zaHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF4xJREFUeJzt3X+M1PWdx/Hn20VBcBFbyHLK1rXWU7i0xZNTco12m/Za\nJLEkJ9FDu4WepuZyHtf+o9fL5dT0j4tp0xjrxcPTQhOSYvrjGqqAXkwnZ0xrSyulFiSSCvKj0i1F\nWNcuSPd9f+zsdhl3d2a/85md+bx5PZKN+535Mnw+vPDN7Gt+mbsjIiKxnNPsBYiISHoa7iIiAWm4\ni4gEpOEuIhKQhruISEAa7iIiAVUd7mb2DTM7Yma/nOCch83sVTP7hZldnXaJIiIyWbXcc18PLBvv\nSjNbDnzA3a8APg88mmhtIiJSUNXh7u7PA8cmOOXTwDfL574IzDGzjjTLExGRIlJ07pcAB0YdHwQW\nJLhdEREpKNUDqlZxrPc0EBFpomkJbuMQ0DnqeEH5sjOYmQa+iEgB7l55B7qqFPfcNwOfBTCzpcCb\n7n5krBPdPezXfffd1/Q1aH/am/YX76uoqvfczexbwEeBuWZ2ALgPOLc8rNe5+xYzW25me4F+4HOF\nV5Oxffv2NXsJDRV5f5H3Btrf2arqcHf3VTWcc3ea5YiISAp6hWoia9asafYSGiry/iLvDbS/s5XV\n0+lM6jcy86n6vUREojAzvEkPqApQKpWavYSGiry/yHuD1tqfmelrgq+UUjwVUkSkZvoJfmyph7tq\nGRGZMuWKodnLaEnj/dmolhERkREa7om0Uq/ZCJH3F3lvEH9/MjYNdxGRgNS5i8iUGatXXrRoCf39\njfs9Z82CXbu2N+43SCR1565ny4hIU/X3w7x5jRu+vb1LGnbbrUy1TCLRe83I+4u8N4i/vxS+8pWv\nsHLlyjMuW7t2LV/4wheatKL66Z67SHCrV98FtDd7GS2tp6eHBx54gOPHj3PhhRdy+vRpnnzySbZt\n29bspRWm4Z5Id3d3s5fQUJH3F3lvQ9obWntMxuuvp32hTirz58/nhhtu4Nvf/jZ33nkn27ZtY968\neVx99dXNXlphqmVERIDVq1ezceNGADZu3EhPT0+TV1QfDfdEoveakfcXeW8AAwN9zV5CFlasWMHO\nnTt5+eWXefrpp7n99tubvaS6aLiLiAAzZsxg5cqV3HbbbVx33XUsWLCg2Uuqi57nLhLcpZcuaZnO\n/Wc/a+3nub/wwgtcf/31rF+/ntWrVzduUWPQ89xFJJRWeoFRZ2cn559/PjfffHOzl1I31TKJRO9t\nI+8v8t5AnXutBgcH+drXvsaqVau44IILmr2cuumeu4ic9fr7++no6OCyyy7L+rnto6lzFwmu1Tt3\nGaL3cxcRkao03BOJ3ttG3l/kvYE697OVhruISEDq3EWCU+eeB3XuIiJSlYZ7ItF728j7i7w3UOd+\nttLz3EWkqZYsWkSj339g+65dhX95d3c3PT093HHHHSOXlUolenp6OHDgQNVfP5lzU9JwTyT6e4JH\n3l/kvQHMmNHiH9TR38/2efMadvNLenvr+vVmhllrvg/9RFTLiMhZr96P2Vu/fj2LFi1i9uzZXH75\n5Tz22GPA0Ctfb7zxRg4fPkx7ezuzZ8/mjTfeSL7+sWi4JxK9t428v8h7A3Xutejp6WHbtm0cP34c\nYORj9obfGbLaM3w6Ojp4+umnOXHiBOvXr+eLX/wiL730ErNmzWLbtm1cfPHF9PX1ceLECebPn9/w\n/YCGu4jIGR+zB5zxMXvuztq1a7noootGvm666aYzqprly5dz2WWXAXDDDTfwyU9+kueffx6o/g9D\no2i4JxK9t428v8h7gww69xYx3sfsmRlf//rXOXbs2MjXU089dcbQ3rp1K0uXLuW9730vF110EVu2\nbOHo0aNN2ccwDXcRESb3MXujB/vJkye5+eabueeee/jtb3/LsWPHWL58+cg5zXowVsM9kei9beT9\nRd4bqHOv1UQfszdRtXLq1ClOnTrF3LlzOeecc9i6dSvPPvvsyPUdHR0cPXqUEydONHT9lfRUSBFp\nrlmz6n66YrXbr9Xq1at5/PHHWb9+/RmXj3Xve/iy9vZ2Hn74YW655RZOnjzJTTfdxIoVK0bOu+qq\nq1i1ahXvf//7GRwcZNeuXVPyoGrV95Yxs2XAQ0Ab8Li7P1hx/YXARqCToX8svuruG8a4Hb23TMYa\n/TmX9ZjMZ2SejfTeMrV7/fXXWbhwIUeOHJnyT2Oa0s9QNbM24BHgE8Ah4Kdmttndd4867R+Bl939\nJjObC+wxs43ufnqyi5HW1d9PywyISr29S5q9BAkg2sfsVevcrwX2uvs+d38H2ASsqDhnEJhd/n42\ncPRsHOzqbfOl7KS/v5/Zs2fz3HPP8cADDzR7OUlU69wvAUa/IcJB4LqKcx4BfmBmh4F24JZ0yxMR\nabxZs2bx1ltvNXsZSVUb7rWUY8uAn7v7x8zscuB/zezD7v6uuwtr1qyhq6sLgDlz5rB48eKR5xgP\n33vK9Xj4slZZTyP219dXor29e+R7oGWO69lfd3d30/98G3k8Y0Z70/MZPpbqSqUSGzZsABiZl0VM\n+ICqmS0F7nf3ZeXjLwGDox9UNbOngP9w9xfKx88B97r79orb0gOqGWulB+Uq9fYuYf/+1lxbK2il\n7Fr9AdVmmuoP69gOXGFmXWZ2HnArsLninNcZesAVM+sArgR+PdmF5E69bb6UnUQ0YS3j7qfN7G7g\nGYaeCvmEu+82s7vK168DvgxsMLOdgAH3uPvvG7xuEclUjm+fmyN9hqrUpJV+tK+kWmZiyi5v+gxV\nEREZoeGeiHrbfCm7vEXPrygNdxGRgDTcE9F7gudL2eUten5FabiLiASk4Z5I9N4vcm+r7PIWPb+i\npvT93C+9tDXfvU9vGSsi0UzpcG/l59rWK3rvF7m3VXZ5i55fUaplREQC0nBPJHrvF7m3VXZ5i55f\nURruIiIBabgnEr33i9zbKru8Rc+vKA13EZGANNwTid77Re5tlV3eoudXlIa7iEhAGu6JRO/9Ive2\nyi5v0fMrSsNdRCQgDfdEovd+kXtbZZe36PkVpeEuIhKQhnsi0Xu/yL2tsstb9PyK0nAXEQlIwz2R\n6L1f5N5W2eUten5FabiLiASk4Z5I9N4vcm+r7PIWPb+iNNxFRALScE8keu8XubdVdnmLnl9RGu4i\nIgFpuCcSvfeL3Nsqu7xFz68oDXcRkYA03BOJ3vtF7m2VXd6i51eUhruISEAa7olE7/0i97bKLm/R\n8ytKw11EJCAN90Si936Re1tll7fo+RWl4S4iElDV4W5my8zsFTN71czuHeecbjN7ycxeNrNS8lVm\nIHrvF7m3VXZ5i55fUdMmutLM2oBHgE8Ah4Cfmtlmd9896pw5wH8Cn3L3g2Y2t5ELFhGR6qrdc78W\n2Ovu+9z9HWATsKLinNuA77r7QQB3/136Zba+6L1f5N5W2eUten5FVRvulwAHRh0fLF822hXAe8zs\nh2a23cx6Ui5QREQmb8JaBvAabuNc4C+BjwMzgR+Z2Y/d/dV6F5eT6L1f5N5W2eUten5FVRvuh4DO\nUcedDN17H+0A8Dt3/wPwBzP7P+DDwLuG+2uvrWH69C4A2trmMHPmYtrbuwHo6ysBNO14+Ee74b8o\nOj7zeGCgj76+Usvkpfwmd9zsfMY7HtbsP59WOi6VSmzYsAGArq4uijL38e+cm9k0YA9D98oPAz8B\nVlU8oHoVQw+6fgqYDrwI3Oruuypuy6+5ppYfBKZeb+8S9u/fXtdtlEql0PcgOjqupLNzT7OXMaZ6\n81N2zaP/96ozM9zdJvvrJrzn7u6nzexu4BmgDXjC3Xeb2V3l69e5+ytmtg3YCQwC/1052EVEZGpV\nq2Vw963A1orL1lUcfxX4atql5SXyPQeI3dsqu7xFz68ovUJVRCQgDfdEoj/XNvJzpZVd3qLnV5SG\nu4hIQBruiUTv/SL3tsoub9HzK6rqA6oiIo3y5uFfseTSS5u9jLHNmsX2Xfk+8U/DnTR/wfoGBmif\nMSPRikZpkb9grdzb1ptfw7KDlsivlbOb6YNsnzevrtso9fXR3Z7+p5Mlvb3Jb3Mqabijv2C5qze/\nRmUHyk+aR517Io0aDq0icm+r7PIWPb+iNNxFRALScE+k1Ne6vWYKrdzb1kvZ5S16fkVpuIuIBKTh\nnkj03i9yb6vs8hY9v6I03EVEAtJwTyR67xe5t1V2eYueX1Ea7iIiAWm4JxK994vc2yq7vEXPrygN\ndxGRgDTcE4ne+0XubZVd3qLnV5SGu4hIQBruiUTv/SL3tsoub9HzK0rDXUQkIA33RKL3fpF7W2WX\nt+j5FaXhLiISkIZ7ItF7v8i9rbLLW/T8itJwFxEJSMM9kei9X+TeVtnlLXp+RWm4i4gEpOGeSPTe\nL3Jvq+zyFj2/ojTcRUQC0nBPJHrvF7m3VXZ5i55fURruIiIBabgnEr33i9zbKru8Rc+vKA13EZGA\nNNwTid77Re5tlV3eoudXlIa7iEhAGu6JRO/9Ive2yi5v0fMrqupwN7NlZvaKmb1qZvdOcN5fmdlp\nM/vbtEsUEZHJmnC4m1kb8AiwDFgErDKzheOc9yCwDbAGrLPlRe/9Ive2yi5v0fMrqto992uBve6+\nz93fATYBK8Y475+A7wC9idcnIiIFVBvulwAHRh0fLF82wswuYWjgP1q+yJOtLiPRe7/Iva2yy1v0\n/IqqNtxrGdQPAf/i7s5QJXNW1jIiIq1kWpXrDwGdo447Gbr3Pto1wCYzA5gL3Ghm77j75sobe+21\nNUyf3gVAW9scZs5cTHt7NwB9fSWAph0P93bD9wIme/zQkSMsnjmz8K8f73hYqTS03u7u7qYcnzhx\nhBkzSi2TV8r8Rv9ZR8xvYKCv6fmMdzysFfPrGxj40/qmMK9SqcSGDRsA6OrqoigbusM9zpVm04A9\nwMeBw8BPgFXuvnuc89cDP3D3741xnV9zTWs2Nod2TOc3iz9Y122U+voa8uPhkt5etu/fn/x2J6uj\n40o6O/c0exljqje/RmUHrZFf5Owg/v97Zoa7T7oRmfCeu7ufNrO7gWeANuAJd99tZneVr19XaLUB\nRe/9Ive2yi5v0fMrqlotg7tvBbZWXDbmUHf3zyVal4iI1EGvUE0k+nNtIz9XWtnlLXp+RWm4i4gE\npOGeSPTeL3Jvq+zyFj2/ojTcRUQC0nBPJHrvF7m3VXZ5i55fURruIiIBabgnEr33i9zbKru8Rc+v\nKA13EZGANNwTid77Re5tlV3eoudXlIa7iEhAGu6JRO/9Ive2yi5v0fMrSsNdRCQgDfdEovd+kXtb\nZZe36PkVpeEuIhKQhnsi0Xu/yL2tsstb9PyK0nAXEQlIwz2R6L1f5N5W2eUten5FabiLiASk4Z5I\n9N4vcm+r7PIWPb+iNNxFRALScE8keu8XubdVdnmLnl9RGu4iIgFpuCcSvfeL3Nsqu7xFz68oDXcR\nkYA03BOJ3vtF7m2VXd6i51eUhruISEAa7olE7/0i97bKLm/R8ytKw11EJCAN90Si936Re1tll7fo\n+RWl4S4iEpCGeyLRe7/Iva2yy1v0/IrScBcRCUjDPZHovV/k3lbZ5S16fkVpuIuIBKThnkj03i9y\nb6vs8hY9v6I03EVEAqppuJvZMjN7xcxeNbN7x7j+djP7hZntNLMXzOxD6Zfa2qL3fpF7W2WXt+j5\nFVV1uJtZG/AIsAxYBKwys4UVp/0auMHdPwR8GXgs9UJFRKR2tdxzvxbY6+773P0dYBOwYvQJ7v4j\ndz9ePnwRWJB2ma0veu8XubdVdnmLnl9RtQz3S4ADo44Pli8bzx3AlnoWJSIi9ZlWwzle642Z2ceA\nvwc+Mtb1r722hunTuwBoa5vDzJmLaW/vBqCvrwTQtOPh3m74XsBkjx86coTFM2cW/vXjHQ8rlYbW\n293d3ZTjEyeOMGNGqWXySpnf6D/riPkNDPQ1PZ/xjoe1Yn59AwN/Wt8U5lUqldiwYQMAXV1dFGXu\nE89uM1sK3O/uy8rHXwIG3f3BivM+BHwPWObue8e4Hb/mmpr/nZhSh3ZM5zeLP1jXbZT6+hry4+GS\n3l6279+f/HYnq6PjSjo79zR7GWOqN79GZQetkV/k7CD+/3tmhrvbZH9dLbXMduAKM+sys/OAW4HN\nFb/5+xga7J8Za7CfDaL3fpF7W2WXt+j5FVW1lnH302Z2N/AM0AY84e67zeyu8vXrgH8HLgIeNTOA\nd9z92sYtW0REJlJL5467bwW2Vly2btT3dwJ3pl1aXhr5o30riPxcaWWXt+j5FaVXqIqIBKThnkj0\new6Re1tll7fo+RWl4S4iEpCGeyLR398icm+r7PIWPb+iNNxFRALScE8keu8XubdVdnmLnl9RGu4i\nIgFpuCcSvfeL3Nsqu7xFz68oDXcRkYA03BOJ3vtF7m2VXd6i51eUhruISEAa7olE7/0i97bKLm/R\n8ytKw11EJCAN90Si936Re1tll7fo+RWl4S4iEpCGeyLRe7/Iva2yy1v0/IrScBcRCUjDPZHovV/k\n3lbZ5S16fkVpuIuIBKThnkj03i9yb6vs8hY9v6I03EVEAtJwTyR67xe5t1V2eYueX1Ea7iIiAWm4\nJxK994vc2yq7vEXPrygNdxGRgDTcE4ne+0XubZVd3qLnV5SGu4hIQBruiUTv/SL3tsoub9HzK0rD\nXUQkIA33RKL3fpF7W2WXt+j5FaXhLiISkIZ7ItF7v8i9rbLLW/T8itJwFxEJSMM9kei9X+TeVtnl\nLXp+RWm4i4gEVHW4m9kyM3vFzF41s3vHOefh8vW/MLOr0y+z9UXv/SL3tsoub9HzK2rC4W5mbcAj\nwDJgEbDKzBZWnLMc+IC7XwF8Hni0QWttaTvefrvZS2ioU6fi7k/Z5S16fkVVu+d+LbDX3fe5+zvA\nJmBFxTmfBr4J4O4vAnPMrCP5Slvcm3/8Y7OX0FCDg3H3p+zyFj2/oqoN90uAA6OOD5Yvq3bOgvqX\nJiIiRVUb7l7j7VjBXxfGvpMnm72Ehjp9Ou7+lF3eoudXlLmPP4fNbClwv7svKx9/CRh09wdHnfNf\nQMndN5WPXwE+6u5HKm7rrBv4IiIpuHvlHeiqplW5fjtwhZl1AYeBW4FVFedsBu4GNpX/MXizcrAX\nXZyIiBQz4XB399NmdjfwDNAGPOHuu83srvL169x9i5ktN7O9QD/wuYavWkREJjRhLSMiInlK/grV\nyC96qrY3M+s2s+Nm9lL569+asc4izOwbZnbEzH45wTlZ5gbV95dzdgBm1mlmPzSzX5nZy2a2dpzz\nssywlv3lmqGZzTCzF81sR3lv949z3uSyc/dkXwxVN3uBLuBcYAewsOKc5cCW8vfXAT9OuYZGfdW4\nt25gc7PXWnB/1wNXA78c5/osc5vE/rLNrrz++cDi8vcXAHui/L83if1lmyEws/zfacCPgevqzS71\nPffIL3qqZW/w7qeFZsHdnweOTXBKrrkBNe0PMs0OwN3fcPcd5e/fAnYDF1eclm2GNe4PMs3Q3Ydf\nZnseQ3ceBytOmXR2qYd75Bc91bI3B/66/GPTFjNbNGWra7xcc6tVmOzKz267Gnix4qoQGU6wv2wz\nNLNzzGwHcAR41t1/WnHKpLOr9lTIyYr8oqda1vhzoNPd3zazG4HvA3/e2GVNqRxzq1WI7MzsAuA7\nwD+X7+G+65SK46wyrLK/bDN090FgsZldCPyPmf2Fu/+q4rRJZZf6nvshoHPUcSdD/8JMdM6C8mWt\nrure3L1v+Mcrd98KnGtm75m6JTZUrrnVJEJ2ZnYu8F1go7t/f4xTss6w2v4iZOjux4EfMvRmjaNN\nOrvUw33kRU9mdh5DL3raXHHOZuCzMPIK2DFf9NSCqu7NzDrMzMrfX8vQU01/P/VLbYhcc6tJ7tmV\n1/4EsMvdHxrntGwzrGV/uWZoZnPNbE75+/OBv2HoMYXRJp1d0lrGA7/oqZa9ASuBfzCz08DbwN81\nbcGTZGbfAj4KzDWzA8B9DD2wk3Vuw6rtj4yzK/sI8Blgp5m9VL7sX4H3QYgMq+6PfDP8M+CbNvQW\n6+cAT5azqmtu6kVMIiIB6WP2REQC0nAXEQlIw11EJCANdxGRgDTcRUQC0nAXEQlIw11EJCANdxGR\ngP4fVq+mNBBtk1kAAAAASUVORK5CYII=\n", 192 | "text/plain": [ 193 | "" 194 | ] 195 | }, 196 | "metadata": {}, 197 | "output_type": "display_data" 198 | } 199 | ], 200 | "source": [ 201 | "#Compare estimate, yHat, to actually score\n", 202 | "bar([0,1,2], y, width = 0.35, alpha=0.8)\n", 203 | "bar([0.35,1.35,2.35],yHat, width = 0.35, color='r', alpha=0.8)\n", 204 | "grid(1)\n", 205 | "legend(['y', 'yHat'])" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "Right now our predictions are pretty inaccurate. To improve our model, we first need to quantify exactly how wrong our predictions are. We'll do this with a cost function. A cost function allows us to express exactly how wrong or \"costly\" our models is, given our examples." 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "One way to compute an overall cost is to take each error value, square it, and add these values together. Multiplying by one half will make things simpler down the road. Now that we have a cost, or job is to minimize it. When someone says they’re training a network, what they really mean is that they're minimizing a cost function. " 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "$$\n", 227 | "J = \\sum \\frac{1}{2}(y-\\hat{y})^2 \\tag{5}\n", 228 | "$$" 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "OUR cost is a function of two things, our examples, and the weights on our synapses. We don't have much control of our data, so we'll minimize our cost by changing the weights." 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "Conceptually, this is pretty simple concept. We have a collection of 9 individual weights, and we're saying that there is some combination of w's that will make our cost, J, as small as possible. When I first saw this problem in machine learning, I thought, I'll just try ALL THE WEIGHTS UNTIL I FIND THE BEST ONE! After all I have a computer!" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "Enter the CURSE OF DIMENSIONALITY. Here's the problem. Let's pretend for a second that we only have 1 weight, instead of 9. To find the ideal value of our weight that will minimize our cost, we need to try a bunch of values for W, let's say we test 1000 values. That doesn't seem so bad, after all, my computer is pretty fast." 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 7, 255 | "metadata": { 256 | "collapsed": false 257 | }, 258 | "outputs": [], 259 | "source": [ 260 | "import time\n", 261 | "\n", 262 | "weightsToTry = np.linspace(-5,5,1000)\n", 263 | "costs = np.zeros(1000)\n", 264 | "\n", 265 | "startTime = time.clock()\n", 266 | "for i in range(1000):\n", 267 | " NN.W1[0,0] = weightsToTry[i]\n", 268 | " yHat = NN.forward(X)\n", 269 | " costs[i] = 0.5*sum((y-yHat)**2)\n", 270 | " \n", 271 | "endTime = time.clock()" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": 8, 277 | "metadata": { 278 | "collapsed": false 279 | }, 280 | "outputs": [ 281 | { 282 | "data": { 283 | "text/plain": [ 284 | "0.065137" 285 | ] 286 | }, 287 | "execution_count": 8, 288 | "metadata": {}, 289 | "output_type": "execute_result" 290 | } 291 | ], 292 | "source": [ 293 | "timeElapsed = endTime-startTime\n", 294 | "timeElapsed" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "It takes about 0.04 seconds to check 1000 different weight values for our neural network. Since we’ve computed the cost for a wide range values of W, we can just pick the one with the smallest cost, let that be our weight, and we’ve trained our network." 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 9, 307 | "metadata": { 308 | "collapsed": false 309 | }, 310 | "outputs": [ 311 | { 312 | "data": { 313 | "text/plain": [ 314 | "" 315 | ] 316 | }, 317 | "execution_count": 9, 318 | "metadata": {}, 319 | "output_type": "execute_result" 320 | }, 321 | { 322 | "data": { 323 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEPCAYAAACp/QjLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFdW19/HvEsUJtR2IKCAthgRM1BYjQXFoFbHlGnEM\ncYhpM1zMFWMI1yhKlHuNA4ljcjMQh6BihDijF8SxoyIOqC0OTMZLBCKR1yhqEhVkvX/sgx57oKu7\nT3Wd2v37PE8/6apTdc5azwm9rL1q7zJ3R0REpNgGWQcgIiLlR8VBREQaUXEQEZFGVBxERKQRFQcR\nEWlExUFERBpJtTiYWY2ZLTCzxWZ2dhOvn2RmL5jZPDObbWa7Jz1XRETSY2nNczCzLsBCYCiwHHgG\nOMHd5xcdsw/wiruvMrMaYIK7D05yroiIpCfNK4dBwKvuvsTdVwNTgRHFB7j7HHdfVdh8CuiV9FwR\nEUlPmsWhJ7C0aHtZYV9zvgPMaOO5IiJSQhum+N6Jx6vM7CDg28CQ1p4rIiKll2ZxWA70LtruTbgC\n+IxCE/oaoMbd327luSoiIiJt4O62vtfTHFaaC/Qzs0oz6wqMBKYXH2BmOwF3ACe7+6utOXcdd4/2\n54ILLsg8BuWn/DpjfjHn5p7sv6lTu3Jw9zVmNhqYBXQBrnP3+WY2qvD6JOB8YGvgN2YGsNrdBzV3\nblqxlqslS5ZkHUKqlF++xZxfzLklleawEu4+E5jZYN+kot+/C3w36bkiItIxNEO6jNXW1mYdQqqU\nX77FnF/MuSWV2iS4jmBmnuf4RUSyYGZ4hg1paae6urqsQ0iV8su3mPOLObekVBxERKQRDSuJiHQy\nGlYSEZE2UXEoY7GPeyq/fIs5v5hzS0rFQUREGlHPQUSkk1HPQURE2kTFoYzFPu6p/PIt5vxizi0p\nFQcREWlEPQcRkU5GPQcREWkTFYcyFvu4p/LLt5jzizm3pFQcRESkEfUcREQ6GfUcRESkTVQcyljs\n457KL99izi/m3JJScRARkUbUcxAR6WTUcxARkTZRcShjsY97Kr98izm/mHNLKtXiYGY1ZrbAzBab\n2dlNvN7fzOaY2QdmNrbBa2PM7CUze9HM/mBmG6cZq4iIfCq1noOZdQEWAkOB5cAzwAnuPr/omO5A\nH+Ao4G13v7ywvyfwGDDA3T80s2nADHe/ocFndFjP4W9/g7lzYcgQqKjokI8UEUlF1j2HQcCr7r7E\n3VcDU4ERxQe4+0p3nwusbuL8DYHNzGxDYDNCgcnMG2/AFVdA796w557wgx/AbbeFoiEiEps0i0NP\nYGnR9rLCvha5+3LgcuB14K/AO+7+YMkjbIWqKnjoIXjrLfj1r6FXL5g8Gfr3Dz/f+x7cdBMsW1a6\nz4x93FP55VvM+cWcW1IbpvjebR7vMbOtgSOBSmAVcKuZneTuNzc8tra2lsrKSgAqKiqoqqqiuroa\n+PQLTmN7n31g0KA6xoyB7bar5tFH4dpr6xg9GnbcsZpDD4UePeqoqoLhw9v2efX19anFXw7byi/f\n27HnF9N2XV0dkydPBvjk72VL0uw5DAYmuHtNYXscsNbdJzZx7AXA+0U9h+OBw9z9u4XtbwKD3f30\nBueV3TyHtWvh+efhgQfCz1NPhWGoQw8NP3vvDRumWZJFRFqQdc9hLtDPzCrNrCswEpjezLENg/wL\nMNjMNjUzIzS1X0kv1NLZYAPYay8455wwDPXmmzB+PLz7Lpx2Gmy/PZx0EkydCm+/nXW0IiJNS604\nuPsaYDQwi/CHfZq7zzezUWY2CsDMepjZUmAMMN7MXjezbu7+NHAb8Bwwr/CWv0sr1jRtthkcdhhc\ndhm88EL4OeAAmDIF+vSBgw6Cyy+HRYsan7vusjBWyi/fYs4v5tySSnWAw91nAjMb7JtU9PsKoHcz\n504AJqQYXiZ69YJRo8LPP/8Zri7uvTcUic03h2OPheOPD0NRIiJZ0dpKZcIdnn023B57661h33HH\nhUKx115g6x0dFBFJLknPQcWhDLlDfX0oErfeCmvWhEJx4onhlloVChFpj6wb0tJGZmFYadiwOhYt\ngrvugq5d4ZhjYPfd4Wc/g+WZTgksjdjHdZVffsWcW1IqDmXODPbYAy66CP78Z/jVr2DxYthtt3Br\n7I03wvvvZx2liMRGw0o59a9/wfTpYVb244+H3sSoUfCVr2QdmYiUO/UcOok33oDrr4drroFttw1F\n4oQTYIstso5MRMqReg45l3Tcc4cd4LzzwrDTRRfBzJlhDsVpp8GLL6YbY3vEPq6r/PIr5tySUnGI\nSJcuUFMDd94ZisKOO4YJeEOHwv/+b1jaQ0QkCQ0rRe6jj2DaNLjyyjDp7oc/hFNOCTO3RaRzUs9B\nPuEOjz4aisTs2WHI6cwzYbvtso5MRDqaeg45V8pxTzM48MAwZ2LOnLAg4Be+AGedBStWlOxjWiX2\ncV3ll18x55aUikMn9PnPw6RJYRHADz+EXXeFM86A11/POjIRKRcaVhJWrAiPQL32Whg5En7yk9DM\nFpE4aVhJEunRIyzJsWgRdOsWZl+fc46eNyHSmak4lLGOHvfcbjv4+c/Don9vvRV6EpdeGu5ySkPs\n47rKL79izi0pFQdppHfvMNv68cfhueegXz+YPFnzJEQ6E/UcpEVPPx1ue12zBq6+GvbdN+uIRKQ9\nNM9BSmbtWrjlFjj77HBL7MSJ4al2IpI/akjnXDmNe26wAZx0EixYAH37hocOXXoprF7d9vcsp/zS\noPzyK+bcklJxkFbp1g0uvBCeeSbMuB44MEyqE5G4aFhJ2sw9PMZ0zBg48ki45BKoqMg6KhFpiYaV\nJFVm8PWvw8svh+1dd4U77sg2JhEpDRWHMpaXcc+KCvjNb8JVxDnnwMknJ5tAl5f82kr55VfMuSWV\nanEwsxozW2Bmi83s7CZe729mc8zsAzMb2+C1CjO7zczmm9krZjY4zVil/YYMCRPott02zLKeMSPr\niESkrVLrOZhZF2AhMBRYDjwDnODu84uO6Q70AY4C3nb3y4teuwH4k7tfb2YbApu7+6oGn6GeQ5l6\n5BH49rfhkEPCuk1bbpl1RCKyTtY9h0HAq+6+xN1XA1OBEcUHuPtKd58LfOaGSDPbCtjf3a8vHLem\nYWGQ8nbQQTBvXuhLDBwIc+dmHZGItEaaxaEnsLRoe1lhXxI7AyvN7Pdm9pyZXWNmne7ZZXkf99xi\ni7AMxyWXwPDh4QqieAmOvOfXEuWXXzHnltSGKb53e8Z7NgQGAqPd/Rkzuwo4Bzi/4YG1tbVUVlYC\nUFFRQVVVFdXV1cCnX3Bet+vr68sqnrZuH398NXvvDcOH1zFtGtxzTzWf+1w8+cX+/XXW/GLarqur\nY/LkyQCf/L1sSZo9h8HABHevKWyPA9a6+8Qmjr0AeH9dz8HMegBz3H3nwvZ+wDnufkSD89RzyJHV\nq+H88+HGG+Hmm6Hw/2ER6WBZ9xzmAv3MrNLMugIjgenNHPuZIN19BbDUzL5Q2DUUeDm1SKVDbLRR\nGGL6/e/hG98Iw0yq7SLlKbXi4O5rgNHALOAVYJq7zzezUWY2CsIVgpktBcYA483sdTPrVniLM4Cb\nzewFYHfg4rRiLVfrLgtjM2wYPPkk/Pa3dZx4IvzjH1lHlI5Yv791Ys4v5tySSrPngLvPBGY22Dep\n6PcVQO9mzn0B2DvN+CQ7lZXwy1+GlV4HD4Y77wzPthaR8qC1lSRT7mF29YQJoRdRU5N1RCLx0/Mc\nJDdmz4bjjoPzzoPRo7OORiRuWTekpZ1iH/cszm/IkFAgfv1rOOOM8NS5vOtM319sYs4tKRUHKRt9\n+8ITT8DChWEJ8HffzToikc5Lw0pSdlavDlcPs2fDvfdCnz5ZRyQSFw0rSS5ttFFoUp96Kuy3H7z0\nUtYRiXQ+Kg5lLPZxz/XlZwY/+hFMnAgHHwyPP95xcZVKZ/7+8i7m3JJScZCyduKJMGUKHH00TG9u\nfr2IlJx6DpILTz8dmtQXXxyeEyEibZek55DqDGmRUhk0CP70JzjsMHjrLTjrrKwjEombhpXKWOzj\nnq3N74tfDL2Ha6+F//7v8l+0T99ffsWcW1K6cpBc6dUrXEEccgh8+CH89KeheS0ipaWeg+TSypVh\nddeDD4bLLlOBEGkNzXOQaHXvDg89BI89FtZiKn78qIi0n4pDGYt93LO9+W2zDTzwANTXw6hR5Vcg\n9P3lV8y5JaXiILm21VYwaxYsWBCuIDTKKFIa6jlIFN59N/QgBg+GK69UD0JkfdRzkE5jyy3hvvtC\nD+Lss3UFIdJeKg5lLPZxz1LnV1ERehCzZsH555f0rdtE319+xZxbUprnIFHZZht48EGoroaNN4bx\n47OOSCSf1HOQKK1YAQceGO5i+tGPso5GpLxobSXptHr0CFcQ++0XriZqa7OOSCRf1HMoY7GPe6ad\nX+/eof8wbhzcfXeqH9UkfX/5FXNuSaVaHMysxswWmNliMzu7idf7m9kcM/vAzMY28XoXM3vezO5J\nM06JV//+cM898L3vgf69iySXWs/BzLoAC4GhwHLgGeAEd59fdEx3oA9wFPC2u1/e4D1+BOwFbOHu\nRzbxGeo5SCKPPAIjR4bbXQcOzDoakWxlPc9hEPCquy9x99XAVGBE8QHuvtLd5wKrG55sZr2A4cC1\ngKY0SbscdBD87ndwxBGwaFHW0YiUvzSLQ09gadH2ssK+pK4EzgLKbMWcjhP7uGdH53fUUXDRRWEm\n9bJl6X+evr/8ijm3pNK8W6nN4z1mdgTwprs/b2bV6zu2traWyspKACoqKqiqqqK6Opyy7gvO63Z9\nfX1ZxRNDfjvvDKefXk1NDVx6aR3dusWVX0dux55fTNt1dXVMnjwZ4JO/ly1Js+cwGJjg7jWF7XHA\nWnef2MSxFwDvr+s5mNnFwDeBNcAmwJbA7e5+SoPz1HOQVnOHH/4Q5s0LPYiNN846IpGOlXXPYS7Q\nz8wqzawrMBKY3syxnwnS3c91997uvjPwDeDhhoVBpK3M4IorYNttw/yHclvqW6QcpFYc3H0NMBqY\nBbwCTHP3+WY2ysxGAZhZDzNbCowBxpvZ62bWram3SyvOcrbusjBWWebXpQvcdBMsXQrnnJPOZ+j7\ny6+Yc0sq1RnS7j4TmNlg36Si31cAvVt4jz8Bf0olQOnUNt00TI4bMiRMmDvjjKwjEikfWltJOr0l\nS0KB+J//gaOPzjoakfRpbSWRBCorwyzqmhrYfnvYd9+sIxLJXos9BzO7Kck+Kb3Yxz3LKb+BA+HG\nG+GYY2DhwtK8Zznll4aY84s5t6SSNKS/XLxhZhsSlrQQiUpNDVxyCRx+OPztb1lHI5KtZnsOZnYu\nMA7YFPhX0Uurgd+5e0r3eCSnnoOkYcIEmDEjrMe0+eZZRyNSekl6Di02pM3s0nIoBE1RcZA0uMOp\np8Lf/w533hluexWJSakmwd27bu6BmX3TzK4wsz4liVDWK/Zxz3LNzyws0vevf8GZZ4Zi0Rblml+p\nxJxfzLkllaQ4/Ab4p5ntAfwIeA24MdWoRDLWtSvcdhs8+miYTS3S2SQZVnre3fcsrH+03N2vNbPn\n3D3zVfE1rCRpW7YM9tknFIjjj886GpHSKNU8h/cKzemTgf0LD/HZqBQBipS7Xr3g3nvh0ENhhx3C\nM6lFOoMkw0ojgQ+BbxeWu+gJ/DzVqASIf9wzL/ntsQdMmQLHHde6ORB5ya+tYs4v5tySarE4uPsb\nwM1AReE5Cx+4u3oO0qkMGwYXXwzDh2sOhHQOSXoOXydcKaxb/O4A4Cx3vzXl2FqknoN0tAsuCM+A\neOQR2GyzrKMRaZtSzXOYBwx19zcL292Bh9x995JF2kYqDtLR3MMzIFatgttv1xwIyadSzXMwYGXR\n9ls0eDiPpCP2cc885mcG11wD778fnia3vv82yWN+rRFzfjHnllSS4nAfMMvMas3sVGAGDZ7RINKZ\ndO0arhrq6uDKK7OORiQd61tbqR+wvbs/bmbHAkMKL70D/MHdX+2gGJulYSXJ0tKlYXlvzYGQvGlX\nz8HM/hcY5+7zGuzfHbjI3b9WskjbSMVBslZfH+5kuvPO8MAgkTxob89h+4aFAaCwb+f2Bicti33c\nM4b8qqrCs6iPPRYWLfrsazHktz4x5xdzbkmtrzhUrOe1TUodiEheHXYYXHRRmAPx5ptZRyNSGusb\nVpoKPOzuv2uw/3uEW1tHdkB866VhJSkn558P998PDz+sORBS3trbc+gB3Al8BDxb2L0XsDFwdGHm\ndKZUHKScuMO3vgXvvRdWdNUcCClX7eo5FNZR2hf4L2AJ8H/Af7n74HIoDJ1B7OOeseVnBtdeGybI\njRkDjzxSl3VIqYrt+ysWc25JrXeegwcPu/sv3P2X7v5waz/AzGrMbIGZLTazs5t4vb+ZzTGzD8xs\nbNH+3mb2iJm9bGYvmdkPWvvZIh2ta1e4444wtHTbbVlHI9J2LS6f0a43D8t7LwSGAsuBZ4AT3H1+\n0THdgT7AUcDb7n55YX8PoIe71xeeRPcscFSDczWsJGXp9dfDHIirrw53MomUk1Itn9Eeg4BX3X2J\nu68GpgIjig9w95XuPhdY3WD/CnevL/z+PjAf2DHleEVKYqed4J574PvfhyeeyDoakdZLuzj0BJYW\nbS8r7GsVM6sE9gSeKklUORH7uGfs+a1aVceNN4Yrh8WLs46m9GL+/mLOLakkT4Jrj3aP+RSGlG4D\nzixcQXxGbW0tlZWVAFRUVFBVVUV1dTXw6Rec1+36+vqyikf5tT6/qiq48MJqDj8cLrusjoqK8olP\n31/n2a6rq2Py5MkAn/y9bEnaPYfBwAR3rylsjwPWuvvEJo69AHh/Xc+hsG8j4F5gprtf1cQ56jlI\nLowfDw8+qDkQUh7KoecwF+hnZpVm1pXwyNHpzRz7mUDNzIDrgFeaKgwieXLhhdCvH5x8Mnz8cdbR\niLQs1eLg7muA0cAs4BVgmrvPN7NRZjYKwl1JZrYUGAOMN7PXC0NJQ4CTgYPM7PnCT02a8ZabdZeF\nsepM+ZnBddfBO+/A2LHNn5MnMX9/MeeWVNo9B9x9Jg2e/+Duk4p+XwH0buLUx0n/ykakw3QtzIHY\nbz/42c/gxz/OOiKR5qXac0ibeg6SR8uWwQEHhOJw2mlZRyOdUZKeQ+pXDiLyWb16heb0gQdCt26h\nDyFSbjRsU8ZiH/fszPn17RtWcD3rrPCgoDyK+fuLObekdOUgkpEBA2DGDKipCbe3HnZY1hGJfEo9\nB5GMPfEEjBgBt98eehEiaSuHeQ4i0oJ994VbboHjjoOnOtUCMVLOVBzKWOzjnsrvU0OHwuTJ8LWv\nwZNPphZSScX8/cWcW1IqDiJlYvjwUCCOPBLmzMk6Guns1HMQKTP33QennAJ33RWGnERKTT0HkRyq\nqYGbboKjjoLZs7OORjorFYcyFvu4p/Jr3mGHwZQpcPTR8NhjpYuplGL+/mLOLSkVB5EyNWwY/OEP\n4WFBM2e2fLxIKannIFLm5swJQ0y/+AWMHJl1NBIDra0kEoF99glrMdXUhCW/R43KOiLpDDSsVMZi\nH/dUfsntths8+ihMnAiXXlqyt22XmL+/mHNLSsVBJCd22QUefzw0qseOhbVrs45IYqaeg0jO/P3v\n4S6m7t3DLa+bbpp1RJI3mucgEqFttgnLfW+8MRx8MKxcmXVEEiMVhzIW+7in8mu7jTcOVw2HHBIa\n1osWpfZRzYr5+4s5t6R0t5JITm2wAfz0p1BZCfvvD9OmQXV11lFJLNRzEInAgw/CSSfBT34Cp58O\ntt7RZOnskvQcVBxEIvHaa2Gy3N57w69+BZtsknVEUq7UkM652Mc9lV9p9e0bnir37rtheOmvf033\n82L+/mLOLalUi4OZ1ZjZAjNbbGZnN/F6fzObY2YfmNnY1pwrIo116wZ//GN4JsTee4P+xklbpTas\nZGZdgIXAUGA58AxwgrvPLzqmO9AHOAp4290vT3pu4TgNK4k04/774VvfgtNOg/HjoUuXrCOScpH1\nsNIg4FV3X+Luq4GpwIjiA9x9pbvPBVa39lwRWb9hw+DZZ8PVw7BhsGJF1hFJnqRZHHoCS4u2lxX2\npX1uNGIf91R+6dtxx3An0/77w8CB8MADpXvvcsgvLTHnllSa8xzaM96T+Nza2loqKysBqKiooKqq\niurCzd7rvuC8btfX15dVPMovv/lNmABbblnHiSfCCSdUc+ml8PTT8eSn7fVv19XVMXnyZIBP/l62\nJM2ew2BggrvXFLbHAWvdfWITx14AvF/Uc0h0rnoOIq3z9tswejTMnQs33ghf/WrWEUkWsu45zAX6\nmVmlmXUFRgLTmzm2YZCtOVdEEtp6a7j5ZrjoIhgxIjSqP/oo66ikHKVWHNx9DTAamAW8Akxz9/lm\nNsrMRgGYWQ8zWwqMAcab2etm1q25c9OKtVytuyyMlfLLznHHQX09zJsHe+0VnjbXWuWcX3vFnFtS\nqa6t5O4zgZkN9k0q+n0F0DvpuSJSOj16wN13h3kRxx4bZldffDFUVGQdmZQDLZ8hIrz9NowbB/fc\nA1ddFa4stD5TvLS2koi0yuzZ4RnVO+wAV14JX/5y1hFJGrJuSEs7xT7uqfzKz5Ah8PzzYfmNgw8O\ndza99VbTx+Yxv6Rizi0pFQcR+YyNNoIzzoD5hVtA+veHq6/WXU2djYaVRGS9Xn4Z/vM/YcECmDAB\nTj5Z6zTlnXoOIlIyjz4K550XhpkuvBCOOUZN67xSzyHnYh/3VH75csABoUBccUWYRNe/fx133QVr\n12YdWenF9t21hYqDiCRmBjU1YfmNk04Kz7DebTe46SZY3XBtZck1DSuJSJu5h1VfL74Y/u//4Kyz\noLYWNt8868hkfTSsJCKpMoNDD4VHHoFbbglLgvfpE4rEkiVZRyftoeJQxmIf91R++dYwv332gbvu\ngqefDlcUX/kKHH10KBx5u8CP/btLQsVBREqqb1+47DL4y19Cf2L0aBgwAH72Mz2NLk/UcxCRVLnD\nE0/AddfBnXfCgQfCd74Dhx8OG6a69Kc0R/McRKSsvPceTJsG118fGtgjR8I3vhEeOqQ5Ex1HDemc\ni33cU/nlW1vy22IL+O53w5VEXV1YHvxb34JddoFzzw3PlyiH/96L/btLQsVBRDLxxS+G5TgWLIDb\nb4ePP4avfQ123TUsH/7kk3FOsMsLDSuJSNlYuzbc7XT33TB9Ovz976FgjBgBhxwCm2ySdYRxUM9B\nRHJt8eJQJO6+G154AfbfH4YNCz9f/KL6FG2lnkPOxT7uqfzyrSPy69cPxo4Nazq99lroT8ybF4pD\nnz6hf3Hrrc0/c6KtYv/uktCNZCKSC9tuC8cfH37cYeFCuP9+uOGGcGvsTjuFxQEPOCBcYfTsmXXE\n+aZhJRHJvTVroL4+XGE89lj42WqrUCT23z/cKjtggJ5DsY56DiLSKa1dG55k99hj8Pjjocn9xhuw\n114waBDsvXf435126px9C/Ucci72cU/ll2/lnN8GG8CXvgSnnQZTpsCiRWE5j3PPDVcUU6aEq4ke\nPeDf/i3snzYt3Fb78cflnVtHSbXnYGY1wFVAF+Bad5/YxDG/AA4H/gnUuvvzhf1jgO8ADrwInOru\nH6YZr4jEa5ttPr3TCULfYtkyePbZcCfUtGnhSXdvvAG9eoXhqD32gN13D3MvunfPNv6Oltqwkpl1\nARYCQ4HlwDPACe4+v+iY4cBodx9uZl8Frnb3wWbWE3gMGODuH5rZNGCGu9/Q4DM0rCQiJfXee/Di\ni+GuqBdeCD/z54d+Rf/+n/0ZMAAqK/PXy0gyrJTmlcMg4FV3X1IIZiowAphfdMyRwA0A7v6UmVWY\n2fZFsW1mZh8DmxEKjIhIqrbYAvbdN/ys4w4rV4YisWBB+HnoofC/K1bA5z8fbrvt2/fTn112Cbfb\ndu2aXS7tkWZx6AksLdpeBnw1wTE93f05M7sceB34FzDL3R9MMdayVFdXR3V1ddZhpEb55VvM+TXM\nzQw+97nwc+CBnz32n/8Mt9X++c9hLsa8eWH12ddeC8NWPXp8Wiz69g1XGjvtFArHDjuU78q0aYaV\ndLyn0aWNmW1NuKqoBFYBt5rZSe5+c8Nja2trqaysBKCiooKqqqpPvtR1TaW8btfX15dVPMpP+XWm\n/FqzveeesGpVHdttBz/+8aevf/wx7LxzNa+9BjNn1hWGq6p5/XVYtKiOVaugZ89qjjwSjjkmvfjq\n6uqYPHkywCd/L1uSZs9hMDDB3WsK2+OAtcVNaTP7LVDn7lML2wuAA4EDgMPc/buF/d8EBrv76Q0+\nQz0HEcmtjz6C5cvhww9DD6OjZH0r61ygn5lVmllXYCQwvcEx04FT4JNi8o67/40wnDTYzDY1MyM0\ntV9JMVYRkQ7XtSvsvHPHFoakUisO7r4GGA3MIvxhn+bu881slJmNKhwzA3jNzF4FJgH/Udj/FHAb\n8Bwwr/CWv0sr1nK17rIwVsov32LOL+bckkq1FeLuM4GZDfZNarA9uplzJwAT0opNRESap+UzREQ6\nmax7DiIiklMqDmUs9nFP5ZdvMecXc25JqTiIiEgj6jmIiHQy6jmIiEibqDiUsdjHPZVfvsWcX8y5\nJaXiICIijajnICLSyajnICIibaLiUMZiH/dUfvkWc34x55aUioOIiDSinoOISCejnoOIiLSJikMZ\ni33cU/nlW8z5xZxbUioOIiLSiHoOIiKdjHoOIiLSJioOZSz2cU/ll28x5xdzbkmpOIiISCPqOYiI\ndDLqOYiISJukWhzMrMbMFpjZYjM7u5ljflF4/QUz27Nof4WZ3WZm883sFTMbnGas5Sj2cU/ll28x\n5xdzbkmlVhzMrAvwP0ANsCtwgpkNaHDMcODz7t4P+HfgN0UvXw3McPcBwO7A/LRiLVf19fVZh5Aq\n5ZdvMecXc25JpXnlMAh41d2XuPtqYCowosExRwI3ALj7U0CFmW1vZlsB+7v79YXX1rj7qhRjLUvv\nvPNO1iGkSvnlW8z5xZxbUmkWh57A0qLtZYV9LR3TC9gZWGlmvzez58zsGjPbLMVYRUSkSJrFIelt\nRA075g6nudgUAAAFzElEQVRsCAwEfu3uA4F/AOeUMLZcWLJkSdYhpEr55VvM+cWcW1Kp3cpaaCBP\ncPeawvY4YK27Tyw65rdAnbtPLWwvAA4kFIw57r5zYf9+wDnufkSDz9B9rCIibdDSrawbpvjZc4F+\nZlYJ/BUYCZzQ4JjpwGhgaqGYvOPufwMws6Vm9gV3XwQMBV5u+AEtJSciIm2TWnFw9zVmNhqYBXQB\nrnP3+WY2qvD6JHefYWbDzexVwtDRqUVvcQZws5l1Bf7c4DUREUlRrmdIi4hIOqKYIW1mZxQmy71k\nZhNbPiN/zGysma01s22yjqWUzOznhe/uBTO7o3Abc64lmfyZV2bW28weMbOXC//efpB1TGkwsy5m\n9ryZ3ZN1LKWWdIJx7ouDmR1EmC+xu7t/Gbgs45BKzsx6A4cCf8k6lhTcD3zJ3fcAFgHjMo6nXZJM\n/sy51cAYd/8SMBg4PbL81jkTeIXkd13mSaIJxrkvDsD3gUsKE+1w95UZx5OGK4AfZx1EGtz9AXdf\nW9h8ijDPJc+STP7MLXdf4e71hd/fJ/xh2THbqErLzHoBw4FraXyrfa61ZoJxDMWhH3CAmT1pZnVm\n9pWsAyolMxsBLHP3eVnH0gG+DczIOoh2SjL5MwqFOxH3JBT1mFwJnAWsbenAHEo8wTjNW1lLxswe\nAHo08dJ5hBy2dvfBZrY38Eegb0fG114t5DcOGFZ8eIcEVULrye9cd7+ncMx5wEfu/ocODa70YhyG\naMTMugG3AWcWriCiYGZHAG+6+/NmVp11PClYN8F4tLs/Y2ZXESYYn9/UgWXP3Q9t7jUz+z5wR+G4\nZwpN223d/a0OC7CdmsvPzL5MqPQvmBmEIZdnzWyQu7/ZgSG2y/q+PwAzqyVcxh/SIQGlaznQu2i7\nN+HqIRpmthFwOzDF3e/KOp4S2xc4srAo6CbAlmZ2o7ufknFcpbKMMBLxTGH7NppZfSKGYaW7gIMB\nzOwLQNc8FYb1cfeX3H17d9+5MFt8GTAwT4WhJWZWQ7iEH+HuH2QdTwl8MvmzMEdnJGGyZxQs/FfK\ndcAr7n5V1vGUmruf6+69C//evgE8HFFhwN1XAEsLfyuhmQnGkJMrhxZcD1xvZi8CHwHRfJFNiHHI\n4pdAV+CBwtXRHHf/j2xDarvmJn9mHFYpDQFOBuaZ2fOFfePc/b4MY0pTjP/mEk0w1iQ4ERFpJIZh\nJRERKTEVBxERaUTFQUREGlFxEBGRRlQcRESkERUHERFpRMVBpAlmdqWZnVm0PcvMrinavtzMxjRz\n7n+Z2Xpne5vZBDMb28T+rQqz/kUypeIg0rTHCUspYGYbANsSluBeZx9gdlMnuvsF7v5QC+/f3ASj\nrYHcTgKUeKg4iDRtDqEAAHwJeAl4r/CglI2BAQCFlYDnmtl9ZtajsG+ymR1b+H144aEqc83sFw0e\nHrNr4cE5fzazMwr7LgV2KTxoJsoHV0k+xLB8hkjJuftfzWxN4UFL+xCKRc/C7+8SnmNwJWFNqP9n\nZiOBi4DvEK4K3Mw2AX5LWD//L2b2Bz69YjCgP1ANbAksNLNfA2cTHn60ZwelKtIkFQeR5j1BGFra\nl/DApZ6F31cRVl8dxqdrQnUB/lp07ro//q+5+7on+N0C/HvhdwfuLTwQ6C0zexPYnhwuyS5xUnEQ\nad5swkJzuwEvEh7i85+E4lAH9HT3fddzfsO+QsM//B8V/f4x+vcoZUQ9B5HmPQEcAbzlwdtABWFo\n6Rag+7qHs5vZRmZW3LB2YCHQ18z6FPaN5LPDSk15D9iitGmItJ6Kg0jzXiLcpfRk0b55wDuFZ5Uf\nB0w0s3rgeT5tYANQeD7FfwD3mdlcQq9i3fN6nSbuWCo8i2S2mb2ohrRkSUt2i6TIzDZ3938Ufv8V\nsMjdr844LJEW6cpBJF3fK9yW+jLhrqRJWQckkoSuHEREpBFdOYiISCMqDiIi0oiKg4iINKLiICIi\njag4iIhIIyoOIiLSyP8HyBw/RNRrVoUAAAAASUVORK5CYII=\n", 324 | "text/plain": [ 325 | "" 326 | ] 327 | }, 328 | "metadata": {}, 329 | "output_type": "display_data" 330 | } 331 | ], 332 | "source": [ 333 | "plot(weightsToTry, costs)\n", 334 | "grid(1)\n", 335 | "ylabel('Cost')\n", 336 | "xlabel('Weight')" 337 | ] 338 | }, 339 | { 340 | "cell_type": "markdown", 341 | "metadata": {}, 342 | "source": [ 343 | "So you may be thinking that 0.04 seconds to train a network is not so bad, and we haven't even optimized anything yet. Plus, there are other, way faster languages than python our there. " 344 | ] 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "metadata": {}, 349 | "source": [ 350 | "Before we optimize through, let's consider the full complexity of the problem. Remember the 0.04 seconds required is only for one weight, and we have 9 total! Let's next consider 2 weights for a moment. To maintain the same precision we now need to check 1000 times 1000, or one million values. This is a lot of work, even for a fast computer." 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 10, 356 | "metadata": { 357 | "collapsed": false 358 | }, 359 | "outputs": [], 360 | "source": [ 361 | "weightsToTry = np.linspace(-5,5,1000)\n", 362 | "costs = np.zeros((1000, 1000))\n", 363 | "\n", 364 | "startTime = time.clock()\n", 365 | "for i in range(1000):\n", 366 | " for j in range(1000):\n", 367 | " NN.W1[0,0] = weightsToTry[i]\n", 368 | " NN.W1[0,1] = weightsToTry[j]\n", 369 | " yHat = NN.forward(X)\n", 370 | " costs[i, j] = 0.5*sum((y-yHat)**2)\n", 371 | " \n", 372 | "endTime = time.clock()" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 11, 378 | "metadata": { 379 | "collapsed": false 380 | }, 381 | "outputs": [ 382 | { 383 | "data": { 384 | "text/plain": [ 385 | "62.06971600000001" 386 | ] 387 | }, 388 | "execution_count": 11, 389 | "metadata": {}, 390 | "output_type": "execute_result" 391 | } 392 | ], 393 | "source": [ 394 | "timeElapsed = endTime-startTime\n", 395 | "timeElapsed" 396 | ] 397 | }, 398 | { 399 | "cell_type": "markdown", 400 | "metadata": {}, 401 | "source": [ 402 | "After our 1 million evaluations we’ve found our solution, but it took an agonizing 40 seconds! The real curse of dimensionality kicks in as we continue to add dimensions. Searching through three weights would take a billion evaluations, or 11 hours! Searching through all 9 weights we need for our simple network would take 1,268,391,679,350,583.5 years. (Over a quardrillion years). So for that reason, the \"just try everything\" or brute force optimization method is clearly not going to work." 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 12, 408 | "metadata": { 409 | "collapsed": false 410 | }, 411 | "outputs": [ 412 | { 413 | "data": { 414 | "text/plain": [ 415 | "1268391679350583.5" 416 | ] 417 | }, 418 | "execution_count": 12, 419 | "metadata": {}, 420 | "output_type": "execute_result" 421 | } 422 | ], 423 | "source": [ 424 | "0.04*(1000**(9-1))/(3600*24*365)" 425 | ] 426 | }, 427 | { 428 | "cell_type": "markdown", 429 | "metadata": {}, 430 | "source": [ 431 | "Let's return to the 1-dimensional case and see if we can be more clever. Let's evaluate our cost function for a specific value of w. If w is 1.1 for example, we can run our cost function, and see that J is 2.8. Now we haven't learned much yet, but let's try to add a little information to what we already know. What if we could figure out which way was downhill? If we could, we would know whether to make W smaller or larger do decrease the cost. We could test the cost function immediately to the left and right of our test point and see which is smaller. This is called numerical estimation, and is sometimes a good approach, but for us, there's a faster way. Let's look at our equations so far." 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": {}, 437 | "source": [ 438 | "$$\n", 439 | "z^{(2)} = XW^{(1)} \\tag{1}\\\\\n", 440 | "$$\n", 441 | "$$\n", 442 | "a^{(2)} = f(z^{(2)}) \\tag{2}\\\\\n", 443 | "$$\n", 444 | "$$\n", 445 | "z^{(3)} = a^{(2)}W^{(2)} \\tag{3}\\\\\n", 446 | "$$\n", 447 | "$$\n", 448 | "\\hat{y} = f(z^{(3)}) \\tag{4}\\\\\n", 449 | "$$\n", 450 | "$$\n", 451 | "J = \\sum \\frac{1}{2}(y-\\hat{y})^2 \\tag{5}\\\\\n", 452 | "$$" 453 | ] 454 | }, 455 | { 456 | "cell_type": "markdown", 457 | "metadata": {}, 458 | "source": [ 459 | "We have 5 equations, but we can really think of them as one big equation.\n" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "And since we have one big equation that uniquely determines our cost, J, from X, y, W1, and W2, we can use our good friend calculus to find what we're looking for. We want to know \"which way is downhill\", that is, what is the rate of change of J with respect to W, also known as the derivative. And in this case, since we’re just considering one weight at a time, the partial derivative. " 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "We can derive an expression for dJdW, that will give us the rate of change of J with respect to W, for any value of W! If dJdW is positive, then the cost function is going uphill. If dJdW is negative the cost function is going downhill. " 474 | ] 475 | }, 476 | { 477 | "cell_type": "markdown", 478 | "metadata": {}, 479 | "source": [ 480 | "Now we can really speed things up. Since we know in which direction the cost decreases, we can save all that time we would have spent searching in the wrong direction. We can save even more computational time by iteratively taking steps downhill and stopping when the cost stops getting smaller. " 481 | ] 482 | }, 483 | { 484 | "cell_type": "markdown", 485 | "metadata": {}, 486 | "source": [ 487 | "This method is known as gradient descent, and although it may not seem so impressive in one dimension, it is capable of incredible speedups in higher dimensions. In fact, in our final video, we’ll show that what would have taken 10^27 function evaluations with our brute force method will take less than 100 evaluations with gradient descent. Gradient descent allows us to find needles in very very very large haystacks. " 488 | ] 489 | }, 490 | { 491 | "cell_type": "markdown", 492 | "metadata": {}, 493 | "source": [ 494 | "Now before we celebrate too much here, there is a restriction. What if our cost function doesn't always go in the same direction? What if it goes up, then back down? The mathematical name for this is non-convex, and it could really throw off our gradient descent algorithm by getting it stuck in a local minima instead of our ideal global minima. One of the reasons we chose our cost function to be the sum of squared errors was to exploit the convex nature of quadratic equations." 495 | ] 496 | }, 497 | { 498 | "cell_type": "markdown", 499 | "metadata": {}, 500 | "source": [ 501 | "We know that the graph of y equals x squared is a nice convex parabola and it turns out that higher dimensional versions are too!" 502 | ] 503 | }, 504 | { 505 | "cell_type": "markdown", 506 | "metadata": {}, 507 | "source": [ 508 | "Another piece of the puzzle here is that depending on how we use our data, it might not matter if or cost function is convex or not. If we use our examples one at a time instead of all at once, sometimes it won't matter if our cost function is convex, we will still find a good solution. This is called stochastic gradient descent. So maybe we shouldn't be afraid of non-convex cost functions, as Neural Network wizard Yann Lecun says in his excellent talk \"Who is afraid on non-convex loss funtions?\"" 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "metadata": {}, 514 | "source": [ 515 | "Link to Yann's Talk:\n", 516 | "http://videolectures.net/eml07_lecun_wia/" 517 | ] 518 | }, 519 | { 520 | "cell_type": "markdown", 521 | "metadata": {}, 522 | "source": [ 523 | "The details of gradient descent are a deep topic for another day, for now we're going to do our gradient descent \"batch\" style, where we use all our example at once, and the way we've setup our cost function will keep things nice and convex. Next time we'll compute and code up our gradients!" 524 | ] 525 | } 526 | ], 527 | "metadata": { 528 | "kernelspec": { 529 | "display_name": "Python 2", 530 | "language": "python", 531 | "name": "python2" 532 | }, 533 | "language_info": { 534 | "codemirror_mode": { 535 | "name": "ipython", 536 | "version": 2 537 | }, 538 | "file_extension": ".py", 539 | "mimetype": "text/x-python", 540 | "name": "python", 541 | "nbconvert_exporter": "python", 542 | "pygments_lexer": "ipython2", 543 | "version": "2.7.10" 544 | } 545 | }, 546 | "nbformat": 4, 547 | "nbformat_minor": 0 548 | } 549 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Part 3 Gradient Descent-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 3: Gradient Descent

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 1, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('5u0jaA3qAGk')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "

Variables

\n", 53 | "\n", 54 | "|Code Symbol | Math Symbol | Definition | Dimensions\n", 55 | "| :-: | :-: | :-: | :-: |\n", 56 | "|X|$$X$$|Input Data, each row in an example| (numExamples, inputLayerSize)|\n", 57 | "|y |$$y$$|target data|(numExamples, outputLayerSize)|\n", 58 | "|W1 | $$W^{(1)}$$ | Layer 1 weights | (inputLayerSize, hiddenLayerSize) |\n", 59 | "|W2 | $$W^{(2)}$$ | Layer 2 weights | (hiddenLayerSize, outputLayerSize) |\n", 60 | "|z2 | $$z^{(2)}$$ | Layer 2 activation | (numExamples, hiddenLayerSize) |\n", 61 | "|a2 | $$a^{(2)}$$ | Layer 2 activity | (numExamples, hiddenLayerSize) |\n", 62 | "|z3 | $$z^{(3)}$$ | Layer 3 activation | (numExamples, outputLayerSize) |\n", 63 | "|J | $$J$$ | Cost | (1, outputLayerSize) |" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "Last time we built a neural network in python that made really bad predictions of your score on a test based on how many hours you slept and how many hours you studied the night before. This time we'll focus on the theory of making those predictions better." 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "We can initialize the network we built last time and pass in our normalized data, X, using our forward method, and have a look at our estimate of y, yHat." 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 1, 83 | "metadata": { 84 | "collapsed": false 85 | }, 86 | "outputs": [ 87 | { 88 | "name": "stdout", 89 | "output_type": "stream", 90 | "text": [ 91 | "Populating the interactive namespace from numpy and matplotlib\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "%pylab inline\n", 97 | "\n", 98 | "#Import code from last time:\n", 99 | "from partTwo import *" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 2, 105 | "metadata": { 106 | "collapsed": false 107 | }, 108 | "outputs": [], 109 | "source": [ 110 | "NN = Neural_Network()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 3, 116 | "metadata": { 117 | "collapsed": false 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "yHat = NN.forward(X)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 4, 127 | "metadata": { 128 | "collapsed": false 129 | }, 130 | "outputs": [ 131 | { 132 | "data": { 133 | "text/plain": [ 134 | "array([[ 0.55850487],\n", 135 | " [ 0.55993739],\n", 136 | " [ 0.56372181]])" 137 | ] 138 | }, 139 | "execution_count": 4, 140 | "metadata": {}, 141 | "output_type": "execute_result" 142 | } 143 | ], 144 | "source": [ 145 | "yHat" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 5, 151 | "metadata": { 152 | "collapsed": false 153 | }, 154 | "outputs": [ 155 | { 156 | "data": { 157 | "text/plain": [ 158 | "array([[ 0.75],\n", 159 | " [ 0.82],\n", 160 | " [ 0.93]])" 161 | ] 162 | }, 163 | "execution_count": 5, 164 | "metadata": {}, 165 | "output_type": "execute_result" 166 | } 167 | ], 168 | "source": [ 169 | "y" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 6, 175 | "metadata": { 176 | "collapsed": false 177 | }, 178 | "outputs": [ 179 | { 180 | "data": { 181 | "text/plain": [ 182 | "" 183 | ] 184 | }, 185 | "execution_count": 6, 186 | "metadata": {}, 187 | "output_type": "execute_result" 188 | }, 189 | { 190 | "data": { 191 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEACAYAAABI5zaHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF4xJREFUeJzt3X+M1PWdx/Hn20VBcBFbyHLK1rXWU7i0xZNTco12m/Za\nJLEkJ9FDu4WepuZyHtf+o9fL5dT0j4tp0xjrxcPTQhOSYvrjGqqAXkwnZ0xrSyulFiSSCvKj0i1F\nWNcuSPd9f+zsdhl3d2a/85md+bx5PZKN+535Mnw+vPDN7Gt+mbsjIiKxnNPsBYiISHoa7iIiAWm4\ni4gEpOEuIhKQhruISEAa7iIiAVUd7mb2DTM7Yma/nOCch83sVTP7hZldnXaJIiIyWbXcc18PLBvv\nSjNbDnzA3a8APg88mmhtIiJSUNXh7u7PA8cmOOXTwDfL574IzDGzjjTLExGRIlJ07pcAB0YdHwQW\nJLhdEREpKNUDqlZxrPc0EBFpomkJbuMQ0DnqeEH5sjOYmQa+iEgB7l55B7qqFPfcNwOfBTCzpcCb\n7n5krBPdPezXfffd1/Q1aH/am/YX76uoqvfczexbwEeBuWZ2ALgPOLc8rNe5+xYzW25me4F+4HOF\nV5Oxffv2NXsJDRV5f5H3Btrf2arqcHf3VTWcc3ea5YiISAp6hWoia9asafYSGiry/iLvDbS/s5XV\n0+lM6jcy86n6vUREojAzvEkPqApQKpWavYSGiry/yHuD1tqfmelrgq+UUjwVUkSkZvoJfmyph7tq\nGRGZMuWKodnLaEnj/dmolhERkREa7om0Uq/ZCJH3F3lvEH9/MjYNdxGRgNS5i8iUGatXXrRoCf39\njfs9Z82CXbu2N+43SCR1565ny4hIU/X3w7x5jRu+vb1LGnbbrUy1TCLRe83I+4u8N4i/vxS+8pWv\nsHLlyjMuW7t2LV/4wheatKL66Z67SHCrV98FtDd7GS2tp6eHBx54gOPHj3PhhRdy+vRpnnzySbZt\n29bspRWm4Z5Id3d3s5fQUJH3F3lvQ9obWntMxuuvp32hTirz58/nhhtu4Nvf/jZ33nkn27ZtY968\neVx99dXNXlphqmVERIDVq1ezceNGADZu3EhPT0+TV1QfDfdEoveakfcXeW8AAwN9zV5CFlasWMHO\nnTt5+eWXefrpp7n99tubvaS6aLiLiAAzZsxg5cqV3HbbbVx33XUsWLCg2Uuqi57nLhLcpZcuaZnO\n/Wc/a+3nub/wwgtcf/31rF+/ntWrVzduUWPQ89xFJJRWeoFRZ2cn559/PjfffHOzl1I31TKJRO9t\nI+8v8t5AnXutBgcH+drXvsaqVau44IILmr2cuumeu4ic9fr7++no6OCyyy7L+rnto6lzFwmu1Tt3\nGaL3cxcRkao03BOJ3ttG3l/kvYE697OVhruISEDq3EWCU+eeB3XuIiJSlYZ7ItF728j7i7w3UOd+\nttLz3EWkqZYsWkSj339g+65dhX95d3c3PT093HHHHSOXlUolenp6OHDgQNVfP5lzU9JwTyT6e4JH\n3l/kvQHMmNHiH9TR38/2efMadvNLenvr+vVmhllrvg/9RFTLiMhZr96P2Vu/fj2LFi1i9uzZXH75\n5Tz22GPA0Ctfb7zxRg4fPkx7ezuzZ8/mjTfeSL7+sWi4JxK9t428v8h7A3Xutejp6WHbtm0cP34c\nYORj9obfGbLaM3w6Ojp4+umnOXHiBOvXr+eLX/wiL730ErNmzWLbtm1cfPHF9PX1ceLECebPn9/w\n/YCGu4jIGR+zB5zxMXvuztq1a7noootGvm666aYzqprly5dz2WWXAXDDDTfwyU9+kueffx6o/g9D\no2i4JxK9t428v8h7gww69xYx3sfsmRlf//rXOXbs2MjXU089dcbQ3rp1K0uXLuW9730vF110EVu2\nbOHo0aNN2ccwDXcRESb3MXujB/vJkye5+eabueeee/jtb3/LsWPHWL58+cg5zXowVsM9kei9beT9\nRd4bqHOv1UQfszdRtXLq1ClOnTrF3LlzOeecc9i6dSvPPvvsyPUdHR0cPXqUEydONHT9lfRUSBFp\nrlmz6n66YrXbr9Xq1at5/PHHWb9+/RmXj3Xve/iy9vZ2Hn74YW655RZOnjzJTTfdxIoVK0bOu+qq\nq1i1ahXvf//7GRwcZNeuXVPyoGrV95Yxs2XAQ0Ab8Li7P1hx/YXARqCToX8svuruG8a4Hb23TMYa\n/TmX9ZjMZ2SejfTeMrV7/fXXWbhwIUeOHJnyT2Oa0s9QNbM24BHgE8Ah4Kdmttndd4867R+Bl939\nJjObC+wxs43ufnqyi5HW1d9PywyISr29S5q9BAkg2sfsVevcrwX2uvs+d38H2ASsqDhnEJhd/n42\ncPRsHOzqbfOl7KS/v5/Zs2fz3HPP8cADDzR7OUlU69wvAUa/IcJB4LqKcx4BfmBmh4F24JZ0yxMR\nabxZs2bx1ltvNXsZSVUb7rWUY8uAn7v7x8zscuB/zezD7v6uuwtr1qyhq6sLgDlz5rB48eKR5xgP\n33vK9Xj4slZZTyP219dXor29e+R7oGWO69lfd3d30/98G3k8Y0Z70/MZPpbqSqUSGzZsABiZl0VM\n+ICqmS0F7nf3ZeXjLwGDox9UNbOngP9w9xfKx88B97r79orb0gOqGWulB+Uq9fYuYf/+1lxbK2il\n7Fr9AdVmmuoP69gOXGFmXWZ2HnArsLninNcZesAVM+sArgR+PdmF5E69bb6UnUQ0YS3j7qfN7G7g\nGYaeCvmEu+82s7vK168DvgxsMLOdgAH3uPvvG7xuEclUjm+fmyN9hqrUpJV+tK+kWmZiyi5v+gxV\nEREZoeGeiHrbfCm7vEXPrygNdxGRgDTcE9F7gudL2eUten5FabiLiASk4Z5I9N4vcm+r7PIWPb+i\npvT93C+9tDXfvU9vGSsi0UzpcG/l59rWK3rvF7m3VXZ5i55fUaplREQC0nBPJHrvF7m3VXZ5i55f\nURruIiIBabgnEr33i9zbKru8Rc+vKA13EZGANNwTid77Re5tlV3eoudXlIa7iEhAGu6JRO/9Ive2\nyi5v0fMrSsNdRCQgDfdEovd+kXtbZZe36PkVpeEuIhKQhnsi0Xu/yL2tsstb9PyK0nAXEQlIwz2R\n6L1f5N5W2eUten5FabiLiASk4Z5I9N4vcm+r7PIWPb+iNNxFRALScE8keu8XubdVdnmLnl9RGu4i\nIgFpuCcSvfeL3Nsqu7xFz68oDXcRkYA03BOJ3vtF7m2VXd6i51eUhruISEAa7olE7/0i97bKLm/R\n8ytKw11EJCAN90Si936Re1tll7fo+RWl4S4iElDV4W5my8zsFTN71czuHeecbjN7ycxeNrNS8lVm\nIHrvF7m3VXZ5i55fUdMmutLM2oBHgE8Ah4Cfmtlmd9896pw5wH8Cn3L3g2Y2t5ELFhGR6qrdc78W\n2Ovu+9z9HWATsKLinNuA77r7QQB3/136Zba+6L1f5N5W2eUten5FVRvulwAHRh0fLF822hXAe8zs\nh2a23cx6Ui5QREQmb8JaBvAabuNc4C+BjwMzgR+Z2Y/d/dV6F5eT6L1f5N5W2eUten5FVRvuh4DO\nUcedDN17H+0A8Dt3/wPwBzP7P+DDwLuG+2uvrWH69C4A2trmMHPmYtrbuwHo6ysBNO14+Ee74b8o\nOj7zeGCgj76+Usvkpfwmd9zsfMY7HtbsP59WOi6VSmzYsAGArq4uijL38e+cm9k0YA9D98oPAz8B\nVlU8oHoVQw+6fgqYDrwI3Oruuypuy6+5ppYfBKZeb+8S9u/fXtdtlEql0PcgOjqupLNzT7OXMaZ6\n81N2zaP/96ozM9zdJvvrJrzn7u6nzexu4BmgDXjC3Xeb2V3l69e5+ytmtg3YCQwC/1052EVEZGpV\nq2Vw963A1orL1lUcfxX4atql5SXyPQeI3dsqu7xFz68ovUJVRCQgDfdEoj/XNvJzpZVd3qLnV5SG\nu4hIQBruiUTv/SL3tsoub9HzK6rqA6oiIo3y5uFfseTSS5u9jLHNmsX2Xfk+8U/DnTR/wfoGBmif\nMSPRikZpkb9grdzb1ptfw7KDlsivlbOb6YNsnzevrtso9fXR3Z7+p5Mlvb3Jb3Mqabijv2C5qze/\nRmUHyk+aR517Io0aDq0icm+r7PIWPb+iNNxFRALScE+k1Ne6vWYKrdzb1kvZ5S16fkVpuIuIBKTh\nnkj03i9yb6vs8hY9v6I03EVEAtJwTyR67xe5t1V2eYueX1Ea7iIiAWm4JxK994vc2yq7vEXPrygN\ndxGRgDTcE4ne+0XubZVd3qLnV5SGu4hIQBruiUTv/SL3tsoub9HzK0rDXUQkIA33RKL3fpF7W2WX\nt+j5FaXhLiISkIZ7ItF7v8i9rbLLW/T8itJwFxEJSMM9kei9X+TeVtnlLXp+RWm4i4gEpOGeSPTe\nL3Jvq+zyFj2/ojTcRUQC0nBPJHrvF7m3VXZ5i55fURruIiIBabgnEr33i9zbKru8Rc+vKA13EZGA\nNNwTid77Re5tlV3eoudXlIa7iEhAGu6JRO/9Ive2yi5v0fMrqupwN7NlZvaKmb1qZvdOcN5fmdlp\nM/vbtEsUEZHJmnC4m1kb8AiwDFgErDKzheOc9yCwDbAGrLPlRe/9Ive2yi5v0fMrqto992uBve6+\nz93fATYBK8Y475+A7wC9idcnIiIFVBvulwAHRh0fLF82wswuYWjgP1q+yJOtLiPRe7/Iva2yy1v0\n/IqqNtxrGdQPAf/i7s5QJXNW1jIiIq1kWpXrDwGdo447Gbr3Pto1wCYzA5gL3Ghm77j75sobe+21\nNUyf3gVAW9scZs5cTHt7NwB9fSWAph0P93bD9wIme/zQkSMsnjmz8K8f73hYqTS03u7u7qYcnzhx\nhBkzSi2TV8r8Rv9ZR8xvYKCv6fmMdzysFfPrGxj40/qmMK9SqcSGDRsA6OrqoigbusM9zpVm04A9\nwMeBw8BPgFXuvnuc89cDP3D3741xnV9zTWs2Nod2TOc3iz9Y122U+voa8uPhkt5etu/fn/x2J6uj\n40o6O/c0exljqje/RmUHrZFf5Owg/v97Zoa7T7oRmfCeu7ufNrO7gWeANuAJd99tZneVr19XaLUB\nRe/9Ive2yi5v0fMrqlotg7tvBbZWXDbmUHf3zyVal4iI1EGvUE0k+nNtIz9XWtnlLXp+RWm4i4gE\npOGeSPTeL3Jvq+zyFj2/ojTcRUQC0nBPJHrvF7m3VXZ5i55fURruIiIBabgnEr33i9zbKru8Rc+v\nKA13EZGANNwTid77Re5tlV3eoudXlIa7iEhAGu6JRO/9Ive2yi5v0fMrSsNdRCQgDfdEovd+kXtb\nZZe36PkVpeEuIhKQhnsi0Xu/yL2tsstb9PyK0nAXEQlIwz2R6L1f5N5W2eUten5FabiLiASk4Z5I\n9N4vcm+r7PIWPb+iNNxFRALScE8keu8XubdVdnmLnl9RGu4iIgFpuCcSvfeL3Nsqu7xFz68oDXcR\nkYA03BOJ3vtF7m2VXd6i51eUhruISEAa7olE7/0i97bKLm/R8ytKw11EJCAN90Si936Re1tll7fo\n+RWl4S4iEpCGeyLRe7/Iva2yy1v0/IrScBcRCUjDPZHovV/k3lbZ5S16fkVpuIuIBKThnkj03i9y\nb6vs8hY9v6I03EVEAqppuJvZMjN7xcxeNbN7x7j+djP7hZntNLMXzOxD6Zfa2qL3fpF7W2WXt+j5\nFVV1uJtZG/AIsAxYBKwys4UVp/0auMHdPwR8GXgs9UJFRKR2tdxzvxbY6+773P0dYBOwYvQJ7v4j\ndz9ePnwRWJB2ma0veu8XubdVdnmLnl9RtQz3S4ADo44Pli8bzx3AlnoWJSIi9ZlWwzle642Z2ceA\nvwc+Mtb1r722hunTuwBoa5vDzJmLaW/vBqCvrwTQtOPh3m74XsBkjx86coTFM2cW/vXjHQ8rlYbW\n293d3ZTjEyeOMGNGqWXySpnf6D/riPkNDPQ1PZ/xjoe1Yn59AwN/Wt8U5lUqldiwYQMAXV1dFGXu\nE89uM1sK3O/uy8rHXwIG3f3BivM+BHwPWObue8e4Hb/mmpr/nZhSh3ZM5zeLP1jXbZT6+hry4+GS\n3l6279+f/HYnq6PjSjo79zR7GWOqN79GZQetkV/k7CD+/3tmhrvbZH9dLbXMduAKM+sys/OAW4HN\nFb/5+xga7J8Za7CfDaL3fpF7W2WXt+j5FVW1lnH302Z2N/AM0AY84e67zeyu8vXrgH8HLgIeNTOA\nd9z92sYtW0REJlJL5467bwW2Vly2btT3dwJ3pl1aXhr5o30riPxcaWWXt+j5FaVXqIqIBKThnkj0\new6Re1tll7fo+RWl4S4iEpCGeyLR398icm+r7PIWPb+iNNxFRALScE8keu8XubdVdnmLnl9RGu4i\nIgFpuCcSvfeL3Nsqu7xFz68oDXcRkYA03BOJ3vtF7m2VXd6i51eUhruISEAa7olE7/0i97bKLm/R\n8ytKw11EJCAN90Si936Re1tll7fo+RWl4S4iEpCGeyLRe7/Iva2yy1v0/IrScBcRCUjDPZHovV/k\n3lbZ5S16fkVpuIuIBKThnkj03i9yb6vs8hY9v6I03EVEAtJwTyR67xe5t1V2eYueX1Ea7iIiAWm4\nJxK994vc2yq7vEXPrygNdxGRgDTcE4ne+0XubZVd3qLnV5SGu4hIQBruiUTv/SL3tsoub9HzK0rD\nXUQkIA33RKL3fpF7W2WXt+j5FaXhLiISkIZ7ItF7v8i9rbLLW/T8itJwFxEJSMM9kei9X+TeVtnl\nLXp+RWm4i4gEVHW4m9kyM3vFzF41s3vHOefh8vW/MLOr0y+z9UXv/SL3tsoub9HzK2rC4W5mbcAj\nwDJgEbDKzBZWnLMc+IC7XwF8Hni0QWttaTvefrvZS2ioU6fi7k/Z5S16fkVVu+d+LbDX3fe5+zvA\nJmBFxTmfBr4J4O4vAnPMrCP5Slvcm3/8Y7OX0FCDg3H3p+zyFj2/oqoN90uAA6OOD5Yvq3bOgvqX\nJiIiRVUb7l7j7VjBXxfGvpMnm72Ehjp9Ou7+lF3eoudXlLmPP4fNbClwv7svKx9/CRh09wdHnfNf\nQMndN5WPXwE+6u5HKm7rrBv4IiIpuHvlHeiqplW5fjtwhZl1AYeBW4FVFedsBu4GNpX/MXizcrAX\nXZyIiBQz4XB399NmdjfwDNAGPOHuu83srvL169x9i5ktN7O9QD/wuYavWkREJjRhLSMiInlK/grV\nyC96qrY3M+s2s+Nm9lL569+asc4izOwbZnbEzH45wTlZ5gbV95dzdgBm1mlmPzSzX5nZy2a2dpzz\nssywlv3lmqGZzTCzF81sR3lv949z3uSyc/dkXwxVN3uBLuBcYAewsOKc5cCW8vfXAT9OuYZGfdW4\nt25gc7PXWnB/1wNXA78c5/osc5vE/rLNrrz++cDi8vcXAHui/L83if1lmyEws/zfacCPgevqzS71\nPffIL3qqZW/w7qeFZsHdnweOTXBKrrkBNe0PMs0OwN3fcPcd5e/fAnYDF1eclm2GNe4PMs3Q3Ydf\nZnseQ3ceBytOmXR2qYd75Bc91bI3B/66/GPTFjNbNGWra7xcc6tVmOzKz267Gnix4qoQGU6wv2wz\nNLNzzGwHcAR41t1/WnHKpLOr9lTIyYr8oqda1vhzoNPd3zazG4HvA3/e2GVNqRxzq1WI7MzsAuA7\nwD+X7+G+65SK46wyrLK/bDN090FgsZldCPyPmf2Fu/+q4rRJZZf6nvshoHPUcSdD/8JMdM6C8mWt\nrure3L1v+Mcrd98KnGtm75m6JTZUrrnVJEJ2ZnYu8F1go7t/f4xTss6w2v4iZOjux4EfMvRmjaNN\nOrvUw33kRU9mdh5DL3raXHHOZuCzMPIK2DFf9NSCqu7NzDrMzMrfX8vQU01/P/VLbYhcc6tJ7tmV\n1/4EsMvdHxrntGwzrGV/uWZoZnPNbE75+/OBv2HoMYXRJp1d0lrGA7/oqZa9ASuBfzCz08DbwN81\nbcGTZGbfAj4KzDWzA8B9DD2wk3Vuw6rtj4yzK/sI8Blgp5m9VL7sX4H3QYgMq+6PfDP8M+CbNvQW\n6+cAT5azqmtu6kVMIiIB6WP2REQC0nAXEQlIw11EJCANdxGRgDTcRUQC0nAXEQlIw11EJCANdxGR\ngP4fVq+mNBBtk1kAAAAASUVORK5CYII=\n", 192 | "text/plain": [ 193 | "" 194 | ] 195 | }, 196 | "metadata": {}, 197 | "output_type": "display_data" 198 | } 199 | ], 200 | "source": [ 201 | "#Compare estimate, yHat, to actually score\n", 202 | "bar([0,1,2], y, width = 0.35, alpha=0.8)\n", 203 | "bar([0.35,1.35,2.35],yHat, width = 0.35, color='r', alpha=0.8)\n", 204 | "grid(1)\n", 205 | "legend(['y', 'yHat'])" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "Right now our predictions are pretty inaccurate. To improve our model, we first need to quantify exactly how wrong our predictions are. We'll do this with a cost function. A cost function allows us to express exactly how wrong or \"costly\" our models is, given our examples." 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "One way to compute an overall cost is to take each error value, square it, and add these values together. Multiplying by one half will make things simpler down the road. Now that we have a cost, or job is to minimize it. When someone says they’re training a network, what they really mean is that they're minimizing a cost function. " 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "$$\n", 227 | "J = \\sum \\frac{1}{2}(y-\\hat{y})^2 \\tag{5}\n", 228 | "$$" 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "OUR cost is a function of two things, our examples, and the weights on our synapses. We don't have much control of our data, so we'll minimize our cost by changing the weights." 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "Conceptually, this is pretty simple concept. We have a collection of 9 individual weights, and we're saying that there is some combination of w's that will make our cost, J, as small as possible. When I first saw this problem in machine learning, I thought, I'll just try ALL THE WEIGHTS UNTIL I FIND THE BEST ONE! After all I have a computer!" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "Enter the CURSE OF DIMENSIONALITY. Here's the problem. Let's pretend for a second that we only have 1 weight, instead of 9. To find the ideal value of our weight that will minimize our cost, we need to try a bunch of values for W, let's say we test 1000 values. That doesn't seem so bad, after all, my computer is pretty fast." 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 7, 255 | "metadata": { 256 | "collapsed": false 257 | }, 258 | "outputs": [], 259 | "source": [ 260 | "import time\n", 261 | "\n", 262 | "weightsToTry = np.linspace(-5,5,1000)\n", 263 | "costs = np.zeros(1000)\n", 264 | "\n", 265 | "startTime = time.clock()\n", 266 | "for i in range(1000):\n", 267 | " NN.W1[0,0] = weightsToTry[i]\n", 268 | " yHat = NN.forward(X)\n", 269 | " costs[i] = 0.5*sum((y-yHat)**2)\n", 270 | " \n", 271 | "endTime = time.clock()" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": 8, 277 | "metadata": { 278 | "collapsed": false 279 | }, 280 | "outputs": [ 281 | { 282 | "data": { 283 | "text/plain": [ 284 | "0.065137" 285 | ] 286 | }, 287 | "execution_count": 8, 288 | "metadata": {}, 289 | "output_type": "execute_result" 290 | } 291 | ], 292 | "source": [ 293 | "timeElapsed = endTime-startTime\n", 294 | "timeElapsed" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "It takes about 0.04 seconds to check 1000 different weight values for our neural network. Since we’ve computed the cost for a wide range values of W, we can just pick the one with the smallest cost, let that be our weight, and we’ve trained our network." 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 9, 307 | "metadata": { 308 | "collapsed": false 309 | }, 310 | "outputs": [ 311 | { 312 | "data": { 313 | "text/plain": [ 314 | "" 315 | ] 316 | }, 317 | "execution_count": 9, 318 | "metadata": {}, 319 | "output_type": "execute_result" 320 | }, 321 | { 322 | "data": { 323 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEPCAYAAACp/QjLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFdW19/HvEsUJtR2IKCAthgRM1BYjQXFoFbHlGnEM\ncYhpM1zMFWMI1yhKlHuNA4ljcjMQh6BihDijF8SxoyIOqC0OTMZLBCKR1yhqEhVkvX/sgx57oKu7\nT3Wd2v37PE8/6apTdc5azwm9rL1q7zJ3R0REpNgGWQcgIiLlR8VBREQaUXEQEZFGVBxERKQRFQcR\nEWlExUFERBpJtTiYWY2ZLTCzxWZ2dhOvn2RmL5jZPDObbWa7Jz1XRETSY2nNczCzLsBCYCiwHHgG\nOMHd5xcdsw/wiruvMrMaYIK7D05yroiIpCfNK4dBwKvuvsTdVwNTgRHFB7j7HHdfVdh8CuiV9FwR\nEUlPmsWhJ7C0aHtZYV9zvgPMaOO5IiJSQhum+N6Jx6vM7CDg28CQ1p4rIiKll2ZxWA70LtruTbgC\n+IxCE/oaoMbd327luSoiIiJt4O62vtfTHFaaC/Qzs0oz6wqMBKYXH2BmOwF3ACe7+6utOXcdd4/2\n54ILLsg8BuWn/DpjfjHn5p7sv6lTu3Jw9zVmNhqYBXQBrnP3+WY2qvD6JOB8YGvgN2YGsNrdBzV3\nblqxlqslS5ZkHUKqlF++xZxfzLklleawEu4+E5jZYN+kot+/C3w36bkiItIxNEO6jNXW1mYdQqqU\nX77FnF/MuSWV2iS4jmBmnuf4RUSyYGZ4hg1paae6urqsQ0iV8su3mPOLObekVBxERKQRDSuJiHQy\nGlYSEZE2UXEoY7GPeyq/fIs5v5hzS0rFQUREGlHPQUSkk1HPQURE2kTFoYzFPu6p/PIt5vxizi0p\nFQcREWlEPQcRkU5GPQcREWkTFYcyFvu4p/LLt5jzizm3pFQcRESkEfUcREQ6GfUcRESkTVQcyljs\n457KL99izi/m3JJScRARkUbUcxAR6WTUcxARkTZRcShjsY97Kr98izm/mHNLKtXiYGY1ZrbAzBab\n2dlNvN7fzOaY2QdmNrbBa2PM7CUze9HM/mBmG6cZq4iIfCq1noOZdQEWAkOB5cAzwAnuPr/omO5A\nH+Ao4G13v7ywvyfwGDDA3T80s2nADHe/ocFndFjP4W9/g7lzYcgQqKjokI8UEUlF1j2HQcCr7r7E\n3VcDU4ERxQe4+0p3nwusbuL8DYHNzGxDYDNCgcnMG2/AFVdA796w557wgx/AbbeFoiEiEps0i0NP\nYGnR9rLCvha5+3LgcuB14K/AO+7+YMkjbIWqKnjoIXjrLfj1r6FXL5g8Gfr3Dz/f+x7cdBMsW1a6\nz4x93FP55VvM+cWcW1IbpvjebR7vMbOtgSOBSmAVcKuZneTuNzc8tra2lsrKSgAqKiqoqqqiuroa\n+PQLTmN7n31g0KA6xoyB7bar5tFH4dpr6xg9GnbcsZpDD4UePeqoqoLhw9v2efX19anFXw7byi/f\n27HnF9N2XV0dkydPBvjk72VL0uw5DAYmuHtNYXscsNbdJzZx7AXA+0U9h+OBw9z9u4XtbwKD3f30\nBueV3TyHtWvh+efhgQfCz1NPhWGoQw8NP3vvDRumWZJFRFqQdc9hLtDPzCrNrCswEpjezLENg/wL\nMNjMNjUzIzS1X0kv1NLZYAPYay8455wwDPXmmzB+PLz7Lpx2Gmy/PZx0EkydCm+/nXW0IiJNS604\nuPsaYDQwi/CHfZq7zzezUWY2CsDMepjZUmAMMN7MXjezbu7+NHAb8Bwwr/CWv0sr1jRtthkcdhhc\ndhm88EL4OeAAmDIF+vSBgw6Cyy+HRYsan7vusjBWyi/fYs4v5tySSnWAw91nAjMb7JtU9PsKoHcz\n504AJqQYXiZ69YJRo8LPP/8Zri7uvTcUic03h2OPheOPD0NRIiJZ0dpKZcIdnn023B57661h33HH\nhUKx115g6x0dFBFJLknPQcWhDLlDfX0oErfeCmvWhEJx4onhlloVChFpj6wb0tJGZmFYadiwOhYt\ngrvugq5d4ZhjYPfd4Wc/g+WZTgksjdjHdZVffsWcW1IqDmXODPbYAy66CP78Z/jVr2DxYthtt3Br\n7I03wvvvZx2liMRGw0o59a9/wfTpYVb244+H3sSoUfCVr2QdmYiUO/UcOok33oDrr4drroFttw1F\n4oQTYIstso5MRMqReg45l3Tcc4cd4LzzwrDTRRfBzJlhDsVpp8GLL6YbY3vEPq6r/PIr5tySUnGI\nSJcuUFMDd94ZisKOO4YJeEOHwv/+b1jaQ0QkCQ0rRe6jj2DaNLjyyjDp7oc/hFNOCTO3RaRzUs9B\nPuEOjz4aisTs2WHI6cwzYbvtso5MRDqaeg45V8pxTzM48MAwZ2LOnLAg4Be+AGedBStWlOxjWiX2\ncV3ll18x55aUikMn9PnPw6RJYRHADz+EXXeFM86A11/POjIRKRcaVhJWrAiPQL32Whg5En7yk9DM\nFpE4aVhJEunRIyzJsWgRdOsWZl+fc46eNyHSmak4lLGOHvfcbjv4+c/Don9vvRV6EpdeGu5ySkPs\n47rKL79izi0pFQdppHfvMNv68cfhueegXz+YPFnzJEQ6E/UcpEVPPx1ue12zBq6+GvbdN+uIRKQ9\nNM9BSmbtWrjlFjj77HBL7MSJ4al2IpI/akjnXDmNe26wAZx0EixYAH37hocOXXoprF7d9vcsp/zS\noPzyK+bcklJxkFbp1g0uvBCeeSbMuB44MEyqE5G4aFhJ2sw9PMZ0zBg48ki45BKoqMg6KhFpiYaV\nJFVm8PWvw8svh+1dd4U77sg2JhEpDRWHMpaXcc+KCvjNb8JVxDnnwMknJ5tAl5f82kr55VfMuSWV\nanEwsxozW2Bmi83s7CZe729mc8zsAzMb2+C1CjO7zczmm9krZjY4zVil/YYMCRPott02zLKeMSPr\niESkrVLrOZhZF2AhMBRYDjwDnODu84uO6Q70AY4C3nb3y4teuwH4k7tfb2YbApu7+6oGn6GeQ5l6\n5BH49rfhkEPCuk1bbpl1RCKyTtY9h0HAq+6+xN1XA1OBEcUHuPtKd58LfOaGSDPbCtjf3a8vHLem\nYWGQ8nbQQTBvXuhLDBwIc+dmHZGItEaaxaEnsLRoe1lhXxI7AyvN7Pdm9pyZXWNmne7ZZXkf99xi\ni7AMxyWXwPDh4QqieAmOvOfXEuWXXzHnltSGKb53e8Z7NgQGAqPd/Rkzuwo4Bzi/4YG1tbVUVlYC\nUFFRQVVVFdXV1cCnX3Bet+vr68sqnrZuH398NXvvDcOH1zFtGtxzTzWf+1w8+cX+/XXW/GLarqur\nY/LkyQCf/L1sSZo9h8HABHevKWyPA9a6+8Qmjr0AeH9dz8HMegBz3H3nwvZ+wDnufkSD89RzyJHV\nq+H88+HGG+Hmm6Hw/2ER6WBZ9xzmAv3MrNLMugIjgenNHPuZIN19BbDUzL5Q2DUUeDm1SKVDbLRR\nGGL6/e/hG98Iw0yq7SLlKbXi4O5rgNHALOAVYJq7zzezUWY2CsIVgpktBcYA483sdTPrVniLM4Cb\nzewFYHfg4rRiLVfrLgtjM2wYPPkk/Pa3dZx4IvzjH1lHlI5Yv791Ys4v5tySSrPngLvPBGY22Dep\n6PcVQO9mzn0B2DvN+CQ7lZXwy1+GlV4HD4Y77wzPthaR8qC1lSRT7mF29YQJoRdRU5N1RCLx0/Mc\nJDdmz4bjjoPzzoPRo7OORiRuWTekpZ1iH/cszm/IkFAgfv1rOOOM8NS5vOtM319sYs4tKRUHKRt9\n+8ITT8DChWEJ8HffzToikc5Lw0pSdlavDlcPs2fDvfdCnz5ZRyQSFw0rSS5ttFFoUp96Kuy3H7z0\nUtYRiXQ+Kg5lLPZxz/XlZwY/+hFMnAgHHwyPP95xcZVKZ/7+8i7m3JJScZCyduKJMGUKHH00TG9u\nfr2IlJx6DpILTz8dmtQXXxyeEyEibZek55DqDGmRUhk0CP70JzjsMHjrLTjrrKwjEombhpXKWOzj\nnq3N74tfDL2Ha6+F//7v8l+0T99ffsWcW1K6cpBc6dUrXEEccgh8+CH89KeheS0ipaWeg+TSypVh\nddeDD4bLLlOBEGkNzXOQaHXvDg89BI89FtZiKn78qIi0n4pDGYt93LO9+W2zDTzwANTXw6hR5Vcg\n9P3lV8y5JaXiILm21VYwaxYsWBCuIDTKKFIa6jlIFN59N/QgBg+GK69UD0JkfdRzkE5jyy3hvvtC\nD+Lss3UFIdJeKg5lLPZxz1LnV1ERehCzZsH555f0rdtE319+xZxbUprnIFHZZht48EGoroaNN4bx\n47OOSCSf1HOQKK1YAQceGO5i+tGPso5GpLxobSXptHr0CFcQ++0XriZqa7OOSCRf1HMoY7GPe6ad\nX+/eof8wbhzcfXeqH9UkfX/5FXNuSaVaHMysxswWmNliMzu7idf7m9kcM/vAzMY28XoXM3vezO5J\nM06JV//+cM898L3vgf69iySXWs/BzLoAC4GhwHLgGeAEd59fdEx3oA9wFPC2u1/e4D1+BOwFbOHu\nRzbxGeo5SCKPPAIjR4bbXQcOzDoakWxlPc9hEPCquy9x99XAVGBE8QHuvtLd5wKrG55sZr2A4cC1\ngKY0SbscdBD87ndwxBGwaFHW0YiUvzSLQ09gadH2ssK+pK4EzgLKbMWcjhP7uGdH53fUUXDRRWEm\n9bJl6X+evr/8ijm3pNK8W6nN4z1mdgTwprs/b2bV6zu2traWyspKACoqKqiqqqK6Opyy7gvO63Z9\nfX1ZxRNDfjvvDKefXk1NDVx6aR3dusWVX0dux55fTNt1dXVMnjwZ4JO/ly1Js+cwGJjg7jWF7XHA\nWnef2MSxFwDvr+s5mNnFwDeBNcAmwJbA7e5+SoPz1HOQVnOHH/4Q5s0LPYiNN846IpGOlXXPYS7Q\nz8wqzawrMBKY3syxnwnS3c91997uvjPwDeDhhoVBpK3M4IorYNttw/yHclvqW6QcpFYc3H0NMBqY\nBbwCTHP3+WY2ysxGAZhZDzNbCowBxpvZ62bWram3SyvOcrbusjBWWebXpQvcdBMsXQrnnJPOZ+j7\ny6+Yc0sq1RnS7j4TmNlg36Si31cAvVt4jz8Bf0olQOnUNt00TI4bMiRMmDvjjKwjEikfWltJOr0l\nS0KB+J//gaOPzjoakfRpbSWRBCorwyzqmhrYfnvYd9+sIxLJXos9BzO7Kck+Kb3Yxz3LKb+BA+HG\nG+GYY2DhwtK8Zznll4aY84s5t6SSNKS/XLxhZhsSlrQQiUpNDVxyCRx+OPztb1lHI5KtZnsOZnYu\nMA7YFPhX0Uurgd+5e0r3eCSnnoOkYcIEmDEjrMe0+eZZRyNSekl6Di02pM3s0nIoBE1RcZA0uMOp\np8Lf/w533hluexWJSakmwd27bu6BmX3TzK4wsz4liVDWK/Zxz3LNzyws0vevf8GZZ4Zi0Rblml+p\nxJxfzLkllaQ4/Ab4p5ntAfwIeA24MdWoRDLWtSvcdhs8+miYTS3S2SQZVnre3fcsrH+03N2vNbPn\n3D3zVfE1rCRpW7YM9tknFIjjj886GpHSKNU8h/cKzemTgf0LD/HZqBQBipS7Xr3g3nvh0ENhhx3C\nM6lFOoMkw0ojgQ+BbxeWu+gJ/DzVqASIf9wzL/ntsQdMmQLHHde6ORB5ya+tYs4v5tySarE4uPsb\nwM1AReE5Cx+4u3oO0qkMGwYXXwzDh2sOhHQOSXoOXydcKaxb/O4A4Cx3vzXl2FqknoN0tAsuCM+A\neOQR2GyzrKMRaZtSzXOYBwx19zcL292Bh9x995JF2kYqDtLR3MMzIFatgttv1xwIyadSzXMwYGXR\n9ls0eDiPpCP2cc885mcG11wD778fnia3vv82yWN+rRFzfjHnllSS4nAfMMvMas3sVGAGDZ7RINKZ\ndO0arhrq6uDKK7OORiQd61tbqR+wvbs/bmbHAkMKL70D/MHdX+2gGJulYSXJ0tKlYXlvzYGQvGlX\nz8HM/hcY5+7zGuzfHbjI3b9WskjbSMVBslZfH+5kuvPO8MAgkTxob89h+4aFAaCwb+f2Bicti33c\nM4b8qqrCs6iPPRYWLfrsazHktz4x5xdzbkmtrzhUrOe1TUodiEheHXYYXHRRmAPx5ptZRyNSGusb\nVpoKPOzuv2uw/3uEW1tHdkB866VhJSkn558P998PDz+sORBS3trbc+gB3Al8BDxb2L0XsDFwdGHm\ndKZUHKScuMO3vgXvvRdWdNUcCClX7eo5FNZR2hf4L2AJ8H/Af7n74HIoDJ1B7OOeseVnBtdeGybI\njRkDjzxSl3VIqYrt+ysWc25JrXeegwcPu/sv3P2X7v5waz/AzGrMbIGZLTazs5t4vb+ZzTGzD8xs\nbNH+3mb2iJm9bGYvmdkPWvvZIh2ta1e4444wtHTbbVlHI9J2LS6f0a43D8t7LwSGAsuBZ4AT3H1+\n0THdgT7AUcDb7n55YX8PoIe71xeeRPcscFSDczWsJGXp9dfDHIirrw53MomUk1Itn9Eeg4BX3X2J\nu68GpgIjig9w95XuPhdY3WD/CnevL/z+PjAf2DHleEVKYqed4J574PvfhyeeyDoakdZLuzj0BJYW\nbS8r7GsVM6sE9gSeKklUORH7uGfs+a1aVceNN4Yrh8WLs46m9GL+/mLOLakkT4Jrj3aP+RSGlG4D\nzixcQXxGbW0tlZWVAFRUVFBVVUV1dTXw6Rec1+36+vqyikf5tT6/qiq48MJqDj8cLrusjoqK8olP\n31/n2a6rq2Py5MkAn/y9bEnaPYfBwAR3rylsjwPWuvvEJo69AHh/Xc+hsG8j4F5gprtf1cQ56jlI\nLowfDw8+qDkQUh7KoecwF+hnZpVm1pXwyNHpzRz7mUDNzIDrgFeaKgwieXLhhdCvH5x8Mnz8cdbR\niLQs1eLg7muA0cAs4BVgmrvPN7NRZjYKwl1JZrYUGAOMN7PXC0NJQ4CTgYPM7PnCT02a8ZabdZeF\nsepM+ZnBddfBO+/A2LHNn5MnMX9/MeeWVNo9B9x9Jg2e/+Duk4p+XwH0buLUx0n/ykakw3QtzIHY\nbz/42c/gxz/OOiKR5qXac0ibeg6SR8uWwQEHhOJw2mlZRyOdUZKeQ+pXDiLyWb16heb0gQdCt26h\nDyFSbjRsU8ZiH/fszPn17RtWcD3rrPCgoDyK+fuLObekdOUgkpEBA2DGDKipCbe3HnZY1hGJfEo9\nB5GMPfEEjBgBt98eehEiaSuHeQ4i0oJ994VbboHjjoOnOtUCMVLOVBzKWOzjnsrvU0OHwuTJ8LWv\nwZNPphZSScX8/cWcW1IqDiJlYvjwUCCOPBLmzMk6Guns1HMQKTP33QennAJ33RWGnERKTT0HkRyq\nqYGbboKjjoLZs7OORjorFYcyFvu4p/Jr3mGHwZQpcPTR8NhjpYuplGL+/mLOLSkVB5EyNWwY/OEP\n4WFBM2e2fLxIKannIFLm5swJQ0y/+AWMHJl1NBIDra0kEoF99glrMdXUhCW/R43KOiLpDDSsVMZi\nH/dUfsntths8+ihMnAiXXlqyt22XmL+/mHNLSsVBJCd22QUefzw0qseOhbVrs45IYqaeg0jO/P3v\n4S6m7t3DLa+bbpp1RJI3mucgEqFttgnLfW+8MRx8MKxcmXVEEiMVhzIW+7in8mu7jTcOVw2HHBIa\n1osWpfZRzYr5+4s5t6R0t5JITm2wAfz0p1BZCfvvD9OmQXV11lFJLNRzEInAgw/CSSfBT34Cp58O\ntt7RZOnskvQcVBxEIvHaa2Gy3N57w69+BZtsknVEUq7UkM652Mc9lV9p9e0bnir37rtheOmvf033\n82L+/mLOLalUi4OZ1ZjZAjNbbGZnN/F6fzObY2YfmNnY1pwrIo116wZ//GN4JsTee4P+xklbpTas\nZGZdgIXAUGA58AxwgrvPLzqmO9AHOAp4290vT3pu4TgNK4k04/774VvfgtNOg/HjoUuXrCOScpH1\nsNIg4FV3X+Luq4GpwIjiA9x9pbvPBVa39lwRWb9hw+DZZ8PVw7BhsGJF1hFJnqRZHHoCS4u2lxX2\npX1uNGIf91R+6dtxx3An0/77w8CB8MADpXvvcsgvLTHnllSa8xzaM96T+Nza2loqKysBqKiooKqq\niurCzd7rvuC8btfX15dVPMovv/lNmABbblnHiSfCCSdUc+ml8PTT8eSn7fVv19XVMXnyZIBP/l62\nJM2ew2BggrvXFLbHAWvdfWITx14AvF/Uc0h0rnoOIq3z9tswejTMnQs33ghf/WrWEUkWsu45zAX6\nmVmlmXUFRgLTmzm2YZCtOVdEEtp6a7j5ZrjoIhgxIjSqP/oo66ikHKVWHNx9DTAamAW8Akxz9/lm\nNsrMRgGYWQ8zWwqMAcab2etm1q25c9OKtVytuyyMlfLLznHHQX09zJsHe+0VnjbXWuWcX3vFnFtS\nqa6t5O4zgZkN9k0q+n0F0DvpuSJSOj16wN13h3kRxx4bZldffDFUVGQdmZQDLZ8hIrz9NowbB/fc\nA1ddFa4stD5TvLS2koi0yuzZ4RnVO+wAV14JX/5y1hFJGrJuSEs7xT7uqfzKz5Ah8PzzYfmNgw8O\ndza99VbTx+Yxv6Rizi0pFQcR+YyNNoIzzoD5hVtA+veHq6/WXU2djYaVRGS9Xn4Z/vM/YcECmDAB\nTj5Z6zTlnXoOIlIyjz4K550XhpkuvBCOOUZN67xSzyHnYh/3VH75csABoUBccUWYRNe/fx133QVr\n12YdWenF9t21hYqDiCRmBjU1YfmNk04Kz7DebTe46SZY3XBtZck1DSuJSJu5h1VfL74Y/u//4Kyz\noLYWNt8868hkfTSsJCKpMoNDD4VHHoFbbglLgvfpE4rEkiVZRyftoeJQxmIf91R++dYwv332gbvu\ngqefDlcUX/kKHH10KBx5u8CP/btLQsVBREqqb1+47DL4y19Cf2L0aBgwAH72Mz2NLk/UcxCRVLnD\nE0/AddfBnXfCgQfCd74Dhx8OG6a69Kc0R/McRKSsvPceTJsG118fGtgjR8I3vhEeOqQ5Ex1HDemc\ni33cU/nlW1vy22IL+O53w5VEXV1YHvxb34JddoFzzw3PlyiH/96L/btLQsVBRDLxxS+G5TgWLIDb\nb4ePP4avfQ123TUsH/7kk3FOsMsLDSuJSNlYuzbc7XT33TB9Ovz976FgjBgBhxwCm2ySdYRxUM9B\nRHJt8eJQJO6+G154AfbfH4YNCz9f/KL6FG2lnkPOxT7uqfzyrSPy69cPxo4Nazq99lroT8ybF4pD\nnz6hf3Hrrc0/c6KtYv/uktCNZCKSC9tuC8cfH37cYeFCuP9+uOGGcGvsTjuFxQEPOCBcYfTsmXXE\n+aZhJRHJvTVroL4+XGE89lj42WqrUCT23z/cKjtggJ5DsY56DiLSKa1dG55k99hj8Pjjocn9xhuw\n114waBDsvXf435126px9C/Ucci72cU/ll2/lnN8GG8CXvgSnnQZTpsCiRWE5j3PPDVcUU6aEq4ke\nPeDf/i3snzYt3Fb78cflnVtHSbXnYGY1wFVAF+Bad5/YxDG/AA4H/gnUuvvzhf1jgO8ADrwInOru\nH6YZr4jEa5ttPr3TCULfYtkyePbZcCfUtGnhSXdvvAG9eoXhqD32gN13D3MvunfPNv6Oltqwkpl1\nARYCQ4HlwDPACe4+v+iY4cBodx9uZl8Frnb3wWbWE3gMGODuH5rZNGCGu9/Q4DM0rCQiJfXee/Di\ni+GuqBdeCD/z54d+Rf/+n/0ZMAAqK/PXy0gyrJTmlcMg4FV3X1IIZiowAphfdMyRwA0A7v6UmVWY\n2fZFsW1mZh8DmxEKjIhIqrbYAvbdN/ys4w4rV4YisWBB+HnoofC/K1bA5z8fbrvt2/fTn112Cbfb\ndu2aXS7tkWZx6AksLdpeBnw1wTE93f05M7sceB34FzDL3R9MMdayVFdXR3V1ddZhpEb55VvM+TXM\nzQw+97nwc+CBnz32n/8Mt9X++c9hLsa8eWH12ddeC8NWPXp8Wiz69g1XGjvtFArHDjuU78q0aYaV\ndLyn0aWNmW1NuKqoBFYBt5rZSe5+c8Nja2trqaysBKCiooKqqqpPvtR1TaW8btfX15dVPMpP+XWm\n/FqzveeesGpVHdttBz/+8aevf/wx7LxzNa+9BjNn1hWGq6p5/XVYtKiOVaugZ89qjjwSjjkmvfjq\n6uqYPHkywCd/L1uSZs9hMDDB3WsK2+OAtcVNaTP7LVDn7lML2wuAA4EDgMPc/buF/d8EBrv76Q0+\nQz0HEcmtjz6C5cvhww9DD6OjZH0r61ygn5lVmllXYCQwvcEx04FT4JNi8o67/40wnDTYzDY1MyM0\ntV9JMVYRkQ7XtSvsvHPHFoakUisO7r4GGA3MIvxhn+bu881slJmNKhwzA3jNzF4FJgH/Udj/FHAb\n8Bwwr/CWv0sr1nK17rIwVsov32LOL+bckkq1FeLuM4GZDfZNarA9uplzJwAT0opNRESap+UzREQ6\nmax7DiIiklMqDmUs9nFP5ZdvMecXc25JqTiIiEgj6jmIiHQy6jmIiEibqDiUsdjHPZVfvsWcX8y5\nJaXiICIijajnICLSyajnICIibaLiUMZiH/dUfvkWc34x55aUioOIiDSinoOISCejnoOIiLSJikMZ\ni33cU/nlW8z5xZxbUioOIiLSiHoOIiKdjHoOIiLSJioOZSz2cU/ll28x5xdzbkmpOIiISCPqOYiI\ndDLqOYiISJukWhzMrMbMFpjZYjM7u5ljflF4/QUz27Nof4WZ3WZm883sFTMbnGas5Sj2cU/ll28x\n5xdzbkmlVhzMrAvwP0ANsCtwgpkNaHDMcODz7t4P+HfgN0UvXw3McPcBwO7A/LRiLVf19fVZh5Aq\n5ZdvMecXc25JpXnlMAh41d2XuPtqYCowosExRwI3ALj7U0CFmW1vZlsB+7v79YXX1rj7qhRjLUvv\nvPNO1iGkSvnlW8z5xZxbUmkWh57A0qLtZYV9LR3TC9gZWGlmvzez58zsGjPbLMVYRUSkSJrFIelt\nRA075g6nudgUAAAFzElEQVRsCAwEfu3uA4F/AOeUMLZcWLJkSdYhpEr55VvM+cWcW1Kp3cpaaCBP\ncPeawvY4YK27Tyw65rdAnbtPLWwvAA4kFIw57r5zYf9+wDnufkSDz9B9rCIibdDSrawbpvjZc4F+\nZlYJ/BUYCZzQ4JjpwGhgaqGYvOPufwMws6Vm9gV3XwQMBV5u+AEtJSciIm2TWnFw9zVmNhqYBXQB\nrnP3+WY2qvD6JHefYWbDzexVwtDRqUVvcQZws5l1Bf7c4DUREUlRrmdIi4hIOqKYIW1mZxQmy71k\nZhNbPiN/zGysma01s22yjqWUzOznhe/uBTO7o3Abc64lmfyZV2bW28weMbOXC//efpB1TGkwsy5m\n9ryZ3ZN1LKWWdIJx7ouDmR1EmC+xu7t/Gbgs45BKzsx6A4cCf8k6lhTcD3zJ3fcAFgHjMo6nXZJM\n/sy51cAYd/8SMBg4PbL81jkTeIXkd13mSaIJxrkvDsD3gUsKE+1w95UZx5OGK4AfZx1EGtz9AXdf\nW9h8ijDPJc+STP7MLXdf4e71hd/fJ/xh2THbqErLzHoBw4FraXyrfa61ZoJxDMWhH3CAmT1pZnVm\n9pWsAyolMxsBLHP3eVnH0gG+DczIOoh2SjL5MwqFOxH3JBT1mFwJnAWsbenAHEo8wTjNW1lLxswe\nAHo08dJ5hBy2dvfBZrY38Eegb0fG114t5DcOGFZ8eIcEVULrye9cd7+ncMx5wEfu/ocODa70YhyG\naMTMugG3AWcWriCiYGZHAG+6+/NmVp11PClYN8F4tLs/Y2ZXESYYn9/UgWXP3Q9t7jUz+z5wR+G4\nZwpN223d/a0OC7CdmsvPzL5MqPQvmBmEIZdnzWyQu7/ZgSG2y/q+PwAzqyVcxh/SIQGlaznQu2i7\nN+HqIRpmthFwOzDF3e/KOp4S2xc4srAo6CbAlmZ2o7ufknFcpbKMMBLxTGH7NppZfSKGYaW7gIMB\nzOwLQNc8FYb1cfeX3H17d9+5MFt8GTAwT4WhJWZWQ7iEH+HuH2QdTwl8MvmzMEdnJGGyZxQs/FfK\ndcAr7n5V1vGUmruf6+69C//evgE8HFFhwN1XAEsLfyuhmQnGkJMrhxZcD1xvZi8CHwHRfJFNiHHI\n4pdAV+CBwtXRHHf/j2xDarvmJn9mHFYpDQFOBuaZ2fOFfePc/b4MY0pTjP/mEk0w1iQ4ERFpJIZh\nJRERKTEVBxERaUTFQUREGlFxEBGRRlQcRESkERUHERFpRMVBpAlmdqWZnVm0PcvMrinavtzMxjRz\n7n+Z2Xpne5vZBDMb28T+rQqz/kUypeIg0rTHCUspYGYbANsSluBeZx9gdlMnuvsF7v5QC+/f3ASj\nrYHcTgKUeKg4iDRtDqEAAHwJeAl4r/CglI2BAQCFlYDnmtl9ZtajsG+ymR1b+H144aEqc83sFw0e\nHrNr4cE5fzazMwr7LgV2KTxoJsoHV0k+xLB8hkjJuftfzWxN4UFL+xCKRc/C7+8SnmNwJWFNqP9n\nZiOBi4DvEK4K3Mw2AX5LWD//L2b2Bz69YjCgP1ANbAksNLNfA2cTHn60ZwelKtIkFQeR5j1BGFra\nl/DApZ6F31cRVl8dxqdrQnUB/lp07ro//q+5+7on+N0C/HvhdwfuLTwQ6C0zexPYnhwuyS5xUnEQ\nad5swkJzuwEvEh7i85+E4lAH9HT3fddzfsO+QsM//B8V/f4x+vcoZUQ9B5HmPQEcAbzlwdtABWFo\n6Rag+7qHs5vZRmZW3LB2YCHQ18z6FPaN5LPDSk15D9iitGmItJ6Kg0jzXiLcpfRk0b55wDuFZ5Uf\nB0w0s3rgeT5tYANQeD7FfwD3mdlcQq9i3fN6nSbuWCo8i2S2mb2ohrRkSUt2i6TIzDZ3938Ufv8V\nsMjdr844LJEW6cpBJF3fK9yW+jLhrqRJWQckkoSuHEREpBFdOYiISCMqDiIi0oiKg4iINKLiICIi\njag4iIhIIyoOIiLSyP8HyBw/RNRrVoUAAAAASUVORK5CYII=\n", 324 | "text/plain": [ 325 | "" 326 | ] 327 | }, 328 | "metadata": {}, 329 | "output_type": "display_data" 330 | } 331 | ], 332 | "source": [ 333 | "plot(weightsToTry, costs)\n", 334 | "grid(1)\n", 335 | "ylabel('Cost')\n", 336 | "xlabel('Weight')" 337 | ] 338 | }, 339 | { 340 | "cell_type": "markdown", 341 | "metadata": {}, 342 | "source": [ 343 | "So you may be thinking that 0.04 seconds to train a network is not so bad, and we haven't even optimized anything yet. Plus, there are other, way faster languages than python our there. " 344 | ] 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "metadata": {}, 349 | "source": [ 350 | "Before we optimize through, let's consider the full complexity of the problem. Remember the 0.04 seconds required is only for one weight, and we have 9 total! Let's next consider 2 weights for a moment. To maintain the same precision we now need to check 1000 times 1000, or one million values. This is a lot of work, even for a fast computer." 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 10, 356 | "metadata": { 357 | "collapsed": false 358 | }, 359 | "outputs": [], 360 | "source": [ 361 | "weightsToTry = np.linspace(-5,5,1000)\n", 362 | "costs = np.zeros((1000, 1000))\n", 363 | "\n", 364 | "startTime = time.clock()\n", 365 | "for i in range(1000):\n", 366 | " for j in range(1000):\n", 367 | " NN.W1[0,0] = weightsToTry[i]\n", 368 | " NN.W1[0,1] = weightsToTry[j]\n", 369 | " yHat = NN.forward(X)\n", 370 | " costs[i, j] = 0.5*sum((y-yHat)**2)\n", 371 | " \n", 372 | "endTime = time.clock()" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 11, 378 | "metadata": { 379 | "collapsed": false 380 | }, 381 | "outputs": [ 382 | { 383 | "data": { 384 | "text/plain": [ 385 | "62.06971600000001" 386 | ] 387 | }, 388 | "execution_count": 11, 389 | "metadata": {}, 390 | "output_type": "execute_result" 391 | } 392 | ], 393 | "source": [ 394 | "timeElapsed = endTime-startTime\n", 395 | "timeElapsed" 396 | ] 397 | }, 398 | { 399 | "cell_type": "markdown", 400 | "metadata": {}, 401 | "source": [ 402 | "After our 1 million evaluations we’ve found our solution, but it took an agonizing 40 seconds! The real curse of dimensionality kicks in as we continue to add dimensions. Searching through three weights would take a billion evaluations, or 11 hours! Searching through all 9 weights we need for our simple network would take 1,268,391,679,350,583.5 years. (Over a quardrillion years). So for that reason, the \"just try everything\" or brute force optimization method is clearly not going to work." 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 12, 408 | "metadata": { 409 | "collapsed": false 410 | }, 411 | "outputs": [ 412 | { 413 | "data": { 414 | "text/plain": [ 415 | "1268391679350583.5" 416 | ] 417 | }, 418 | "execution_count": 12, 419 | "metadata": {}, 420 | "output_type": "execute_result" 421 | } 422 | ], 423 | "source": [ 424 | "0.04*(1000**(9-1))/(3600*24*365)" 425 | ] 426 | }, 427 | { 428 | "cell_type": "markdown", 429 | "metadata": {}, 430 | "source": [ 431 | "Let's return to the 1-dimensional case and see if we can be more clever. Let's evaluate our cost function for a specific value of w. If w is 1.1 for example, we can run our cost function, and see that J is 2.8. Now we haven't learned much yet, but let's try to add a little information to what we already know. What if we could figure out which way was downhill? If we could, we would know whether to make W smaller or larger do decrease the cost. We could test the cost function immediately to the left and right of our test point and see which is smaller. This is called numerical estimation, and is sometimes a good approach, but for us, there's a faster way. Let's look at our equations so far." 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": {}, 437 | "source": [ 438 | "$$\n", 439 | "z^{(2)} = XW^{(1)} \\tag{1}\\\\\n", 440 | "$$\n", 441 | "$$\n", 442 | "a^{(2)} = f(z^{(2)}) \\tag{2}\\\\\n", 443 | "$$\n", 444 | "$$\n", 445 | "z^{(3)} = a^{(2)}W^{(2)} \\tag{3}\\\\\n", 446 | "$$\n", 447 | "$$\n", 448 | "\\hat{y} = f(z^{(3)}) \\tag{4}\\\\\n", 449 | "$$\n", 450 | "$$\n", 451 | "J = \\sum \\frac{1}{2}(y-\\hat{y})^2 \\tag{5}\\\\\n", 452 | "$$" 453 | ] 454 | }, 455 | { 456 | "cell_type": "markdown", 457 | "metadata": {}, 458 | "source": [ 459 | "We have 5 equations, but we can really think of them as one big equation.\n" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "And since we have one big equation that uniquely determines our cost, J, from X, y, W1, and W2, we can use our good friend calculus to find what we're looking for. We want to know \"which way is downhill\", that is, what is the rate of change of J with respect to W, also known as the derivative. And in this case, since we’re just considering one weight at a time, the partial derivative. " 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "We can derive an expression for dJdW, that will give us the rate of change of J with respect to W, for any value of W! If dJdW is positive, then the cost function is going uphill. If dJdW is negative the cost function is going downhill. " 474 | ] 475 | }, 476 | { 477 | "cell_type": "markdown", 478 | "metadata": {}, 479 | "source": [ 480 | "Now we can really speed things up. Since we know in which direction the cost decreases, we can save all that time we would have spent searching in the wrong direction. We can save even more computational time by iteratively taking steps downhill and stopping when the cost stops getting smaller. " 481 | ] 482 | }, 483 | { 484 | "cell_type": "markdown", 485 | "metadata": {}, 486 | "source": [ 487 | "This method is known as gradient descent, and although it may not seem so impressive in one dimension, it is capable of incredible speedups in higher dimensions. In fact, in our final video, we’ll show that what would have taken 10^27 function evaluations with our brute force method will take less than 100 evaluations with gradient descent. Gradient descent allows us to find needles in very very very large haystacks. " 488 | ] 489 | }, 490 | { 491 | "cell_type": "markdown", 492 | "metadata": {}, 493 | "source": [ 494 | "Now before we celebrate too much here, there is a restriction. What if our cost function doesn't always go in the same direction? What if it goes up, then back down? The mathematical name for this is non-convex, and it could really throw off our gradient descent algorithm by getting it stuck in a local minima instead of our ideal global minima. One of the reasons we chose our cost function to be the sum of squared errors was to exploit the convex nature of quadratic equations." 495 | ] 496 | }, 497 | { 498 | "cell_type": "markdown", 499 | "metadata": {}, 500 | "source": [ 501 | "We know that the graph of y equals x squared is a nice convex parabola and it turns out that higher dimensional versions are too!" 502 | ] 503 | }, 504 | { 505 | "cell_type": "markdown", 506 | "metadata": {}, 507 | "source": [ 508 | "Another piece of the puzzle here is that depending on how we use our data, it might not matter if or cost function is convex or not. If we use our examples one at a time instead of all at once, sometimes it won't matter if our cost function is convex, we will still find a good solution. This is called stochastic gradient descent. So maybe we shouldn't be afraid of non-convex cost functions, as Neural Network wizard Yann Lecun says in his excellent talk \"Who is afraid on non-convex loss funtions?\"" 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "metadata": {}, 514 | "source": [ 515 | "Link to Yann's Talk:\n", 516 | "http://videolectures.net/eml07_lecun_wia/" 517 | ] 518 | }, 519 | { 520 | "cell_type": "markdown", 521 | "metadata": {}, 522 | "source": [ 523 | "The details of gradient descent are a deep topic for another day, for now we're going to do our gradient descent \"batch\" style, where we use all our example at once, and the way we've setup our cost function will keep things nice and convex. Next time we'll compute and code up our gradients!" 524 | ] 525 | } 526 | ], 527 | "metadata": { 528 | "kernelspec": { 529 | "display_name": "Python 2", 530 | "language": "python", 531 | "name": "python2" 532 | }, 533 | "language_info": { 534 | "codemirror_mode": { 535 | "name": "ipython", 536 | "version": 2 537 | }, 538 | "file_extension": ".py", 539 | "mimetype": "text/x-python", 540 | "name": "python", 541 | "nbconvert_exporter": "python", 542 | "pygments_lexer": "ipython2", 543 | "version": "2.7.10" 544 | } 545 | }, 546 | "nbformat": 4, 547 | "nbformat_minor": 0 548 | } 549 | -------------------------------------------------------------------------------- /Part 4 Backpropagation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Neural Networks Demystified

\n", 8 | "

Part 4: Backpropagation

\n", 9 | "\n", 10 | "\n", 11 | "

@stephencwelch

" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "text/html": [ 24 | "\n", 25 | " \n", 32 | " " 33 | ], 34 | "text/plain": [ 35 | "" 36 | ] 37 | }, 38 | "execution_count": 1, 39 | "metadata": {}, 40 | "output_type": "execute_result" 41 | } 42 | ], 43 | "source": [ 44 | "from IPython.display import YouTubeVideo\n", 45 | "YouTubeVideo('GlcnxUlrtek')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "

Variables

\n", 53 | "\n", 54 | "|Code Symbol | Math Symbol | Definition | Dimensions\n", 55 | "| :-: | :-: | :-: | :-: |\n", 56 | "|X|$$X$$|Input Data, each row in an example| (numExamples, inputLayerSize)|\n", 57 | "|y |$$y$$|target data|(numExamples, outputLayerSize)|\n", 58 | "|W1 | $$W^{(1)}$$ | Layer 1 weights | (inputLayerSize, hiddenLayerSize) |\n", 59 | "|W2 | $$W^{(2)}$$ | Layer 2 weights | (hiddenLayerSize, outputLayerSize) |\n", 60 | "|z2 | $$z^{(2)}$$ | Layer 2 activation | (numExamples, hiddenLayerSize) |\n", 61 | "|a2 | $$a^{(2)}$$ | Layer 2 activity | (numExamples, hiddenLayerSize) |\n", 62 | "|z3 | $$z^{(3)}$$ | Layer 3 activation | (numExamples, outputLayerSize) |\n", 63 | "|J | $$J$$ | Cost | (1, outputLayerSize) |\n", 64 | "|dJdz3 | $$\\frac{\\partial J}{\\partial z^{(3)} } = \\delta^{(3)}$$ | Partial derivative of cost with respect to $z^{(3)}$ | (numExamples,outputLayerSize)|\n", 65 | "|dJdW2|$$\\frac{\\partial J}{\\partial W^{(2)}}$$|Partial derivative of cost with respect to $W^{(2)}$|(hiddenLayerSize, outputLayerSize)|\n", 66 | "|dz3dz2|$$\\frac{\\partial z^{(3)}}{\\partial z^{(2)}}$$|Partial derivative of $z^{(3)}$ with respect to $z^{(2)}$|(numExamples, hiddenLayerSize)|\n", 67 | "|dJdW1|$$\\frac{\\partial J}{\\partial W^{(1)}}$$|Partial derivative of cost with respect to $W^{(1)}$|(inputLayerSize, hiddenLayerSize)|\n", 68 | "|delta2|$$\\delta^{(2)}$$|Backpropagating Error 2|(numExamples,hiddenLayerSize)|\n", 69 | "|delta3|$$\\delta^{(3)}$$|Backpropagating Error 1|(numExamples,outputLayerSize)|" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "Last time, we decided to use gradient descent to train our Neural Network, so it could make better predictions of your score on a test based on how many hours you slept, and how many hours you studied the night before. To perform gradient descent, we need an equation and some code for our gradient, dJ/dW. " 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "Our weights, W, are spread across two matrices, W1 and W2. We’ll separate our dJ/dW computation in the same way, by computing dJdW1 and dJdW2 independently. We should have just as many gradient values as weight values, so when we’re done, our matrices dJdW1 and dJdW2 will be the same size as W1 and W2." 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "$$\n", 91 | "\\frac{\\partial J}{\\partial W^{(2)}} = \\frac{\\partial \\sum \\frac{1}{2}(y-\\hat{y})^2}{\\partial W^{(2)}}\n", 92 | "$$" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "Let’s work on dJdW2 first. The sum in our cost function adds the error from each example to create our overall cost. We’ll take advantage of the sum rule in differentiation, which says that the derivative of the sums equals the sum of the derivatives. We can move our sigma outside and just worry about the derivative of the inside expression first. " 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "$$\n", 107 | "\\frac{\\partial J}{\\partial W^{(2)}} = \\sum \\frac{\\partial \\frac{1}{2}(y-\\hat{y})^2}{\\partial W^{(2)}}\n", 108 | "$$" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "To keep things simple, we’ll temporarily forget about our summation. Once we’ve computed dJdW for a single example, we’ll add all our individual derivative terms together. " 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "We can now evaluate our derivative. The power rule tells us to bring down our exponent, 2, and multiply. To finish our derivative, we’ll need to apply the chain rule. " 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "The chain rule tells us how to take the derivative of a function inside of a function, and generally says we take the derivative of the outside function and then multiply it by the derivative of the inside function. " 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "One way to express the chain rule is as the product of derivatives, this will come in very handy as we progress through backpropagation. In fact, a better name for backpropagation might be: don’t stop doing the chain rule. ever. " 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": {}, 142 | "source": [ 143 | "We’ve taken the derivative of the outside of our cost function - now we need to multiply it by the derivative of the inside." 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "Y is just our test scores, which won’t change, so the derivative of y, a constant, with respect to W two is 0! yHat, on the other hand, does change with respect to W two, so we’ll apply the chain rule and multiply our results by minus dYhat/dW2. " 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "$$\n", 158 | "\\frac{\\partial J}{\\partial W^{(2)}} = -(y-\\hat{y}) \\frac{\\partial \\hat{y}}{\\partial W^{(2)}}\n", 159 | "$$" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "We now need to think about the derivative of yHat with respect to W2. Equation 4 tells us that yHat is our activation function of z3, so we it will be helpful to apply the chain rule again to break dyHat/dW2 into dyHat/dz3 times dz3/dW2. " 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "$$\n", 174 | "\\frac{\\partial J}{\\partial W^{(2)}} = \n", 175 | "-(y-\\hat{y})\n", 176 | "\\frac{\\partial \\hat{y}}{\\partial z^{(3)}} \n", 177 | "\\frac{\\partial z^{(3)}}{\\partial W^{(2)}}\n", 178 | "$$" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "To find the rate of change of yHat with respect to z3, we need to differentiate our sigmoid activation function with respect to z. " 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "$$\n", 193 | "f(z) = \\frac{1}{1+e^{-z}}\n", 194 | "$$" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "$$\n", 202 | "f^\\prime(z) = \\frac{e^{-z}}{(1+e^{-z})^2}\n", 203 | "$$" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "Now is a good time to add a new python method for the derivative of our sigmoid function, sigmoid Prime. Our derivative should be the largest where our sigmoid function is the steepest, at the value z equals zero." 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 1, 216 | "metadata": { 217 | "collapsed": false 218 | }, 219 | "outputs": [ 220 | { 221 | "name": "stdout", 222 | "output_type": "stream", 223 | "text": [ 224 | "Populating the interactive namespace from numpy and matplotlib\n" 225 | ] 226 | } 227 | ], 228 | "source": [ 229 | "%pylab inline\n", 230 | "#Import code from last time\n", 231 | "from partTwo import *" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 2, 237 | "metadata": { 238 | "collapsed": false 239 | }, 240 | "outputs": [], 241 | "source": [ 242 | "def sigmoid(z):\n", 243 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 244 | " return 1/(1+np.exp(-z))" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 3, 250 | "metadata": { 251 | "collapsed": false 252 | }, 253 | "outputs": [], 254 | "source": [ 255 | "def sigmoidPrime(z):\n", 256 | " #Derivative of sigmoid function\n", 257 | " return np.exp(-z)/((1+np.exp(-z))**2)" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": 4, 263 | "metadata": { 264 | "collapsed": false 265 | }, 266 | "outputs": [ 267 | { 268 | "data": { 269 | "text/plain": [ 270 | "" 271 | ] 272 | }, 273 | "execution_count": 4, 274 | "metadata": {}, 275 | "output_type": "execute_result" 276 | }, 277 | { 278 | "data": { 279 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEACAYAAACuzv3DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8Dff6wPHPVzaxBrElEVFB7bFWXe21095bWrTo9ktv\nKaqtbmq7KkWLtrdV1QWluihFucVtlZaUovbatyASEmsFEVkk398fk0ZEliM558w5k+f9ep2XMzlf\nM89j5MnkmZnvKK01Qggh3FcJswMQQghRNFLIhRDCzUkhF0IINyeFXAgh3JwUciGEcHNSyIUQws0V\nWMiVUnOUUmeUUnvyGTNNKXVEKbVLKdXMviEKIYTIjy1H5J8D3fP6UCl1PxCqta4DPAN8YqfYhBBC\n2KDAQq61Xg9czGdID+CLzLGbAT+lVFX7hCeEEKIg9uiRBwKx2ZZPAkF2WK8QQggb2Otkp8qxLPf9\nCyGEk3jaYR2ngBrZloMyv3YTpZQUdyGEKAStdc6D5ZvY44h8GfAkgFKqDZCgtT6TRzCWfY0bN870\nGCQ/yU3y06Sna+LjNdu2af77X81HH2lGjdI8+aSmUydN48aaatU0np4ao3lwe69Bg5ybny0KPCJX\nSs0H/g74K6VigXGAV2ZhnqG1/kEpdb9SKgq4Cjxl05YtJjo62uwQHMrK+Vk5N7BmflevwrFjcPQo\nrFgRzdmzEBVlLMfGQlqabespWxb8/aFy5RuvSpWgQgUoXx78/IxX9vcVKzo2t8IosJBrrfvbMOY5\n+4QjhBA3JCfDwYOwbx/s3XvjlfNn0/btNy9XqgSBgRAUdPOfgYFQtapRsP39oWRJp6XiUPbokQsg\nPDzc7BAcysr5WTk3cJ/80tKMIr1lC2zdavy5fz+kp9861ssLatWC2rXBxyecv//deF+7NoSEQKlS\nTg/fVMrWHkyRN6SUdta2hBCu7+pV2LgR1q6FX3+FHTuMI/DsSpSAOnWgUaObX6Gh4FlMDkOVUugC\nTnZKIbeTyMhI2rdvb3YYDmPl/Fw1N6Xy/d4VFpRbjbSlkBeTn2lCuCcrH/yImxXlB7cckQvhojKP\nxMwOQzhJXvvbliNymcZWCCHcnBRyO4mMjDQ7BIeycn5Wzk0UD1LIhRB2M2nSJAYOHOhy2w0JCeGX\nX35xYkTOJT1yIVyU9Mjtp1atWsyePZuOHTuaHUqepEcuhBDFmBRyO7F6n9XK+Vk5N0eaMmUKQUFB\nlCtXjjvvvJM1a9YQERHBE088kTXmyy+/pGbNmvj7+zNx4kRCQkJYs2YNABERETz88MM88cQTlCtX\njiZNmnDkyBEmTZpE1apVCQ4OZvXq1VnriouLo0ePHlSqVIk6derw2WefZX2Wc7tfffVV1nbfeust\nJ/xrmEsKuRBuSin7vArj0KFDfPTRR2zbto3Lly+zatUqQkJCbroWev/+/QwdOpT58+cTHx/PpUuX\niIuLu2k9K1as4Mknn+TixYs0a9aMbt26AUbRfv311xk0aFDW2H79+hEcHEx8fDyLFy9m9OjRrF27\nNvPf4ubtPvvss8ybN4+4uDguXLjAyZMnC5eom5BCbieueGegPVk5Pyvn5igeHh6kpKSwb98+0tLS\nCA4O5o477ripx7t48WJ69OhB27Zt8fLyYvz48bfc9HLvvffSpUsXPDw86NOnD+fOnWPkyJF4eHjQ\nt29foqOjuXz5MrGxsWzcuJEpU6bg7e1N06ZNGTBgAF9++SXALdt94IEHaNeuHd7e3kyYMIESJaxd\n6qydnRAWprV9XoURGhrK1KlTiYiIoGrVqvTv35/4+PibxsTFxREUdOOpj76+vlSqVOmmMVWqVLnp\nc39//6xi7+vrC0BiYiJxcXFUrFiR0qVLZ40PDg7m1KlbnmFzy3ZLlSp1y3atRgq5nVi9z2rl/Kyc\nmyP179+f9evXc+LECZRSjBgx4qYj7oCAgJtaGteuXePChQuF2lZAQAB//vkniYmJWV+LiYm5qWBn\nHxsbe+MxwklJSYXerruQQi6EuG2HDx9mzZo1pKSk4OPjQ8mSJfHw8LhpTO/evVm+fDmbNm0iNTWV\niIiIQl9OWaNGDdq2bcuoUaNISUlh9+7dzJkzh8cff/yWsb1792bFihVs2LCB1NRUXn/9dTIyMgq1\nXXchhdxOrN5ntXJ+Vs7NUVJSUhg1ahSVK1emevXqnD9/nkmTJgE3Tjw2bNiQDz/8kH79+hEQEEDZ\nsmWpUqUKPj4+WeNy9szzW54/fz7R0dEEBATQq1cvxo8fn3VdePZ1NWzYkI8++ohHH32UgIAAKlas\nSI0aNbAyuSFICBdltRuCEhMTqVChAlFRUdSsWdPscFyO3BDkAqzeZ7VyflbOzWzLly8nKSmJq1ev\n8uqrr9KkSRMp4g4ghVwI4TDLli0jMDCQwMBAjh49yoIFC8wOyZKktSKEi7Jaa0XkT1orQghRjEkh\ntxOr91mtnJ+VcxPFgxRyIYRwc9IjF8JFSY+8eJEeuRBCFGNSyO3E6n1WK+dn5dyczYqPehsyZAgT\nJ04sbGhO4Wl2AEII6xg1apRLbjf7LfwRERG8+eablCxZEk9PTxo0aMB//vMf2rRpk+vf/eSTT+we\nr73JEbmdWH2+DivnZ+XcxK2UUvTv358rV65w7tw52rVrR69evXId6y6TbUkhF0IUirs+6k1rnXVS\n0dPTkyeffJLTp09z4cIFwsPDGTJkCPfffz9lypRh7dq1hIeHM3bsWMBowwUFBfHOO+9QtWpVAgIC\n+P777/nhhx+oV68elSpVypo87K9tTZ48mdDQUPz9/enbty8XL160414wSCG3E6v3Wa2cn7vmpt5Q\ndnkVhlUe9ZaSksLcuXMJDg7OevjE/PnzGTt2LImJibRr1+6WWRrPnDlDSkoKcXFxjB8/ngEDBvDN\nN9+wY8cO1q9fz4QJEzhx4gQA06ZNY9myZaxbt474+HgqVKjA0KFDC/Vvnh8p5EKI2+buj3pbuHAh\nFSpUIDg4mJ07d7J06dKszx588EHuvvtugKwpd7Ov38vLizFjxmTFeOHCBYYNG0bp0qVp0KABDRo0\nYNeuXQB8+umnTJw4kYCAALy8vBg3bhyLFy+2e8tGTnbaidX7rFbOz11z0+PMu8Y8+6Pe9u3bR7du\n3XjvvfduGuOMR71t27btlthsedRb3759s34IZKeUyvWpQ9lVqlTplhirVq16Ux5/PcnoxIkTPPTQ\nQzf9IPH09OTMmTNUr1493+3cDjkiF0IUijs/6u12b7TK+ZuErYKDg1m5ciUXL17MeiUlJdm1iIMU\ncrtx1z6rraycn5VzcxSrPuott/iynxy9XYMHD2b06NHExMQAcO7cOZYtW1aodeVHCrkQ4ra586Pe\ncttufp/l/FpBMWc3bNgwevToQdeuXSlXrhx33303W7ZsyXN8YRU414pSqjswFfAAPtNaT8nxeXng\na6AGRs/9Xa313FzWI3OtCHEbrDbXijzqLX8Om2tFKeUBTAe6Aw2A/kqp+jmGDQX2aq3DgPbAf5RS\nchJVCCGPenOSglorrYEorXW01joNWAD0zDEmAyiX+b4ccEFrfd2+Ybo+q/dZrZyflXMzmzzqzTkK\nOnIOBGKzLZ8E7soxZjqwXCkVB5QFHrFfeEIUT1bpqMyaNYtZs2aZHYblFVTIbfnv1B3YobXuoJSq\nDaxWSjXVWl/JOTA8PJyQkBAA/Pz8CAsLy7qG96+jIndd/utrrhKP5Gf7cvv27V0qHoAXXjCWRfET\nGRnJ3LlzAbLqZUHyPdmplGoDRGitu2cujwIysp/wVEqtACZprTdkLv8CjNBab8uxLjnZKYQNNm+G\ndu3g+nVrnewU+XPkgyW2AXWUUiFKKW+gL5DzIsgYoHPmBqsC9YBjNsZuGVbvs1o5P1fKLSEB+vWD\n68XuLJMoinxbK1rr60qp54CfMC4/nK21PqCUGpT5+QxgAjBXKbUbUMBrWus/HRy3EJajNTzzDERH\nQ4sWsH174e8oFMWLPLNTCBcxYwYMHgxly8KOHRAaanZEwhXY0lqRQi6EC9izB1q3huRk+OYb6N/f\n7IiEq5CHLzuRK/VZHcHK+ZmdW0oKPPaYUcSfftr+Rdzs/BzN6vnZQgq5ECaLiDCOyEND4YMPzI5G\nuCNprQhhoo0b4Z57jPfr10PbtubGI1yPtFaEcGFXr8L//R9kZMDw4VLEReFJIbcTq/fprJyfWbm9\n9hpERUHjxvDGG47bjpX3HVg/P1tIIRfCBD//DB9/DF5e8NVXkDlFtxCFIj1yIZzs6lXjKPz4cZg4\nEcaMMTsi4cqkRy6ECxo3zijiTZsa7RUhikoKuZ1YvU9n5fycmdv27fD++1CiBHz2mdFacTQr7zuw\nfn62kEIuhJOkpcGAAcZVKsOGQcuWZkckrEJ65EI4ydtvw4gREBICe/dC6dJmRyTcgcy1IoSLOHoU\nGjUybsNfuRK6dTM7IuEu5GSnE1m9T2fl/Bydm9bw7LNGEX/sMecXcSvvO7B+fraQQi6Egy1dCqtW\ngZ8fvPee2dEIK5LWihAOlJQE9etDTAxMnw5Dh5odkXA30loRwmSTJxtFPCzMeGiEEI4ghdxOrN6n\ns3J+jsrt6FHjShUwjsY9PByymQJZed+B9fOzhRRyIRzkxReNh0Y8+ST87W9mRyOsTHrkQjjAihXw\nwANQrhwcOgTVqpkdkXBX0iMXwgTJycadm2BMTytFXDiaFHI7sXqfzsr52Tu3adPg2DFo2NA1rlKx\n8r4D6+dnCynkQtjR2bPw5pvG+/ffd86kWEJIj1wIO3r2WfjkE7jvPvjhB7OjEVYgc60I4UT790OT\nJsb73buhQQNz4xHWICc7ncjqfTor52ev3IYPh/R0eOYZ1yriVt53YP38bCGFXAg7WLXKaKWULQsR\nEWZHI4obaa0IUUTp6dCsGezZY9ySP2KE2REJK5HWihBO8PnnRhGvWfPG9eNCOJMUcjuxep/OyvkV\nJberV2HsWOP95MlQsqR9YrInK+87sH5+tpBCLkQRfPABnD4NrVpB375mRyOKK+mRC1FIFy7AHXfA\n5cvwyy/QsaPZEQkrkh65EA40ebJRxLt0kSIuzCWF3E6s3qezcn6Fye3kSfjwQ+P9pEn2jcferLzv\nwPr52UIKuRCF8MYbxlzjjzwCLVqYHY0o7grskSulugNTAQ/gM631lFzGtAfeB7yA81rr9rmMkR65\nsISDB42ZDZUybsuvW9fsiISV2dIj9yxgBR7AdKAzcArYqpRaprU+kG2MH/AR0E1rfVIp5V/00IVw\nXf/+N2RkGLfiSxEXrqCg1kprIEprHa21TgMWAD1zjHkU+E5rfRJAa33e/mG6Pqv36ayc3+3ktnUr\nfPedcb346687LiZ7svK+A+vnZ4uCCnkgEJtt+WTm17KrA1RUSq1VSm1TSj1hzwCFcBVaw8iRxvth\nwyAw53eCECbJt0eulOoNdNdaD8xcfhy4S2v9fLYx04HmQCegFLAJ+IfW+kiOdUmPXLi11auha1fw\n8zOeAFShgtkRieKgyD1yjL54jWzLNTCOyrOLxTjBeQ24ppRaBzQFjuQYR3h4OCEhIQD4+fkRFhZG\n+/btgRu/HsmyLLvi8tq1kZnzqLTntddg1y7Xik+WrbMcGRnJ3LlzAbLqZYG01nm+MAr9USAE8Ab+\nAOrnGHMn8DPGVS2lgD1Ag1zWpa1s7dq1ZofgUFbOz5bcfvhBa9Da31/rK1ccH5M9WXnfaW39/DJr\nZ761Ot8jcq31daXUc8BPmYV6ttb6gFJqUObnM7TWB5VSK4HdQAYwS2u937YfI0K4Pq1vnNgcMQLK\nlDE3HiFykrlWhCjAihXwwANQpYrRGy9d2uyIRHEic60IUURaw7hxxvuRI6WIC9ckhdxO/jpZYVVW\nzi+/3JYtgx07oFo1GDzYeTHZk5X3HVg/P1tIIRciD1rfeP7myJHg62tqOELkSXrkQuRh6VLo1QsC\nAiAqSgq5MIf0yIUopIyMG73xUaOkiAvXJoXcTqzep7NyfrnltmSJ8UDlwEAYMMD5MdmTlfcdWD8/\nW0ghFyKHjIwbvfExY1zzgcpCZCc9ciFy+PZb6NcPatSAI0fAx8fsiERxJj1yIW5Terrx9B8w5h2X\nIi7cgRRyO7F6n87K+WXP7dtv4cABqFkTwsNNC8murLzvwPr52UIKuRCZ0tNh/Hjj/b//Dd7e5sYj\nhK2kRy5Epq+/hieegFq14NAh8PIyOyIhpEcuhM2uX7/RGx87Voq4cC9SyO3E6n06K+cXGRnJvHnG\n3Zu1axtH5VZi5X0H1s/PFlLIRbF3/TpMmGC8f/118CzouVlCuBjpkYtib84cePppqFMH9u+XQi5c\ni/TIhShAWhpMnGi8HzdOirhwT1LI7cTqfTqr5vfFF3D8eCR33mnczWlFVt13f7F6fraQQi6KrdTU\nm3vjHh7mxiNEYUmPXBRbM2YYT/1p0AB275ZCLlyTLT1yKeSiWEpJMU5uxsYat+U/8ojZEQmROznZ\n6URW79NZLb/Zs40i3qgR+PtHmh2OQ1lt3+Vk9fxsIYVcFDvJyfDWW8b7iAgoId8Fws1Ja0UUOx9+\nCC+8AE2awM6dUsiFa5MeuRA5XLtm3IYfH288zu2hh8yOSIj8SY/ciazep7NKfjNmGEW8WTN48EHj\na1bJLS+Sn/VJIRfFxtWrMGmS8X78eFD5HuMI4T6ktSKKjXffheHDoVUr2LxZCrlwD9IjFyJTYqLx\nwIjz5+HHH6F7d7MjEsI20iN3Iqv36dw9v+nTjSLepg1063bzZ+6eW0EkP+uTQi4s7/JleOcd4730\nxoUVSWtFWN7Eicbj29q1g3XrpJAL9yI9clHsXboEISGQkABr1kCHDmZHJMTtkR65E1m9T+eu+U2d\nahTx9u3zLuLumputJD/rk0IuLOviRXjvPeP9G2+YG4sQjiStFWFZY8ca/fHOnWH1arOjEaJw7NJa\nUUp1V0odVEodUUqNyGdcK6XUdaVUr8IEK4Q9XbhgtFVAjsaF9eVbyJVSHsB0oDvQAOivlKqfx7gp\nwEqgWF4TYPU+nbvl9847xk1A3bpB27b5j3W33G6X5Gd9BR2RtwaitNbRWus0YAHQM5dxzwOLgXN2\njk+I23b6NEybZrwfP97cWIRwhnx75EqpPkA3rfXAzOXHgbu01s9nGxMIfA10BOYAy7XWS3JZl/TI\nhVM89xx89JExu+HSpWZHI0TR2KNHbkvlnQqMzKzSimLaWhGu4fhxmDnTuOlnwgSzoxHCOTwL+PwU\nUCPbcg3gZI4xLYAFyrhdzh+4TymVprVelnNl4eHhhISEAODn50dYWBjt27cHbvS53HV56tSplsrH\nXfObM6c9aWnQpUsk588DFPz3s/dYzY7fEcuSn3stR0ZGMnfuXICselmQglornsAhoBMQB2wB+mut\nD+Qx/nOKaWslMjIya6dYkTvkt3ev8fg2T084dMiY7dAW7pBbUUh+7s0ut+grpe7DaJ94ALO11pOU\nUoMAtNYzcowttoVcmO/BB+H772HoUGO2QyGsQOZaEcXG5s3GFLW+vnDsGFSrZnZEQtiHzLXiRNn7\ndFbk6vmNHm38OWzY7RdxV8+tqCQ/65NCLtzezz8bMxv6+cFrr5kdjRDOJ60V4da0hrvugq1b4a23\nYNQosyMSwr6kRy4s79tvoV8/o50SFQWlS5sdkRD2JT1yJ7J6n84V80tJgZEjjffjxxe+iLtibvYk\n+VmfFHLhtj76CKKjoUEDeOops6MRwjzSWhFu6c8/ITTUeHjEihXwj3+YHZEQjiGtFWFZb75pFPGO\nHeH++82ORghzSSG3E6v36Vwpv2PHbty5+c47xgRZReFKuTmC5Gd9UsiF2xk9GlJT4YknoHlzs6MR\nwnzSIxdu5a9b8X184PBhCA42OyIhHEt65MJStIZXXjHev/SSFHEh/iKF3E6s3qdzhfzmz4cNG6BK\nlRvXj9uDK+TmSJKf9UkhF24hMRGGDzfeT5oE5cubG48QrkR65MItjBljzKXSsqXRJy8hhyCimJC5\nVoQlHD1q3L2ZmgobN8Ldd5sdkRDOIyc7ncjqfToz83vllRuXGzqiiMu+c29Wz88WUsiFS1u92nh8\nW+nSMHmy2dEI4ZqktSJcVmoqhIXBgQNGER8xwuyIhHA+aa0It/buu0YRr1MHXnzR7GiEcF1SyO3E\n6n06Z+d37BhMmGC8//hj405OR5F9596snp8tpJALl6M1DB0Kycnw2GPQubPZEQnh2qRHLlzOokXw\nyCPGw5QPHoSqVc2OSAjzSI9cuJ3Ll2HYMOP95MlSxIWwhRRyO7F6n85Z+f373xAfb8xwOHCgUzYp\n+87NWT0/W0ghFy5jwwbjgREeHvDpp3IbvhC2kh65cAnXrhnXjB8+bMyrMnGi2REJ4RqkRy7cxuuv\nG0W8YUMYO9bsaIRwL3JEbieRkZG0b9/e7DAcxpH5bd4Mbdsa73//HVq1su/60zPS2Xt2L9vitrE9\nfjtHLx4l5lIMpxNPk3I9hZSoFHxCffAr6UelUpUIrRjKnZXuJKxaGPfUvIeAsgH2DcjJ5P+me7Pl\niNzTWcEIkZvkZHjqKcjIgNdes18Rv5p6lRWHV7DiyApWRq3kfNL5vAdruHb9GtcSrxGfGM/es3tv\n+ji0Yig96vagd4PetAlqQwklv8gK1yJH5MJUo0YZlxnWqwd//AElSxZ+XVprfj/5O7N3zmbhvoVc\nSb2S9VnN8jVpE9SGlgEtqe9fn+DywQSUDcDXyxevEl6kpKeQkJzAmcQzHL5wmIPnD7L51GY2xG4g\nMTUxaz1B5YIY2HwgA5sPpHrZ6kVJXQibyHzkwqWtWwft24NSsH79jfbK7crQGSw/tJwpG6aw6eSm\nrK+3CWpDn/p9uL/O/dzpfydK5fu9kKvrGdfZfHIz3x34jsX7FxN7ORYAzxKe9GnQh9HtRtO4auPC\nBS6EDWwp5GitnfIyNmVda9euNTsEh7J3fhcvah0crDVoPWZM4daRkZGhl+xfoutPr6+JQBOBrjC5\ngh6+arjef3a/zeuxNbeMjAz989Gfda9ve2mPNzyyttn729561+ldhUvCCeT/pnvLrJ351lfpkQtT\nDB0KMTFGT3zcuNv/+xtjNzJ89XA2xm4EoEa5Grx898sMaD6AMt5l7BytQSlFpzs60emOTsReiuWd\nje8wc/tMvjvwHUsOLOGpsKd4s9ObVCtTzSHbFyIv0loRTjdvHjz+uPGwiJ07jWlqbXU68TQv//Qy\n8/fOB6ByqcqM+/s4BrYYiLeHt4Mizlv8lXimbJjCx1s/Ji0jjTLeZRhzzxheavMSPp4OnLJRFBvS\nIxcuJzoamjY15lSZNQsGDLDt72XoDGZtn8XIX0aSkJyAr6cvr7Z9lVfbvko5n3IOjdkWRy4c4dXV\nr7Ls0DIA6vvX57Men9G2RiEb/0JkstsNQUqp7kqpg0qpI0qpW57TopR6TCm1Sym1Wym1QSnVpLBB\nuyurz/dgj/xSUuDhh40i/tBD8PTTtv29A+cO0G5OOwb/bzAJyQncF3of+57dx/gO4+1SxO2RW51K\ndfi+3/esenwVdSvV5cB5I+bnf3ieKylXCl6BA8n/TesrsJArpTyA6UB3oAHQXylVP8ewY8C9Wusm\nwARgpr0DFe7v5Zdh2zYICYHZs42rVfKToTOYtnkazWc2Z9PJTVQrU42FfRbyv0f/R60KtZwS8+3q\nUrsLuwbvYnS70XiU8GD61uk0+qQRv0b/anZowsIKbK0ope4Gxmmtu2cujwTQWuf6KFylVAVgj9Y6\nKMfXpbVSjC1YAP37g7e3MTlWy5b5jz95+STh/w3nl+O/ABAeFs773d7Hr6SfE6K1j12nd/H0sqfZ\nHr8dheK1v73G+A7jTenlC/dlr9ZKIBCbbflk5tfy8jTwgw3rFcXEgQM3euFTpxZcxOfvmU/jTxrz\ny/Ff8C/lz5JHlvB5z8/dqogDNK3WlE1Pb2LsvWNRSjFlwxTafNaGA+cOmB2asBhbLj+0+TBaKdUB\n+Bfwt9w+Dw8PJyQkBAA/Pz/CwsKy5kj4q8/lrstTp061VD72yq958/b06QNXr0bSsSMMHpz3+OTr\nySy8upDP//gcjkObGm1YOmQp1cpUc2h+2Xusjlj/+A7jqXKuCm+ue5Od7KT5zOYM8R/CA3UfoEOH\nDnbfnrPzM3vZavlFRkYyd+5cgKx6WaCCLjQH2gArsy2PAkbkMq4JEAWE5rEeh1ws7yqsflNCYfK7\nfl3rBx4wbvqpX1/rK1fyHnvg3AHd6ONGmgh0yYkl9YxtM3RGRkbhA74Nztp3l5Iv6fD/hmfdSNR/\ncX99Ofmyw7cr/zfdGzbcEGRLj9wTOAR0AuKALUB/rfWBbGOCgTXA41rr3/NYjy5oW8Ja/ppHpUIF\n2LIFQkNzHzdv9zwGrRjE1bSr1KtUj0UPL7L0be/Z861bqS6LHl5Ek6rF7kIvYSO7XUeulLoPmAp4\nALO11pOUUoMAtNYzlFKfAQ8BMZl/JU1r3TrHOqSQFyN/3fTj4QGrVkHHjreOuZZ2jWErhzFrxywA\nHm38KJ/+41PK+pR1crTOd+j8IR5e9DB7zu6hpGdJpnWfxoDmAwo1H4ywNplrxYms/uvd7eS3ebPW\nPj5GS2X69NzHHDp/SDf9pKkmAu0zwceprZSczNp3SalJeuCygVmtlse+e0xfScmn/1RI8n/TvWFD\na0UmVhZ2deQI/POfxs0/gwbBs8/eOubbvd/SYmYLdp3ZRWjFUH4f8DvPtHim2B2N+nr5MvOBmXz9\n0NeU9irNvD3zaDmzJXvO7DE7NOFm5BZ9YTdnzsDdd8Px49CtGyxfDl5eNz5PuZ7Cyz+9zMfbPgbg\nkYaPMOuBWS5xi73ZDp4/yMOLHmbv2b34evry8T8+Jjws3OywhAuQuVaE01y5YswtvmOHcZ342rVQ\nJtskhMcvHufhRQ+zPX473h7evN/tfYa0HFLsjsLzk5SWxHM/PGdcfolxE9RH939EKa9SJkcmzCQP\nX3ai7NeyWlF++aWkQO/eRhGvXRv+97+bi/iyQ8toPrM52+O3E+IXwoZ/beDZVs+6TBF3lX1XyqsU\nc3rO4fMxdHnQAAAROUlEQVSen+Pr6cvcP+Zy12d3cfD8wSKt11XycxSr52cLKeSiSNLSoG9fWL0a\nqlSBn34y/gRIS0/jtdWv0XNBTxKSE+hRrwc7ntlBy4ACbu0s5sLDwtkycAv1KtVj79m9tJzZkm/2\nfGN2WMKFSWtFFFpamjF/ynffgZ8frFkDzZoZn526fIp+3/Xjt5jf8FAeTOo0iVfbvuoyR+HuIDE1\nkUErBmUV8UEtBjG1+1RKehbhwabC7UiPXDjM9evGdeLffgvly8PPP9+YQ+XnYz/z6HePci7pHAFl\nA1jQewH31LzH3IDdlNaamdtnMmzlMFLSUwirFsaihxcRWjGPu6uE5UiP3Ims3qfLnl9aGvzf/xlF\nvGxZo53SsiWkZ6Qz/tfxdP2qK+eSztGpVid2Dtrp8kXclfedUopBLQex6elN1K5Qmz9O/0GLmS34\nbv93Nq/DlfOzB6vnZwsp5OK2XLsGvXrBN98YJzRXroS77oLYS7F0+rIT4yKNB3CO+/s4fnr8J6qU\nrmJyxNbQrHoztj+znd71e3M55TJ9FvVh2I/DSE1PNTs04QKktSJsdukS9OgB69ZBxYrw44/QujUs\nPbCUp5c9zcXki1QrU40vH/ySLrW7mB2uJWmtmb5lOq+seoW0jDRaBbRiQZ8F3FHhDrNDEw4iPXJh\nN2fPwn33GZcYBgQYV6mE1Eni5Z9eZsb2GQD8o84/+Lzn51QuXdnkaK1vy6ktPLLoEU5cOkEZ7zJM\n6z6N8LBwOZlsQdIjdyIr9+n27oUmTSKzrhPfsAHSKu6i1axWzNg+A28Pbz7o/gHL+y93yyLujvuu\ndWBrdgzaQZ8GfUhMTeRfy/5F74W9OZ90/pax7pjf7bB6fraQQi7ytXIltG1r3H7fujWs/TWNr05M\noOWsluw/t587/e9ky4AtvHDXC3I06GQVfSuysM9CvnjwC8p6l2XpwaU0/qQxPx750ezQhJNJa0Xk\nSmv48EN46SXIyIBHHoHX3t3HoB//j+3x2wEY2mooUzpPobR3aZOjFdEJ0Ty59EnWx6wHYEjLIUzp\nPKVYTAlsddIjF4WSmGjMXPhN5s2E/349nTJd/sPrkWNJTU+lZvmazOk5h461cplkXJgmPSOddze+\ny9i1Y0nLSCO4fDAz/jmD7qHdzQ5NFIH0yJ3IKn26/fuNFso330Dp0vDW5ztYGdSGkZ+NIDU9lYHN\nB7J7yG5LFXGr7DuPEh6MaDeCrQO30qJ6C2IuxXDfvPvoOqErF5IumB2ew1hl/xWFFHIBGK2UL76A\nVq2Mp97Xa3KF3jNf4t8xrdgWtw3/0v78+NiPzHxgpkw76+KaVmvK7wN+5+3Ob1PSsySrj66m/kf1\nmb9nPvJbsTVJa0Vw7pzRSlm6FEBz76D/crTuC5y6cpISqgQv3vUib3R4gzLeZQpalXAxUX9GMXD5\nQCKjIwHoENKBafdNo1GVRuYGJmwmPXJRoO+/h2eeMa4TLxWyl1qDX2Zf8moAWgW0YsY/Z9CsejOT\noxRFkaEzmLNzDiN/HsmFaxfwUB4MbTWUNzq8gV9JP7PDEwWQHrkTuVufLi7OmH72wQfh7NWzBDwz\nmOSnmrIveTV+Jf348L4P2fT0pqwi7m753Q4r5waw7td1DGg+gMPPH2Zoq6FoNNO2TKPuh3WZuX0m\n1zOumx1ikVh9/9lCCnkxk54O06dD/fqwcGkSXu2nUHJ4HeICZqBQPNfqOaKej+K51s/hUcLD7HCF\nHVX0rcj0+6ez45kd3BN8D+eSzjFoxSAaftyQxfsXS//cjUlrpRiJjIRXXoEdu5OhxQx8Ok8ixesM\nAPfXuZ93u7xL/cr1zQ1SOIXWmkX7FzFmzRii/owCoGVASyZ3mkynOzqZHJ3ITnrkAjAuKRwxAlb8\nmALNPsejw0TSS58CjG/eNzu+SdfaXU2OUpghLT2N2Ttn88avb3A68TQA99a8l9HtRtO1dle5W9cF\nSI/ciVyxT3fsGAwYAI1aJrDi4hTUS7Xgn0NIL32KJlWb8H2/79kyYItNRdwV87MXK+cG+efn5eHF\n4JaDiXo+irc6voVfST/WnVhH93ndaTWrFUsOLCFDZzgv2EKw+v6zhRRyCzp8GMLDoU7LWGbHvoJ+\nsQZ0GYkuE0/jKo1Z2GchOwftpEe9HnLEJQAo7V2aUfeM4sSLJ5jcaTJVSldhe/x2ei/sTcOPG/LJ\n1k9ITE00O0yRB2mtWITWsHEjTP0gg+92/oJu/inc+T2USAegY62ODG87nG61u0nxFgW6lnaNOTvn\n8PbGt4m5FANAOZ9yPBX2FENbDaVOpTomR1h8SI+8GEhONh659p9Pz7HH43NoMQMqHgPAQ3nwcMOH\nGd52OM2rNzc5UuGO0tLTWHJgCR9u+ZANsRuyvt61dleeCnuKnvV64uvla2KE1ieF3IkiIyNp3769\nU7alNezcCbPmXuXrbd+TWGse1F4FHsb1wIFlghnS6hn+1exfVC9b3S7bdGZ+zmbl3MB++e2M38n0\nLdP5Zu83JF9PBqC8T3n6NuxLeFg4bYLamPLbntX3ny2F3NNZwYiii42Fed9e49PVP3Oi9EKovxS6\nXQWgBB50r/1Pnm09mO6h3eUacGF3zao3Y3bP2bzd5W0W7F3A3F1z2Ra3jZk7ZjJzx0xq+dWiV/1e\n9GnQh9aBrSmh5BScs8gRuYuLioIvF1/g6y0rOO79PdT+CbyTsj5vXKENz7R5jEcaPiIPOhZOt+/s\nPr7Y9QVf7/6a+MT4rK8Hlg2kV/1e9KzXk3bB7fDx9DExSvcmrRU3dO0arF2XyldrNvNL9M+cK/Mz\nBP0OJW5cAlbbtwVPtHqQx5v2p3bF2iZGK4QhPSOdTSc3sXj/YpYcWELs5disz0p5laJ9SHu61+5O\nt9Bu1KlYR0643wYp5E5U2D5daips2prMot92sPrgRqKuryUj+Ffwvpo1poT2oknZDvxfm570adyD\noHJBdozcNlbuQ1o5N3B+flprtsZtZcmBJfwY9SO7z+y+6fPg8sHcE3wP9wTfQ7vgdtSvXL9IbRir\n7z/pkbugs2c1//sthmU7trAlfiPxHpvQ1XaARxqE3Bjnn9GQe4M689jdnehc5+8yB7hwG0opWge2\npnVgayZ3nkz8lXhWHV3FyqMrWX10NTGXYpi3Zx7z9swDjDlg/lbjb9wVeBfNqzenRUALaRPeJjki\nd5CMDNh/+Bortuxj/ZFd7Du/i/iMXaRW2A2+CTcP1ooKaQ0J829L75b30Cusk92uNhHClaRnpLP3\n7F5+i/mN9THrWR+znrgrcbeMCywbSIuAFjSv1pxGVRrRoHIDQiuG4uXhZULU5pLWihP8eTGddbti\n2HDoMHtOHSYq4RBnrh/masnD6HIxoG7N2Su1CkEezbk76G56tbqbzvVbU75keROiF8JcWmuiE6L5\nLeY3tsdvZ0f8Dnae3pnrXaSeJTypU7EO9SvXp4F/A+pWqkutCrWo5VeL6mWrW/YqGbsUcqVUd2Aq\n4AF8prWeksuYacB9QBIQrrXemcsYtyvkySkZHDmZwO5jceyJieHwmRhiLsVwOjmGhIwYrnnFklHm\npHH99nGgVo4VpHtSJrUOwd5hNKvelI4Nm9I9LIyActXMSKdIrNyHtHJu4H75ZegMjlw4klXU95/b\nz/5z+4lOiEaTSw05Dj6hPtT0q8kdFe6gll8tgssHE1A24KZXeZ/ybnmStcg9cqWUBzAd6AycArYq\npZZprQ9kG3M/EKq1rqOUugv4BGhT5OjtLD1dc+ZiEjFnEjh54SJxfyZw5lIC8QkXiLt8lrNXz/Bn\nylkuZ5whSZ0l1esM2vdc1k02WUplvrLxvBaAzwlf6tXszJ2V69IipC73NqxL0+BalvlV8I8//nCr\nYnA7rJwbuF9+JVQJ6vnXo55/Pfo37p/19aS0JA6dP8T+c/s5cP4AUX9GceziMfZt30dSehKHLxzm\n8IXDea7X19M3q6hXKV2FSr6V8C/lT6VSmX/mWC7nU85tjvILOtnZGojSWkcDKKUWAD2BA9nG9AC+\nANBab1ZK+Smlqmqtzzgg3gL9tvcE/WYPJykjgRSVQKrHRdI9E9A+CbcW5ex8M185qJTy+KRWpxzB\nVPEOpka5YGpXDqZhYDDNategSUgQvl4liYiIICIiwlFpmS4hIaHgQW7KyrmBdfIr5VWKZtWb3fLo\nwYhTEbwy8hWiE6I5nnCc4xePc/LySeIS44i7YrxOXT7F1bSrHL14lKMXj9q0PYWirE9ZyvuUp3zJ\n8ll/9qjbg0EtBzkixUIrqJAHArHZlk8Cd9kwJggwpZAnp6Zyym9R7h+m+eKRWgHPdD98tB+++FHa\nowKVSlalapmqBJavQk3/qtxRtQp3BlWlbmBlfL3lRgYhXF1Zn7I0rtqYxlUb5znmSsqVrMJ+Puk8\n55POc+HahZv/TLqxfDnlctYr+3XxdSq63oRhBRVyW5vaOfs3pjXDm4UG8mLQAqqU8yOggh+BlSpQ\no7IfQf7lKV3ScUU5OjraYet2BVbOz8q5geT3l7I+ZannY7RsbJGekc6V1CtcSr7EpZRLXEq+REJy\nAjX9ahYhWsfI92SnUqoNEKG17p65PArIyH7CUyn1KRCptV6QuXwQ+HvO1opSuVy+IYQQokBFvSFo\nG1BHKRUCxAF9gf45xiwDngMWZBb+hNz64wUFIoQQonDyLeRa6+tKqeeAnzAuP5yttT6glBqU+fkM\nrfUPSqn7lVJRwFXgKYdHLYQQIovTbggSQgjhGE69SFIp9bxS6oBSaq9S6pYbi6xAKfWKUipDKVXR\n7FjsSSn1Tua+26WUWqKUssStqEqp7kqpg0qpI0qpEWbHY09KqRpKqbVKqX2Z33MvmB2TvSmlPJRS\nO5VSy82Oxd4yL+VenPl9tz+zdZ0rpxVypVQHjGvOm2itGwHvOmvbzqKUqgF0AU6YHYsDrAIaaq2b\nAoeBUSbHU2TZbnjrDjQA+iul6psblV2lAS9prRti3KQ31GL5AQwD9mPilXIO9AHwg9a6PtCEm+/f\nuYkzj8iHAJO01mkAWutzTty2s7wHvGZ2EI6gtV6ttf5rUvTNGPcKuLusG94y/1/+dcObJWitT2ut\n/8h8n4hRCALMjcp+lFJBwP3AZ9x6CbRby/yN9x6t9RwwzldqrS/lNd6ZhbwOcK9S6nelVKRSqqUT\nt+1wSqmewEmt9e4CB7u/fwE/mB2EHeR2M1ugSbE4VOaVZ80wfghbxfvAcCCjoIFuqBZwTin1uVJq\nh1JqllKqVF6D7TofuVJqNZDbjFBjMrdVQWvdRinVClgI3GHP7TtaAfmNArpmH+6UoOwon/xGa62X\nZ44ZA6Rqrb9xanCOYcVfx2+hlCoDLAaGZR6Zuz2l1D+Bs1rrnUqp9mbH4wCeQHPgOa31VqXUVGAk\n8Hpeg+1Ga90lr8+UUkOAJZnjtmaeEKyktb5gzxgcKa/8lFKNMH6C7sqcXS0I2K6Uaq21PuvEEIsk\nv/0HoJQKx/hVtpNTAnK8U0CNbMs1MI7KLUMp5QV8B3yttf6v2fHYUVugR+akfSWBckqpL7XWT5oc\nl72cxPgNf2vm8mKMQp4rZ7ZW/gt0BFBK1QW83amI50drvVdrXVVrXUtrXQtjJzR3pyJekMzpjIcD\nPbXWyWbHYydZN7wppbwxbnhbZnJMdqOMo4rZwH6t9VSz47EnrfVorXWNzO+3fsAaCxVxtNangdjM\nWgnGDLT78hrvzEe9zQHmKKX2AKmAZf7Rc2HFX9k/BLyB1Zm/dWzSWj9rbkhFk9cNbyaHZU9/Ax4H\ndiul/npGwCit9UoTY3IUK37PPQ/MyzzIOEo+N1vKDUFCCOHm3GPWdCGEEHmSQi6EEG5OCrkQQrg5\nKeRCCOHmpJALIYSbk0IuhBBuTgq5EEK4OSnkQgjh5v4fkCR/CNy3Sa4AAAAASUVORK5CYII=\n", 280 | "text/plain": [ 281 | "" 282 | ] 283 | }, 284 | "metadata": {}, 285 | "output_type": "display_data" 286 | } 287 | ], 288 | "source": [ 289 | "testValues = np.arange(-5,5,0.01)\n", 290 | "plot(testValues, sigmoid(testValues), linewidth=2)\n", 291 | "plot(testValues, sigmoidPrime(testValues), linewidth=2)\n", 292 | "grid(1)\n", 293 | "legend(['sigmoid', 'sigmoidPrime'])" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "We can now replace dyHat/dz3 with f prime of z 3." 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "$$\n", 308 | "\\frac{\\partial J}{\\partial W^{(2)}}= \n", 309 | "-(y-\\hat{y}) f^\\prime(z^{(3)}) \\frac{\\partial z^{(3)}}{\\partial W^{(2)}}\n", 310 | "$$" 311 | ] 312 | }, 313 | { 314 | "cell_type": "markdown", 315 | "metadata": {}, 316 | "source": [ 317 | "Our final piece of the puzzle is dz3dW2, this term represents the change of z, our third layer activity, with respect to the weights in the second layer." 318 | ] 319 | }, 320 | { 321 | "cell_type": "markdown", 322 | "metadata": {}, 323 | "source": [ 324 | "Z three is the matrix product of our activities, a two, and our weights, w two. The activities from layer two are multiplied by their correspond weights and added together to yield z3. If we focus on a single synapse for a moment, we see a simple linear relationship between W and z, where a is the slope. So for each synapse, dz/dW(2) is just the activation, a on that synapse!" 325 | ] 326 | }, 327 | { 328 | "cell_type": "markdown", 329 | "metadata": {}, 330 | "source": [ 331 | "$$\n", 332 | "z^{(3)} = a^{(2)}W^{(2)} \\tag{3}\\\\\n", 333 | "$$" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "metadata": {}, 339 | "source": [ 340 | "Another way to think about what the calculus is doing here is that it is “backpropagating” the error to each weight, by multiplying by the activity on each synapses, the weights that contribute more to the error will have larger activations, and yield larger dJ/dW2 values, and those weights will be changed more when we perform gradient descent. " 341 | ] 342 | }, 343 | { 344 | "cell_type": "markdown", 345 | "metadata": {}, 346 | "source": [ 347 | "We need to be careful with our dimensionality here, and if we’re clever, we can take care of that summation we got rid of earlier. " 348 | ] 349 | }, 350 | { 351 | "cell_type": "markdown", 352 | "metadata": {}, 353 | "source": [ 354 | "The first part of our equation, y minus yHat is of the same dimension as our output data, 3 by 1. " 355 | ] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": {}, 360 | "source": [ 361 | "F prime of z three is of the same size, 3 by 1, and our first operation is scalar multiplication. Our resulting 3 by 1 matrix is referred to as the backpropagating error, delta 3." 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": {}, 367 | "source": [ 368 | "We determined that dz3/dW2 is equal to the activity of each synapse. Each value in delta 3 needs to be multiplied by each activity. We can achieve this by transposing a2 and matrix multiplying by delta3. " 369 | ] 370 | }, 371 | { 372 | "cell_type": "markdown", 373 | "metadata": {}, 374 | "source": [ 375 | "$$\n", 376 | "\\frac{\\partial J}{\\partial W^{(2)}} = \n", 377 | "(a^{(2)})^T\\delta^{(3)}\\tag{6}\n", 378 | "$$" 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": {}, 384 | "source": [ 385 | "$$\n", 386 | "\\delta^{(3)} = -(y-\\hat{y}) f^\\prime(z^{(3)}) \n", 387 | "$$" 388 | ] 389 | }, 390 | { 391 | "cell_type": "markdown", 392 | "metadata": {}, 393 | "source": [ 394 | "What’s cool here is that the matrix multiplication also takes care of our earlier omission – it adds up the dJ/dW terms across all our examples. " 395 | ] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "metadata": {}, 400 | "source": [ 401 | "Another way to think about what’s happening here is that is that each example our algorithm sees has a certain cost and a certain gradient. The gradient with respect to each example pulls our gradient descent algorithm in a certain direction. It's like every example gets a vote on which way is downhill, and when we perform batch gradient descent we just add together everyone’s vote, call it downhill, and move in that direction." 402 | ] 403 | }, 404 | { 405 | "cell_type": "markdown", 406 | "metadata": {}, 407 | "source": [ 408 | "We’ll code up our gradients in python in a new method, cost function prime. Numpy’s multiply method performs element-wise multiplication, and the dot method performs matrix multiplication." 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": 5, 414 | "metadata": { 415 | "collapsed": false 416 | }, 417 | "outputs": [], 418 | "source": [ 419 | "# Part of NN Class (won't work alone, needs to be included in class as \n", 420 | "# shown in below and in partFour.py):\n", 421 | "\n", 422 | "def costFunctionPrime(self, X, y):\n", 423 | " #Compute derivative with respect to W and W2 for a given X and y:\n", 424 | " self.yHat = self.forward(X)\n", 425 | "\n", 426 | " delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n", 427 | " dJdW2 = np.dot(self.a2.T, delta3)" 428 | ] 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": {}, 433 | "source": [ 434 | "We have one final term to compute: dJ/dW1. The derivation begins the same way, computing the derivative through our final layer: first dJ/dyHat, then dyHat/dz3, and we called these two taken together form our backpropagating error, delta3. We now take the derivative “across” our synapses, this is a little different from out job last time, computing the derivative with respect to the weights on our synapses. " 435 | ] 436 | }, 437 | { 438 | "cell_type": "markdown", 439 | "metadata": {}, 440 | "source": [ 441 | "$$\n", 442 | "\\frac{\\partial J}{\\partial W^{(1)}} = (y-\\hat{y})\n", 443 | "\\frac{\\partial \\hat{y}}{\\partial W^{(1)}}\n", 444 | "$$\n", 445 | "\n", 446 | "$$\n", 447 | "\\frac{\\partial J}{\\partial W^{(1)}} = (y-\\hat{y})\n", 448 | "\\frac{\\partial \\hat{y}}{\\partial z^{(3)}}\n", 449 | "\\frac{\\partial z^{(3)}}{\\partial W^{(1)}}\n", 450 | "$$\n", 451 | "\n", 452 | "$$\n", 453 | "\\frac{\\partial J}{\\partial W^{(1)}} = -(y-\\hat{y}) f^\\prime(z^{(3)}) \\frac{\\partial z^{(3)}}{\\partial W^{(1)}}\n", 454 | "$$\n", 455 | "\n", 456 | "$$\n", 457 | "\\frac{\\partial z^{(3)}}{\\partial W^{(1)}} = \\frac{\\partial z^{(3)}}{\\partial a^{(2)}}\\frac{\\partial a^{(2)}}{\\partial W^{(1)}}\n", 458 | "$$\n" 459 | ] 460 | }, 461 | { 462 | "cell_type": "markdown", 463 | "metadata": {}, 464 | "source": [ 465 | "There’s still a nice linear relationship along each synapse, but now we’re interested in the rate of change of z(3) with respect to a(2). Now the slope is just equal to the weight value for that synapse. We can achieve this mathematically by multiplying by W(2) transpose. " 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": {}, 471 | "source": [ 472 | "$$\n", 473 | "\\frac{\\partial J}{\\partial W^{(1)}} = \\delta^{(3)} \n", 474 | "(W^{(2)})^{T}\n", 475 | "\\frac{\\partial a^{(2)}}{\\partial W^{(1)}}\n", 476 | "$$" 477 | ] 478 | }, 479 | { 480 | "cell_type": "markdown", 481 | "metadata": {}, 482 | "source": [ 483 | "$$\n", 484 | "\\frac{\\partial J}{\\partial W^{(1)}} = \\delta^{(3)} \n", 485 | "(W^{(2)})^{T}\n", 486 | "\\frac{\\partial a^{(2)}}{\\partial z^{(2)}}\n", 487 | "\\frac{\\partial z^{(2)}}{\\partial W^{(1)}}\n", 488 | "$$" 489 | ] 490 | }, 491 | { 492 | "cell_type": "markdown", 493 | "metadata": {}, 494 | "source": [ 495 | "Our next term to work on is da(2)/dz(2) – this step is just like the derivative across our layer 3 neurons, so we can just multiply by f prime(z2). " 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "metadata": {}, 501 | "source": [ 502 | "$$\n", 503 | "\\frac{\\partial J}{\\partial W^{(1)}} = \\delta^{(3)} \n", 504 | "(W^{(2)})^{T}\n", 505 | "f^\\prime(z^{(2)})\n", 506 | "\\frac{\\partial z^{(2)}}{\\partial W^{(1)}}\n", 507 | "$$" 508 | ] 509 | }, 510 | { 511 | "cell_type": "markdown", 512 | "metadata": {}, 513 | "source": [ 514 | "Our final computation here is dz2/dW1. This is very similar to our dz3/dW2 computation, there is a simple linear relationship on the synapses between z2 and w1, in this case though, the slope is the input value, X. We can use the same technique as last time by multiplying by X transpose, effectively applying the derivative and adding our dJ/dW1’s together across all our examples. " 515 | ] 516 | }, 517 | { 518 | "cell_type": "markdown", 519 | "metadata": {}, 520 | "source": [ 521 | "$$\n", 522 | "\\frac{\\partial J}{\\partial W^{(1)}} = \n", 523 | "X^{T}\n", 524 | "\\delta^{(3)} \n", 525 | "(W^{(2)})^{T}\n", 526 | "f^\\prime(z^{(2)})\n", 527 | "$$" 528 | ] 529 | }, 530 | { 531 | "cell_type": "markdown", 532 | "metadata": {}, 533 | "source": [ 534 | "Or:" 535 | ] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": {}, 540 | "source": [ 541 | "$$\n", 542 | "\\frac{\\partial J}{\\partial W^{(1)}} = \n", 543 | "X^{T}\\delta^{(2)} \\tag{7}\n", 544 | "$$" 545 | ] 546 | }, 547 | { 548 | "cell_type": "markdown", 549 | "metadata": {}, 550 | "source": [ 551 | "Where:" 552 | ] 553 | }, 554 | { 555 | "cell_type": "markdown", 556 | "metadata": {}, 557 | "source": [ 558 | "$$\n", 559 | "\\delta^{(2)} = \\delta^{(3)} \n", 560 | "(W^{(2)})^{T}\n", 561 | "f^\\prime(z^{(2)})\n", 562 | "$$" 563 | ] 564 | }, 565 | { 566 | "cell_type": "markdown", 567 | "metadata": {}, 568 | "source": [ 569 | "All that’s left is to code this equation up in python. What’s cool here is that if we want to make a deeper neural network, we could just stack a bunch of these operations together. " 570 | ] 571 | }, 572 | { 573 | "cell_type": "code", 574 | "execution_count": 6, 575 | "metadata": { 576 | "collapsed": false 577 | }, 578 | "outputs": [], 579 | "source": [ 580 | "# Whole Class with additions:\n", 581 | "class Neural_Network(object):\n", 582 | " def __init__(self): \n", 583 | " #Define Hyperparameters\n", 584 | " self.inputLayerSize = 2\n", 585 | " self.outputLayerSize = 1\n", 586 | " self.hiddenLayerSize = 3\n", 587 | " \n", 588 | " #Weights (parameters)\n", 589 | " self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)\n", 590 | " self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)\n", 591 | " \n", 592 | " def forward(self, X):\n", 593 | " #Propogate inputs though network\n", 594 | " self.z2 = np.dot(X, self.W1)\n", 595 | " self.a2 = self.sigmoid(self.z2)\n", 596 | " self.z3 = np.dot(self.a2, self.W2)\n", 597 | " yHat = self.sigmoid(self.z3) \n", 598 | " return yHat\n", 599 | " \n", 600 | " def sigmoid(self, z):\n", 601 | " #Apply sigmoid activation function to scalar, vector, or matrix\n", 602 | " return 1/(1+np.exp(-z))\n", 603 | " \n", 604 | " def sigmoidPrime(self,z):\n", 605 | " #Gradient of sigmoid\n", 606 | " return np.exp(-z)/((1+np.exp(-z))**2)\n", 607 | " \n", 608 | " def costFunction(self, X, y):\n", 609 | " #Compute cost for given X,y, use weights already stored in class.\n", 610 | " self.yHat = self.forward(X)\n", 611 | " J = 0.5*sum((y-self.yHat)**2)\n", 612 | " return J\n", 613 | " \n", 614 | " def costFunctionPrime(self, X, y):\n", 615 | " #Compute derivative with respect to W and W2 for a given X and y:\n", 616 | " self.yHat = self.forward(X)\n", 617 | " \n", 618 | " delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))\n", 619 | " dJdW2 = np.dot(self.a2.T, delta3)\n", 620 | " \n", 621 | " delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)\n", 622 | " dJdW1 = np.dot(X.T, delta2) \n", 623 | " \n", 624 | " return dJdW1, dJdW2" 625 | ] 626 | }, 627 | { 628 | "cell_type": "markdown", 629 | "metadata": {}, 630 | "source": [ 631 | "So how should we change our W’s to decrease our cost? We can now compute dJ/dW, which tells us which way is uphill in our 9 dimensional optimization space. " 632 | ] 633 | }, 634 | { 635 | "cell_type": "code", 636 | "execution_count": 7, 637 | "metadata": { 638 | "collapsed": false 639 | }, 640 | "outputs": [], 641 | "source": [ 642 | "NN = Neural_Network()" 643 | ] 644 | }, 645 | { 646 | "cell_type": "code", 647 | "execution_count": 8, 648 | "metadata": { 649 | "collapsed": false 650 | }, 651 | "outputs": [], 652 | "source": [ 653 | "cost1 = NN.costFunction(X,y)" 654 | ] 655 | }, 656 | { 657 | "cell_type": "code", 658 | "execution_count": 9, 659 | "metadata": { 660 | "collapsed": false 661 | }, 662 | "outputs": [], 663 | "source": [ 664 | "dJdW1, dJdW2 = NN.costFunctionPrime(X,y)" 665 | ] 666 | }, 667 | { 668 | "cell_type": "code", 669 | "execution_count": 10, 670 | "metadata": { 671 | "collapsed": false 672 | }, 673 | "outputs": [ 674 | { 675 | "data": { 676 | "text/plain": [ 677 | "array([[ 0.00292997, 0.01709262, 0.05603318],\n", 678 | " [ 0.00183221, 0.01376613, 0.04525805]])" 679 | ] 680 | }, 681 | "execution_count": 10, 682 | "metadata": {}, 683 | "output_type": "execute_result" 684 | } 685 | ], 686 | "source": [ 687 | "dJdW1" 688 | ] 689 | }, 690 | { 691 | "cell_type": "code", 692 | "execution_count": 11, 693 | "metadata": { 694 | "collapsed": false 695 | }, 696 | "outputs": [ 697 | { 698 | "data": { 699 | "text/plain": [ 700 | "array([[-0.05659312],\n", 701 | " [-0.13972542],\n", 702 | " [-0.19348711]])" 703 | ] 704 | }, 705 | "execution_count": 11, 706 | "metadata": {}, 707 | "output_type": "execute_result" 708 | } 709 | ], 710 | "source": [ 711 | "dJdW2" 712 | ] 713 | }, 714 | { 715 | "cell_type": "markdown", 716 | "metadata": {}, 717 | "source": [ 718 | "If we move this way by adding a scalar times our derivative to our weights, our cost will increase, and if we do the opposite, subtract our gradient from our weights, we will move downhill and reduce our cost. This simple step downhill is the core of gradient descent and a key part of how even very sophisticated learning algorithms are trained. " 719 | ] 720 | }, 721 | { 722 | "cell_type": "code", 723 | "execution_count": 12, 724 | "metadata": { 725 | "collapsed": false 726 | }, 727 | "outputs": [], 728 | "source": [ 729 | "scalar = 3\n", 730 | "NN.W1 = NN.W1 + scalar*dJdW1\n", 731 | "NN.W2 = NN.W2 + scalar*dJdW2\n", 732 | "cost2 = NN.costFunction(X,y)" 733 | ] 734 | }, 735 | { 736 | "cell_type": "code", 737 | "execution_count": 13, 738 | "metadata": { 739 | "collapsed": false 740 | }, 741 | "outputs": [ 742 | { 743 | "name": "stdout", 744 | "output_type": "stream", 745 | "text": [ 746 | "0.42407153522 0.623202502031\n" 747 | ] 748 | } 749 | ], 750 | "source": [ 751 | "print cost1, cost2" 752 | ] 753 | }, 754 | { 755 | "cell_type": "code", 756 | "execution_count": 14, 757 | "metadata": { 758 | "collapsed": false 759 | }, 760 | "outputs": [], 761 | "source": [ 762 | "dJdW1, dJdW2 = NN.costFunctionPrime(X,y)\n", 763 | "NN.W1 = NN.W1 - scalar*dJdW1\n", 764 | "NN.W2 = NN.W2 - scalar*dJdW2\n", 765 | "cost3 = NN.costFunction(X, y)" 766 | ] 767 | }, 768 | { 769 | "cell_type": "code", 770 | "execution_count": 15, 771 | "metadata": { 772 | "collapsed": false 773 | }, 774 | "outputs": [ 775 | { 776 | "name": "stdout", 777 | "output_type": "stream", 778 | "text": [ 779 | "0.623202502031 0.429162161135\n" 780 | ] 781 | } 782 | ], 783 | "source": [ 784 | "print cost2, cost3" 785 | ] 786 | }, 787 | { 788 | "cell_type": "markdown", 789 | "metadata": {}, 790 | "source": [ 791 | "Next time we’ll perform numerical gradient checking check to make sure our math is correct. " 792 | ] 793 | }, 794 | { 795 | "cell_type": "code", 796 | "execution_count": null, 797 | "metadata": { 798 | "collapsed": true 799 | }, 800 | "outputs": [], 801 | "source": [] 802 | } 803 | ], 804 | "metadata": { 805 | "kernelspec": { 806 | "display_name": "Python 2", 807 | "language": "python", 808 | "name": "python2" 809 | }, 810 | "language_info": { 811 | "codemirror_mode": { 812 | "name": "ipython", 813 | "version": 2 814 | }, 815 | "file_extension": ".py", 816 | "mimetype": "text/x-python", 817 | "name": "python", 818 | "nbconvert_exporter": "python", 819 | "pygments_lexer": "ipython2", 820 | "version": "2.7.10" 821 | } 822 | }, 823 | "nbformat": 4, 824 | "nbformat_minor": 0 825 | } 826 | --------------------------------------------------------------------------------