├── FCLayer.ipynb ├── FcLayerOpt.ipynb ├── LICENSE ├── adversarial_mnist.py ├── cnn.py ├── gd.ipynb ├── momentum.ipynb ├── relu.log └── sigmoid.log /FcLayerOpt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 26, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "\n", 13 | "trace = False\n", 14 | "trace_forward = False\n", 15 | "\n", 16 | "class FC:\n", 17 | " '''\n", 18 | " This class is not thread safe.\n", 19 | " '''\n", 20 | " def __init__(self, in_num, out_num, lr = 0.1):\n", 21 | " self._in_num = in_num\n", 22 | " self._out_num = out_num\n", 23 | " self.w = np.random.randn(in_num, out_num)\n", 24 | " self.b = np.zeros((out_num, 1))\n", 25 | " self.lr = lr\n", 26 | " def _sigmoid(self, in_data):\n", 27 | " return 1 / (1 + np.exp(-in_data))\n", 28 | " def forward(self, in_data):\n", 29 | " \n", 30 | " self.topVal = self._sigmoid(np.dot(self.w.T, in_data) + self.b)\n", 31 | " if trace_forward:\n", 32 | " print '=== topVal {0} ==='.format(self.topVal.shape)\n", 33 | " print self.topVal\n", 34 | " self.bottomVal = in_data\n", 35 | " return self.topVal\n", 36 | " def backward(self, loss):\n", 37 | " residual_z = loss * self.topVal * (1 - self.topVal)\n", 38 | " grad_w = np.dot(self.bottomVal, residual_z.T)\n", 39 | " grad_b = np.sum(residual_z)\n", 40 | " self.w -= self.lr * grad_w\n", 41 | " self.b -= self.lr * grad_b\n", 42 | " residual_x = np.dot(self.w, residual_z)\n", 43 | " if trace:\n", 44 | " print '=== z {0}==='.format(residual_z.shape)\n", 45 | " print residual_z\n", 46 | " print '=== grad_w {0}==='.format(grad_w.shape)\n", 47 | " print grad_w\n", 48 | " print '=== grad_b {0}==='.format(grad_b.shape)\n", 49 | " print grad_b\n", 50 | " print '=== self.w {0}==='.format(self.w.shape)\n", 51 | " print self.w\n", 52 | " print '=== self.b {0} ==='.format(self.b.shape)\n", 53 | " print self.b\n", 54 | " print '=== residual {0} ==='.format(residual_x.shape)\n", 55 | " print residual_x\n", 56 | " return residual_x\n", 57 | "\n", 58 | "class SquareLoss:\n", 59 | " '''\n", 60 | " Same as above, not thread safe\n", 61 | " '''\n", 62 | " def forward(self, y, t):\n", 63 | " self.loss = y - t\n", 64 | " if trace:\n", 65 | " print '=== Loss ==='.format(self.loss.shape)\n", 66 | " print self.loss\n", 67 | " return np.sum(self.loss * self.loss) / self.loss.shape[1] / 2\n", 68 | " def backward(self):\n", 69 | " if trace:\n", 70 | " print '=== loss {0} ==='.format(self.loss.shape)\n", 71 | " print self.loss\n", 72 | " return self.loss\n", 73 | "\n", 74 | "class Net:\n", 75 | " def __init__(self, input_num=2, hidden_num=4, out_num=1, lr=0.1):\n", 76 | " self.fc1 = FC(input_num, hidden_num, lr)\n", 77 | " self.fc2 = FC(hidden_num, out_num, lr)\n", 78 | " self.loss = SquareLoss()\n", 79 | " def train(self, X, y): # X are arranged by col\n", 80 | " for i in range(10000):\n", 81 | " # forward step\n", 82 | " layer1out = self.fc1.forward(X)\n", 83 | " layer2out = self.fc2.forward(layer1out)\n", 84 | " loss = self.loss.forward(layer2out, y)\n", 85 | " if i % 1000 == 0:\n", 86 | " print 'iter = {0}, loss ={1}'.format(i, loss)\n", 87 | " print '=== Label vs Prediction ==='\n", 88 | " print 't={0}'.format(y)\n", 89 | " print 'y={0}'.format(layer2out)\n", 90 | " # backward step\n", 91 | " layer2loss = self.loss.backward()\n", 92 | " layer1loss = self.fc2.backward(layer2loss)\n", 93 | " saliency = self.fc1.backward(layer1loss)\n", 94 | " layer1out = self.fc1.forward(X)\n", 95 | " layer2out = self.fc2.forward(layer1out)\n", 96 | " print '=== Final ==='\n", 97 | " print 'X={0}'.format(X)\n", 98 | " print 't={0}'.format(y)\n", 99 | " print 'y={0}'.format(layer2out)\n", 100 | "\n" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 25, 106 | "metadata": { 107 | "collapsed": false 108 | }, 109 | "outputs": [ 110 | { 111 | "name": "stdout", 112 | "output_type": "stream", 113 | "text": [ 114 | "iter = 0, loss =0.29837110876\n", 115 | "=== Label vs Prediction ===\n", 116 | "t=[[ 0.01]\n", 117 | " [ 0.99]]\n", 118 | "y=[[ 0.75136507]\n", 119 | " [ 0.77292847]]\n", 120 | "iter = 1000, loss =0.00034637971269\n", 121 | "=== Label vs Prediction ===\n", 122 | "t=[[ 0.01]\n", 123 | " [ 0.99]]\n", 124 | "y=[[ 0.02873778]\n", 125 | " [ 0.97151609]]\n", 126 | "iter = 2000, loss =0.000119529215903\n", 127 | "=== Label vs Prediction ===\n", 128 | "t=[[ 0.01]\n", 129 | " [ 0.99]]\n", 130 | "y=[[ 0.0210022 ]\n", 131 | " [ 0.97913675]]\n", 132 | "iter = 3000, loss =6.0258461253e-05\n", 133 | "=== Label vs Prediction ===\n", 134 | "t=[[ 0.01]\n", 135 | " [ 0.99]]\n", 136 | "y=[[ 0.01781127]\n", 137 | " [ 0.98228631]]\n", 138 | "iter = 4000, loss =3.54968683217e-05\n", 139 | "=== Label vs Prediction ===\n", 140 | "t=[[ 0.01]\n", 141 | " [ 0.99]]\n", 142 | "y=[[ 0.01599555]\n", 143 | " [ 0.98407994]]\n", 144 | "iter = 5000, loss =2.27849062666e-05\n", 145 | "=== Label vs Prediction ===\n", 146 | "t=[[ 0.01]\n", 147 | " [ 0.99]]\n", 148 | "y=[[ 0.01480398]\n", 149 | " [ 0.98525747]]\n", 150 | "iter = 6000, loss =1.5445159009e-05\n", 151 | "=== Label vs Prediction ===\n", 152 | "t=[[ 0.01]\n", 153 | " [ 0.99]]\n", 154 | "y=[[ 0.01395575]\n", 155 | " [ 0.98609585]]\n", 156 | "iter = 7000, loss =1.08721488321e-05\n", 157 | "=== Label vs Prediction ===\n", 158 | "t=[[ 0.01]\n", 159 | " [ 0.99]]\n", 160 | "y=[[ 0.01331933]\n", 161 | " [ 0.98672489]]\n", 162 | "iter = 8000, loss =7.86715065902e-06\n", 163 | "=== Label vs Prediction ===\n", 164 | "t=[[ 0.01]\n", 165 | " [ 0.99]]\n", 166 | "y=[[ 0.01282401]\n", 167 | " [ 0.98721445]]\n", 168 | "iter = 9000, loss =5.81352036324e-06\n", 169 | "=== Label vs Prediction ===\n", 170 | "t=[[ 0.01]\n", 171 | " [ 0.99]]\n", 172 | "y=[[ 0.01242797]\n", 173 | " [ 0.98760584]]\n", 174 | "=== Final ===\n", 175 | "X=[[ 0.05]\n", 176 | " [ 0.1 ]]\n", 177 | "t=[[ 0.01]\n", 178 | " [ 0.99]]\n", 179 | "y=[[ 0.01210474]\n", 180 | " [ 0.98792522]]\n" 181 | ] 182 | } 183 | ], 184 | "source": [ 185 | "# example from https://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/\n", 186 | "X = np.array([[0.05, 0.1]]).T\n", 187 | "y = np.array([[0.01, 0.99]]).T\n", 188 | "\n", 189 | "net = Net(2,2,2,0.5)\n", 190 | "net.fc1.w = np.array([[.15,.25], [.2, .3]])\n", 191 | "net.fc1.b = np.array([[.35], [.35]])\n", 192 | "net.fc2.w = np.array([[.4,.5], [.45,.55]])\n", 193 | "net.fc2.b = np.array([[.6], [.6]])\n", 194 | "net.train(X,y)" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 27, 200 | "metadata": { 201 | "collapsed": false 202 | }, 203 | "outputs": [ 204 | { 205 | "name": "stdout", 206 | "output_type": "stream", 207 | "text": [ 208 | "iter = 0, loss =0.105256639066\n", 209 | "=== Label vs Prediction ===\n", 210 | "t=[[0 0 0 1]]\n", 211 | "y=[[ 0.40930536 0.4617139 0.36923076 0.4299025 ]]\n", 212 | "iter = 1000, loss =0.0229368486589\n", 213 | "=== Label vs Prediction ===\n", 214 | "t=[[0 0 0 1]]\n", 215 | "y=[[ 0.04445123 0.22684496 0.17747671 0.68605373]]\n", 216 | "iter = 2000, loss =0.00657594469044\n", 217 | "=== Label vs Prediction ===\n", 218 | "t=[[0 0 0 1]]\n", 219 | "y=[[ 0.01057127 0.11332809 0.11016211 0.83411794]]\n", 220 | "iter = 3000, loss =0.00322081318498\n", 221 | "=== Label vs Prediction ===\n", 222 | "t=[[0 0 0 1]]\n", 223 | "y=[[ 0.00517544 0.07831654 0.07871461 0.88419737]]\n", 224 | "iter = 4000, loss =0.00201059297485\n", 225 | "=== Label vs Prediction ===\n", 226 | "t=[[0 0 0 1]]\n", 227 | "y=[[ 0.00336374 0.06171018 0.0624756 0.90855558]]\n", 228 | "iter = 5000, loss =0.00142205310651\n", 229 | "=== Label vs Prediction ===\n", 230 | "t=[[0 0 0 1]]\n", 231 | "y=[[ 0.00249895 0.05189239 0.05257126 0.92309992]]\n", 232 | "iter = 6000, loss =0.00108341055769\n", 233 | "=== Label vs Prediction ===\n", 234 | "t=[[0 0 0 1]]\n", 235 | "y=[[ 0.00200067 0.04532728 0.04585262 0.93287134]]\n", 236 | "iter = 7000, loss =0.000866734887908\n", 237 | "=== Label vs Prediction ===\n", 238 | "t=[[0 0 0 1]]\n", 239 | "y=[[ 0.00167856 0.04058314 0.04096262 0.9399489 ]]\n", 240 | "iter = 8000, loss =0.000717647908313\n", 241 | "=== Label vs Prediction ===\n", 242 | "t=[[0 0 0 1]]\n", 243 | "y=[[ 0.00145369 0.03696819 0.0372232 0.94534786]]\n", 244 | "iter = 9000, loss =0.000609513241467\n", 245 | "=== Label vs Prediction ===\n", 246 | "t=[[0 0 0 1]]\n", 247 | "y=[[ 0.00128784 0.03410575 0.03425751 0.94962473]]\n", 248 | "=== Final ===\n", 249 | "X=[[0 0 1 1]\n", 250 | " [0 1 0 1]]\n", 251 | "t=[[0 0 0 1]]\n", 252 | "y=[[ 0.00116042 0.03177232 0.03183889 0.95311123]]\n" 253 | ] 254 | } 255 | ], 256 | "source": [ 257 | "# and operation\n", 258 | "X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]).T\n", 259 | "y = np.array([[0],[0],[0],[1]]).T\n", 260 | "\n", 261 | "net = Net(2,4,1,0.1)\n", 262 | "net.train(X,y)" 263 | ] 264 | } 265 | ], 266 | "metadata": { 267 | "kernelspec": { 268 | "display_name": "Python 2", 269 | "language": "python", 270 | "name": "python2" 271 | }, 272 | "language_info": { 273 | "codemirror_mode": { 274 | "name": "ipython", 275 | "version": 2 276 | }, 277 | "file_extension": ".py", 278 | "mimetype": "text/x-python", 279 | "name": "python", 280 | "nbconvert_exporter": "python", 281 | "pygments_lexer": "ipython2", 282 | "version": "2.7.10" 283 | } 284 | }, 285 | "nbformat": 4, 286 | "nbformat_minor": 0 287 | } 288 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 hsmyy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /adversarial_mnist.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | #from __future__ import print_function 3 | import os 4 | import struct 5 | from array import array 6 | 7 | from keras.models import Sequential 8 | from keras.layers.core import Dense, Dropout, Activation, Flatten 9 | from keras.layers.convolutional import Convolution2D, MaxPooling2D 10 | from keras.regularizers import l2, l1 11 | from keras.constraints import maxnorm 12 | from keras.optimizers import SGD, Adam, RMSprop, Adagrad 13 | from keras.utils import np_utils, generic_utils 14 | import numpy as np 15 | import matplotlib.pyplot as plt 16 | 17 | class MNIST(object): 18 | def __init__(self, path='.'): 19 | self.path = path 20 | 21 | self.test_img_fname = 't10k-images-idx3-ubyte' 22 | self.test_lbl_fname = 't10k-labels-idx1-ubyte' 23 | 24 | self.train_img_fname = 'train-images-idx3-ubyte' 25 | self.train_lbl_fname = 'train-labels-idx1-ubyte' 26 | 27 | self.test_images = [] 28 | self.test_labels = [] 29 | 30 | self.train_images = [] 31 | self.train_labels = [] 32 | 33 | def load_testing(self): 34 | ims, labels = self.load(os.path.join(self.path, self.test_img_fname), 35 | os.path.join(self.path, self.test_lbl_fname)) 36 | 37 | self.test_images = np.array(ims) 38 | self.test_labels = np.array(labels) 39 | 40 | return ims, labels 41 | 42 | def load_training(self): 43 | ims, labels = self.load(os.path.join(self.path, self.train_img_fname), 44 | os.path.join(self.path, self.train_lbl_fname)) 45 | 46 | self.train_images = np.array(ims) 47 | self.train_labels = np.array(labels) 48 | 49 | np.random.seed(1337) 50 | np.random.shuffle(self.train_images) 51 | np.random.seed(1337) 52 | np.random.shuffle(self.train_labels) 53 | 54 | return ims, labels 55 | 56 | @classmethod 57 | def load(cls, path_img, path_lbl): 58 | with open(path_lbl, 'rb') as file: 59 | magic, size = struct.unpack(">II", file.read(8)) 60 | if magic != 2049: 61 | raise ValueError('Magic number mismatch, expected 2049,' 62 | 'got %d' % magic) 63 | 64 | labels = array("B", file.read()) 65 | 66 | with open(path_img, 'rb') as file: 67 | magic, size, rows, cols = struct.unpack(">IIII", file.read(16)) 68 | if magic != 2051: 69 | raise ValueError('Magic number mismatch, expected 2051,' 70 | 'got %d' % magic) 71 | 72 | image_data = array("B", file.read()) 73 | 74 | images = [] 75 | for i in xrange(size): 76 | images.append([0]*rows*cols) 77 | 78 | for i in xrange(size): 79 | images[i][:] = image_data[i*rows*cols : (i+1)*rows*cols] 80 | 81 | return images, labels 82 | 83 | def test(self): 84 | test_img, test_label = self.load_testing() 85 | train_img, train_label = self.load_training() 86 | assert len(test_img) == len(test_label) 87 | assert len(test_img) == 10000 88 | assert len(train_img) == len(train_label) 89 | assert len(train_img) == 60000 90 | print ("Showing num:" , train_label[0]) 91 | print (self.display(train_img[0])) 92 | print 93 | return True 94 | 95 | @classmethod 96 | def display(cls, img, width=28): 97 | render = '' 98 | for i in range(len(img)): 99 | if i % width == 0: render += '\n' 100 | if img[i] > 200: 101 | render += '1' 102 | else: 103 | render += '0' 104 | return render 105 | 106 | def image_generator(img, batch_size): 107 | dataset = np.zeros((64, 1, 28, 28)) 108 | for i in range(batch_size): 109 | dataset[i] = img + np.random.uniform(low=-0.1, high=0.1, size=(1, 28, 28)) 110 | return dataset 111 | 112 | def build_model(): 113 | nb_classes = 10 114 | model = Sequential() 115 | model.add(Convolution2D(32, 3, 3, input_shape=(1,28,28))) 116 | model.add(Activation('relu')) 117 | model.add(MaxPooling2D(pool_size=(2, 2))) 118 | model.add(Convolution2D(64, 3, 3)) 119 | model.add(Activation('relu')) 120 | model.add(MaxPooling2D(pool_size=(2, 2))) 121 | model.add(Flatten()) 122 | model.add(Dense(256, init='normal')) 123 | model.add(Activation('relu')) 124 | model.add(Dropout(0.5)) 125 | model.add(Dense(nb_classes, init='normal')) 126 | model.add(Activation('softmax')) 127 | sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) 128 | model.compile(loss='categorical_crossentropy', optimizer=sgd) 129 | return model 130 | 131 | def gen(X_train, Y_train, X_test, Y_test): 132 | batch_size = 64 133 | nb_classes = 10 134 | nb_epoch = 20 135 | 136 | img = X_train[2] 137 | 138 | img = img.astype("float32").reshape((1,28,28)) 139 | label = Y_train[2] 140 | img /= 255.0 141 | print 'label=' + str(label) 142 | plt.imshow(img.reshape((28,28)),cmap = plt.cm.gray) 143 | plt.show() 144 | 145 | model = build_model() 146 | model.load_weights('mnist_cnn') 147 | 148 | for iterator in range(200): 149 | 150 | ds = image_generator(img, 64) 151 | pred = model.predict(ds, batch_size=64) 152 | pred_label = np.argmax(pred, axis=1) 153 | flag = False 154 | for i in range(64): 155 | if pred_label[i] == label: 156 | choosed_img = ds[i] 157 | flag = True 158 | break 159 | if flag == False: 160 | print 'iter=' + str(iterator) + ", break" 161 | break 162 | else: 163 | img = choosed_img 164 | print 'iter=' + str(iterator) + ", label = " + str(label) 165 | if iterator == 50 or iterator == 100 or iterator == 150: 166 | plt.imshow(img.reshape((28,28)),cmap = plt.cm.gray) 167 | plt.show() 168 | print img 169 | plt.imshow(img.reshape((28,28)),cmap = plt.cm.gray) 170 | plt.show() 171 | 172 | # choose the best 173 | def gen2(X_train, Y_train, X_test, Y_test): 174 | batch_size = 64 175 | nb_classes = 10 176 | nb_epoch = 20 177 | 178 | img = X_train[2] 179 | 180 | img = img.astype("float32").reshape((1,28,28)) 181 | label = Y_train[2] 182 | img /= 255.0 183 | print 'label=' + str(label) 184 | 185 | model = build_model() 186 | model.load_weights('mnist_cnn') 187 | 188 | for iterator in range(1000): 189 | ds = image_generator(img, 64) 190 | pred = model.predict(ds, batch_size=64) 191 | pred_label = np.argmax(pred, axis=1) 192 | flag = False 193 | for i in range(64): 194 | if pred_label[i] != label: 195 | choosed_idx = i 196 | flag = True 197 | break 198 | if flag == False: 199 | print 'iter=' + str(iterator) + ", no change" 200 | img = ds[0] 201 | else: 202 | img = ds[choosed_idx] 203 | print 'iter=' + str(iterator) + ", label = " + str(pred_label[choosed_idx]) 204 | break 205 | plt.imshow(img.reshape((28,28)),cmap = plt.cm.gray) 206 | plt.show() 207 | 208 | 209 | def CNN(X_train, Y_train, X_test, Y_test): 210 | batch_size = 64 211 | nb_classes = 10 212 | nb_epoch = 20 213 | 214 | X_train = X_train.reshape(60000, 1, 28, 28) 215 | X_test = X_test.reshape(10000, 1, 28, 28) 216 | X_train = X_train.astype("float32") 217 | X_test = X_test.astype("float32") 218 | X_train /= 255 219 | X_test /= 255 220 | print(X_train.shape, 'train samples') 221 | print(Y_train.shape, 'train labels') 222 | print(X_test.shape, 'test smaples') 223 | 224 | Y_train = np_utils.to_categorical(Y_train, nb_classes) 225 | Y_test = np_utils.to_categorical(Y_test, nb_classes) 226 | 227 | model = build_model() 228 | model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=30) 229 | score = model.evaluate(X_test, Y_test, batch_size=batch_size) 230 | model.save_weights('mnist_cnn') 231 | print('Test score:', score) 232 | 233 | if __name__ == "__main__": 234 | print ('Testing') 235 | mn = MNIST('.') 236 | if mn.test(): 237 | print ('Passed') 238 | #CNN(mn.train_images, mn.train_labels, mn.test_images, mn.test_labels) 239 | gen2(mn.train_images, mn.train_labels, mn.test_images, mn.test_labels) 240 | -------------------------------------------------------------------------------- /cnn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | 4 | def conv2(X, k): 5 | # as a demo code, here we ignore the shape check 6 | x_row, x_col = X.shape 7 | k_row, k_col = k.shape 8 | ret_row, ret_col = x_row - k_row + 1, x_col - k_col + 1 9 | ret = np.empty((ret_row, ret_col)) 10 | for y in range(ret_row): 11 | for x in range(ret_col): 12 | sub = X[y : y + k_row, x : x + k_col] 13 | ret[y,x] = np.sum(sub * k) 14 | return ret 15 | 16 | def rot180(in_data): 17 | ret = in_data.copy() 18 | yEnd = ret.shape[0] - 1 19 | xEnd = ret.shape[1] - 1 20 | for y in range(ret.shape[0] / 2): 21 | for x in range(ret.shape[1]): 22 | ret[yEnd - y][x] = ret[y][x] 23 | for y in range(ret.shape[0]): 24 | for x in range(ret.shape[1] / 2): 25 | ret[y][xEnd - x] = ret[y][x] 26 | return ret 27 | 28 | def padding(in_data, size): 29 | cur_r, cur_w = in_data.shape[0], in_data.shape[1] 30 | new_r = cur_r + size * 2 31 | new_w = cur_w + size * 2 32 | ret = np.zeros((new_r, new_w)) 33 | ret[size:cur_r + size, size:cur_w+size] = in_data 34 | return ret 35 | 36 | def discreterize(in_data, size): 37 | num = in_data.shape[0] 38 | ret = np.zeros((num, size)) 39 | for i, idx in enumerate(in_data): 40 | ret[i, idx] = 1 41 | return ret 42 | 43 | class ConvLayer: 44 | def __init__(self, in_channel, out_channel, kernel_size, lr=0.01, momentum=0.9, name='Conv'): 45 | self.w = np.random.randn(in_channel, out_channel, kernel_size, kernel_size) 46 | self.b = np.zeros((out_channel)) 47 | self.layer_name = name 48 | self.lr = lr 49 | self.momentum = momentum 50 | 51 | self.prev_gradient_w = np.zeros_like(self.w) 52 | self.prev_gradient_b = np.zeros_like(self.b) 53 | # def _relu(self, x): 54 | # x[x < 0] = 0 55 | # return x 56 | def forward(self, in_data): 57 | # assume the first index is channel index 58 | print 'conv forward:' + str(in_data.shape) 59 | in_batch, in_channel, in_row, in_col = in_data.shape 60 | out_channel, kernel_size = self.w.shape[1], self.w.shape[2] 61 | self.top_val = np.zeros((in_batch, out_channel, in_row - kernel_size + 1, in_col - kernel_size + 1)) 62 | self.bottom_val = in_data 63 | 64 | for b_id in range(in_batch): 65 | for o in range(out_channel): 66 | for i in range(in_channel): 67 | self.top_val[b_id, o] += conv2(in_data[b_id, i], self.w[i, o]) 68 | self.top_val[b_id, o] += self.b[o] 69 | return self.top_val 70 | 71 | def backward(self, residual): 72 | in_channel, out_channel, kernel_size = self.w.shape 73 | in_batch = residual.shape[0] 74 | # gradient_b 75 | self.gradient_b = residual.sum(axis=3).sum(axis=2).sum(axis=0) / self.batch_size 76 | # gradient_w 77 | self.gradient_w = np.zeros_like(self.w) 78 | for b_id in range(in_batch): 79 | for i in range(in_channel): 80 | for o in range(out_channel): 81 | self.gradient_w[i, o] += conv2(self.bottom_val[b_id], residual[o]) 82 | self.gradient_w /= self.batch_size 83 | # gradient_x 84 | gradient_x = np.zeros_like(self.bottom_val) 85 | for b_id in range(in_batch): 86 | for i in range(in_channel): 87 | for o in range(out_channel): 88 | gradient_x[b_id, i] += conv2(padding(residual, kernel_size - 1), rot180(self.w[i, o])) 89 | gradient_x /= self.batch_size 90 | # update 91 | self.prev_gradient_w = self.prev_gradient_w * self.momentum - self.gradient_w 92 | self.w += self.lr * self.prev_gradient_w 93 | self.prev_gradient_b = self.prev_gradient_b * self.momentum - self.gradient_b 94 | self.b += self.lr * self.prev_gradient_b 95 | return gradient_x 96 | 97 | class FCLayer: 98 | def __init__(self, in_num, out_num, lr = 0.01, momentum=0.9): 99 | self._in_num = in_num 100 | self._out_num = out_num 101 | self.w = np.random.randn(in_num, out_num) 102 | self.b = np.zeros((out_num, 1)) 103 | self.lr = lr 104 | self.momentum = momentum 105 | self.prev_grad_w = np.zeros_like(self.w) 106 | self.prev_grad_b = np.zeros_like(self.b) 107 | # def _sigmoid(self, in_data): 108 | # return 1 / (1 + np.exp(-in_data)) 109 | def forward(self, in_data): 110 | print 'fc forward=' + str(in_data.shape) 111 | self.topVal = np.dot(self.w.T, in_data) + self.b 112 | self.bottomVal = in_data 113 | return self.topVal 114 | def backward(self, loss): 115 | batch_size = loss.shape[0] 116 | 117 | # residual_z = loss * self.topVal * (1 - self.topVal) 118 | grad_w = np.dot(self.bottomVal, loss.T) / batch_size 119 | grad_b = np.sum(loss) / batch_size 120 | residual_x = np.dot(self.w, loss) 121 | self.prev_grad_w = self.prev_grad_w * momentum - grad_w 122 | self.prev_grad_b = self.prev_grad_b * momentum - grad_b 123 | self.w -= self.lr * self.prev_grad_w 124 | self.b -= self.lr * self.prev_grad_b 125 | return residual_x 126 | 127 | class ReLULayer: 128 | def __init__(self, name='ReLU'): 129 | pass 130 | 131 | def forward(self, in_data): 132 | self.top_val = in_data 133 | ret = in_data.copy() 134 | ret[ret < 0] = 0 135 | return ret 136 | def backward(self, residual): 137 | gradient_x = residual.copy() 138 | gradient_x[self.top_val < 0] = 0 139 | return gradient_x 140 | 141 | class MaxPoolingLayer: 142 | def __init__(self, kernel_size, name='MaxPool'): 143 | self.kernel_size = kernel_size 144 | 145 | def forward(self, in_data): 146 | in_batch, in_channel, in_row, in_col = in_data.shape 147 | k = self.kernel_size 148 | out_row = in_row / k + (1 if in_row % k != 0 else 0) 149 | out_col = in_col / k + (1 if in_col % k != 0 else 0) 150 | 151 | self.flag = np.zeros_like(in_data) 152 | ret = np.empty((in_batch, in_channel, out_row, out_col)) 153 | for b_id in range(in_batch): 154 | for c in range(in_channel): 155 | for oy in range(out_row): 156 | for ox in range(out_col): 157 | height = k if (oy + 1) * k <= in_row else in_row - oy * k 158 | width = k if (ox + 1) * k <= in_col else in_col - ox * k 159 | idx = np.argmax(in_data[b_id, c, oy * k: oy * k + height, ox * k: ox * k + width]) 160 | offset_r = idx / width 161 | offset_c = idx % width 162 | self.flag[b_id, c, oy * k + offset_r, ox * k + offset_c] = 1 163 | ret[b_id, c, oy, ox] = in_data[b_id, c, oy * k + offset_r, ox * k + offset_c] 164 | return ret 165 | def backward(self, residual): 166 | in_batch, in_channel, in_row, in_col = self.flag 167 | k = self.kernel_size 168 | out_row, out_col = residual.shape[2], residual.shape[3] 169 | 170 | gradient_x = np.zeros_like(self.flag) 171 | for b_id in range(in_batch): 172 | for c in range(in_channel): 173 | for oy in range(out_row): 174 | for ox in range(out_col): 175 | height = k if (oy + 1) * k <= in_row else in_row - oy * k 176 | width = k if (ox + 1) * k <= in_col else in_col - ox * k 177 | gradient_x[b_id, c, oy * k + offset_r, ox * k + offset_c] = residual[b_id, c, oy, ox] 178 | gradient_x[self.flag == 0] = 0 179 | return gradient_x 180 | 181 | class FlattenLayer: 182 | def __init__(self, name='Flatten'): 183 | pass 184 | def forward(self, in_data): 185 | self.in_batch, self.in_channel, self.r, self.c = in_data.shape 186 | return in_data.reshape(self.in_batch, self.in_channel * self.r * self.c) 187 | def backward(self, residual): 188 | return residual.reshape(self.in_batch, self.in_channel, self.r, self.c) 189 | 190 | class SoftmaxLayer: 191 | def __init__(self, name='Softmax'): 192 | pass 193 | def forward(self, in_data): 194 | exp_out = np.exp(in_data) 195 | self.top_val = exp_out / np.sum(exp_out, axis=1) 196 | return self.top_val 197 | def backward(self, residual): 198 | return self.top_val - residual 199 | 200 | class Net: 201 | def __init__(self): 202 | self.layers = [] 203 | def addLayer(self, layer): 204 | self.layers.append(layer) 205 | def train(self, trainData, trainLabel, validData, validLabel, batch_size, iteration): 206 | train_num = trainData.shape[0] 207 | for iter in range(iteration): 208 | print 'iter=' + str(iter) 209 | for batch_iter in range(0, train_num, batch_size): 210 | if batch_iter + batch_size < train_num: 211 | self.train_inner(trainData[batch_iter: batch_iter + batch_size], 212 | trainLabel[batch_iter: batch_iter + batch_size]) 213 | else: 214 | self.train_inner(trainData[batch_iter: train_num], 215 | trainLabel[batch_iter: train_num]) 216 | print "eval=" + str(self.eval(validData, validLabel)) 217 | def train_inner(self, data, label): 218 | lay_num = len(self.layers) 219 | in_data = data 220 | for i in range(lay_num): 221 | out_data = self.layers[i].forward(in_data) 222 | in_data = out_data 223 | residual_in = label 224 | for i in range(0, lay_num, -1): 225 | residual_out = self.layers[i].backward(residual_in) 226 | residual_in = residual_out 227 | def eval(self, data, label): 228 | lay_num = len(self.layers) 229 | in_data = data 230 | for i in range(lay_num): 231 | out_data = self.layers[i].forward(in_data) 232 | in_data = out_data 233 | out_idx = np.argmax(in_data, axis=1) 234 | label_idx = np.argmax(label, axis=1) 235 | return np.sum(out_idx == label_idx) / float(out_idx.shape[0]) 236 | 237 | if __name__ == '__main__': 238 | import struct 239 | from array import array 240 | 241 | def load_data(data_path, label_path): 242 | with open(label_path, 'rb') as file: 243 | magic, size = struct.unpack(">II", file.read(8)) 244 | if magic != 2049: 245 | raise ValueError('Magic number mismatch, expected 2049,' 246 | 'got %d' % magic) 247 | labels = array("B", file.read()) 248 | with open(data_path, 'rb') as file: 249 | magic, size, rows, cols = struct.unpack(">IIII", file.read(16)) 250 | if magic != 2051: 251 | raise ValueError('Magic number mismatch, expected 2051,' 252 | 'got %d' % magic) 253 | image_data = array("B", file.read()) 254 | images = [] 255 | for i in xrange(size): 256 | images.append([0]*rows*cols) 257 | for i in xrange(size): 258 | images[i][:] = image_data[i*rows*cols : (i+1)*rows*cols] 259 | return np.array(images), np.array(labels) 260 | 261 | train_feature_raw, train_label_raw = load_data('train.feat', 'train.label') 262 | valid_feature_raw, valid_label_raw = load_data('valid.feat', 'valid.label') 263 | print 'load ok' 264 | train_feature = train_feature_raw.reshape(60000, 1, 28, 28) 265 | valid_feature = valid_feature_raw.reshape(10000, 1, 28, 28) 266 | train_label = discreterize(train_label_raw, 10) 267 | valid_label = discreterize(valid_label_raw, 10) 268 | 269 | net = Net() 270 | net.addLayer(ConvLayer(1, 20, 4, 0.01, 0.9)) 271 | net.addLayer(ReLULayer()) 272 | net.addLayer(MaxPoolingLayer(2)) 273 | 274 | net.addLayer(ConvLayer(20, 40, 5, 0.01, 0.9)) 275 | net.addLayer(ReLULayer()) 276 | net.addLayer(MaxPoolingLayer(3)) 277 | 278 | net.addLayer(FlattenLayer()) 279 | net.addLayer(FCLayer(40 * 3 * 3, 150, 0.01, 0.9)) 280 | net.addLayer(ReLULayer()) 281 | net.addLayer(FCLayer(150, 10, 0.01, 0.9)) 282 | net.addLayer(SoftmaxLayer()) 283 | print 'net build ok' 284 | net.train(train_feature, train_label, valid_feature, valid_label, 100 ,10) -------------------------------------------------------------------------------- /gd.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat_minor": 0, "cells": [{"execution_count": 2, "cell_type": "code", "source": "import numpy as np\nimport matplotlib.pyplot as plt", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 10, "cell_type": "code", "source": "def f(x):\n return x * x - 2 * x + 1\n\ndef g(x):\n return 2 * x - 2\n\nx = np.linspace(-5,7,100)\ny = f(x)", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 11, "cell_type": "code", "source": "print x", "outputs": [{"output_type": "stream", "name": "stdout", "text": "[-5. -4.87878788 -4.75757576 -4.63636364 -4.51515152 -4.39393939\n -4.27272727 -4.15151515 -4.03030303 -3.90909091 -3.78787879 -3.66666667\n -3.54545455 -3.42424242 -3.3030303 -3.18181818 -3.06060606 -2.93939394\n -2.81818182 -2.6969697 -2.57575758 -2.45454545 -2.33333333 -2.21212121\n -2.09090909 -1.96969697 -1.84848485 -1.72727273 -1.60606061 -1.48484848\n -1.36363636 -1.24242424 -1.12121212 -1. -0.87878788 -0.75757576\n -0.63636364 -0.51515152 -0.39393939 -0.27272727 -0.15151515 -0.03030303\n 0.09090909 0.21212121 0.33333333 0.45454545 0.57575758 0.6969697\n 0.81818182 0.93939394 1.06060606 1.18181818 1.3030303 1.42424242\n 1.54545455 1.66666667 1.78787879 1.90909091 2.03030303 2.15151515\n 2.27272727 2.39393939 2.51515152 2.63636364 2.75757576 2.87878788\n 3. 3.12121212 3.24242424 3.36363636 3.48484848 3.60606061\n 3.72727273 3.84848485 3.96969697 4.09090909 4.21212121 4.33333333\n 4.45454545 4.57575758 4.6969697 4.81818182 4.93939394 5.06060606\n 5.18181818 5.3030303 5.42424242 5.54545455 5.66666667 5.78787879\n 5.90909091 6.03030303 6.15151515 6.27272727 6.39393939 6.51515152\n 6.63636364 6.75757576 6.87878788 7. ]\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 12, "cell_type": "code", "source": "print y", "outputs": [{"output_type": "stream", "name": "stdout", "text": "[ 3.60000000e+01 3.45601469e+01 3.31496786e+01 3.17685950e+01\n 3.04168962e+01 2.90945822e+01 2.78016529e+01 2.65381084e+01\n 2.53039486e+01 2.40991736e+01 2.29237833e+01 2.17777778e+01\n 2.06611570e+01 1.95739210e+01 1.85160698e+01 1.74876033e+01\n 1.64885216e+01 1.55188246e+01 1.45785124e+01 1.36675849e+01\n 1.27860422e+01 1.19338843e+01 1.11111111e+01 1.03177227e+01\n 9.55371901e+00 8.81910009e+00 8.11386593e+00 7.43801653e+00\n 6.79155188e+00 6.17447199e+00 5.58677686e+00 5.02846648e+00\n 4.49954086e+00 4.00000000e+00 3.52984389e+00 3.08907254e+00\n 2.67768595e+00 2.29568411e+00 1.94306703e+00 1.61983471e+00\n 1.32598714e+00 1.06152433e+00 8.26446281e-01 6.20752984e-01\n 4.44444444e-01 2.97520661e-01 1.79981635e-01 9.18273646e-02\n 3.30578512e-02 3.67309458e-03 3.67309458e-03 3.30578512e-02\n 9.18273646e-02 1.79981635e-01 2.97520661e-01 4.44444444e-01\n 6.20752984e-01 8.26446281e-01 1.06152433e+00 1.32598714e+00\n 1.61983471e+00 1.94306703e+00 2.29568411e+00 2.67768595e+00\n 3.08907254e+00 3.52984389e+00 4.00000000e+00 4.49954086e+00\n 5.02846648e+00 5.58677686e+00 6.17447199e+00 6.79155188e+00\n 7.43801653e+00 8.11386593e+00 8.81910009e+00 9.55371901e+00\n 1.03177227e+01 1.11111111e+01 1.19338843e+01 1.27860422e+01\n 1.36675849e+01 1.45785124e+01 1.55188246e+01 1.64885216e+01\n 1.74876033e+01 1.85160698e+01 1.95739210e+01 2.06611570e+01\n 2.17777778e+01 2.29237833e+01 2.40991736e+01 2.53039486e+01\n 2.65381084e+01 2.78016529e+01 2.90945822e+01 3.04168962e+01\n 3.17685950e+01 3.31496786e+01 3.45601469e+01 3.60000000e+01]\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 13, "cell_type": "code", "source": "%matplotlib inline\nplt.plot(x, y)\n", "outputs": [{"execution_count": 13, "output_type": "execute_result", "data": {"text/plain": "[]"}, "metadata": {}}, {"output_type": "display_data", "data": {"image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAEACAYAAAB8nvebAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu85XO9x/HX261IJakhlI5ChYaTclRs4hw5JeecXIqS\nLij3Qyd0To3qJEJOF5XbuCuJiSLGZVOS6xiXIXWiogyOSwqV5n3++P0m29gze+291trf9Vvr/Xw8\n9mP2Wnv91nobez7ru75X2SYiIpplsdIBIiJi/FK8IyIaKMU7IqKBUrwjIhooxTsiooFSvCMiGqil\n4i1pcUmzJJ1f315e0kxJd0q6WNJy3Y0ZEREjtdry3geYA8yfFH4gMNP2GsCl9e2IiJgkYxZvSasA\nWwHHA6rv3ho4uf7+ZGCbrqSLiIhRtdLy/hLwcWDeiPum2J5bfz8XmNLpYBERsXCLLN6S3gHcb3sW\nT7e6n8HV+vqssY+ImERLjPHzjYCtJW0FPBd4gaRTgbmSVrR9n6SVgPtHu1hSinpExATYHrXBPJ9a\n3ZhK0ibAAbbfKelw4P9sHybpQGA5288atJTksQL0MknTbE8rnWMimpwdkr+05C+rldo53nne8yv9\nF4AtJN0JbFbfjoiISTJWt8nf2L4CuKL+/iFg826FioiIRcsKy0UbLh2gDcOlA7RpuHSANg2XDtCm\n4dIB2jRcOkC3tdznPaEnb3ifd0RECd3o846IiB6Q4h0R0UAp3hERDZTiHRHRQCneERENlOIdEdFA\nKd4REQ2U4h0R0UAp3hERDZTiHRHRQCneERENlOIdEdFAKd4REQ2U4h0R0UAp3hERDdTY4i3xGYkV\nSueIiMEi8V8SLyudo7HFG3gVsF3pEBExOCRWBvYDHiqdZcziLem5kq6RdJOkOZIOre+fJukeSbPq\nry27H/cZTgPeN8mvGRGD7b3AOTZPlg7S0jFokpax/bikJYAfAwcAbwMes33UIq7r2jFoEksC9wBv\ntvlFN14jImIkidnAPnZ3z8js2DFoth+vv10KWBx4eP5rTDxee2z+AnwL2KlUhogYHBLrAi8Criyd\nBVos3pIWk3QTMBe43PZt9Y/2kjRb0gmSlutayoU7DdhJKvcmEhEDYyfgdJt5pYNA6y3vebanAqsA\nG0saAr4OvBKYCvwOOLJbIRfheuApYMMCrx0RA0Jicar+7lNLZ5lvifE82Pajkn4AvMH28Pz7JR0P\nnD/aNZKmjbg5PPK6dtlYqlrfwNWdet6IiAVsAtxvM6cbT143iIfGdc1YA5aSVgCesv2IpKWBi4BD\ngNts31c/Zj9gA9vvXeDarg1YPv0avBK4FljZ5s/dfK2IGEwS04Gbbb40Oa83du1speW9EnCypMWo\nullOtX2ppFMkTQUM3AXs1nbiCbC5S+J24O3A90pkiIj+JbEMsA1wcOksI7U0VXDCTz4JLe/qdfgw\n8Habf+v2a0XEYJHYEdjRZqvJe80OTRVsgO8Am0u8uHSQiOg77wdOKR1iQX1RvG0eBS4Eti+dJSL6\nR70cfgN6sEu2L4p37WSqd8iIiE7ZEfiuzROlgyyon4r3TOAVEmuWDhIRzVcv/tuZHuwygT4q3jZP\nAWeQzaoiojPWA5YGriodZDR9U7xrpwDvk/ruvysiJt/OwKm9shx+QX1V5GxmU+2zu2npLBHRXBJL\nAe+hR7tMoM+Kd+0k4AOFM0REs/0zcLvN/5YOsjD9WLzPAN4p8YLSQSKisT5A1RDsWX1XvG0eAC4D\nti2dJSKaR2IK1UZU3ymdZVH6rnjXTgJ2KR0iIhppR2CGzR9KB1mUfi3eFwKvknh16SAR0Rz13O5d\ngOmls4ylL4t3fUTa6VRTfSIiWrUe8DzgR6WDjKUvi3ftJGDn+gSMiIhW7AKc3Ktzu0fq2+Jtcwtw\nH7B56SwR0fsknks1t/ukwlFa0rfFu3YC8KHSISKiEbYBbrT5Vekgrej34v0t4B8lVigdJCJ63geB\nE0uHaFVfF2+bR4DvU039iYgYlcQrgPWBGaWztKqvi3ftBOBD9RSgiIjRfAA40+bJ0kFatcjiLem5\nkq6RdJOkOZIOre9fXtJMSXdKuljScpMTd0KuAJaleleNiHiGehfSXWhQlwmMUbxtPwlsansqsC6w\nqaS3AAcCM22vAVxa3+5J9ZSf6WTgMiJGtxnwsM2s0kHGY8xuE9uP198uBSwOPAxsTXXsGPWf23Ql\nXeecBGwvsUzpIBHRcz5Mw1rd0ELxlrSYpJuAucDltm8DptieWz9kLjClixnbZvMb4KfAu0tniYje\nUc9E2xI4rXSW8VpirAfYngdMlfRC4CJJmy7wc0vywq6XNG3EzWHbwxPM2q7jgX+nhzdXj4hJ9z7g\nfJuHS4aQNAQMjesae6F1d7QX+C/gCaqPGUO275O0ElWLfK1RHm/bPTHLQ2JJ4NfAZja3l84TEWXV\nM9BuBT5mc0XpPCO1UjvHmm2ywvyZJJKWBrYAZgHn8fSmTzvTgLmR9WZVJ5GBy4iobAgsCVxZOshE\nLLLlLWkdqgHJxeqvU21/UdLywFnAy4G7ge1sPzLK9T3T8gaQeBXwE2BVmz+VzhMR5UicCNxhc3jp\nLAtqpXaOq9ukGwEmm8RlwNft3j4lIyK6pz4m8VfAWjZzx3r8ZGu726RPHQfsWjpERBT1XuCyXizc\nrRrE4n0O8HqJ1UsHiYjJVw9U7gZ8s3SWdgxc8a77uk8BPlI6S0QUsQHwAuCS0kHaMXDFu3Yc8AGJ\npUoHiYhJtxtwXBNOy1mUgSzeNj8Dbqf3l/VHRAdJvBD4VxpwwPBYBrJ4175J9Q4cEYNjR2Bmkwcq\n5xvk4n0usLbEq0sHiYju65eByvkGtnjXA5cnk2mDEYPiTcDzgMtLB+mEgVukM1K94vJqqhWXjTlB\nIyLGT+IU4GabI0pnGUsW6YzB5hfAjcC2pbNERPdIvJjqHILGD1TON9DFu3YM8NHSISKiq3YBzrP5\nv9JBOiXFG34ArCoxtXSQiOi8+ozK3YGvl87SSQNfvG2eAo4lre+IfrUF8BjVaVp9Y6AHLOeTWAmY\nA7zC5vel80RE50jMAC6wObZ0llZlwLJFNr8DZgLvL50lIjpHYlVgY+CM0lk6LcX7aV8D9qgn8kdE\nf9gdOM3mD6WDdFqK99OuBJ4CNisdJCLaJ/EcqvN2jymdpRtSvGs2pm59l84SER2xLTDb5o7SQboh\nxfuZTgM2kXh56SAR0bY9qRpkfWnM4i1pVUmXS7pN0q2S9q7vnybpHkmz6q8tux+3u+p+sVOp+ski\noqEk3gCsBHy/dJZuGXOqoKQVgRVt3yRpWeAGqn2wtwMes33UIq5txFTBkSTWAH4MvDz7nUQ0k8R0\nqpPhDyudZSI6MlXQ9n22b6q//wPVIQYrz3+NtlP2GJs7gVnA9qWzRMT4SaxA1cA8oXSWbhpXn7ek\n1YD1eHql0l6SZks6QdJyHc5W0peBvTJtMKKRdgXOtXmwdJBuanmFZd1lMgx8zvYMSS8FHqh//Flg\nJdsfWuAaA4eMuGvY9nC7obut3gvhTmBnm6tK54mI1kgsCdwFvMPmptJ5WiVpCBgacdenx+o2aal4\nS1qSquP/QttHj/Lz1YDzba+zwP2N6/OeT2IfYCM73ScRTSGxHbCHzSals7SjI33ekkTVdzRnZOGW\ntNKIh/0LcMtEg/ao6cAWEquUDhIRLduHqtuz77Uy2+QtVKsPbwbmP/hg4D3A1Pq+u4DdbM9d4NrG\ntrwBJL4MPGbzydJZImLR6umB3wVWr3cLbaxWamd2FVyE+nDiq6h2G3yidJ6IWLj6mLNbbQ4vnaVd\n2VWwTTY/B64DdiydJSIWTmJF4J30+fTAkVK8x/YlYN9MG4zoaXsA3+qnY87GkuI9tkuBeVSncURE\nj5FYGtgNeNZMuH6W4j2GerfBo4H9SmeJiFHtBFxr87PSQSZTBixbIPFc4G5gU5vbC8eJiFrdnXkr\nsLfNpaXzdEoGLDuk3qDqG8C+pbNExDP8I9UhKpeVDjLZ0vJukcQU4A5gDftv2wJEREESF1ENVE4v\nnaWT0vLuIJu5VAsAPlo6S0SAxLrAuvTh4cKtSMt7HCReC1wOrJZFOxFlSZwE3Gnz+dJZOi0t7w6z\nmUO1aOd9pbNEDDKJlwHvohqLGkgp3uN3BPDv9baxEVHGXsCpNg+VDlJKCtD4XQE8Bvxz6SARg0ji\n+cBHGLBFOQtK8R6netHOEcABpbNEDKgPApfZ/LJ0kJIyYDkBEksAPwd2sLmmdJ6IQVGflPNzYPt+\n/reXAcsuqfcKPgr4ROksEQNme+Cufi7crUrLe4Iknkd1CMXGNneUzhPR7+ql8DcDH7f5Yek83ZSW\ndxfZ/BH4GvDx0lkiBsRWwF+Bi0oH6QVpebdB4sVU/W/r2NxbOk9EP5O4EviG3f8rKtPy7rJ64/dT\nyYZVEV0l8Q/AqsBZpbP0ilZOj19V0uWSbpN0q6S96/uXlzRT0p2SLpa0XPfj9qQjgQ9KvKh0kIg+\ndiBwZNMPFu6kVk6PXxFY0fZNkpYFbgC2AXYBHrR9uKRPAC+yfeAC1/Z1t8l89R4L/2vz2dJZIvqN\nxDrAxcDfDcqeQl05PV7SDOCr9dcmtufWBX7Y9lrjDdAPJNYCrqT65fpD6TwR/UTidOBmm8NKZ5ks\nHe/zlrQasB5wDTDF9tz6R3OBKRPI2BfqqYJXUC3ZjYgOkVgd+Cfg66Wz9JolWn1g3WXyXWAf249J\nT78p2LakUZvwkqaNuDlse3hiUXve54HzJY6x+VPpMBF94j+Ar9v8vnSQbpI0BAyN65pWuk0kLQl8\nH7jQ9tH1fXcAQ7bvk7QScPmgdpvMJ3EBMMPm2NJZIppOYmXgFqrTqx4snWcydaTbRFUT+wRgzvzC\nXTsP2Ln+fmdgxkSD9pHPA5+o9z6JiPYcAJw8aIW7Va3MNnkL1WDczcD8Bx8EXEs15/LlVCerb2f7\nkQWuHaiWN4DE5cB0m1NKZ4loqvrM2NuBtW1+WzrPZOvKbJNOB+g3EpsC3wReY/PX0nkimkjicGAZ\nmz1LZykhKyzLGAbup9r9LCLGSWIF4MMwOFMDJyLFu8Pqwxo+A/xnjkqLmJD9gLNsflM6SC9LcemO\nmcDvgX8rHSSiSSSWB3YHvlA6S69L8e6CEa3vT6X1HTEu+wLfs7m7dJBel8LSPRcCj5PWd0RL6lb3\nHsDnSmdpghTvLqlb39OAaRKLF44T0QT7A+cM+sHCrUrx7q4fUvV9b1s6SEQvq2eY7A78d+ksTZHi\n3UV16/vTwKfT+o5YpP2Bs9PX3boU7+6bCTwE7FA6SEQvkngJsCtpdY9LVlhOAom3UW1p+dqcBBLx\nTBJHUK2m/FjpLL0iy+N7hISAy4DTbE4onSeiV0i8DLiVAd3DZGFSvHuIxEbAmVTbW2a/7whA4hjg\njzYfL52ll6R49xiJ7wMX2XyldJaI0iReCVwPrJltX58pxbvHSEylWrzzKps/ls4TUVJ9cPevbD5d\nOkuvSfHuQRJnAbNsDi2dJaKUEYd2v9rm0dJ5ek2Kdw+SWBP4MdVHxYdK54koQeIc4GqbL5bO0otS\nvHuUxLHAIzb/UTpLxGST2BD4DtXg/ROl8/SiFO8eVR+sejPwept7SueJmCz1tNlh4JRMm124nKTT\no2zuBY6j2rgqYpC8HXgpcHLpIE3XyunxJ0qaK+mWEfdNk3SPpFn115bdjdmXDgPeJfGa0kEiJkO9\nt/2hwMFZady+Vlre04EFi7OBo2yvV3/9sPPR+pvNw8DhkFknMTB2otrjfkbpIP1gzOJt+0fAw6P8\nKH3Z7fsKMFVi49JBIrpJYmmqQxb2r3fbjDa10+e9l6TZkk6QtFzHEg0QmyeBg4Ejclxa9Ll9gWtt\nflI6SL9oabaJpNWA822vU99+KfBA/ePPAivZ/tAo1xk4ZMRdw7aH24vcX+qifQ1wlM2ZpfNEdJrE\nS4E5wIY2vyidpxdJGgKGRtz16Y5MFVyweI/jZ5kq2AKJTYCTgLWyaVX0G4mvAn+12ad0lqbo2lRB\nSSuNuPkvwC0Le2yMzeYKqnnfe5fOEtFJ9TL47ak+oUcHjdnylnQmsAmwAjCX6livIWAq1ayTu4Dd\nbM8d5dq0vFsksQZwFfA6m/tL54nohHonzctsjiqdpUmywrJhJI6iOlFk99JZItol8U/AV6kaJH8u\nnadJUrwbRuJFwB3AFjY3l84TMVESSwCzgYNsziudp2myPL5h6oU7hwBH1XtARDTVrsDvgPNLB+lX\naXn3mLrFchPwn3ZWokXzSCwP3E4+QU5Yuk0aqj5t/jiqvsJsmRmNUk8NlM0epbM0VYp3g0mcDcy2\nM8UqmkPi9cDFwGty2MjEpXg3mMQrgBuAv7f5Vek8EWOpx2muAE63+WbpPE2WAcsGqwv2V4AjSmeJ\naNEOwLLA8aWDDIK0vHtYvRPbbcCuNpeUzhOxMBLPp9q/ZAebq0rnabq0vBuuHqzcG/iaxHNK54lY\nhGnAzBTuyZOWdwNIzACus/nv0lkiFiSxLnAJ1eyoB8Z6fIwtA5Z9YsTg5QY2d5XOEzFfvaXxj4CT\nbY4tnadfpNukT9SDl0cAX8nKy+gxOwNLkEHKSZeWd0NILMXTKy/PKZ0nQuIlVNtBb2VzY+k8/STd\nJn1G4i3At6j6Fh8tnScGm8QpwAM2+5fO0m9SvPuQxDeBp7L0OEqS2Jyqq2Rtmz+UztNvUrz7kMRy\nVHO/t81hrlGCxDLUJz/ZXFA6Tz/KgGUfsnkE2A84tu4Hj5hs/wVcn8JdVlreDVTPOPkecKPNtMJx\nYoBI/D1wAbCuzbOOPozOSLdJH5NYGZgFbJ49k2My1J/0rgO+aHNa6Tz9rCPdJpJOlDRX0i0j7lte\n0kxJd0q6WNJynQgcrbO5FzgIOLE+wCGi2w4E7gFOLx0kWuvzng5sucB9BwIzba8BXFrfjsl3IvAw\ncEDpINHfJNYG9gJ2s+nex/VoWUvdJpJWA863vU59+w5gE9tzJa0IDNtea5Tr0m3SZRKrUX2UHbK5\nrXCc6EMSSwJXA9+0Oa50nkHQzdkmU2zPH6yYC0yZ4PNEm2zuBg4GTq7/kUV02kHAA2QJfE9pu6/U\ntiUttPkuadqIm8O2h9t9zXiW44F/pfpH9pnCWaKPSKwP7Amsl+6S7pE0BAyN65o2uk2GbN8naSXg\n8nSblFXPPrkReHv2mYhOqPeQvwE41M4g5WTqZrfJeVS7iVH/OWOCzxMdUs8+2Q84tT6BJ6JdnwN+\nBpxROkg825gtb0lnApsAK1D1b3+KaoHIWcDLgbuB7Ww/Msq1aXlPonrxzpnA/TZ7l84TzSWxGXAK\nMNXmwdJ5Bk0W6QwgiRdRbR27u82FpfNE80gsD8wGPmRzcek8gyjFe0BJDFF91J1qc3/hONEg9ae3\ns4B7bfYtnWdQZWOqAWUzDJwKnJCTd2KcPgCsRRbe9by0vPtUvQ/Fj4EzbI4unSd6n8RrgCvJgq/i\n0m0y4CT+DrgG2NLmhtJ5onfVM5SuAb6SVZTlpdtkwNn8EtgD+LbEC0rniZ52FDCHrKJsjLS8B0B9\ndNoLgfdklVwsSGI74FBg/ZyN2hvS8o759gXWhJx7Gc8ksRbwVeDdKdzNkpb3gJBYnWpnuK1tflo6\nT5Qn8Tyqfu6j7XSX9JIMWMYzSLwL+DLw91k1N9jqKaSnAk8Bu6Q7rbe0UjtzAssAsfmexEbAtyS2\ntHmqdKYoZk9gHeAfUribKS3vASOxOHAhcIvN/qXzxOSrV+B+m6pw/7JwnBhFBizjWWz+CuwAbCOx\nY+k8MbkkXk61edlOKdzNlpb3gJJYB7iMLOAZGBLLAD+iWnV7ZOk8sXAZsIxFkvhX4H+ADev9wKNP\nSSxGteHUE8D708/d2zJgGYtkc47EmsB5Ehvb/LF0puiazwArAm9L4e4PaXkPuHrK2EnAssC2NvPK\nJopOk3gfcAjwJpsHSueJsWXAMsZUt8J2BV4CHFY4TnRYPbPkSOCdKdz9JcU7sPkTsA3wDom9SueJ\nzpBYm2pK4A7Z4rX/tNXnLelu4PfAX4G/2H5jJ0LF5LN5SOLtwFUS99qcUzpTTJzEysAFwH42l5XO\nE53X7oClgSHbD3UiTJRlc7fEO4GLJB60ubJ0phi/+hzTC4Fj7Jz83q860W2SAck+YnMj8B7gbIn1\nS+eJ8ak3m/oBcAkZw+hr7RZvA5dIul7SRzoRKMqzuQTYHfhBPZUwGkDiOcA5wM+AAzIlsL+1223y\nZtu/k/QSYKakO2z/qBPBoqx6DvgLgYslNrG5u3SmWDiJJYDTgT8CH8mUz/7XVvG2/bv6zwcknQu8\nkWr57d9Imjbi5rDt4XZeMyaPzfT6Y/hlEkM2vy6dKZ6tLtynAs8Dtslukc0jaQgYGtc1E12kI2kZ\nYHHbj0l6HnAxcIjti0c8Jot0+oDEflSn8GySZfS9pd4l8iSq1ZNb2zxRNlF0QreXx08BzpU0/3lO\nH1m4o3/YfKlu3V0usZnNPaUzxd8K9wnAysA7UrgHS5bHR8skDgA+RrU/xl2l8wwyiSWpukpeTNVV\nkn1p+kg2poqOsjlC4nHgConNbe4snWkQ1bNKzqKaLfZOmycLR4oCUrxjXGyOkXgCGJZ4Rz0vPCaJ\nxPOppgM+Auxo8+fCkaKQ7G0S42YzneoMxB9KvK10nkEhMQUYBn5JtV9JCvcAS/GOCan3PtkWOFNi\nh9J5+p3E6sBVwPnA7vVxdjHAUrxjwmyuADYHDpf4ZL03eHSYxFupCvcRNtOycjIgs02iAyReBpwH\n3AbsWm8xGx0gsRNwFPA+m4tK54nJkcMYYlLY/BbYBHg+cKnESoUjNZ7E4hJfAD4LbJrCHQtK8Y6O\nqOcZvxuYCVwnsVHhSI0l8WLgh8AGwBtzkEKMJsU7OsZmns0hVDsSzpDYM/3g4yPxBuB64Cbgn3J0\nWSxM+ryjKyReRXUE16+AD9vkwI5FkFgM2Bc4EPiYzdmFI0VB6fOOYmx+AWxEVbxn1TMmYhT1/O3z\ngO2oTnhP4Y4xpXhH19j8yf7bjoTfljhCYunSuXqJxLuB2fXXW7NnTLQq3SYxKSRWAI4B1gE+aHN1\n4UhFSbwU+B9gfeD9NtcUjhQ9JN0m0TNsHrTZDvg08F2Jb9QH5Q4UicUkPgzcCtwLrJfCHROR4h2T\nyuYs4LXAPOA2iV3qwbq+V88kuRL4MLCFzQE2jxeOFQ2VbpMoRuKNwNHAc4H96uX2fUdiZeDzwBbA\np4Dp2ZskFiXdJtHTbK4F3gwcBpwkcWFd0PuCxEsljgRuBu4B1rQ5PoU7OiHFO4qysc23gTWB71H1\nh58v8eamLvCRWEXii8DtwJLAOjaftHmscLToIyne0RNs/mzzDeDVwIXAycDVEtvWR371PIn1JU6h\namkvCUy12bve+yWio9oq3pK2lHSHpJ9L+kSnQsXgsnnS5hiqlvhhwF7AryUOrVdt9hSJF0jsJnE9\n1Qk3c4DVbfa1+U3heNHHJly8JS0OfBXYkmr2wHskvaZTwXqBpKHSGSaqydkreqvNuTYbA5tStWSv\nkrhO4gCJ1YolE8+X2EHiXOA3wD8Cn6Qq2l+webjpf//J3/vaaXm/EfiF7btt/wX4FvCuzsTqGUOl\nA7RhqHSANg3N/8bmDpsDgJWp9v5YA7hW4naJoyS2kli+W0EklpJ4k8QnJC4HfgvsTNVH/wqbf7O5\naIGByKHRnqtBhkoHaNNQ6QDd1s4BxCvDMz4W3gO8qb04EQtn8xRwKdWe4bsD6wFvB/YHNpC4h2pH\nvlupDob4BfCbVudSSywBvAxYjerT5NrA1Prr58CPgSOA4XoL3Ihi2ineOYopirGZB9xQf32uLrzr\nUBX01wFvA/4OWEXiceBB4PfAo8BT9dMsBiwLvBBYDngJ8ADVZlpzqN4EzgWuzUyR6DUTXqQjaUNg\nmu0t69sHAfNsHzbiMSnwERETMNYinXaK9xLAz6haOL8FrgXeY/v2CT1hRES0bMLdJrafkrQncBGw\nOHBCCndExOTo6t4mERHRHV1fYSlpL0m3S7pV0mFjX9F7JO0vaZ6krk1H6wZJX6z/7mdLOkfSC0tn\nakWTF39JWlXS5ZJuq3/n9y6dabwkLS5plqTzS2cZL0nLSTq7/r2fU4/NNYakg+rfnVsknSHpOQt7\nbFeLt6RNga2BdW2vTTXNqlEkrUq1G9yvSmeZgIuB19l+PXAncFDhPGPqg8VffwH2s/06YENgj4bl\nB9iHarZNEz+W/w9wge3XAOtS7S/TCJJWAz4CrG97Haru6B0W9vhut7w/ChxaL+LBdhNPwj4K+I/S\nISbC9kzb8+qb1wCrlMzTokYv/rJ9n+2b6u//QFU8XlY2VeskrQJsBRwPzdoYrP5k+VbbJ0I1Lmf7\n0cKxxuP3VG/+y9QTQpahOrBjVN0u3q8GNpb0U0nDkt7Q5dfrKEnvAu6xfXPpLB3wQeCC0iFaMNri\nr5ULZWlL3ZJaDxp1Us6XgI9THZbRNK8EHpA0XdKNko6TtEzpUK2y/RBwJPBrqhl8j9i+ZGGPb2eR\nDgCSZgIrjvKjT9bP/yLbG0raADiLauFEzxgj/0FU+1b87eGTEmocFpH/YNvn14/5JPBn22dMariJ\naeJH9WeRtCxwNrBP3QLveZLeAdxve1ZD9wZZgupM0D1tXyfpaKrtFD5VNlZrJK0O7Eu1wvdR4DuS\ndrR9+miPb7t4295iEWE+SrXTGvVf5jxJL7b9f+2+bqcsLL+ktaneyWdLgqrL4QZJb7R9/yRGXKRF\n/f0DSPoA1cfgt01KoPbdC6w64vaqVK3vxpC0JPBd4DTbM0rnGYeNgK0lbUV1utELJJ1i+/2Fc7Xq\nHqpPytfVt8+mKt5N8QbgJ/Pro6RzqP6fjFq8u91tMgPYrA6yBrBULxXuRbF9q+0ptl9p+5VUvxjr\n91LhHoukLak+Ar/L9pOl87ToeuDVklaTtBSwPXBe4UwtU/VOfwIwx/bRpfOMh+2Dba9a/77vAFzW\noMKN7fuA39S1BmBzqj1umuIOYENJS9e/R5tTDRyPqu2W9xhOBE6UdAvwZ6AxvwijaOLH+a8ASwEz\n608PV9ssom5lAAAAc0lEQVT+WNlIi9YHi7/eDOwE3CxpVn3fQbZ/WDDTRDXxd34v4PT6jf9/gV0K\n52mZ7dmSTqFqwMwDbgSOXdjjs0gnIqKBcgxaREQDpXhHRDRQindERAOleEdENFCKd0REA6V4R0Q0\nUIp3REQDpXhHRDTQ/wOtWIG0Mm6fvAAAAABJRU5ErkJggg==\n", "text/plain": ""}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 33, "cell_type": "code", "source": "def gd(x_start, step, g): # gd\u4ee3\u8868\u4e86Gradient Descent\n x = x_start\n for i in range(20):\n grad = g(x)\n x -= grad * step\n print '[ Epoch {0} ] grad = {1}, x = {2}'.format(i, grad, x)\n if abs(grad) < 1e-6:\n break;\n return x\ngd(5,0.1,g)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "[ Epoch 0 ] grad = 8, x = 4.2\n[ Epoch 1 ] grad = 6.4, x = 3.56\n[ Epoch 2 ] grad = 5.12, x = 3.048\n[ Epoch 3 ] grad = 4.096, x = 2.6384\n[ Epoch 4 ] grad = 3.2768, x = 2.31072\n[ Epoch 5 ] grad = 2.62144, x = 2.048576\n[ Epoch 6 ] grad = 2.097152, x = 1.8388608\n[ Epoch 7 ] grad = 1.6777216, x = 1.67108864\n[ Epoch 8 ] grad = 1.34217728, x = 1.536870912\n[ Epoch 9 ] grad = 1.073741824, x = 1.4294967296\n[ Epoch 10 ] grad = 0.8589934592, x = 1.34359738368\n[ Epoch 11 ] grad = 0.68719476736, x = 1.27487790694\n[ Epoch 12 ] grad = 0.549755813888, x = 1.21990232556\n[ Epoch 13 ] grad = 0.43980465111, x = 1.17592186044\n[ Epoch 14 ] grad = 0.351843720888, x = 1.14073748836\n[ Epoch 15 ] grad = 0.281474976711, x = 1.11258999068\n[ Epoch 16 ] grad = 0.225179981369, x = 1.09007199255\n[ Epoch 17 ] grad = 0.180143985095, x = 1.07205759404\n[ Epoch 18 ] grad = 0.144115188076, x = 1.05764607523\n[ Epoch 19 ] grad = 0.115292150461, x = 1.04611686018\n"}, {"execution_count": 33, "output_type": "execute_result", "data": {"text/plain": "1.0461168601842739"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 34, "cell_type": "code", "source": "gd(5,100,g)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "[ Epoch 0 ] grad = 8, x = -795\n[ Epoch 1 ] grad = -1592, x = 158405\n[ Epoch 2 ] grad = 316808, x = -31522395\n[ Epoch 3 ] grad = -63044792, x = 6272956805\n[ Epoch 4 ] grad = 12545913608, x = -1248318403995\n[ Epoch 5 ] grad = -2496636807992, x = 248415362395205\n[ Epoch 6 ] grad = 496830724790408, x = -49434657116645595\n[ Epoch 7 ] grad = -98869314233291192, x = 9837496766212473605\n[ Epoch 8 ] grad = 19674993532424947208, x = -1957661856476282247195\n[ Epoch 9 ] grad = -3915323712952564494392, x = 389574709438780167192005\n[ Epoch 10 ] grad = 779149418877560334384008, x = -77525367178317253271208795\n[ Epoch 11 ] grad = -155050734356634506542417592, x = 15427548068485133400970550405\n[ Epoch 12 ] grad = 30855096136970266801941100808, x = -3070082065628541546793139530395\n[ Epoch 13 ] grad = -6140164131257083093586279060792, x = 610946331060079767811834766548805\n[ Epoch 14 ] grad = 1221892662120159535623669533097608, x = -121578319880955873794555118543211995\n[ Epoch 15 ] grad = -243156639761911747589110237086423992, x = 24194085656310218885116468590099187205\n[ Epoch 16 ] grad = 48388171312620437770232937180198374408, x = -4814623045605733558138177249429738253595\n[ Epoch 17 ] grad = -9629246091211467116276354498859476507192, x = 958109986075540978069497272636517912465605\n[ Epoch 18 ] grad = 1916219972151081956138994545273035824931208, x = -190663887229032654635829957254667064580655195\n[ Epoch 19 ] grad = -381327774458065309271659914509334129161310392, x = 37942113558577498272530161493678745851550384005\n"}, {"execution_count": 34, "output_type": "execute_result", "data": {"text/plain": "37942113558577498272530161493678745851550384005L"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 35, "cell_type": "code", "source": "gd(4,1,g)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "[ Epoch 0 ] grad = 6, x = -2\n[ Epoch 1 ] grad = -6, x = 4\n[ Epoch 2 ] grad = 6, x = -2\n[ Epoch 3 ] grad = -6, x = 4\n[ Epoch 4 ] grad = 6, x = -2\n[ Epoch 5 ] grad = -6, x = 4\n[ Epoch 6 ] grad = 6, x = -2\n[ Epoch 7 ] grad = -6, x = 4\n[ Epoch 8 ] grad = 6, x = -2\n[ Epoch 9 ] grad = -6, x = 4\n[ Epoch 10 ] grad = 6, x = -2\n[ Epoch 11 ] grad = -6, x = 4\n[ Epoch 12 ] grad = 6, x = -2\n[ Epoch 13 ] grad = -6, x = 4\n[ Epoch 14 ] grad = 6, x = -2\n[ Epoch 15 ] grad = -6, x = 4\n[ Epoch 16 ] grad = 6, x = -2\n[ Epoch 17 ] grad = -6, x = 4\n[ Epoch 18 ] grad = 6, x = -2\n[ Epoch 19 ] grad = -6, x = 4\n"}, {"execution_count": 35, "output_type": "execute_result", "data": {"text/plain": "4"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 32, "cell_type": "code", "source": "def f2(x):\n return 4 * x * x - 4 * x + 1\ndef g2(x):\n return 8 * x -4\ngd(5,0.25,g2)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "[ Epoch 0 ] grad = 36, x = -4.0\n[ Epoch 1 ] grad = -36.0, x = 5.0\n[ Epoch 2 ] grad = 36.0, x = -4.0\n[ Epoch 3 ] grad = -36.0, x = 5.0\n[ Epoch 4 ] grad = 36.0, x = -4.0\n[ Epoch 5 ] grad = -36.0, x = 5.0\n[ Epoch 6 ] grad = 36.0, x = -4.0\n[ Epoch 7 ] grad = -36.0, x = 5.0\n[ Epoch 8 ] grad = 36.0, x = -4.0\n[ Epoch 9 ] grad = -36.0, x = 5.0\n[ Epoch 10 ] grad = 36.0, x = -4.0\n[ Epoch 11 ] grad = -36.0, x = 5.0\n[ Epoch 12 ] grad = 36.0, x = -4.0\n[ Epoch 13 ] grad = -36.0, x = 5.0\n[ Epoch 14 ] grad = 36.0, x = -4.0\n[ Epoch 15 ] grad = -36.0, x = 5.0\n[ Epoch 16 ] grad = 36.0, x = -4.0\n[ Epoch 17 ] grad = -36.0, x = 5.0\n[ Epoch 18 ] grad = 36.0, x = -4.0\n[ Epoch 19 ] grad = -36.0, x = 5.0\n"}, {"execution_count": 32, "output_type": "execute_result", "data": {"text/plain": "5.0"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.10", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}} --------------------------------------------------------------------------------